From a80824255e2772d94c63ee90d15d6ae3fd88b25f Mon Sep 17 00:00:00 2001 From: RandNMR73 Date: Mon, 2 Feb 2026 04:38:24 +0000 Subject: [PATCH 001/214] Add data preprocess pipeline for WanGame --- .../preprocess_wangame_data_i2v.sh | 27 ++ fastvideo/dataset/dataloader/schema.py | 40 +++ fastvideo/models/dits/hyworld/pose.py | 210 +++++++++++- .../pipelines/preprocess/v1_preprocess.py | 8 +- .../wangame/wangame_preprocess_pipeline.py | 303 ++++++++++++++++++ 5 files changed, 585 insertions(+), 3 deletions(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/preprocess_wangame_data_i2v.sh create mode 100644 fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline.py diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/preprocess_wangame_data_i2v.sh b/examples/training/finetune/WanGame2.1_1.3b_i2v/preprocess_wangame_data_i2v.sh new file mode 100644 index 000000000..85a4fd0d2 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/preprocess_wangame_data_i2v.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +GPU_NUM=1 # 2,4,8 +MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" +DATA_MERGE_PATH="mc_wasd_10/merge.txt" +OUTPUT_DIR="mc_wasd_10/preprocessed/" + +# export CUDA_VISIBLE_DEVICES=0 +export MASTER_ADDR=localhost +export MASTER_PORT=29500 +export RANK=0 +export WORLD_SIZE=1 + +python fastvideo/pipelines/preprocess/v1_preprocess.py \ + --model_path $MODEL_PATH \ + --data_merge_path $DATA_MERGE_PATH \ + --preprocess_video_batch_size 10 \ + --seed 42 \ + --max_height 352 \ + --max_width 640 \ + --num_frames 77 \ + --dataloader_num_workers 0 \ + --output_dir=$OUTPUT_DIR \ + --samples_per_file 10 \ + --train_fps 25 \ + --flush_frequency 10 \ + --preprocess_task wangame \ No newline at end of file diff --git a/fastvideo/dataset/dataloader/schema.py b/fastvideo/dataset/dataloader/schema.py index 048c7686c..50982b66d 100644 --- a/fastvideo/dataset/dataloader/schema.py +++ b/fastvideo/dataset/dataloader/schema.py @@ -157,3 +157,43 @@ pa.field("duration_sec", pa.float64()), pa.field("fps", pa.float64()), ]) + +pyarrow_schema_wangame = pa.schema([ + pa.field("id", pa.string()), + # --- Image/Video VAE latents --- + # Tensors are stored as raw bytes with shape and dtype info for loading + pa.field("vae_latent_bytes", pa.binary()), + # e.g., [C, T, H, W] or [C, H, W] + pa.field("vae_latent_shape", pa.list_(pa.int64())), + # e.g., 'float32' + pa.field("vae_latent_dtype", pa.string()), + #I2V + pa.field("clip_feature_bytes", pa.binary()), + pa.field("clip_feature_shape", pa.list_(pa.int64())), + pa.field("clip_feature_dtype", pa.string()), + pa.field("first_frame_latent_bytes", pa.binary()), + pa.field("first_frame_latent_shape", pa.list_(pa.int64())), + pa.field("first_frame_latent_dtype", pa.string()), + # --- Action --- + pa.field("mouse_cond_bytes", pa.binary()), + pa.field("mouse_cond_shape", pa.list_(pa.int64())), # [T, 2] + pa.field("mouse_cond_dtype", pa.string()), + pa.field("keyboard_cond_bytes", pa.binary()), + pa.field("keyboard_cond_shape", pa.list_(pa.int64())), # [T, 4] + pa.field("keyboard_cond_dtype", pa.string()), + # I2V Validation + pa.field("pil_image_bytes", pa.binary()), + pa.field("pil_image_shape", pa.list_(pa.int64())), + pa.field("pil_image_dtype", pa.string()), + # --- Metadata --- + pa.field("file_name", pa.string()), + pa.field("caption", pa.string()), + pa.field("media_type", pa.string()), # 'image' or 'video' + pa.field("width", pa.int64()), + pa.field("height", pa.int64()), + # -- Video-specific (can be null/default for images) --- + # Number of frames processed (e.g., 1 for image, N for video) + pa.field("num_frames", pa.int64()), + pa.field("duration_sec", pa.float64()), + pa.field("fps", pa.float64()), +]) \ No newline at end of file diff --git a/fastvideo/models/dits/hyworld/pose.py b/fastvideo/models/dits/hyworld/pose.py index b1b3f5df3..d41bf7bf5 100644 --- a/fastvideo/models/dits/hyworld/pose.py +++ b/fastvideo/models/dits/hyworld/pose.py @@ -15,7 +15,7 @@ from scipy.spatial.transform import Rotation as R from typing import Union, Optional -from .trajectory import generate_camera_trajectory_local +from fastvideo.models.dits.hyworld.trajectory import generate_camera_trajectory_local # Mapping from one-hot action encoding to single label @@ -411,3 +411,211 @@ def compute_num_frames(latent_num: int) -> int: Number of video frames """ return (latent_num - 1) * 4 + 1 + +def reformat_keyboard_and_mouse_tensors(keyboard_tensor, mouse_tensor): + """ + Reformat the keyboard and mouse tensors to the format compatible with HyWorld. + """ + num_frames = keyboard_tensor.shape[0] + assert (num_frames - 1) % 4 == 0, "num_frames must be a multiple of 4" + assert mouse_tensor.shape[0] == num_frames, "mouse_tensor must have the same number of frames as keyboard_tensor" + keyboard_tensor = keyboard_tensor[1:, :] + mouse_tensor = mouse_tensor[1:, :] + groups = keyboard_tensor.view(-1, 4, keyboard_tensor.shape[1]) + assert (groups == groups[:, 0:1]).all(dim=1).all(), "keyboard_tensor must have the same value for each group" + groups = mouse_tensor.view(-1, 4, mouse_tensor.shape[1]) + assert (groups == groups[:, 0:1]).all(dim=1).all(), "mouse_tensor must have the same value for each group" + + return keyboard_tensor[::4], mouse_tensor[::4] + +def process_custom_actions(keyboard_tensor, mouse_tensor, forward_speed=DEFAULT_FORWARD_SPEED): + """ + Process custom keyboard and mouse tensors into model inputs (viewmats, intrinsics, action_labels). + Assumes inputs correspond to each LATENT frame. + """ + keyboard_tensor, mouse_tensor = reformat_keyboard_and_mouse_tensors(keyboard_tensor, mouse_tensor) + + motions = [] + + # 1. Translate tensors to motions for trajectory generation + for t in range(keyboard_tensor.shape[0]): + frame_motion = {} + + # --- Translation --- + # MatrixGame convention: 0:W, 1:S, 2:A, 3:D + fwd = 0.0 + if keyboard_tensor[t, 0] > 0.5: fwd += forward_speed # W + if keyboard_tensor[t, 1] > 0.5: fwd -= forward_speed # S + if fwd != 0: frame_motion["forward"] = fwd + + rgt = 0.0 + if keyboard_tensor[t, 2] > 0.5: rgt -= forward_speed # A (Left is negative Right) + if keyboard_tensor[t, 3] > 0.5: rgt += forward_speed # D (Right) + if rgt != 0: frame_motion["right"] = rgt + + # --- Rotation --- + # MatrixGame convention: mouse is [Pitch, Yaw] (or Y, X) + # Apply scaling (e.g. to match HyWorld distribution) + pitch = mouse_tensor[t, 0].item() + yaw = mouse_tensor[t, 1].item() + + if abs(pitch) > 1e-4: frame_motion["pitch"] = pitch + if abs(yaw) > 1e-4: frame_motion["yaw"] = yaw + + motions.append(frame_motion) + + # 2. Generate Camera Trajectory + # generate_camera_trajectory_local returns T+1 poses (starting at Identity) + # We take the first T poses to match the latent count. + # Pose 0 is Identity. Pose 1 is Identity + Motion[0]. + poses = generate_camera_trajectory_local(motions) + # poses = np.array(poses[:T]) + + # 3. Compute Viewmats (w2c) and Intrinsics + w2c_list = [] + intrinsic_list = [] + + # Setup default intrinsic (normalized) + K = np.array(DEFAULT_INTRINSIC) + K[0, 0] /= K[0, 2] * 2 + K[1, 1] /= K[1, 2] * 2 + K[0, 2] = 0.5 + K[1, 2] = 0.5 + + for i in range(len(poses)): + c2w = np.array(poses[i]) + w2c = np.linalg.inv(c2w) + w2c_list.append(w2c) + intrinsic_list.append(K) + + viewmats = torch.as_tensor(np.array(w2c_list)) + intrinsics = torch.as_tensor(np.array(intrinsic_list)) + + # 4. Generate Action Labels DIRECTLY from inputs + # HyWorld Label Logic: + # Trans One-Hot: [Forward, Backward, Right, Left] (Indices 0, 1, 2, 3) + # Rotate One-Hot: [Right, Left, Up, Down] (Indices 0, 1, 2, 3) + + trans_one_hot = torch.zeros((keyboard_tensor.shape[0], 4), dtype=torch.long) + trans_one_hot[:, 0] = (keyboard_tensor[:, 0] > 0.5).long() # Forward + trans_one_hot[:, 1] = (keyboard_tensor[:, 1] > 0.5).long() # Backward + trans_one_hot[:, 2] = (keyboard_tensor[:, 3] > 0.5).long() # Right + trans_one_hot[:, 3] = (keyboard_tensor[:, 2] > 0.5).long() # Left + + rotate_one_hot = torch.zeros((mouse_tensor.shape[0], 4), dtype=torch.long) + rotate_one_hot[:, 0] = (mouse_tensor[:, 1] > 1e-4).long() # Yaw Right + rotate_one_hot[:, 1] = (mouse_tensor[:, 1] < -1e-4).long() # Yaw Left + rotate_one_hot[:, 2] = (mouse_tensor[:, 0] > 1e-4).long() # Pitch Up + rotate_one_hot[:, 3] = (mouse_tensor[:, 0] < -1e-4).long() # Pitch Down + + # Convert to single labels + trans_label = one_hot_to_one_dimension(trans_one_hot) + rotate_label = one_hot_to_one_dimension(rotate_one_hot) + action_labels = trans_label * 9 + rotate_label + + action_labels = torch.cat([torch.tensor([0]), action_labels]) + + return viewmats, intrinsics, action_labels + +if __name__ == "__main__": + print("Running comparison test between process_custom_actions and pose_to_input...") + + def test_process_custom_actions(pose_string: str, keyboard: torch.Tensor, mouse: torch.Tensor, latent_num: int): + # Run process_custom_actions + # Note: We need to pass float tensors + print("Running process_custom_actions...") + viewmats_1, intrinsics_1, labels_1 = process_custom_actions( + keyboard, mouse + ) + + print(f"Running pose_to_input with string: '{pose_string}'...") + viewmats_2, intrinsics_2, labels_2 = pose_to_input( + pose_string, latent_num=latent_num + ) + + # print(f"Viewmats: {viewmats_1} vs \n {viewmats_2}") + # print(f"Intrinsics: {intrinsics_1} vs \n {intrinsics_2}") + # print(f"Labels: {labels_1} vs \n {labels_2}") + # 3. Compare Results + print("\nComparison Results:") + + # Check Shapes + print(f"Shapes (Viewmats): {viewmats_1.shape} vs {viewmats_2.shape}") + assert viewmats_1.shape == viewmats_2.shape, "Shape mismatch for viewmats" + + # Check Values + # Viewmats + diff_viewmats = (viewmats_1 - viewmats_2).abs().max().item() + print(f"Max difference in Viewmats: {diff_viewmats}") + if diff_viewmats < 1e-5: + print("✅ Viewmats match.") + else: + print("❌ Viewmats mismatch.") + + # Check intrinsics + diff_intrinsics = (intrinsics_1 - intrinsics_2).abs().max().item() + print(f"Max difference in Intrinsics: {diff_intrinsics}") + if diff_intrinsics < 1e-5: + print("✅ Intrinsics match.") + else: + print("❌ Intrinsics mismatch.") + + # Check labels + diff_labels = (labels_1 - labels_2).abs().max().item() + print(f"Max difference in Labels: {diff_labels}") + if diff_labels < 1e-5: + print("✅ Labels match.") + else: + print("❌ Labels mismatch.") + + print("All checks passed.") + + # Define shared parameters + + latent_num = 13 + pose_string = "w-2, a-3, s-1, d-6" + + num_frames = 4 * (latent_num - 1) + 1 + keyboard = torch.zeros((num_frames, 6)) + mouse = torch.zeros((num_frames, 2)) + + # Frame 0 is ignored/start + # Frames 1-8: Press W (index 0) + keyboard[1:9, 0] = 1.0 + # Frames 9-20: Press A (index 2) + keyboard[9:21, 2] = 1.0 + # Frames 21-24: Press S (index 1) + keyboard[21:25, 1] = 1.0 + # Frames 25-48: Press D (index 3) + keyboard[25:49, 3] = 1.0 + + test_process_custom_actions(pose_string, keyboard, mouse, latent_num) + + # Test keyboard AND mouse + latent_num = 25 + pose_string = "w-2, up-2, a-3, down-4, s-1, left-2, d-6, right-4" + + num_frames = 4 * (latent_num - 1) + 1 + keyboard = torch.zeros((num_frames, 6)) + mouse = torch.zeros((num_frames, 2)) + + # Frame 0 is ignored/start + # Frames 1-8: Press W (index 0) + keyboard[1:9, 0] = 1.0 + # Frames 17-28: Press A (index 2) + keyboard[17:29, 2] = 1.0 + # Frames 45-48: Press S (index 1) + keyboard[45:49, 1] = 1.0 + # Frames 57-80: Press D (index 3) + keyboard[57:81, 3] = 1.0 + + # Frames 9-16: Press Up (index 4) + mouse[9:17, 0] = DEFAULT_PITCH_SPEED + # Frames 25-32: Press Down (index 5) + mouse[29:45, 0] = -DEFAULT_PITCH_SPEED + # Frames 41-48: Press Left (index 6) + mouse[49:57, 1] = -DEFAULT_YAW_SPEED + # Frames 57-64: Press Right (index 7) + mouse[81:, 1] = DEFAULT_YAW_SPEED + + test_process_custom_actions(pose_string, keyboard, mouse, latent_num) diff --git a/fastvideo/pipelines/preprocess/v1_preprocess.py b/fastvideo/pipelines/preprocess/v1_preprocess.py index 18455d70f..bcce47292 100644 --- a/fastvideo/pipelines/preprocess/v1_preprocess.py +++ b/fastvideo/pipelines/preprocess/v1_preprocess.py @@ -18,6 +18,8 @@ PreprocessPipeline_Text) from fastvideo.pipelines.preprocess.matrixgame.matrixgame_preprocess_pipeline import ( PreprocessPipeline_MatrixGame) +from fastvideo.pipelines.preprocess.wangame.wangame_preprocess_pipeline import ( + PreprocessPipeline_WanGame) from fastvideo.utils import maybe_download_model logger = init_logger(__name__) @@ -64,10 +66,12 @@ def main(args) -> None: PreprocessPipeline = PreprocessPipeline_ODE_Trajectory elif args.preprocess_task == "matrixgame": PreprocessPipeline = PreprocessPipeline_MatrixGame + elif args.preprocess_task == "wangame": + PreprocessPipeline = PreprocessPipeline_WanGame else: raise ValueError( f"Invalid preprocess task: {args.preprocess_task}. " - f"Valid options: t2v, i2v, ode_trajectory, text_only, matrixgame") + f"Valid options: t2v, i2v, ode_trajectory, text_only, matrixgame, wangame") logger.info("Preprocess task: %s using %s", args.preprocess_task, PreprocessPipeline.__name__) @@ -115,7 +119,7 @@ def main(args) -> None: "--preprocess_task", type=str, default="t2v", - choices=["t2v", "i2v", "text_only", "ode_trajectory", "matrixgame"], + choices=["t2v", "i2v", "text_only", "ode_trajectory", "matrixgame", "wangame"], help="Type of preprocessing task to run") parser.add_argument("--train_fps", type=int, default=30) parser.add_argument("--use_image_num", type=int, default=0) diff --git a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline.py b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline.py new file mode 100644 index 000000000..40c958d76 --- /dev/null +++ b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline.py @@ -0,0 +1,303 @@ +# SPDX-License-Identifier: Apache-2.0 +from typing import Any + +import numpy as np +import torch +from PIL import Image + +from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.forward_context import set_forward_context +from fastvideo.pipelines.preprocess.preprocess_pipeline_base import ( + BasePreprocessPipeline) +from fastvideo.pipelines.stages import ImageEncodingStage + + +class PreprocessPipeline_WanGame(BasePreprocessPipeline): + """I2V preprocessing pipeline implementation.""" + + _required_config_modules = ["vae", "image_encoder", "image_processor"] + + def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): + self.add_stage(stage_name="image_encoding_stage", + stage=ImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + )) + + def get_pyarrow_schema(self): + """Return the PyArrow schema for I2V pipeline.""" + return pyarrow_schema_wangame + + def get_extra_features(self, valid_data: dict[str, Any], + fastvideo_args: FastVideoArgs) -> dict[str, Any]: + + # TODO(will): move these to cpu at some point + self.get_module("image_encoder").to(get_local_torch_device()) + self.get_module("vae").to(get_local_torch_device()) + + features = {} + """Get CLIP features from the first frame of each video.""" + first_frame = valid_data["pixel_values"][:, :, 0, :, :].permute( + 0, 2, 3, 1) # (B, C, T, H, W) -> (B, H, W, C) + _, _, num_frames, height, width = valid_data["pixel_values"].shape + # latent_height = height // self.get_module( + # "vae").spatial_compression_ratio + # latent_width = width // self.get_module("vae").spatial_compression_ratio + + processed_images = [] + # Frame has values between -1 and 1 + for frame in first_frame: + frame = (frame + 1) * 127.5 + frame_pil = Image.fromarray(frame.cpu().numpy().astype(np.uint8)) + processed_img = self.get_module("image_processor")( + images=frame_pil, return_tensors="pt") + processed_images.append(processed_img) + + # Get CLIP features + pixel_values = torch.cat( + [img['pixel_values'] for img in processed_images], + dim=0).to(get_local_torch_device()) + with torch.no_grad(): + image_inputs = {'pixel_values': pixel_values} + with set_forward_context(current_timestep=0, attn_metadata=None): + clip_features = self.get_module("image_encoder")(**image_inputs) + clip_features = clip_features.last_hidden_state + + features["clip_feature"] = clip_features + """Get VAE features from the first frame of each video""" + video_conditions = [] + for frame in first_frame: + processed_img = frame.to(device="cpu", dtype=torch.float32) + processed_img = processed_img.unsqueeze(0).permute(0, 3, 1, + 2).unsqueeze(2) + # (B, H, W, C) -> (B, C, 1, H, W) + video_condition = torch.cat([ + processed_img, + processed_img.new_zeros(processed_img.shape[0], + processed_img.shape[1], num_frames - 1, + height, width) + ], + dim=2) + video_condition = video_condition.to( + device=get_local_torch_device(), dtype=torch.float32) + video_conditions.append(video_condition) + + video_conditions = torch.cat(video_conditions, dim=0) + + with torch.autocast(device_type="cuda", + dtype=torch.float32, + enabled=True): + encoder_outputs = self.get_module("vae").encode(video_conditions) + + latent_condition = encoder_outputs.mean + if (hasattr(self.get_module("vae"), "shift_factor") + and self.get_module("vae").shift_factor is not None): + if isinstance(self.get_module("vae").shift_factor, torch.Tensor): + latent_condition -= self.get_module("vae").shift_factor.to( + latent_condition.device, latent_condition.dtype) + else: + latent_condition -= self.get_module("vae").shift_factor + + if isinstance(self.get_module("vae").scaling_factor, torch.Tensor): + latent_condition = latent_condition * self.get_module( + "vae").scaling_factor.to(latent_condition.device, + latent_condition.dtype) + else: + latent_condition = latent_condition * self.get_module( + "vae").scaling_factor + + # mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, + # latent_width) + # mask_lat_size[:, :, list(range(1, num_frames))] = 0 + # first_frame_mask = mask_lat_size[:, :, 0:1] + # first_frame_mask = torch.repeat_interleave( + # first_frame_mask, + # dim=2, + # repeats=self.get_module("vae").temporal_compression_ratio) + # mask_lat_size = torch.concat( + # [first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2) + # mask_lat_size = mask_lat_size.view( + # batch_size, -1, + # self.get_module("vae").temporal_compression_ratio, latent_height, + # latent_width) + # mask_lat_size = mask_lat_size.transpose(1, 2) + # mask_lat_size = mask_lat_size.to(latent_condition.device) + + # image_latent = torch.concat([mask_lat_size, latent_condition], dim=1) + + features["first_frame_latent"] = latent_condition + + if "action_path" in valid_data and valid_data["action_path"]: + keyboard_cond_list = [] + mouse_cond_list = [] + num_bits = 6 + for action_path in valid_data["action_path"]: + if action_path: + action_data = np.load(action_path, allow_pickle=True) + if isinstance( + action_data, + np.ndarray) and action_data.dtype == np.dtype('O'): + action_dict = action_data.item() + if "keyboard" in action_dict: + keyboard_raw = action_dict["keyboard"] + # Convert 1D bit-flag values to 2D multi-hot encoding + if isinstance(keyboard_raw, np.ndarray): + if keyboard_raw.ndim == 1: + # [T] -> [T, num_bits] + T = len(keyboard_raw) + multi_hot = np.zeros((T, num_bits), + dtype=np.float32) + action_values = keyboard_raw.astype(int) + for bit_idx in range(num_bits): + target_idx = ( + 2 - + (bit_idx % 3)) + 3 * (bit_idx // 3) + if target_idx < num_bits: + multi_hot[:, target_idx] = ( + (action_values >> bit_idx) + & 1).astype(np.float32) + keyboard_cond_list.append(multi_hot) + else: + # If already 2D, pad to num_bits if necessary + k_data = keyboard_raw.astype(np.float32) + if k_data.ndim == 2 and k_data.shape[ + -1] < num_bits: + padding = np.zeros( + (k_data.shape[0], + num_bits - k_data.shape[-1]), + dtype=np.float32) + k_data = np.concatenate( + [k_data, padding], axis=-1) + keyboard_cond_list.append(k_data) + else: + keyboard_cond_list.append(keyboard_raw) + if "mouse" in action_dict: + mouse_cond_list.append(action_dict["mouse"]) + else: + if isinstance(action_data, + np.ndarray) and action_data.ndim == 1: + T = len(action_data) + multi_hot = np.zeros((T, num_bits), + dtype=np.float32) + action_values = action_data.astype(int) + for bit_idx in range(num_bits): + target_idx = ( + 2 - (bit_idx % 3)) + 3 * (bit_idx // 3) + if target_idx < num_bits: + multi_hot[:, target_idx] = ( + (action_values >> bit_idx) & 1).astype( + np.float32) + keyboard_cond_list.append(multi_hot) + else: + # If already 2D, pad to num_bits if necessary + k_data = action_data.astype(np.float32) + if k_data.ndim == 2 and k_data.shape[-1] < num_bits: + padding = np.zeros( + (k_data.shape[0], + num_bits - k_data.shape[-1]), + dtype=np.float32) + k_data = np.concatenate([k_data, padding], + axis=-1) + keyboard_cond_list.append(k_data) + if keyboard_cond_list: + features["keyboard_cond"] = keyboard_cond_list + if mouse_cond_list: + features["mouse_cond"] = mouse_cond_list + + return features + + def create_record( + self, + video_name: str, + vae_latent: np.ndarray, + text_embedding: np.ndarray, + valid_data: dict[str, Any], + idx: int, + extra_features: dict[str, Any] | None = None) -> dict[str, Any]: + """Create a record for the Parquet dataset with CLIP features.""" + record = super().create_record(video_name=video_name, + vae_latent=vae_latent, + text_embedding=text_embedding, + valid_data=valid_data, + idx=idx, + extra_features=extra_features) + + if extra_features and "clip_feature" in extra_features: + clip_feature = extra_features["clip_feature"] + record.update({ + "clip_feature_bytes": clip_feature.tobytes(), + "clip_feature_shape": list(clip_feature.shape), + "clip_feature_dtype": str(clip_feature.dtype), + }) + else: + record.update({ + "clip_feature_bytes": b"", + "clip_feature_shape": [], + "clip_feature_dtype": "", + }) + + if extra_features and "first_frame_latent" in extra_features: + first_frame_latent = extra_features["first_frame_latent"] + record.update({ + "first_frame_latent_bytes": + first_frame_latent.tobytes(), + "first_frame_latent_shape": + list(first_frame_latent.shape), + "first_frame_latent_dtype": + str(first_frame_latent.dtype), + }) + else: + record.update({ + "first_frame_latent_bytes": b"", + "first_frame_latent_shape": [], + "first_frame_latent_dtype": "", + }) + + if extra_features and "pil_image" in extra_features: + pil_image = extra_features["pil_image"] + record.update({ + "pil_image_bytes": pil_image.tobytes(), + "pil_image_shape": list(pil_image.shape), + "pil_image_dtype": str(pil_image.dtype), + }) + else: + record.update({ + "pil_image_bytes": b"", + "pil_image_shape": [], + "pil_image_dtype": "", + }) + + if extra_features and "keyboard_cond" in extra_features: + keyboard_cond = extra_features["keyboard_cond"] + record.update({ + "keyboard_cond_bytes": keyboard_cond.tobytes(), + "keyboard_cond_shape": list(keyboard_cond.shape), + "keyboard_cond_dtype": str(keyboard_cond.dtype), + }) + else: + record.update({ + "keyboard_cond_bytes": b"", + "keyboard_cond_shape": [], + "keyboard_cond_dtype": "", + }) + + if extra_features and "mouse_cond" in extra_features: + mouse_cond = extra_features["mouse_cond"] + record.update({ + "mouse_cond_bytes": mouse_cond.tobytes(), + "mouse_cond_shape": list(mouse_cond.shape), + "mouse_cond_dtype": str(mouse_cond.dtype), + }) + else: + record.update({ + "mouse_cond_bytes": b"", + "mouse_cond_shape": [], + "mouse_cond_dtype": "", + }) + + return record + + +EntryClass = PreprocessPipeline_WanGame From b1abb5c2516bce34a15d5389870060b094a5163c Mon Sep 17 00:00:00 2001 From: JerryZhou54 Date: Mon, 2 Feb 2026 05:20:06 +0000 Subject: [PATCH 002/214] Update action_labels --- fastvideo/models/dits/hyworld/pose.py | 86 ++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/fastvideo/models/dits/hyworld/pose.py b/fastvideo/models/dits/hyworld/pose.py index d41bf7bf5..6274b3a93 100644 --- a/fastvideo/models/dits/hyworld/pose.py +++ b/fastvideo/models/dits/hyworld/pose.py @@ -491,30 +491,84 @@ def process_custom_actions(keyboard_tensor, mouse_tensor, forward_speed=DEFAULT_ viewmats = torch.as_tensor(np.array(w2c_list)) intrinsics = torch.as_tensor(np.array(intrinsic_list)) - # 4. Generate Action Labels DIRECTLY from inputs - # HyWorld Label Logic: - # Trans One-Hot: [Forward, Backward, Right, Left] (Indices 0, 1, 2, 3) - # Rotate One-Hot: [Right, Left, Up, Down] (Indices 0, 1, 2, 3) + # 4. Generate Action Labels by analyzing the generated trajectory + # This ensures consistency with complex simultaneous movements, exactly as pose_to_input does. - trans_one_hot = torch.zeros((keyboard_tensor.shape[0], 4), dtype=torch.long) - trans_one_hot[:, 0] = (keyboard_tensor[:, 0] > 0.5).long() # Forward - trans_one_hot[:, 1] = (keyboard_tensor[:, 1] > 0.5).long() # Backward - trans_one_hot[:, 2] = (keyboard_tensor[:, 3] > 0.5).long() # Right - trans_one_hot[:, 3] = (keyboard_tensor[:, 2] > 0.5).long() # Left + # Calculate relative camera-to-world transforms + # c2ws = inverse(viewmats) + c2ws = np.linalg.inv(np.array(w2c_list)) - rotate_one_hot = torch.zeros((mouse_tensor.shape[0], 4), dtype=torch.long) - rotate_one_hot[:, 0] = (mouse_tensor[:, 1] > 1e-4).long() # Yaw Right - rotate_one_hot[:, 1] = (mouse_tensor[:, 1] < -1e-4).long() # Yaw Left - rotate_one_hot[:, 2] = (mouse_tensor[:, 0] > 1e-4).long() # Pitch Up - rotate_one_hot[:, 3] = (mouse_tensor[:, 0] < -1e-4).long() # Pitch Down + # Calculate relative movement between frames + # relative_c2w[i] = inv(c2ws[i-1]) @ c2ws[i] + C_inv = np.linalg.inv(c2ws[:-1]) + relative_c2w = np.zeros_like(c2ws) + relative_c2w[0, ...] = c2ws[0, ...] # First is anchor + relative_c2w[1:, ...] = C_inv @ c2ws[1:, ...] + + # Initialize one-hot action encodings + trans_one_hot = np.zeros((relative_c2w.shape[0], 4), dtype=np.int32) + rotate_one_hot = np.zeros((relative_c2w.shape[0], 4), dtype=np.int32) + + move_norm_valid = 0.0001 + + # Skip index 0 (anchor/identity) + for i in range(1, relative_c2w.shape[0]): + move_dirs = relative_c2w[i, :3, 3] # direction vector + move_norms = np.linalg.norm(move_dirs) + + if move_norms > move_norm_valid: # threshold for movement + move_norm_dirs = move_dirs / move_norms + angles_rad = np.arccos(move_norm_dirs.clip(-1.0, 1.0)) + trans_angles_deg = angles_rad * (180.0 / np.pi) # convert to degrees + else: + trans_angles_deg = np.zeros(3) + + R_rel = relative_c2w[i, :3, :3] + r = R.from_matrix(R_rel) + rot_angles_deg = r.as_euler("xyz", degrees=True) + + # Determine movement actions based on trajectory + # Note: HyWorld logic checks if rotation is small before assigning translation labels + # to avoid ambiguity in TPS mode, but here we generally want to capture the dominant movement. + tps = False # Default assumption, can be made an arg if needed + + if move_norms > move_norm_valid: + if (not tps) or ( + tps and abs(rot_angles_deg[1]) < 5e-2 and abs(rot_angles_deg[0]) < 5e-2 + ): + # Z-axis (Forward/Back) + if trans_angles_deg[2] < 60: + trans_one_hot[i, 0] = 1 # forward + elif trans_angles_deg[2] > 120: + trans_one_hot[i, 1] = 1 # backward + + # X-axis (Right/Left) + if trans_angles_deg[0] < 60: + trans_one_hot[i, 2] = 1 # right + elif trans_angles_deg[0] > 120: + trans_one_hot[i, 3] = 1 # left + + # Determine rotation actions + # Y-axis (Yaw) + if rot_angles_deg[1] > 5e-2: + rotate_one_hot[i, 0] = 1 # right + elif rot_angles_deg[1] < -5e-2: + rotate_one_hot[i, 1] = 1 # left + + # X-axis (Pitch) + if rot_angles_deg[0] > 5e-2: + rotate_one_hot[i, 2] = 1 # up + elif rot_angles_deg[0] < -5e-2: + rotate_one_hot[i, 3] = 1 # down + + trans_one_hot = torch.tensor(trans_one_hot) + rotate_one_hot = torch.tensor(rotate_one_hot) # Convert to single labels trans_label = one_hot_to_one_dimension(trans_one_hot) rotate_label = one_hot_to_one_dimension(rotate_one_hot) action_labels = trans_label * 9 + rotate_label - action_labels = torch.cat([torch.tensor([0]), action_labels]) - return viewmats, intrinsics, action_labels if __name__ == "__main__": From f3a7a37312a8d41e4b7e7f8de6e6eae0435a51aa Mon Sep 17 00:00:00 2001 From: JerryZhou54 Date: Tue, 3 Feb 2026 05:11:48 +0000 Subject: [PATCH 003/214] Overfitting running for MC 10 --- .../WanGame2.1_1.3b_i2v/finetune_i2v.sh | 93 ++++ .../WanGame2.1_1.3b_i2v/finetune_i2v.slurm | 120 +++++ fastvideo/configs/models/dits/wangamevideo.py | 115 +++++ fastvideo/configs/pipelines/registry.py | 1 + fastvideo/configs/sample/registry.py | 2 + fastvideo/configs/sample/wan.py | 12 +- fastvideo/models/dits/hyworld/pose.py | 4 + fastvideo/models/dits/wangame/__init__.py | 5 + .../dits/wangame/hyworld_action_module.py | 231 ++++++++++ fastvideo/models/dits/wangame/model.py | 424 ++++++++++++++++++ fastvideo/models/loader/component_loader.py | 4 + fastvideo/models/loader/fsdp_load.py | 7 +- fastvideo/models/registry.py | 1 + .../basic/wan/wangame_i2v_pipeline.py | 74 +++ fastvideo/pipelines/pipeline_registry.py | 1 + fastvideo/pipelines/stages/denoising.py | 16 + .../training/wangame_training_pipeline.py | 250 +++++++++++ 17 files changed, 1352 insertions(+), 8 deletions(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.sh create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.slurm create mode 100644 fastvideo/configs/models/dits/wangamevideo.py create mode 100644 fastvideo/models/dits/wangame/__init__.py create mode 100644 fastvideo/models/dits/wangame/hyworld_action_module.py create mode 100644 fastvideo/models/dits/wangame/model.py create mode 100644 fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py create mode 100644 fastvideo/training/wangame_training_pipeline.py diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.sh b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.sh new file mode 100644 index 000000000..e9869e1f5 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +export WANDB_BASE_URL="https://api.wandb.ai" +export WANDB_MODE=offline +export TOKENIZERS_PARALLELISM=false +export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN + +MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" +DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" +VALIDATION_DATASET_FILE="mc_wasd_10/validation.json" +NUM_GPUS=4 +# export CUDA_VISIBLE_DEVICES=0,1,2,3 +# IP=[MASTER NODE IP] + +# Training arguments +training_args=( + --tracker_project_name "wangame_1.3b_overfit" + --output_dir "wangame_1.3b_overfit" + --max_train_steps 1500 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 20 + --num_height 352 + --num_width 640 + --num_frames 77 + --enable_gradient_checkpointing_type "full" +) + +# Parallel arguments +parallel_args=( + --num_gpus $NUM_GPUS + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim $NUM_GPUS +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 1 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 100 + --validation_sampling_steps "40" + --validation_guidance_scale "1.0" +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 2e-5 + --mixed_precision "bf16" + --weight_only_checkpointing_steps 1000 + --training_state_checkpointing_steps 1000 + --weight_decay 1e-4 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.1 + --multi_phased_distill_schedule "4000-1" + --not_apply_cfg_solver + --dit_precision "fp32" + --num_euler_timesteps 50 + --ema_start_step 0 +) + +# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t +torchrun \ + --nnodes 1 \ + --nproc_per_node $NUM_GPUS \ + fastvideo/training/wangame_training_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.slurm new file mode 100644 index 000000000..18a77d2da --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.slurm @@ -0,0 +1,120 @@ +#!/bin/bash +#SBATCH --job-name=wangame_1.3b_overfit +#SBATCH --partition=main +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=wangame_1.3b_overfit_output/wangame_1.3b_overfit_%j.out +#SBATCH --error=wangame_1.3b_overfit_output/wangame_1.3b_overfit_%j.err +#SBATCH --exclusive + +# Basic Info +export NCCL_P2P_DISABLE=1 +export TORCH_NCCL_ENABLE_MONITORING=0 +export NCCL_DEBUG_SUBSYS=INIT,NET +# different cache dir for different processes +export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} +export MASTER_PORT=29500 +export NODE_RANK=$SLURM_PROCID +nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) +export MASTER_ADDR=${nodes[0]} +export TOKENIZERS_PARALLELISM=false +# export WANDB_API_KEY="8d9f4b39abd68eb4e29f6fc010b7ee71a2207cde" +export WANDB_API_KEY="50632ebd88ffd970521cec9ab4a1a2d7e85bfc45" +# export WANDB_API_KEY='your_wandb_api_key_here' +export WANDB_BASE_URL="https://api.wandb.ai" +export WANDB_MODE=online +export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN + +source ~/conda/miniconda/bin/activate +conda activate wei-fv-distill +export HOME="/mnt/weka/home/hao.zhang/wei" + +MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" +DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" +VALIDATION_DATASET_FILE="mc_wasd_10/validation.json" +# Configs +NUM_GPUS=8 + +# Training arguments +training_args=( + --tracker_project_name "wangame_1.3b_overfit" + --output_dir "wangame_1.3b_overfit" + --max_train_steps 1500 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 20 + --num_height 352 + --num_width 640 + --num_frames 77 + --enable_gradient_checkpointing_type "full" +) + +# Parallel arguments +parallel_args=( + --num_gpus $NUM_GPUS + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim $NUM_GPUS +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 1 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 100 + --validation_sampling_steps "40" + --validation_guidance_scale "1.0" +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 2e-5 + --mixed_precision "bf16" + --weight_only_checkpointing_steps 1000 + --training_state_checkpointing_steps 1000 + --weight_decay 1e-4 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.1 + --multi_phased_distill_schedule "4000-1" + --not_apply_cfg_solver + --dit_precision "fp32" + --num_euler_timesteps 50 + --ema_start_step 0 +) + +# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t +torchrun \ + --nnodes 1 \ + --nproc_per_node $NUM_GPUS \ + fastvideo/training/wangame_training_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ No newline at end of file diff --git a/fastvideo/configs/models/dits/wangamevideo.py b/fastvideo/configs/models/dits/wangamevideo.py new file mode 100644 index 000000000..b43442543 --- /dev/null +++ b/fastvideo/configs/models/dits/wangamevideo.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass, field + +from fastvideo.configs.models.dits.base import DiTArchConfig, DiTConfig + + +def is_blocks(n: str, m) -> bool: + return "blocks" in n and str.isdigit(n.split(".")[-1]) + + +@dataclass +class WanGameVideoArchConfig(DiTArchConfig): + _fsdp_shard_conditions: list = field(default_factory=lambda: [is_blocks]) + + param_names_mapping: dict = field( + default_factory=lambda: { + r"^patch_embedding\.(.*)$": + r"patch_embedding.proj.\1", + r"^condition_embedder\.text_embedder\.linear_1\.(.*)$": + r"condition_embedder.text_embedder.fc_in.\1", + r"^condition_embedder\.text_embedder\.linear_2\.(.*)$": + r"condition_embedder.text_embedder.fc_out.\1", + r"^condition_embedder\.time_embedder\.linear_1\.(.*)$": + r"condition_embedder.time_embedder.mlp.fc_in.\1", + r"^condition_embedder\.time_embedder\.linear_2\.(.*)$": + r"condition_embedder.time_embedder.mlp.fc_out.\1", + r"^condition_embedder\.time_proj\.(.*)$": + r"condition_embedder.time_modulation.linear.\1", + r"^condition_embedder\.image_embedder\.ff\.net\.0\.proj\.(.*)$": + r"condition_embedder.image_embedder.ff.fc_in.\1", + r"^condition_embedder\.image_embedder\.ff\.net\.2\.(.*)$": + r"condition_embedder.image_embedder.ff.fc_out.\1", + r"^blocks\.(\d+)\.attn1\.to_q\.(.*)$": + r"blocks.\1.to_q.\2", + r"^blocks\.(\d+)\.attn1\.to_k\.(.*)$": + r"blocks.\1.to_k.\2", + r"^blocks\.(\d+)\.attn1\.to_v\.(.*)$": + r"blocks.\1.to_v.\2", + r"^blocks\.(\d+)\.attn1\.to_out\.0\.(.*)$": + r"blocks.\1.to_out.\2", + r"^blocks\.(\d+)\.attn1\.norm_q\.(.*)$": + r"blocks.\1.norm_q.\2", + r"^blocks\.(\d+)\.attn1\.norm_k\.(.*)$": + r"blocks.\1.norm_k.\2", + r"^blocks\.(\d+)\.attn2\.to_out\.0\.(.*)$": + r"blocks.\1.attn2.to_out.\2", + r"^blocks\.(\d+)\.ffn\.net\.0\.proj\.(.*)$": + r"blocks.\1.ffn.fc_in.\2", + r"^blocks\.(\d+)\.ffn\.net\.2\.(.*)$": + r"blocks.\1.ffn.fc_out.\2", + r"^blocks\.(\d+)\.norm2\.(.*)$": + r"blocks.\1.self_attn_residual_norm.norm.\2", + }) + + # Reverse mapping for saving checkpoints: custom -> hf + reverse_param_names_mapping: dict = field(default_factory=lambda: {}) + + # Some LoRA adapters use the original official layer names instead of hf layer names, + # so apply this before the param_names_mapping + lora_param_names_mapping: dict = field( + default_factory=lambda: { + r"^blocks\.(\d+)\.self_attn\.q\.(.*)$": r"blocks.\1.attn1.to_q.\2", + r"^blocks\.(\d+)\.self_attn\.k\.(.*)$": r"blocks.\1.attn1.to_k.\2", + r"^blocks\.(\d+)\.self_attn\.v\.(.*)$": r"blocks.\1.attn1.to_v.\2", + r"^blocks\.(\d+)\.self_attn\.o\.(.*)$": + r"blocks.\1.attn1.to_out.0.\2", + r"^blocks\.(\d+)\.cross_attn\.q\.(.*)$": r"blocks.\1.attn2.to_q.\2", + r"^blocks\.(\d+)\.cross_attn\.k\.(.*)$": r"blocks.\1.attn2.to_k.\2", + r"^blocks\.(\d+)\.cross_attn\.v\.(.*)$": r"blocks.\1.attn2.to_v.\2", + r"^blocks\.(\d+)\.cross_attn\.o\.(.*)$": + r"blocks.\1.attn2.to_out.0.\2", + r"^blocks\.(\d+)\.ffn\.0\.(.*)$": r"blocks.\1.ffn.fc_in.\2", + r"^blocks\.(\d+)\.ffn\.2\.(.*)$": r"blocks.\1.ffn.fc_out.\2", + }) + + patch_size: tuple[int, int, int] = (1, 2, 2) + text_len = 512 + num_attention_heads: int = 40 + attention_head_dim: int = 128 + in_channels: int = 16 + out_channels: int = 16 + text_dim: int = 4096 + freq_dim: int = 256 + ffn_dim: int = 13824 + num_layers: int = 40 + cross_attn_norm: bool = True + qk_norm: str = "rms_norm_across_heads" + eps: float = 1e-6 + image_dim: int | None = None + added_kv_proj_dim: int | None = None + rope_max_seq_len: int = 1024 + pos_embed_seq_len: int | None = None + exclude_lora_layers: list[str] = field(default_factory=lambda: ["embedder"]) + + # Wan MoE + boundary_ratio: float | None = None + + # Causal Wan + local_attn_size: int = -1 # Window size for temporal local attention (-1 indicates global attention) + sink_size: int = 0 # Size of the attention sink, we keep the first `sink_size` frames unchanged when rolling the KV cache + num_frames_per_block: int = 3 + sliding_window_num_frames: int = 21 + + def __post_init__(self): + super().__post_init__() + self.out_channels = self.out_channels or self.in_channels + self.hidden_size = self.num_attention_heads * self.attention_head_dim + self.num_channels_latents = self.out_channels + + +@dataclass +class WanGameVideoConfig(DiTConfig): + arch_config: DiTArchConfig = field(default_factory=WanGameVideoArchConfig) + + prefix: str = "WanGame" diff --git a/fastvideo/configs/pipelines/registry.py b/fastvideo/configs/pipelines/registry.py index 3228f400a..d4f9dfcba 100644 --- a/fastvideo/configs/pipelines/registry.py +++ b/fastvideo/configs/pipelines/registry.py @@ -42,6 +42,7 @@ "FastVideo/HY-WorldPlay-Bidirectional-Diffusers": HYWorldConfig, "Wan-AI/Wan2.1-T2V-1.3B-Diffusers": WanT2V480PConfig, "weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers": WanI2V480PConfig, + "weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers": WanI2V480PConfig, "IRMChen/Wan2.1-Fun-1.3B-Control-Diffusers": WANV2VConfig, "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers": WanI2V480PConfig, "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers": WanI2V720PConfig, diff --git a/fastvideo/configs/sample/registry.py b/fastvideo/configs/sample/registry.py index 22014b06b..f8253781b 100644 --- a/fastvideo/configs/sample/registry.py +++ b/fastvideo/configs/sample/registry.py @@ -58,6 +58,8 @@ "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers": WanI2V_14B_720P_SamplingParam, "weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers": Wan2_1_Fun_1_3B_InP_SamplingParam, + "weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers": + Wan2_1_Fun_1_3B_InP_SamplingParam, "IRMChen/Wan2.1-Fun-1.3B-Control-Diffusers": Wan2_1_Fun_1_3B_Control_SamplingParam, diff --git a/fastvideo/configs/sample/wan.py b/fastvideo/configs/sample/wan.py index 85f76a7ac..9cd7b2940 100644 --- a/fastvideo/configs/sample/wan.py +++ b/fastvideo/configs/sample/wan.py @@ -113,13 +113,13 @@ class FastWanT2V480P_SamplingParam(WanT2V_1_3B_SamplingParam): @dataclass class Wan2_1_Fun_1_3B_InP_SamplingParam(SamplingParam): """Sampling parameters for Wan2.1 Fun 1.3B InP model.""" - height: int = 480 - width: int = 832 - num_frames: int = 81 - fps: int = 16 + height: int = 352 + width: int = 640 + num_frames: int = 77 + fps: int = 25 negative_prompt: str | None = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" - guidance_scale: float = 6.0 - num_inference_steps: int = 50 + guidance_scale: float = 1.0 + num_inference_steps: int = 40 @dataclass diff --git a/fastvideo/models/dits/hyworld/pose.py b/fastvideo/models/dits/hyworld/pose.py index 6274b3a93..3ee0b9856 100644 --- a/fastvideo/models/dits/hyworld/pose.py +++ b/fastvideo/models/dits/hyworld/pose.py @@ -433,6 +433,10 @@ def process_custom_actions(keyboard_tensor, mouse_tensor, forward_speed=DEFAULT_ Process custom keyboard and mouse tensors into model inputs (viewmats, intrinsics, action_labels). Assumes inputs correspond to each LATENT frame. """ + if keyboard_tensor.ndim == 3: + keyboard_tensor = keyboard_tensor.squeeze(0) + if mouse_tensor.ndim == 3: + mouse_tensor = mouse_tensor.squeeze(0) keyboard_tensor, mouse_tensor = reformat_keyboard_and_mouse_tensors(keyboard_tensor, mouse_tensor) motions = [] diff --git a/fastvideo/models/dits/wangame/__init__.py b/fastvideo/models/dits/wangame/__init__.py new file mode 100644 index 000000000..8b7d5c92b --- /dev/null +++ b/fastvideo/models/dits/wangame/__init__.py @@ -0,0 +1,5 @@ +from .model import WanGameActionTransformer3DModel + +__all__ = [ + "WanGameActionTransformer3DModel", +] diff --git a/fastvideo/models/dits/wangame/hyworld_action_module.py b/fastvideo/models/dits/wangame/hyworld_action_module.py new file mode 100644 index 000000000..cdc928bb2 --- /dev/null +++ b/fastvideo/models/dits/wangame/hyworld_action_module.py @@ -0,0 +1,231 @@ +import torch +import torch.nn as nn + +from fastvideo.layers.visual_embedding import TimestepEmbedder, ModulateProjection, timestep_embedding +from fastvideo.platforms import AttentionBackendEnum +from fastvideo.attention import DistributedAttention +from fastvideo.forward_context import set_forward_context +from fastvideo.models.dits.wanvideo import WanImageEmbedding + +from fastvideo.models.dits.hyworld.camera_rope import prope_qkv +from fastvideo.layers.rotary_embedding import _apply_rotary_emb +from fastvideo.layers.mlp import MLP + +class WanGameActionTimeImageEmbedding(nn.Module): + def __init__( + self, + dim: int, + time_freq_dim: int, + image_embed_dim: int | None = None, + ): + super().__init__() + + self.time_freq_dim = time_freq_dim + self.time_embedder = TimestepEmbedder( + dim, frequency_embedding_size=time_freq_dim, act_layer="silu") + self.time_modulation = ModulateProjection(dim, + factor=6, + act_layer="silu") + + self.image_embedder = None + if image_embed_dim is not None: + self.image_embedder = WanImageEmbedding(image_embed_dim, dim) + + self.action_embedder = MLP( + time_freq_dim, + dim, + dim, + bias=True, + act_type="silu" + ) + # Initialize with zeros for residual-like behavior + nn.init.zeros_(self.action_embedder.fc_out.weight) + if self.action_embedder.fc_out.bias is not None: + nn.init.zeros_(self.action_embedder.fc_out.bias) + + def forward( + self, + timestep: torch.Tensor, + action: torch.Tensor, + encoder_hidden_states: torch.Tensor, # Kept for interface compatibility + encoder_hidden_states_image: torch.Tensor | None = None, + timestep_seq_len: int | None = None, + ): + temb = self.time_embedder(timestep, timestep_seq_len) + + action_emb = timestep_embedding(action.flatten(), self.time_freq_dim) + action_embedder_dtype = next(iter(self.action_embedder.parameters())).dtype + if ( + action_emb.dtype != action_embedder_dtype + and action_embedder_dtype != torch.int8 + ): + action_emb = action_emb.to(action_embedder_dtype) + action_emb = self.action_embedder(action_emb).type_as(temb) + temb = temb + action_emb + + timestep_proj = self.time_modulation(temb) + + # MatrixGame does not use text embeddings, so we ignore encoder_hidden_states + + if encoder_hidden_states_image is not None: + assert self.image_embedder is not None + encoder_hidden_states_image = self.image_embedder( + encoder_hidden_states_image) + + encoder_hidden_states = torch.zeros((timestep.shape[0], 0, temb.shape[-1]), + device=temb.device, + dtype=temb.dtype) + + return temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image + +class WanGameActionSelfAttention(nn.Module): + """ + Self-attention module with support for: + - Standard RoPE-based attention + - Camera PRoPE-based attention (when viewmats and Ks are provided) + - KV caching for autoregressive generation + """ + + def __init__(self, + dim: int, + num_heads: int, + local_attn_size: int = -1, + sink_size: int = 0, + qk_norm=True, + eps=1e-6) -> None: + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.local_attn_size = local_attn_size + self.sink_size = sink_size + self.qk_norm = qk_norm + self.eps = eps + self.max_attention_size = 32760 if local_attn_size == -1 else local_attn_size * 1560 + + # Scaled dot product attention (using DistributedAttention for SP support) + self.attn = DistributedAttention( + num_heads=num_heads, + head_size=self.head_dim, + softmax_scale=None, + causal=False, + supported_attention_backends=(AttentionBackendEnum.FLASH_ATTN, + AttentionBackendEnum.TORCH_SDPA)) + + def forward(self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + kv_cache: dict | None = None, + current_start: int = 0, + cache_start: int | None = None, + viewmats: torch.Tensor | None = None, + Ks: torch.Tensor | None = None, + is_cache: bool = False, + attention_mask: torch.Tensor | None = None): + """ + Forward pass with camera PRoPE attention combining standard RoPE and projective positional encoding. + + Args: + q, k, v: Query, key, value tensors [B, L, num_heads, head_dim] + freqs_cis: RoPE frequency cos/sin tensors + kv_cache: KV cache dict (may have None values for training) + current_start: Current position for KV cache + cache_start: Cache start position + viewmats: Camera view matrices for PRoPE [B, cameras, 4, 4] + Ks: Camera intrinsics for PRoPE [B, cameras, 3, 3] + is_cache: Whether to store to KV cache (for inference) + attention_mask: Attention mask [B, L] (1 = attend, 0 = mask) + """ + if cache_start is None: + cache_start = current_start + + # Apply RoPE manually + cos, sin = freqs_cis + query_rope = _apply_rotary_emb(q, cos, sin, is_neox_style=False).type_as(v) + key_rope = _apply_rotary_emb(k, cos, sin, is_neox_style=False).type_as(v) + value_rope = v + + # Get PRoPE transformed q, k, v + query_prope, key_prope, value_prope, apply_fn_o = prope_qkv( + q.transpose(1, 2), # [B, num_heads, L, head_dim] + k.transpose(1, 2), + v.transpose(1, 2), + viewmats=viewmats, + Ks=Ks, + patches_x=40, # hardcoded for now + patches_y=22, + ) + # PRoPE returns [B, num_heads, L, head_dim], convert to [B, L, num_heads, head_dim] + query_prope = query_prope.transpose(1, 2) + key_prope = key_prope.transpose(1, 2) + value_prope = value_prope.transpose(1, 2) + + # KV cache handling + if kv_cache is not None: + cache_key = kv_cache.get("k", None) + cache_value = kv_cache.get("v", None) + + if cache_value is not None and not is_cache: + cache_key_rope, cache_key_prope = cache_key.chunk(2, dim=-1) + cache_value_rope, cache_value_prope = cache_value.chunk(2, dim=-1) + + key_rope = torch.cat([cache_key_rope, key_rope], dim=1) + value_rope = torch.cat([cache_value_rope, value_rope], dim=1) + key_prope = torch.cat([cache_key_prope, key_prope], dim=1) + value_prope = torch.cat([cache_value_prope, value_prope], dim=1) + + if is_cache: + # Store to cache (update input dict directly) + kv_cache["k"] = torch.cat([key_rope, key_prope], dim=-1) + kv_cache["v"] = torch.cat([value_rope, value_prope], dim=-1) + + # Concatenate rope and prope paths (matching original) + query_all = torch.cat([query_rope, query_prope], dim=0) + key_all = torch.cat([key_rope, key_prope], dim=0) + value_all = torch.cat([value_rope, value_prope], dim=0) + + # Check if Q and KV have different sequence lengths (KV cache mode) + # In this case, use LocalAttention (supports different Q/KV lengths) + if query_all.shape[1] != key_all.shape[1]: + raise ValueError("Q and KV have different sequence lengths") + # KV cache mode: Q has new tokens only, KV has cached + new tokens + # Use LocalAttention which supports different Q/KV lengths + # LocalAttention will use the appropriate backend (SageAttn, FlashAttn, etc.) + if not hasattr(self, '_kv_cache_attn'): + from fastvideo.attention import LocalAttention + self._kv_cache_attn = LocalAttention( + num_heads=self.num_heads, + head_size=self.head_dim, + causal=False, + supported_attention_backends=(AttentionBackendEnum.SAGE_ATTN, + AttentionBackendEnum.FLASH_ATTN, + AttentionBackendEnum.TORCH_SDPA) + ) + hidden_states_all = self._kv_cache_attn(query_all, key_all, value_all) + else: + # Same sequence length: use DistributedAttention (supports SP) + # Create default attention mask if not provided + if attention_mask is None: + batch_size, seq_len = q.shape[0], q.shape[1] + attention_mask = torch.ones(batch_size, seq_len, device=q.device, dtype=q.dtype) + + if q.dtype == torch.float32: + from fastvideo.attention.backends.sdpa import SDPAMetadataBuilder + attn_metadata_builder = SDPAMetadataBuilder + else: + from fastvideo.attention.backends.flash_attn import FlashAttnMetadataBuilder + attn_metadata_builder = FlashAttnMetadataBuilder + attn_metadata = attn_metadata_builder().build( + current_timestep=0, + attn_mask=attention_mask, + ) + with set_forward_context(current_timestep=0, attn_metadata=attn_metadata): + hidden_states_all, _ = self.attn(query_all, key_all, value_all, attention_mask=attention_mask) + + hidden_states_rope, hidden_states_prope = hidden_states_all.chunk(2, dim=0) + hidden_states_prope = apply_fn_o(hidden_states_prope.transpose(1, 2)).transpose(1, 2) + + return hidden_states_rope, hidden_states_prope \ No newline at end of file diff --git a/fastvideo/models/dits/wangame/model.py b/fastvideo/models/dits/wangame/model.py new file mode 100644 index 000000000..685f2299f --- /dev/null +++ b/fastvideo/models/dits/wangame/model.py @@ -0,0 +1,424 @@ +# SPDX-License-Identifier: Apache-2.0 + +import math +from typing import Any + +import torch +import torch.nn as nn + +from fastvideo.configs.models.dits.wangamevideo import WanGameVideoConfig +from fastvideo.distributed.parallel_state import get_sp_world_size +from fastvideo.layers.layernorm import (FP32LayerNorm, LayerNormScaleShift, + RMSNorm, ScaleResidual, + ScaleResidualLayerNormScaleShift) +from fastvideo.layers.linear import ReplicatedLinear +from fastvideo.layers.mlp import MLP +from fastvideo.layers.rotary_embedding import (_apply_rotary_emb, + get_rotary_pos_embed) +from fastvideo.layers.visual_embedding import PatchEmbed +from fastvideo.logger import init_logger +from fastvideo.models.dits.base import BaseDiT +from fastvideo.models.dits.wanvideo import WanI2VCrossAttention +from fastvideo.platforms import AttentionBackendEnum, current_platform + +# Import ActionModule +from fastvideo.models.dits.wangame.hyworld_action_module import WanGameActionTimeImageEmbedding, WanGameActionSelfAttention + +logger = init_logger(__name__) + + +class WanGameCrossAttention(WanI2VCrossAttention): + def forward(self, x, context, context_lens=None): + r""" + Args: + x(Tensor): Shape [B, L1, C] + context(Tensor): Shape [B, L2, C] + context_lens(Tensor): Shape [B] + """ + context_img = context + b, n, d = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.norm_q(self.to_q(x)[0]).view(b, -1, n, d) + k_img = self.norm_added_k(self.add_k_proj(context_img)[0]).view( + b, -1, n, d) + v_img = self.add_v_proj(context_img)[0].view(b, -1, n, d) + img_x = self.attn(q, k_img, v_img) + + # output + x = img_x.flatten(2) + x, _ = self.to_out(x) + return x + +class WanGameActionTransformerBlock(nn.Module): + """ + Transformer block for WAN Action model with support for: + - Self-attention with RoPE and camera PRoPE + - Cross-attention with text/image context + - Feed-forward network with AdaLN modulation + """ + + def __init__(self, + dim: int, + ffn_dim: int, + num_heads: int, + local_attn_size: int = -1, + sink_size: int = 0, + qk_norm: str = "rms_norm_across_heads", + cross_attn_norm: bool = False, + eps: float = 1e-6, + added_kv_proj_dim: int | None = None, + supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None, + prefix: str = ""): + super().__init__() + + # 1. Self-attention + self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False) + self.to_q = ReplicatedLinear(dim, dim, bias=True) + self.to_k = ReplicatedLinear(dim, dim, bias=True) + self.to_v = ReplicatedLinear(dim, dim, bias=True) + self.to_out = ReplicatedLinear(dim, dim, bias=True) + + self.attn1 = WanGameActionSelfAttention( + dim, + num_heads, + local_attn_size=local_attn_size, + sink_size=sink_size, + qk_norm=qk_norm, + eps=eps) + + self.hidden_dim = dim + self.num_attention_heads = num_heads + self.local_attn_size = local_attn_size + dim_head = dim // num_heads + + if qk_norm == "rms_norm": + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + elif qk_norm == "rms_norm_across_heads": + self.norm_q = RMSNorm(dim, eps=eps) + self.norm_k = RMSNorm(dim, eps=eps) + else: + raise ValueError(f"QK Norm type {qk_norm} not supported") + + assert cross_attn_norm is True + self.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( + dim, + norm_type="layer", + eps=eps, + elementwise_affine=True, + compute_dtype=torch.float32) + + # 2. Cross-attention (I2V only for now) + self.attn2 = WanGameCrossAttention(dim, + num_heads, + qk_norm=qk_norm, + eps=eps) + # norm3 for FFN input + self.norm3 = LayerNormScaleShift(dim, norm_type="layer", eps=eps, + elementwise_affine=False) + + # 3. Feed-forward + self.ffn = MLP(dim, ffn_dim, act_type="gelu_pytorch_tanh") + self.mlp_residual = ScaleResidual() + + self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + + # PRoPE output projection (initialized via add_discrete_action_parameters on the model) + self.to_out_prope = nn.ModuleList([ + nn.Linear(dim, dim, bias=True), + ]) + nn.init.zeros_(self.to_out_prope[0].weight) + if self.to_out_prope[0].bias is not None: + nn.init.zeros_(self.to_out_prope[0].bias) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + kv_cache: dict | None = None, + crossattn_cache: dict | None = None, + current_start: int = 0, + cache_start: int | None = None, + viewmats: torch.Tensor | None = None, + Ks: torch.Tensor | None = None, + is_cache: bool = False, + ) -> torch.Tensor: + if hidden_states.dim() == 4: + hidden_states = hidden_states.squeeze(1) + + num_frames = temb.shape[1] + frame_seqlen = hidden_states.shape[1] // num_frames + bs, seq_length, _ = hidden_states.shape + orig_dtype = hidden_states.dtype + + # Cast temb to float32 for scale/shift computation + e = self.scale_shift_table + temb.float() + assert e.shape == (bs, num_frames, 6, self.hidden_dim) + shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(6, dim=2) + + # 1. Self-attention + norm_hidden_states = (self.norm1(hidden_states.float()).unflatten(dim=1, sizes=(num_frames, frame_seqlen)) * + (1 + scale_msa) + shift_msa).to(orig_dtype).flatten(1, 2) + + query, _ = self.to_q(norm_hidden_states) + key, _ = self.to_k(norm_hidden_states) + value, _ = self.to_v(norm_hidden_states) + + if self.norm_q is not None: + query = self.norm_q.forward_native(query) + if self.norm_k is not None: + key = self.norm_k.forward_native(key) + + query = query.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + + # Self-attention with optional camera PRoPE + attn_output_rope, attn_output_prope = self.attn1( + query, key, value, freqs_cis, + kv_cache, current_start, cache_start, viewmats, Ks, + is_cache=is_cache + ) + # Combine rope and prope outputs + attn_output_rope = attn_output_rope.flatten(2) + attn_output_rope, _ = self.to_out(attn_output_rope) + attn_output_prope = attn_output_prope.flatten(2) + attn_output_prope = self.to_out_prope[0](attn_output_prope) + attn_output = attn_output_rope.squeeze(1) + attn_output_prope.squeeze(1) + + # Self-attention residual + norm in float32 + null_shift = null_scale = torch.zeros(1, device=hidden_states.device, dtype=torch.float32) + norm_hidden_states, hidden_states = self.self_attn_residual_norm( + hidden_states.float(), attn_output.float(), gate_msa, null_shift, null_scale) + hidden_states = hidden_states.type_as(attn_output) + norm_hidden_states = norm_hidden_states.type_as(attn_output) + + # 2. Cross-attention + attn_output = self.attn2(norm_hidden_states.to(orig_dtype), + context=encoder_hidden_states, + context_lens=None) + # Cross-attention residual in bfloat16 + hidden_states = hidden_states + attn_output + + # norm3 for FFN input in float32 + norm_hidden_states = self.norm3( + hidden_states.float(), c_shift_msa, c_scale_msa + ).type_as(hidden_states) + + # 3. Feed-forward + ff_output = self.ffn(norm_hidden_states.to(orig_dtype)) + hidden_states = self.mlp_residual(hidden_states.float(), ff_output.float(), c_gate_msa) + hidden_states = hidden_states.to(orig_dtype) # Cast back to original dtype + + return hidden_states + +class WanGameActionTransformer3DModel(BaseDiT): + """ + WAN Action Transformer 3D Model for video generation with action conditioning. + + Extends the base WAN video model with: + - Action embedding support for controllable generation + - camera PRoPE attention for 3D-aware generation + - KV caching for autoregressive inference + """ + _fsdp_shard_conditions = WanGameVideoConfig()._fsdp_shard_conditions + _compile_conditions = WanGameVideoConfig()._compile_conditions + _supported_attention_backends = WanGameVideoConfig()._supported_attention_backends + param_names_mapping = WanGameVideoConfig().param_names_mapping + reverse_param_names_mapping = WanGameVideoConfig().reverse_param_names_mapping + lora_param_names_mapping = WanGameVideoConfig().lora_param_names_mapping + + def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> None: + super().__init__(config=config, hf_config=hf_config) + + inner_dim = config.num_attention_heads * config.attention_head_dim + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + self.attention_head_dim = config.attention_head_dim + self.in_channels = config.in_channels + self.out_channels = config.out_channels + self.num_channels_latents = config.num_channels_latents + self.patch_size = config.patch_size + self.local_attn_size = config.local_attn_size + self.inner_dim = inner_dim + + # 1. Patch & position embedding + self.patch_embedding = PatchEmbed(in_chans=config.in_channels, + embed_dim=inner_dim, + patch_size=config.patch_size, + flatten=False) + + # 2. Condition embeddings (with action support) + self.condition_embedder = WanGameActionTimeImageEmbedding( + dim=inner_dim, + time_freq_dim=config.freq_dim, + image_embed_dim=config.image_dim, + ) + + # 3. Transformer blocks + self.blocks = nn.ModuleList([ + WanGameActionTransformerBlock( + inner_dim, + config.ffn_dim, + config.num_attention_heads, + config.local_attn_size, + config.sink_size, + config.qk_norm, + config.cross_attn_norm, + config.eps, + config.added_kv_proj_dim, + supported_attention_backends=self._supported_attention_backends, + prefix=f"{config.prefix}.blocks.{i}") + for i in range(config.num_layers) + ]) + + # 4. Output norm & projection + self.norm_out = LayerNormScaleShift(inner_dim, + norm_type="layer", + eps=config.eps, + elementwise_affine=False, + dtype=torch.float32) + self.proj_out = nn.Linear( + inner_dim, config.out_channels * math.prod(config.patch_size)) + self.scale_shift_table = nn.Parameter(torch.randn(1, 2, inner_dim) / inner_dim**0.5) + + self.gradient_checkpointing = False + + # Causal-specific + self.num_frame_per_block = config.arch_config.num_frames_per_block + assert self.num_frame_per_block <= 3 + + self.__post_init__() + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor], + guidance=None, + action: torch.Tensor | None = None, + viewmats: torch.Tensor | None = None, + Ks: torch.Tensor | None = None, + kv_cache: list[dict] | None = None, + crossattn_cache: list[dict] | None = None, + current_start: int = 0, + cache_start: int = 0, + start_frame: int = 0, + is_cache: bool = False, + **kwargs + ) -> torch.Tensor: + """ + Forward pass for both training and inference with KV caching. + + Args: + hidden_states: Video latents [B, C, T, H, W] + encoder_hidden_states: Text embeddings [B, L, D] + timestep: Timestep tensor + encoder_hidden_states_image: Optional image embeddings + action: Action tensor [B, T] for per-frame conditioning + viewmats: Camera view matrices for PRoPE [B, T, 4, 4] + Ks: Camera intrinsics for PRoPE [B, T, 3, 3] + kv_cache: KV cache for autoregressive inference (list of dicts per layer) + crossattn_cache: Cross-attention cache for inference + current_start: Current position for KV cache + cache_start: Cache start position + start_frame: RoPE offset for new frames in autoregressive mode + is_cache: If True, populate KV cache and return early (cache-only mode) + """ + orig_dtype = hidden_states.dtype + # if not isinstance(encoder_hidden_states, torch.Tensor): + # encoder_hidden_states = encoder_hidden_states[0] + if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: + encoder_hidden_states_image = encoder_hidden_states_image[0] + # else: + # encoder_hidden_states_image = None + + batch_size, num_channels, num_frames, height, width = hidden_states.shape + p_t, p_h, p_w = self.patch_size + post_patch_num_frames = num_frames // p_t + post_patch_height = height // p_h + post_patch_width = width // p_w + + # Get rotary embeddings + d = self.hidden_size // self.num_attention_heads + rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] + freqs_cos, freqs_sin = get_rotary_pos_embed( + (post_patch_num_frames * get_sp_world_size(), post_patch_height, post_patch_width), + self.hidden_size, + self.num_attention_heads, + rope_dim_list, + dtype=torch.float32 if current_platform.is_mps() else torch.float64, + rope_theta=10000, + start_frame=start_frame + ) + freqs_cos = freqs_cos.to(hidden_states.device) + freqs_sin = freqs_sin.to(hidden_states.device) + freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None + + hidden_states = self.patch_embedding(hidden_states) + hidden_states = hidden_states.flatten(2).transpose(1, 2) + + if timestep.dim() == 2: + timestep = timestep.flatten() + + temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( + timestep, action, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) + # Reshape timestep_proj: [T, 6*dim] -> [B, T, 6, dim] + # For training: batch_size=1, T=num_frames (diffusion forcing) + # For inference: batch_size can vary + timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) + if timestep_proj.shape[0] == post_patch_num_frames and batch_size == 1: + # Training mode: timestep_proj is [T, 6, dim], add batch dim -> [1, T, 6, dim] + timestep_proj = timestep_proj.unsqueeze(0) + else: + # Inference mode: reshape based on timestep shape + timestep_proj = timestep_proj.unflatten(dim=0, sizes=timestep.shape) + + encoder_hidden_states = encoder_hidden_states_image + + # Transformer blocks + for block_idx, block in enumerate(self.blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + block, hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, + kv_cache[block_idx] if kv_cache else None, + crossattn_cache[block_idx] if crossattn_cache else None, + current_start, cache_start, + viewmats, Ks, is_cache) + else: + hidden_states = block( + hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, + kv_cache[block_idx] if kv_cache else None, + crossattn_cache[block_idx] if crossattn_cache else None, + current_start, cache_start, + viewmats, Ks, is_cache) + + # If cache-only mode, return early + if is_cache: + return kv_cache + + # Output norm, projection & unpatchify + # Reshape temb to match timestep_proj shape: [T, dim] -> [B, T, 1, dim] + if temb.shape[0] == post_patch_num_frames and batch_size == 1: + # Training mode: temb is [T, dim] -> [1, T, 1, dim] + temb = temb.unsqueeze(0).unsqueeze(2) + else: + # Inference mode: reshape based on timestep shape + temb = temb.unflatten(dim=0, sizes=timestep.shape).unsqueeze(2) + + shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) + hidden_states = self.norm_out(hidden_states, shift, scale) + hidden_states = self.proj_out(hidden_states) + + hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, + post_patch_height, + post_patch_width, p_t, p_h, p_w, + -1) + hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) + output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + return output \ No newline at end of file diff --git a/fastvideo/models/loader/component_loader.py b/fastvideo/models/loader/component_loader.py index bf53d324c..18dc392a9 100644 --- a/fastvideo/models/loader/component_loader.py +++ b/fastvideo/models/loader/component_loader.py @@ -806,6 +806,10 @@ def load(self, model_path: str, fastvideo_args: FastVideoArgs): cls_name.startswith("Cosmos25") or cls_name == "Cosmos25Transformer3DModel" or getattr(fastvideo_args.pipeline_config, "prefix", "") == "Cosmos25" + ) and not ( + cls_name.startswith("WanGame") + or cls_name == "WanGameActionTransformer3DModel" + or getattr(fastvideo_args.pipeline_config, "prefix", "") == "WanGame" ) model = maybe_load_fsdp_model( model_cls=model_cls, diff --git a/fastvideo/models/loader/fsdp_load.py b/fastvideo/models/loader/fsdp_load.py index 9ba60320a..9b590a86c 100644 --- a/fastvideo/models/loader/fsdp_load.py +++ b/fastvideo/models/loader/fsdp_load.py @@ -138,7 +138,7 @@ def maybe_load_fsdp_model( weight_iterator = safetensors_weights_iterator(weight_dir_list) param_names_mapping_fn = get_param_names_mapping(model.param_names_mapping) - load_model_from_full_model_state_dict( + incompatible_keys, unexpected_keys = load_model_from_full_model_state_dict( model, weight_iterator, device, @@ -147,6 +147,9 @@ def maybe_load_fsdp_model( cpu_offload=cpu_offload, param_names_mapping=param_names_mapping_fn, ) + if incompatible_keys or unexpected_keys: + logger.warning("Incompatible keys: %s", incompatible_keys) + logger.warning("Unexpected keys: %s", unexpected_keys) for n, p in chain(model.named_parameters(), model.named_buffers()): if p.is_meta: raise RuntimeError( @@ -340,7 +343,7 @@ def load_model_from_full_model_state_dict( unused_keys) # List of allowed parameter name patterns - ALLOWED_NEW_PARAM_PATTERNS = ["gate_compress", "proj_l"] # Can be extended as needed + ALLOWED_NEW_PARAM_PATTERNS = ["gate_compress", "proj_l", "to_out_prope", "action_embedder"] # Can be extended as needed for new_param_name in unused_keys: if not any(pattern in new_param_name for pattern in ALLOWED_NEW_PARAM_PATTERNS): diff --git a/fastvideo/models/registry.py b/fastvideo/models/registry.py index 954b96049..7861918a3 100644 --- a/fastvideo/models/registry.py +++ b/fastvideo/models/registry.py @@ -42,6 +42,7 @@ # "HunyuanVideoTransformer3DModel": ("dits", "hunyuanvideo", "HunyuanVideoDiT"), "WanTransformer3DModel": ("dits", "wanvideo", "WanTransformer3DModel"), "CausalWanTransformer3DModel": ("dits", "causal_wanvideo", "CausalWanTransformer3DModel"), + "WanGameActionTransformer3DModel": ("dits", "wangame", "WanGameActionTransformer3DModel"), "MatrixGameWanModel": ("dits", "matrixgame", "MatrixGameWanModel"), "CausalMatrixGameWanModel": ("dits", "matrixgame", "CausalMatrixGameWanModel"), } diff --git a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py new file mode 100644 index 000000000..e2ef763ee --- /dev/null +++ b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Wan video diffusion pipeline implementation. + +This module contains an implementation of the Wan video diffusion pipeline +using the modular pipeline architecture. +""" + +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.logger import init_logger +from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase +from fastvideo.pipelines.lora_pipeline import LoRAPipeline + +# isort: off +from fastvideo.pipelines.stages import ( + ImageEncodingStage, ConditioningStage, DecodingStage, DenoisingStage, + ImageVAEEncodingStage, InputValidationStage, LatentPreparationStage, + TimestepPreparationStage) +# isort: on +from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler) + +logger = init_logger(__name__) + + +class WanGameActionImageToVideoPipeline(LoRAPipeline, ComposedPipelineBase): + + _required_config_modules = [ + "vae", "transformer", "scheduler", \ + "image_encoder", "image_processor" + ] + + def initialize_pipeline(self, fastvideo_args: FastVideoArgs): + self.modules["scheduler"] = FlowUniPCMultistepScheduler( + shift=fastvideo_args.pipeline_config.flow_shift) + + def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): + """Set up pipeline stages with proper dependency injection.""" + + self.add_stage(stage_name="input_validation_stage", + stage=InputValidationStage()) + + self.add_stage( + stage_name="image_encoding_stage", + stage=ImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + )) + + self.add_stage(stage_name="conditioning_stage", + stage=ConditioningStage()) + + self.add_stage(stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"))) + + self.add_stage(stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer"))) + + self.add_stage(stage_name="image_latent_preparation_stage", + stage=ImageVAEEncodingStage(vae=self.get_module("vae"))) + + self.add_stage(stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"))) + + self.add_stage(stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"))) + + +EntryClass = WanGameActionImageToVideoPipeline diff --git a/fastvideo/pipelines/pipeline_registry.py b/fastvideo/pipelines/pipeline_registry.py index dc00f8e42..0540868b2 100644 --- a/fastvideo/pipelines/pipeline_registry.py +++ b/fastvideo/pipelines/pipeline_registry.py @@ -21,6 +21,7 @@ "WanPipeline": "wan", "WanDMDPipeline": "wan", "WanImageToVideoPipeline": "wan", + "WanGameActionImageToVideoPipeline": "wan", "WanVideoToVideoPipeline": "wan", "WanCausalDMDPipeline": "wan", "TurboDiffusionPipeline": "turbodiffusion", diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index a6c235b5d..3791e725c 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -168,6 +168,20 @@ def forward( }, ) + if batch.mouse_cond is not None and batch.keyboard_cond is not None: + from fastvideo.models.dits.hyworld.pose import process_custom_actions + viewmats, intrinsics, action_labels = process_custom_actions(batch.keyboard_cond, batch.mouse_cond) + camera_action_kwargs = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + "viewmats": viewmats.unsqueeze(0).to(get_local_torch_device(), dtype=target_dtype), + "Ks": intrinsics.unsqueeze(0).to(get_local_torch_device(), dtype=target_dtype), + "action": action_labels.unsqueeze(0).to(get_local_torch_device(), dtype=target_dtype), + }, + ) + else: + camera_action_kwargs = {} + action_kwargs = self.prepare_extra_func_kwargs( self.transformer.forward, { @@ -406,6 +420,7 @@ def forward( **image_kwargs, **pos_cond_kwargs, **action_kwargs, + **camera_action_kwargs, ) if batch.do_classifier_free_guidance: @@ -423,6 +438,7 @@ def forward( **image_kwargs, **neg_cond_kwargs, **action_kwargs, + **camera_action_kwargs, ) noise_pred_text = noise_pred diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py new file mode 100644 index 000000000..5beea6abc --- /dev/null +++ b/fastvideo/training/wangame_training_pipeline.py @@ -0,0 +1,250 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys +from copy import deepcopy +from typing import Any + +import torch + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.logger import init_logger +from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler) +from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import WanGameActionImageToVideoPipeline +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch +from fastvideo.training.training_pipeline import TrainingPipeline +from fastvideo.utils import is_vsa_available, shallow_asdict + +vsa_available = is_vsa_available() + +logger = init_logger(__name__) + + +class WanGameTrainingPipeline(TrainingPipeline): + """ + A training pipeline for WanGame-2.1-Fun-1.3B-InP. + """ + _required_config_modules = ["scheduler", "transformer", "vae"] + + def initialize_pipeline(self, fastvideo_args: FastVideoArgs): + self.modules["scheduler"] = FlowUniPCMultistepScheduler( + shift=fastvideo_args.pipeline_config.flow_shift) + + def create_training_stages(self, training_args: TrainingArgs): + """ + May be used in future refactors. + """ + pass + + def set_schemas(self): + self.train_dataset_schema = pyarrow_schema_wangame + + def initialize_validation_pipeline(self, training_args: TrainingArgs): + logger.info("Initializing validation pipeline...") + # args_copy.pipeline_config.vae_config.load_encoder = False + # validation_pipeline = WanImageToVideoValidationPipeline.from_pretrained( + self.validation_pipeline = WanGameActionImageToVideoPipeline.from_pretrained( + training_args.model_path, + args=None, + inference_mode=True, + loaded_modules={ + "transformer": self.get_module("transformer"), + }, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + dit_cpu_offload=False) + + def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: + batch = next(self.train_loader_iter, None) # type: ignore + if batch is None: + self.current_epoch += 1 + logger.info("Starting epoch %s", self.current_epoch) + # Reset iterator for next epoch + self.train_loader_iter = iter(self.train_dataloader) + # Get first batch of new epoch + batch = next(self.train_loader_iter) + + latents = batch['vae_latent'] + latents = latents[:, :, :self.training_args.num_latent_t] + # encoder_hidden_states = batch['text_embedding'] + # encoder_attention_mask = batch['text_attention_mask'] + clip_features = batch['clip_feature'] + image_latents = batch['first_frame_latent'] + image_latents = image_latents[:, :, :self.training_args.num_latent_t] + pil_image = batch['pil_image'] + infos = batch['info_list'] + + training_batch.latents = latents.to(get_local_torch_device(), + dtype=torch.bfloat16) + training_batch.encoder_hidden_states = None + training_batch.encoder_attention_mask = None + # MatrixGame doesn't use text encoder + training_batch.preprocessed_image = pil_image.to( + get_local_torch_device()) + training_batch.image_embeds = clip_features.to(get_local_torch_device()) + training_batch.image_latents = image_latents.to( + get_local_torch_device()) + training_batch.infos = infos + + # Action conditioning + if 'mouse_cond' in batch and batch['mouse_cond'].numel() > 0: + training_batch.mouse_cond = batch['mouse_cond'].to( + get_local_torch_device(), dtype=torch.bfloat16) + else: + training_batch.mouse_cond = None + + if 'keyboard_cond' in batch and batch['keyboard_cond'].numel() > 0: + training_batch.keyboard_cond = batch['keyboard_cond'].to( + get_local_torch_device(), dtype=torch.bfloat16) + else: + training_batch.keyboard_cond = None + + return training_batch + + def _prepare_dit_inputs(self, + training_batch: TrainingBatch) -> TrainingBatch: + """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" + + # First, call parent method to prepare noise, timesteps, etc. for video latents + training_batch = super()._prepare_dit_inputs(training_batch) + + assert isinstance(training_batch.image_latents, torch.Tensor) + image_latents = training_batch.image_latents.to( + get_local_torch_device(), dtype=torch.bfloat16) + + temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (self.training_args.num_latent_t - + 1) * temporal_compression_ratio + 1 + batch_size, num_channels, _, latent_height, latent_width = image_latents.shape + mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, + latent_width) + mask_lat_size[:, :, 1:] = 0 + + first_frame_mask = mask_lat_size[:, :, :1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, dim=2, repeats=temporal_compression_ratio) + mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], + dim=2) + mask_lat_size = mask_lat_size.view(batch_size, -1, + temporal_compression_ratio, + latent_height, latent_width) + mask_lat_size = mask_lat_size.transpose(1, 2) + mask_lat_size = mask_lat_size.to( + image_latents.device).to(dtype=torch.bfloat16) + + training_batch.noisy_model_input = torch.cat( + [training_batch.noisy_model_input, mask_lat_size, image_latents], + dim=1) + + return training_batch + + def _build_input_kwargs(self, + training_batch: TrainingBatch) -> TrainingBatch: + + # Image Embeds for conditioning + image_embeds = training_batch.image_embeds + assert torch.isnan(image_embeds).sum() == 0 + image_embeds = image_embeds.to(get_local_torch_device(), + dtype=torch.bfloat16) + encoder_hidden_states_image = image_embeds + + from fastvideo.models.dits.hyworld.pose import process_custom_actions + viewmats, intrinsics, action_labels = process_custom_actions(training_batch.keyboard_cond, training_batch.mouse_cond) + viewmats = viewmats.unsqueeze(0).to(get_local_torch_device(), dtype=torch.bfloat16) + intrinsics = intrinsics.unsqueeze(0).to(get_local_torch_device(), dtype=torch.bfloat16) + action_labels = action_labels.unsqueeze(0).to(get_local_torch_device(), dtype=torch.bfloat16) + + # NOTE: noisy_model_input already contains concatenated image_latents from _prepare_dit_inputs + training_batch.input_kwargs = { + "hidden_states": + training_batch.noisy_model_input, + "encoder_hidden_states": + training_batch.encoder_hidden_states, # None for MatrixGame + "timestep": + training_batch.timesteps.to(get_local_torch_device(), + dtype=torch.bfloat16), + # "encoder_attention_mask": + # training_batch.encoder_attention_mask, + "encoder_hidden_states_image": + encoder_hidden_states_image, + # Action conditioning + "viewmats": viewmats, + "Ks": intrinsics, + "action": action_labels, + "return_dict": + False, + } + return training_batch + + def _prepare_validation_batch(self, sampling_param: SamplingParam, + training_args: TrainingArgs, + validation_batch: dict[str, Any], + num_inference_steps: int) -> ForwardBatch: + sampling_param.prompt = validation_batch['prompt'] + sampling_param.height = training_args.num_height + sampling_param.width = training_args.num_width + sampling_param.image_path = validation_batch.get( + 'image_path') or validation_batch.get('video_path') + sampling_param.num_inference_steps = num_inference_steps + sampling_param.data_type = "video" + assert self.seed is not None + sampling_param.seed = self.seed + + latents_size = [(sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, sampling_param.width // 8] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (training_args.num_latent_t - + 1) * temporal_compression_factor + 1 + sampling_param.num_frames = num_frames + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=torch.Generator(device="cpu").manual_seed(self.seed), + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=training_args.VSA_sparsity, + ) + if "image" in validation_batch and validation_batch["image"] is not None: + batch.pil_image = validation_batch["image"] + + if "keyboard_cond" in validation_batch and validation_batch[ + "keyboard_cond"] is not None: + keyboard_cond = validation_batch["keyboard_cond"] + keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) + keyboard_cond = keyboard_cond.unsqueeze(0) + batch.keyboard_cond = keyboard_cond + + if "mouse_cond" in validation_batch and validation_batch[ + "mouse_cond"] is not None: + mouse_cond = validation_batch["mouse_cond"] + mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) + mouse_cond = mouse_cond.unsqueeze(0) + batch.mouse_cond = mouse_cond + + return batch + + +def main(args) -> None: + logger.info("Starting training pipeline...") + + pipeline = WanGameTrainingPipeline.from_pretrained( + args.pretrained_model_name_or_path, args=args) + args = pipeline.training_args + pipeline.train() + logger.info("Training pipeline done") + + +if __name__ == "__main__": + argv = sys.argv + from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.utils import FlexibleArgumentParser + parser = FlexibleArgumentParser() + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args() + args.dit_cpu_offload = False + main(args) \ No newline at end of file From bdaf2e2e458088fcab546f295182229a87fb5d1e Mon Sep 17 00:00:00 2001 From: JerryZhou54 Date: Tue, 3 Feb 2026 20:23:13 +0000 Subject: [PATCH 004/214] Support custom action trajectories for validation --- .../action/000000_action.npy | Bin 0 -> 2994 bytes .../action/000001_action.npy | Bin 0 -> 2994 bytes .../action/000002_action.npy | Bin 0 -> 2994 bytes .../action/000003_action.npy | Bin 0 -> 2994 bytes .../action/000004_action.npy | Bin 0 -> 2994 bytes .../action/000005_action.npy | Bin 0 -> 2994 bytes .../action/000006_action.npy | Bin 0 -> 2994 bytes .../action/000007_action.npy | Bin 0 -> 2994 bytes .../action/000008_action.npy | Bin 0 -> 2994 bytes .../action/000009_action.npy | Bin 0 -> 2994 bytes .../action/000010_action.npy | Bin 0 -> 2994 bytes .../action/000011_action.npy | Bin 0 -> 2994 bytes .../action/000012_action.npy | Bin 0 -> 2994 bytes .../action/000013_action.npy | Bin 0 -> 2994 bytes .../action/000014_action.npy | Bin 0 -> 2994 bytes .../action/000015_action.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/action/README.md | 18 + .../WanGame2.1_1.3b_i2v/finetune_i2v.slurm | 8 +- .../WanGame2.1_1.3b_i2v/generate_actions.py | 193 +++++++++ .../WanGame2.1_1.3b_i2v/validation.json | 404 ++++++++++++++++++ fastvideo/dataset/validation_dataset.py | 21 + 21 files changed, 640 insertions(+), 4 deletions(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000000_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000001_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000002_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000003_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000004_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000005_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000006_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000007_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000008_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000009_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000010_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000011_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000012_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000013_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000014_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000015_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/README.md create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/generate_actions.py create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000000_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000000_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..8a6ab2ab14b855f0691ea93076134c495f72926b GIT binary patch literal 2994 zcmeHIF-yZh6i!lO#X37UWlIVrlY@v*sNi5QRl&hQ!X>#9vB`zYMF_UgrP{!4d4m5| zbBRMJZVE22TBwCFv;Vzu)Tm#Y{&VWJt9g`0sO$DI3P*JXdW=A zfa^^q@O~1uB^5Zz8qQT8-ryE2M;%5zu7|8hv8dOkCs1zMqaIfm;JWVRXB`bVV+MZt z086m=-V-i-f@lbl7U_aC>VH!@H_q-WL<6qPj=OAUTWpG8n|$;|Q>$^?+cwI4tBBM= z>d<`~(ht%P(hpq)q#wF!{LTLW^@QenO!5r>IOer_%paALjn=(duB8J1r+~0*j%u*K E06JO3F8}}l literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000001_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000001_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..e649b6c7273d76092c69f063d53cc78ae717dfbf GIT binary patch literal 2994 zcmeHIze~eF6uzX!igk8y%62J~Ob#MKp@M_KR0RhI376zb#3mOmi4bg|i`u|#d4m6~ z=7&NQw+=4%hIjAYJ-)|%e0Q(3Z!Rux74(Rnuul_@<6|8!rg)^|8RID@k;Ax8>+~6! zC$!2Z0f}j~H-;m9sc&YxtF1RYK#ht0C=27QuJD_nzNDZ`*AG(XK_TGy-7BPiDGQBNofuxIzKR`91g%+|bp?{8fwHosW#bm8^+sarn@P7={lFdlv F%uh&;#4i8< literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000002_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000002_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..3ee9a9c38c0fc4b02f535415193cac4201c448a3 GIT binary patch literal 2994 zcmeH|ze~eF6vr>AF=Cw^oU&aCC6j}QP^jQwFjc|9LBb`ua$=K|OCkhY=%O}oTb|&5 ztN9@`h?|1Tz2R~1-Qx@Q<-@(wxw*K!RnQ}P!U0Wuj*ktznBbv-r!h}CiQJe6v~Hh~ zSwgFR!pNLfwK*6XOXGL3yV`ogebj{5kFs!{Tdp5-YK7S>y(gTL9K@c>sUJs4$}>NO z5`@-Fg3zO&9Vx*^&v2@E@Cw&p+RA|SgchXvoYwQ|$KVv{SEL#8no^cAl=zfIxNe`J zS&BtJC1{St+8zwerTM$qU2VNteWeM$pXBkpaQu)m?8NyjyGM+n0{EWKa7dFhV|kcC z1zc~Cz>OqoAr-jFGn}dcyuvlut~ww+u18rh$2DG?96`Bmw|ZP%faiJVA0nFMjxq?! zJ6M9vcOGyNP{bl28qsS~Vti4$SMK%;kUrO@hh4UlZ){4 DWcb7{ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000004_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000004_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..9ce5230372d7f3a39ac9eb871b638708e37e2d55 GIT binary patch literal 2994 zcmeH|ze~eF6vva)7_rU{PT7(|$>bm+6e>6vOjU4jkZ?(^L~MR=xd_1)x>Q@ZEl=>j z)%@_lh?|3h_lA4C?>)ZoK0J7(b8~TdD@%{klM&#|C&saE%qPZBH>L@p1jTMb0$j^y zXqMqh&q9>qsx}8heWCv@c9&bPMqg@z?Z$#5$#x!a;U$O!5GtWsn}*t_rF~^@KSS7O%JlH7+sQXKsdgK`Y4uRcQ9B#YZ6$QU zffMNNhuD)Y4^D}5;aTXMK$n-`)8)Y_aV|UyofGKt`X79ty3kCGl03sdo_VD@^9P&p TTIoU(l_2m}1nQN|QT13Kr_=zg literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000005_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000005_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..02cd7ab2e0691d85f98d3d6140435496260cf2dd GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGacRJ{HmZH3|;+@f9~Tc6;6 ztNmf2h}#V=d62xkm&X_K@g=Xc?yj!yCG?D5utyV@;ZqGSXLzjPdBjpiLOWs}tVY?BdMvyJiN5UA%fgjk6x>1;*7ni3sz+mwlxM_0=Cp^TynL1AoZn`EeiZ>=zoU(Wakjdn<(2lXi4*=S99{ zLhvAX=-x!_g&u?+gdX|~2tD-K*#3WjszY-nh_ZzK+2-Y{%^wxhjod$0tOW!A$3QLF Ij8v2H4YmmYtpET3 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000007_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000007_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..b0f8a4d3e79be73f379dc65f8aca9f50c5a87b48 GIT binary patch literal 2994 zcmeH|u}i~16vi*9F=Cw^oU$c_lF30tC{%DTn5y95Amy4|iP$FLk_f>Tx>Q@ZEl=>j z)ui!(5jO`B?hW_defRj_ejL2Qy}7u&RmdZGGJTeK+&nhS*}&`?<~ZUhr{OT-KCA3A zI!RdRCjpIFdA2%TV{ZJeb{AW(W}9qE?1$Mj&h4QWab{1mNqSE?r#XndA!l9`CMnOn z6bca9rW1r7`gKbI4q3vfGJ+Sl2Fp=8K}%?Un#Zi_R|iM1Ue#AEq0GQ_-ShXdHjC3s zVYI%3Iap%nf#su!avuUU(5lct|BQ65ob6`_+Cm*4eoZ_1mPV?y{+nLEijYOG-l0k# z2{gG#K207-3Avz39|<&hNj^;;NC~;1N*@U{c}YG^9!Lqfp#Fa!$_XvBX_O`G$1$&# YWBy>>U#a+1sgl6o5~#jxM$5(i0K`53tpET3 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000008_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000008_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..8a754f020b29a79843f069690896426397c4eb96 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarJu literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000010_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000010_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..e97638cfc59b49de2c798709bf1dc5e988c22fd0 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarc literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000015_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000015_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..a8c7fba01479245739ac912e26c76249940eec91 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa[{k2}]" + + # Format Mouse Description + if m1 == "stop" and m2 == "stop": + m_desc = "Static" + elif m1 == m2: + m_desc = f"Hold [{m1}]" + else: + m_desc = f"Switch [{m1}]->[{m2}]" + + return f"{k_desc} + {m_desc}" + +# ========================================== +# Main Generation Logic +# ========================================== + +configs = [] +readme_content = [] + +# Group 1: Constant Keyboard, No Mouse (0-7) +keys_basic = ['W', 'S', 'A', 'D', 'WA', 'WD', 'SA', 'SD'] +for k in keys_basic: + configs.append(((k, k), ("stop", "stop"))) + +# Group 2: No Keyboard, Constant Mouse (8-15) +mouse_basic = ['up', 'down', 'left', 'right', 'up_right', 'up_left', 'down_right', 'down_left'] +for m in mouse_basic: + configs.append((("", ""), (m, m))) + +# Group 3: Split Keyboard, No Mouse (16-23) +split_keys = [ + ('W', 'S'), ('S', 'W'), + ('A', 'D'), ('D', 'A'), + ('W', 'A'), ('W', 'D'), + ('S', 'A'), ('S', 'D') +] +for k1, k2 in split_keys: + configs.append(((k1, k2), ("stop", "stop"))) + +# Group 4: No Keyboard, Split Mouse (24-31) +split_mouse = [ + ('left', 'right'), ('right', 'left'), + ('up', 'down'), ('down', 'up'), + ('up_left', 'up_right'), ('up_right', 'up_left'), + ('left', 'up'), ('right', 'down') +] +for m1, m2 in split_mouse: + configs.append((("", ""), (m1, m2))) + +# Group 5: Constant Keyboard + Constant Mouse (32-47) +combo_keys = ['W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD', 'W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD'] +combo_mice = ['left', 'left', 'right', 'right', 'up', 'up', 'down', 'down', 'up_left', 'up_left', 'up_right', 'up_right', 'down_left', 'down_right', 'right', 'left'] +for i in range(16): + configs.append(((combo_keys[i], combo_keys[i]), (combo_mice[i], combo_mice[i]))) + +# Group 6: Constant Keyboard, Split Mouse (48-55) +complex_1_keys = ['W'] * 8 +complex_1_mice = [ + ('left', 'right'), ('right', 'left'), + ('up', 'down'), ('down', 'up'), + ('left', 'up'), ('right', 'up'), + ('left', 'down'), ('right', 'down') +] +for i in range(8): + configs.append(((complex_1_keys[i], complex_1_keys[i]), complex_1_mice[i])) + +# Group 7: Split Keyboard, Constant Mouse (56-63) +complex_2_keys = [ + ('W', 'S'), ('S', 'W'), + ('A', 'D'), ('D', 'A'), + ('W', 'A'), ('W', 'D'), + ('S', 'A'), ('S', 'D') +] +complex_2_mouse = 'up' +for k1, k2 in complex_2_keys: + configs.append(((k1, k2), (complex_2_mouse, complex_2_mouse))) + + +# Execution +print(f"Preparing to generate {len(configs)} action files...") + +for i, (key_seq, mouse_seq) in enumerate(configs): + if i >= 16: break + + # Generate Data + kb_arr, ms_arr = generate_sequence(key_seq, mouse_seq) + filename = save_action(i, kb_arr, ms_arr) + + # Generate Description for README + description = generate_description(key_seq, mouse_seq) + readme_entry = f"{i:02d}. {description}" + readme_content.append(readme_entry) + + print(f"Generated {filename} -> {description}") + +# Write README +readme_path = os.path.join(BASE_OUTPUT_DIR, 'README.md') +with open(readme_path, 'w', encoding='utf-8') as f: + f.write(f"Total Files: {len(readme_content)}\n\n") + for line in readme_content: + f.write(line + '\n') + +print(f"\nProcessing complete.") +print(f"64 .npy files generated in {VIDEO_OUTPUT_DIR}") +print(f"Manifest saved to {readme_path}") \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json new file mode 100644 index 000000000..5af4cca74 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json @@ -0,0 +1,404 @@ +{ + "data": [ + { + "caption": "0", + "image_path": "../../../../mc_wasd_10/validate/000000.jpg", + "action_path": "../../../../mc_wasd_10/videos/000000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1", + "image_path": "../../../../mc_wasd_10/validate/000001.jpg", + "action_path": "../../../../mc_wasd_10/videos/000001_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "2", + "image_path": "../../../../mc_wasd_10/validate/000002.jpg", + "action_path": "../../../../mc_wasd_10/videos/000002_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "3", + "image_path": "../../../../mc_wasd_10/validate/000003.jpg", + "action_path": "../../../../mc_wasd_10/videos/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "4", + "image_path": "../../../../mc_wasd_10/validate/000004.jpg", + "action_path": "../../../../mc_wasd_10/videos/000004_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "5", + "image_path": "../../../../mc_wasd_10/validate/000005.jpg", + "action_path": "../../../../mc_wasd_10/videos/000005_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "6", + "image_path": "../../../../mc_wasd_10/validate/000006.jpg", + "action_path": "../../../../mc_wasd_10/videos/000006_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "7", + "image_path": "../../../../mc_wasd_10/validate/000007.jpg", + "action_path": "../../../../mc_wasd_10/videos/000007_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "00. Hold [W] + Static", + "image_path": "../../../../mc_wasd_10/validate/000000.jpg", + "action_path": "action/000000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "01. Hold [S] + Static", + "image_path": "../../../../mc_wasd_10/validate/000001.jpg", + "action_path": "action/000001_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "02. Hold [A] + Static", + "image_path": "../../../../mc_wasd_10/validate/000002.jpg", + "action_path": "action/000002_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "03. Hold [D] + Static", + "image_path": "../../../../mc_wasd_10/validate/000003.jpg", + "action_path": "action/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "04. Hold [WA] + Static", + "image_path": "../../../../mc_wasd_10/validate/000004.jpg", + "action_path": "action/000004_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "05. Hold [WD] + Static", + "image_path": "../../../../mc_wasd_10/validate/000005.jpg", + "action_path": "action/000005_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "06. Hold [SA] + Static", + "image_path": "../../../../mc_wasd_10/validate/000006.jpg", + "action_path": "action/000006_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "07. Hold [SD] + Static", + "image_path": "../../../../mc_wasd_10/validate/000007.jpg", + "action_path": "action/000007_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "08. No Key + Hold [up]", + "image_path": "../../../../mc_wasd_10/validate/000000.jpg", + "action_path": "action/000008_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "09. No Key + Hold [down]", + "image_path": "../../../../mc_wasd_10/validate/000001.jpg", + "action_path": "action/000009_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "10. No Key + Hold [left]", + "image_path": "../../../../mc_wasd_10/validate/000002.jpg", + "action_path": "action/000010_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "11. No Key + Hold [right]", + "image_path": "../../../../mc_wasd_10/validate/000003.jpg", + "action_path": "action/000011_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "12. No Key + Hold [up_right]", + "image_path": "../../../../mc_wasd_10/validate/000004.jpg", + "action_path": "action/000012_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "13. No Key + Hold [up_left]", + "image_path": "../../../../mc_wasd_10/validate/000005.jpg", + "action_path": "action/000013_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "14. No Key + Hold [down_right]", + "image_path": "../../../../mc_wasd_10/validate/000006.jpg", + "action_path": "action/000014_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "15. No Key + Hold [down_left]", + "image_path": "../../../../mc_wasd_10/validate/000007.jpg", + "action_path": "action/000015_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "00. Hold [W] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000000.jpg", + "action_path": "action/000000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "01. Hold [S] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000001.jpg", + "action_path": "action/000001_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "02. Hold [A] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000002.jpg", + "action_path": "action/000002_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "03. Hold [D] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000003.jpg", + "action_path": "action/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "04. Hold [WA] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000004.jpg", + "action_path": "action/000004_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "05. Hold [WD] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000005.jpg", + "action_path": "action/000005_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "06. Hold [SA] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000006.jpg", + "action_path": "action/000006_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "07. Hold [SD] + Static", + "image_path": "../../../../mc_wasd_10/validate/gen_000007.jpg", + "action_path": "action/000007_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "08. No Key + Hold [up]", + "image_path": "../../../../mc_wasd_10/validate/gen_000000.jpg", + "action_path": "action/000008_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "09. No Key + Hold [down]", + "image_path": "../../../../mc_wasd_10/validate/gen_000001.jpg", + "action_path": "action/000009_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "10. No Key + Hold [left]", + "image_path": "../../../../mc_wasd_10/validate/gen_000002.jpg", + "action_path": "action/000010_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "11. No Key + Hold [right]", + "image_path": "../../../../mc_wasd_10/validate/gen_000003.jpg", + "action_path": "action/000011_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "12. No Key + Hold [up_right]", + "image_path": "../../../../mc_wasd_10/validate/gen_000004.jpg", + "action_path": "action/000012_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "13. No Key + Hold [up_left]", + "image_path": "../../../../mc_wasd_10/validate/gen_000005.jpg", + "action_path": "action/000013_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "14. No Key + Hold [down_right]", + "image_path": "../../../../mc_wasd_10/validate/gen_000006.jpg", + "action_path": "action/000014_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "15. No Key + Hold [down_left]", + "image_path": "../../../../mc_wasd_10/validate/gen_000007.jpg", + "action_path": "action/000015_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + } + ] +} \ No newline at end of file diff --git a/fastvideo/dataset/validation_dataset.py b/fastvideo/dataset/validation_dataset.py index 5ab467d75..8d831d5df 100644 --- a/fastvideo/dataset/validation_dataset.py +++ b/fastvideo/dataset/validation_dataset.py @@ -4,6 +4,7 @@ import pathlib import datasets +import numpy as np from torch.utils.data import IterableDataset from fastvideo.distributed import (get_sp_world_size, get_world_rank, @@ -160,5 +161,25 @@ def __iter__(self): else: sample["control_video"] = load_video(control_video_path) + if sample.get("action_path", None) is not None: + action_path = sample["action_path"] + action_path = os.path.join(self.dir, action_path) + sample["action_path"] = action_path + if not pathlib.Path(action_path).is_file(): + logger.warning("Action file %s does not exist.", action_path) + else: + try: + action_data = np.load(action_path, allow_pickle=True) + num_frames = sample["num_frames"] + if action_data.dtype == object: action_data = action_data.item() + if isinstance(action_data, dict): + sample["keyboard_cond"] = action_data["keyboard"][:num_frames] + sample["mouse_cond"] = action_data["mouse"][:num_frames] + else: + sample["keyboard_cond"] = action_data[:num_frames] + except Exception as e: + logger.error("Error loading action file %s: %s", + action_path, e) + sample = {k: v for k, v in sample.items() if v is not None} yield sample From ca16275ca39d0994d4046a72b62d1c7c29fdde74 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Fri, 6 Feb 2026 00:13:12 +0000 Subject: [PATCH 005/214] no text --- .../finetune_wangame.slurm | 146 ++++++++ .../scripts/generate_validation_static_w.py | 48 +++ fastvideo/configs/models/encoders/clip.py | 6 +- fastvideo/fastvideo_args.py | 9 + fastvideo/models/dits/hyworld/pose.py | 8 +- fastvideo/models/dits/matrixgame/utils.py | 351 ++++++++++++------ .../dits/wangame/hyworld_action_module.py | 36 +- fastvideo/models/dits/wangame/model.py | 26 +- fastvideo/training/training_pipeline.py | 19 + .../training/wangame_training_pipeline.py | 106 +++++- 10 files changed, 607 insertions(+), 148 deletions(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm new file mode 100644 index 000000000..6b34764af --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm @@ -0,0 +1,146 @@ +#!/bin/bash +#SBATCH --job-name=wangame_1.3b +#SBATCH --partition=main +#SBATCH --nodes=8 +#SBATCH --ntasks=8 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=wangame_1.3b_output/wangame_1.3b_%j.out +#SBATCH --error=wangame_1.3b_output/wangame_1.3b_%j.err +#SBATCH --exclusive + +# Basic Info +export NCCL_P2P_DISABLE=1 +export TORCH_NCCL_ENABLE_MONITORING=0 +export NCCL_DEBUG_SUBSYS=INIT,NET +# different cache dir for different processes +export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} +export MASTER_PORT=29500 +export NODE_RANK=$SLURM_PROCID +nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) +export MASTER_ADDR=${nodes[0]} +export TOKENIZERS_PARALLELISM=false +export WANDB_API_KEY="d5b02b05e30d8cb34c7b31c6ae10416fc26dcb66" +export WANDB_BASE_URL="https://api.wandb.ai" +export WANDB_MODE=online +export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN + +source ~/conda/miniconda/bin/activate +conda activate mhuo-fv +export HOME="/mnt/weka/home/hao.zhang/mhuo" + +# Configs +NUM_GPUS=8 +NUM_NODES=8 +NUM_TOTAL_GPUS=$((NUM_GPUS * NUM_NODES)) +BS_PER_GPU=4 +GRADIENT_ACCUMULATION_STEPS=1 +WANDB_RUN_NAME="bs256" +FREEZE_DIT=False + +MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" +DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed" +VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" +# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_doom.json" +# DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_overfit.json" + +if [[ "$DATA_DIR" == *"mc_wasd_10"* ]]; then + RUN_DIR="wangame_1.3b_overfit" + CHECKPOINTING_STEPS=100000 +else + RUN_DIR="wangame_1.3b" + CHECKPOINTING_STEPS=1000 +fi + +# Training arguments +training_args=( + --tracker_project_name $RUN_DIR + --output_dir $RUN_DIR + --wandb_run_name $WANDB_RUN_NAME + --max_train_steps 10000 + --train_batch_size $BS_PER_GPU + --train_sp_batch_size $BS_PER_GPU + --gradient_accumulation_steps $GRADIENT_ACCUMULATION_STEPS + --num_latent_t 20 + --num_height 352 + --num_width 640 + --num_frames 77 + --enable_gradient_checkpointing_type "full" + --train_action_only $FREEZE_DIT +) + +# Parallel arguments +parallel_args=( + --num_gpus $NUM_TOTAL_GPUS + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim $NUM_TOTAL_GPUS +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 1 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 100 + --validation_sampling_steps "40" + --validation_guidance_scale "1.0" + --validation_num_samples $NUM_TOTAL_GPUS +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 2e-5 + --mixed_precision "bf16" + --weight_only_checkpointing_steps 100000 + --training_state_checkpointing_steps $CHECKPOINTING_STEPS + --weight_decay 1e-4 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 2 + --training_cfg_rate 0.1 + --multi_phased_distill_schedule "4000-1" + --not_apply_cfg_solver + --dit_precision "fp32" + --num_euler_timesteps 50 + --ema_start_step 0 +) + +# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t +# torchrun \ +# --nnodes 1 \ +# --nproc_per_node $NUM_GPUS \ +srun torchrun \ + --nnodes $NUM_NODES \ + --nproc_per_node $NUM_GPUS \ + --rdzv_backend c10d \ + --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT \ + --node_rank $SLURM_PROCID \ + fastvideo/training/wangame_training_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py new file mode 100644 index 000000000..15ce1fde3 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py @@ -0,0 +1,48 @@ +import json +import os + +# Paths +image_dir = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/first_frame" +action_still = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/videos/000000_action.npy" +action_w = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/videos/001050_action.npy" + +# Output path +output_path = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_static_w.json" + +# Fixed fields +fixed_fields = { + "video_path": None, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 +} + +data = [] + +# 32 images, each with 2 actions (Still and W) +for i in range(32): + image_path = os.path.join(image_dir, f"{i:06d}.png") + + # Still action + data.append({ + "caption": f"{i:02d} - Still", + "image_path": image_path, + "action_path": action_still, + **fixed_fields + }) + + # W action + data.append({ + "caption": f"{i:02d} - W", + "image_path": image_path, + "action_path": action_w, + **fixed_fields + }) + +# Write to file +output = {"data": data} +with open(output_path, "w") as f: + json.dump(output, f, indent=4) + +print(f"Generated {len(data)} entries to {output_path}") diff --git a/fastvideo/configs/models/encoders/clip.py b/fastvideo/configs/models/encoders/clip.py index d233872f1..7cc5c70d5 100644 --- a/fastvideo/configs/models/encoders/clip.py +++ b/fastvideo/configs/models/encoders/clip.py @@ -87,10 +87,10 @@ class CLIPVisionConfig(ImageEncoderConfig): arch_config: ImageEncoderArchConfig = field( default_factory=CLIPVisionArchConfig) - num_hidden_layers_override: int | None = None + num_hidden_layers_override: int | None = 31 require_post_norm: bool | None = None - enable_scale: bool = True - is_causal: bool = True + enable_scale: bool = False + is_causal: bool = False prefix: str = "clip" diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index 33c06efcc..25e4aa187 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -919,6 +919,9 @@ class TrainingArgs(FastVideoArgs): lora_alpha: int | None = None lora_training: bool = False + # Action-only training (freeze base model, only train action params) + train_action_only: bool = False + # distillation args generator_update_interval: int = 5 dfake_gen_update_ratio: int = 5 # self-forcing: how often to train generator vs critic @@ -1269,6 +1272,12 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: parser.add_argument("--lora-rank", type=int, help="LoRA rank") parser.add_argument("--lora-alpha", type=int, help="LoRA alpha") + # Action-only training (freeze base model, only train action params) + parser.add_argument("--train-action-only", + action=StoreBoolean, + help="Whether to only train action-related parameters " + "(action_embedder and to_out_prope) while freezing base model") + # V-MoBA parameters parser.add_argument( "--moba-config-path", diff --git a/fastvideo/models/dits/hyworld/pose.py b/fastvideo/models/dits/hyworld/pose.py index 3ee0b9856..99d535308 100644 --- a/fastvideo/models/dits/hyworld/pose.py +++ b/fastvideo/models/dits/hyworld/pose.py @@ -10,6 +10,7 @@ """ import json +import logging import numpy as np import torch from scipy.spatial.transform import Rotation as R @@ -17,6 +18,7 @@ from fastvideo.models.dits.hyworld.trajectory import generate_camera_trajectory_local +logger = logging.getLogger(__name__) # Mapping from one-hot action encoding to single label mapping = { @@ -422,9 +424,11 @@ def reformat_keyboard_and_mouse_tensors(keyboard_tensor, mouse_tensor): keyboard_tensor = keyboard_tensor[1:, :] mouse_tensor = mouse_tensor[1:, :] groups = keyboard_tensor.view(-1, 4, keyboard_tensor.shape[1]) - assert (groups == groups[:, 0:1]).all(dim=1).all(), "keyboard_tensor must have the same value for each group" + if not (groups == groups[:, 0:1]).all(dim=1).all(): + logger.warning(f"keyboard_tensor has different values for each group: {groups}") groups = mouse_tensor.view(-1, 4, mouse_tensor.shape[1]) - assert (groups == groups[:, 0:1]).all(dim=1).all(), "mouse_tensor must have the same value for each group" + if not (groups == groups[:, 0:1]).all(dim=1).all(): + logger.warning(f"mouse_tensor has different values for each group: {groups}") return keyboard_tensor[::4], mouse_tensor[::4] diff --git a/fastvideo/models/dits/matrixgame/utils.py b/fastvideo/models/dits/matrixgame/utils.py index 4dd937699..c7dfd6743 100644 --- a/fastvideo/models/dits/matrixgame/utils.py +++ b/fastvideo/models/dits/matrixgame/utils.py @@ -301,119 +301,238 @@ def parse_config(config, mode="universal"): # NOTE: drawing functions are commented out to avoid cv2/libGL dependency. # -# def draw_rounded_rectangle(image, top_left, bottom_right, color, radius=10, alpha=0.5): -# overlay = image.copy() -# x1, y1 = top_left -# x2, y2 = bottom_right -# -# cv2.rectangle(overlay, (x1 + radius, y1), (x2 - radius, y2), color, -1) -# cv2.rectangle(overlay, (x1, y1 + radius), (x2, y2 - radius), color, -1) -# cv2.ellipse(overlay, (x1 + radius, y1 + radius), (radius, radius), 180, 0, 90, color, -1) -# cv2.ellipse(overlay, (x2 - radius, y1 + radius), (radius, radius), 270, 0, 90, color, -1) -# cv2.ellipse(overlay, (x1 + radius, y2 - radius), (radius, radius), 90, 0, 90, color, -1) -# cv2.ellipse(overlay, (x2 - radius, y2 - radius), (radius, radius), 0, 0, 90, color, -1) -# cv2.addWeighted(overlay, alpha, image, 1 - alpha, 0, image) -# -# def draw_keys_on_frame(frame, keys, key_size=(80, 50), spacing=20, bottom_margin=30, mode='universal'): -# h, w, _ = frame.shape -# horison_shift = 90 -# vertical_shift = -20 -# horizon_shift_all = 50 -# key_positions = { -# "W": (w // 2 - key_size[0] // 2 - horison_shift - horizon_shift_all, -# h - bottom_margin - key_size[1] * 2 + vertical_shift - 20), -# "A": (w // 2 - key_size[0] * 2 + 5 - horison_shift - horizon_shift_all, -# h - bottom_margin - key_size[1] + vertical_shift), -# "S": (w // 2 - key_size[0] // 2 - horison_shift - horizon_shift_all, -# h - bottom_margin - key_size[1] + vertical_shift), -# "D": (w // 2 + key_size[0] - 5 - horison_shift - horizon_shift_all, -# h - bottom_margin - key_size[1] + vertical_shift), -# } -# key_icon = {"W": "W", "A": "A", "S": "S", "D": "D", "left": "left", "right": "right"} -# if mode == 'templerun': -# key_positions.update({ -# "left": (w // 2 + key_size[0] * 2 + spacing * 2 - horison_shift - horizon_shift_all, -# h - bottom_margin - key_size[1] + vertical_shift), -# "right": (w // 2 + key_size[0] * 3 + spacing * 7 - horison_shift - horizon_shift_all, -# h - bottom_margin - key_size[1] + vertical_shift) -# }) -# -# for key, (x, y) in key_positions.items(): -# is_pressed = keys.get(key, False) -# top_left = (x, y) -# if key in ["left", "right"]: -# bottom_right = (x + key_size[0] + 40, y + key_size[1]) -# else: -# bottom_right = (x + key_size[0], y + key_size[1]) -# -# color = (0, 255, 0) if is_pressed else (200, 200, 200) -# alpha = 0.8 if is_pressed else 0.5 -# draw_rounded_rectangle(frame, top_left, bottom_right, color, radius=10, alpha=alpha) -# -# text_size = cv2.getTextSize(key, cv2.FONT_HERSHEY_SIMPLEX, 0.8, 2)[0] -# if key in ["left", "right"]: -# text_x = x + (key_size[0] + 40 - text_size[0]) // 2 -# else: -# text_x = x + (key_size[0] - text_size[0]) // 2 -# text_y = y + (key_size[1] + text_size[1]) // 2 -# cv2.putText(frame, key_icon[key], (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2) -# -# def overlay_icon(frame, icon, position, scale=1.0, rotation=0): -# x, y = position -# h, w, _ = icon.shape -# -# scaled_width = int(w * scale) -# scaled_height = int(h * scale) -# icon_resized = cv2.resize(icon, (scaled_width, scaled_height), interpolation=cv2.INTER_AREA) -# -# center = (scaled_width // 2, scaled_height // 2) -# rotation_matrix = cv2.getRotationMatrix2D(center, rotation, 1.0) -# icon_rotated = cv2.warpAffine( -# icon_resized, rotation_matrix, (scaled_width, scaled_height), -# flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0, 0) -# ) -# -# h, w, _ = icon_rotated.shape -# frame_h, frame_w, _ = frame.shape -# -# top_left_x = max(0, int(x - w // 2)) -# top_left_y = max(0, int(y - h // 2)) -# bottom_right_x = min(frame_w, int(x + w // 2)) -# bottom_right_y = min(frame_h, int(y + h // 2)) -# -# icon_x_start = max(0, int(-x + w // 2)) -# icon_y_start = max(0, int(-y + h // 2)) -# icon_x_end = icon_x_start + (bottom_right_x - top_left_x) -# icon_y_end = icon_y_start + (bottom_right_y - top_left_y) -# -# icon_region = icon_rotated[icon_y_start:icon_y_end, icon_x_start:icon_x_end] -# alpha = icon_region[:, :, 3] / 255.0 -# icon_rgb = icon_region[:, :, :3] -# -# frame_region = frame[top_left_y:bottom_right_y, top_left_x:bottom_right_x] -# for c in range(3): -# frame_region[:, :, c] = (1 - alpha) * frame_region[:, :, c] + alpha * icon_rgb[:, :, c] -# frame[top_left_y:bottom_right_y, top_left_x:bottom_right_x] = frame_region -# -# def process_video(input_video, output_video, config, mouse_icon_path, -# mouse_scale=1.0, mouse_rotation=0, process_icon=True, mode='universal'): -# key_data, mouse_data = parse_config(config, mode=mode) -# fps = 12 -# -# mouse_icon = cv2.imread(mouse_icon_path, cv2.IMREAD_UNCHANGED) -# -# out_video = [] -# for frame_idx, frame in enumerate(input_video): -# frame = np.ascontiguousarray(frame) -# if process_icon: -# keys = key_data.get(frame_idx, {"W": False, "A": False, "S": False, "D": False, "left": False, "right": False}) -# draw_keys_on_frame(frame, keys, key_size=(50, 50), spacing=10, bottom_margin=20, mode=mode) -# if mode == 'universal': -# frame_width = frame.shape[1] -# frame_height = frame.shape[0] -# mouse_position = mouse_data.get(frame_idx, (frame_width // 2, frame_height // 2)) -# overlay_icon(frame, mouse_icon, mouse_position, scale=mouse_scale, rotation=mouse_rotation) -# out_video.append(frame / 255) -# -# export_to_video(out_video, output_video, fps=fps) -# logger.info(f"Video saved to {output_video}") +import cv2 +import numpy as np +from diffusers.utils import export_to_video + +def draw_rounded_rectangle(image, top_left, bottom_right, color, radius=10, alpha=0.5): + overlay = image.copy() + x1, y1 = top_left + x2, y2 = bottom_right + + cv2.rectangle(overlay, (x1 + radius, y1), (x2 - radius, y2), color, -1) + cv2.rectangle(overlay, (x1, y1 + radius), (x2, y2 - radius), color, -1) + cv2.ellipse(overlay, (x1 + radius, y1 + radius), (radius, radius), 180, 0, 90, color, -1) + cv2.ellipse(overlay, (x2 - radius, y1 + radius), (radius, radius), 270, 0, 90, color, -1) + cv2.ellipse(overlay, (x1 + radius, y2 - radius), (radius, radius), 90, 0, 90, color, -1) + cv2.ellipse(overlay, (x2 - radius, y2 - radius), (radius, radius), 0, 0, 90, color, -1) + cv2.addWeighted(overlay, alpha, image, 1 - alpha, 0, image) + +def draw_keys_on_frame(frame, keys, key_size=(30, 30), spacing=5, top_margin=15, mode='universal'): + """Draw WASD keys on the left top of the frame.""" + h, w, _ = frame.shape + + # Left top positioning + left_margin = 15 + gap = 3 # Gap between keys + + key_positions = { + "W": (left_margin + key_size[0] + gap, + top_margin), + "A": (left_margin, + top_margin + key_size[1] + gap), + "S": (left_margin + key_size[0] + gap, + top_margin + key_size[1] + gap), + "D": (left_margin + (key_size[0] + gap) * 2, + top_margin + key_size[1] + gap), + } + key_icon = {"W": "W", "A": "A", "S": "S", "D": "D", "left": "L", "right": "R"} + if mode == 'templerun': + key_positions.update({ + "left": (left_margin + (key_size[0] + gap) * 3 + 10, + top_margin + key_size[1] + gap), + "right": (left_margin + (key_size[0] + gap) * 4 + 15, + top_margin + key_size[1] + gap) + }) + + for key, (x, y) in key_positions.items(): + is_pressed = keys.get(key, False) + top_left = (x, y) + bottom_right = (x + key_size[0], y + key_size[1]) + + color = (0, 255, 0) if is_pressed else (200, 200, 200) + alpha = 0.8 if is_pressed else 0.5 + draw_rounded_rectangle(frame, top_left, bottom_right, color, radius=5, alpha=alpha) + + text_size = cv2.getTextSize(key_icon[key], cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)[0] + text_x = x + (key_size[0] - text_size[0]) // 2 + text_y = y + (key_size[1] + text_size[1]) // 2 + cv2.putText(frame, key_icon[key], (text_x, text_y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + +def overlay_icon(frame, icon, position, scale=1.0, rotation=0): + x, y = position + h, w, _ = icon.shape + + scaled_width = int(w * scale) + scaled_height = int(h * scale) + icon_resized = cv2.resize(icon, (scaled_width, scaled_height), interpolation=cv2.INTER_AREA) + + center = (scaled_width // 2, scaled_height // 2) + rotation_matrix = cv2.getRotationMatrix2D(center, rotation, 1.0) + icon_rotated = cv2.warpAffine( + icon_resized, rotation_matrix, (scaled_width, scaled_height), + flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0, 0) + ) + + h, w, _ = icon_rotated.shape + frame_h, frame_w, _ = frame.shape + + top_left_x = max(0, int(x - w // 2)) + top_left_y = max(0, int(y - h // 2)) + bottom_right_x = min(frame_w, int(x + w // 2)) + bottom_right_y = min(frame_h, int(y + h // 2)) + + icon_x_start = max(0, int(-x + w // 2)) + icon_y_start = max(0, int(-y + h // 2)) + icon_x_end = icon_x_start + (bottom_right_x - top_left_x) + icon_y_end = icon_y_start + (bottom_right_y - top_left_y) + + icon_region = icon_rotated[icon_y_start:icon_y_end, icon_x_start:icon_x_end] + alpha = icon_region[:, :, 3] / 255.0 + icon_rgb = icon_region[:, :, :3] + + frame_region = frame[top_left_y:bottom_right_y, top_left_x:bottom_right_x] + for c in range(3): + frame_region[:, :, c] = (1 - alpha) * frame_region[:, :, c] + alpha * icon_rgb[:, :, c] + frame[top_left_y:bottom_right_y, top_left_x:bottom_right_x] = frame_region + +def process_video(input_video, output_video, config, mouse_icon_path, + mouse_scale=1.0, mouse_rotation=0, process_icon=True, mode='universal'): + key_data, mouse_data = parse_config(config, mode=mode) + fps = 12 + + mouse_icon = cv2.imread(mouse_icon_path, cv2.IMREAD_UNCHANGED) + + out_video = [] + for frame_idx, frame in enumerate(input_video): + frame = np.ascontiguousarray(frame) + if process_icon: + keys = key_data.get(frame_idx, {"W": False, "A": False, "S": False, "D": False, "left": False, "right": False}) + draw_keys_on_frame(frame, keys, key_size=(50, 50), spacing=10, bottom_margin=20, mode=mode) + if mode == 'universal': + frame_width = frame.shape[1] + frame_height = frame.shape[0] + mouse_position = mouse_data.get(frame_idx, (frame_width // 2, frame_height // 2)) + overlay_icon(frame, mouse_icon, mouse_position, scale=mouse_scale, rotation=mouse_rotation) + out_video.append(frame / 255) + + export_to_video(out_video, output_video, fps=fps) + logger.info(f"Video saved to {output_video}") + + +def parse_npy_action(action_path): + """Convert npy action file to key_data and mouse_data dict format.""" + action_data = np.load(action_path, allow_pickle=True).item() + keyboard_data = action_data['keyboard'] # shape: (num_frames, 6) -> [W, S, A, D, left, right] + mouse_data = action_data.get('mouse', None) # shape: (num_frames, 2) -> [Pitch, Yaw] + + # MatrixGame convention: 0:W, 1:S, 2:A, 3:D, 4:left, 5:right + key_names = ["W", "S", "A", "D", "left", "right"] + key_data = {} + for frame_idx, keys in enumerate(keyboard_data): + key_data[frame_idx] = {key_names[i]: bool(keys[i]) for i in range(len(key_names))} + + # MatrixGame convention: mouse is [Pitch, Yaw] + mouse_dict = {} + if mouse_data is not None: + for frame_idx, (pitch, yaw) in enumerate(mouse_data): + mouse_dict[frame_idx] = {"pitch": float(pitch), "yaw": float(yaw)} + + return key_data, mouse_dict + + +def draw_mouse_on_frame(frame, pitch, yaw, top_margin=15): + """Draw crosshair with direction arrow on the right top of the frame.""" + h, w, _ = frame.shape + + # Right top positioning + right_margin = 15 + crosshair_radius = 25 + + # Position crosshair on the right top + crosshair_x = w - right_margin - crosshair_radius + crosshair_y = top_margin + crosshair_radius + + # Yaw affects horizontal direction, pitch affects vertical + dx = int(yaw * crosshair_radius * 8) # Scale for visibility + dy = int(-pitch * crosshair_radius * 8) # Negative because y increases downward + + # Clamp arrow length + max_arrow = crosshair_radius - 5 + dx = max(-max_arrow, min(max_arrow, dx)) + dy = max(-max_arrow, min(max_arrow, dy)) + + # Draw crosshair background + cv2.circle(frame, (crosshair_x, crosshair_y), crosshair_radius, (50, 50, 50), -1) + cv2.circle(frame, (crosshair_x, crosshair_y), crosshair_radius, (200, 200, 200), 1) + cv2.line(frame, (crosshair_x - crosshair_radius + 5, crosshair_y), + (crosshair_x + crosshair_radius - 5, crosshair_y), (100, 100, 100), 1) + cv2.line(frame, (crosshair_x, crosshair_y - crosshair_radius + 5), + (crosshair_x, crosshair_y + crosshair_radius - 5), (100, 100, 100), 1) + + # Draw direction arrow + if abs(dx) > 1 or abs(dy) > 1: + cv2.arrowedLine(frame, (crosshair_x, crosshair_y), (crosshair_x + dx, crosshair_y + dy), + (0, 255, 0), 2, tipLength=0.3) + + +def process_video_with_npy(input_video, output_video, action_path, fps=12, mode='universal'): + """Process video with overlay using npy action file. + + Uses existing draw_keys_on_frame function. + """ + key_data, mouse_data = parse_npy_action(action_path) + + out_video = [] + for frame_idx, frame in enumerate(input_video): + frame = np.ascontiguousarray(frame) + keys = key_data.get(frame_idx, {"W": False, "A": False, "S": False, "D": False, "left": False, "right": False}) + draw_keys_on_frame(frame, keys, mode=mode) + + # Draw pitch and yaw + mouse = mouse_data.get(frame_idx, {"pitch": 0.0, "yaw": 0.0}) + draw_mouse_on_frame(frame, mouse["pitch"], mouse["yaw"]) + + out_video.append(frame / 255.0) + + export_to_video(out_video, output_video, fps=fps) + logger.info(f"Video saved to {output_video}") + + +if __name__ == "__main__": + import argparse + import cv2 + + parser = argparse.ArgumentParser(description="Overlay keyboard actions on video") + parser.add_argument("--video", type=str, required=True, help="Path to input video (.mp4)") + parser.add_argument("--action", type=str, required=True, help="Path to action file (.npy)") + parser.add_argument("--output", type=str, default=None, help="Path to output video (default: input_with_overlay.mp4)") + parser.add_argument("--fps", type=int, default=12, help="Output video FPS") + args = parser.parse_args() + + # Load video frames using cv2 + cap = cv2.VideoCapture(args.video) + if not cap.isOpened(): + raise ValueError(f"Cannot open video: {args.video}") + + frames = [] + while True: + ret, frame = cap.read() + if not ret: + break + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + frames.append(frame) + cap.release() + + print(f"Loaded {len(frames)} frames from video") + + # Set output path + if args.output is None: + base_name = args.video.rsplit('.', 1)[0] + output_path = f"{base_name}_with_overlay.mp4" + else: + output_path = args.output + + # Process video with overlay using existing functions + process_video_with_npy(frames, output_path, args.action, fps=args.fps) + print(f"Video with overlay saved to: {output_path}") diff --git a/fastvideo/models/dits/wangame/hyworld_action_module.py b/fastvideo/models/dits/wangame/hyworld_action_module.py index cdc928bb2..b98e3ff34 100644 --- a/fastvideo/models/dits/wangame/hyworld_action_module.py +++ b/fastvideo/models/dits/wangame/hyworld_action_module.py @@ -51,19 +51,43 @@ def forward( encoder_hidden_states_image: torch.Tensor | None = None, timestep_seq_len: int | None = None, ): + """ + Args: + timestep: [B] diffusion timesteps (one per batch sample) + action: [B, T] action labels (one per frame per batch sample) + + Returns: + temb: [B*T, dim] combined timestep + action embedding + timestep_proj: [B*T, 6*dim] modulation projection + """ + # timestep: [B] -> temb: [B, dim] temb = self.time_embedder(timestep, timestep_seq_len) - - action_emb = timestep_embedding(action.flatten(), self.time_freq_dim) + + # Handle action embedding for batch > 1 + # action shape: [B, T] where B=batch_size, T=num_frames + batch_size = action.shape[0] + num_frames = action.shape[1] + + # Compute action embeddings: [B, T] -> [B*T] -> [B*T, dim] + action_flat = action.flatten() # [B*T] + action_emb = timestep_embedding(action_flat, self.time_freq_dim) action_embedder_dtype = next(iter(self.action_embedder.parameters())).dtype if ( action_emb.dtype != action_embedder_dtype and action_embedder_dtype != torch.int8 ): action_emb = action_emb.to(action_embedder_dtype) - action_emb = self.action_embedder(action_emb).type_as(temb) - temb = temb + action_emb + action_emb = self.action_embedder(action_emb).type_as(temb) # [B*T, dim] + + # Expand temb to match action_emb: [B, dim] -> [B, T, dim] -> [B*T, dim] + # Each batch's temb is repeated for all its frames + temb_expanded = temb.unsqueeze(1).expand(-1, num_frames, -1) # [B, T, dim] + temb_expanded = temb_expanded.reshape(batch_size * num_frames, -1) # [B*T, dim] + + # Add action embedding to expanded temb + temb = temb_expanded + action_emb # [B*T, dim] - timestep_proj = self.time_modulation(temb) + timestep_proj = self.time_modulation(temb) # [B*T, 6*dim] # MatrixGame does not use text embeddings, so we ignore encoder_hidden_states @@ -72,7 +96,7 @@ def forward( encoder_hidden_states_image = self.image_embedder( encoder_hidden_states_image) - encoder_hidden_states = torch.zeros((timestep.shape[0], 0, temb.shape[-1]), + encoder_hidden_states = torch.zeros((batch_size, 0, temb.shape[-1]), device=temb.device, dtype=temb.dtype) diff --git a/fastvideo/models/dits/wangame/model.py b/fastvideo/models/dits/wangame/model.py index 685f2299f..3e77bfaba 100644 --- a/fastvideo/models/dits/wangame/model.py +++ b/fastvideo/models/dits/wangame/model.py @@ -367,16 +367,13 @@ def forward( temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( timestep, action, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) - # Reshape timestep_proj: [T, 6*dim] -> [B, T, 6, dim] - # For training: batch_size=1, T=num_frames (diffusion forcing) - # For inference: batch_size can vary - timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) - if timestep_proj.shape[0] == post_patch_num_frames and batch_size == 1: - # Training mode: timestep_proj is [T, 6, dim], add batch dim -> [1, T, 6, dim] - timestep_proj = timestep_proj.unsqueeze(0) - else: - # Inference mode: reshape based on timestep shape - timestep_proj = timestep_proj.unflatten(dim=0, sizes=timestep.shape) + + # condition_embedder returns: + # - temb: [B*T, dim] where T = post_patch_num_frames + # - timestep_proj: [B*T, 6*dim] + # Reshape to [B, T, 6, dim] for transformer blocks + timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) # [B*T, 6, dim] + timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, self.hidden_size) # [B, T, 6, dim] encoder_hidden_states = encoder_hidden_states_image @@ -402,13 +399,8 @@ def forward( return kv_cache # Output norm, projection & unpatchify - # Reshape temb to match timestep_proj shape: [T, dim] -> [B, T, 1, dim] - if temb.shape[0] == post_patch_num_frames and batch_size == 1: - # Training mode: temb is [T, dim] -> [1, T, 1, dim] - temb = temb.unsqueeze(0).unsqueeze(2) - else: - # Inference mode: reshape based on timestep shape - temb = temb.unflatten(dim=0, sizes=timestep.shape).unsqueeze(2) + # temb is [B*T, dim], reshape to [B, T, 1, dim] + temb = temb.view(batch_size, post_patch_num_frames, -1).unsqueeze(2) # [B, T, 1, dim] shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) hidden_states = self.norm_out(hidden_states, shift, scale) diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index bc23a3b07..2e4456f91 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -793,6 +793,22 @@ def _prepare_validation_batch(self, sampling_param: SamplingParam, return batch + def _post_process_validation_frames(self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: + """Post-process validation frames before saving. + + Override this method in subclasses to add custom processing, + e.g., overlay action indicators for action-conditioned models. + + Args: + frames: List of numpy arrays (H, W, C) representing video frames + batch: The ForwardBatch containing input data (may include action data) + + Returns: + Processed frames (same format as input) + """ + return frames + @torch.no_grad() def _log_validation(self, transformer, training_args, global_step) -> None: """ @@ -871,6 +887,9 @@ def _log_validation(self, transformer, training_args, global_step) -> None: x = torchvision.utils.make_grid(x, nrow=6) x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) frames.append((x * 255).numpy().astype(np.uint8)) + + # Apply optional post-processing (e.g., overlay for action-conditioned models) + frames = self._post_process_validation_frames(frames, batch) step_videos.append(frames) # Only sp_group leaders (rank_in_sp_group == 0) need to send their diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index 5beea6abc..ba5c60f32 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -3,6 +3,7 @@ from copy import deepcopy from typing import Any +import numpy as np import torch from fastvideo.configs.sample import SamplingParam @@ -41,6 +42,48 @@ def create_training_stages(self, training_args: TrainingArgs): def set_schemas(self): self.train_dataset_schema = pyarrow_schema_wangame + def set_trainable(self) -> None: + """ + Override to only train newly added action-related parameters: + - condition_embedder.action_embedder: embeds action into timestep + - blocks.*.to_out_prope: projects PRoPE attention output + + This freezes the base model (q/k/v projections, FFN, etc.) while + allowing the action-conditioning path to be trained. + """ + train_action_only = getattr(self.fastvideo_args, "train_action_only", False) + + if not train_action_only: + # Default behavior: train all parameters + super().set_trainable() + return + + # Freeze all transformer parameters first + transformer = self.get_module("transformer") + transformer.train() + transformer.requires_grad_(False) + + # Define which parameter name patterns to train + action_param_patterns = [ + "condition_embedder.action_embedder", # Action embedding MLP + "to_out_prope", # PRoPE output projections in each block + ] + + # Enable gradients for action-related parameters only + trainable_count = 0 + frozen_count = 0 + for name, param in transformer.named_parameters(): + should_train = any(pattern in name for pattern in action_param_patterns) + if should_train: + param.requires_grad_(True) + trainable_count += 1 + logger.info(f"Trainable: {name} ({param.numel()} params)") + else: + frozen_count += 1 + + logger.info(f"Action-only training: {trainable_count} trainable param groups, " + f"{frozen_count} frozen param groups") + def initialize_validation_pipeline(self, training_args: TrainingArgs): logger.info("Initializing validation pipeline...") # args_copy.pipeline_config.vae_config.load_encoder = False @@ -152,10 +195,19 @@ def _build_input_kwargs(self, encoder_hidden_states_image = image_embeds from fastvideo.models.dits.hyworld.pose import process_custom_actions - viewmats, intrinsics, action_labels = process_custom_actions(training_batch.keyboard_cond, training_batch.mouse_cond) - viewmats = viewmats.unsqueeze(0).to(get_local_torch_device(), dtype=torch.bfloat16) - intrinsics = intrinsics.unsqueeze(0).to(get_local_torch_device(), dtype=torch.bfloat16) - action_labels = action_labels.unsqueeze(0).to(get_local_torch_device(), dtype=torch.bfloat16) + + # Process actions for each batch sample + batch_size = training_batch.noisy_model_input.shape[0] + viewmats_list, intrinsics_list, action_labels_list = [], [], [] + for b in range(batch_size): + v, i, a = process_custom_actions( + training_batch.keyboard_cond[b], training_batch.mouse_cond[b]) + viewmats_list.append(v) + intrinsics_list.append(i) + action_labels_list.append(a) + viewmats = torch.stack(viewmats_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) + action_labels = torch.stack(action_labels_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) # NOTE: noisy_model_input already contains concatenated image_latents from _prepare_dit_inputs training_batch.input_kwargs = { @@ -227,6 +279,52 @@ def _prepare_validation_batch(self, sampling_param: SamplingParam, return batch + def _post_process_validation_frames(self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: + """Apply action overlay to validation frames for WanGame. + + Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. + """ + # Check if action data is available + keyboard_cond = getattr(batch, 'keyboard_cond', None) + mouse_cond = getattr(batch, 'mouse_cond', None) + + if keyboard_cond is None and mouse_cond is None: + return frames + + # Import overlay functions + from fastvideo.models.dits.matrixgame.utils import ( + draw_keys_on_frame, draw_mouse_on_frame) + + # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) + if keyboard_cond is not None: + keyboard_cond = keyboard_cond.squeeze(0).cpu().float().numpy() # (T, 6) + if mouse_cond is not None: + mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) + + # MatrixGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] + key_names = ["W", "S", "A", "D", "left", "right"] + + processed_frames = [] + for frame_idx, frame in enumerate(frames): + frame = np.ascontiguousarray(frame.copy()) + + # Draw keyboard overlay + if keyboard_cond is not None and frame_idx < len(keyboard_cond): + keys = {key_names[i]: bool(keyboard_cond[frame_idx, i]) + for i in range(min(len(key_names), keyboard_cond.shape[1]))} + draw_keys_on_frame(frame, keys, mode='universal') + + # Draw mouse overlay + if mouse_cond is not None and frame_idx < len(mouse_cond): + pitch = float(mouse_cond[frame_idx, 0]) + yaw = float(mouse_cond[frame_idx, 1]) + draw_mouse_on_frame(frame, pitch, yaw) + + processed_frames.append(frame) + + return processed_frames + def main(args) -> None: logger.info("Starting training pipeline...") From c07724971b099b4703495aaf16f6376f79c0c88b Mon Sep 17 00:00:00 2001 From: mignonjia Date: Fri, 6 Feb 2026 00:15:16 +0000 Subject: [PATCH 006/214] text --- fastvideo/dataset/validation_dataset.py | 9 ++- fastvideo/fastvideo_args.py | 4 ++ fastvideo/models/dits/wangame/__init__.py | 3 + .../dits/wangame/hyworld_action_module.py | 37 ++++++++---- fastvideo/models/dits/wangame/model.py | 33 ++++++++--- .../basic/wan/wangame_i2v_pipeline.py | 10 +++- fastvideo/training/distillation_pipeline.py | 3 +- fastvideo/training/training_pipeline.py | 3 +- .../training/wangame_training_pipeline.py | 58 +++++++++++++++++-- 9 files changed, 133 insertions(+), 27 deletions(-) diff --git a/fastvideo/dataset/validation_dataset.py b/fastvideo/dataset/validation_dataset.py index 8d831d5df..cf97e8bc0 100644 --- a/fastvideo/dataset/validation_dataset.py +++ b/fastvideo/dataset/validation_dataset.py @@ -17,8 +17,9 @@ class ValidationDataset(IterableDataset): - def __init__(self, filename: str): + def __init__(self, filename: str, num_samples: int | None = None): super().__init__() + self.num_samples = num_samples self.filename = pathlib.Path(filename) # get directory of filename @@ -59,6 +60,12 @@ def __init__(self, filename: str): # Convert to list to get total samples self.all_samples = list(data) + + # Limit number of samples if specified + if self.num_samples is not None and self.num_samples < len(self.all_samples): + self.all_samples = self.all_samples[:self.num_samples] + logger.info("Limiting validation samples to %s", self.num_samples) + self.original_total_samples = len(self.all_samples) # Extend samples to be a multiple of DP degree (num_sp_groups) diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index 25e4aa187..fb00d6be6 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -859,6 +859,7 @@ class TrainingArgs(FastVideoArgs): validation_sampling_steps: str = "" validation_guidance_scale: str = "" validation_steps: float = 0.0 + validation_num_samples: int | None = None # Limit number of validation samples (None = use all) log_validation: bool = False trackers: list[str] = dataclasses.field(default_factory=list) tracker_project_name: str = "" @@ -1094,6 +1095,9 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: parser.add_argument("--validation-steps", type=float, help="Number of validation steps") + parser.add_argument("--validation-num-samples", + type=int, + help="Limit number of validation samples (default: use all)") parser.add_argument("--log-validation", action=StoreBoolean, help="Whether to log validation results") diff --git a/fastvideo/models/dits/wangame/__init__.py b/fastvideo/models/dits/wangame/__init__.py index 8b7d5c92b..a79fde639 100644 --- a/fastvideo/models/dits/wangame/__init__.py +++ b/fastvideo/models/dits/wangame/__init__.py @@ -1,5 +1,8 @@ from .model import WanGameActionTransformer3DModel +from .hyworld_action_module import WanGameActionTimeTextImageEmbedding, WanGameActionSelfAttention __all__ = [ "WanGameActionTransformer3DModel", + "WanGameActionTimeTextImageEmbedding", + "WanGameActionSelfAttention", ] diff --git a/fastvideo/models/dits/wangame/hyworld_action_module.py b/fastvideo/models/dits/wangame/hyworld_action_module.py index b98e3ff34..82fe7d905 100644 --- a/fastvideo/models/dits/wangame/hyworld_action_module.py +++ b/fastvideo/models/dits/wangame/hyworld_action_module.py @@ -11,21 +11,33 @@ from fastvideo.layers.rotary_embedding import _apply_rotary_emb from fastvideo.layers.mlp import MLP -class WanGameActionTimeImageEmbedding(nn.Module): +class WanGameActionTimeTextImageEmbedding(nn.Module): + """ + Embedding module that incorporates action signals in addition to timestep, text, and image embeddings. + Action embeddings are combined with timestep embeddings before projection. + """ + def __init__( self, dim: int, time_freq_dim: int, + text_embed_dim: int, image_embed_dim: int | None = None, ): super().__init__() - + self.dim = dim self.time_freq_dim = time_freq_dim + self.time_embedder = TimestepEmbedder( dim, frequency_embedding_size=time_freq_dim, act_layer="silu") self.time_modulation = ModulateProjection(dim, factor=6, act_layer="silu") + self.text_embedder = MLP(text_embed_dim, + dim, + dim, + bias=True, + act_type="gelu_pytorch_tanh") if text_embed_dim > 0 else None self.image_embedder = None if image_embed_dim is not None: @@ -38,8 +50,8 @@ def __init__( bias=True, act_type="silu" ) - # Initialize with zeros for residual-like behavior - nn.init.zeros_(self.action_embedder.fc_out.weight) + # Initialize with small values for residual-like behavior (allows gradients to flow) + nn.init.normal_(self.action_embedder.fc_out.weight, std=0.01) if self.action_embedder.fc_out.bias is not None: nn.init.zeros_(self.action_embedder.fc_out.bias) @@ -47,7 +59,7 @@ def forward( self, timestep: torch.Tensor, action: torch.Tensor, - encoder_hidden_states: torch.Tensor, # Kept for interface compatibility + encoder_hidden_states: torch.Tensor, encoder_hidden_states_image: torch.Tensor | None = None, timestep_seq_len: int | None = None, ): @@ -55,10 +67,13 @@ def forward( Args: timestep: [B] diffusion timesteps (one per batch sample) action: [B, T] action labels (one per frame per batch sample) + encoder_hidden_states: [B, L, D] text embeddings Returns: temb: [B*T, dim] combined timestep + action embedding timestep_proj: [B*T, 6*dim] modulation projection + encoder_hidden_states: [B, L, dim] processed text embeddings + encoder_hidden_states_image: [B, L_img, dim] processed image embeddings """ # timestep: [B] -> temb: [B, dim] temb = self.time_embedder(timestep, timestep_seq_len) @@ -89,17 +104,19 @@ def forward( timestep_proj = self.time_modulation(temb) # [B*T, 6*dim] - # MatrixGame does not use text embeddings, so we ignore encoder_hidden_states + # Process text embeddings through text_embedder if available + if self.text_embedder is not None and encoder_hidden_states is not None: + encoder_hidden_states = self.text_embedder(encoder_hidden_states) + else: + encoder_hidden_states = torch.zeros((batch_size, 0, temb.shape[-1]), + device=temb.device, + dtype=temb.dtype) if encoder_hidden_states_image is not None: assert self.image_embedder is not None encoder_hidden_states_image = self.image_embedder( encoder_hidden_states_image) - encoder_hidden_states = torch.zeros((batch_size, 0, temb.shape[-1]), - device=temb.device, - dtype=temb.dtype) - return temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image class WanGameActionSelfAttention(nn.Module): diff --git a/fastvideo/models/dits/wangame/model.py b/fastvideo/models/dits/wangame/model.py index 3e77bfaba..c3c427b18 100644 --- a/fastvideo/models/dits/wangame/model.py +++ b/fastvideo/models/dits/wangame/model.py @@ -22,7 +22,7 @@ from fastvideo.platforms import AttentionBackendEnum, current_platform # Import ActionModule -from fastvideo.models.dits.wangame.hyworld_action_module import WanGameActionTimeImageEmbedding, WanGameActionSelfAttention +from fastvideo.models.dits.wangame.hyworld_action_module import WanGameActionTimeTextImageEmbedding, WanGameActionSelfAttention logger = init_logger(__name__) @@ -242,6 +242,7 @@ def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> Non self.out_channels = config.out_channels self.num_channels_latents = config.num_channels_latents self.patch_size = config.patch_size + self.text_len = config.text_len self.local_attn_size = config.local_attn_size self.inner_dim = inner_dim @@ -251,10 +252,11 @@ def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> Non patch_size=config.patch_size, flatten=False) - # 2. Condition embeddings (with action support) - self.condition_embedder = WanGameActionTimeImageEmbedding( + # 2. Condition embeddings (with action and text support) + self.condition_embedder = WanGameActionTimeTextImageEmbedding( dim=inner_dim, time_freq_dim=config.freq_dim, + text_embed_dim=config.text_dim, image_embed_dim=config.image_dim, ) @@ -330,12 +332,16 @@ def forward( is_cache: If True, populate KV cache and return early (cache-only mode) """ orig_dtype = hidden_states.dtype - # if not isinstance(encoder_hidden_states, torch.Tensor): - # encoder_hidden_states = encoder_hidden_states[0] + # Handle encoder_hidden_states (text embeddings) - can be None, list, or tensor + if encoder_hidden_states is None: + pass # Will be handled by condition_embedder (returns zeros) + elif isinstance(encoder_hidden_states, list): + encoder_hidden_states = encoder_hidden_states[0] if len(encoder_hidden_states) > 0 else None + # Handle encoder_hidden_states_image if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: encoder_hidden_states_image = encoder_hidden_states_image[0] - # else: - # encoder_hidden_states_image = None + else: + encoder_hidden_states_image = None batch_size, num_channels, num_frames, height, width = hidden_states.shape p_t, p_h, p_w = self.patch_size @@ -365,6 +371,13 @@ def forward( if timestep.dim() == 2: timestep = timestep.flatten() + # Pad text embeddings to text_len if provided + if encoder_hidden_states is not None and encoder_hidden_states.size(1) > 0: + encoder_hidden_states = torch.cat([ + encoder_hidden_states, + encoder_hidden_states.new_zeros(batch_size, self.text_len - encoder_hidden_states.size(1), encoder_hidden_states.size(2)) + ], dim=1) + temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( timestep, action, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) @@ -375,7 +388,11 @@ def forward( timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) # [B*T, 6, dim] timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, self.hidden_size) # [B, T, 6, dim] - encoder_hidden_states = encoder_hidden_states_image + # Concatenate text and image embeddings for cross-attention + if encoder_hidden_states_image is not None: + encoder_hidden_states = torch.concat([encoder_hidden_states_image, encoder_hidden_states], dim=1) + + encoder_hidden_states = encoder_hidden_states.to(orig_dtype) if current_platform.is_mps() else encoder_hidden_states # Transformer blocks for block_idx, block in enumerate(self.blocks): diff --git a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py index e2ef763ee..2b10912ba 100644 --- a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py @@ -15,7 +15,7 @@ from fastvideo.pipelines.stages import ( ImageEncodingStage, ConditioningStage, DecodingStage, DenoisingStage, ImageVAEEncodingStage, InputValidationStage, LatentPreparationStage, - TimestepPreparationStage) + TimestepPreparationStage, TextEncodingStage) # isort: on from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( FlowUniPCMultistepScheduler) @@ -27,7 +27,7 @@ class WanGameActionImageToVideoPipeline(LoRAPipeline, ComposedPipelineBase): _required_config_modules = [ "vae", "transformer", "scheduler", \ - "image_encoder", "image_processor" + "image_encoder", "image_processor", "text_encoder", "tokenizer" ] def initialize_pipeline(self, fastvideo_args: FastVideoArgs): @@ -40,6 +40,12 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) + self.add_stage(stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + )) + self.add_stage( stage_name="image_encoding_stage", stage=ImageEncodingStage( diff --git a/fastvideo/training/distillation_pipeline.py b/fastvideo/training/distillation_pipeline.py index c9ba23588..d70379756 100644 --- a/fastvideo/training/distillation_pipeline.py +++ b/fastvideo/training/distillation_pipeline.py @@ -1181,7 +1181,8 @@ def _log_validation(self, transformer, training_args, global_step) -> None: training_args.validation_dataset_file, local_main_process_only=False) validation_dataset = ValidationDataset( - training_args.validation_dataset_file) + training_args.validation_dataset_file, + num_samples=training_args.validation_num_samples) validation_dataloader = DataLoader(validation_dataset, batch_size=None, num_workers=0) diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index 2e4456f91..e8d214798 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -832,7 +832,8 @@ def _log_validation(self, transformer, training_args, global_step) -> None: training_args.validation_dataset_file, local_main_process_only=False) validation_dataset = ValidationDataset( - training_args.validation_dataset_file) + training_args.validation_dataset_file, + num_samples=training_args.validation_num_samples) validation_dataloader = DataLoader(validation_dataset, batch_size=None, num_workers=0) diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index ba5c60f32..80eb99dd1 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -10,6 +10,7 @@ from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame from fastvideo.distributed import get_local_torch_device from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.forward_context import set_forward_context from fastvideo.logger import init_logger from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( FlowUniPCMultistepScheduler) @@ -27,11 +28,56 @@ class WanGameTrainingPipeline(TrainingPipeline): """ A training pipeline for WanGame-2.1-Fun-1.3B-InP. """ - _required_config_modules = ["scheduler", "transformer", "vae"] + _required_config_modules = ["scheduler", "transformer", "vae", "text_encoder", "tokenizer"] + + # Cached empty string embedding (computed once at initialization) + _empty_string_embedding: torch.Tensor | None = None def initialize_pipeline(self, fastvideo_args: FastVideoArgs): self.modules["scheduler"] = FlowUniPCMultistepScheduler( shift=fastvideo_args.pipeline_config.flow_shift) + + # Compute and cache the empty string embedding + self._compute_empty_string_embedding(fastvideo_args) + + @torch.no_grad() + def _compute_empty_string_embedding(self, fastvideo_args: FastVideoArgs): + """ + Compute the true empty string embedding using the text encoder. + This is called once during initialization and cached for training. + """ + logger.info("Computing empty string embedding using text encoder...") + + tokenizer = self.get_module("tokenizer") + text_encoder = self.get_module("text_encoder") + + # Get encoder config for tokenizer settings + encoder_config = fastvideo_args.pipeline_config.text_encoder_configs[0] + tok_kwargs = dict(encoder_config.tokenizer_kwargs) + + # Tokenize empty string + text_inputs = tokenizer([""], **tok_kwargs).to(get_local_torch_device()) + input_ids = text_inputs["input_ids"] + attention_mask = text_inputs["attention_mask"] + + # Encode empty string + with set_forward_context(current_timestep=0, attn_metadata=None): + outputs = text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + ) + + # Apply postprocess function (same as used in inference) + postprocess_func = fastvideo_args.pipeline_config.postprocess_text_funcs[0] + try: + prompt_embeds = postprocess_func(outputs) + except Exception: + prompt_embeds, _ = postprocess_func(outputs, attention_mask) + + # Cache the embedding (shape: [1, seq_len, hidden_dim]) + self._empty_string_embedding = prompt_embeds.to(dtype=torch.bfloat16) + logger.info(f"Empty string embedding shape: {self._empty_string_embedding.shape}") def create_training_stages(self, training_args: TrainingArgs): """ @@ -122,9 +168,13 @@ def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: training_batch.latents = latents.to(get_local_torch_device(), dtype=torch.bfloat16) - training_batch.encoder_hidden_states = None + # Use the pre-computed empty string embedding (true BOS token embedding) + # Expand to batch size: [1, seq_len, hidden_dim] -> [batch_size, seq_len, hidden_dim] + batch_size = latents.shape[0] + assert self._empty_string_embedding is not None, "Empty string embedding not initialized" + training_batch.encoder_hidden_states = self._empty_string_embedding.expand( + batch_size, -1, -1).to(get_local_torch_device()) training_batch.encoder_attention_mask = None - # MatrixGame doesn't use text encoder training_batch.preprocessed_image = pil_image.to( get_local_torch_device()) training_batch.image_embeds = clip_features.to(get_local_torch_device()) @@ -214,7 +264,7 @@ def _build_input_kwargs(self, "hidden_states": training_batch.noisy_model_input, "encoder_hidden_states": - training_batch.encoder_hidden_states, # None for MatrixGame + training_batch.encoder_hidden_states, # Zero embedding (empty prompt) "timestep": training_batch.timesteps.to(get_local_torch_device(), dtype=torch.bfloat16), From 7aedf3d62986d611cfc4ce1f400ddf93669e9399 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Fri, 6 Feb 2026 02:55:57 +0000 Subject: [PATCH 007/214] zero init fix --- docs/wangame/zero_init_fixes.md | 50 ++++++++++++++ .../scripts/generate_validation_static_w.py | 45 +++++++++--- .../dits/wangame/hyworld_action_module.py | 37 +++++++++- fastvideo/models/dits/wangame/model.py | 20 ++++-- fastvideo/models/loader/fsdp_load.py | 38 ++++++++--- .../training/wangame_training_pipeline.py | 68 +++++++++++++++++++ 6 files changed, 228 insertions(+), 30 deletions(-) create mode 100644 docs/wangame/zero_init_fixes.md diff --git a/docs/wangame/zero_init_fixes.md b/docs/wangame/zero_init_fixes.md new file mode 100644 index 000000000..b418cb5dd --- /dev/null +++ b/docs/wangame/zero_init_fixes.md @@ -0,0 +1,50 @@ +# Zero Initialization Fixes Summary + +## Problem +New parameters (`action_embedder`, `to_out_prope`) were not learning - weights stayed at zero after training. + +## Root Causes & Fixes + +### 1. FSDP Loader Overwriting Model Initialization + +**File:** `fastvideo/models/loader/fsdp_load.py` + +**Problem:** FSDP loader initialized ALL new parameters (not in checkpoint) with zeros, overwriting the model's `__init__` initialization. + +**Fix:** Added `KAIMING_INIT_PATTERNS` to selectively apply proper initialization: + +```python +ALLOWED_NEW_PARAM_PATTERNS = ["gate_compress", "proj_l", "to_out_prope", "action_embedder"] +KAIMING_INIT_PATTERNS = ["fc_in.weight", "lora_A"] # Input projections need non-zero init + +for new_param_name in unused_keys: + use_kaiming = any(pattern in new_param_name for pattern in KAIMING_INIT_PATTERNS) + if use_kaiming: + nn.init.kaiming_uniform_(tensor, a=math.sqrt(5)) # Non-zero for gradient flow + else: + torch.zeros_like(...) # Zero for output projections (residual behavior) +``` + +**Why:** +- Input projections (`fc_in.weight`) need non-zero weights for gradients to flow +- Output projections (`fc_out.weight`) should be zero-initialized for stable residual learning (ControlNet/adapter pattern) + +### 2. Attention Mask Shape Mismatch + +**File:** `fastvideo/models/dits/wangame/hyworld_action_module.py` + +**Problem:** Attention mask had shape `[B, L]` but query tensor had shape `[2*B, L, ...]` (rope + prope concatenated). The prope batch (second half) had no mask coverage → output was zeros. + +**Fix:** +```python +# Before (wrong): +attention_mask = torch.ones(batch_size, seq_len, ...) # [B, L] + +# After (correct): +attention_mask = torch.ones(batch_size * 2, seq_len, ...) # [2*B, L] +``` + +## Files Modified + +1. `fastvideo/models/loader/fsdp_load.py` - KAIMING_INIT_PATTERNS +2. `fastvideo/models/dits/wangame/hyworld_action_module.py` - attention mask shape diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py index 15ce1fde3..10e285750 100644 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py @@ -1,8 +1,11 @@ import json import os -# Paths -image_dir = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/first_frame" +# Paths for two image directories +image_dir_val = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/first_frame" +image_dir_train = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_same_1st_frame_static_plus_w_only/first_frame" + +# Action paths (used for both) action_still = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/videos/000000_action.npy" action_w = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/videos/001050_action.npy" @@ -20,22 +23,42 @@ data = [] -# 32 images, each with 2 actions (Still and W) -for i in range(32): - image_path = os.path.join(image_dir, f"{i:06d}.png") +# 16 images from each directory, alternating: val (0,1), train (2,3), val (4,5), train (6,7), ... +for i in range(16): + # Val images: indices 0,1, 4,5, 8,9, ... (pair index 0, 2, 4, ...) + image_path_val = os.path.join(image_dir_val, f"{i:06d}.png") + + # Still action for val + data.append({ + "caption": f"val {i:02d} - Still", + "image_path": image_path_val, + "action_path": action_still, + **fixed_fields + }) + + # W action for val + data.append({ + "caption": f"val {i:02d} - W", + "image_path": image_path_val, + "action_path": action_w, + **fixed_fields + }) + + # Train images: indices 2,3, 6,7, 10,11, ... (pair index 1, 3, 5, ...) + image_path_train = os.path.join(image_dir_train, f"{i:06d}.png") - # Still action + # Still action for train data.append({ - "caption": f"{i:02d} - Still", - "image_path": image_path, + "caption": f"train {i:02d} - Still", + "image_path": image_path_train, "action_path": action_still, **fixed_fields }) - # W action + # W action for train data.append({ - "caption": f"{i:02d} - W", - "image_path": image_path, + "caption": f"train {i:02d} - W", + "image_path": image_path_train, "action_path": action_w, **fixed_fields }) diff --git a/fastvideo/models/dits/wangame/hyworld_action_module.py b/fastvideo/models/dits/wangame/hyworld_action_module.py index b98e3ff34..8bf26b67c 100644 --- a/fastvideo/models/dits/wangame/hyworld_action_module.py +++ b/fastvideo/models/dits/wangame/hyworld_action_module.py @@ -1,3 +1,5 @@ +import math + import torch import torch.nn as nn @@ -38,7 +40,9 @@ def __init__( bias=True, act_type="silu" ) - # Initialize with zeros for residual-like behavior + # Initialize fc_in with kaiming_uniform (same as nn.Linear default) + nn.init.kaiming_uniform_(self.action_embedder.fc_in.weight, a=math.sqrt(5)) + # Initialize fc_out with zeros for residual-like behavior nn.init.zeros_(self.action_embedder.fc_out.weight) if self.action_embedder.fc_out.bias is not None: nn.init.zeros_(self.action_embedder.fc_out.bias) @@ -172,6 +176,14 @@ def forward(self, key_rope = _apply_rotary_emb(k, cos, sin, is_neox_style=False).type_as(v) value_rope = v + # # DEBUG: Check camera matrices + # if self.training and torch.distributed.get_rank() == 0: + # vm_info = f"viewmats={viewmats.shape if viewmats is not None else None}" + # ks_info = f"Ks={Ks.shape if Ks is not None else None}" + # vm_nonzero = (viewmats != 0).sum().item() if viewmats is not None else 0 + # ks_nonzero = (Ks != 0).sum().item() if Ks is not None else 0 + # print(f"[DEBUG] PRoPE input: {vm_info} nonzero={vm_nonzero}, {ks_info} nonzero={ks_nonzero}", flush=True) + # Get PRoPE transformed q, k, v query_prope, key_prope, value_prope, apply_fn_o = prope_qkv( q.transpose(1, 2), # [B, num_heads, L, head_dim] @@ -186,6 +198,13 @@ def forward(self, query_prope = query_prope.transpose(1, 2) key_prope = key_prope.transpose(1, 2) value_prope = value_prope.transpose(1, 2) + + # # DEBUG: Check prope_qkv output + # if self.training and torch.distributed.get_rank() == 0: + # q_nz = (query_prope != 0).sum().item() + # k_nz = (key_prope != 0).sum().item() + # v_nz = (value_prope != 0).sum().item() + # print(f"[DEBUG] prope_qkv output: q_nonzero={q_nz}, k_nonzero={k_nz}, v_nonzero={v_nz}", flush=True) # KV cache handling if kv_cache is not None: @@ -232,9 +251,10 @@ def forward(self, else: # Same sequence length: use DistributedAttention (supports SP) # Create default attention mask if not provided + # NOTE: query_all has shape [2*B, L, ...] (rope+prope concatenated), so mask needs 2*B if attention_mask is None: batch_size, seq_len = q.shape[0], q.shape[1] - attention_mask = torch.ones(batch_size, seq_len, device=q.device, dtype=q.dtype) + attention_mask = torch.ones(batch_size * 2, seq_len, device=q.device, dtype=q.dtype) if q.dtype == torch.float32: from fastvideo.attention.backends.sdpa import SDPAMetadataBuilder @@ -250,6 +270,19 @@ def forward(self, hidden_states_all, _ = self.attn(query_all, key_all, value_all, attention_mask=attention_mask) hidden_states_rope, hidden_states_prope = hidden_states_all.chunk(2, dim=0) + + # # DEBUG: Check attention output and apply_fn_o + # if self.training and torch.distributed.get_rank() == 0: + # attn_all_nz = (hidden_states_all != 0).sum().item() + # rope_nz = (hidden_states_rope != 0).sum().item() + # prope_before = (hidden_states_prope != 0).sum().item() + # print(f"[DEBUG] attn output: all_nonzero={attn_all_nz}, rope_nonzero={rope_nz}, prope_before_apply={prope_before}", flush=True) + hidden_states_prope = apply_fn_o(hidden_states_prope.transpose(1, 2)).transpose(1, 2) + + # # DEBUG: Check after apply_fn_o + # if self.training and torch.distributed.get_rank() == 0: + # prope_after = (hidden_states_prope != 0).sum().item() + # print(f"[DEBUG] prope_after_apply_fn_o={prope_after}", flush=True) return hidden_states_rope, hidden_states_prope \ No newline at end of file diff --git a/fastvideo/models/dits/wangame/model.py b/fastvideo/models/dits/wangame/model.py index 3e77bfaba..ac6331a71 100644 --- a/fastvideo/models/dits/wangame/model.py +++ b/fastvideo/models/dits/wangame/model.py @@ -125,12 +125,10 @@ def __init__(self, self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) # PRoPE output projection (initialized via add_discrete_action_parameters on the model) - self.to_out_prope = nn.ModuleList([ - nn.Linear(dim, dim, bias=True), - ]) - nn.init.zeros_(self.to_out_prope[0].weight) - if self.to_out_prope[0].bias is not None: - nn.init.zeros_(self.to_out_prope[0].bias) + self.to_out_prope = ReplicatedLinear(dim, dim, bias=True) + nn.init.zeros_(self.to_out_prope.weight) + if self.to_out_prope.bias is not None: + nn.init.zeros_(self.to_out_prope.bias) def forward( self, @@ -186,7 +184,15 @@ def forward( attn_output_rope = attn_output_rope.flatten(2) attn_output_rope, _ = self.to_out(attn_output_rope) attn_output_prope = attn_output_prope.flatten(2) - attn_output_prope = self.to_out_prope[0](attn_output_prope) + + # # DEBUG: Check if prope input is zero + # if self.training and torch.distributed.get_rank() == 0: + # prope_nonzero = (attn_output_prope != 0).sum().item() + # prope_total = attn_output_prope.numel() + # if prope_nonzero == 0: + # print(f"[DEBUG] to_out_prope INPUT is ALL ZEROS! shape={attn_output_prope.shape}", flush=True) + + attn_output_prope, _ = self.to_out_prope(attn_output_prope) attn_output = attn_output_rope.squeeze(1) + attn_output_prope.squeeze(1) # Self-attention residual + norm in float32 diff --git a/fastvideo/models/loader/fsdp_load.py b/fastvideo/models/loader/fsdp_load.py index 9b590a86c..05596dcd3 100644 --- a/fastvideo/models/loader/fsdp_load.py +++ b/fastvideo/models/loader/fsdp_load.py @@ -342,8 +342,12 @@ def load_model_from_full_model_state_dict( logger.warning("Found unloaded parameters in meta state dict: %s", unused_keys) - # List of allowed parameter name patterns - ALLOWED_NEW_PARAM_PATTERNS = ["gate_compress", "proj_l", "to_out_prope", "action_embedder"] # Can be extended as needed + # List of allowed parameter name patterns (whitelist for new params not in checkpoint) + ALLOWED_NEW_PARAM_PATTERNS = ["gate_compress", "proj_l", "to_out_prope", "action_embedder"] + + # Patterns for params that need kaiming_uniform init (input projections need non-zero for gradient flow) + KAIMING_INIT_PATTERNS = ["fc_in.weight"] + for new_param_name in unused_keys: if not any(pattern in new_param_name for pattern in ALLOWED_NEW_PARAM_PATTERNS): @@ -353,17 +357,31 @@ def load_model_from_full_model_state_dict( f"New parameter '{new_param_name}' is not supported. " f"Currently only parameters containing {ALLOWED_NEW_PARAM_PATTERNS} are allowed." ) + + # Check if this param needs kaiming init (non-zero) for gradient flow + use_kaiming = any(pattern in new_param_name for pattern in KAIMING_INIT_PATTERNS) + meta_sharded_param = meta_sd.get(new_param_name) if not hasattr(meta_sharded_param, "device_mesh"): - # Initialize with zeros - sharded_tensor = torch.zeros_like(meta_sharded_param, - device=device, - dtype=param_dtype) + # Non-sharded tensor + if use_kaiming: + import math + sharded_tensor = torch.empty_like(meta_sharded_param, device=device, dtype=param_dtype) + nn.init.kaiming_uniform_(sharded_tensor, a=math.sqrt(5)) + logger.info(f"Initialized {new_param_name} with kaiming_uniform_") + else: + # Initialize with zeros (output projections for residual behavior) + sharded_tensor = torch.zeros_like(meta_sharded_param, device=device, dtype=param_dtype) else: - # Initialize with zeros and distribute - full_tensor = torch.zeros_like(meta_sharded_param, - device=device, - dtype=param_dtype) + # Sharded tensor (DTensor) + if use_kaiming: + import math + full_tensor = torch.empty_like(meta_sharded_param, device=device, dtype=param_dtype) + nn.init.kaiming_uniform_(full_tensor, a=math.sqrt(5)) + logger.info(f"Initialized {new_param_name} with kaiming_uniform_") + else: + # Initialize with zeros and distribute + full_tensor = torch.zeros_like(meta_sharded_param, device=device, dtype=param_dtype) sharded_tensor = distribute_tensor( full_tensor, meta_sharded_param.device_mesh, diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index ba5c60f32..ac8697f6a 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -84,6 +84,74 @@ def set_trainable(self) -> None: logger.info(f"Action-only training: {trainable_count} trainable param groups, " f"{frozen_count} frozen param groups") + # def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: + # """Override to add debug logging after first step.""" + # current_step = training_batch.current_timestep + # + # # Call parent's train_one_step + # training_batch = super().train_one_step(training_batch) + # + # # === DEBUG: Print weights after optimizer step === + # if current_step == 2: # After second step (so fc_out is non-zero) + # transformer = self.get_module("transformer") + # rank = self.global_rank + # + # print(f"\n{'='*60}", flush=True) + # print(f"[Rank {rank}] DEBUG: Weights after step {current_step}", flush=True) + # print(f"{'='*60}", flush=True) + # + # # First, list all to_out_prope params to debug + # prope_params = [n for n, _ in transformer.named_parameters() if "to_out_prope" in n] + # print(f"[Rank {rank}] to_out_prope params found: {prope_params[:5]}...") # Show first 5 + # + # for name, param in transformer.named_parameters(): + # # Check action_embedder + # if "action_embedder" in name: + # data = param.data + # if hasattr(data, 'full_tensor'): + # data = data.full_tensor() + # nonzero = (data != 0).sum().item() + # print(f"[Rank {rank}] {name}:") + # print(f" shape={tuple(data.shape)}, requires_grad={param.requires_grad}") + # print(f" nonzero={nonzero}/{data.numel()}") + # print(f" min={data.min().item():.6f}, max={data.max().item():.6f}") + # if param.grad is not None: + # grad = param.grad + # if hasattr(grad, 'full_tensor'): + # grad = grad.full_tensor() + # grad_nonzero = (grad != 0).sum().item() + # print(f" grad_nonzero={grad_nonzero}/{grad.numel()}, grad_sum={grad.sum().item():.6f}") + # else: + # print(f" grad=None") + # + # # Check to_out_prope (just block 0) - should match blocks.0.to_out_prope.weight/bias + # if "to_out_prope" in name and "blocks.0" in name: + # data = param.data + # if hasattr(data, 'full_tensor'): + # data = data.full_tensor() + # nonzero = (data != 0).sum().item() + # print(f"[Rank {rank}] {name}:") + # print(f" shape={tuple(data.shape)}, requires_grad={param.requires_grad}") + # print(f" nonzero={nonzero}/{data.numel()}") + # print(f" min={data.min().item():.6f}, max={data.max().item():.6f}") + # if param.grad is not None: + # grad = param.grad + # if hasattr(grad, 'full_tensor'): + # grad = grad.full_tensor() + # grad_nonzero = (grad != 0).sum().item() + # print(f" grad_nonzero={grad_nonzero}/{grad.numel()}, grad_sum={grad.sum().item():.6f}") + # else: + # print(f" grad=None") + # + # print(f"{'='*60}") + # print(f"[Rank {rank}] DEBUG COMPLETE - Exiting after step 1") + # print(f"{'='*60}\n") + # + # import sys + # sys.exit(0) + # + # return training_batch + def initialize_validation_pipeline(self, training_args: TrainingArgs): logger.info("Initializing validation pipeline...") # args_copy.pipeline_config.vae_config.load_encoder = False From 8cb516dfb4800cee8968b5a1e40811ee3daf22cf Mon Sep 17 00:00:00 2001 From: mignonjia Date: Fri, 6 Feb 2026 03:41:39 +0000 Subject: [PATCH 008/214] Revert "Merge pull request #2 from mignonjia/wangame-text" This reverts commit 4c1d9a9905d78d4fcf61e620c5498bb323e8d7a7, reversing changes made to 7aedf3d62986d611cfc4ce1f400ddf93669e9399. --- fastvideo/models/dits/wangame/__init__.py | 4 +- .../dits/wangame/hyworld_action_module.py | 34 +++-------- fastvideo/models/dits/wangame/model.py | 33 +++-------- .../basic/wan/wangame_i2v_pipeline.py | 10 +--- .../training/wangame_training_pipeline.py | 56 +------------------ 5 files changed, 24 insertions(+), 113 deletions(-) diff --git a/fastvideo/models/dits/wangame/__init__.py b/fastvideo/models/dits/wangame/__init__.py index a79fde639..7dbe1cb64 100644 --- a/fastvideo/models/dits/wangame/__init__.py +++ b/fastvideo/models/dits/wangame/__init__.py @@ -1,8 +1,8 @@ from .model import WanGameActionTransformer3DModel -from .hyworld_action_module import WanGameActionTimeTextImageEmbedding, WanGameActionSelfAttention +from .hyworld_action_module import WanGameActionTimeImageEmbedding, WanGameActionSelfAttention __all__ = [ "WanGameActionTransformer3DModel", - "WanGameActionTimeTextImageEmbedding", + "WanGameActionTimeImageEmbedding", "WanGameActionSelfAttention", ] diff --git a/fastvideo/models/dits/wangame/hyworld_action_module.py b/fastvideo/models/dits/wangame/hyworld_action_module.py index 9c6e4d5be..8bf26b67c 100644 --- a/fastvideo/models/dits/wangame/hyworld_action_module.py +++ b/fastvideo/models/dits/wangame/hyworld_action_module.py @@ -13,33 +13,21 @@ from fastvideo.layers.rotary_embedding import _apply_rotary_emb from fastvideo.layers.mlp import MLP -class WanGameActionTimeTextImageEmbedding(nn.Module): - """ - Embedding module that incorporates action signals in addition to timestep, text, and image embeddings. - Action embeddings are combined with timestep embeddings before projection. - """ - +class WanGameActionTimeImageEmbedding(nn.Module): def __init__( self, dim: int, time_freq_dim: int, - text_embed_dim: int, image_embed_dim: int | None = None, ): super().__init__() - self.dim = dim - self.time_freq_dim = time_freq_dim + self.time_freq_dim = time_freq_dim self.time_embedder = TimestepEmbedder( dim, frequency_embedding_size=time_freq_dim, act_layer="silu") self.time_modulation = ModulateProjection(dim, factor=6, act_layer="silu") - self.text_embedder = MLP(text_embed_dim, - dim, - dim, - bias=True, - act_type="gelu_pytorch_tanh") if text_embed_dim > 0 else None self.image_embedder = None if image_embed_dim is not None: @@ -63,7 +51,7 @@ def forward( self, timestep: torch.Tensor, action: torch.Tensor, - encoder_hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, # Kept for interface compatibility encoder_hidden_states_image: torch.Tensor | None = None, timestep_seq_len: int | None = None, ): @@ -71,13 +59,10 @@ def forward( Args: timestep: [B] diffusion timesteps (one per batch sample) action: [B, T] action labels (one per frame per batch sample) - encoder_hidden_states: [B, L, D] text embeddings Returns: temb: [B*T, dim] combined timestep + action embedding timestep_proj: [B*T, 6*dim] modulation projection - encoder_hidden_states: [B, L, dim] processed text embeddings - encoder_hidden_states_image: [B, L_img, dim] processed image embeddings """ # timestep: [B] -> temb: [B, dim] temb = self.time_embedder(timestep, timestep_seq_len) @@ -108,18 +93,17 @@ def forward( timestep_proj = self.time_modulation(temb) # [B*T, 6*dim] - # Process text embeddings through text_embedder if available - if self.text_embedder is not None and encoder_hidden_states is not None: - encoder_hidden_states = self.text_embedder(encoder_hidden_states) - else: - encoder_hidden_states = torch.zeros((batch_size, 0, temb.shape[-1]), - device=temb.device, - dtype=temb.dtype) + # MatrixGame does not use text embeddings, so we ignore encoder_hidden_states + if encoder_hidden_states_image is not None: assert self.image_embedder is not None encoder_hidden_states_image = self.image_embedder( encoder_hidden_states_image) + encoder_hidden_states = torch.zeros((batch_size, 0, temb.shape[-1]), + device=temb.device, + dtype=temb.dtype) + return temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image class WanGameActionSelfAttention(nn.Module): diff --git a/fastvideo/models/dits/wangame/model.py b/fastvideo/models/dits/wangame/model.py index a9b59aeb3..ac6331a71 100644 --- a/fastvideo/models/dits/wangame/model.py +++ b/fastvideo/models/dits/wangame/model.py @@ -22,7 +22,7 @@ from fastvideo.platforms import AttentionBackendEnum, current_platform # Import ActionModule -from fastvideo.models.dits.wangame.hyworld_action_module import WanGameActionTimeTextImageEmbedding, WanGameActionSelfAttention +from fastvideo.models.dits.wangame.hyworld_action_module import WanGameActionTimeImageEmbedding, WanGameActionSelfAttention logger = init_logger(__name__) @@ -248,7 +248,6 @@ def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> Non self.out_channels = config.out_channels self.num_channels_latents = config.num_channels_latents self.patch_size = config.patch_size - self.text_len = config.text_len self.local_attn_size = config.local_attn_size self.inner_dim = inner_dim @@ -258,11 +257,10 @@ def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> Non patch_size=config.patch_size, flatten=False) - # 2. Condition embeddings (with action and text support) - self.condition_embedder = WanGameActionTimeTextImageEmbedding( + # 2. Condition embeddings (with action support) + self.condition_embedder = WanGameActionTimeImageEmbedding( dim=inner_dim, time_freq_dim=config.freq_dim, - text_embed_dim=config.text_dim, image_embed_dim=config.image_dim, ) @@ -338,16 +336,12 @@ def forward( is_cache: If True, populate KV cache and return early (cache-only mode) """ orig_dtype = hidden_states.dtype - # Handle encoder_hidden_states (text embeddings) - can be None, list, or tensor - if encoder_hidden_states is None: - pass # Will be handled by condition_embedder (returns zeros) - elif isinstance(encoder_hidden_states, list): - encoder_hidden_states = encoder_hidden_states[0] if len(encoder_hidden_states) > 0 else None - # Handle encoder_hidden_states_image + # if not isinstance(encoder_hidden_states, torch.Tensor): + # encoder_hidden_states = encoder_hidden_states[0] if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: encoder_hidden_states_image = encoder_hidden_states_image[0] - else: - encoder_hidden_states_image = None + # else: + # encoder_hidden_states_image = None batch_size, num_channels, num_frames, height, width = hidden_states.shape p_t, p_h, p_w = self.patch_size @@ -377,13 +371,6 @@ def forward( if timestep.dim() == 2: timestep = timestep.flatten() - # Pad text embeddings to text_len if provided - if encoder_hidden_states is not None and encoder_hidden_states.size(1) > 0: - encoder_hidden_states = torch.cat([ - encoder_hidden_states, - encoder_hidden_states.new_zeros(batch_size, self.text_len - encoder_hidden_states.size(1), encoder_hidden_states.size(2)) - ], dim=1) - temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( timestep, action, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) @@ -394,11 +381,7 @@ def forward( timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) # [B*T, 6, dim] timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, self.hidden_size) # [B, T, 6, dim] - # Concatenate text and image embeddings for cross-attention - if encoder_hidden_states_image is not None: - encoder_hidden_states = torch.concat([encoder_hidden_states_image, encoder_hidden_states], dim=1) - - encoder_hidden_states = encoder_hidden_states.to(orig_dtype) if current_platform.is_mps() else encoder_hidden_states + encoder_hidden_states = encoder_hidden_states_image # Transformer blocks for block_idx, block in enumerate(self.blocks): diff --git a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py index 2b10912ba..e2ef763ee 100644 --- a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py @@ -15,7 +15,7 @@ from fastvideo.pipelines.stages import ( ImageEncodingStage, ConditioningStage, DecodingStage, DenoisingStage, ImageVAEEncodingStage, InputValidationStage, LatentPreparationStage, - TimestepPreparationStage, TextEncodingStage) + TimestepPreparationStage) # isort: on from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( FlowUniPCMultistepScheduler) @@ -27,7 +27,7 @@ class WanGameActionImageToVideoPipeline(LoRAPipeline, ComposedPipelineBase): _required_config_modules = [ "vae", "transformer", "scheduler", \ - "image_encoder", "image_processor", "text_encoder", "tokenizer" + "image_encoder", "image_processor" ] def initialize_pipeline(self, fastvideo_args: FastVideoArgs): @@ -40,12 +40,6 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) - self.add_stage(stage_name="prompt_encoding_stage", - stage=TextEncodingStage( - text_encoders=[self.get_module("text_encoder")], - tokenizers=[self.get_module("tokenizer")], - )) - self.add_stage( stage_name="image_encoding_stage", stage=ImageEncodingStage( diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index 9f52054c1..f8cade34e 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -28,57 +28,12 @@ class WanGameTrainingPipeline(TrainingPipeline): """ A training pipeline for WanGame-2.1-Fun-1.3B-InP. """ - _required_config_modules = ["scheduler", "transformer", "vae", "text_encoder", "tokenizer"] - - # Cached empty string embedding (computed once at initialization) - _empty_string_embedding: torch.Tensor | None = None + _required_config_modules = ["scheduler", "transformer", "vae"] def initialize_pipeline(self, fastvideo_args: FastVideoArgs): self.modules["scheduler"] = FlowUniPCMultistepScheduler( shift=fastvideo_args.pipeline_config.flow_shift) - # Compute and cache the empty string embedding - self._compute_empty_string_embedding(fastvideo_args) - - @torch.no_grad() - def _compute_empty_string_embedding(self, fastvideo_args: FastVideoArgs): - """ - Compute the true empty string embedding using the text encoder. - This is called once during initialization and cached for training. - """ - logger.info("Computing empty string embedding using text encoder...") - - tokenizer = self.get_module("tokenizer") - text_encoder = self.get_module("text_encoder") - - # Get encoder config for tokenizer settings - encoder_config = fastvideo_args.pipeline_config.text_encoder_configs[0] - tok_kwargs = dict(encoder_config.tokenizer_kwargs) - - # Tokenize empty string - text_inputs = tokenizer([""], **tok_kwargs).to(get_local_torch_device()) - input_ids = text_inputs["input_ids"] - attention_mask = text_inputs["attention_mask"] - - # Encode empty string - with set_forward_context(current_timestep=0, attn_metadata=None): - outputs = text_encoder( - input_ids=input_ids, - attention_mask=attention_mask, - output_hidden_states=True, - ) - - # Apply postprocess function (same as used in inference) - postprocess_func = fastvideo_args.pipeline_config.postprocess_text_funcs[0] - try: - prompt_embeds = postprocess_func(outputs) - except Exception: - prompt_embeds, _ = postprocess_func(outputs, attention_mask) - - # Cache the embedding (shape: [1, seq_len, hidden_dim]) - self._empty_string_embedding = prompt_embeds.to(dtype=torch.bfloat16) - logger.info(f"Empty string embedding shape: {self._empty_string_embedding.shape}") - def create_training_stages(self, training_args: TrainingArgs): """ May be used in future refactors. @@ -236,12 +191,7 @@ def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: training_batch.latents = latents.to(get_local_torch_device(), dtype=torch.bfloat16) - # Use the pre-computed empty string embedding (true BOS token embedding) - # Expand to batch size: [1, seq_len, hidden_dim] -> [batch_size, seq_len, hidden_dim] - batch_size = latents.shape[0] - assert self._empty_string_embedding is not None, "Empty string embedding not initialized" - training_batch.encoder_hidden_states = self._empty_string_embedding.expand( - batch_size, -1, -1).to(get_local_torch_device()) + training_batch.encoder_hidden_states = None training_batch.encoder_attention_mask = None training_batch.preprocessed_image = pil_image.to( get_local_torch_device()) @@ -332,7 +282,7 @@ def _build_input_kwargs(self, "hidden_states": training_batch.noisy_model_input, "encoder_hidden_states": - training_batch.encoder_hidden_states, # Zero embedding (empty prompt) + training_batch.encoder_hidden_states, # None (no text conditioning) "timestep": training_batch.timesteps.to(get_local_torch_device(), dtype=torch.bfloat16), From 35bd0adbba88fe1f714044c287c8b91f1300bacc Mon Sep 17 00:00:00 2001 From: mignonjia Date: Fri, 6 Feb 2026 21:52:17 +0000 Subject: [PATCH 009/214] wsad --- .../finetune_wangame.slurm | 69 ++-- .../WanGame2.1_1.3b_i2v/validation_wsad.json | 324 ++++++++++++++++++ fastvideo/fastvideo_args.py | 12 + .../training/wangame_training_pipeline.py | 129 ++++--- 4 files changed, 440 insertions(+), 94 deletions(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm index 6b34764af..5802cab0c 100644 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm @@ -1,8 +1,8 @@ #!/bin/bash #SBATCH --job-name=wangame_1.3b #SBATCH --partition=main -#SBATCH --nodes=8 -#SBATCH --ntasks=8 +#SBATCH --nodes=2 +#SBATCH --ntasks=2 #SBATCH --ntasks-per-node=1 #SBATCH --gres=gpu:8 #SBATCH --cpus-per-task=128 @@ -33,34 +33,36 @@ export HOME="/mnt/weka/home/hao.zhang/mhuo" # Configs NUM_GPUS=8 -NUM_NODES=8 +NUM_NODES=2 NUM_TOTAL_GPUS=$((NUM_GPUS * NUM_NODES)) -BS_PER_GPU=4 +BS_PER_GPU=2 GRADIENT_ACCUMULATION_STEPS=1 -WANDB_RUN_NAME="bs256" +WANDB_RUN_NAME="MC_WASD_bs32_with_warmup_lr_1e-4" FREEZE_DIT=False +RUN_DIR="wangame_1.3b_with_warmup_lr_1e-4" +CHECKPOINTING_STEPS=1000 +SCENE_PROMPT="" +ACTION_WARMUP_STEPS=200 +LEARNING_RATE=1e-4 MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed" -VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" +# MC w only +DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed" +VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json" + +# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" # DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed" # VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_doom.json" # DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" # VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_overfit.json" -if [[ "$DATA_DIR" == *"mc_wasd_10"* ]]; then - RUN_DIR="wangame_1.3b_overfit" - CHECKPOINTING_STEPS=100000 -else - RUN_DIR="wangame_1.3b" - CHECKPOINTING_STEPS=1000 -fi # Training arguments training_args=( - --tracker_project_name $RUN_DIR + --tracker_project_name "wangame_1.3b" --output_dir $RUN_DIR - --wandb_run_name $WANDB_RUN_NAME + --wandb_run_name "$WANDB_RUN_NAME" --max_train_steps 10000 --train_batch_size $BS_PER_GPU --train_sp_batch_size $BS_PER_GPU @@ -71,6 +73,7 @@ training_args=( --num_frames 77 --enable_gradient_checkpointing_type "full" --train_action_only $FREEZE_DIT + --action_warmup_steps $ACTION_WARMUP_STEPS ) # Parallel arguments @@ -106,7 +109,7 @@ validation_args=( # Optimizer arguments optimizer_args=( - --learning_rate 2e-5 + --learning_rate $LEARNING_RATE --mixed_precision "bf16" --weight_only_checkpointing_steps 100000 --training_state_checkpointing_steps $CHECKPOINTING_STEPS @@ -127,15 +130,26 @@ miscellaneous_args=( ) # If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t -# torchrun \ -# --nnodes 1 \ -# --nproc_per_node $NUM_GPUS \ -srun torchrun \ - --nnodes $NUM_NODES \ - --nproc_per_node $NUM_GPUS \ - --rdzv_backend c10d \ - --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT \ - --node_rank $SLURM_PROCID \ + +if [ $NUM_NODES -eq 1 ]; then + torchrun \ + --nnodes $NUM_NODES \ + --nproc_per_node $NUM_GPUS \ + fastvideo/training/wangame_training_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" +else + srun torchrun \ + --nnodes $NUM_NODES \ + --nproc_per_node $NUM_GPUS \ + --rdzv_backend c10d \ + --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT \ + --node_rank $SLURM_PROCID \ fastvideo/training/wangame_training_pipeline.py \ "${parallel_args[@]}" \ "${model_args[@]}" \ @@ -143,4 +157,5 @@ srun torchrun \ "${training_args[@]}" \ "${optimizer_args[@]}" \ "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ No newline at end of file + "${miscellaneous_args[@]}" +fi \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json new file mode 100644 index 000000000..be9a13b82 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json @@ -0,0 +1,324 @@ +{ + "data": [ + { + "caption": "Validation 00 - W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 00 - S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 00 - A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 00 - D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 00 - Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 00 - W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 00 - S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 00 - A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 00 - D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 01 - W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/002000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 01 - S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/002000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 01 - A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/002000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 01 - D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/002000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 02 - W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/004000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 02 - S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/004000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 02 - A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/004000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "train 02 - D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/004000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + } + ] +} \ No newline at end of file diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index fb00d6be6..0673679dc 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -923,6 +923,10 @@ class TrainingArgs(FastVideoArgs): # Action-only training (freeze base model, only train action params) train_action_only: bool = False + # Action warmup: keep action modules (action_embedder, to_out_prope) at zero + # for this many steps to let the base model stabilize first, then enable them. + action_warmup_steps: int = 0 + # distillation args generator_update_interval: int = 5 dfake_gen_update_ratio: int = 5 # self-forcing: how often to train generator vs critic @@ -1282,6 +1286,14 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: help="Whether to only train action-related parameters " "(action_embedder and to_out_prope) while freezing base model") + # Action warmup: keep action modules frozen for N steps + parser.add_argument("--action-warmup-steps", + type=int, + default=0, + help="Number of steps to keep action modules " + "(action_embedder, to_out_prope) frozen to let " + "the base model stabilize first") + # V-MoBA parameters parser.add_argument( "--moba-config-path", diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index f8cade34e..d54dfc137 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -85,73 +85,46 @@ def set_trainable(self) -> None: logger.info(f"Action-only training: {trainable_count} trainable param groups, " f"{frozen_count} frozen param groups") - # def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: - # """Override to add debug logging after first step.""" - # current_step = training_batch.current_timestep - # - # # Call parent's train_one_step - # training_batch = super().train_one_step(training_batch) - # - # # === DEBUG: Print weights after optimizer step === - # if current_step == 2: # After second step (so fc_out is non-zero) - # transformer = self.get_module("transformer") - # rank = self.global_rank - # - # print(f"\n{'='*60}", flush=True) - # print(f"[Rank {rank}] DEBUG: Weights after step {current_step}", flush=True) - # print(f"{'='*60}", flush=True) - # - # # First, list all to_out_prope params to debug - # prope_params = [n for n, _ in transformer.named_parameters() if "to_out_prope" in n] - # print(f"[Rank {rank}] to_out_prope params found: {prope_params[:5]}...") # Show first 5 - # - # for name, param in transformer.named_parameters(): - # # Check action_embedder - # if "action_embedder" in name: - # data = param.data - # if hasattr(data, 'full_tensor'): - # data = data.full_tensor() - # nonzero = (data != 0).sum().item() - # print(f"[Rank {rank}] {name}:") - # print(f" shape={tuple(data.shape)}, requires_grad={param.requires_grad}") - # print(f" nonzero={nonzero}/{data.numel()}") - # print(f" min={data.min().item():.6f}, max={data.max().item():.6f}") - # if param.grad is not None: - # grad = param.grad - # if hasattr(grad, 'full_tensor'): - # grad = grad.full_tensor() - # grad_nonzero = (grad != 0).sum().item() - # print(f" grad_nonzero={grad_nonzero}/{grad.numel()}, grad_sum={grad.sum().item():.6f}") - # else: - # print(f" grad=None") - # - # # Check to_out_prope (just block 0) - should match blocks.0.to_out_prope.weight/bias - # if "to_out_prope" in name and "blocks.0" in name: - # data = param.data - # if hasattr(data, 'full_tensor'): - # data = data.full_tensor() - # nonzero = (data != 0).sum().item() - # print(f"[Rank {rank}] {name}:") - # print(f" shape={tuple(data.shape)}, requires_grad={param.requires_grad}") - # print(f" nonzero={nonzero}/{data.numel()}") - # print(f" min={data.min().item():.6f}, max={data.max().item():.6f}") - # if param.grad is not None: - # grad = param.grad - # if hasattr(grad, 'full_tensor'): - # grad = grad.full_tensor() - # grad_nonzero = (grad != 0).sum().item() - # print(f" grad_nonzero={grad_nonzero}/{grad.numel()}, grad_sum={grad.sum().item():.6f}") - # else: - # print(f" grad=None") - # - # print(f"{'='*60}") - # print(f"[Rank {rank}] DEBUG COMPLETE - Exiting after step 1") - # print(f"{'='*60}\n") - # - # import sys - # sys.exit(0) - # - # return training_batch + # ── Action module warmup ────────────────────────────────────────────── + # For the first `action_warmup_steps`, action modules (action_embedder, + # to_out_prope) have requires_grad=False so the base model stabilizes + # first. After warmup the gradients are re-enabled. + + _ACTION_PARAM_PATTERNS = [ + "condition_embedder.action_embedder", + "to_out_prope", + ] + + def _set_action_params_grad(self, requires_grad: bool) -> None: + """Toggle requires_grad for action-related parameters.""" + transformer = self.get_module("transformer") + count = 0 + for name, param in transformer.named_parameters(): + if any(p in name for p in self._ACTION_PARAM_PATTERNS): + param.requires_grad_(requires_grad) + count += 1 + state = "enabled" if requires_grad else "disabled" + logger.info("Gradients %s for %d action parameter groups", state, count) + + def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: + step = training_batch.current_timestep + warmup_steps = self.training_args.action_warmup_steps + + if warmup_steps > 0: + if step == 1: + # Freeze action params at the very first step + self._set_action_params_grad(False) + logger.info( + "Action warmup: freezing action modules for the first " + "%d steps to stabilize base model", warmup_steps) + elif step == warmup_steps + 1: + # Unfreeze action params once warmup is done + self._set_action_params_grad(True) + logger.info( + "Action warmup complete — action modules unfrozen at " + "step %d", step) + + return super().train_one_step(training_batch) def initialize_validation_pipeline(self, training_args: TrainingArgs): logger.info("Initializing validation pipeline...") @@ -213,6 +186,19 @@ def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: else: training_batch.keyboard_cond = None + # Validate action temporal dimensions match video num_frames + expected_num_frames = (self.training_args.num_latent_t - 1) * 4 + 1 + if training_batch.keyboard_cond is not None: + assert training_batch.keyboard_cond.shape[1] == expected_num_frames, ( + f"keyboard_cond temporal dim {training_batch.keyboard_cond.shape[1]} " + f"!= expected {expected_num_frames} " + f"(num_latent_t={self.training_args.num_latent_t})") + if training_batch.mouse_cond is not None: + assert training_batch.mouse_cond.shape[1] == expected_num_frames, ( + f"mouse_cond temporal dim {training_batch.mouse_cond.shape[1]} " + f"!= expected {expected_num_frames} " + f"(num_latent_t={self.training_args.num_latent_t})") + return training_batch def _prepare_dit_inputs(self, @@ -277,6 +263,15 @@ def _build_input_kwargs(self, intrinsics = torch.stack(intrinsics_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) action_labels = torch.stack(action_labels_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) + # Validate processed action latent dim matches video latent dim + num_latent_t = training_batch.noisy_model_input.shape[2] + assert action_labels.shape[1] == num_latent_t, ( + f"action_labels temporal dim {action_labels.shape[1]} != " + f"video latent temporal dim {num_latent_t}") + assert viewmats.shape[1] == num_latent_t, ( + f"viewmats temporal dim {viewmats.shape[1]} != " + f"video latent temporal dim {num_latent_t}") + # NOTE: noisy_model_input already contains concatenated image_latents from _prepare_dit_inputs training_batch.input_kwargs = { "hidden_states": From 868ce50a3b15aa1c079f9fffcf39250b28ea883a Mon Sep 17 00:00:00 2001 From: mignonjia Date: Sun, 8 Feb 2026 01:32:33 +0000 Subject: [PATCH 010/214] load ckpt using safetensor --- .../WanGame2.1_1.3b_i2v/action/README.md | 84 +- .../finetune_wangame.slurm | 31 +- .../WanGame2.1_1.3b_i2v/validation_mc.json | 924 ++++++++++++++++++ 3 files changed, 1007 insertions(+), 32 deletions(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/README.md b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/README.md index cdddcfab6..ec488ec39 100644 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/README.md +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/README.md @@ -1,18 +1,66 @@ -Total Files: 16 - -00. Hold [W] + Static -01. Hold [S] + Static -02. Hold [A] + Static -03. Hold [D] + Static -04. Hold [WA] + Static -05. Hold [WD] + Static -06. Hold [SA] + Static -07. Hold [SD] + Static -08. No Key + Hold [up] -09. No Key + Hold [down] -10. No Key + Hold [left] -11. No Key + Hold [right] -12. No Key + Hold [up_right] -13. No Key + Hold [up_left] -14. No Key + Hold [down_right] -15. No Key + Hold [down_left] +Total Files: 16 + +00. Hold [W] + Static +01. Hold [S] + Static +02. Hold [A] + Static +03. Hold [D] + Static +04. Hold [WA] + Static +05. Hold [WD] + Static +06. Hold [SA] + Static +07. Hold [SD] + Static +08. No Key + Hold [up] +09. No Key + Hold [down] +10. No Key + Hold [left] +11. No Key + Hold [right] +12. No Key + Hold [up_right] +13. No Key + Hold [up_left] +14. No Key + Hold [down_right] +15. No Key + Hold [down_left] +16. Switch [W]->[S] + Static +17. Switch [S]->[W] + Static +18. Switch [A]->[D] + Static +19. Switch [D]->[A] + Static +20. Switch [W]->[A] + Static +21. Switch [W]->[D] + Static +22. Switch [S]->[A] + Static +23. Switch [S]->[D] + Static +24. No Key + Switch [left]->[right] +25. No Key + Switch [right]->[left] +26. No Key + Switch [up]->[down] +27. No Key + Switch [down]->[up] +28. No Key + Switch [up_left]->[up_right] +29. No Key + Switch [up_right]->[up_left] +30. No Key + Switch [left]->[up] +31. No Key + Switch [right]->[down] +32. Hold [W] + Hold [left] +33. Hold [S] + Hold [left] +34. Hold [W] + Hold [right] +35. Hold [S] + Hold [right] +36. Hold [A] + Hold [up] +37. Hold [D] + Hold [up] +38. Hold [WA] + Hold [down] +39. Hold [WD] + Hold [down] +40. Hold [W] + Hold [up_left] +41. Hold [S] + Hold [up_left] +42. Hold [W] + Hold [up_right] +43. Hold [S] + Hold [up_right] +44. Hold [A] + Hold [down_left] +45. Hold [D] + Hold [down_right] +46. Hold [WA] + Hold [right] +47. Hold [WD] + Hold [left] +48. Hold [W] + Switch [left]->[right] +49. Hold [W] + Switch [right]->[left] +50. Hold [W] + Switch [up]->[down] +51. Hold [W] + Switch [down]->[up] +52. Hold [W] + Switch [left]->[up] +53. Hold [W] + Switch [right]->[up] +54. Hold [W] + Switch [left]->[down] +55. Hold [W] + Switch [right]->[down] +56. Switch [W]->[S] + Hold [up] +57. Switch [S]->[W] + Hold [up] +58. Switch [A]->[D] + Hold [up] +59. Switch [D]->[A] + Hold [up] +60. Switch [W]->[A] + Hold [up] +61. Switch [W]->[D] + Hold [up] +62. Switch [S]->[A] + Hold [up] +63. Switch [S]->[D] + Hold [up] diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm index 5802cab0c..bdb0a34b6 100644 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm @@ -1,8 +1,8 @@ #!/bin/bash #SBATCH --job-name=wangame_1.3b #SBATCH --partition=main -#SBATCH --nodes=2 -#SBATCH --ntasks=2 +#SBATCH --nodes=4 +#SBATCH --ntasks=4 #SBATCH --ntasks-per-node=1 #SBATCH --gres=gpu:8 #SBATCH --cpus-per-task=128 @@ -33,25 +33,27 @@ export HOME="/mnt/weka/home/hao.zhang/mhuo" # Configs NUM_GPUS=8 -NUM_NODES=2 +NUM_NODES=4 NUM_TOTAL_GPUS=$((NUM_GPUS * NUM_NODES)) -BS_PER_GPU=2 +BS_PER_GPU=1 GRADIENT_ACCUMULATION_STEPS=1 -WANDB_RUN_NAME="MC_WASD_bs32_with_warmup_lr_1e-4" +WANDB_RUN_NAME="MC_wsad_random_lr_1e-5" FREEZE_DIT=False -RUN_DIR="wangame_1.3b_with_warmup_lr_1e-4" +RUN_DIR="wangame_1.3b_wsad_random_lr_1e-5" CHECKPOINTING_STEPS=1000 -SCENE_PROMPT="" -ACTION_WARMUP_STEPS=200 -LEARNING_RATE=1e-4 +ACTION_WARMUP_STEPS=0 +LEARNING_RATE=1e-5 MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -# MC w only -DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed" -VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json" +CKPT_SAFETENSOR="wangame_1.3b_with_warmup_lr_1e-5/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" +# MC wasd only +# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json" + +# MC random +DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed" +VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" -# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" # DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed" # VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_doom.json" # DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" @@ -74,6 +76,7 @@ training_args=( --enable_gradient_checkpointing_type "full" --train_action_only $FREEZE_DIT --action_warmup_steps $ACTION_WARMUP_STEPS + --init_weights_from_safetensors $CKPT_SAFETENSOR ) # Parallel arguments diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json new file mode 100644 index 000000000..8488ee725 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json @@ -0,0 +1,924 @@ +{ + "data": [ + { + "caption": "Validation 00 - W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 00 - A => D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/ad.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 00 - Random", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 00 - Random", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000500_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - S => A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/sa.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - Random", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - Random", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000500_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - S => W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/sw.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - Random", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - Random", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001500_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - W => D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/wd.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - Random", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - Random", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001500_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 00 - Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01 - Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02 - Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03 - Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Train 00", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Train 01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000500.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000500_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Train 02", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Train 03", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001500.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001500_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Doom 00 - W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Doom 01 - S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Doom 02 - A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Doom 03 - D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Doom 00 - W => S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "action/000016_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Doom 01 - A => D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "action/000018_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Doom 02 - W => A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "action/000020_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Doom 03 - S => A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "action/000022_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Train 00 - W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Train 01 - S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Train 02 - A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Train 03 - D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 00", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 02", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "action/000018_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 03", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "action/000019_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 04", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "action/000020_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 05", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "action/000021_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 06", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "action/000022_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 07", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "action/000023_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 08", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "action/000016_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 09", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "action/000017_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 10", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "action/000018_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 11", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "action/000019_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 12", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "action/000020_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 13", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "action/000021_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 14", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "action/000022_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 15", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "action/000023_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 16", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "action/000020_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 17", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "action/000021_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 18", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "action/000022_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 19", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "action/000023_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 20", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "action/000024_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 21", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "action/000025_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 22", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "action/000026_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "Validation 23", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "action/000027_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "32", + "image_path": "humanplay/000000.jpg", + "action_path": "humanplay/000000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "33", + "image_path": "humanplay/000001.jpg", + "action_path": "humanplay/000001_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "34", + "image_path": "humanplay/000002.jpg", + "action_path": "humanplay/000002_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "35", + "image_path": "humanplay/000003.jpg", + "action_path": "humanplay/000003_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "36", + "image_path": "humanplay/000004.jpg", + "action_path": "action/000004_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "37", + "image_path": "humanplay/000005.jpg", + "action_path": "action/000005_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "38", + "image_path": "humanplay/000006.jpg", + "action_path": "action/000006_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "39", + "image_path": "humanplay/000007.jpg", + "action_path": "action/000007_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "40", + "image_path": "humanplay/000008.jpg", + "action_path": "action/000008_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "41", + "image_path": "humanplay/000009.jpg", + "action_path": "action/000009_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "42", + "image_path": "humanplay/000010.jpg", + "action_path": "action/000010_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "43", + "image_path": "humanplay/000011.jpg", + "action_path": "action/000011_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "44", + "image_path": "humanplay/000012.jpg", + "action_path": "action/000012_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "45", + "image_path": "humanplay/000013.jpg", + "action_path": "action/000013_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "46", + "image_path": "humanplay/000014.jpg", + "action_path": "action/000014_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "47", + "image_path": "humanplay/000015.jpg", + "action_path": "action/000015_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "48", + "image_path": "humanplay/000016.jpg", + "action_path": "action/000016_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "49", + "image_path": "humanplay/000017.jpg", + "action_path": "action/000017_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "50", + "image_path": "humanplay/000018.jpg", + "action_path": "action/000018_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "51", + "image_path": "humanplay/000019.jpg", + "action_path": "action/000019_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "52", + "image_path": "humanplay/000020.jpg", + "action_path": "action/000020_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "53", + "image_path": "humanplay/000021.jpg", + "action_path": "action/000021_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "54", + "image_path": "humanplay/000022.jpg", + "action_path": "action/000022_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "55", + "image_path": "humanplay/000023.jpg", + "action_path": "action/000023_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "56", + "image_path": "humanplay/000024.jpg", + "action_path": "action/000024_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "57", + "image_path": "humanplay/000025.jpg", + "action_path": "action/000025_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "58", + "image_path": "humanplay/000026.jpg", + "action_path": "action/000026_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "59", + "image_path": "humanplay/000027.jpg", + "action_path": "action/000027_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "60", + "image_path": "humanplay/000028.jpg", + "action_path": "action/000028_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "61", + "image_path": "humanplay/000029.jpg", + "action_path": "action/000029_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "62", + "image_path": "humanplay/000030.jpg", + "action_path": "action/000030_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "63", + "image_path": "humanplay/000031.jpg", + "action_path": "action/000031_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + } + ] +} \ No newline at end of file From 8b4dec5c2bd9ce723af1db4511cd5163abb2ae1d Mon Sep 17 00:00:00 2001 From: mignonjia Date: Sun, 8 Feb 2026 23:29:43 +0000 Subject: [PATCH 011/214] shuffle each epoch --- .../dataset/parquet_dataset_map_style.py | 30 ++++++++++++++----- fastvideo/fastvideo_args.py | 5 ++++ fastvideo/training/training_pipeline.py | 5 +++- .../training/wangame_training_pipeline.py | 2 ++ 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/fastvideo/dataset/parquet_dataset_map_style.py b/fastvideo/dataset/parquet_dataset_map_style.py index fa4a4460e..866faa51f 100644 --- a/fastvideo/dataset/parquet_dataset_map_style.py +++ b/fastvideo/dataset/parquet_dataset_map_style.py @@ -36,6 +36,7 @@ def __init__( global_rank: int, drop_last: bool = True, drop_first_row: bool = False, + reshuffle_each_epoch: bool = True, seed: int = 0, ): self.batch_size = batch_size @@ -45,34 +46,40 @@ def __init__( self.num_sp_groups = num_sp_groups self.global_rank = global_rank self.sp_world_size = sp_world_size + self.drop_first_row = drop_first_row + self.reshuffle_each_epoch = reshuffle_each_epoch + self._build_indices(0) # build indices for epoch 0 to initialize the sampler + + def _build_indices(self, epoch: int) -> None: # ── epoch-level RNG ──────────────────────────────────────────────── - rng = torch.Generator().manual_seed(self.seed) + rng = torch.Generator().manual_seed(self.seed + epoch) # Create a random permutation of all indices global_indices = torch.randperm(self.dataset_size, generator=rng) - if drop_first_row: + dataset_size = self.dataset_size + if self.drop_first_row: # drop 0 in global_indices global_indices = global_indices[global_indices != 0] - self.dataset_size = self.dataset_size - 1 + dataset_size = dataset_size - 1 if self.drop_last: # For drop_last=True, we: # 1. Ensure total samples is divisible by (batch_size * num_sp_groups) # 2. This guarantees each SP group gets same number of complete batches # 3. Prevents uneven batch sizes across SP groups at end of epoch - num_batches = self.dataset_size // self.batch_size + num_batches = dataset_size // self.batch_size num_global_batches = num_batches // self.num_sp_groups global_indices = global_indices[:num_global_batches * self.num_sp_groups * self.batch_size] else: - if self.dataset_size % (self.num_sp_groups * self.batch_size) != 0: + if dataset_size % (self.num_sp_groups * self.batch_size) != 0: # add more indices to make it divisible by (batch_size * num_sp_groups) padding_size = self.num_sp_groups * self.batch_size - ( - self.dataset_size % (self.num_sp_groups * self.batch_size)) + dataset_size % (self.num_sp_groups * self.batch_size)) logger.info("Padding the dataset from %d to %d", - self.dataset_size, self.dataset_size + padding_size) + dataset_size, dataset_size + padding_size) global_indices = torch.cat( [global_indices, global_indices[:padding_size]]) @@ -84,6 +91,11 @@ def __init__( logger.info("Dataset size for each sp group: %d", len(sp_group_local_indices)) + def set_epoch(self, epoch: int) -> None: + if not self.reshuffle_each_epoch: + return + self._build_indices(epoch) + def __iter__(self): indices = self.sp_group_local_indices for i in range(0, len(indices), self.batch_size): @@ -231,6 +243,7 @@ def __init__( seed: int = 42, drop_last: bool = True, drop_first_row: bool = False, + reshuffle_each_epoch: bool = False, text_padding_length: int = 512, ): super().__init__() @@ -253,6 +266,7 @@ def __init__( global_rank=get_world_rank(), drop_last=drop_last, drop_first_row=drop_first_row, + reshuffle_each_epoch=reshuffle_each_epoch, seed=seed, ) logger.info("Dataset initialized with %d parquet files and %d rows", @@ -325,6 +339,7 @@ def build_parquet_map_style_dataloader( cfg_rate=0.0, drop_last=True, drop_first_row=False, + reshuffle_each_epoch=False, text_padding_length=512, seed=42) -> tuple[LatentsParquetMapStyleDataset, StatefulDataLoader]: dataset = LatentsParquetMapStyleDataset( @@ -333,6 +348,7 @@ def build_parquet_map_style_dataloader( cfg_rate=cfg_rate, drop_last=drop_last, drop_first_row=drop_first_row, + reshuffle_each_epoch=reshuffle_each_epoch, text_padding_length=text_padding_length, parquet_schema=parquet_schema, seed=seed) diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index 0673679dc..4b0db30c1 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -831,6 +831,7 @@ class TrainingArgs(FastVideoArgs): """ data_path: str = "" dataloader_num_workers: int = 0 + reshuffle_each_epoch: bool = True num_height: int = 0 num_width: int = 0 num_frames: int = 0 @@ -1010,6 +1011,10 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: type=int, required=True, help="Number of workers for dataloader") + parser.add_argument("--reshuffle-each-epoch", + action=StoreBoolean, + default=TrainingArgs.reshuffle_each_epoch, + help="Whether to reshuffle dataset order each epoch") parser.add_argument("--num-height", type=int, required=True, diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index e8d214798..c4f16460b 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -184,7 +184,8 @@ def initialize_training_pipeline(self, training_args: TrainingArgs): text_padding_length=training_args.pipeline_config. text_encoder_configs[0].arch_config. text_len, # type: ignore[attr-defined] - seed=self.seed) + seed=self.seed, + reshuffle_each_epoch=training_args.reshuffle_each_epoch) self.noise_scheduler = noise_scheduler if self.training_args.boundary_ratio is not None: @@ -248,6 +249,8 @@ def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: if batch is None: self.current_epoch += 1 logger.info("Starting epoch %s", self.current_epoch) + # Reshuffle dataset order each epoch + self.train_dataset.sampler.set_epoch(self.current_epoch) # Reset iterator for next epoch self.train_loader_iter = iter(self.train_dataloader) # Get first batch of new epoch diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index d54dfc137..6a178d676 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -147,6 +147,8 @@ def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: if batch is None: self.current_epoch += 1 logger.info("Starting epoch %s", self.current_epoch) + # Reshuffle dataset order each epoch + self.train_dataset.sampler.set_epoch(self.current_epoch) # Reset iterator for next epoch self.train_loader_iter = iter(self.train_dataloader) # Get first batch of new epoch From d4a3349e99bf85b8dcc7e7aefb77675c5965d414 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Mon, 9 Feb 2026 07:31:09 +0000 Subject: [PATCH 012/214] actions --- .../action/000000_action.npy | Bin 2994 -> 0 bytes .../action/000001_action.npy | Bin 2994 -> 0 bytes .../action/000002_action.npy | Bin 2994 -> 0 bytes .../action/000003_action.npy | Bin 2994 -> 0 bytes .../action/000004_action.npy | Bin 2994 -> 0 bytes .../action/000005_action.npy | Bin 2994 -> 0 bytes .../action/000006_action.npy | Bin 2994 -> 0 bytes .../action/000007_action.npy | Bin 2994 -> 0 bytes .../action/000012_action.npy | Bin 2994 -> 0 bytes .../action/000013_action.npy | Bin 2994 -> 0 bytes .../action/000014_action.npy | Bin 2994 -> 0 bytes .../action/000015_action.npy | Bin 2994 -> 0 bytes .../WanGame2.1_1.3b_i2v/action/README.md | 66 -- .../WanGame2.1_1.3b_i2v/actions/A.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/A_d.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/A_dl.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/A_dr.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/A_l.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/A_r.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/A_u.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/A_ul.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/A_ur.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/D.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/D_d.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/D_dl.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/D_dr.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/D_l.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/D_r.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/D_u.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/D_ul.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/D_ur.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/README.md | 147 +++ .../WanGame2.1_1.3b_i2v/actions/S.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SA.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SA_d.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SA_dl.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SA_dr.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SA_l.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SA_r.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SA_u.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SA_ul.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SA_ur.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SD.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SD_d.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SD_dl.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SD_dr.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SD_l.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SD_r.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SD_u.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SD_ul.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/SD_ur.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/S_d.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/S_dl.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/S_dr.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/S_l.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/S_r.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/S_u.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/S_ul.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/S_ur.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/W.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WA.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WA_d.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WA_dl.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WA_dr.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WA_l.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WA_r.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WA_u.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WA_ul.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WA_ur.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WD.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WD_d.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WD_dl.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WD_dr.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WD_l.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WD_r.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WD_u.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WD_ul.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/WD_ur.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/W_d.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/W_dl.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/W_dr.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/W_l.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/W_r.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/W_u.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/W_ul.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/W_ur.npy | Bin 0 -> 2866 bytes .../camera_1_action_rand_1.npy} | Bin 2994 -> 2866 bytes .../camera_1_action_rand_1_f4.npy} | Bin 2994 -> 2866 bytes .../actions/camera_1_action_rand_2.npy | Bin 0 -> 2866 bytes .../actions/camera_1_action_rand_2_f4.npy | Bin 0 -> 2866 bytes .../camera_1_action_rand_3.npy} | Bin 2994 -> 2866 bytes .../actions/camera_1_action_rand_3_f4.npy | Bin 0 -> 2866 bytes .../actions/camera_1_action_rand_4.npy | Bin 0 -> 2866 bytes .../actions/camera_1_action_rand_4_f4.npy | Bin 0 -> 2866 bytes .../actions/camera_2_action_rand_1.npy | Bin 0 -> 2866 bytes .../actions/camera_2_action_rand_1_f4.npy | Bin 0 -> 2866 bytes .../actions/camera_2_action_rand_2.npy | Bin 0 -> 2866 bytes .../actions/camera_2_action_rand_2_f4.npy | Bin 0 -> 2866 bytes .../actions/camera_2_action_rand_3.npy | Bin 0 -> 2866 bytes .../actions/camera_2_action_rand_3_f4.npy | Bin 0 -> 2866 bytes .../actions/camera_2_action_rand_4.npy | Bin 0 -> 2866 bytes .../actions/camera_2_action_rand_4_f4.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/d.npy | Bin 0 -> 2866 bytes .../000009_action.npy => actions/dl.npy} | Bin 2994 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/dr.npy | Bin 0 -> 2866 bytes .../actions/key_1_action_rand_1.npy | Bin 0 -> 2866 bytes .../actions/key_1_action_rand_1_f4.npy | Bin 0 -> 2866 bytes .../actions/key_1_action_rand_2.npy | Bin 0 -> 2866 bytes .../actions/key_1_action_rand_2_f4.npy | Bin 0 -> 2866 bytes .../actions/key_1_action_rand_3.npy | Bin 0 -> 2866 bytes .../actions/key_1_action_rand_3_f4.npy | Bin 0 -> 2866 bytes .../actions/key_1_action_rand_4.npy | Bin 0 -> 2866 bytes .../actions/key_1_action_rand_4_f4.npy | Bin 0 -> 2866 bytes .../actions/key_2_action_rand_1.npy | Bin 0 -> 2866 bytes .../actions/key_2_action_rand_1_f4.npy | Bin 0 -> 2866 bytes .../actions/key_2_action_rand_2.npy | Bin 0 -> 2866 bytes .../actions/key_2_action_rand_2_f4.npy | Bin 0 -> 2866 bytes .../actions/key_2_action_rand_3.npy | Bin 0 -> 2866 bytes .../actions/key_2_action_rand_3_f4.npy | Bin 0 -> 2866 bytes .../actions/key_2_action_rand_4.npy | Bin 0 -> 2866 bytes .../actions/key_2_action_rand_4_f4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_1_action_rand_1.npy | Bin 0 -> 2866 bytes .../actions/key_camera_1_action_rand_1_f4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_1_action_rand_2.npy | Bin 0 -> 2866 bytes .../actions/key_camera_1_action_rand_2_f4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_1_action_rand_3.npy | Bin 0 -> 2866 bytes .../actions/key_camera_1_action_rand_3_f4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_1_action_rand_4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_1_action_rand_4_f4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_2_action_rand_1.npy | Bin 0 -> 2866 bytes .../actions/key_camera_2_action_rand_1_f4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_2_action_rand_2.npy | Bin 0 -> 2866 bytes .../actions/key_camera_2_action_rand_2_f4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_2_action_rand_3.npy | Bin 0 -> 2866 bytes .../actions/key_camera_2_action_rand_3_f4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_2_action_rand_4.npy | Bin 0 -> 2866 bytes .../actions/key_camera_2_action_rand_4_f4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_1_action_rand_1.npy | Bin 0 -> 2866 bytes .../key_camera_excl_1_action_rand_1_f4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_1_action_rand_2.npy | Bin 0 -> 2866 bytes .../key_camera_excl_1_action_rand_2_f4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_1_action_rand_3.npy | Bin 0 -> 2866 bytes .../key_camera_excl_1_action_rand_3_f4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_1_action_rand_4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_1_action_rand_4_f4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_2_action_rand_1.npy | Bin 0 -> 2866 bytes .../key_camera_excl_2_action_rand_1_f4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_2_action_rand_2.npy | Bin 0 -> 2866 bytes .../key_camera_excl_2_action_rand_2_f4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_2_action_rand_3.npy | Bin 0 -> 2866 bytes .../key_camera_excl_2_action_rand_3_f4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_2_action_rand_4.npy | Bin 0 -> 2866 bytes .../key_camera_excl_2_action_rand_4_f4.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/l.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/r.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/still.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/u.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/ul.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/actions/ur.npy | Bin 0 -> 2866 bytes .../WanGame2.1_1.3b_i2v/generate_actions.py | 193 ---- .../scripts/generate_actions.py | 278 ++++++ .../scripts/generate_validation.py | 214 ++++ .../WanGame2.1_1.3b_i2v/validation_mc.json | 924 ------------------ .../WanGame2.1_1.3b_i2v/validation_wsad.json | 324 ------ 164 files changed, 639 insertions(+), 1507 deletions(-) delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000000_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000001_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000002_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000003_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000004_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000005_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000006_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000007_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000012_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000013_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000014_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/000015_action.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/action/README.md create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_ur.npy rename examples/training/finetune/WanGame2.1_1.3b_i2v/{action/000008_action.npy => actions/camera_1_action_rand_1.npy} (66%) rename examples/training/finetune/WanGame2.1_1.3b_i2v/{action/000010_action.npy => actions/camera_1_action_rand_1_f4.npy} (66%) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2_f4.npy rename examples/training/finetune/WanGame2.1_1.3b_i2v/{action/000011_action.npy => actions/camera_1_action_rand_3.npy} (66%) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/d.npy rename examples/training/finetune/WanGame2.1_1.3b_i2v/{action/000009_action.npy => actions/dl.npy} (83%) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/still.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/ur.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/generate_actions.py create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000000_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000000_action.npy deleted file mode 100644 index 8a6ab2ab14b855f0691ea93076134c495f72926b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeHIF-yZh6i!lO#X37UWlIVrlY@v*sNi5QRl&hQ!X>#9vB`zYMF_UgrP{!4d4m5| zbBRMJZVE22TBwCFv;Vzu)Tm#Y{&VWJt9g`0sO$DI3P*JXdW=A zfa^^q@O~1uB^5Zz8qQT8-ryE2M;%5zu7|8hv8dOkCs1zMqaIfm;JWVRXB`bVV+MZt z086m=-V-i-f@lbl7U_aC>VH!@H_q-WL<6qPj=OAUTWpG8n|$;|Q>$^?+cwI4tBBM= z>d<`~(ht%P(hpq)q#wF!{LTLW^@QenO!5r>IOer_%paALjn=(duB8J1r+~0*j%u*K E06JO3F8}}l diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000001_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000001_action.npy deleted file mode 100644 index e649b6c7273d76092c69f063d53cc78ae717dfbf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeHIze~eF6uzX!igk8y%62J~Ob#MKp@M_KR0RhI376zb#3mOmi4bg|i`u|#d4m6~ z=7&NQw+=4%hIjAYJ-)|%e0Q(3Z!Rux74(Rnuul_@<6|8!rg)^|8RID@k;Ax8>+~6! zC$!2Z0f}j~H-;m9sc&YxtF1RYK#ht0C=27QuJD_nzNDZ`*AG(XK_TGy-7BPiDGQBNofuxIzKR`91g%+|bp?{8fwHosW#bm8^+sarn@P7={lFdlv F%uh&;#4i8< diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000002_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000002_action.npy deleted file mode 100644 index 3ee9a9c38c0fc4b02f535415193cac4201c448a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeH|ze~eF6vr>AF=Cw^oU&aCC6j}QP^jQwFjc|9LBb`ua$=K|OCkhY=%O}oTb|&5 ztN9@`h?|1Tz2R~1-Qx@Q<-@(wxw*K!RnQ}P!U0Wuj*ktznBbv-r!h}CiQJe6v~Hh~ zSwgFR!pNLfwK*6XOXGL3yV`ogebj{5kFs!{Tdp5-YK7S>y(gTL9K@c>sUJs4$}>NO z5`@-Fg3zO&9Vx*^&v2@E@Cw&p+RA|SgchXvoYwQ|$KVv{SEL#8no^cAl=zfIxNe`J zS&BtJC1{St+8zwerTM$qU2VNteWeM$pXBkpaQu)m?8NyjyGM+n0{EWKa7dFhV|kcC z1zc~Cz>OqoAr-jFGn}dcyuvlut~ww+u18rh$2DG?96`Bmw|ZP%faiJVA0nFMjxq?! zJ6M9vcOGyNP{bl28qsS~Vti4$SMK%;kUrO@hh4UlZ){4 DWcb7{ diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000004_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000004_action.npy deleted file mode 100644 index 9ce5230372d7f3a39ac9eb871b638708e37e2d55..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeH|ze~eF6vva)7_rU{PT7(|$>bm+6e>6vOjU4jkZ?(^L~MR=xd_1)x>Q@ZEl=>j z)%@_lh?|3h_lA4C?>)ZoK0J7(b8~TdD@%{klM&#|C&saE%qPZBH>L@p1jTMb0$j^y zXqMqh&q9>qsx}8heWCv@c9&bPMqg@z?Z$#5$#x!a;U$O!5GtWsn}*t_rF~^@KSS7O%JlH7+sQXKsdgK`Y4uRcQ9B#YZ6$QU zffMNNhuD)Y4^D}5;aTXMK$n-`)8)Y_aV|UyofGKt`X79ty3kCGl03sdo_VD@^9P&p TTIoU(l_2m}1nQN|QT13Kr_=zg diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000005_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000005_action.npy deleted file mode 100644 index 02cd7ab2e0691d85f98d3d6140435496260cf2dd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGacRJ{HmZH3|;+@f9~Tc6;6 ztNmf2h}#V=d62xkm&X_K@g=Xc?yj!yCG?D5utyV@;ZqGSXLzjPdBjpiLOWs}tVY?BdMvyJiN5UA%fgjk6x>1;*7ni3sz+mwlxM_0=Cp^TynL1AoZn`EeiZ>=zoU(Wakjdn<(2lXi4*=S99{ zLhvAX=-x!_g&u?+gdX|~2tD-K*#3WjszY-nh_ZzK+2-Y{%^wxhjod$0tOW!A$3QLF Ij8v2H4YmmYtpET3 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000007_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000007_action.npy deleted file mode 100644 index b0f8a4d3e79be73f379dc65f8aca9f50c5a87b48..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeH|u}i~16vi*9F=Cw^oU$c_lF30tC{%DTn5y95Amy4|iP$FLk_f>Tx>Q@ZEl=>j z)ui!(5jO`B?hW_defRj_ejL2Qy}7u&RmdZGGJTeK+&nhS*}&`?<~ZUhr{OT-KCA3A zI!RdRCjpIFdA2%TV{ZJeb{AW(W}9qE?1$Mj&h4QWab{1mNqSE?r#XndA!l9`CMnOn z6bca9rW1r7`gKbI4q3vfGJ+Sl2Fp=8K}%?Un#Zi_R|iM1Ue#AEq0GQ_-ShXdHjC3s zVYI%3Iap%nf#su!avuUU(5lct|BQ65ob6`_+Cm*4eoZ_1mPV?y{+nLEijYOG-l0k# z2{gG#K207-3Avz39|<&hNj^;;NC~;1N*@U{c}YG^9!Lqfp#Fa!$_XvBX_O`G$1$&# YWBy>>U#a+1sgl6o5~#jxM$5(i0K`53tpET3 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000012_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000012_action.npy deleted file mode 100644 index 0392aeeb87d199d45702744ee0fb736009a8cef4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarc diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000015_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/action/000015_action.npy deleted file mode 100644 index a8c7fba01479245739ac912e26c76249940eec91..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa[S] + Static -17. Switch [S]->[W] + Static -18. Switch [A]->[D] + Static -19. Switch [D]->[A] + Static -20. Switch [W]->[A] + Static -21. Switch [W]->[D] + Static -22. Switch [S]->[A] + Static -23. Switch [S]->[D] + Static -24. No Key + Switch [left]->[right] -25. No Key + Switch [right]->[left] -26. No Key + Switch [up]->[down] -27. No Key + Switch [down]->[up] -28. No Key + Switch [up_left]->[up_right] -29. No Key + Switch [up_right]->[up_left] -30. No Key + Switch [left]->[up] -31. No Key + Switch [right]->[down] -32. Hold [W] + Hold [left] -33. Hold [S] + Hold [left] -34. Hold [W] + Hold [right] -35. Hold [S] + Hold [right] -36. Hold [A] + Hold [up] -37. Hold [D] + Hold [up] -38. Hold [WA] + Hold [down] -39. Hold [WD] + Hold [down] -40. Hold [W] + Hold [up_left] -41. Hold [S] + Hold [up_left] -42. Hold [W] + Hold [up_right] -43. Hold [S] + Hold [up_right] -44. Hold [A] + Hold [down_left] -45. Hold [D] + Hold [down_right] -46. Hold [WA] + Hold [right] -47. Hold [WD] + Hold [left] -48. Hold [W] + Switch [left]->[right] -49. Hold [W] + Switch [right]->[left] -50. Hold [W] + Switch [up]->[down] -51. Hold [W] + Switch [down]->[up] -52. Hold [W] + Switch [left]->[up] -53. Hold [W] + Switch [right]->[up] -54. Hold [W] + Switch [left]->[down] -55. Hold [W] + Switch [right]->[down] -56. Switch [W]->[S] + Hold [up] -57. Switch [S]->[W] + Hold [up] -58. Switch [A]->[D] + Hold [up] -59. Switch [D]->[A] + Hold [up] -60. Switch [W]->[A] + Hold [up] -61. Switch [W]->[D] + Hold [up] -62. Switch [S]->[A] + Hold [up] -63. Switch [S]->[D] + Hold [up] diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy new file mode 100644 index 0000000000000000000000000000000000000000..475dd3f9265f51403e35936028508b36cc84a7db GIT binary patch literal 2866 zcmeHIO-sW-5Z$E4NcHT&Q*KG2S+W@d*6^LXrR-0SJ(je;J~qZyJcVCIQoE@$S@Fy{&5498x=LQ=QS z@ggHtKchG$Rc#H1#>)6z^w&FY?|sySIE?csEo?7H7_p;#!S66*xBzkBF%l$k##tV4 z*nrTQNf3G*wj&#G&=Z^~KD@vcSdKEFJ)woXNJ%}f&PGtawog5wEWvf%i;pTAaK;?` z@)lNLiM@ML_z7kqP?c&mDb>FyovE|?40K$NvD;1r=J9s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4FHpgqv>HZ zJs^j`XnH^njge74Oaa!QJ*>I;rNyaOt6?T!H7wQXl)=QnaQ4iZy`TUZrANbomgxYL KBuf*4gdPC9(Xw*@ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_dl.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_dl.npy new file mode 100644 index 0000000000000000000000000000000000000000..4caa89b442189f2f54702a632c759e629e5ad218 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4FHpgqv>HZ zJs^j`XnH^njge74Oaa!QJ*>I;rNyaOt6?T!H7wQXl)=QnaQ4iZy`y+E3}~GOKuNMR I5lH9(01(5YyZ`_I literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..9f5891bbe0adf0f0d64bcb0ddd742d5555c5645f GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4FHpgqv>HZ zJs^j`XnH^njge74Oaa!QJ*>I;rNyaOt6?T!H7wQXl)=QnaQ4iZy+CX`N{@yEEzs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4FHpgqv>HZ zJs^j`XnH^njge74Oaa!QJ*>I;rNyaOt6?T!H7wQXl)(fEnzLul>;CteYRXPAA K$s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4FHpgqv>HZ zJs^j`XnH^njge74Oaa!QJ*>I;rNyaOt6?T!H7wQXl)(fEnzLul*n(K2^k_KHDjk55 KWN9Lh&;tOJY>V>% literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_u.npy new file mode 100644 index 0000000000000000000000000000000000000000..f898c097da070b141bc59b4d655077988ae400f9 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4FHpgqv>HZ zJs^j`XnH^njge74Oaa!QJ*>I;rNyaOt6?T!H7wQXl)=QnaQ4g@TTlRv(xc%(%X9!r KlBJ12LJt7Hfs6A1 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..aa71bbe207ad7a68dbbb93fa5e894335c73bea1f GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4FHpgqv>HZ zJs^j`XnH^njge74Oaa!QJ*>I;rNyaOt6?T!H7wQXl)=QnaQ4g@TOi&$N{@yEEzs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4FHpgqv>HZ zJs^j`XnH^njge74Oaa!QJ*>I;rNyaOt6?T!H7wQXl)=QnaQ4g@+fh6k2DDBCpd?wE H2qg3XvLjZ! literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy new file mode 100644 index 0000000000000000000000000000000000000000..90371fcb0c6ddd8f3fc2dab25075d37d15dbeb8d GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MTs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MTs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MTs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MTs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MTs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MTs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MTs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MTs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MTuT FJpjx|R=fZJ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md new file mode 100644 index 000000000..96f37334e --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md @@ -0,0 +1,147 @@ +Total Files: 145 + +00: W +01: S +02: A +03: D +04: WA +05: WD +06: SA +07: SD +08: u +09: d +10: l +11: r +12: ur +13: ul +14: dr +15: dl +16: still +17: W_u +18: W_d +19: W_l +20: W_r +21: W_ur +22: W_ul +23: W_dr +24: W_dl +25: S_u +26: S_d +27: S_l +28: S_r +29: S_ur +30: S_ul +31: S_dr +32: S_dl +33: A_u +34: A_d +35: A_l +36: A_r +37: A_ur +38: A_ul +39: A_dr +40: A_dl +41: D_u +42: D_d +43: D_l +44: D_r +45: D_ur +46: D_ul +47: D_dr +48: D_dl +49: WA_u +50: WA_d +51: WA_l +52: WA_r +53: WA_ur +54: WA_ul +55: WA_dr +56: WA_dl +57: WD_u +58: WD_d +59: WD_l +60: WD_r +61: WD_ur +62: WD_ul +63: WD_dr +64: WD_dl +65: SA_u +66: SA_d +67: SA_l +68: SA_r +69: SA_ur +70: SA_ul +71: SA_dr +72: SA_dl +73: SD_u +74: SD_d +75: SD_l +76: SD_r +77: SD_ur +78: SD_ul +79: SD_dr +80: SD_dl +81: key_1_action_rand_1_f4 +82: key_1_action_rand_2_f4 +83: key_1_action_rand_3_f4 +84: key_1_action_rand_4_f4 +85: key_1_action_rand_1 +86: key_1_action_rand_2 +87: key_1_action_rand_3 +88: key_1_action_rand_4 +89: camera_1_action_rand_1_f4 +90: camera_1_action_rand_2_f4 +91: camera_1_action_rand_3_f4 +92: camera_1_action_rand_4_f4 +93: camera_1_action_rand_1 +94: camera_1_action_rand_2 +95: camera_1_action_rand_3 +96: camera_1_action_rand_4 +97: key_camera_1_action_rand_1_f4 +98: key_camera_1_action_rand_2_f4 +99: key_camera_1_action_rand_3_f4 +100: key_camera_1_action_rand_4_f4 +101: key_camera_1_action_rand_1 +102: key_camera_1_action_rand_2 +103: key_camera_1_action_rand_3 +104: key_camera_1_action_rand_4 +105: key_2_action_rand_1_f4 +106: key_2_action_rand_2_f4 +107: key_2_action_rand_3_f4 +108: key_2_action_rand_4_f4 +109: key_2_action_rand_1 +110: key_2_action_rand_2 +111: key_2_action_rand_3 +112: key_2_action_rand_4 +113: camera_2_action_rand_1_f4 +114: camera_2_action_rand_2_f4 +115: camera_2_action_rand_3_f4 +116: camera_2_action_rand_4_f4 +117: camera_2_action_rand_1 +118: camera_2_action_rand_2 +119: camera_2_action_rand_3 +120: camera_2_action_rand_4 +121: key_camera_2_action_rand_1_f4 +122: key_camera_2_action_rand_2_f4 +123: key_camera_2_action_rand_3_f4 +124: key_camera_2_action_rand_4_f4 +125: key_camera_2_action_rand_1 +126: key_camera_2_action_rand_2 +127: key_camera_2_action_rand_3 +128: key_camera_2_action_rand_4 +129: key_camera_excl_1_action_rand_1_f4 +130: key_camera_excl_1_action_rand_2_f4 +131: key_camera_excl_1_action_rand_3_f4 +132: key_camera_excl_1_action_rand_4_f4 +133: key_camera_excl_1_action_rand_1 +134: key_camera_excl_1_action_rand_2 +135: key_camera_excl_1_action_rand_3 +136: key_camera_excl_1_action_rand_4 +137: key_camera_excl_2_action_rand_1_f4 +138: key_camera_excl_2_action_rand_2_f4 +139: key_camera_excl_2_action_rand_3_f4 +140: key_camera_excl_2_action_rand_4_f4 +141: key_camera_excl_2_action_rand_1 +142: key_camera_excl_2_action_rand_2 +143: key_camera_excl_2_action_rand_3 +144: key_camera_excl_2_action_rand_4 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy new file mode 100644 index 0000000000000000000000000000000000000000..6791ea46db8598a44f695952505ac05a39f618f9 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)rNEj>&D)}TGCx%s8VsaUIFCSWxz)#;SMgc(SqBBLQdix2=M$W#rrTWPqJk z;ucn5@V$GIyAftSP?;(9}qg&%|;gdesQ_-}rIYJ+kmh_ZzIxQ69w4IdQ~Dt{MAAVPt^P@uMChO1s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(J&htZ?rkdPP+htY6=q&@{Ls?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(J&htZ?rkdPP+htY6=q&@{Ls?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(J&htZ?rkdPP+htY6=q&@{LOZftKk2 Llq5?NfrK6ah1RyO literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_l.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_l.npy new file mode 100644 index 0000000000000000000000000000000000000000..3744d98a7375381ffce3b5c8a286a717664711b6 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(J&htZ?rkdPP+htY6=q&@{Ls?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(J&htZ?rkdPP+htY6=q&@{LNQF=5SXq65? LNwPE%Naz6oYtO}I literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_u.npy new file mode 100644 index 0000000000000000000000000000000000000000..c748520b412c1ef6d2f9842534a8eb19546f9e32 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(J&htZ?rkdPP+htY6=q&@{Ls?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(J&htZ?rkdPP+htY6=q&@{LOZftKk2 Llq5?NfrK6a>&LdQ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_ur.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SA_ur.npy new file mode 100644 index 0000000000000000000000000000000000000000..bf5ea12b6ce90b062b0a2d9213d4c2b8eeee8e98 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(J&htZ?rkdPP+htY6=q&@{LF8}}l literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD.npy new file mode 100644 index 0000000000000000000000000000000000000000..9db373e873a498fb20dc17beec1c8f02f9990e5f GIT binary patch literal 2866 zcmeH|u}i~16vi*9F;bl!oU)~blF6ZnP^jQwXexq(gM@2x<-|5AmqZA*(4}hOwm$y` z|4dC99~g0yZuf@wUA{a%Ebw-6IuE3TTu^uKL$N`nF^#i?{kewq YVhx|H`&`GXC{+Y@OQ8HEGg>s~3zJg%od5s; literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_d.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_d.npy new file mode 100644 index 0000000000000000000000000000000000000000..79a05c157ed3ab75e65b899fd3d4e01c0464f074 GIT binary patch literal 2866 zcmeH|F-yZh6vr>AF;bl!oU)~b(#fHSP^jQwXexq(gOqD><-|5AmqZA*(4}hOwmv_D zpTciYKa(bn4~)1;w|m3=@Ba7rll(Y%EBCs4d83gB@@V@k@`Qb2+0&lgu{cUzpBo6Uf-%@Q}QrK#*?(!_d>y%<75=yQ6Xpw@}Mu6 z7Y0!*5-)}oNMq9p(hU5fWd#mdz?n9HXSjlzqt$su8h)HkSl+MqTCjX6t}0TSg6p~$ zpIL1dryav!c?&bB$-R4)4nit?;5s+*P;P!lI$dY?3HZL$hsQtDPJgA5Jgw6*i@zdd z-Y#~i%0~iaE{ac?2U0>VsLDqIWnPL;nFmrrE~v^!0%cx`Pnic&LN2J=_aU30(in$H j#Qq$^dNziSmL0AW^08jO=O3h0aoFZN$e(0Fv!(e3ty$~l literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_dl.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_dl.npy new file mode 100644 index 0000000000000000000000000000000000000000..aab59c573296c409140e6e6970a66b7529b5cd31 GIT binary patch literal 2866 zcmeH|!AiqG5QaCYF;YEy@RVCxC_Q;7A`~ikFf{cUzo^c4Uf-%@6Y?-f#*?(!_d>y%<75=yQ6Xpw@}Mu6 z7Y0!*5-)}oNMln8(hU5fWCadcz?n9HXSjlzqt$su8h)HkSYEI9TCjX6o+?tCg6p~$ zpIK}+ryav!c?&bB$-R4)4nit?;5s*QQ*M4+I$dY?3HZL$hsQtNPJg+P+^y3xi(e5k zZx<)bN=E`^E{ac?2U0>Vn3awM%DfbxG7qGLTrevg36yy$K4l(A3Ate2z7E+0mBu(s hBKGGP*0V8uwCr%5kdO8HT`k36+dYuCWJ0s8`2_+?-7f$D literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..90c62dd5244b12e68db87ba4c66c99d8f16e373a GIT binary patch literal 2866 zcmeH|PfNov7{=4iI+>n5c#kGd+(SsF`*k{&}Y0dgp%d|!jOBfCZ z5erCepW$&#gdbBpCE{#1Tjt#SUo{q6uWt=h6yFci$ux5Ye#D3~Nyo`8W(;S*_Xdpk zQ5YvI^%GbDH;PW+^)M(~R^XyXIMs&m43}WLT9eke5hU4^Zb$c%FCu zDXQ4qP6ET_4a~vjJ9i`-MwkUab!z0HwEi9G_T23!pu1cj9sW!^{*{LEv~Igz{uM%t zPWdHO`I11Di{ew|At^~NQk5?WRCy^rRUVR(C;H}8X(pxl^5 kX-xhc!@3y52g@$i5&BrK-^DsnsyJ-&9pq0k#bR^50H6D|uK)l5 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_l.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_l.npy new file mode 100644 index 0000000000000000000000000000000000000000..96f19147518f52171a1f524377adb899deb2c4b4 GIT binary patch literal 2866 zcmeH|F-yZh6vr>AF;bl!oU)~b(#fHSP^jQwXexq(gOqD><-|5AmqZA*(4}hOwmv_D zpTciYKa(bn4~)1;w|m3=@Ba7rll(Y%EBCs4d83gB@@V@k@`Qb2+0&lgu{cUzpBo6Uf-%@Q}QrK#*?(!_d>y%<75=yQ6Xpw@}Mu6 z7Y0!*5-)}oNMq9p(hU5fWd#mdz?n9HXSjlzqt$su8h)HkSl+MqTCjX6t}0TSg6p~$ zpIL1dryav!c?&bB$-R4)4nit?;5s+*P;P!lI$dY?3HZL$hsQtDPJgA5Jgw6*i@zdd z-Y#~i%0~iaE{ac?2U0>VsLDqIWnPL;nFmrrE~v^!0%cx`Pnic&LN2J=_aU30(in$H j#Qq$^dNziSmL0AapYpL@zvnxpio>?vLH;BYnk~&Q=r!x+ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_r.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_r.npy new file mode 100644 index 0000000000000000000000000000000000000000..6841f0a0bd053273976bea51b1ab92275188a315 GIT binary patch literal 2866 zcmeH|F-yZh6vr>Au~MBKoU$c_(#fHSQmEiyXexq(gOqD><-|5AmqZA*(4}hOwmv_D zU#5O0O&T8%2w>%_FCJ*#0_!%)P62K`X@ zEVnP{C}No(aXMky*{(Otnfbq}&3E43Yh+XMFi6IewAuGU!J6Y_6yH-JXbSS6FPIkw zQ7jTKh80L-(+RQ~_(jVK9I}9OZ2&KD4Ys4zc|{t2oK9HYulHK8d@ZgjQk#P7x|g3> zZ5F2;!(e#_GqB~}14{=X6+UpC8+j`;}D1j<|#pE3`mgj`USj|9rR6rVB=q=Z~hm5&6Au~MBKoU$c_(#fHSQmEiyXexq(gOqD><-|5AmqZA*(4}hOwmv_D zU#5O0O&T8%2w>%_FCJ*#0_!%)P62K`X@ zEVnP{C}No(aXMky*{(Otnfbq}&3E43Yh+XMFi6IewAuGU!J6Y_6yH-JXbSS6FPIkw zQ7jTKh80L-(+RQ~_(jVK9I}9OZ2&KD4Ys4zc|{t2oK9HYulHK8d@ZgjQk#P7x|g3> zZ5F2;!(e#_GqB~}14{=X6+UpC8+j`;}D1j<|#pE3`mgj`USj|9rR6rVB=q=Z~hm5&6n5c#kGd+(SsF`*k{&}Y0dgp%d|!jOBfCZ z5erCepW$&#gdbBpCE{#1Tjt#SUo{q6uWt=h6yFci$ux5Ye#D3~Nyo`8W(;S*_Xdpk zQ5YvI^%GbDH;PW+^)M(~R^XyXIMs&m43}WLT9eke5hU4^Zb$c%FCu zDXQ4qP6ET_4a~vjJ9i`-MwkUab!z0HwEi9G_T23!pu1cj9sW!^{*{LEv~Igz{uM%t zPWdHO`I11Di{ew|At^~NQk5?WRCy^rRUVR(C;H}8X(pxl^5 kX-xhc!@3y52g@$i5&BrKPsDmxsyJ-&9pq0k#bR^500sNDuK)l5 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_ur.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/SD_ur.npy new file mode 100644 index 0000000000000000000000000000000000000000..c047c6226c4b9f90d85e73a0a72055ce4c68f3f5 GIT binary patch literal 2866 zcmeH|!AiqG5QaCYu~I#I@RVCpC_Q;7q7*84Ff(* zSuS7DQN%Jm;&j5YyIpUXGxL8@oA126*T^R1VUUa`X|wNzf;GpdL36!}gK4l(A3AtcaIua=JQhdrhkP>pitaKz$=B4$jqpOfF literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_d.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_d.npy new file mode 100644 index 0000000000000000000000000000000000000000..d281c8f4219676daf3b3b5a2f4590a16e3571cf5 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)rNEj>&D)}TGCx%s8VsaUIFCSWxz)#;SM#K3U&%$dEQ;2EVy!-1CR0F)$4 I6M=*t0HLh1a{vGU literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_dl.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_dl.npy new file mode 100644 index 0000000000000000000000000000000000000000..0633f767d17f5713bfa5fe81ef089e3317cd31dc GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)rNEj>&D)}TGCx%s8VsaUIFCSWxz)#;SM#K3U&%$dETcr*-Xod!TjvNRD$ G=m7xskE6T* literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..774025749a05f8575b2d458156795f1b8a901051 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)rNEj>&D)}TGCx%s8VsaUIFCSWxz)#;SM#K3U&%$dDFY&%Mih663r0Vqk9 ICISgP0F+LBJOBUy literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_l.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_l.npy new file mode 100644 index 0000000000000000000000000000000000000000..051ccc73f879acdeeadbf427974aac1e15439d2e GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)rNEj>&D)}TGCx%s8VsaUIFCSWxz)#;SM1PPe4XU^;eu}109aG+H>042%N IL?EFD0NjYOa{vGU literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_r.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_r.npy new file mode 100644 index 0000000000000000000000000000000000000000..e29b95433fc2f954cdaddb68aa2c060102494040 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)rNEj>&D)}TGCx%s8VsaUIFCSWxz)#;SM1PPe4XU^DySfli4IM6B`fRbcs IB9PDn0D4D@^8f$< literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_u.npy new file mode 100644 index 0000000000000000000000000000000000000000..876708d0e7c093d5743a03b4c2c5a27df8f97f21 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)rNEj>&D)}TGCx%s8VsaUIFCSWxz)#;SM#K3U&%o$ry@Ql)<;XuoD07{aj Ii9kXR0Hs`u^8f$< literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..912e6c218dd7e81f0be5cd4d8c862cbada2a18f9 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)rNEj>&D)}TGCx%s8VsaUIFCSWxz)#;SM#K3U&%o$rC-aATs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)rNEj>&D)}TGCx%s8VsaUIFCSWxz)#;SM#K3U&%o*EJJQ@bHP6MDMS(*qW F^Z=LsR=fZJ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy new file mode 100644 index 0000000000000000000000000000000000000000..8b72f483efc7553af91190edca116741781ce24b GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m#?0W~Tz8UnNk0Z@`GO#~8p0BwN5 ADgXcg literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA.npy new file mode 100644 index 0000000000000000000000000000000000000000..8e56ec665a0f9d9d1965926ed1863db949f68703 GIT binary patch literal 2866 zcmeH|O-sW-5QdY~7^$8;c*-p)l$<;i5egMN7@CUU!GnZNvSr2QgH0j?Tj-@~U~e6N zgFjRA;eruwB6!$2>@e>RFU-ThY@O@r<&7*oNRLKxkI@&1 U6QXp75S1YCR|M*n%u)4NA72Riod5s; literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_d.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_d.npy new file mode 100644 index 0000000000000000000000000000000000000000..7da3fdccef32fc4ad6b7224db38eb4aac683afb9 GIT binary patch literal 2866 zcmeH|!AiqG5QaCYF;YEy@RVCxC^>m3A`~ikFf)K6dy+$cJMn_*D4tieS~IMIgi2p7579SP~8u;CsiJjlH?+@(j|c^FU6s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@beuL2v-l+I!IKV;xHGPbx6V&j4 zg%T`WVDzXsECf))YczaN!vhvduyBFVqvEg-Kn<_K5k6CZHE0iOZhmQTD%NV430MtF dbvk7*F)*Axb7t=-9t{IprvXrsEKLLwdH~Jx-7f$D literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..2307c8c64d7c796249474ec8172471b4742fb8f5 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@beuL2v-l+I!IKV;xHGPbx6V&j4 zg%T`WVDzXsECf))YczaN!vhvduyBFVqvEg-Kn<_K5k6CZHE0iOZhmQTD%NV430MtF gbvk7*F)*Axb7n6P+m6zs;XuoD07{aji9kXR0Bfs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@beuL2v-l+I!IKV;xHGPbx6V&j4 zg%T`WVDzXsECf))YczaN!vhvduyBFVqvEg-Kn<_K5k6CZHE0iOZhmQTD%NV430MtF fbvk7*K^%Yf%$dC))+jw14zx-Kpd?wE2qg3Xv+L{T literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_r.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_r.npy new file mode 100644 index 0000000000000000000000000000000000000000..8d94d59d623318c88fc8eb4205bffe1050a9cf4d GIT binary patch literal 2866 zcmeH|&r8EF6vx}nTA7|bc->2D?lxSHdSp=p9kJs~K@<=Jkv&AIuzXfC#1UYp7$*nXG=ao!nu5yhP#o2IvjQj`PR8&T{< zVUp6!OJM~}-&6uKLcc0mfuk(pR2{<;T!QVWEz)4RpXM><_1drt#dCGlVCoE9*FAqP zW3xED6vo93%)w?mcQ_wMi26V@qH|MXd|EmKXZsPzF4HE5U)_$sxhdSO-!=HB7QO0k zB3CD=OAdlSy+351)OiR>f{Vyf7X<3OB%eADK}m2CS?Yp7o!9^11LXu|dJts^{&5Uz eNmi literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_u.npy new file mode 100644 index 0000000000000000000000000000000000000000..571f4080c5d2aa52f5481bc2a178bc391fc2dcd8 GIT binary patch literal 2866 zcmeH|!AiqG5QaCYF;YEy@RVCxC_Q;7q7*84FfUR9i4GFhi#}q>?y5(## zmzB~L2T7p1A8JpkJR~K_MP{W-0##m$PnCzHB)Q0}bV;Df>woZp{DN|05~VTu@eJ$v d89rF`sE*LvW^*D&P+4*K%RLY)nd1Cue*k>H#by8i literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..43ef5c3f78d6b953d28f05b64609d550774a3ae6 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@beuL2v-l+I!IKV;xHGPbx6V&j4 zg%T`WVDzXsECf))YczaN!vhvduyBFVqvEg-Kn<_K5k6CZHE0iOZhmQTD%NV430MtF gbvk7*F)*AxbH)~k_m0w|;XuoD07{aji9kXR0MVkhuK)l5 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_ur.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA_ur.npy new file mode 100644 index 0000000000000000000000000000000000000000..58902f71e5fd42d5a58133b666fdc0450e802b1a GIT binary patch literal 2866 zcmeH|&q~8U5XLvDF;YEy@RVCpC_Q;7q7*84Ff!8;W*!;lY-qL(a}u(I;b0Up zkCgHmp2nomV~QhEyscK-m>a)~=3?vRwTU*t_k+}rvd+j28R_`xG`Yo$;SBiRh!Hmo z;)JDc0xRI!rV_XwcvZ;?Y_x<^WeiVn36`z2XoG8Bl0~GfSBG85pR2D1S7zWi&iQ*0 zo6GJcFwSpa4i?|JBiT5_%mb=Ytu&?jr=>lxw;zG-a&>a})$RD38!FxUUA=5-(W`nB zxq3-mau5XS^C9P?&O=ZVTtt?-AW-Ke`P6v`N`i~XQWpg3y#5CtC^jh9{4kBlk84;h b*6=~zrz%2k>-C8|lEGikKsk~rF0Rf8Z()!x literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD.npy new file mode 100644 index 0000000000000000000000000000000000000000..49bd1cfa4737bf939a0fc08272f5a94a7532578e GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8ns?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8L6XftKk2lq5?NfrK6amgL1| literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WD_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..a887f33d4c550226ad7bd4802a3c90e54795fe44 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m!L3}?@r*$WDyQF=5SXqgT`NwPE% HNaz6ohljFr literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_dl.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_dl.npy new file mode 100644 index 0000000000000000000000000000000000000000..5dfc422fe5cf2b184ebdbff2f29b1f4201d9f51e GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m!L3}?@r**l6y!+_Rl0F)$46M=*t E0NiS$yZ`_I literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..3fc9be95ccb1b7803be3e087fdae2f13ae64e802 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m!L3}?@r*$c$Bqx5Jv&@vr>l4NNj HkkA7FdMSN8 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_l.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_l.npy new file mode 100644 index 0000000000000000000000000000000000000000..ae390202e5f0e20c16853f48e72f0cf65a9d4efe GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m#)z&U&7%w7;{lpYNSTBQR}k}ORG G5_$l_W3qDq literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_r.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_r.npy new file mode 100644 index 0000000000000000000000000000000000000000..727119295a5ca5350114c1bc2981e706cfa00561 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m#)z&U&7j4g;YN{@yEts?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m!L3}?@ru>}RuC_Neuv`hz}Bw3mW GB=i7_H;eNC literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..932bc35da40560a5fd0e5ebe7921f0de5c48d5b7 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m!L3}?@ru?6D2qx5Jv&@vr>l4NNj HkkA7F;2M2A literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_ur.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W_ur.npy new file mode 100644 index 0000000000000000000000000000000000000000..0c7e8b4f8b83b5702401bea1b6de6ab1f56f95fe GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m!L3}?@ru^q*uVLs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk_IjH8CCvuDoi1(`!b+7@UwE%9Ec`Lv>G>2`XUjW>Ql LNwPE%Naz6on_Rr6 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaS5Z)yB zpQ3M2pGh;x9*QJED0s@i>__H*J_55eZhBX@5_u#~noWI+X(y^S>uXI_8+j~XraSPM zP0@T|j(r;2eaD>A_`BX{s&jQasXgy}eAY-J{Ll?2)2KDDJVslSa2(v5jF}Peg8`$K z=lTH)tpJw5l_U~))wR=*CFtY@&ZQx|!8O!%so_+(Vh7QbV!YgM!{R-4Rk$<*!!Rzt zV@V$UEP&zS4(3qjdk-`kdM2~MksSq#I#p3p@9Dd*;OujGbi7&Z^j9^4)w=C!x>fm> z43rF%43rF%43rG~jRA-+C|4$4=+n$IEXQZ~Xwh|KLcZ4P)jw%R^eIC0<>^ZJG7*Wv z`gtgNV{V>ygs%wEw~ekuekO|9J2{xU8^)vH{CQa9OOn(4yxzET=FA)R!WjG|Los?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk^Dac9q*u?4Z{K<|axGvsJs*z5&`4-IKrnoVoB!{QTX NIszrh(nKJk2LM}<&qx3O literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..e8b66d94fdb8849ee834db34d087414c62a707cf GIT binary patch literal 2866 zcmeHIO-sW-5KU5JrF!<@DYvvxdh$?25h{2vG!?;v2PtW?vSOQ*O(Fza@Zv{dZykSw zzoP%3{wCdNi->g7iq{>O9kOrs?b}CS#_nbF{7TpEwFf&OvCr%S%O14ss%76qEMZ>Q zj#xl&`_${kMC`}Z>l5)^u2ij|^}i@RZoIvhv{~@oFzxlTTHB8psrAxsa_ceXWx#jZ zjQCL)CoJ_77y~zFoxqD>FmD-yqdmci-hmgmfU=`kXn~tSlJyDp8?8EwUguDO>jQ9I z_v}-|Y;leg=!~vm2xY!?N3u@ju>hz+O(dno1*Ow;HlKlRbK_?JC)?pKHVxSt_2T?h zYE=#>2b2TK0p);lU_A#wOi*t2qBJH;$FL#B@ZP9F4Nd!)PLI~iiRjBqM&WbZ%$;iVxRn#Lyp_mo2hL|hE%6h2Git3#;nOI qoC%W^I1}Q|o;kCZ0SIVAPnP3Mo1Ds+0JKhqYf8zKq|!tnp$7nxUOs04 delta 95 zcmdlawn=^#Mb4Efs=I^b(m(bZ%$-1VV_*Um^OJTha7L9H&fe`45?114kiYM Z$pKt3lN~rc@L&_JDJ4^qN)v&E9spgp9OVE2 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..ac1e637b1fff71b1779413b10a800498be5bd179 GIT binary patch literal 2866 zcmeHI%}T>S5KdBKrF!<@DYvvxdh$?25h{2vG!?;v2PtW?WyLlrn?wk<;KiT7-a0;m zPtiB1&!oFa4n^`)@VWy#vzec7WHqpA}`GsN?iK^NkWG*qMV98A-^ z`eBkN<2-;-d=Cp~(!EC%jXaAufEBEAP+ZR-jlQw_26&$;8pem z3$@tl71+FYDSlObQQu8Jzn2N`!Q&;ia7I3VHe->k#IC64xMgRR)#uu>ck>wk JB}0p;&M(QCn;-xH literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..148649e834016592a0b9f4efaa6a7ce1b4da2b69 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk_IjH8CCvuDoO(ircBnold5rtSxYG0^>cX+?v~re*jL NN+X~oS(*qW^Z?FDwjKZg literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..880cd20c25ab906a2d01ee605a6f50d8b67a0147 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR*aGohLNv%?kQ!RiP&a_wffYmT-HVF`>BA)l z5(fc%?#CxbNH3A{1j88Q29Q6s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk^Dac9q**$ZOPfwqO(GfG3kd&GnTC`pzk0tr0;BSl#N literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..2e969a6e53156b0023a5b245a0f0750c01d262f6 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR>;)N01Z@jdM;1-U;Y8Sv*PTR|kJsIV^b;eG e&ksO3y!I0zPtgCA*bnj_E%k$vWN9Lh&;tOTC^+B% literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..438ca95800418b5b534b64335c66d1a0409d83d9 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR*n$kDHBFquDN_aX!(OOgs7}+;Y?!as?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR*n$itL=#d&NI#|Wggj2je5m{P(wN2@miYVt ebU$9V;nPcup9twE#tdS@7?(YuBw3mWB=i7(r773| literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..a0e013a697b636ebe87e51c6d0c6d748825da2fd GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk_IjH8ySvuDoi1(`)d+7@a))oB{K85}0GayPxqB@{lO KBw3mWB=i83Q9ech literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..18327d33763b4b5d23ce2860cb7a69a133d150c1 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk^Dac9q*u?4XR(S$4q>fZ}B2ZJWYY)bXxb00plak-fk paj05C;fN6~$aa9(_y9gX;FANH36v+~HbU|gxgC@wOA~>F9sv2!I$!_* literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..fefa780ae46e9d39a55f129cea70be06961e5151 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR>;)N02ig{D&yb|)6h<_Q3t;@xG~5Y=4=72N ICISgP07MXH00000 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_4_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..7c0649a141522f8f0c5e1032a66f280c9a735a75 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk^Dac9q**$ZM3qVZV{)N2bhgDe`K*_4s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX u?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR>;)M*N{@yEEzs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX s?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR>>b6UVLs?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarJu diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..35c6472fe9d7d2fcf8b568d8c43d8343d166f351 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX v?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR>;+=mQF=5SXqgT`NwPE%Naz6oyz4{B literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy new file mode 100644 index 0000000000000000000000000000000000000000..e78573fe0437edb158c8e7bd10cca3cf73b1d8f7 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4F1r6G{MkuFX<|4#wdpgpX)`K86FSgT6F1l VO!SSa84Ur-LjaT{OA~>F9sv0o${qj! literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..5aca2b834c519a4f1db86118da5396a644024a2b GIT binary patch literal 2866 zcmeHIO-sW-5KU5JrF!<@DYvvxdh$?2C{*xZFcrar2PtW?C1RVDO(Fza=*5r1-a7sU zf2L+z2g4HjVa-A49`^0Lym>o28`!CNb9r?uOOMi%?vTKy`iZ6wTY6Q~yFLx6<+Xk4 z5Kf<4y?`Wq;93Kc#N|p=8)=)3(s<|nqa@7+JM^OdAg;A-pORWX>V@|frB)2=piPPG zdqGGeJA^4Pb*2PX^qje53WoFyXL1K#;Tp<@TyYCbb;5W+xL#@1Ve&TrRABNDOw+vh zOvY?78XLJHi-o5swAF;bl!oU)~b(#fHSP^jQwXexq(gM@2x<-|5AmqZA*(8V8t+xq+) z{4+Jzc-Y2*gSg!r?%n&|;|urU!CSi5{mUB-J)lQBAhFNv6U&|r?3QJZB9<^5dJzkV zkk9crCWRhTJS9clXtu1m^|z=mcHZ9WXchc0OefQ7;3un6Rl4LL^S1dBZ%Qv2Y~vs_rGGQ1vXHgR)mi&p khhl?rV-lq?*}8`HVhtbVJ*rDiB_KvJ#l_M71XjQ3wEzGB literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..f418f818e5ca75b56b0e002f4d241643247326af GIT binary patch literal 2866 zcmeH|KTE?v7{=4oTB*(sPTkT%>EuvEC{%DTv=za@K}wojiP$FPk_g2Xy7;GXTVFqe zA5(L!4}_pt6N?DFA-}xOz0Y%ZA#n57W%K+hqueR?n#DYmYez+G+}0{Zt>=qP)8(u>2>Skw!MQOavDe|u zbREy+CHYi-%tKsXM!-36;doEgN-RgIi>)c20)c`Hx7wsc%1$y(`q-VKnVHmZr$GMlCN7 zHLWkt1}>}ruj3q&@v~trt)ABZ-99kCE@J;6)(WBmVykk$2(fr@kkqaUlHJC() z7fg~NJ@A9biSnztG&;N>)@)#E&4eW4aRd;)uj);VA3r3!Du!z z!DtV(B>hoOJQFpDbx?zUr&D-?irl~rJof1|tcKU{!K`7c*y4tpFRyzKX literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..14cb0d35efa8a9e12aeb03ca4199b368b9efb590 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x26{Na_%Iq4W-vaC z9uCd(2v8>kKuNMR5lH9(0Ho^59smFU literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..dbbb6fe2a70d8994ed569ea9b42a37085d2916ef GIT binary patch literal 2866 zcmeHIO>4qH5KUrJi|x6Go^p$Vcdvp35 z`ZL?C>0k(vN;L(+J?z_gJ9+bF2<%M1ZC&4q!jtf#rDG}-(3ED7_)j-6mI3~L?JrKC0p`u>AKsSyG@>QG|3 zj_1?B^kD`}o@W9pI#!%A15J2=3$Y7ta06vctk?x6TYfksJYQ`wTnNJT1(r11^bXmz0;)Hc}h*-YR!5v?ke6V zjbyYq=fx*=^EmSklDdnIHp^$#;M_LwIFl@|>&`C^P4xR8i2f0=CdP<=QMn3a3my}r}P6M43M7J0%xv+QZlZdmp(6tSQ|KNLPw z_9Y!fEb}8yCoIcr^@cUG{ub5w-up+DtV13L$#{}B`(7wmbDWIg2Py856n@Ee0ISv&RArC oIqL7~A={wR7>7y3wyt44Tf-;I4%e}LsU?B`C7^!Egl0$c3)yDnxc~qF literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_4_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..6e73c3a082fff2591c81ba225e334f38d253b9d5 GIT binary patch literal 2866 zcmeHIO-sW-6iiZMrF!<@skgLHdh$?2C{*xZFcrar2PtW?C1RVDO(Fza=*5r1-unCv z{!GobK1i29)uck9d)V1GnVq+p1a@ZJTwdME(xddG*~GV~c2d?xZLL<;dL9j^>2^G7 z6HcF-eV@d<@0ddp*Oh9mJT7M&rOD3wM@gCwcIbwKVN~x}9wqfb*bnYaO3et^L5C8{ zbNzsZRsb_#>Rbt|=-La(40P!k&g3q{Y6sDfaJ|xQ!1QhLsKDeA7>052 z8ISp-Hv{NS?_dlSw)a4yu4hsk9K}(&sZ;#6)Gzhj7jX8O(mPJNoeFLecWX6@i?@=b zNfTGg_#b>)56*1@znoV7)w*-Q6Q^a(pXJ44pLJng=S=fS<2EUqJkOfUBDqd}*SvYr z4{Jp1jWyybKAFo;_n;1ORS!`o>aj-rjUKg#qDNMq`-vW653CXYqEFkyxvV_yM-#N!{q%lQ-9a0aM z@+F!@xX>el=D0ZPy}r6s{}Os*z@DS=myQ)0 zjKRt7VF@}rc*Lm_BI*H=h+LWy<=fI6oBJ<74w*DPZMvQRa1%s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuLg!}u*YsMwZuWDQ_Cz|=FrOy>>))x zQC$8T3gSb-jo8u&t=&z%)G;t7(KCE#VHYjT#1#|tvdN(C{%DTG!?hH(36T#6FqN75Xd`>#L_nKV6JS3CcuCX{NUfl{BU_N(D~*2KKt z&WCep=hD2KCgQCA&Z?otBTisD0sgOQKASj{-A%pS>QHP@Ci`I$;np=Q6>In;8xd*y R1@bN@@LvSVCz+ta8Q=N1uR;I- literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_2_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..c4a7bb1ab9d70f8b9fa62cf5b2d936448d4f97aa GIT binary patch literal 2866 zcmeH|O-sW-5QdY~SgD>pc*-p;l$<;i5egMN7@CUU!Gn}dvSr2QgH0j?Tj<4)!rnUm z27jhz8w;iMW+xY&!(=lVo_FUZFbn6le|@JaPs+3D)5PQEnPE-_X4^1F5l=Y@hY|Ow zEMJmwLW_FBNKA|OW~*&Xjo(FMw)gSbP?jN%!z_q%Yv@IsT0u5W9|-3p2XQpy)QiF- z<(Zel0))Qo1W^zDvSk6b@&XsC3vX}(O)ufPt4HacCWoo~<-N$Yj$a;RCioNY5> z*H?N~>8$j2^{nFFRcP$Wk8K>x`(v`#XTjFN=F3^IG_bhq{$rmpXTW4*`%)Dj%Ofmq l=?fGaROmsJC3Nc=)`~TJGVd|1DjWZG{<;J5m&{0!tZ#lGvOWL+ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..e865d985550df715aed112a2ae37c8637543270b GIT binary patch literal 2866 zcmeH|ze~eF6vva)7^%(dN(C{%DTG!?#jU5K5A7zH^h7=oqx=8L2r6>b-U{madL7dRk zO`rfKZ!3W*fmfFlUH(36T!|9pQ_`H8yH7y&nKVCcs-1pQ6DqaINU3jC`}MfF`&#pO zJ0H%bolTYDd^n$)KsOG}UcPQ8!2j9r@2j0$*9?>sl*xXW#`woEER|#UD4!6i>jAuv O6Zji}>Pe=kbjAmn+_gFY literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..087f49a4d394cbb0779a51e272a1508f1ed90c94 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9;)KBb~hq~7#Qej7qY#S zu<7Y$TG&MkGjW9{E$yO}*|ahjR~S)SoL=syl|8gFm)hY>jCn)ZFSK#{6krY7!@IsgCw literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..e7bd832ff7bc621fe0f78978335b9c62761b3001 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5TJetG}vP|b5tCAxQvRThQMe# zfrJOHaDastF8NV${NYDg_)G!TpgpX)`K86FSgT6F0)i>6U}Gz4fH0-z*W Ing}HH0DjS>j{pDw literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_4_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..e7603c7f8cd5d8c2d99189139e4c03c9ee3edef5 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$rnNZ8CR*D;Yx5DIMLiZh z-AoI+XkjLL=+eUubUUfZrt;RjfF4Gnqs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(@|Jt__h0n~6o47(Pg&T|yqesPIA%Gf|l!ea}U=7;Cnwwu*oQky? zW&&2jQk_m2Ob|=Yo;kA@#G)k)G}{(x2d(J6AUi;Qpao51w}aeGGq;1ngx2mSWIiZK JmL>uTJpeIT;a&g$ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..0298750cc6793c7ce2e5ddf247207b68cb783cd5 GIT binary patch literal 2866 zcmc&#%WB&|6qS{@E@{?Xbmg5K2wrs&N<$!pEGpuJKo(sTBU=*`8e3*01xypjCJ)2x zeDe+Y3jaa+O|7o&L&Zd6JARM>&Cxw`?zubx}-MiQZ&PM6w9O;Q=e3Es3cWt`y)x8Ta{XThvx@bCcY-iz|= z#~!-RvU%jrV%|*5&Fb(zNuPJd+%ouiGa+{tO?mQWKnsKqYM_~?^afSMR+vO0_Zqg- zYxq-dKd>$9&-3$c`V3}7)|#FH2VyS+n)<*G#yL#5JN7jp_pz4=`E1f->I2Wccz=OY mvLUnSw|U{0^DV|+%$}5w{h%}9wEk)5CST_HFPV7hR{aMEph92( literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..7102ca4416ca56928b218592b30eb73968603071 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(@|Jt__h0n~6o4j*JTvg%QB zL}-A*0$X^DiX+n1X!t{era(k)@qmu zSPe^cI%P1ylJM}L&z?E67vvLK&_FY7p=Q&HrkUGm>~2_i>;;88E(}VNrHMd74*>eW BfJ6WQ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..de9a17bdcee9398f483d64e9ea43632f21078429 GIT binary patch literal 2866 zcmdT_O-sW-5Zy*=MLm1))LUARo;-*sf`SKwtq2}GNJ*P55!-tOC(H*dFv9ahgzPA_uohTZBGcbh`rDe8AMeY2>y9N`JG zTX%%Tkv=lpE)RIuHhVl+m&%*P`{Hb|^04^&w!%gs*SmhF*RRx@j^LG!-}bIdAEOf1L$`B0p2#2`_Ji{53j9l4Xka^4N_c-QjwQU%@jJFnK z?hdNe>hV_)qmQxgL1S#8`R)dsS+!Emdtdq{dL~ zV7o9r6=M(k<3Ve}ta-H$%~izyQIV9VJVMf*ntaM74Evmw z&)R?3FRcl~{?qbmeHL@{qw12sdGk>{keii{s-LI#d1^&@Rk_Otx20t-u@Rt@Pf*G$MIbr`3!5pGrTc4urs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSQ97L!iMPyUtN@NVtH)0b5v%iX*}Y zTMEJ!W*~JVO&k_BBhBB$_!n8tC>xSeki!9%U(m&6 zr&9(K1H;)fXZF$w+k!j?w2M~sUa0vrp=oJ0DE#SRKFAJQ0caYs#YKaXWN9Lh&;tN8 C5XDXa literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..6fb6efeb4e54fe781f2f8c87136b573327db6a6f GIT binary patch literal 2866 zcmd5-O-lkn7+%-Ru+AMid0PeX)IlN<3_7%^5p?JfvbrN1sVlR)fn?yN4`FV;{f2%; z|G<7z?Wivf)3TPi)xgg4em?Ir!@`WK=atiom~x}sY9?`-R69s(gPN90Yb~3))acf2 zY7!nkHrfvHa3*PAva`Ht6iuMA3!KCqoS zB~82QxYTR9Fb0;GRsu_Q%~{D9bmbn7VhwnNGsx(%tQBVo)9v>NuUBgY7(LBWaTXgu zwOT!S_t$jOOD;4s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u>zQK8TPSWg&+EEZx9p zm>i4`qesPIApoIa;R2&!vM@f39`52W|AA<9Kf~lky7)-s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u>zPKBM7+JzNkehk;=< zj0lGja(IohM?-*|5E#v8kWfPo2V^#Km{Lc43a|$4Va?4iEl$N+4Ko3&VX0213?>GK lvuDoOf*e3=8fwShp-R&!d}tjPGz))1;R8yNrHMd74*(Skj0*q& literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..87983b36aa0c26b902b0a7cf1d16b840dd09195f GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQqLnzU%!RpqsGAS+2QB?T zOS55dKubT+!~L|hW2l;q9JfO`erXy0$YDe;dyws4C22JK%&g=!Th@fqu>M&?RdWke2uRB0yPyz_K5h#!0FOYp8aYA~D naTg&oh>*wWKA1ak(I7wIl7oul4SPcR37BmQN|L3CKtc}y>g;vr literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..82ae3cfca58903141efdad9d17063d3239e9e951 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUYrD;!|qg-d=^92P>O zX$%$;xY7hnew4-x0bF4=Dn6R8V5tRHT7bz@pPmA&L3>zp^Gl0Uu~x%Oz-m~k(3il%8C&^mr$ae#{kCCSo6AfX2U&jXYD literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..aa4669e4ece2007d911842d536cbabb190edbd5d GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTf|Elv-+hl3sThzInz9}4{8 z6fU&(3oXp2g_*e09j)!6rTMfnn^xwc#{s?ULARHheCqpQ3a|$4Va?4iEl$N+4Ko3& zVX0213?>GKvuDoOf;>ZrrqsjKFdt~fUZ`7XMdJ+%LSaA+KjN~3klBR71{!X7{fEyC QBJ9U&HYiD!CISgP0B6EV(*OVf literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..d9b7e40388bdbd9879c7ade40aebf2562e438246 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x#%MT@9uA{nhCR$i z!v%Y|Kt%D&jfNd^D2=kQg#c~S$P{1=+QXWgUs{}swHjsuR>M-AP8m$toJKpbvuDoO of=s6c4K#Bv)NGp2v~oAHxga*J04>5Cs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$rnNZ8CR*4*3o{X+LK`>J z(|nj+v}UVJ2WTEY<0h!32r&vuDoi1+nnaKsj5e8mu(LB=SjO%*Uq&pB&6p_|)K& wL()$uoUn!sA+wR>@tO~F2QC`P91M2=s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x2Ci^G4y#c%{;s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQqO~}(T|<#g>+qqK`Lr?@ zJq~GQ53S5ac0VocprzR$KhV;ATAGa<{lsnYM2RF4NG-8WiT-?oIP`9FUS!@(6&%@SZRE6K(lQ@7@r!b9A3A8)Q}4Z r`440+AvY2skJJ5>=!N;2kQp#_*l1$(5~Bv2*~Ex}l4NNjkkA7FP_I?n literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy new file mode 100644 index 0000000000000000000000000000000000000000..4ae59560a8fe2cc34f2fbc8d61dc0dc1fdb8daf4 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5TH#6G}uGTh0!ouV0;)o62)PD zhtVT33_yN{`5#8Z!U37aB{wRL97>~X`h~z0U=7;Cnwwu*oQky?W&&2jQk_m2ObiTX o&z!LZ;=Qz{L7oTMLra=|0X+0M1|`YTL?EFD01uvY&Hw-a literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..c4628e390913e35519f6320a7a7082801a744798 GIT binary patch literal 2866 zcmeHIO-lkn7+%*l!#a29I8+F=}=N=`AzTIpwtL} z?bIkSown;y-}GPrOc_}MOSP@B$pAFz0ge-Oc!G1tYKfekWQyel9m4J9Y7zR+W^!Xcu68)3CULcL zF*UAAl6obvn#5)ElhM|@cn9LWU&cBfYZ>c!&T;)!;o~{4*LVlST?!OdVlVJF{Iu`1 zhS&>S)L!_lGV+?Sy8nw`_1<9lWBuZGi}4V?piF5wzDv+&SPq}zy?)u2u@rFbeK>pz z=g}pI{HeC6Zr%fQM!BK2b00kF{Oose1wX$MZW-y!yk8Lu PLtOQX<-eqFgrCAEFU?(1 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..9d3240149505f4651eaa2a2042a326cdc9aacd18 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2cULnw65AhF- zhWQT0htZ^qPXX4TJ*>I;rNyaOt6?T!H7wQXl)=QnaQ4g@TRP#rPwTNp{^+-{>RW+}u4TpNv zXxAKS5l){OO_zkcYa1OB+Sy!Q=_z6>)8Bf1%Se;O_S=4|6BKHuLrJ0KH@#bfQX>Gi zSEIyq+O9`^(}N+f)FcvEx@}EEhM-B0aGI#YGh9MeOXTb%OIcpfAv|8L7Gdx*btPG% z3%ahKe}rrD(n=oG2RG1zEZeywLESN^1-5LbxTu|;Noo~s`w8q_CN~bFYR54(iK~^1 z>FKE?sb9j~JpOim{xttqd;<7=5z&t}q9~8~h;iMY@|d?NTvwmVba>y(eCy!3#v+|{ z@td0aU(k)tebf8syrQ*4c%F+|h^U7)q6icINX1Yhwuhvsg~or zWZ^X|hu84Iplr)1A&&3k@yUv3H1jV*p1;gD67BFiZQhrtzG(i6*-YlbYhbT&6s^DM MeEv)NM))dx0yq;l82|tP literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..0aa5c5ed4f5772e575391a54bdbe1ca9cee469e7 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u>zQK8TPSWg&+EEZx9p zm>i4`qerSZ$lvH;fF2ewK8zko>PM1aLGeqfe_?52lpcr>m;$UpdsuVxON&#nR>Mrd tYFMh%DT9fAjyrqi%wAez`uSvN+Cs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTf|Esk#AD4)I|fE>THVAI1l z$nHkAk6Pl$W)3wrav0EpP3!QWg`1Jhq_z7&=Hs%D8sacBU^L7>Fg}bXR~)8qDAVX+ z0P_pw;!}V%Xb)>{era(k)@qmuSPe^cI%P1ytRzgIJ#%I+NEI%ckO@G2wgm8AklDB} zk>-Q+gY3be3F#+B9%L>^9WEN58Q9ze6ob1JpI&;&f!qVK3mXk}E6BZAF(EgCl4NNj HkkA7FzNkSx literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..963d2d182fa1a6864a2b045be6b1d7aa7868b493 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u>zQK8TPSWub?_XxbS~ zJLqYG6h0{ZU`s=z;-rM)$W;qVJ1`m+5->iDCS80Aums?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuZB0-V2|B@)Dp*T`rsEu_CGDy^b7%H z_aoa&EpcQsN7?ib0cyn~vN?>mQ12n16>RJq1{U z_ORyWmlmgDt%jL^)v#2jQw9?Q!`U-u_JTY~h$dDIz5oN-VGA_}g9gcg+<=S5XD&<* xpZPE~xM+~wAUQlVLHFU&3-%Y(ds?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%WwQ8Wo3x2Ci^G4y#f2Xb6mk zz-S1JhQM$RfhoWmw1+h}zqB|NYcs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuLh26u*YsUwZyTTPJK~oxfho?)OYcy ziKK)8u6UtToKpQTKj1P4CQoY`m;KZdr?vZOVLr9|j>{ZcxS5`2;_?@@#A)l7DZm=E zhc!39v^W)OHOvI8hNU{4GMK36jI(FX*n&*MMH8|Kmp&x%y-@S9(uB;Xhdh#7v4#ac lHvsiw^%qPW>UXT>!tAAj`2^j7%MO?uaM7S7S(*qW^Z+>%19AWW literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..4cf0ac8fb54b51ecb79e24a7398b54265d19a54e GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVD1A<$rt-94k?*u!O1 zl%63l1z3aju;%8M7N=sZhM9oXuvDj01{1_lXV09m1+i#N(>e%-y8DNE_yEIyFEkxt M(4Zt)ng}HH04|aKod5s; literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_2_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..f7684c45e6f7a9b15503470b33439206b563346b GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu;s^blyUM;C$csU?oxbn1&z z%e@0-4)sHXy77oBTxjL);bbnZu%nhZD6FVu7OFW@fHi0jYi@pNaVpkomswXxLy4Hza#$ Q7Iz@~K}oVS5lH9(0K1y?-v9sr literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..0a225bc4a1e7900a7b2e0e11a2c0469e310d2d7f GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVDXA<$rt-A$w7*u!O1 zlwKh)1z3aju;%8M7N=sZhM9oXuvDj01{1_VXV0A33u4iMwuRb56B^>1Q4&`O0OOY? P;ZBUXpd?wE2qg3XHpc@j literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..a97b899a7c37a978a718548d976c19f74ee1ea3d GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuZAbtV2|Bj)Dj1oPA#)=nS6F0)vHt9tGkZZSN@?1724O1DeYQ|{l0_p-!p_2HK0Z09oA8DqAvJ`;9=nT46D8ya Q(o7>wH7H4zCISgP04_@nBme*a literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..9e0de61ccbaee5a7892ea73867d447f2e5d483d9 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSQ97LZHDOyRK1jNVtrK1O9Npo_a<_ zM?+vV1O|TyOaa!QJ*>I;rNyaOt6?T!H7wQXl)=QnaQ4iZy&ws?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$rnNZ2ChD_>vfHU28uT}D zH2iUgIX%LLmUht6Y*47t+I-wGGjwF=5k~Z|gH|Cs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX u?P1N$FD*{RS`9M+t6`~5rwk^Dac9q**$ZNg(xc%(t8@TLlBJ12LJt51Gs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX u?P1N$FD*{RS`9M+t6`~5rwk^Dac9q*u?4Y4>CteYRXPAA$dN(5Gptrnu_4yAmNf+IkCxw%S8yb;NqXaZF&6+ zeoVb24k8YPuJ;ZP+e8&32tUrx4QPIJvj+3V&D*L3W6ZG{F2j&f5suI z?_dqC*n7ZLL=j7X7{qKtiS-ld&-~q2Ap62t9RE%`{gbA(Y14_-ze=ehpa>`eihv@Z n2y8|GWP=JbrDcKttzkpf@KHS_#%6KKr3h?Gp#731lHz>>utJBo literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/u.npy new file mode 100644 index 0000000000000000000000000000000000000000..55d7e270967e3afca132b0557ad32219e1b22a4f GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX u?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR*n$ilrANbomgxYLBuf*4gdPCR`BH!Y literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..ea94d4e37d18bca76de9094a5e3a1efbbe0c6fc2 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX v?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR*aGq1QF=5SXqgT`NwPE%Naz6oBI`rR literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/ur.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/ur.npy new file mode 100644 index 0000000000000000000000000000000000000000..5e84bb919c74fd42a50b6a6bca850cc769db8f10 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX s?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR*pA}SFralB042%NL?EFD0Kl6aNdN!< literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/generate_actions.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/generate_actions.py deleted file mode 100644 index edc3ad6dd..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/generate_actions.py +++ /dev/null @@ -1,193 +0,0 @@ -import os -import numpy as np - -# Configuration -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -BASE_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'action') -VIDEO_OUTPUT_DIR = BASE_OUTPUT_DIR -os.makedirs(VIDEO_OUTPUT_DIR, exist_ok=True) - -FRAME_COUNT = 81 -CAM_VALUE = 0.1 - -# Action Mapping -KEY_TO_INDEX = { - 'W': 0, 'S': 1, 'A': 2, 'D': 3, -} - -VIEW_ACTION_TO_MOUSE = { - "stop": [0.0, 0.0], - "up": [CAM_VALUE, 0.0], - "down": [-CAM_VALUE, 0.0], - "left": [0.0, -CAM_VALUE], - "right": [0.0, CAM_VALUE], - "up_right": [CAM_VALUE, CAM_VALUE], - "up_left": [CAM_VALUE, -CAM_VALUE], - "down_right": [-CAM_VALUE, CAM_VALUE], - "down_left": [-CAM_VALUE, -CAM_VALUE], -} - -def get_multihot_vector(keys_str): - """Convert string like 'WA' to [1, 0, 1, 0, 0, 0]""" - vector = [0.0] * 6 - if not keys_str: - return vector - for char in keys_str.upper(): - if char in KEY_TO_INDEX: - vector[KEY_TO_INDEX[char]] = 1.0 - return vector - -def get_mouse_vector(view_str): - """Convert view string to [x, y]""" - return VIEW_ACTION_TO_MOUSE.get(view_str.lower(), [0.0, 0.0]) - -def generate_sequence(key_seq, mouse_seq): - """ - Generates action arrays based on sequences. - """ - keyboard_arr = np.zeros((FRAME_COUNT, 6), dtype=np.float32) - mouse_arr = np.zeros((FRAME_COUNT, 2), dtype=np.float32) - - mid_point = FRAME_COUNT // 2 - - # First Half - k_vec1 = get_multihot_vector(key_seq[0]) - m_vec1 = get_mouse_vector(mouse_seq[0]) - keyboard_arr[:mid_point] = k_vec1 - mouse_arr[:mid_point] = m_vec1 - - # Second Half - k_vec2 = get_multihot_vector(key_seq[1]) - m_vec2 = get_mouse_vector(mouse_seq[1]) - keyboard_arr[mid_point:] = k_vec2 - mouse_arr[mid_point:] = m_vec2 - - return keyboard_arr, mouse_arr - -def save_action(index, keyboard_arr, mouse_arr): - filename = f"{index:06d}_action.npy" - filepath = os.path.join(VIDEO_OUTPUT_DIR, filename) - - action_dict = { - 'keyboard': keyboard_arr, - 'mouse': mouse_arr - } - np.save(filepath, action_dict) - return filename - -def generate_description(key_seq, mouse_seq): - """Generates a human-readable string for the combination.""" - k1, k2 = key_seq - m1, m2 = mouse_seq - - # Format Keyboard Description - if not k1 and not k2: - k_desc = "No Key" - elif k1 == k2: - k_desc = f"Hold [{k1}]" - else: - k_desc = f"Switch [{k1}]->[{k2}]" - - # Format Mouse Description - if m1 == "stop" and m2 == "stop": - m_desc = "Static" - elif m1 == m2: - m_desc = f"Hold [{m1}]" - else: - m_desc = f"Switch [{m1}]->[{m2}]" - - return f"{k_desc} + {m_desc}" - -# ========================================== -# Main Generation Logic -# ========================================== - -configs = [] -readme_content = [] - -# Group 1: Constant Keyboard, No Mouse (0-7) -keys_basic = ['W', 'S', 'A', 'D', 'WA', 'WD', 'SA', 'SD'] -for k in keys_basic: - configs.append(((k, k), ("stop", "stop"))) - -# Group 2: No Keyboard, Constant Mouse (8-15) -mouse_basic = ['up', 'down', 'left', 'right', 'up_right', 'up_left', 'down_right', 'down_left'] -for m in mouse_basic: - configs.append((("", ""), (m, m))) - -# Group 3: Split Keyboard, No Mouse (16-23) -split_keys = [ - ('W', 'S'), ('S', 'W'), - ('A', 'D'), ('D', 'A'), - ('W', 'A'), ('W', 'D'), - ('S', 'A'), ('S', 'D') -] -for k1, k2 in split_keys: - configs.append(((k1, k2), ("stop", "stop"))) - -# Group 4: No Keyboard, Split Mouse (24-31) -split_mouse = [ - ('left', 'right'), ('right', 'left'), - ('up', 'down'), ('down', 'up'), - ('up_left', 'up_right'), ('up_right', 'up_left'), - ('left', 'up'), ('right', 'down') -] -for m1, m2 in split_mouse: - configs.append((("", ""), (m1, m2))) - -# Group 5: Constant Keyboard + Constant Mouse (32-47) -combo_keys = ['W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD', 'W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD'] -combo_mice = ['left', 'left', 'right', 'right', 'up', 'up', 'down', 'down', 'up_left', 'up_left', 'up_right', 'up_right', 'down_left', 'down_right', 'right', 'left'] -for i in range(16): - configs.append(((combo_keys[i], combo_keys[i]), (combo_mice[i], combo_mice[i]))) - -# Group 6: Constant Keyboard, Split Mouse (48-55) -complex_1_keys = ['W'] * 8 -complex_1_mice = [ - ('left', 'right'), ('right', 'left'), - ('up', 'down'), ('down', 'up'), - ('left', 'up'), ('right', 'up'), - ('left', 'down'), ('right', 'down') -] -for i in range(8): - configs.append(((complex_1_keys[i], complex_1_keys[i]), complex_1_mice[i])) - -# Group 7: Split Keyboard, Constant Mouse (56-63) -complex_2_keys = [ - ('W', 'S'), ('S', 'W'), - ('A', 'D'), ('D', 'A'), - ('W', 'A'), ('W', 'D'), - ('S', 'A'), ('S', 'D') -] -complex_2_mouse = 'up' -for k1, k2 in complex_2_keys: - configs.append(((k1, k2), (complex_2_mouse, complex_2_mouse))) - - -# Execution -print(f"Preparing to generate {len(configs)} action files...") - -for i, (key_seq, mouse_seq) in enumerate(configs): - if i >= 16: break - - # Generate Data - kb_arr, ms_arr = generate_sequence(key_seq, mouse_seq) - filename = save_action(i, kb_arr, ms_arr) - - # Generate Description for README - description = generate_description(key_seq, mouse_seq) - readme_entry = f"{i:02d}. {description}" - readme_content.append(readme_entry) - - print(f"Generated {filename} -> {description}") - -# Write README -readme_path = os.path.join(BASE_OUTPUT_DIR, 'README.md') -with open(readme_path, 'w', encoding='utf-8') as f: - f.write(f"Total Files: {len(readme_content)}\n\n") - for line in readme_content: - f.write(line + '\n') - -print(f"\nProcessing complete.") -print(f"64 .npy files generated in {VIDEO_OUTPUT_DIR}") -print(f"Manifest saved to {readme_path}") \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py new file mode 100644 index 000000000..a1449c35a --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py @@ -0,0 +1,278 @@ +import os +import numpy as np + +# Configuration +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +BASE_OUTPUT_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "actions")) +VIDEO_OUTPUT_DIR = BASE_OUTPUT_DIR + +os.makedirs(VIDEO_OUTPUT_DIR, exist_ok=True) + +CAM_VALUE = 0.1 +FRAME_COUNT = 76 + +# Action Mapping +KEY_TO_INDEX = { + 'W': 0, 'S': 1, 'A': 2, 'D': 3, +} + +VIEW_ACTION_TO_MOUSE = { + "stop": [0.0, 0.0], + "up": [CAM_VALUE, 0.0], + "down": [-CAM_VALUE, 0.0], + "left": [0.0, -CAM_VALUE], + "right": [0.0, CAM_VALUE], + "up_right": [CAM_VALUE, CAM_VALUE], + "up_left": [CAM_VALUE, -CAM_VALUE], + "down_right": [-CAM_VALUE, CAM_VALUE], + "down_left": [-CAM_VALUE, -CAM_VALUE], +} + +def get_multihot_vector(keys_str): + """Convert string like 'WA' to [1, 0, 1, 0, 0, 0]""" + vector = [0.0] * 6 + if not keys_str: + return vector + for char in keys_str.upper(): + if char in KEY_TO_INDEX: + vector[KEY_TO_INDEX[char]] = 1.0 + return vector + +def get_mouse_vector(view_str): + """Convert view string to [x, y]""" + return VIEW_ACTION_TO_MOUSE.get(view_str.lower(), [0.0, 0.0]) + +def generate_sequence(key_seq, mouse_seq): + """ + Generates action arrays based on sequences. + key_seq and mouse_seq must be length FRAME_COUNT. + Duplicates the first frame at the beginning, so output length is FRAME_COUNT + 1. + """ + if len(key_seq) != FRAME_COUNT or len(mouse_seq) != FRAME_COUNT: + raise ValueError("key_seq and mouse_seq must be length FRAME_COUNT") + + keyboard_arr = np.zeros((FRAME_COUNT, 6), dtype=np.float32) + mouse_arr = np.zeros((FRAME_COUNT, 2), dtype=np.float32) + + for i in range(FRAME_COUNT): + keyboard_arr[i] = get_multihot_vector(key_seq[i]) + mouse_arr[i] = get_mouse_vector(mouse_seq[i]) + + keyboard_arr = np.vstack([keyboard_arr[0:1], keyboard_arr]) + mouse_arr = np.vstack([mouse_arr[0:1], mouse_arr]) + + return keyboard_arr, mouse_arr + +def save_action(filename, keyboard_arr, mouse_arr): + if not filename.endswith(".npy"): + filename = f"{filename}.npy" + filepath = os.path.join(VIDEO_OUTPUT_DIR, filename) + + action_dict = { + 'keyboard': keyboard_arr, + 'mouse': mouse_arr + } + np.save(filepath, action_dict) + return filename + + +def build_constant_sequence(value): + return [value] * FRAME_COUNT + + +def build_random_sequence(actions, granularity, rng): + sequence = [] + remaining = FRAME_COUNT + while remaining > 0: + block = granularity if remaining >= granularity else remaining + action = rng.choice(actions) + sequence.extend([action] * block) + remaining -= block + return sequence + + +def build_random_sequence_either_or(key_actions, mouse_actions, granularity, rng): + """Build key_seq and mouse_seq where each block has either key OR mouse, not both.""" + key_seq = [] + mouse_seq = [] + remaining = FRAME_COUNT + while remaining > 0: + block = granularity if remaining >= granularity else remaining + use_key = rng.choice([True, False]) + if use_key: + key_action = rng.choice(key_actions) + mouse_action = "" + else: + key_action = "" + mouse_action = rng.choice(mouse_actions) + key_seq.extend([key_action] * block) + mouse_seq.extend([mouse_action] * block) + remaining -= block + return key_seq, mouse_seq + + +def mouse_short_name(view_str): + mapping = { + "up": "u", + "down": "d", + "left": "l", + "right": "r", + "up_right": "ur", + "up_left": "ul", + "down_right": "dr", + "down_left": "dl", + } + return mapping.get(view_str, "NA") + + +if __name__ == "__main__": + configs = [] + readme_content = [] + rng = np.random.default_rng(42) + + # configs = list of entries + # a entry is a tuple of (key_seq, mouse_seq) + # key_seq is a list of strings, length of FRAME_COUNT, each string is a key in 'W', 'S', 'A', 'D', 'WA', 'WD', 'SA', 'SD' + # mouse_seq is a list of strings, length of FRAME_COUNT, each string is a mouse action in 'up', 'down', 'left', 'right', 'up_right', 'up_left', 'down_right', 'down_left' + + # Naming: 1=WASDudlr (key: W.npy, SA.npy; camera: u.npy; key+camera: W_u.npy, SA_dl.npy). 2-6=rand names below. + # Group 1: Constant Keyboard, No Mouse. W.npy, S.npy, WA.npy, SA.npy, ... + keys_basic = ["W", "S", "A", "D", "WA", "WD", "SA", "SD"] + for key in keys_basic: + configs.append( + (key, build_constant_sequence(key), build_constant_sequence("")) + ) + + # Group 2: No Keyboard, Constant Mouse. u.npy, d.npy, ur.npy, ... + mouse_basic = [ + "up", + "down", + "left", + "right", + "up_right", + "up_left", + "down_right", + "down_left", + ] + for mouse in mouse_basic: + name = mouse_short_name(mouse) + configs.append( + (name, build_constant_sequence(""), build_constant_sequence(mouse)) + ) + + # Group 3: Still. still.npy + configs.append(("still", build_constant_sequence(""), build_constant_sequence(""))) + + # Group 4: Constant key + camera. W_u.npy, SA_dl.npy, ... + for key in keys_basic: + for mouse in mouse_basic: + configs.append( + ( + f"{key}_{mouse_short_name(mouse)}", + build_constant_sequence(key), + build_constant_sequence(mouse), + ) + ) + + # Random groups: allow still ("") as an option (WASD+still, UDLR+still, and full sets+still) + keys_basic_still = keys_basic + [""] + mouse_basic_still = mouse_basic + [""] + + # Group 5: key_1_action_rand (full key set). key_1_action_rand_1..4, key_1_action_rand_1_f4..4_f4 + for granularity in (4, 12): + suffix = "_f4" if granularity == 4 else "" + for i in range(1, 5): + key_seq = build_random_sequence(keys_basic_still, granularity, rng) + configs.append( + (f"key_1_action_rand_{i}{suffix}", key_seq, build_constant_sequence("")) + ) + + # Group 6: camera_1_action_rand (full camera set) + for granularity in (4, 12): + suffix = "_f4" if granularity == 4 else "" + for i in range(1, 5): + mouse_seq = build_random_sequence(mouse_basic_still, granularity, rng) + configs.append( + (f"camera_1_action_rand_{i}{suffix}", build_constant_sequence(""), mouse_seq) + ) + + # Group 7: key_camera_1_action_rand (both full sets) + for granularity in (4, 12): + suffix = "_f4" if granularity == 4 else "" + for i in range(1, 5): + key_seq = build_random_sequence(keys_basic_still, granularity, rng) + mouse_seq = build_random_sequence(mouse_basic_still, granularity, rng) + configs.append( + (f"key_camera_1_action_rand_{i}{suffix}", key_seq, mouse_seq) + ) + + # WASD-only (no combined keys) and u/d/l/r-only (no combined directions), with still as option + keys_wasd_only = ["W", "S", "A", "D"] + mouse_udlr_only = ["up", "down", "left", "right"] + keys_wasd_still = keys_wasd_only + [""] + mouse_udlr_still = mouse_udlr_only + [""] + + # Group 8: key_2_action_rand (WASD+still only) + for granularity in (4, 12): + suffix = "_f4" if granularity == 4 else "" + for i in range(1, 5): + key_seq = build_random_sequence(keys_wasd_still, granularity, rng) + configs.append( + (f"key_2_action_rand_{i}{suffix}", key_seq, build_constant_sequence("")) + ) + + # Group 9: camera_2_action_rand (UDLR+still only) + for granularity in (4, 12): + suffix = "_f4" if granularity == 4 else "" + for i in range(1, 5): + mouse_seq = build_random_sequence(mouse_udlr_still, granularity, rng) + configs.append( + (f"camera_2_action_rand_{i}{suffix}", build_constant_sequence(""), mouse_seq) + ) + + # Group 10: key_camera_2_action_rand (WASD+still, UDLR+still) + for granularity in (4, 12): + suffix = "_f4" if granularity == 4 else "" + for i in range(1, 5): + key_seq = build_random_sequence(keys_wasd_still, granularity, rng) + mouse_seq = build_random_sequence(mouse_udlr_still, granularity, rng) + configs.append( + (f"key_camera_2_action_rand_{i}{suffix}", key_seq, mouse_seq) + ) + + # Group 11a: key_camera_excl_1_action_rand (either key OR camera per block, full key + full camera set) + for granularity in (4, 12): + suffix = "_f4" if granularity == 4 else "" + for i in range(1, 5): + key_seq, mouse_seq = build_random_sequence_either_or(keys_basic_still, mouse_basic_still, granularity, rng) + configs.append( + (f"key_camera_excl_1_action_rand_{i}{suffix}", key_seq, mouse_seq) + ) + + # Group 11b: key_camera_excl_2_action_rand (either key OR camera per block, WASD/UDLR+still) + for granularity in (4, 12): + suffix = "_f4" if granularity == 4 else "" + for i in range(1, 5): + key_seq, mouse_seq = build_random_sequence_either_or(keys_wasd_still, mouse_udlr_still, granularity, rng) + configs.append( + (f"key_camera_excl_2_action_rand_{i}{suffix}", key_seq, mouse_seq) + ) + + # Execution + print(f"Preparing to generate {len(configs)} action files...") + + for name, key_seq, mouse_seq in configs: + # Generate Data + kb_arr, ms_arr = generate_sequence(key_seq, mouse_seq) + filename = save_action(name, kb_arr, ms_arr) + readme_content.append(filename.replace(".npy", "")) + + print(f"Generated {filename}") + + readme_path = os.path.join(VIDEO_OUTPUT_DIR, "README.md") + with open(readme_path, "w") as f: + f.write(f"Total Files: {len(readme_content)}\n\n") + for idx, name in enumerate(readme_content): + f.write(f"{idx:02d}: {name}\n") + + print(f"{len(configs)} .npy files generated in {VIDEO_OUTPUT_DIR}") \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py new file mode 100644 index 000000000..11b08a03f --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py @@ -0,0 +1,214 @@ +import json +import os +import shutil + +import cv2 + +# Output path +output_path = ( + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/" + "WanGame2.1_1.3b_i2v/validation_random.json" +) + +# Fixed fields +fixed_fields = { + "video_path": None, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77, +} + +action_dir = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions" +# WASDudlr: single key W.npy, single camera u.npy, key+camera w_u.npy +still = os.path.join(action_dir, "still.npy") +key_W = os.path.join(action_dir, "W.npy") +key_S = os.path.join(action_dir, "S.npy") +key_A = os.path.join(action_dir, "A.npy") +key_D = os.path.join(action_dir, "D.npy") +key_wa = os.path.join(action_dir, "WA.npy") +key_s_u = os.path.join(action_dir, "S_u.npy") +camera_u = os.path.join(action_dir, "u.npy") +camera_d = os.path.join(action_dir, "d.npy") +camera_l = os.path.join(action_dir, "l.npy") +camera_r = os.path.join(action_dir, "r.npy") +# key_1_action_rand, camera_1_action_rand (full set); _f4 suffix for granularity 4 +key_1_action_rand_1 = os.path.join(action_dir, "key_1_action_rand_1.npy") +key_1_action_rand_2 = os.path.join(action_dir, "key_1_action_rand_2.npy") +key_1_action_rand_1_f4 = os.path.join(action_dir, "key_1_action_rand_1_f4.npy") +key_1_action_rand_2_f4 = os.path.join(action_dir, "key_1_action_rand_2_f4.npy") +camera_1_action_rand_1 = os.path.join(action_dir, "camera_1_action_rand_1.npy") +camera_1_action_rand_2 = os.path.join(action_dir, "camera_1_action_rand_2.npy") +camera_1_action_rand_1_f4 = os.path.join(action_dir, "camera_1_action_rand_1_f4.npy") +camera_1_action_rand_2_f4 = os.path.join(action_dir, "camera_1_action_rand_2_f4.npy") +key_camera_1_action_rand_1 = os.path.join(action_dir, "key_camera_1_action_rand_1.npy") +key_camera_1_action_rand_2 = os.path.join(action_dir, "key_camera_1_action_rand_2.npy") +key_camera_1_action_rand_1_f4 = os.path.join(action_dir, "key_camera_1_action_rand_1_f4.npy") +key_camera_1_action_rand_2_f4 = os.path.join(action_dir, "key_camera_1_action_rand_2_f4.npy") +key_camera_excl_1_action_rand_1 = os.path.join(action_dir, "key_camera_excl_1_action_rand_1.npy") +key_camera_excl_1_action_rand_2 = os.path.join(action_dir, "key_camera_excl_1_action_rand_2.npy") +key_camera_excl_1_action_rand_1_f4 = os.path.join(action_dir, "key_camera_excl_1_action_rand_1_f4.npy") +key_camera_excl_1_action_rand_2_f4 = os.path.join(action_dir, "key_camera_excl_1_action_rand_2_f4.npy") +# key_2_action_rand, camera_2_action_rand (WASD/UDLR+still) +key_2_action_rand_1 = os.path.join(action_dir, "key_2_action_rand_1.npy") +key_2_action_rand_1_f4 = os.path.join(action_dir, "key_2_action_rand_1_f4.npy") +camera_2_action_rand_1 = os.path.join(action_dir, "camera_2_action_rand_1.npy") +camera_2_action_rand_1_f4 = os.path.join(action_dir, "camera_2_action_rand_1_f4.npy") +key_camera_2_action_rand_1 = os.path.join(action_dir, "key_camera_2_action_rand_1.npy") +key_camera_2_action_rand_1_f4 = os.path.join(action_dir, "key_camera_2_action_rand_1_f4.npy") +key_camera_excl_2_action_rand_1 = os.path.join(action_dir, "key_camera_excl_2_action_rand_1.npy") +key_camera_excl_2_action_rand_1_f4 = os.path.join(action_dir, "key_camera_excl_2_action_rand_1_f4.npy") + +val_img_mc_list = [ + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg", +] + +# Get doom Val data list +val_img_doom_list = [ + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000002.jpg", + "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000003.jpg", +] + + +# Get train data list +train_data_dir = "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1" +train_idx_list = ["000000", "000500", "001000", "001500", "002000", "002500", "003000", "003500"] +train_img_list = [] +train_action_list = [] + +for idx in train_idx_list: + video_path = os.path.join(train_data_dir, f"videos/{idx}.mp4") + # extract the first frame as image + image_path = os.path.join(train_data_dir, f"first_frame/{idx}.jpg") + os.makedirs(os.path.dirname(image_path), exist_ok=True) + cap = cv2.VideoCapture(video_path) + ret, frame = cap.read() + cap.release() + if ret: + cv2.imwrite(image_path, frame) + train_img_list.append(image_path) + train_action_list.append(os.path.join(train_data_dir, f"videos/{idx}_action.npy")) + + +holder = 0 # placeholder +# 32 placeholders (idx 0-31). Fill in manually. +a0 = ["00 Val-00: W", val_img_mc_list[0], key_W] +a1 = ["01 Val-01: S", val_img_mc_list[1], key_S] +a2 = ["02 Val-02: A", val_img_mc_list[2], key_A] +a3 = ["03 Val-03: D", val_img_mc_list[3], key_D] +a4 = ["04 Val-04: u", val_img_mc_list[4], camera_u] +a5 = ["05 Val-05: d", val_img_mc_list[5], camera_d] +a6 = ["06 Val-06: l", val_img_mc_list[6], camera_l] +a7 = ["07 Val-07: r", val_img_mc_list[7], camera_r] +a8 = ["08 Val-00: key rand", val_img_mc_list[0], key_1_action_rand_1] +a9 = ["09 Val-01: key rand", val_img_mc_list[1], key_1_action_rand_2] +a10 = ["10 Val-02: camera rand", val_img_mc_list[2], camera_1_action_rand_1] +a11 = ["11 Val-03: camera rand", val_img_mc_list[3], camera_1_action_rand_2] +a12 = ["12 Val-00: key+camera excl rand", val_img_mc_list[0], key_camera_excl_1_action_rand_1] +a13 = ["13 Val-01: key+camera excl rand", val_img_mc_list[1], key_camera_excl_1_action_rand_2] +a14 = ["14 Val-02: key+camera rand", val_img_mc_list[2], key_camera_1_action_rand_1] +a15 = ["15 Val-03: key+camera rand", val_img_mc_list[3], key_camera_1_action_rand_2] +a16 = ["16 Val-04: (simultaneous) key rand", val_img_mc_list[4], key_2_action_rand_1] +a17 = ["17 Val-05: (simultaneous) camera rand", val_img_mc_list[5], camera_2_action_rand_1] +a18 = ["18 Val-06: (simultaneous) key+camera excl rand", val_img_mc_list[6], key_camera_excl_2_action_rand_1] +a19 = ["19 Val-07: (simultaneous) key+camera rand", val_img_mc_list[7], key_camera_2_action_rand_1] +a20 = ["20 Val-08: W+A", val_img_mc_list[8], key_wa] +a21 = ["21 Val-09: S+u", val_img_mc_list[9], key_s_u] +a22 = ["22 Val-08: Still", val_img_mc_list[8], still] +a23 = ["23 Val-09: Still", val_img_mc_list[9], still] +a24 = ["24 Val-06: key+camera excl rand Frame 4", val_img_mc_list[6], key_camera_excl_1_action_rand_1_f4] +a25 = ["25 Val-07: key+camera excl rand Frame 4", val_img_mc_list[7], key_camera_excl_1_action_rand_2_f4] +a26 = ["26 Train-00", train_img_list[0], train_action_list[0]] +a27 = ["27 Train-01", train_img_list[1], train_action_list[1]] +a28 = ["28 Doom-00: W", val_img_doom_list[0], key_W] +a29 = ["29 Doom-01: key rand", val_img_doom_list[1], key_1_action_rand_1] +a30 = ["30 Doom-02: camera rand", val_img_doom_list[2], camera_1_action_rand_1] +a31 = ["31 Doom-03: key+camera excl rand", val_img_doom_list[3], key_camera_excl_1_action_rand_1] + +Val_entries = { + 0: a0, + 1: a1, + 2: a2, + 3: a3, + 4: a4, + 5: a5, + 6: a6, + 7: a7, + 8: a8, + 9: a9, + 10: a10, + 11: a11, + 12: a12, + 13: a13, + 14: a14, + 15: a15, + 16: a16, + 17: a17, + 18: a18, + 19: a19, + 20: a20, + 21: a21, + 22: a22, + 23: a23, + 24: a24, + 25: a25, + 26: a26, + 27: a27, + 28: a28, + 29: a29, + 30: a30, + 31: a31, +} + +data = [] +for idx in range(32): + if idx not in Val_entries: + raise ValueError(f"Missing entry for idx {idx}") + caption, image_path, action_path = Val_entries[idx] + data.append( + { + "caption": caption, + "image_path": image_path, + "action_path": action_path, + **fixed_fields, + } + ) + +output = {"data": data} +with open(output_path, "w") as f: + json.dump(output, f, indent=4) + +print(f"Generated {len(data)} entries to {output_path}") + +# Check file all exists + +with open(output_path) as f: + data = json.load(f) + +missing = [] +for i, item in enumerate(data['data']): + for key in ('image_path', 'action_path'): + path = item.get(key) + if path: + import os + if not os.path.isfile(path): + missing.append((i, key, path)) +if missing: + print('Missing paths:') + for idx, key, path in missing: + print(f' [{idx}] {key}: {path}') +else: + print('All paths exist.') + + diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json deleted file mode 100644 index 8488ee725..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json +++ /dev/null @@ -1,924 +0,0 @@ -{ - "data": [ - { - "caption": "Validation 00 - W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 00 - A => D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/ad.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 00 - Random", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000003_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 00 - Random", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000500_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - S => A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/sa.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - Random", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000003_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - Random", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000500_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - S => W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/sw.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - Random", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001000_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - Random", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001500_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - W => D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/wd.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - Random", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001000_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - Random", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001500_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 00 - Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Train 00", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000003_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Train 01", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000500.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/000500_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Train 02", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001000_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Train 03", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001500.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/first_frame/001500_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Doom 00 - W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Doom 01 - S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Doom 02 - A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Doom 03 - D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Doom 00 - W => S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "action/000016_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Doom 01 - A => D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "action/000018_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Doom 02 - W => A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "action/000020_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Doom 03 - S => A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "action/000022_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Train 00 - W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Train 01 - S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Train 02 - A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Train 03 - D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 00", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "action/000018_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "action/000019_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 04", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "action/000020_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 05", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "action/000021_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 06", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "action/000022_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 07", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "action/000023_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 08", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "action/000016_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 09", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "action/000017_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 10", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "action/000018_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 11", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "action/000019_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 12", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "action/000020_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 13", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "action/000021_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 14", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "action/000022_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 15", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "action/000023_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 16", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", - "action_path": "action/000020_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 17", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", - "action_path": "action/000021_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 18", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", - "action_path": "action/000022_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 19", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", - "action_path": "action/000023_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 20", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", - "action_path": "action/000024_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 21", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", - "action_path": "action/000025_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 22", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", - "action_path": "action/000026_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 23", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", - "action_path": "action/000027_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "32", - "image_path": "humanplay/000000.jpg", - "action_path": "humanplay/000000_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "33", - "image_path": "humanplay/000001.jpg", - "action_path": "humanplay/000001_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "34", - "image_path": "humanplay/000002.jpg", - "action_path": "humanplay/000002_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "35", - "image_path": "humanplay/000003.jpg", - "action_path": "humanplay/000003_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "36", - "image_path": "humanplay/000004.jpg", - "action_path": "action/000004_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "37", - "image_path": "humanplay/000005.jpg", - "action_path": "action/000005_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "38", - "image_path": "humanplay/000006.jpg", - "action_path": "action/000006_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "39", - "image_path": "humanplay/000007.jpg", - "action_path": "action/000007_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "40", - "image_path": "humanplay/000008.jpg", - "action_path": "action/000008_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "41", - "image_path": "humanplay/000009.jpg", - "action_path": "action/000009_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "42", - "image_path": "humanplay/000010.jpg", - "action_path": "action/000010_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "43", - "image_path": "humanplay/000011.jpg", - "action_path": "action/000011_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "44", - "image_path": "humanplay/000012.jpg", - "action_path": "action/000012_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "45", - "image_path": "humanplay/000013.jpg", - "action_path": "action/000013_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "46", - "image_path": "humanplay/000014.jpg", - "action_path": "action/000014_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "47", - "image_path": "humanplay/000015.jpg", - "action_path": "action/000015_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "48", - "image_path": "humanplay/000016.jpg", - "action_path": "action/000016_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "49", - "image_path": "humanplay/000017.jpg", - "action_path": "action/000017_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "50", - "image_path": "humanplay/000018.jpg", - "action_path": "action/000018_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "51", - "image_path": "humanplay/000019.jpg", - "action_path": "action/000019_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "52", - "image_path": "humanplay/000020.jpg", - "action_path": "action/000020_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "53", - "image_path": "humanplay/000021.jpg", - "action_path": "action/000021_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "54", - "image_path": "humanplay/000022.jpg", - "action_path": "action/000022_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "55", - "image_path": "humanplay/000023.jpg", - "action_path": "action/000023_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "56", - "image_path": "humanplay/000024.jpg", - "action_path": "action/000024_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "57", - "image_path": "humanplay/000025.jpg", - "action_path": "action/000025_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "58", - "image_path": "humanplay/000026.jpg", - "action_path": "action/000026_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "59", - "image_path": "humanplay/000027.jpg", - "action_path": "action/000027_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "60", - "image_path": "humanplay/000028.jpg", - "action_path": "action/000028_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "61", - "image_path": "humanplay/000029.jpg", - "action_path": "action/000029_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "62", - "image_path": "humanplay/000030.jpg", - "action_path": "action/000030_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "63", - "image_path": "humanplay/000031.jpg", - "action_path": "action/000031_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - } - ] -} \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json deleted file mode 100644 index be9a13b82..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json +++ /dev/null @@ -1,324 +0,0 @@ -{ - "data": [ - { - "caption": "Validation 00 - W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 00 - S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 00 - A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 00 - D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 00 - Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 01 - Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 02 - Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "Validation 03 - Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 00 - W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 00 - S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 00 - A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 00 - D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 01 - W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/002000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 01 - S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/002000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 01 - A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/002000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 01 - D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/002000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 02 - W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/004000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/w.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 02 - S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/004000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/s.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 02 - A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/004000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/a.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "train 02 - D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/1_wasd_only/first_frame/004000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - } - ] -} \ No newline at end of file From ecc8f565f765ae04b48563b66ce3e2caee8d31fc Mon Sep 17 00:00:00 2001 From: mignonjia Date: Mon, 9 Feb 2026 21:11:12 +0000 Subject: [PATCH 013/214] actions --- .../WanGame2.1_1.3b_i2v/actions/README.md | 112 +++--- .../actions/camera_1_action_rand_1.npy | Bin 2866 -> 2866 bytes .../actions/camera_1_action_rand_1_f4.npy | Bin 2866 -> 2866 bytes .../actions/camera_1_action_rand_2.npy | Bin 2866 -> 2866 bytes .../actions/camera_1_action_rand_2_f4.npy | Bin 2866 -> 2866 bytes .../actions/camera_1_action_rand_3.npy | Bin 2866 -> 2866 bytes .../actions/camera_1_action_rand_3_f4.npy | Bin 2866 -> 2866 bytes .../actions/camera_1_action_rand_4.npy | Bin 2866 -> 2866 bytes .../actions/camera_1_action_rand_4_f4.npy | Bin 2866 -> 2866 bytes .../actions/camera_2_action_rand_1.npy | Bin 2866 -> 2866 bytes .../actions/camera_2_action_rand_1_f4.npy | Bin 2866 -> 2866 bytes .../actions/camera_2_action_rand_2.npy | Bin 2866 -> 2866 bytes .../actions/camera_2_action_rand_2_f4.npy | Bin 2866 -> 2866 bytes .../actions/camera_2_action_rand_3.npy | Bin 2866 -> 2866 bytes .../actions/camera_2_action_rand_3_f4.npy | Bin 2866 -> 2866 bytes .../actions/camera_2_action_rand_4.npy | Bin 2866 -> 2866 bytes .../actions/camera_2_action_rand_4_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_1_action_rand_1.npy | Bin 2866 -> 2866 bytes .../actions/key_1_action_rand_1_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_1_action_rand_2.npy | Bin 2866 -> 2866 bytes .../actions/key_1_action_rand_2_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_1_action_rand_3.npy | Bin 2866 -> 2866 bytes .../actions/key_1_action_rand_3_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_1_action_rand_4.npy | Bin 2866 -> 2866 bytes .../actions/key_1_action_rand_4_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_2_action_rand_1.npy | Bin 2866 -> 2866 bytes .../actions/key_2_action_rand_1_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_2_action_rand_2.npy | Bin 2866 -> 2866 bytes .../actions/key_2_action_rand_2_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_2_action_rand_3.npy | Bin 2866 -> 2866 bytes .../actions/key_2_action_rand_3_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_2_action_rand_4.npy | Bin 2866 -> 2866 bytes .../actions/key_2_action_rand_4_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_1_action_rand_1.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_1_action_rand_1_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_1_action_rand_2.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_1_action_rand_2_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_1_action_rand_3.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_1_action_rand_3_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_1_action_rand_4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_1_action_rand_4_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_2_action_rand_1.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_2_action_rand_1_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_2_action_rand_2.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_2_action_rand_2_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_2_action_rand_3.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_2_action_rand_3_f4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_2_action_rand_4.npy | Bin 2866 -> 2866 bytes .../actions/key_camera_2_action_rand_4_f4.npy | Bin 2866 -> 2866 bytes .../key_camera_excl_1_action_rand_1.npy | Bin 2866 -> 2866 bytes .../key_camera_excl_1_action_rand_1_f4.npy | Bin 2866 -> 2866 bytes .../key_camera_excl_1_action_rand_2.npy | Bin 2866 -> 2866 bytes .../key_camera_excl_1_action_rand_2_f4.npy | Bin 2866 -> 2866 bytes .../key_camera_excl_1_action_rand_3.npy | Bin 2866 -> 2866 bytes .../key_camera_excl_1_action_rand_3_f4.npy | Bin 2866 -> 2866 bytes .../key_camera_excl_1_action_rand_4.npy | Bin 2866 -> 2866 bytes .../key_camera_excl_1_action_rand_4_f4.npy | Bin 2866 -> 2866 bytes .../scripts/generate_actions.py | 32 +- .../validation_random.json | 324 ++++++++++++++++++ 59 files changed, 396 insertions(+), 72 deletions(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md index 96f37334e..fa58e4bfe 100644 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md @@ -81,54 +81,54 @@ Total Files: 145 78: SD_ul 79: SD_dr 80: SD_dl -81: key_1_action_rand_1_f4 -82: key_1_action_rand_2_f4 -83: key_1_action_rand_3_f4 -84: key_1_action_rand_4_f4 -85: key_1_action_rand_1 -86: key_1_action_rand_2 -87: key_1_action_rand_3 -88: key_1_action_rand_4 -89: camera_1_action_rand_1_f4 -90: camera_1_action_rand_2_f4 -91: camera_1_action_rand_3_f4 -92: camera_1_action_rand_4_f4 -93: camera_1_action_rand_1 -94: camera_1_action_rand_2 -95: camera_1_action_rand_3 -96: camera_1_action_rand_4 -97: key_camera_1_action_rand_1_f4 -98: key_camera_1_action_rand_2_f4 -99: key_camera_1_action_rand_3_f4 -100: key_camera_1_action_rand_4_f4 -101: key_camera_1_action_rand_1 -102: key_camera_1_action_rand_2 -103: key_camera_1_action_rand_3 -104: key_camera_1_action_rand_4 -105: key_2_action_rand_1_f4 -106: key_2_action_rand_2_f4 -107: key_2_action_rand_3_f4 -108: key_2_action_rand_4_f4 -109: key_2_action_rand_1 -110: key_2_action_rand_2 -111: key_2_action_rand_3 -112: key_2_action_rand_4 -113: camera_2_action_rand_1_f4 -114: camera_2_action_rand_2_f4 -115: camera_2_action_rand_3_f4 -116: camera_2_action_rand_4_f4 -117: camera_2_action_rand_1 -118: camera_2_action_rand_2 -119: camera_2_action_rand_3 -120: camera_2_action_rand_4 -121: key_camera_2_action_rand_1_f4 -122: key_camera_2_action_rand_2_f4 -123: key_camera_2_action_rand_3_f4 -124: key_camera_2_action_rand_4_f4 -125: key_camera_2_action_rand_1 -126: key_camera_2_action_rand_2 -127: key_camera_2_action_rand_3 -128: key_camera_2_action_rand_4 +81: key_2_action_rand_1_f4 +82: key_2_action_rand_2_f4 +83: key_2_action_rand_3_f4 +84: key_2_action_rand_4_f4 +85: key_2_action_rand_1 +86: key_2_action_rand_2 +87: key_2_action_rand_3 +88: key_2_action_rand_4 +89: camera_2_action_rand_1_f4 +90: camera_2_action_rand_2_f4 +91: camera_2_action_rand_3_f4 +92: camera_2_action_rand_4_f4 +93: camera_2_action_rand_1 +94: camera_2_action_rand_2 +95: camera_2_action_rand_3 +96: camera_2_action_rand_4 +97: key_camera_2_action_rand_1_f4 +98: key_camera_2_action_rand_2_f4 +99: key_camera_2_action_rand_3_f4 +100: key_camera_2_action_rand_4_f4 +101: key_camera_2_action_rand_1 +102: key_camera_2_action_rand_2 +103: key_camera_2_action_rand_3 +104: key_camera_2_action_rand_4 +105: key_1_action_rand_1_f4 +106: key_1_action_rand_2_f4 +107: key_1_action_rand_3_f4 +108: key_1_action_rand_4_f4 +109: key_1_action_rand_1 +110: key_1_action_rand_2 +111: key_1_action_rand_3 +112: key_1_action_rand_4 +113: camera_1_action_rand_1_f4 +114: camera_1_action_rand_2_f4 +115: camera_1_action_rand_3_f4 +116: camera_1_action_rand_4_f4 +117: camera_1_action_rand_1 +118: camera_1_action_rand_2 +119: camera_1_action_rand_3 +120: camera_1_action_rand_4 +121: key_camera_1_action_rand_1_f4 +122: key_camera_1_action_rand_2_f4 +123: key_camera_1_action_rand_3_f4 +124: key_camera_1_action_rand_4_f4 +125: key_camera_1_action_rand_1 +126: key_camera_1_action_rand_2 +127: key_camera_1_action_rand_3 +128: key_camera_1_action_rand_4 129: key_camera_excl_1_action_rand_1_f4 130: key_camera_excl_1_action_rand_2_f4 131: key_camera_excl_1_action_rand_3_f4 @@ -137,11 +137,11 @@ Total Files: 145 134: key_camera_excl_1_action_rand_2 135: key_camera_excl_1_action_rand_3 136: key_camera_excl_1_action_rand_4 -137: key_camera_excl_2_action_rand_1_f4 -138: key_camera_excl_2_action_rand_2_f4 -139: key_camera_excl_2_action_rand_3_f4 -140: key_camera_excl_2_action_rand_4_f4 -141: key_camera_excl_2_action_rand_1 -142: key_camera_excl_2_action_rand_2 -143: key_camera_excl_2_action_rand_3 -144: key_camera_excl_2_action_rand_4 +137: key_camera_excl_1_action_rand_1_f4 +138: key_camera_excl_1_action_rand_2_f4 +139: key_camera_excl_1_action_rand_3_f4 +140: key_camera_excl_1_action_rand_4_f4 +141: key_camera_excl_1_action_rand_1 +142: key_camera_excl_1_action_rand_2 +143: key_camera_excl_1_action_rand_3 +144: key_camera_excl_1_action_rand_4 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy index e74bedf181827968ab59c5990c5912ffe9430f04..6aa5f58729c462fd393d10cae1d2d0c104ce8813 100644 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk^Dac9q**$ZOPfwqO(GfG3kd&GnTC`pzk0tr0;BSl#N literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk_IjH8CCvuDoi1(`!b+7@UwE%9Ec`Lv>G>2`XUjW>Ql LNwPE%Naz6on_Rr6 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1_f4.npy index f7eced5816695269cae67ee08f3f1b37feb3e148..2e969a6e53156b0023a5b245a0f0750c01d262f6 100644 GIT binary patch delta 178 zcmdlawn=Qm5st|vTq2VjxH!Q03XuH($i~DhlT#TLCQsoMnY@6L0}F=8BlLs#E>QK{ slY_Y=Cg11cL^GFtayFL(RE~A>1WpN{xiGabJ_py7k|{~0i9kXR0E|XF`v3p{ literal 2866 zcmeHI%}T>S5Z)yB zpQ3M2pGh;x9*QJED0s@i>__H*J_55eZhBX@5_u#~noWI+X(y^S>uXI_8+j~XraSPM zP0@T|j(r;2eaD>A_`BX{s&jQasXgy}eAY-J{Ll?2)2KDDJVslSa2(v5jF}Peg8`$K z=lTH)tpJw5l_U~))wR=*CFtY@&ZQx|!8O!%so_+(Vh7QbV!YgM!{R-4Rk$<*!!Rzt zV@V$UEP&zS4(3qjdk-`kdM2~MksSq#I#p3p@9Dd*;OujGbi7&Z^j9^4)w=C!x>fm> z43rF%43rF%43rG~jRA-+C|4$4=+n$IEXQZ~Xwh|KLcZ4P)jw%R^eIC0<>^ZJG7*Wv z`gtgNV{V>ygs%wEw~ekuekO|9J2{xU8^)vH{CQa9OOn(4yxzET=FA)R!WjG|Los?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk^Dac9q*u?4Z{K<|axGvsJs*z5&`4-IKrnoVoB!{QTX NIszrh(nKJk2LM}<&qx3O diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2_f4.npy index e8b66d94fdb8849ee834db34d087414c62a707cf..cb83730b18a909c1c65261f43f0bb69915ec53e8 100644 GIT binary patch delta 125 zcmdlawn=Qm5st}UI3*?@;NqFQf=glY2TmCvTW1oG3le(6B{R8zk!A80pcD(Y%47v@ l4!jto?hTjCg7iq{>O9kOrs?b}CS#_nbF{7TpEwFf&OvCr%S%O14ss%76qEMZ>Q zj#xl&`_${kMC`}Z>l5)^u2ij|^}i@RZoIvhv{~@oFzxlTTHB8psrAxsa_ceXWx#jZ zjQCL)CoJ_77y~zFoxqD>FmD-yqdmci-hmgmfU=`kXn~tSlJyDp8?8EwUguDO>jQ9I z_v}-|Y;leg=!~vm2xY!?N3u@ju>hz+O(dno1*Ow;HlKlRbK_?JC)?pKHVxSt_2T?h zYE=#>2b2TK0p);lU_A#wOi*t2qBJH;$FL#B@ZP9F4Nd!)PLI~iiRjBqUL9$mCSUgvnDlWw@r4Oi3zD1QL1x7TgYV diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_3_f4.npy index ac1e637b1fff71b1779413b10a800498be5bd179..18327d33763b4b5d23ce2860cb7a69a133d150c1 100644 GIT binary patch delta 126 zcmdlawn=QmagNE~IYcKP;N+hCpM!NW52wuJ0!Eg}I$Sc7H*iXW#WFaRCvWDIK$BOV we4axAh-D{#;E)08=b3y0Nb_*nOnw4X{~jm~F_RZbPJwGm$&{qhL?EFD0Ke5MTL1t6 delta 281 zcmdlawn=QmagNF6T%wZ?a0*Xe%_Tee4Tp@v*)wPM;=_|Sa4Ago;8d0+q)J&5r;5oN zI5j5waM?^&;A96{ED5sI7HTP08mK`z5~~cFPLLX0crVlj40;l`3fGj9DM_V?Ktc}y DQ~R=m diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4.npy index 148649e834016592a0b9f4efaa6a7ce1b4da2b69..fefa780ae46e9d39a55f129cea70be06961e5151 100644 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR>;)N02ig{D&yb|)6h<_Q3t;@xG~5Y=4=72N ICISgP07MXH00000 literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk_IjH8CCvuDoO(ircBnold5rtSxYG0^>cX+?v~re*jL NN+X~oS(*qW^Z?FDwjKZg diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_4_f4.npy index 880cd20c25ab906a2d01ee605a6f50d8b67a0147..7c0649a141522f8f0c5e1032a66f280c9a735a75 100644 GIT binary patch delta 110 zcmdlawn=Qm0glN9oD!28I60u0W%3IynaN)`944nSDu8K(JV;Ju@(&J)$tO4!fbtEK bOE^W4^eIm+=M+TZD{xIInUYkR2qg3XSgTGRw9NNJlXgj53N>XVekkA7F88XiP diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1.npy index 6aa5f58729c462fd393d10cae1d2d0c104ce8813..e74bedf181827968ab59c5990c5912ffe9430f04 100644 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk_IjH8CCvuDoi1(`!b+7@UwE%9Ec`Lv>G>2`XUjW>Ql LNwPE%Naz6on_Rr6 literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk^Dac9q**$ZOPfwqO(GfG3kd&GnTC`pzk0tr0;BSl#N diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1_f4.npy index 2e969a6e53156b0023a5b245a0f0750c01d262f6..f7eced5816695269cae67ee08f3f1b37feb3e148 100644 GIT binary patch literal 2866 zcmeHI%}T>S5Z)yB zpQ3M2pGh;x9*QJED0s@i>__H*J_55eZhBX@5_u#~noWI+X(y^S>uXI_8+j~XraSPM zP0@T|j(r;2eaD>A_`BX{s&jQasXgy}eAY-J{Ll?2)2KDDJVslSa2(v5jF}Peg8`$K z=lTH)tpJw5l_U~))wR=*CFtY@&ZQx|!8O!%so_+(Vh7QbV!YgM!{R-4Rk$<*!!Rzt zV@V$UEP&zS4(3qjdk-`kdM2~MksSq#I#p3p@9Dd*;OujGbi7&Z^j9^4)w=C!x>fm> z43rF%43rF%43rG~jRA-+C|4$4=+n$IEXQZ~Xwh|KLcZ4P)jw%R^eIC0<>^ZJG7*Wv z`gtgNV{V>ygs%wEw~ekuekO|9J2{xU8^)vH{CQa9OOn(4yxzET=FA)R!WjG|LoQK{ slY_Y=Cg11cL^GFtayFL(RE~A>1WpN{xiGabJ_py7k|{~0i9kXR0E|XF`v3p{ diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_2.npy index 438ca95800418b5b534b64335c66d1a0409d83d9..d910803913a778f895cff57a89fe34a25a6c25db 100644 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk^Dac9q*u?4Z{K<|axGvsJs*z5&`4-IKrnoVoB!{QTX NIszrh(nKJk2LM}<&qx3O delta 47 zcmdlawn=Qm0S=b4XU^D8KERPOIh8SGvI%F*WF5|g$p)Mteht@@k|{~0i9kXR0LiTq AIsgCw diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_2_f4.npy index cb83730b18a909c1c65261f43f0bb69915ec53e8..e8b66d94fdb8849ee834db34d087414c62a707cf 100644 GIT binary patch literal 2866 zcmeHIO-sW-5KU5JrF!<@DYvvxdh$?25h{2vG!?;v2PtW?vSOQ*O(Fza@Zv{dZykSw zzoP%3{wCdNi->g7iq{>O9kOrs?b}CS#_nbF{7TpEwFf&OvCr%S%O14ss%76qEMZ>Q zj#xl&`_${kMC`}Z>l5)^u2ij|^}i@RZoIvhv{~@oFzxlTTHB8psrAxsa_ceXWx#jZ zjQCL)CoJ_77y~zFoxqD>FmD-yqdmci-hmgmfU=`kXn~tSlJyDp8?8EwUguDO>jQ9I z_v}-|Y;leg=!~vm2xY!?N3u@ju>hz+O(dno1*Ow;HlKlRbK_?JC)?pKHVxSt_2T?h zYE=#>2b2TK0p);lU_A#wOi*t2qBJH;$FL#B@ZP9F4Nd!)PLI~iiRjBqUL9$mCSUgvnDlWw@r4Oi3zD1QL1x7TgYV delta 53 zcmdlawn=P*2Iu4gMwZDRI3p%;aZe8C3Z3l0$uju`=Mf<70mLFewgT6bk|{~0i9kXR E0PYs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk_IjH8CCvuDoO(ircBnold5rtSxYG0^>cX+?v~re*jL NN+X~oS(*qW^Z?FDwjKZg literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR>;)N02ig{D&yb|)6h<_Q3t;@xG~5Y=4=72N ICISgP07MXH00000 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_4_f4.npy index 7c0649a141522f8f0c5e1032a66f280c9a735a75..880cd20c25ab906a2d01ee605a6f50d8b67a0147 100644 GIT binary patch delta 348 zcmdlawn=Qm0S=9`XU^CH@m@moTGRw9NNJlXgj53N>XVekkA7F88XiP delta 110 zcmdlawn=Qm0glN9oD!28I60u0W%3IynaN)`944nSDu8K(JV;Ju@(&J)$tO4!fbtEK bOE^W4^eIm+=M+TZD{xIInUYkR2qg3XSg$CGPPX5ozvw1#~3L}D7sSi7jgLRF1h!0KeG~ APXGV_ delta 199 zcmdlawn=P5I^*VS#u%o_*^E3i!K;}?CO0#RFg4guZe$dr0;R}NGWi0>sm=FUiWoOf I<(SI^0H%C6?f?J) diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1_f4.npy index 5aca2b834c519a4f1db86118da5396a644024a2b..da076a4e1e27376b03de5be081552f8751cd039e 100644 GIT binary patch delta 120 zcmdlawn=P5Jmch}92Jw_vjj};XB9!Cb0%M4`S#^wtgHcSwq53C$O`LNAX IIp#6}0CKSv7XSbN literal 2866 zcmeH|ze~eF6vr>AF;bl!oU)~b(#fHSP^jQwXexq(gM@2x<-|5AmqZA*(8V8t+xq+) z{4+Jzc-Y2*gSg!r?%n&|;|urU!CSi5{mUB-J)lQBAhFNv6U&|r?3QJZB9<^5dJzkV zkk9crCWRhTJS9clXtu1m^|z=mcHZ9WXchc0OefQ7;3un6Rl4LL^S1dBZ%Qv2Y~vs_rGGQ1vXHgR)mi&p khhl?rV-lq?*}8`HVhtbVJ*rDiB_KvJ#l_M71XjQ3wEzGB diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2_f4.npy index f418f818e5ca75b56b0e002f4d241643247326af..c4a7bb1ab9d70f8b9fa62cf5b2d936448d4f97aa 100644 GIT binary patch delta 98 zcmdlawn=P5I^$$-rhv)(YzmW@IVS&SjhK9%BLPCQi%hO(kC}XdBL>X3X9Chp0$?^j cn+HS-3y`}5s3Zlh9sT^~e03g&RQ2+n{ delta 244 zcmdlawn=P5I^$$@4hejCvpTB@2fw1WIn_pdFU1)D*>^A$~O0MEM)=!FULRG diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3.npy index 14cb0d35efa8a9e12aeb03ca4199b368b9efb590..e865d985550df715aed112a2ae37c8637543270b 100644 GIT binary patch delta 44 ycmdlawn=P5I^*VSMgzvp_gPdJH_Nk^F;3QHKehP+hX#nJz&MF5X!BH#xl8~}A`Tn? delta 235 zcmdlawn=P5I^*VQMg_*n6Bt=0*E0$LF~{Wep-FE}XEb5l#HPTwxt>u5=A*F9Q#s}` F0RSdbK*9h3 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_3_f4.npy index dbbb6fe2a70d8994ed569ea9b42a37085d2916ef..087f49a4d394cbb0779a51e272a1508f1ed90c94 100644 GIT binary patch delta 116 zcmdlawn=Qmb;im4j2@HsGpbC^XZD!9o=s%(1V)z0^^6gd@3SOKj%Q1O@J^1?Ndn& diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_4.npy index 6a527f0aa2e902147a801f6785d325c02d03855f..e7bd832ff7bc621fe0f78978335b9c62761b3001 100644 GIT binary patch delta 82 zcmdlawn=P5GUMix93f1T{W;tw`*L`|@a78~N{r}&lP_>^VTeIA32dIqF_#GdT*Mjq literal 2866 zcmeH|!AiqG5QaCYF;YEy@RVCxC_Q;7A`~ikFfQMn3a3my}r}P6M43M7J0%xv+QZlZdmp(6tSQ|KNLPw z_9Y!fEb}8yCoIcr^@cUG{ub5w-up+DtV13L$#{}B`(7wmbDWIg2Py856n@Ee0ISv&RArC oIqL7~A={wR7>7y3wyt44Tf-;I4%e}LsU?B`C7^!Egl0$c3)yDnxc~qF diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_4_f4.npy index 6e73c3a082fff2591c81ba225e334f38d253b9d5..e7603c7f8cd5d8c2d99189139e4c03c9ee3edef5 100644 GIT binary patch delta 84 zcmdlawn=P5GUMcO#*E49nI$IIvm^lV79h?6;*iPJj0uw$FtGsH2Y@&Oh-07_q%H-D QLF^+yb$*+ta?E7{075b!tpET3 delta 265 zcmdlawn=P5GUMcQ#tNne`^gU(#Ym$PS;}x~Sjg5i`2l;(ZXRVn0m{A1PEo=?;3=BYD6*7w9)R@l5w|OeZTqXdh Caa4N% diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1.npy index 3f285a789d580503a0a4b1e5d8edb3e3cd4d41f5..e78573fe0437edb158c8e7bd10cca3cf73b1d8f7 100644 GIT binary patch delta 199 zcmdlawn=P5I^*VS#u%o_*^E3i!K;}?CO0#RFg4guZe$dr0;R}NGWi0>sm=FUiWoOf I<(SI^0H%C6?f?J) delta 52 zcmdlawn=P5I^*VQMjNKd^O;pP>$CGPPX5ozvw1#~3L}D7sSi7jgLRF1h!0KeG~ APXGV_ diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1_f4.npy index da076a4e1e27376b03de5be081552f8751cd039e..5aca2b834c519a4f1db86118da5396a644024a2b 100644 GIT binary patch delta 294 zcmdlawn=P5JmchK=8DM$j4VtI_LBpd#84@=273kupu|H)smZI^B;dH3c?n3*AF;bl!oU)~b(#fHSP^jQwXexq(gM@2x<-|5AmqZA*(8V8t+xq+) z{4+Jzc-Y2*gSg!r?%n&|;|urU!CSi5{mUB-J)lQBAhFNv6U&|r?3QJZB9<^5dJzkV zkk9crCWRhTJS9clXtu1m^|z=mcHZ9WXchc0OefQ7;3un6Rl4LL^S1dBZ%Qv2Y~vs_rGGQ1vXHgR)mi&p khhl?rV-lq?*}8`HVhtbVJ*rDiB_KvJ#l_M71XjQ3wEzGB delta 60 zcmdlawn=P5GUMds%p#ll*?1Tyv9e5l&myq-KdTJm`S#^wtgHcSwq53C$O`LNAX IIp#6}0CKSv7XSbN diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_2_f4.npy index c4a7bb1ab9d70f8b9fa62cf5b2d936448d4f97aa..f418f818e5ca75b56b0e002f4d241643247326af 100644 GIT binary patch delta 244 zcmdlawn=P5I^$$@4hejCvpTB@2fw1WIn_pdFU1)D*>^A$~O0MEM)=!FULRG delta 98 zcmdlawn=P5I^$$-rhv)(YzmW@IVS&SjhK9%BLPCQi%hO(kC}XdBL>X3X9Chp0$?^j cn+HS-3y`}5s3Zlh9sT^~e03g&RQ2+n{ diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3.npy index e865d985550df715aed112a2ae37c8637543270b..14cb0d35efa8a9e12aeb03ca4199b368b9efb590 100644 GIT binary patch delta 235 zcmdlawn=P5I^*VQMg_*n6Bt=0*E0$LF~{Wep-FE}XEb5l#HPTwxt>u5=A*F9Q#s}` F0RSdbK*9h3 delta 44 ycmdlawn=P5I^*VSMgzvp_gPdJH_Nk^F;3QHKehP+hX#nJz&MF5X!BH#xl8~}A`Tn? diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_3_f4.npy index 087f49a4d394cbb0779a51e272a1508f1ed90c94..dbbb6fe2a70d8994ed569ea9b42a37085d2916ef 100644 GIT binary patch delta 261 zcmdlawn=Qmb;il{j69RMStQ_iHJ^1?Ndn& delta 116 zcmdlawn=Qmb;im4j2@HsGpbC^XZD!9o=s%(1V)z0^^6gd@3SOKj%Q1O@QMn3a3my}r}P6M43M7J0%xv+QZlZdmp(6tSQ|KNLPw z_9Y!fEb}8yCoIcr^@cUG{ub5w-up+DtV13L$#{}B`(7wmbDWIg2Py856n@Ee0ISv&RArC oIqL7~A={wR7>7y3wyt44Tf-;I4%e}LsU?B`C7^!Egl0$c3)yDnxc~qF delta 82 zcmdlawn=P5GUMix93f1T{W;tw`*L`|@a78~N{r}&lP_>^VTeIA32dIqF_#GdT*Mjq diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_4_f4.npy index e7603c7f8cd5d8c2d99189139e4c03c9ee3edef5..6e73c3a082fff2591c81ba225e334f38d253b9d5 100644 GIT binary patch delta 265 zcmdlawn=P5GUMcQ#tNne`^gU(#Ym$PS;}x~Sjg5i`2l;(ZXRVn0m{A1PEo=?;3=BYD6*7w9)R@l5w|OeZTqXdh Caa4N% delta 84 zcmdlawn=P5GUMcO#*E49nI$IIvm^lV79h?6;*iPJj0uw$FtGsH2Y@&Oh-07_q%H-D QLF^+yb$*+ta?E7{075b!tpET3 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1.npy index 2d09ca6fa982289a484dd8bfdd77e682ad7cb664..35ff433dd580fd3453cf90b58c18845cfa509828 100644 GIT binary patch delta 91 zcmdlawn=P5GUMj&tO|^i7cj9*-p?$sc|WrXBapCR+Pr{C9wf{Q>UUc`AqY sWCPCV$^SWdCm-O*n0$d_$K(xM>64FhL{8S=RN$IYG9{@r5lH9(0OA`VYXATM literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(@|Jt__h0n~6o47(Pg&T|yqesPIA%Gf|l!ea}U=7;Cnwwu*oQky? zW&&2jQk_m2Ob|=Yo;kA@#G)k)G}{(x2d(J6AUi;Qpao51w}aeGGq;1ngx2mSWIiZK JmL>uTJpeIT;a&g$ diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1_f4.npy index 0298750cc6793c7ce2e5ddf247207b68cb783cd5..87983b36aa0c26b902b0a7cf1d16b840dd09195f 100644 GIT binary patch delta 270 zcmdlawn=P5I^*PSmXgW+EE^`TXDOIGfsti$J!8t`e8z~$3s^ZO%d_l25(@#UO8|=V zY?fz{U<50G2t)%Fu|g%If#Ttlr*dddHsF$&?7+nV#q5*Cxg;2ECkt>&Pqyb`MU#c8 z0?{m!r?NUsHsKP%p>ry$3X&|cP7oVx&t!Qn0T2e-w}DG$vI0<4pGy)+mUXfKmpr%a Nl#(e)rHMd74*<{+N-zKb literal 2866 zcmc&#%WB&|6qS{@E@{?Xbmg5K2wrs&N<$!pEGpuJKo(sTBU=*`8e3*01xypjCJ)2x zeDe+Y3jaa+O|7o&L&Zd6JARM>&Cxw`?zubx}-MiQZ&PM6w9O;Q=e3Es3cWt`y)x8Ta{XThvx@bCcY-iz|= z#~!-RvU%jrV%|*5&Fb(zNuPJd+%ouiGa+{tO?mQWKnsKqYM_~?^afSMR+vO0_Zqg- zYxq-dKd>$9&-3$c`V3}7)|#FH2VyS+n)<*G#yL#5JN7jp_pz4=`E1f->I2Wccz=OY mvLUnSw|U{0^DV|+%$}5w{h%}9wEk)5CST_HFPV7hR{aMEph92( diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2.npy index 7102ca4416ca56928b218592b30eb73968603071..82ae3cfca58903141efdad9d17063d3239e9e951 100644 GIT binary patch delta 122 zcmdlawn=P5JmY3&HW|jr2bfqU&u5ka(VOMjB^W2KXBJp2&xVk=z_DlY1&$c7#`L4?x*GlaF%9a7`(hl2n=q GB=i8S-zqr( delta 363 zcmdlawn=P5JR=(e5H#3NE@Tv&+|S4{xtvje7Bo|X{boUSJw_ls`5?2{;HDHg5vz(ZGgd=A13(fiV z*(Zy03Ig?UK=mphnPUTGvrk^lsW@2#XeLA*h%Y(0gHw63JSQhq46L7PO39R@(nKJk F2LKxtOVt1X literal 2866 zcmdT_O-sW-5Zy*=MLm1))LUARo;-*sf`SKwtq2}GNJ*P55!-tOC(H*dFv9ahgzPA_uohTZBGcbh`rDe8AMeY2>y9N`JG zTX%%Tkv=lpE)RIuHhVl+m&%*P`{Hb|^04^&w!%gs*SmhF*RRx@j^LG!-}bIdAEOf1L$`B0p2#2`_Ji{53j9l4Xka^4N_c-QjwQU%@jJFnK z?hdNe>hV_)qmQxgL1S#8`R)dsS+!Emdtdq{dL~ zV7o9r6=M(k<3Ve}ta-H$%~izyQIV9VJVMf*ntaM74Evmw z&)R?3FRcl~{?qbmeHL@{qw12sdGk>{keii{s-LI#d1^&@Rk_Otx20t-u@Rt@Pf*G$MIbr`3!5pGrTc4urs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSQ97L!iMPyUtN@NVtH)0b5v%iX*}Y zTMEJ!W*~JVO&k_BBhBB$_!n8tC>xSeki!9%U(m&6 zr&9(K1H;)fXZF$w+k!j?w2M~sUa0vrp=oJ0DE#SRKFAJQ0caYs#YKaXWN9Lh&;tN8 C5XDXa diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_3_f4.npy index 6fb6efeb4e54fe781f2f8c87136b573327db6a6f..8ebe6f57dcfe12830fe89e1c66d703123e55e232 100644 GIT binary patch delta 194 zcmdlawn=P5I^*PQ#+u3UEFqKEGx1FBXAA+d6M#5l@_x1hP&SC40>m*;I$^Usiv;83 zdPbOF)a0og`jbVtR3>+DDono0=rDN#rwj`N5KQjjRGvJUQxXeDfJ+@DGkFfD02m7Z kwO?ga0BTj3EW#-=S%H%uNnQjh#=$kEWJ*$LB9PDn0DaFkrvLx| literal 2866 zcmd5-O-lkn7+%-Ru+AMid0PeX)IlN<3_7%^5p?JfvbrN1sVlR)fn?yN4`FV;{f2%; z|G<7z?Wivf)3TPi)xgg4em?Ir!@`WK=atiom~x}sY9?`-R69s(gPN90Yb~3))acf2 zY7!nkHrfvHa3*PAva`Ht6iuMA3!KCqoS zB~82QxYTR9Fb0;GRsu_Q%~{D9bmbn7VhwnNGsx(%tQBVo)9v>NuUBgY7(LBWaTXgu zwOT!S_t$jOOD;429s!c00qjsi$NX!2B!$xI9k jllO5XO`giMXYv7#tqcsmB~4rF_WKgsRQ}EleoEo6vyP_ h9FkBaAhs+}8mJ7Y?FE+tlx@Q`rDRG{X(Eu&0{}$NI2ix{ literal 2866 zcmdT_%WA?v6pgXgV!Q65tL&m6uDXy?5iE4kU@V0$x+oHpk(6p8lcW%;;HD2@cHVwN zzoLI&zp2frLqo_zYAkjjIWys?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(@|Jt__h0n~6o47(Pg&T|yqesPIA%Gf|l!ea}U=7;Cnwwu*oQky? zW&&2jQk_m2Ob|=Yo;kA@#G)k)G}{(x2d(J6AUi;Qpao51w}aeGGq;1ngx2mSWIiZK JmL>uTJpeIT;a&g$ delta 91 zcmdlawn=P5GUMj&tO|^i7cj9*-p?$sc|WrXBapCR+Pr{C9wf{Q>UUc`AqY sWCPCV$^SWdCm-O*n0$d_$K(xM>64FhL{8S=RN$IYG9{@r5lH9(0OA`VYXATM diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1_f4.npy index 87983b36aa0c26b902b0a7cf1d16b840dd09195f..0298750cc6793c7ce2e5ddf247207b68cb783cd5 100644 GIT binary patch literal 2866 zcmc&#%WB&|6qS{@E@{?Xbmg5K2wrs&N<$!pEGpuJKo(sTBU=*`8e3*01xypjCJ)2x zeDe+Y3jaa+O|7o&L&Zd6JARM>&Cxw`?zubx}-MiQZ&PM6w9O;Q=e3Es3cWt`y)x8Ta{XThvx@bCcY-iz|= z#~!-RvU%jrV%|*5&Fb(zNuPJd+%ouiGa+{tO?mQWKnsKqYM_~?^afSMR+vO0_Zqg- zYxq-dKd>$9&-3$c`V3}7)|#FH2VyS+n)<*G#yL#5JN7jp_pz4=`E1f->I2Wccz=OY mvLUnSw|U{0^DV|+%$}5w{h%}9wEk)5CST_HFPV7hR{aMEph92( delta 270 zcmdlawn=P5I^*PSmXgW+EE^`TXDOIGfsti$J!8t`e8z~$3s^ZO%d_l25(@#UO8|=V zY?fz{U<50G2t)%Fu|g%If#Ttlr*dddHsF$&?7+nV#q5*Cxg;2ECkt>&Pqyb`MU#c8 z0?{m!r?NUsHsKP%p>ry$3X&|cP7oVx&t!Qn0T2e-w}DG$vI0<4pGy)+mUXfKmpr%a Nl#(e)rHMd74*<{+N-zKb diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2.npy index 82ae3cfca58903141efdad9d17063d3239e9e951..7102ca4416ca56928b218592b30eb73968603071 100644 GIT binary patch delta 363 zcmdlawn=P5JR=(e5H#3NE@Tv&+|S4{xtvje7Bo|X{boUSJw_ls`5?2{;HDHg5vz(ZGgd=A13(f#`L4?x*GlaF%9a7`(hl2n=q GB=i8S-zqr( diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_2_f4.npy index aa4669e4ece2007d911842d536cbabb190edbd5d..de9a17bdcee9398f483d64e9ea43632f21078429 100644 GIT binary patch literal 2866 zcmdT_O-sW-5Zy*=MLm1))LUARo;-*sf`SKwtq2}GNJ*P55!-tOC(H*dFv9ahgzPA_uohTZBGcbh`rDe8AMeY2>y9N`JG zTX%%Tkv=lpE)RIuHhVl+m&%*P`{Hb|^04^&w!%gs*SmhF*RRx@j^LG!-}bIdAEOf1L$`B0p2#2`_Ji{53j9l4Xka^4N_c-QjwQU%@jJFnK z?hdNe>hV_)qmQxgL1S#8`R)dsS+!Emdtdq{dL~ zV7o9r6=M(k<3Ve}ta-H$%~izyQIV9VJVMf*ntaM74Evmw z&)R?3FRcl~{?qbmeHL@{qw12sdGk>{keii{s-LI#d1^&@Rk_Otx20t-u@Rt@Pf*G$MIbr`3!5pGrTc4uriV z*(Zy03Ig?UK=mphnPUTGvrk^lsW@2#XeLA*h%Y(0gHw63JSQhq46L7PO39R@(nKJk F2LKxtOVt1X diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_3.npy index d9b7e40388bdbd9879c7ade40aebf2562e438246..a283ed1a9eb769952e7e6c7f44657b9af6070c4c 100644 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSQ97L!iMPyUtN@NVtH)0b5v%iX*}Y zTMEJ!W*~JVO&k_BBhBB$_!n8tC>xSeki!9%U(m&6 zr&9(K1H;)fXZF$w+k!j?w2M~sUa0vrp=oJ0DE#SRKFAJQ0caYs#YKaXWN9Lh&;tN8 C5XDXa delta 96 zcmdlawn=P5Hsj{mOe&0<=QAlVPM*&s0OUz9ZobU|6uQl#!MM4AQG{{wY^J!$Q#rIJ vCo|?uHsFkAw4FSGGh*@rPL9bsobfa3*PAva`Ht6iuMA3!KCqoS zB~82QxYTR9Fb0;GRsu_Q%~{D9bmbn7VhwnNGsx(%tQBVo)9v>NuUBgY7(LBWaTXgu zwOT!S_t$jOOD;4m*;I$^Usiv;83 zdPbOF)a0og`jbVtR3>+DDono0=rDN#rwj`N5KQjjRGvJUQxXeDfJ+@DGkFfD02m7Z kwO?ga0BTj3EW#-=S%H%uNnQjh#=$kEWJ*$LB9PDn0DaFkrvLx| diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_4.npy index a4eecd2ed55404747ba9c27127ce0bf7ce6b44ae..4c3d8faa4032b3c70d850c38f657c30b5ddbc5d2 100644 GIT binary patch delta 302 zcmdlawn=P5I^*VSMjxih3z%3YpJ(Qnyq#HKh|&@uKWqm2ff48d0SIG1iwaQGZ}L=* x$&*)bB}`t!RWtbjm&@c0T*;HoIYTF3;NVu@d-lv3TOh_tPbry_RGJ7R^Z=D$b!Y$p delta 86 zcmdlawn=P5I^*VQMis`%^^6jm7chx1Zmwt40TCjMi>29s!c00qjsi$NX!2B!$xI9k jllO5XO`giMXYv7#tqcsmB~4rF_WKgsRQ}EleoEo6vyP_ h9FkBaAhs+}8mJ7Y?FE+tlx@Q`rDRG{X(Eu&0{}$NI2ix{ diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy index 4ae59560a8fe2cc34f2fbc8d61dc0dc1fdb8daf4..893339f537b310b39b3a63a30d8efaddb3e2ebb0 100644 GIT binary patch delta 293 zcmdlawn=P5Hsj>?EF6>HvkK6d7TH|D=*s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5TH#6G}uGTh0!ouV0;)o62)PD zhtVT33_yN{`5#8Z!U37aB{wRL97>~X`h~z0U=7;Cnwwu*oQky?W&&2jQk_m2ObiTX o&z!LZ;=Qz{L7oTMLra=|0X+0M1|`YTL?EFD01uvY&Hw-a diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1_f4.npy index c4628e390913e35519f6320a7a7082801a744798..91454630f910d67b9377bde2d450cb838bbc49b9 100644 GIT binary patch delta 143 zcmdlawn=P5GUMc%j1`m5Gbc=bz?cD|Hy5yIGXgoGKu!d*1W;@Xlz#w9$4#Ef5eyVp tn4H1s0K^KDMYtp;D{y)M#aw`V8z`R#g$+@o0#eU4rDRG{X(Eu&0{{eEGtdA4 delta 381 zcmdlawn=P5GUMd!YzdS9v)!0nz!)?6JR8sC+l<)pYo;8ayayB;Y+_bn1hP4R`USAj zObzyvFEU;N84F^5WE8`s6q%ZU@&QmI#U@YX2v#_I=FDCow#7wHF5r}y+`#EEnTN{; vs9D7ZpJJ#XAOo>ts9_vH16^crDFTUu6o72P>ofuesBldwnUYkR2qg3X7u(K+ diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2.npy index 9d3240149505f4651eaa2a2042a326cdc9aacd18..4cf0ac8fb54b51ecb79e24a7398b54265d19a54e 100644 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVD1A<$rt-94k?*u!O1 zl%63l1z3aju;%8M7N=sZhM9oXuvDj01{1_lXV09m1+i#N(>e%-y8DNE_yEIyFEkxt M(4Zt)ng}HH04|aKod5s; literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2cULnw65AhF- zhWQT0htZ^qPXX4TJ*>I;rNyaOt6?T!H7wQXl)=QnaQ4g@TRP#rPJdOqt8pgNAp^I20S+q1?nHP}!7 z$S#ISDRRhdE?^X41RBIM`8GQ~oP*CoOp_7T0UZ!Ec`8S7%-J(%Yzg4KP#3v_kc}%{*VK0l*2(YRRT+);ObFxn6;j#e+U<%iik|{~0i9kXR0E-*D A82|tP diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_3.npy index 0aa5c5ed4f5772e575391a54bdbe1ca9cee469e7..0a225bc4a1e7900a7b2e0e11a2c0469e310d2d7f 100644 GIT binary patch delta 248 zcmdlawn=P5I^$+u_729)0_?3&qL$H$Y4QT5q{&k`^i$5BIkT4m2xv)9=HZr^tia7d e2ydRsq`){?fLnr)LM-wuTvJM>B$Xxt2|WNTF-(U5 literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u>zQK8TPSWg&+EEZx9p zm>i4`qerSZ$lvH;fF2ewK8zko>PM1aLGeqfe_?52lpcr>m;$UpdsuVxON&#nR>Mrd tYFMh%DT9fAjyrqi%wAez`uSvN+CJ$UrrN6o3pu sp@9lKpf<@Upi83jCnqwtOg_V@05cI}8U_X`QsA0WG9{@r5lH9(0H`Iar2qf` diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_4.npy index 963d2d182fa1a6864a2b045be6b1d7aa7868b493..9e0de61ccbaee5a7892ea73867d447f2e5d483d9 100644 GIT binary patch delta 95 zcmdlawn=P5I^*WKOmd8q7cdEJu4hzW1QIq(n-?%iGlB`Ac+TXh9NLrLa3)Ot!^tt( jhbsmsmN8ie$p69_0T-LV$O2R&!!@O3N>XVekkA7F9^oOS literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u>zQK8TPSWub?_XxbS~ zJLqYG6h0{ZU`s=z;-rM)$W;qVJ1`m+5->iDCS80Aum>cJwQ5Y@>CA( z$;pg4lTUCePyWxrJNW_!&twg*l*v7uHb9eFfZ__1d4RZpQ3fi;!ZoF2N>XVekkA7F DK0PT` delta 345 zcmdlawn=P5I^*QIOgARiGwzwZfGGw@^Gvp8!i6P(Y7#(dH!on4W}N(hkrT)XVQR3S zoX98!WO7VSXB0rA3xIrF<`E+fvUAhqsT|sqbvQL98*r*jUc;p@`3$Fw!r3!tY=L+$ xE_(7sMirn!8=yvINt`NyvMQ4^I8!EH;7D-8WfW8#ry(F`OevX?RGJ7R^Z>i~mT&+7 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py index a1449c35a..d20a90559 100644 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py @@ -135,7 +135,7 @@ def mouse_short_name(view_str): # key_seq is a list of strings, length of FRAME_COUNT, each string is a key in 'W', 'S', 'A', 'D', 'WA', 'WD', 'SA', 'SD' # mouse_seq is a list of strings, length of FRAME_COUNT, each string is a mouse action in 'up', 'down', 'left', 'right', 'up_right', 'up_left', 'down_right', 'down_left' - # Naming: 1=WASDudlr (key: W.npy, SA.npy; camera: u.npy; key+camera: W_u.npy, SA_dl.npy). 2-6=rand names below. + # Naming: 1=WASDudlr (key: W.npy, SA.npy; camera: u.npy; key+camera: W_u.npy, SA_dl.npy). Rand: 1_action=WASD/UDLR+still only, 2_action=full set. 2-6=rand names below. # Group 1: Constant Keyboard, No Mouse. W.npy, S.npy, WA.npy, SA.npy, ... keys_basic = ["W", "S", "A", "D", "WA", "WD", "SA", "SD"] for key in keys_basic: @@ -178,32 +178,32 @@ def mouse_short_name(view_str): keys_basic_still = keys_basic + [""] mouse_basic_still = mouse_basic + [""] - # Group 5: key_1_action_rand (full key set). key_1_action_rand_1..4, key_1_action_rand_1_f4..4_f4 + # Group 5: key_2_action_rand (full key set). key_2_action_rand_1..4, key_2_action_rand_1_f4..4_f4 for granularity in (4, 12): suffix = "_f4" if granularity == 4 else "" for i in range(1, 5): key_seq = build_random_sequence(keys_basic_still, granularity, rng) configs.append( - (f"key_1_action_rand_{i}{suffix}", key_seq, build_constant_sequence("")) + (f"key_2_action_rand_{i}{suffix}", key_seq, build_constant_sequence("")) ) - # Group 6: camera_1_action_rand (full camera set) + # Group 6: camera_2_action_rand (full camera set) for granularity in (4, 12): suffix = "_f4" if granularity == 4 else "" for i in range(1, 5): mouse_seq = build_random_sequence(mouse_basic_still, granularity, rng) configs.append( - (f"camera_1_action_rand_{i}{suffix}", build_constant_sequence(""), mouse_seq) + (f"camera_2_action_rand_{i}{suffix}", build_constant_sequence(""), mouse_seq) ) - # Group 7: key_camera_1_action_rand (both full sets) + # Group 7: key_camera_2_action_rand (both full sets) for granularity in (4, 12): suffix = "_f4" if granularity == 4 else "" for i in range(1, 5): key_seq = build_random_sequence(keys_basic_still, granularity, rng) mouse_seq = build_random_sequence(mouse_basic_still, granularity, rng) configs.append( - (f"key_camera_1_action_rand_{i}{suffix}", key_seq, mouse_seq) + (f"key_camera_2_action_rand_{i}{suffix}", key_seq, mouse_seq) ) # WASD-only (no combined keys) and u/d/l/r-only (no combined directions), with still as option @@ -212,35 +212,35 @@ def mouse_short_name(view_str): keys_wasd_still = keys_wasd_only + [""] mouse_udlr_still = mouse_udlr_only + [""] - # Group 8: key_2_action_rand (WASD+still only) + # Group 8: key_1_action_rand (WASD+still only) for granularity in (4, 12): suffix = "_f4" if granularity == 4 else "" for i in range(1, 5): key_seq = build_random_sequence(keys_wasd_still, granularity, rng) configs.append( - (f"key_2_action_rand_{i}{suffix}", key_seq, build_constant_sequence("")) + (f"key_1_action_rand_{i}{suffix}", key_seq, build_constant_sequence("")) ) - # Group 9: camera_2_action_rand (UDLR+still only) + # Group 9: camera_1_action_rand (UDLR+still only) for granularity in (4, 12): suffix = "_f4" if granularity == 4 else "" for i in range(1, 5): mouse_seq = build_random_sequence(mouse_udlr_still, granularity, rng) configs.append( - (f"camera_2_action_rand_{i}{suffix}", build_constant_sequence(""), mouse_seq) + (f"camera_1_action_rand_{i}{suffix}", build_constant_sequence(""), mouse_seq) ) - # Group 10: key_camera_2_action_rand (WASD+still, UDLR+still) + # Group 10: key_camera_1_action_rand (WASD+still, UDLR+still) for granularity in (4, 12): suffix = "_f4" if granularity == 4 else "" for i in range(1, 5): key_seq = build_random_sequence(keys_wasd_still, granularity, rng) mouse_seq = build_random_sequence(mouse_udlr_still, granularity, rng) configs.append( - (f"key_camera_2_action_rand_{i}{suffix}", key_seq, mouse_seq) + (f"key_camera_1_action_rand_{i}{suffix}", key_seq, mouse_seq) ) - # Group 11a: key_camera_excl_1_action_rand (either key OR camera per block, full key + full camera set) + # Group 11a: key_camera_excl_2_action_rand (either key OR camera per block, full key + full camera set) for granularity in (4, 12): suffix = "_f4" if granularity == 4 else "" for i in range(1, 5): @@ -249,13 +249,13 @@ def mouse_short_name(view_str): (f"key_camera_excl_1_action_rand_{i}{suffix}", key_seq, mouse_seq) ) - # Group 11b: key_camera_excl_2_action_rand (either key OR camera per block, WASD/UDLR+still) + # Group 11b: key_camera_excl_1_action_rand (either key OR camera per block, WASD/UDLR+still) for granularity in (4, 12): suffix = "_f4" if granularity == 4 else "" for i in range(1, 5): key_seq, mouse_seq = build_random_sequence_either_or(keys_wasd_still, mouse_udlr_still, granularity, rng) configs.append( - (f"key_camera_excl_2_action_rand_{i}{suffix}", key_seq, mouse_seq) + (f"key_camera_excl_1_action_rand_{i}{suffix}", key_seq, mouse_seq) ) # Execution diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json new file mode 100644 index 000000000..41e51fc79 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json @@ -0,0 +1,324 @@ +{ + "data": [ + { + "caption": "00 Val-00: W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "01 Val-01: S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "02 Val-02: A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "03 Val-03: D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "04 Val-04: u", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/u.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "05 Val-05: d", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "06 Val-06: l", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/l.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "07 Val-07: r", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/r.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "08 Val-00: key rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "09 Val-01: key rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "10 Val-02: camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "11 Val-03: camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "12 Val-00: key+camera excl rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "13 Val-01: key+camera excl rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "14 Val-02: key+camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "15 Val-03: key+camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "16 Val-04: (simultaneous) key rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "17 Val-05: (simultaneous) camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "18 Val-06: (simultaneous) key+camera excl rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "19 Val-07: (simultaneous) key+camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "20 Val-08: W+A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "21 Val-09: S+u", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_u.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "22 Val-08: Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "23 Val-09: Still", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/still.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "24 Val-06: key+camera excl rand Frame 4", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1_f4.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "25 Val-07: key+camera excl rand Frame 4", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2_f4.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "26 Train-00", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/first_frame/000500.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/videos/000500_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "27 Train-01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/first_frame/001000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/videos/001000_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "28 Doom-00: W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "29 Doom-01: key rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "30 Doom-02: camera rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "31 Doom-03: key+camera excl rand", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + } + ] +} \ No newline at end of file From abc026bbab8e92a5c1c8d3807589b6e116abd2c1 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Mon, 9 Feb 2026 21:12:04 +0000 Subject: [PATCH 014/214] compute correct trainable params --- fastvideo/training/training_pipeline.py | 33 ++++++++++++++----- fastvideo/training/training_utils.py | 18 ++++++++++ .../training/wangame_training_pipeline.py | 10 ++++++ 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index c4f16460b..e51eab670 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -42,9 +42,9 @@ initialize_trackers, Trackers) from fastvideo.training.training_utils import ( clip_grad_norm_while_handling_failing_dtensor_cases, - compute_density_for_timestep_sampling, count_trainable, get_scheduler, - get_sigmas, load_checkpoint, normalize_dit_input, save_checkpoint, - shard_latents_across_sp) + compute_density_for_timestep_sampling, count_trainable, + count_trainable_total, get_scheduler, get_sigmas, load_checkpoint, + normalize_dit_input, save_checkpoint, shard_latents_across_sp) from fastvideo.utils import (is_vmoba_available, is_vsa_available, set_random_seed, shallow_asdict) @@ -582,15 +582,30 @@ def train(self) -> None: local_main_process_only=False) if not self.post_init_called: self.post_init() - num_trainable_params = count_trainable(self.transformer) - logger.info("Starting training with %s B trainable parameters", - round(num_trainable_params / 1e9, 3)) + local_trainable = count_trainable(self.transformer) + total_trainable = count_trainable_total( + self.transformer, + get_local_torch_device(), + ) + logger.info( + "Starting training with %s B trainable parameters (total); " + "this rank shard: %s B", + round(total_trainable / 1e9, 3), + round(local_trainable / 1e9, 3), + ) if getattr(self, "transformer_2", None) is not None: - num_trainable_params = count_trainable(self.transformer_2) + local_trainable_2 = count_trainable(self.transformer_2) + total_trainable_2 = count_trainable_total( + self.transformer_2, + get_local_torch_device(), + ) logger.info( - "Transformer 2: Starting training with %s B trainable parameters", - round(num_trainable_params / 1e9, 3)) + "Transformer 2: %s B trainable parameters (total); " + "this rank shard: %s B", + round(total_trainable_2 / 1e9, 3), + round(local_trainable_2 / 1e9, 3), + ) # Set random seeds for deterministic training self.noise_random_generator = torch.Generator(device="cpu").manual_seed( diff --git a/fastvideo/training/training_utils.py b/fastvideo/training/training_utils.py index 321bfb339..a0deb3621 100644 --- a/fastvideo/training/training_utils.py +++ b/fastvideo/training/training_utils.py @@ -1630,9 +1630,27 @@ def _local_numel(p: torch.Tensor) -> int: def count_trainable(model: torch.nn.Module) -> int: + """Return this rank's trainable parameter count (FSDP local shard).""" return sum(_local_numel(p) for p in model.parameters() if p.requires_grad) +def count_trainable_total( + model: torch.nn.Module, + device: torch.device | None = None, +) -> int: + """Return total trainable parameter count across all ranks (FSDP-safe). + + When device is provided and dist is initialized, torch.distributed.all_reduce(SUM) + with the default world group is used. Otherwise returns local count. + """ + local = count_trainable(model) + if device is not None and dist.is_initialized(): + t = torch.tensor([local], dtype=torch.long, device=device) + dist.all_reduce(t, op=dist.ReduceOp.SUM) + return t.item() + return local + + class EMA_FSDP: """ FSDP2-friendly EMA with two modes: diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index 6a178d676..8cb6f491f 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -17,6 +17,7 @@ from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import WanGameActionImageToVideoPipeline from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch from fastvideo.training.training_pipeline import TrainingPipeline +from fastvideo.training.training_utils import count_trainable, count_trainable_total from fastvideo.utils import is_vsa_available, shallow_asdict vsa_available = is_vsa_available() @@ -114,9 +115,18 @@ def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: if step == 1: # Freeze action params at the very first step self._set_action_params_grad(False) + local_trainable = count_trainable(self.transformer) + total_trainable = count_trainable_total( + self.transformer, get_local_torch_device() + ) logger.info( "Action warmup: freezing action modules for the first " "%d steps to stabilize base model", warmup_steps) + logger.info( + "Trainable during warmup: %s B (total); this rank shard: %s B", + round(total_trainable / 1e9, 3), + round(local_trainable / 1e9, 3), + ) elif step == warmup_steps + 1: # Unfreeze action params once warmup is done self._set_action_params_grad(True) From d70da3b4fbae34ae98b6f48a52197ac5ca7603bc Mon Sep 17 00:00:00 2001 From: mignonjia Date: Mon, 9 Feb 2026 21:12:51 +0000 Subject: [PATCH 015/214] allow multiple data path --- .../finetune_wangame.slurm | 54 +++++--- .../dataset/parquet_dataset_map_style.py | 131 +++++++++++++++--- fastvideo/fastvideo_args.py | 2 +- fastvideo/models/dits/wangame/model.py | 2 +- 4 files changed, 146 insertions(+), 43 deletions(-) diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm index bdb0a34b6..a6051f192 100644 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm @@ -17,7 +17,7 @@ export TORCH_NCCL_ENABLE_MONITORING=0 export NCCL_DEBUG_SUBSYS=INIT,NET # different cache dir for different processes export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} -export MASTER_PORT=29500 +export MASTER_PORT=29501 export NODE_RANK=$SLURM_PROCID nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) export MASTER_ADDR=${nodes[0]} @@ -26,6 +26,7 @@ export WANDB_API_KEY="d5b02b05e30d8cb34c7b31c6ae10416fc26dcb66" export WANDB_BASE_URL="https://api.wandb.ai" export WANDB_MODE=online export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN +export FASTVIDEO_MAP_STYLE_CACHE_DIR="/mnt/weka/home/hao.zhang/mhuo/FastVideo/map_style_cache" source ~/conda/miniconda/bin/activate conda activate mhuo-fv @@ -37,27 +38,44 @@ NUM_NODES=4 NUM_TOTAL_GPUS=$((NUM_GPUS * NUM_NODES)) BS_PER_GPU=1 GRADIENT_ACCUMULATION_STEPS=1 -WANDB_RUN_NAME="MC_wsad_random_lr_1e-5" +WANDB_RUN_NAME="MC_1action_rand_from_scratch" FREEZE_DIT=False -RUN_DIR="wangame_1.3b_wsad_random_lr_1e-5" +RUN_DIR="wangame_1.3b_1action_rand_from_scratch" CHECKPOINTING_STEPS=1000 ACTION_WARMUP_STEPS=0 LEARNING_RATE=1e-5 MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -CKPT_SAFETENSOR="wangame_1.3b_with_warmup_lr_1e-5/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" -# MC wasd only -# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json" - -# MC random -DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed" -VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" - -# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_doom.json" -# DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_overfit.json" +# CKPT_SAFETENSOR="wangame_1.3b_with_warmup_lr_1e-5/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" +# +# Data dirs (use one of the following): +# - DATA_DIR_ALL: all datasets below combined (comma-separated) +# - Or a single path / subset; optional ":N" = repeat, ":0" = skip +# +DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0" # Random +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0" # Doom +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1" # Static + w only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1" # w/s/a/d only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1" # wasd only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1" # camera l-only and r-only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1" # camera only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1" # key_camera_excl_1_action_rand + +VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json" +# +# Single-dir / validation alternatives (comment out DATA_DIR above and uncomment one block): +# MC wasd only: +# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json" +# MC random: +# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" +# Doom: +# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_doom.json" +# Overfit: +# DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_overfit.json" # Training arguments @@ -65,7 +83,7 @@ training_args=( --tracker_project_name "wangame_1.3b" --output_dir $RUN_DIR --wandb_run_name "$WANDB_RUN_NAME" - --max_train_steps 10000 + --max_train_steps 20000 --train_batch_size $BS_PER_GPU --train_sp_batch_size $BS_PER_GPU --gradient_accumulation_steps $GRADIENT_ACCUMULATION_STEPS @@ -76,7 +94,7 @@ training_args=( --enable_gradient_checkpointing_type "full" --train_action_only $FREEZE_DIT --action_warmup_steps $ACTION_WARMUP_STEPS - --init_weights_from_safetensors $CKPT_SAFETENSOR + # --init_weights_from_safetensors $CKPT_SAFETENSOR ) # Parallel arguments diff --git a/fastvideo/dataset/parquet_dataset_map_style.py b/fastvideo/dataset/parquet_dataset_map_style.py index 866faa51f..1ff9d3dc3 100644 --- a/fastvideo/dataset/parquet_dataset_map_style.py +++ b/fastvideo/dataset/parquet_dataset_map_style.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +import hashlib import os import pickle import random @@ -106,10 +107,90 @@ def __len__(self): return len(self.sp_group_local_indices) // self.batch_size +def _parse_data_path_specs(path: str) -> list[tuple[str, int]]: + """ + Parse data_path into a list of (directory, repeat_count). + Syntax: comma-separated entries; each entry is "path" (default 1) or "path:N" (N = repeat count). + N=0 means skip that path (convenience to disable without removing). If no ":" present, default is 1. + Example: "/dir1:2,/dir2,/dir3:0" -> dir1 2x, dir2 1x, dir3 skipped. + """ + specs: list[tuple[str, int]] = [] + for part in path.split(","): + part = part.strip() + if not part: + continue + if ":" in part: + p, _, count_str = part.rpartition(":") + p = p.strip() + try: + count = int(count_str.strip()) + except ValueError: + raise ValueError( + f"data_path repeat count must be an integer, got {count_str!r}" + ) from None + if count < 0: + raise ValueError( + f"data_path repeat count must be >= 0, got {count}" + ) + specs.append((p, count)) + else: + specs.append((part, 1)) + return specs + + +def _scan_parquet_files_for_path(p: str) -> tuple[list[str], list[int]]: + """Return (file_paths, row_lengths) for a single directory.""" + file_names: list[str] = [] + for root, _, files in os.walk(p): + for file in sorted(files): + if file.endswith(".parquet"): + file_names.append(os.path.join(root, file)) + lengths = [] + for file_path in tqdm.tqdm( + file_names, desc="Reading parquet files to get lengths"): + lengths.append(pq.ParquetFile(file_path).metadata.num_rows) + logger.info("Found %d parquet files with %d total rows", len(file_names), sum(lengths)) + return file_names, lengths + + def get_parquet_files_and_length(path: str): - # Check if cached info exists - cache_dir = os.path.join(path, "map_style_cache") - cache_file = os.path.join(cache_dir, "file_info.pkl") + """ + Collect parquet file paths and row lengths from one or more directories. + path: single directory, or comma-separated "path" or "path:N" (N = repeat count). + E.g. "/dir1:2,/dir2:1" -> dir1's files appear 2x (oversampled), dir2 once. + """ + path_specs = _parse_data_path_specs(path) + if not path_specs: + raise ValueError( + "data_path must be a non-empty path or comma-separated path specs" + ) + # Use first path with count > 0 for cache_dir (single-path case only) + first_path = next( + (p for p, c in path_specs if c > 0), + path_specs[0][0], + ) + is_single_no_repeat = ( + len(path_specs) == 1 and path_specs[0][1] == 1 + ) + effective_path = path.strip() + # Single path, no repeat: cache under that path (backward compatible). + # Multi-path or repeat: cache in a neutral dir keyed by hash of full path spec, + # so we never reuse "first path's" cache and the cached list is the merged list. + if is_single_no_repeat: + cache_dir = os.path.join(first_path, "map_style_cache") + cache_suffix = "file_info.pkl" + else: + neutral_root = os.environ.get( + "FASTVIDEO_MAP_STYLE_CACHE_DIR", + os.path.join(os.path.expanduser("~"), ".cache", "fastvideo", "map_style_cache"), + ) + cache_dir = neutral_root + cache_suffix = ( + "file_info_" + + hashlib.md5(effective_path.encode()).hexdigest()[:16] + + ".pkl" + ) + cache_file = os.path.join(cache_dir, cache_suffix) # Only rank 0 checks for cache and scans files if needed if get_world_rank() == 0: @@ -132,26 +213,30 @@ def get_parquet_files_and_length(path: str): # If cache not loaded (either doesn't exist or failed to load), scan files if not cache_loaded: - logger.info("Scanning parquet files to get lengths") - lengths = [] - file_names = [] - for root, _, files in os.walk(path): - for file in sorted(files): - if file.endswith('.parquet'): - file_path = os.path.join(root, file) - file_names.append(file_path) - for file_path in tqdm.tqdm( - file_names, desc="Reading parquet files to get lengths"): - num_rows = pq.ParquetFile(file_path).metadata.num_rows - lengths.append(num_rows) - # sort according to file name to ensure all rank has the same order - file_names_sorted, lengths_sorted = zip(*sorted(zip(file_names, - lengths, - strict=True), - key=lambda x: x[0]), - strict=True) - assert len( - file_names_sorted) != 0, "No parquet files found in the dataset" + logger.info( + "Scanning parquet files (path specs: %s)", + [(p, c) for p, c in path_specs], + ) + # Build list with repeats; use (path, length, sort_index) for stable order + # Skip paths with count 0 (no I/O for disabled paths) + combined: list[tuple[str, int, int]] = [] + sort_index = 0 + for p, count in path_specs: + if count == 0: + continue + fnames, lens = _scan_parquet_files_for_path(p) + for _ in range(count): + for f, ln in zip(fnames, lens, strict=True): + combined.append((f, ln, sort_index)) + sort_index += 1 + if not combined: + raise ValueError( + "No parquet files found in the dataset (paths: %s)" + % [p for p, _ in path_specs] + ) + combined.sort(key=lambda x: (x[0], x[2])) + file_names_sorted = tuple(x[0] for x in combined) + lengths_sorted = tuple(x[1] for x in combined) # Save the cache os.makedirs(cache_dir, exist_ok=True) diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index 4b0db30c1..896c40e77 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -1006,7 +1006,7 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: parser.add_argument("--data-path", type=str, required=True, - help="Path to parquet files") + help="Path to parquet files (comma-separated for multiple; path:N for repeat count)") parser.add_argument("--dataloader-num-workers", type=int, required=True, diff --git a/fastvideo/models/dits/wangame/model.py b/fastvideo/models/dits/wangame/model.py index ac6331a71..61f539821 100644 --- a/fastvideo/models/dits/wangame/model.py +++ b/fastvideo/models/dits/wangame/model.py @@ -174,7 +174,7 @@ def forward( key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - # Self-attention with optional camera PRoPE + # Self-attention with camera PRoPE attn_output_rope, attn_output_prope = self.attn1( query, key, value, freqs_cis, kv_cache, current_start, cache_start, viewmats, Ks, From fb219f82b76575eeac8c975de0cc8137c5df2ca2 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Mon, 9 Feb 2026 14:44:20 -0800 Subject: [PATCH 016/214] update inference and wangame lingtbot --- examples/inference/basic/basic_wangame.py | 46 +++++ .../inference/basic/basic_wangame_lingbot.py | 46 +++++ .../action/000000_action.npy | Bin 0 -> 3030 bytes .../action/000001_action.npy | Bin 0 -> 3030 bytes .../action/000002_action.npy | Bin 0 -> 3030 bytes .../action/000003_action.npy | Bin 0 -> 3030 bytes .../action/000004_action.npy | Bin 0 -> 3030 bytes .../action/000005_action.npy | Bin 0 -> 3030 bytes .../action/000006_action.npy | Bin 0 -> 3030 bytes .../action/000007_action.npy | Bin 0 -> 3030 bytes .../action/000008_action.npy | Bin 0 -> 3030 bytes .../action/000009_action.npy | Bin 0 -> 3030 bytes .../action/000010_action.npy | Bin 0 -> 3030 bytes .../action/000011_action.npy | Bin 0 -> 3030 bytes .../action/000012_action.npy | Bin 0 -> 3030 bytes .../action/000013_action.npy | Bin 0 -> 3030 bytes .../action/000014_action.npy | Bin 0 -> 3030 bytes .../action/000015_action.npy | Bin 0 -> 3030 bytes .../action/000016_action.npy | Bin 0 -> 3030 bytes .../action/000017_action.npy | Bin 0 -> 3030 bytes .../action/000018_action.npy | Bin 0 -> 3030 bytes .../action/000019_action.npy | Bin 0 -> 3030 bytes .../action/000020_action.npy | Bin 0 -> 3030 bytes .../action/000021_action.npy | Bin 0 -> 3030 bytes .../action/000022_action.npy | Bin 0 -> 3030 bytes .../action/000023_action.npy | Bin 0 -> 3030 bytes .../action/000024_action.npy | Bin 0 -> 3030 bytes .../action/000025_action.npy | Bin 0 -> 3030 bytes .../action/000026_action.npy | Bin 0 -> 3030 bytes .../action/000027_action.npy | Bin 0 -> 3030 bytes .../action/000028_action.npy | Bin 0 -> 3030 bytes .../action/000029_action.npy | Bin 0 -> 3030 bytes .../action/000030_action.npy | Bin 0 -> 3030 bytes .../action/000031_action.npy | Bin 0 -> 3030 bytes .../action/000032_action.npy | Bin 0 -> 3030 bytes .../action/000033_action.npy | Bin 0 -> 3030 bytes .../action/000034_action.npy | Bin 0 -> 3030 bytes .../action/000035_action.npy | Bin 0 -> 3030 bytes .../action/000036_action.npy | Bin 0 -> 3030 bytes .../action/000037_action.npy | Bin 0 -> 3030 bytes .../action/000038_action.npy | Bin 0 -> 3030 bytes .../action/000039_action.npy | Bin 0 -> 3030 bytes .../action/000040_action.npy | Bin 0 -> 3030 bytes .../action/000041_action.npy | Bin 0 -> 3030 bytes .../action/000042_action.npy | Bin 0 -> 3030 bytes .../action/000043_action.npy | Bin 0 -> 3030 bytes .../action/000044_action.npy | Bin 0 -> 3030 bytes .../action/000045_action.npy | Bin 0 -> 3030 bytes .../action/000046_action.npy | Bin 0 -> 3030 bytes .../action/000047_action.npy | Bin 0 -> 3030 bytes .../action/000048_action.npy | Bin 0 -> 3030 bytes .../action/000049_action.npy | Bin 0 -> 3030 bytes .../action/000050_action.npy | Bin 0 -> 3030 bytes .../action/000051_action.npy | Bin 0 -> 3030 bytes .../action/000052_action.npy | Bin 0 -> 3030 bytes .../action/000053_action.npy | Bin 0 -> 3030 bytes .../action/000054_action.npy | Bin 0 -> 3030 bytes .../action/000055_action.npy | Bin 0 -> 3030 bytes .../action/000056_action.npy | Bin 0 -> 3030 bytes .../action/000057_action.npy | Bin 0 -> 3030 bytes .../action/000058_action.npy | Bin 0 -> 3030 bytes .../action/000059_action.npy | Bin 0 -> 3030 bytes .../action/000060_action.npy | Bin 0 -> 3030 bytes .../action/000061_action.npy | Bin 0 -> 3030 bytes .../action/000062_action.npy | Bin 0 -> 3030 bytes .../action/000063_action.npy | Bin 0 -> 3030 bytes .../action/README.md | 18 ++ .../doom/000000.jpg | Bin 0 -> 44127 bytes .../doom/000001.jpg | Bin 0 -> 27703 bytes .../doom/000002.jpg | Bin 0 -> 28997 bytes .../doom/000003.jpg | Bin 0 -> 37116 bytes .../doom/000004.jpg | Bin 0 -> 18463 bytes .../doom/000005.jpg | Bin 0 -> 23782 bytes .../doom/000006.jpg | Bin 0 -> 24237 bytes .../doom/000007.jpg | Bin 0 -> 24609 bytes .../finetune_i2v.sh | 102 +++++++++ .../finetune_i2v.slurm | 120 +++++++++++ .../generate_actions.py | 193 ++++++++++++++++++ .../humanplay/000000.jpg | Bin 0 -> 31367 bytes .../humanplay/000000_action.npy | Bin 0 -> 2866 bytes .../humanplay/000001.jpg | Bin 0 -> 31098 bytes .../humanplay/000001_action.npy | Bin 0 -> 2866 bytes .../humanplay/000002.jpg | Bin 0 -> 32192 bytes .../humanplay/000002_action.npy | Bin 0 -> 2866 bytes .../humanplay/000003.jpg | Bin 0 -> 32308 bytes .../humanplay/000003_action.npy | Bin 0 -> 2866 bytes .../humanplay/000004.jpg | Bin 0 -> 29376 bytes .../humanplay/000004_action.npy | Bin 0 -> 2866 bytes .../humanplay/000005.jpg | Bin 0 -> 35341 bytes .../humanplay/000005_action.npy | Bin 0 -> 2866 bytes .../humanplay/000006.jpg | Bin 0 -> 32213 bytes .../humanplay/000006_action.npy | Bin 0 -> 2866 bytes .../humanplay/000007.jpg | Bin 0 -> 34252 bytes .../humanplay/000007_action.npy | Bin 0 -> 2866 bytes .../humanplay/000008.jpg | Bin 0 -> 34836 bytes .../humanplay/000008_action.npy | Bin 0 -> 2866 bytes .../humanplay/000009.jpg | Bin 0 -> 32197 bytes .../humanplay/000009_action.npy | Bin 0 -> 2866 bytes .../humanplay/000010.jpg | Bin 0 -> 26205 bytes .../humanplay/000010_action.npy | Bin 0 -> 2866 bytes .../humanplay/000011.jpg | Bin 0 -> 29045 bytes .../humanplay/000011_action.npy | Bin 0 -> 2866 bytes .../humanplay/000012.jpg | Bin 0 -> 29384 bytes .../humanplay/000012_action.npy | Bin 0 -> 2866 bytes .../humanplay/000013.jpg | Bin 0 -> 28293 bytes .../humanplay/000013_action.npy | Bin 0 -> 2866 bytes .../humanplay/000014.jpg | Bin 0 -> 29096 bytes .../humanplay/000014_action.npy | Bin 0 -> 2866 bytes .../humanplay/000015.jpg | Bin 0 -> 32846 bytes .../humanplay/000015_action.npy | Bin 0 -> 2866 bytes .../humanplay/000016.jpg | Bin 0 -> 28131 bytes .../humanplay/000016_action.npy | Bin 0 -> 2866 bytes .../humanplay/000017.jpg | Bin 0 -> 31577 bytes .../humanplay/000017_action.npy | Bin 0 -> 2866 bytes .../humanplay/000018.jpg | Bin 0 -> 31892 bytes .../humanplay/000018_action.npy | Bin 0 -> 2866 bytes .../humanplay/000019.jpg | Bin 0 -> 31525 bytes .../humanplay/000019_action.npy | Bin 0 -> 2866 bytes .../humanplay/000020.jpg | Bin 0 -> 33493 bytes .../humanplay/000020_action.npy | Bin 0 -> 2866 bytes .../humanplay/000021.jpg | Bin 0 -> 29167 bytes .../humanplay/000021_action.npy | Bin 0 -> 2866 bytes .../humanplay/000022.jpg | Bin 0 -> 27977 bytes .../humanplay/000022_action.npy | Bin 0 -> 2866 bytes .../humanplay/000023.jpg | Bin 0 -> 30427 bytes .../humanplay/000023_action.npy | Bin 0 -> 2866 bytes .../humanplay/000024.jpg | Bin 0 -> 30925 bytes .../humanplay/000024_action.npy | Bin 0 -> 2866 bytes .../humanplay/000025.jpg | Bin 0 -> 19554 bytes .../humanplay/000025_action.npy | Bin 0 -> 2866 bytes .../humanplay/000026.jpg | Bin 0 -> 30925 bytes .../humanplay/000026_action.npy | Bin 0 -> 2866 bytes .../humanplay/000027.jpg | Bin 0 -> 34454 bytes .../humanplay/000027_action.npy | Bin 0 -> 2866 bytes .../humanplay/000028.jpg | Bin 0 -> 30879 bytes .../humanplay/000028_action.npy | Bin 0 -> 2866 bytes .../humanplay/000029.jpg | Bin 0 -> 27994 bytes .../humanplay/000029_action.npy | Bin 0 -> 2866 bytes .../humanplay/000030.jpg | Bin 0 -> 29098 bytes .../humanplay/000030_action.npy | Bin 0 -> 2866 bytes .../humanplay/000031.jpg | Bin 0 -> 33151 bytes .../humanplay/000031_action.npy | Bin 0 -> 2866 bytes .../humanplay/000032.jpg | Bin 0 -> 30016 bytes .../humanplay/000032_action.npy | Bin 0 -> 2866 bytes .../humanplay/000033.jpg | Bin 0 -> 31795 bytes .../humanplay/000033_action.npy | Bin 0 -> 2866 bytes .../humanplay/000034.jpg | Bin 0 -> 30287 bytes .../humanplay/000034_action.npy | Bin 0 -> 2866 bytes .../humanplay/000035.jpg | Bin 0 -> 35676 bytes .../humanplay/000035_action.npy | Bin 0 -> 2866 bytes .../humanplay/000036.jpg | Bin 0 -> 29895 bytes .../humanplay/000036_action.npy | Bin 0 -> 2866 bytes .../humanplay/000037.jpg | Bin 0 -> 29044 bytes .../humanplay/000037_action.npy | Bin 0 -> 2866 bytes .../humanplay/000038.jpg | Bin 0 -> 27844 bytes .../humanplay/000038_action.npy | Bin 0 -> 2866 bytes .../launch_preprocess_slurm.sh | 20 ++ .../preprocess_wangame_data_i2v.sh | 27 +++ .../preprocess_worker.slurm | 61 ++++++ .../validation.json | 14 ++ .../validation_vizdoom.json | 84 ++++++++ fastvideo/configs/models/dits/__init__.py | 5 +- fastvideo/configs/models/dits/wangamevideo.py | 7 + fastvideo/configs/pipelines/__init__.py | 3 + fastvideo/configs/pipelines/registry.py | 8 +- fastvideo/configs/pipelines/wan.py | 16 ++ fastvideo/configs/sample/registry.py | 6 + fastvideo/dataset/dataloader/schema.py | 4 +- fastvideo/layers/visual_embedding.py | 57 +++++- .../models/dits/wangame_lingbot/__init__.py | 5 + fastvideo/models/loader/component_loader.py | 3 + fastvideo/models/registry.py | 1 + fastvideo/pipelines/pipeline_registry.py | 1 + 173 files changed, 843 insertions(+), 4 deletions(-) create mode 100644 examples/inference/basic/basic_wangame.py create mode 100644 examples/inference/basic/basic_wangame_lingbot.py create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000000_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000001_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000002_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000003_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000004_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000005_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000006_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000007_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000008_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000009_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000010_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000011_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000012_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000013_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000014_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000015_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000016_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000017_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000018_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000019_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000020_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000021_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000022_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000023_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000024_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000025_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000026_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000027_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000028_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000029_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000030_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000031_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000032_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000033_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000034_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000035_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000036_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000037_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000038_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000039_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000040_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000041_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000042_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000043_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000044_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000045_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000046_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000047_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000048_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000049_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000050_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000051_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000052_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000053_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000054_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000055_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000056_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000057_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000058_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000059_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000060_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000061_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000062_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000063_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/README.md create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000000.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000001.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000002.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000003.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000004.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000005.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000006.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000007.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.sh create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.slurm create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/generate_actions.py create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000000.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000000_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000001.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000001_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000002.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000002_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000003.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000003_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000004.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000004_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000005.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000005_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000006.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000006_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000007.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000007_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000008.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000008_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000009.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000009_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000010.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000010_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000011.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000011_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000012.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000012_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000013.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000013_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000014.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000014_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000015.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000015_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000016.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000016_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000017.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000017_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000018.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000018_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000019.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000019_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000020.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000020_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000021.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000021_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000022.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000022_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000023.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000023_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000024.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000024_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000025.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000025_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000026.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000026_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000027.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000027_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000028.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000028_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000029.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000029_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000030.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000030_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000031.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000031_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000032.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000032_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000033.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000033_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000034.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000034_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000035.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000035_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000036.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000036_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000037.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000037_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000038.jpg create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000038_action.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/launch_preprocess_slurm.sh create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_wangame_data_i2v.sh create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation.json create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation_vizdoom.json create mode 100644 fastvideo/models/dits/wangame_lingbot/__init__.py diff --git a/examples/inference/basic/basic_wangame.py b/examples/inference/basic/basic_wangame.py new file mode 100644 index 000000000..8ce3fa36c --- /dev/null +++ b/examples/inference/basic/basic_wangame.py @@ -0,0 +1,46 @@ +from fastvideo import VideoGenerator +from fastvideo.configs.pipelines import WanGameI2V480PConfig +from fastvideo.models.dits.matrixgame.utils import create_action_presets + +BASE_MODEL_PATH = "weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" +WEIGHTS_PATH = "wangame_1.3b_overfit/checkpoint-10000/transformer/diffusion_pytorch_model.safetensors" + +OUTPUT_PATH = "video_samples_wangame" +IMAGE_PATH = "https://raw.githubusercontent.com/SkyworkAI/Matrix-Game/main/Matrix-Game-2/demo_images/universal/0000.png" + + +def main(): + generator = VideoGenerator.from_pretrained( + BASE_MODEL_PATH, + pipeline_config=WanGameI2V480PConfig(), + num_gpus=1, + use_fsdp_inference=False, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=True, + pin_cpu_memory=True, + override_pipeline_cls_name="WanGameActionImageToVideoPipeline", + override_transformer_cls_name="WanGameActionTransformer3DModel", + init_weights_from_safetensors=WEIGHTS_PATH, + ) + + num_frames = 77 + actions = create_action_presets(num_frames, keyboard_dim=4) + + generator.generate_video( + prompt="", + image_path=IMAGE_PATH, + mouse_cond=actions["mouse"].unsqueeze(0), + keyboard_cond=actions["keyboard"].unsqueeze(0), + num_frames=num_frames, + height=352, + width=640, + num_inference_steps=40, + guidance_scale=1.0, + output_path=OUTPUT_PATH, + save_video=True, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/inference/basic/basic_wangame_lingbot.py b/examples/inference/basic/basic_wangame_lingbot.py new file mode 100644 index 000000000..b30d0f562 --- /dev/null +++ b/examples/inference/basic/basic_wangame_lingbot.py @@ -0,0 +1,46 @@ +from fastvideo import VideoGenerator +from fastvideo.configs.pipelines import WanLingBotI2V480PConfig +from fastvideo.models.dits.matrixgame.utils import create_action_presets + +BASE_MODEL_PATH = "weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" +WEIGHTS_PATH = "wangame_lingbot_test/checkpoint-100/transformer/diffusion_pytorch_model.safetensors" + +OUTPUT_PATH = "video_samples_wangame_lingbot" +IMAGE_PATH = "https://raw.githubusercontent.com/SkyworkAI/Matrix-Game/main/Matrix-Game-2/demo_images/universal/0000.png" + + +def main(): + generator = VideoGenerator.from_pretrained( + BASE_MODEL_PATH, + pipeline_config=WanLingBotI2V480PConfig(), + num_gpus=1, + use_fsdp_inference=False, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=True, + pin_cpu_memory=True, + override_pipeline_cls_name="WanLingBotImageToVideoPipeline", + override_transformer_cls_name="WanLingBotTransformer3DModel", + init_weights_from_safetensors=WEIGHTS_PATH, + ) + + num_frames = 77 + actions = create_action_presets(num_frames, keyboard_dim=4) + + generator.generate_video( + prompt="", + image_path=IMAGE_PATH, + mouse_cond=actions["mouse"].unsqueeze(0), + keyboard_cond=actions["keyboard"].unsqueeze(0), + num_frames=num_frames, + height=352, + width=640, + num_inference_steps=40, + guidance_scale=1.0, + output_path=OUTPUT_PATH, + save_video=True, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000000_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000000_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..a8c753691c88c9e88b8e28750e44cf662dbc2ffd GIT binary patch literal 3030 zcmeHJ%SyvQ6rHrz*7$zZ_am;7Lg~tdh)}5D!eFWaAugokF(qP~^iC#1u!U}Ho3$JN zRws>xic4L%n6tPrXD)N#aAxkjX?NE*_X2uGFS3IJo5*L9+^x%HNpAXt64Pt=#K8%D zX|@8)_`o$gn5~skS?Wo@gV@j7UfAwL727Ac(hghn(Imu-#0J5(?*){EHWdM-U83it zyCA6O6*3os1=%6Y=#W!Xq4d?{(5Qn&m^Cs8q53K0g)j4+*G1@0_%yyh7QGv z#5rt!mp~M@thn}`XNRMHHK{+r`P38c1NUM2cJMs#Jn%eBJHYcW?Z)4{53o&P$1-y4 vZGyIp7r2i-eHzEAM@bxm~TF`hugbH0llI(*};KL}PE+Y=5R3$SJB&`YCc~)WIUm8ySR9{T%Y~Xb1%z78EsW2#o83sA<~G7sGf^ zs}vTk_z9MJu&fvbR`P-)%QuMwD=u=!VyiAwTGL^jDXI7B`~WuGRHAtXn~B7h4#kPY zC2Y@1APPHHT>H$k%g5+fC#%_+^W2~5MDRTDJn%eBJHYcW?FN4z{wfc!OJUD4a_qjg t9|joqC%yk5>HUW~9HqTKGk{~a7}rM02ZMt<@DCh-6AF?QniNj0$~TSAy5s-= literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000002_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000002_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..a3fe51f4826b49dedc7c52456d2b47bbef2203fb GIT binary patch literal 3030 zcmeHJOKZYF5Z-95Hhn+r`;nd!K|J+PNoi^x}!+^NY$Np5(A62q-~#Ktjw zYcze#c;7MFnC+E9QR+(bLF}b1H)yrPlI0OxY6VUDY7kwg!BLwG{`8bQ20x-Y1qahEb1wQQ0)@3(Qp7c4VDx&tqY9nfLN_o?|v9Y zi(00zY(_7z(uGw;&#{y@9Gad%Y*=%UGZI^On9_y@n@mZ4R2TcO5IYpf%gONhiM0RKTNwZ-}?aj6b?*1 x!z#ACz{hYnp8ZGh>_67vB$@rGKAbxFs4|Q{7#!Szf8YR|QIO2Qpm1)MegT{Wy5s-= literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000003_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000003_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..3431d7e9a9ab9ffefc919c24442037d09b6b0d99 GIT binary patch literal 3030 zcmeHJOH0E*5Z<)b*7$zZ_amN?Lg~qah)}5D!C9C-v>6*ZD9S}85yZK@m zEozyx;L!98V#A7qoDtco!n8H^bnxyGpFwmi;!i_1X5DY<2b zU63^L2C+-Yg!mBVbVy`1Dfc~O^G=(}kgUcCA(b;ojYmUB>oA|y;#Em#U69M=^7W^v z$)OcFESTd*SnR=4wwe|r&$MH@2D4$=AZh;~SZwN$ zL5mC6`e{KpY@6fKM@n2y`o-X!V#uZ(`;m?bgowEyVi!4Lq+Pi- z1VY4I5V4CKG14)CD7V=p@O>a+4>{uB>afFM*Q_SQS2SJUqp*j&e?Qp$2Ra;vyFWI7 cBPTO1je;MHsG~2hK{0)U!--k=0{-RqQvd(} literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000005_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000005_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..9343858c4eac53757538e0447d3a8cb5cc0b73e1 GIT binary patch literal 3030 zcmeH}$xFjP6vij5U5xuu_a&Z^Lg~qah)}5D!CDZyxg-9v{4U*Y0m_9|ZJ*US$V|Hj&RIx!;m&lH3UhC8pmFh=bGg zmDvq3%ZIMn!)&iqYtlgaoy1|@_9L$s*X@Adx)*inlSzmfi*17Kzz->jY$^iEszl$( zPC-!78{{qp3-V)_)1jcKLiM}Ip>Yq3P&9G~q1G9clF1m#I?O9--Vj)>52B`N*Pl$& zpf)HhSji(S4q!Cq7cL0Z1(S` lv;RPc!)*5FMsVa-lIA4+U~usc{DTg_F$KwrObRDf{R<5#_fr4> literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000006_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000006_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..cdbeb6669053524c8c96102746a785cf71ae3f50 GIT binary patch literal 3030 zcmeHJNlU{(7@f4%*0?WqU*ah#l%70@2!#qB45k_o;z3H5DG}SG-((^LTj<5MIeYQn z>ZGwyz4qc^9?ZbJ`IrZfH}lPh_HcXmD4pil1Pq2g{04U?nd&vV4;`u;Lh*g`LMIeYQH z)oIfQ?b(Y$-XS0FdqckPW8mep^mupwB#}4rt~%7WnR=39}v5iEQk+a!GMIOOWElmhqqf)hGZp12q|AcYBU@`+JHq(k5?q&bwDnaO1Iyl zCJ(*9VaXaj!*Ul^v`ShWd8KX3GnoUcE^#MeYpzIIH()~~=?{9c51VcnQ9FmNam2O( z8H~7so%smDVb>ZJzf$7mlj;@bb!wBypXY)+B0fkU;xkJE?*}T*ut$x2BoHccLB%Qd zsF9BZB5t!J@P44;413i7<0Zi$TQ&ge+UF6WHgGI<28H7;%2nz9V07V_96*X%JEY}55)3nP^ z7UMxZqcCH|_b}UoImIZll1Cg_zDXRIcab}WEx3%dsKXK?sW)oA56f_m| zG(nue+D`_BZl3@~g>djEFP`*(EM nO?!W)4|{GYt_|4_#>Z%Mn%sbYxre`e2e3~;vO<%>fmQtiLakHA literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000009_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000009_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..b392e51a5e53dd0a721979d8c72d175a40c8098a GIT binary patch literal 3030 zcmeH}NlU{(6o4nKU5xuu_a&Z^Lg~qah#*w(U@+By5D!waOo`YgeUph0Y{83dbN1q& z@gLORq?5)TJbTc?yu*Zf^O$euW8mdgySco&70@Gkk{uk_L_U_}Ze6ZOa?>Z2m|nvt z4o=eNW-GufAGl@*v$;~PNImIyAwFkqFKl6csPKf4%3R7H3XLHf~aZQ z#V3pLpq^5gvEnQI^>PGIdP0#R7E;@Vq*ZO(%m;0Cw>Zh#x$2L6o!*r2d!898>KwjTx<8mxGj}5NzY^=1Y2-p+pOLA zGQL55CY>~P;o5~R=3mS({~YEYJ_gRbYB!fxw*q=ZPqKppo5;tK+^x$MNpAXt64Pt= z#KDPuZngr<{J=Fkn5~s^Me0evllYvqy|CShszTy-A1}i4B5n-wP-SZ7Kpv zt3=O9cR^6m3uMj&3$g>4(jlj)LixMMp-~5mkT)_2q52UN;^6>_I!r5S))1K21yR$q zi%+KUpq^5gvEnBF8|ifhBNw_Sk*7>X;a1k literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000012_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000012_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..24c59b6836d29751435d7bb9ce538632e0551925 GIT binary patch literal 3030 zcmeH}NlU{(6o4nKU5xuu_a&Z^Lg~qah$2+*U@+By5D!waOo`YeeUph0Y{83dbN1qo zs=rC6jXikwpoe)d1M}uF-|#W;=1sf3y1o<86MB{%9NI)amgG)du1In-Ae5MXBOneQ zr7z4@h*>^#%{FFhrCgD^((gii$=ZJ8wd1NC5M1@57JV=YF=MeoupRgzC6P@!l%XL81H0|<} z#duQBD9l*NJafH}>W!N3!LpkonuoA5Lagdg znjlVK?I!|JShteedx0I!gB#!mxB+f}8{h{1jRDx8uxS}Nb|Ei_LJV7z-oHKS{X07B kroBJYgFUyD)CTN>@i82p^1&Va^&5bF3X&C>6b`KF7ZYqL7XSbN literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000013_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000013_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..b8cea270f81e70839573b232b53edb58c064da29 GIT binary patch literal 3030 zcmeH}%SyvQ6hJ4feHh=5`qsu(QYc-y5K)8*E)1p`5aL2g9#bN=N$+GL1Y2-p+pOLA zGyXySCY?5R;o5~R=E7u{o6DTxFmUHpySco&70@Gkk{uk_L_U(_PE9ULa>FN-m|oo{ z4j!e?&1QgEK5)%8W^1KXmb%jKLVQl!Uf60!727Ac(h8gO-Xz3~#5%#Y?*){EHWdLS zRifu4e?d^u3#85j3(|d<(jlX$Lg~B6p-~%)kTp^Wq1qwj;z1wsI!r5S+7MW-1EQvB z7oRM~AN7>Nj1}L(Y!~JfBhN}6aAf%=abVs>?ijYZs6Y-fDHA_d6`O$z%~FN-m|oo{ z4j!e?&1QgEK5)%8W^1KXmb%jKLVQl!Uf60!727Ac(h8gO-Xz3~#5%#Y?*){EHWdLS zRifu4e?d^u3#85j3(|d<(jlX$Lg~B6p-~%)kTp^Wq1qwj;z1wsI!r5S+7MW-1EQvB z7oRM~AN7>Nj1}L(Y!~JfBhN}6aAf%=abVs>?ijYZs6Y-fDHA_d6`O$z%~|sc_V`msvkii8T6s3!?dDi4T0r4AZnU+ z@yTL5sizcXtmF=6yD+C1MON~FV=FL;1M@C&$FK#Lkrs7WVkGri&G%s0O%csQSQ#N! zbtp{`C$RPtfheq7N$st`4(GuQa0A=`H^2>W1OLVVY*5&=j2yd=7epb3tx50S9`*hm m9d^^+pXtG#TS{sJ_QCiV4&V9U4*vQLz&-`ZicAUzR`m;MqIwqq literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000016_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000016_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..be04d5a765da0f762dcf2b17ee6e0d96f9a57952 GIT binary patch literal 3030 zcmeHJNlU{(6rQwpG44yuY) zzA&3UX1wp39n98Bu_X1Rzd`J$?RL=Wgk{?!xZDbw^uZ*=48=OZw%7J432Z6?N~*-R zlVm|q(F>%`1q;$cn9(7ls6z3x$f02ei;y)^2%*{urszE>l|8VTCEFH)?hOt8OCEIEJ-YVqJ&A zMB)rKzDghpn^shP%dx{zzY@pq@l%fTiC^3Y?!)xW;r+n-f%n6-1H2!m-T3|c09zEc zEhEErx4gi|urrzcyYcMb(_ufE{iy*QxP_=Xia!_}+<|}K031?~tiYsjWR*VvkjuK{ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000017_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000017_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..8b6ff5bbc537d98808a99c9552c57e4e0dc081e2 GIT binary patch literal 3030 zcmeH}$xFjP6vij5U5xuu_a&Z^Lg~qah)}5D!CR-@KU*ehl;GRlB{qz7x z7iQDPO!r;0gV|gumZYBaw-EbjyB)MTVcGTwF1LaveJ}|zL$OYef(7Xz%;=C&RH68pw>6h+T}Zo zaidl!%vsSr%=chHG4d?s5r>v%5(gGtwp(u5vzd`IKEjMVk!jk0?Txtdl`d|=Zgkp_g%X5870*i`(;w;g% z<0uFUdV$2bU_!DFGa97wict7WvT4}DBBb>MLa2HInc<)hSq*0MN>UdX*9K8l)ysE= z(V&zm%$dV`nD4+sUeB_WM;w}-L2OubkTVina+uPx1}jWSxlz(RSao8F`Z26U66+e| z#u8_+@l^s**ffWgw+uUcf^K;-{Wke2#`)MP?gRH>`qJ=v;Pt@kVcG#+57Tb^{yxAK zg>6$$v0Y6s@G{~=Wm*j3;E=h9JBa|3!!y`71 z`dDWd<=V&*}osp{sRpTli8mdz>!mks-yUW!Gja{2L#}lf@B5;g%h*<1(^uC A=5)v?s!;qcvT4}CB4q0+gi!4aa?xlAc^&2zHC-1N*9B42wChiX zaZoE17R=}o7JIOy)bp(58Hc845F3^q@P6R^!24m^0p1VOZv5_jfE^0E zWef(7Xz%;=C&RH67Oa%kAWB4mveLa25Exo9+mybiO9nl=Q+bwSiL?ed*r zJg5~4b5?W@^F3Hlj69P(;?VL;;=rPd+_Bk`%bb>VSYb};jhY?6s+(9ej$tjfSl6L2 zu{eW`uNH{HrWIA+a_n%_uf*fe_)f$5BsZP|&tduo@p|C(!0TZ;0bUQ&Y5e{^z!rsV y%gC@@Eidpf>`Z$9ZruC#bl6XNe`){+ZXv3U;tvJ~Pv9RU0EZMLD=;Y>S>+G2ExP0Y literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000022_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000022_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..69926a5bbace725bd5714119944aabf1cde01dd8 GIT binary patch literal 3030 zcmeHJNlU{(6rQx!*0?WqU*ah#l%70@2!#qB45k_o;z3H5DG}SGZ!!^zE%ajBoW1yO zb<$X95xscvFz+zSdmr<`$2arlO}o3kxfjqgdXXI**hD^)wu_f+SLb( z(V&(o%v;d|EOcQ}F>(y@gxi*H5(k!CZ_oxISE;$=k&J!2Q7eFsT6d!=xJ1?*nX8 z*s+WZLp1#`z_2^+{d;lm-`C+F>HVob9J=|aGK@bMJh%e?Kmju&D?r$r4*m z5(Pm;&yYG5EJzPvN{5W13dN5khlXt|Le@wjgf5RD7YzrH*I`;w(}uvf4v3njoxd@R z2DL(A#)|G>whMELk!L9nIJ7*II56)bcO?RWRLs*F=R&^+h zB~D=Ns|2F3ZbjAC96Nl3US+)ce*P)W`QQEH{&0ULZxpWwUJtw;CLQ4QFzLqU`v4mh zHZ3E=qD?RGF>H-{|90H_cXZfIdVi`9du}1B4&x672Y28fH~{+;Br7l}99ZRdura#i literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000024_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000024_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..2807108ebb4d02efaf22c6089e98dc62be8aba03 GIT binary patch literal 3030 zcmeH}%SyvQ6ox0Qy%_J8dcVX~QYc-y5K)8*E)1p`5aL2gE>j}5NzY^=6kBj(+pOLA zGQL55CY?0x!nGhS=3mTV&K%A^d<>j<({8V>?*#OOo@ECIHj$4dxl@%(l3e!*C8k&N ziG$4?S;)&yKMUemz!aOKA41&9NKPS5wb=KAyhqrTr}uIUWaK#O&bEsbwJcK z?edFhw5VqkW~}HQX1g$_7!xD2+Z`5oLmfggneh4eE z#i|a4iNy)5{j@+7)~%@Wo@33qH~~(86W|0m0Z!oG5P%H|o0gGb7i#)pfMIJg`?urS vzoWx$GW%0K*mDa}We|Tbm_CQYk8wQZam0gjNZ!Stae#dak`241 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000025_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000025_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..c268759063c403ec467f5e47792c6e99929de39c GIT binary patch literal 3030 zcmeH}%SyvQ6ox0Qy%_J8dcVX~QYc-y5K)8*E)1p`5aL2gE>j}5NzY^=6kBj(+pOLA zGQL55CY?0x!nGhS=3mTV&K%A^d<>j<({8V>?*#OOo@ECIHj$4dxl@%(l3e!*C8k&N ziG$4?S;)&yKMUemz!aOKA41&9NKPS5wb=KAyhqrTr}uIUWaK#O&bEsbwJcK z?edFhw5VqkW~}HQX1g$_7!xD2+Z`5oLmfggneh4eE z#i|a4iNy)5{j@+7)~%@Wo@33qH~~(86W|0m0Z!oG5P%H|o0gGb7i#)pfMIJg`?urS vzoWx$GW%0K*mDa}We|Tbm_CQY({cRZam0gjNZ!Stae#dak`AjE}~T&6^9lb*>$2)5wHwpqLJ zWqgDBOgd@o!nF%s%)gk!oH@)t^D%JdRlB*ox)snPdXgO+*hD^(wu_f z+Qlc+Xi!fnOk43C%yeN^F$%2Y0Y{c^5(nm7AjE}~T&6^9lb*>$2)5wHwpqLJ zWqgDBOgd@o!nF%s%)gk!oH@)t^D%JdRlB*ox)snPdXgO+*hD^(wu_f z+Qlc+Xi!fnOk43C%yeN^F$%2Y0Y{c^5(nm7I!r5S+7MW*1EQvB zmtV|AgL+0`#)|G?whMELk!K~3xNZ3+abVs>?nrFGWlD=WEHNeZM$PtM*-a$shp-Y$ ztm;shNSwghPYFa}-HIyjId(WVPJk2O1ULasfD`yP1Ym>0re$Q3DVG2M literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000029_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000029_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..7a83c7721f7aa4588287d9e31f716d30a1c303fb GIT binary patch literal 3030 zcmeH}%SyvQ6hJ4fwKcvU_5Fyeq)@tYA)*KsTo_C>AjE}~Jf=i!litZh2)5wHwpqLJ zXZ(ZuO*(B_(6t~g=E7uV?p)>!XLIwa-CSPX3g{6%$qo)|A|FX|t0EUAx#kl}Ot0z_ z2M^NcW<9_xAGl@{v%OL%N^R+PAwH*VFKjfUlI;^*YJ_!qZxUigVwGUq_X0{nn~H#v zD$#S2QxH`20;w~>f^-)qbjT>GP#6_CG-_fICXEzAsB#F|xYvc84pWMnHUySyfv9QP z#V3p5M?Ix5ZN+ym(}r2a$gz?K99h0e9GG*FJA}=SPC&+yqfOfaL*V zMTh(taSW?J5s1Q?6_?+#>~bF505`x5a0A=`H}G!^z&eEu%gC??HT*EZusQDiTZ7)e tt;0^z`%@j*b@Oq#$9^zA`u&q{<9)Q|bK-@2_#+RnM?tbelfu4L`U0kDQ5OIJ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000030_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000030_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..7ca7fa8e67d5cca0e6ea5dd681415029cae40146 GIT binary patch literal 3030 zcmeH}NlU{(6vrp6U5xuu_a&Z^Lg~qah$2+*U@+By5D!waOo`YgeUph$Y{83dbN1qw z@f+07q?4vScoxLN{9(5D9`hf5mp5pr2x^lCnF za2&oc8v$nVz%^T#ot0uq>Po+p_>#80u-R&tZJ*$BGi=ZYlMu5l)(EzJFQ6o}sR$^k z5s#&Gul~O)Tn%uo7FW z>QIpr2x^lCnF za2&oc8v$nVz%^T#ot0uq>Po+p_>#80u-R&tZJ*$BGi=ZYlMu5l)(EzJFQ6o}sR$^k z5s#&Gul~O)Tn%uo7FW z>QIaTi*Mte@@P`~=pMqqCCWQm5{0(a{d&U3& literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000032_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000032_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..3f869e0f86295bd08285fb0b66c963c9c0994145 GIT binary patch literal 3030 zcmeHJ%SyvQ6rHrz*7$zZ_am;-Lg~tdh)}5D!eFWaAugokF(qP~^iC#1v4w7Io3$H% z#y_avq?5)%#ifFqIg2~Yoy$FNI5T%%)SJtzTLC?yC)vinMdTAnZdc@jB-cDbiQ!f~ zV&f=%Zq$9u^1fp2T18j0Q6~C8-N6*9K8l z)r${?QBX=0X3g*p<~lH+(=)8(0k=%gAT}&G$Qh9>I*e&agJs5~Tr1N(SaD*C+7YZq z6l)q}#}ucq{+$9**f7KLYnnAj-BJ{Pg6CsRxDH&0$=kvG!2Q7eFsT6d!=xI2^FF{P zg)LK0vCa)I@G)$UXa7z#`*$_ii)VkL2m4MoEDxd&2E+Ss_{LYR!+-h!4k$=wU{E+T Fi=QfVxU~QP literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000033_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000033_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..6c98c970fd7511bb037dc5a945122343d7a16970 GIT binary patch literal 3030 zcmeHJ%SyvQ6rHs8VSGR8`w>@3p>*X!L?~2nVKCKz5EoMNm=du~dM6Vh*g`klUNuK9!#)2sT# z!7+Vd)&tD=z%`qgot1n+YD>R^*h$)6*l0#Y+b6i#2yVpBoWaIV2}EJj8kFBMtoR6>(q#8Da-R2R+7a9j+z;Fj(;eV`nC=FDAO6Y@utj0p zGE%Hw!w&-tJCoVJ8_)hd9rnkwKhcE)H#aEv;|~VI$8h-0Bk#k1`T!0oNLFZ4II@ag DIH|a` literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000034_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000034_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..bd729811f3cbaf1ff5e9552f7cfa26081c4f2b00 GIT binary patch literal 3030 zcmeHJ%SyvQ6rHrz*7$zZ_am;-Lg~tdh*GHF!eFWaAugokF(qP~^iC#1v4w7Io3$H1 zs(zDB8VePd3U1~s?l5;Q_rT%I+<8@RudeR|^n{*e8~YZKPb9frkqeSs^9Ut|TlI*I zqx6MQ_c6=+j?u(ypUW4dj`TZ--K6CPjb^K8c?1_5L7hGrgcvQcO0ea*J|%%gML=xQ8Iy9OO!r{Li79GFuo_XU zX^k2T>sa2+OZ2loT_1NXzE0^ARiYW&Un0GkxH zOg+UqH@v{dusxprJJIal)nG54{fQpzJK3;2h&~t$@5A94U%3wd=>s^RAen(d;m|C8 E0#I3yH2?qr literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000035_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000035_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..3a993e8bd95dc6663c1a12753a71be4eae11bf40 GIT binary patch literal 3030 zcmeHJ%SyvQ6rHrz*7$zZ_am;7Lg~tdh*GHF!eFWap)RE4F(qP~^iC#1v4w7Io3$H1 zs(zDB8Vl`O!Ofh-+?g|%d*E`IJMY^4&FzDLUeK#-W8Wh3nIyN%a$b^a9-+i=s~)j& zL|++oA2Z%}j3#DhC6|{v((fR4la?DanyrH65nN~lb^2rwVzk66!ItOxlmr$P0mWIO zYsa@BsOSw6mx2k&KFsKlQdA-LlVsCY6N@lgNg#yEr;rW@eaPrAr>Mz_z_>PunxoJ5-=^T>%UKdgtEM%0pF0fb=L{(L9 zf0zvir9ffP>_5X&3zjo_ilw~bz;q2_!-|8PA=s+Jkk&LCxWP)!Da}t zr9pZGaS7Yg5QxH#*)M)2S@E&83uAZ6;qizL&I9LR@?vm3a6NE6OftarFv-Sj?*r^o z*faG6t5|b=55xYb_aB74|4@UYsQ1S@aO|Y}#UA@$d=Ca!;Su=EdH7EqzzGG(^bHE9 GX8sq!_>eUK literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000037_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000037_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..f63ccd3ae5aad2231fcbdf9657625f12caa6aaf4 GIT binary patch literal 3030 zcmeHJ%TB^T6rF+y*7pP7kGd)(gp~_pG$fd~uo1M0NnIGy$B-JK5fnd??C`1R6sFQ~!tqE7t^?O$d@0z1`+@s``(aW6?uSV=e*Zqe zHiaESOR&@GuJ2*k9rgabu=nq)a1izWSQifMbWrTG560(Ua1qk;nd|VMK7b<%lHuzV Hj*a{`&@GTP literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000038_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000038_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..bd7c40686fbc62a6db8f2bcb0b6637ce1708c90f GIT binary patch literal 3030 zcmeHJ%SyvQ6rHr%YJ5NH`w>@(AYHi-5egMt7)&)F)P+bMQzEua?_?qb72Ige+KoTs zAJlKsN#n)3cHt(on8P{uk~17G+`Jm+C#M$@xg~e1O9O|g`-*yBQ+E`#;WN&x_La|E zn#f01GoV5bJgY+m*S2?*2jy=^e#|@Vu+@pmj?ZYh6*l>`#h4YzSByG-JK!vIxC|ui zlG|=t1xY7QkUNxY$ai7LgkepWw!fQP9(AY;g?f$o zV%~%jS{%T_Zwta<(T=NcMe%X+SV`_5OxX{|dZuFnDPk^&SVfMQ>6k#4+h7tnABb2( zj+p6~K#G_PB36+jW;!O29 l`*Y8*;g#a*i}=C#==I){XJ?q@@Sk%4n;aB7v^Z?p4DJhnt^lg?%%1Y79EwmEz8 z%lHlIXVOh$51u{rvU8Z-ogMZ!41bt?*Y0m_9|ZJ*US%8m7Lm^-xm%Mhd$mC^Dsi~Ekz!R)LQOHxnzpNajnUoy(j6>5ihz-jQawf19hmlrwSYsshPR$Nr-ANG5Q`m?R zn>rMxhzr>Ii$D~%&8Yg3V~?YLCH9H&J?{h80M`K50M`K5z$`TYI}~=!dWO}|_5vTn w-gNfw$Fu)Hhr?v{rv`B36r$>geK5Yp@3p>*X!L@88oVKCKz5EoMNm=du~dM6X1*g`kHyzLX5uZ2~5ZxUiQ#4^FQ?*){EHWdLy zS)%7ery!{48DeLG1@SIS>5x!VA^Vl&&_*4LkThZlq0$kg`n@isb(mJvxFImC38JQH z7auG}lUkrKWA*P~wgq#Fk!C3mxMBGwabVs>?ucx`WlW1YEHNhaT1|Ff*^MYFhp;lF zSk)mjrZ|DM?-Yo_y45efrdV;*E)3_N!SP5Ft^?O$@^)}Pa6fQAOe(@3p>*X!L@88oVX)PJ5EoMNm=dvVdM6Vh*g`k9&DxDW z;~&&-(rII%T`Rbm3-da2xd#r1x${=KyS}*>&@+0GZR}Y@K9%HFS89u;#8x|Bj%~GClV7dmeVbMX(h-}GWOv@UqFedd@O?F|`i72Yaur{Pv z*B~>dID?I83PfSk42$n6R(uql!gx2ma-8ck=?Lxz?g#FNNd>qcCe`5Y!(aIUwkT|y zdVK*%P^S+DAx2BA5^Q;{Pf1`=5l~zu zx^{dDf`VQkaW0sU?8B4>shlF@zlv+*mVv5=ktVR@T z8f3>5r?CE=0#Vp7!}432HAmf2lz#@tV@#LEPBi;>HQ0-1f1(HbPBttL*azc%I6UKvYw+J5fCCDW85k4}&Eh9^ CO=X4v literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000043_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000043_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..fde12c7f04caec5a12de6c278cf5d9fab8c67695 GIT binary patch literal 3030 zcmeHJ%SyvQ6rHrz*7$zZ_am;-Lg~tdh*GHF!eFWaAugokF(qP~^iC#1u!U}Ho3$H1 zs(zDB8w>4P!OdKl*O|*ba5&7JclG|}_CY`|=vB6{ZxQ)Sl3NwIAjvh4P-3`MkJxw! zUm0~DGu(HKCT8bcz96-w--XyoT5iy2hDFOGxY!8l^vNK^2*oPFmgoAE1Qrzm#aW_j z$G0FT=nWE=f(gkU%xI9xDMJ1y$);fwi!iGv5JHtxNJsr1WHgw|DM?*mSPMi|Rj&o6{7C!j>78KhmuDC_1I_ZhGZ7*JsiZ+z;Fj+z*oqa6e3{!QY3!@&jyB*fI4K ztJm-XAH(jr_wNmR|GowXaqmxb;n2xO@Zp>*X!L?~2nVKCKz5EoMNm=du~dM6Vh*g`kL_U|~W?9Zla@8Z07;eQQ zHXg!PM$N|z_Z_2w**crcOD$=(5ZiIf4eE`sV0i==>OqY@8H5<2SRvT*T%VG_q9UNE zN_6dL7X$^pLF`g6A>M^K4H8*J$o&@CG;ClIl6njwR6c`Lzt@Gd2J=}Zt_uunf~czM z^*4*rq!cMEnEgjsY{61iPqUI|9Gad%Y*==XGa_4Y7}Kf-Ym7;`Q<5E6cOr`FDQpZW zHZ{nMDK22^F9o8oZT3qaDR%fI+QmuwC*gP`2iJk?Fnuw&AGja5AEp)HewbEcw)X*c zDD0Ygf}L3R0w2TPxcBc5d;ft3hf(j3b>PU!^h-VV!T1^sKKbAp{I>_-n1W;m289!| F@B`@WwuS%z literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000045_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000045_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..47c5d0df110cc6d1735446522c9c9d1e54fe66be GIT binary patch literal 3030 zcmeHJNlU{(6rQwpG44yZ6I)Ts4XU+z&hiM(Rf7tBGzc;3Vu@hObA3tzi;954 zEYY>YQxFvN3Xu!JglGq5G>D}YA@h@D(|Qey5Z5CJq2dW7dfg5rHJD8+QC(oM28gPv zUVSkePD-A_oY{MT`6eu+^(0Gq!gbR#hz*Mla)w|_4ntbjV1*$mw@SPXt4;_}K8Cdc z#JUEl5yTm6OhX_Fn`W=@o?ykNtd*Z~#}1E&RB#?R593S0He3%}4_ptE3~)V6vhn-( z0k$Y?n|h2@t9pTtVQ19)cL%+HPlNri_ea`r;G}wmF8g47_WK`fob#3Q@Si$>Lkf}^ J7!;1o+&3K-js^e# literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000046_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000046_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..2489bae60e680e4483edb44fda6b72542f216def GIT binary patch literal 3030 zcmeHJ%SyvQ6rHr%*7$zZ_am;-Lg~tdh*GHF!eG^aP#042m=dvVdM6X1*g`kC_PB;wQyp`^*Ztf-WOkPx*dKObp6t#V=<`lK+GR}-f#bq`P zW3xp^I4$Y0ES$7kEzyG&C$y*@!D?W! zrb8MnPGSAG1>vw^4vX(eaXIN0g7XJM_QSCs>6k!>m6) zjC4#OM9c*dyT}nE9TSLhn@j@V2O{>6BmS=rn;f>x^0fGhy6bxswsH6G1iOD%hrMw3 k$9k~uq=&^p@Pi@hV>CL8Fv{US-vJzOP)y(8aA@Yg0MDe%qyPW_ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000047_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000047_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..cab325912354f607edc0f5dfc6747bf6f8941460 GIT binary patch literal 3030 zcmeH}%SyvQ6ox0Qy%_J8dcVX~QYc-y5D^L$To_C>Ak>AFT&6^9lb*>$2)59TZL@ac z%lHQMnRL?Fg=-hx%)dA@bLKGLF#K@lUAw=zeGt$KdX;VLTSPvWJDinWw>6h z+VwZnB&Zb%3ug2Pi#=FU>Umc3j6>5ihz-jQawf19hmlrwSYsshPR$Nr-ANG5Q`m?R zn>rMxhzr>Ii$D~%&8Yg3V~3-DC60;soWBG20QUg*0QUg*z$`rgI}~=!dWPMh?FBxD vy=m{?k9+@t4u?taPYvM6DMZy#{J~)Q8jnBu$lqa>K7eBik{K8jPR#NTxAgTS literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000048_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000048_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..43afab4cb06fef99eaa024d634a23587d0c4c16c GIT binary patch literal 3030 zcmeHJ%SyvQ6rHs8VSGR8`w>@Zp>*X!L@88oVK8bys0%51Oo`Ywy_1O$Y@r+5X6?qG z@ek@Z>7;3);!<%nXK{zQbGZi&XXehUdV6_&C!i~cBIHIz7WJE0gk(8}5V|^oRM781T7&7V5-$rZ*8)*h z)r(JtVNi+`W{lt-X4^2AEvH$@BlZnPCl<`x$R3g{*o!Sj(OTnDbho;C|qKm{frKVN#91c^_bd z!lqG9u+DYIb1`gk#?k-*kX| M3X@Zp>*X!L@88oVK8bys0%51Oo`Ywy_1O$Y@r+5X6?qG z@ek@Z>7;3);!<%nXK{zQbGZi&XXehUdV6_&C!i~cBIHIz7WJE0gk(8}5V|^oRM781T7&7V5-$rZ*8)*h z)r(JtVNi+`W{lt-X4^2AEvH$@BlZnPCl<`x$R3g{*o!Sj(OTnDbho;C|qKm{frKVN#91c^_bd z!lqG9u+DYIb1`gk#?k-*kX| M3Xq`q4eZIL@8A8U@+ByP!Ce_*b=dAI-8A9Y@rw1=Iq5k z<3FgsNjHs!ibn-6JBJx|c9?JZ_;zPsw42MTTLC?yC)viHMdV{iZkOboBv)NRiP5OI z#KvLz+^Bh&J zqAIarN4p@X=ow;Xf(h|HOzDtNR3ZCSWYbm?i;yhG5JIIRNCksFq;;59)OcB7xi*NJ zrd@n68#%Q=Va5#ZV73EuN;%C+9&pQa4PwK*gPalAg2Rv&by#9Z>b08e!Lk!UR1aY# zgjm%fGln>UweJv!!nzq0UsJ3(>J~!((|A7Og!908n7kca4_psi50eaVJxsFkH}3;% zP}nrf3D&vp`W}X@@$BCYXa9~4yV2~A^q`q4eZIL@8A8U@+ByP!Ce_*b=dAI-8A9Y@rw1=Iq5k z<3FgsNjHs!ibn-6JBJx|c9?JZ_;zPsw42MTTLC?yC)viHMdV{iZkOboBv)NRiP5OI z#KvLz+^Bh&J zqAIarN4p@X=ow;Xf(h|HOzDtNR3ZCSWYbm?i;yhG5JIIRNCksFq;;59)OcB7xi*NJ zrd@n68#%Q=Va5#ZV73EuN;%C+9&pQa4PwK*gPalAg2Rv&by#9Z>b08e!Lk!UR1aY# zgjm%fGln>UweJv!!nzq0UsJ3(>J~!((|A7Og!908n7kca4_psi50eaVJxsFkH}3;% zP}nrf3D&vp`W}X@@$BCYXa9~4yV2~A^rRq4eZIL@8A8U@&Sxs0S%|Y>C(=oy|rlw$O`hbN1q& z@gLORq?@LNibuuM4(#mix5Iq!F}t&`>h0zAoq(Ruvut6{B=WH&x31*8Bv)NRiSATf zV&M=z*J~bTxM%AP%=Xz_UTRCfGqDpl9lzda7EG7mLfx;?2b~bTDOLzJUB{!uH>n6H ziV__wIt4*NFAzHu42bt&N`pjJ5pts-i#8irgk(8}5V|^oRM781T7&7V5-$r3Yk{b$ z>cuCsp;L+!W{lt-X4^2AEvH$?BW@b5PAr(Wkv$|^uo=^$21|@dxlxi`ShgdI>LILz z6ssC!#uO*8_MHMzST};wTZ$z|onq*J8pk6|xDH&0$=ku}f!71Che-u^Jxr?cH}3;% zP}nre36{C;`W}X@aqr&_d;g9GyHW3tbz#rW1f_oX!C>|=7`(G_I$HBN(uC{qk2=6U M1K*%P^S+DAx2BA5^Q;{Pf1`=5l|c@ zx^{dDf`VQkaW0sU?8B4>shlF@zk+PqYGM(l^#nqwat!Hk(1(l$GdU%x3k++6sH*Da z2eXk=N)%?z@E+zmFrU*iEaVZlOwS-TEI7y+ku5rmX-R`+#-!XR(>+*mVv5=ktVR@T z8f3>5r?CE=0#Vp7!}432B}d&-* zGW8V8-0%V)!}hrM??kh@P9hM0R_nn J3<`&4@e`W{kTn1R literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000054_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000054_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..e56305704fab991e3d01966eabea77e65f0c4d8c GIT binary patch literal 3030 zcmeHJOH0E*5Z<)b*7$zZ_amN?Lg~qah)}5D!CLbT%6y*g`M1&Do28 z#(z+MlWrOd6_1Li9oX63Z-@EdV|Hg>w42MTTLC?yC)viHMdTAnZkOe}B-dO*iP5aO z#KsYPZqz-@aL+MXnC+EZUg}7{GqIbrntr1d6fBqELc_1qdxH=o5UT`RZquX0x2Omx zjuM-8d?ueHVs->gy~8GAyhtwbU5fkMu!Jtie+xNzK3CZ-1~Q;-oLBEUflZ=J=k}$VQCP3Fqpj$hi}msulXEn!gcsR9pHe1 LWcmh$L$mM+6)?E9 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000055_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000055_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..1c293f20f597d13afd791b109e4a26c6fe64884f GIT binary patch literal 3030 zcmeHJOH0E*5Z<)*VSGR8`w>rRq4eZIL@8A8U@&Sxs0S%|Y>C(=oy|rlw$O`hbN1q& z@gLORq?@LNibuuM4(#mix5Iq!F}t&`>h0zAoq(Ruvut6{B=WH&x31*8Bv)NRiSATf zV&M=z*J~bTxM%AP%=Xz_UTRCfGqDpl9lzda7EG7mLfx;?2b~bTDOLzJUB{!uH>n6H ziV__wIt4*NFAzHu42bt&N`pjJ5pts-i#8irgk(8}5V|^oRM781T7&7V5-$r3Yk{b$ z>cuCsp;L+!W{lt-X4^2AEvH$?BW@b5PAr(Wkv$|^uo=^$21|@dxlxi`ShgdI>LILz z6ssC!#uO*8_MHMzST};wTZ$z|onq*J8pk6|xDH&0$=ku}f!71Che-u^Jxr?cH}3;% zP}nre36{C;`W}X@aqr&_d;g9GyHW3tbz#rW1f_oX!C>|=7@V^4K3elR(uC{qk2=6U M1@3p>*X!L@88oVKCKzP#042m=dvVdM6X1*g`k9&DxD0 zRli9mjfDydf(tiu7k8L5cjmz1%-ng=ZZ5BG1@wrXWE*=Hk&h+0U6ON>Ty+U0Mx){q z8;9_@QS&gvJ;!KbHdnGasU!U@#BSVb`1NKhZ@C2L>wb;i8-y4wu|lxrHatpvi;94v zDzRZlt01W88DeLG3GqHm>5x!VA^TZm(^eCUkSxa#LZu@}1%p1Mb(mJvcv)ar8$?ah zF5X#;l3JiJV+MCH+krWyoMt5txMjKqv0>gp&WLQmVN8oUEHNhaT21y~*@-Bshp-Y- ztm=>%Q=GusR|-U7-3*GaDRwyO7Q*~}{ItgLNH4Af*J1MJa6fQAa6e2c!2K|(#_!(; z*r2d!mJ@7u-Ss^TTjSop9rpem9d@JMAM3%MlL?9g_Q7}?4o^cmK5`xY(+99mK{9=V I!hxCp0Jgu7H2?qr literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000057_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000057_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..c9937c13c46f3d1339431050c436201594fc2dd9 GIT binary patch literal 3030 zcmeH}NlU{(6vrp6U5xuu_a&avLg~qah*GHF!C^+;yOC9NNA$H?t(`&T+g6R-kXn1vcuM?vCVwGUiX}XknCKUli zRbtbMvLGnv8DeLG0r5UeX^_Y&LhiH3qJ9gDkgUWILggb!1%p1MHJHvS@ruB-Hi)XK zUc9pyJ(MDa86&uZ*$&KQD`{5pfPKT!i3RgEvPWhMHgj6kV2L>?*GjSn%XVZ@JA{?c zVpW68*y04%zFQy)>qby|O|g%UqFWsAzSf_#Fw6Cb5$C`;LrRq4eZIL@8A8V6fGIP!Ce_*b=dAI-89UY@rw1=Iq5E zRezIi8e1wN9z1y2Iqc5P%w$+*J=)CamUh{n61;9tkjnNW@0C5G~9aA%NaJoxw>1U_ZlIZCsqhH?1n>$YfuqT zSS2>hFbjf$o*{B3=n(C}lnSx5B4j>`OzJhU2=Q_RAyhhogx~K$QibWX5-kfX)&j9u zEMB}b8xBgI!i?_U!E77m(&Z#8dBC1-Ys7?k3t2<31&bjqs<6b6lxrp4g=H&*s2;*f z0I{k~I8ye8P;W9#I{?l4SRp23wu_|FZ9_5<83fF4Pf}pdH7Eqz&-^@ KcQpzJdhP?$DUdY) literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000059_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000059_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..7579ae0f2d0d34021177e592ef9e9ff5c0c4d22e GIT binary patch literal 3030 zcmeH}NlU{(6vrp6U5xuu_a&Z^Lg~qah*GHF!Ch*g`M1&Do0| zs(vP&G!|O$A|AZVJIwI@^O8UOc+8tO?e6;KUO><2MYgeL5&2Y-+a)>16dCMg@U-xVD(ICWVi4}q^x8YIZTT}!T zRf!EdIt4*RuMoQsOo;bkMu&u=3fb=>o3@%*gk(8}5GtKODj4)3t;4LM#>)cJ+8}D0 zcJ;|(w5SCNb7t@W^Bq`F%4t^egj=R-5E~X9@Zp>*X!L@88oVKCKzP#042m=dvVdM6Vh*g`k9&DxD0 zRli9mjfJ`>2rk^rUEE>L+?fN1Gjr!ny}Q1-7tk|$kuB_+L_U?|c3I9#a?K%>=*_A_ zEF8j@dfmkgcWu3e**u%eOC9NNA$H?t(`&T+g6R-kXn1w{s1u_5VwGUiX}XknCKUli zRbtbMRzXnEE5t4Y1LA#{(IAmkgxq(LMg0~QAz6tbgvuw73I=^hYcQKt;uV2mZ4gye zz4~M^N=lK!oDn?0d&>TP4|pRXd`n9m85k zv93X8OmPMqKPeD}O(Q71r`X}BTMYB`n6}RGNHeYj*J1MZa6fQAa6e2c!2K|(#_!$- z*rKp)R1$1w!|_}UJLBHJ8}|M^4fdnnAM3$^oe4?<_QCiV4$ng>K5`xY(+6-!K{7m@ I!jVz<0>|)>H2?qr literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000061_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/000061_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..60dab0943f7280952cd4d0c48b7134f3e22a1db5 GIT binary patch literal 3030 zcmeHJ%SyvQ6rHs8VSGR8`w>@3p>*X!L@88oVKCKzP#042m=dvVdM6X1*g`k9&DxD0 zRli9mjfJ{U7cSh)UEE>roS6fMGjr!fySco&70@Gkl5Ol+L_U_}c1g}ja@8f27>$Zc zY#f%)jhcs9-E)j4W^*N*lRDDhLhQz^hF@>C@|H_*zV6rPy+Mf45-S8-Zo{L*x2Omx zLWvDKS_MHx&k#EkOo;bkN{57^3fWJPO+8}D0 zcJa<)l+*%+88f(p*$&JpPjU zv8qF6OmPBhUnvlUbu%cwrr6=ATL|;d@g0WOBi*rRq4eZIL@8A8V6bXHhzBWoY>C)5oy|rFw$O`hbN1qo zs=rA$jfED`iw7?|hn?M-VLtfyc4uG9x0ly<0(wHvvV~of$j6f0x{~vfTy+Q~dZXeH z3y0xzz2;&TcWu3i**TlbOKs_ICU)Xx!>c#_g6R-ksCzZ~pcA6|VufJSX}FYlCKUli zRbs=6vLGnv1!8A{0r4J8sgTGjLhh@`qJ9&Lkkn!bp{pZE1^phRRhZ5yaZO;c7Kr6? z`Qn4wa8ZgBW{lt-X4^2A)zYlw5&MRt6AR{TWDmg>Y=*R`!V*JLZj@vfmhA|ldI&2a z#HtFJ5yT0sO+p|F>qby|OR?rd>l8=3@BXJ8kN?gc=ZEt%ew(-+xE{D3#u?yx7-wVh zeSi%Ln}(KP6?MmRF>H-y|8_Y0cU0JoW`C>;dv+!$_1OpGeK0r;TXN5N_)i_cJ_X6} JbP5MX;SjpsQQ@2eaEO{c2+W3sU`i*#CF_rgIc|jvpj-xwV+Dx3_^^CSSHx=T%VG_q9UMS zmFU{TEC?!kg4n5GLc9wTIwTZT$b1ypv{A<*OqOB@q2eJV!(JCsI!q~Qyd{+6-@DrUkP~DaA_eal`ZsV#Az+oB`Op!;ltqSY$}*l{(phC1(gxIe_H| zVnv7a2;vx4zd|4iYi3w@NwVexYv)J1@BL4C9{-&`&JX8j{6=v-a6NE6j5EOXFwVy3 z`vB_{Hq26jWousGW7r(c{;g>CZ|kr#oc*y5>^kYN&|@Er*M9#bYR5h2;Xic%dlV!y JFevPsxp&qjkTn1R literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/README.md b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/README.md new file mode 100644 index 000000000..cdddcfab6 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/README.md @@ -0,0 +1,18 @@ +Total Files: 16 + +00. Hold [W] + Static +01. Hold [S] + Static +02. Hold [A] + Static +03. Hold [D] + Static +04. Hold [WA] + Static +05. Hold [WD] + Static +06. Hold [SA] + Static +07. Hold [SD] + Static +08. No Key + Hold [up] +09. No Key + Hold [down] +10. No Key + Hold [left] +11. No Key + Hold [right] +12. No Key + Hold [up_right] +13. No Key + Hold [up_left] +14. No Key + Hold [down_right] +15. No Key + Hold [down_left] diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000000.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000000.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b25d9c35346761b221c57a35e0a7c625af4a8f32 GIT binary patch literal 44127 zcmYhBWmr^S*zSjvkOomCq!|#9Mx>+%7;=C=DM}6v(v8w7LrF`_3^SB;hoqFm3?Ll} zI5g6Ie9wpXoae({>w2zht@UB;y|3rKf6x8T`#S)&_FG>Y$rnOmB0?fBL;?2;0A&CP z2?;3)Df#~miiZ^cFDNM~X=rHvJt-(CDXFPxXlUtZ|0QVX=;-O`A3tVfWM*dJ-~itD z0rCL=d;&Z?Jiz}30X`8PA%OV7zvTbK|KVJ>S+8-FE|MNbnr-!wK*>0QfWj0vf>mJm4Y0 zzjZYK<_dtzy0Waq;V@MFO|=Wjk@)t7ey=&FN7HY&GGwZL4iH!D5Wv9WMHFN`+P0{dL0cD1h;XRbt-O10h0r@LfQcHtR+DgiyFbNo*nI&j!uZ?dB0 z`PmHGs~;hD75X%(17FEiVZj+RtealLZND5nZ~ogSlnF_x4YqV==@OP5%T-R6#u)hx z4Ie46DU4Bc9dXpwTig6dgn9FMk{fXzS$@ToV#CzRl5Pj|nzSBhMW7%j+Xz;KUMFak zwQ&03Ofu*BOA1w%KY!aRCEoFnk#_y4MSFOocgi&eip@U{YxUb(Pw|J^u(q@oPBrsG z83N&u^w1@)iTzzQy|uDdtpaDVlRrIcfAmZ6eyVfeB3$ z$myQdDDx;mq)=$)A+IxdP(b}PJCUDzxa{~^?DsJ*72_tgsgz9R>GksTI5Qc0(=4fi zX4cH6KmeLdg-2Km`x(~MsySBW?0x<{FfZi4x25V8)wFjZ#^f>U+lL1|RHv}AtP10AnMl!Ybi`@fSZeGNB<&lV1_%NpxZXxv29_6@7c+cQY|1(>B zIl;$`lSLw(x~+8&Z;U8KzEq+7G|&IZXDqMyLG6qKt}@bAUCBIH|m7;j&jndblJ(3Gdf>@a27ChtjE6C|kx0rl%rPqX#)E})cg`b(@ z=_XP>c}&_0g=G>v%(3*-!snCE9>{TTd8RbwR%)ObcK&!g(}k)uvEPd4vzdu6x5=<^ z#62Kas2Ur&Sg(Gg>Njgj)V$L^7}SZ4AHRN^*YQ z<1OMNLG6kb4io(CCt^ThqEV`j^r?>#Q(;@hNlWa*bYMg*R~t(HNI#mJ8Ptu?{Rv+cGIoEX{p-Kt{d%OxGK-Aca>hfe z$DvuiXixg7BwTU1-^kmiJ}QUYuQP-md@P=7m1%4*7F1e)$y?x;z~f`N!Sn0VZ(P04 zh*tPv(hEX%*(l$p;Smv1k9xWcP~-k6gGP)zy@5dEi)Y;$^EBGAdF-IZh2OBY(Cqrq z+yPqmmKVyTt*PKQ_W;kttG?gax8{l@c%Ncx8@R&VHs+u@!N(N3Ws-AT3t^wD4oS6e z;7>d2%^uqNk}~BY1y+O~?8gUl6QlSc(;s5F1=Wf>i?@;a=Y3axA${n)*Uy-|)q8;# z+YGRV!m2Szq4!iQ2DpHBikxOALIxHYdrT;QzvvY-mCY&O&E*TYHY$SMPzFeb3^N8g z6b3crM+h2jYh-OnHe$gG4I_VhJdB)eiibOX9YdT9@=?3E?ow#xgYFI-ZQ$+)}Hwxw<9`u&eoQJ%wg6PjhL=NjvO zG}<{JQP)x!xN$}0>noXem-V5A{b|lEWiQgwEa$r_)dUI!)?#p_BForPMre@;+V~!@ z;1W`4Erq$DUMHHZFmoh`W&DK>vxb!W_~NE3nZuYs-G3G|ieeE0e-pXgxpR`7mC}^( z)(}9bVryUh(Edp7XmS=Y5xUq1RDGQ>_4GKL8u(^lhLGn)#1nQJ4&GY&1AjlRD{fHhS%{?-V_(m2lG-$zq3I1PwghOV;7bk z3WfHBm2T&;f=2xH>)cQsj-K3e<7=US{E1aZotgvE-rM0BOPf~6(#>D$#1Svv`Gx-O zLJM~?8Sf|nh6eGeqcJenbi87vz~oM|!DdpsJewtwsO{9eEx^`+s(`E zuL`t*`R$H+3&*1S{X+iyc{F642xSQUOXB5pbEl@L=1|%XC!@AG9vIze<;zv4>h75@ zX-S}D)YD2~2LYdnbmY*c)@IeCCVI9e4K)48{?rWDW#<8i*9|prDb!52j#o%ae-R`p zYW7BeoyW*wkA{91=1-mOACG;yKbANEx95K9M60d&QnTs#GPA0Xcy#upppIUvXmI74 zyTRsy%2Wa|wB*}i`dA%ZMMhDUujoCXhgYmFV-7}@C<8^?XvT`#(f12pHO#d?C*i-r z!*IcDUO8)<>pIZLPru~sJ!Ga3&$l{j>xvM>+af9-W6UIJMjM<$To2Cbq>om0!LQ#5 zjehvu#uzi+_mx{KmdSSwWw_Ru&w3npp0FI$GYYo=j)-PS8HT;Oor6_UVLzhOngXCU zw#wsuqH*d{9{DMp$=L))JZ6~^EuxMP8(moM$tZ$g#^W_UCW{b|p z=?iB=xK}5{i~YAVzDpG;GQ8lWfWM*sw3(IYdT95URl`H#BI5Q7{3>;-%QcRUM%Tl7GR6hBT|nqNH>c2dE2b^UGTGqCfkc_- z;B(C?)%VtZo4N`ki)w9fu5HW!lL)-pcFjX{JOca)0kOrz=fn>&^I$_7kOQyeCe4Ex zbcn4{pJ4Ce4O6>-zb#dYs5$ouPXYW24NTc}aW^Ci*Ne)QW~Ed!Q=#R3g%+9;C{JOHY*P4s z9;oYXg=fp!<4Q;UNC8!Pl6wzuN^woI@~5MO^Lw8V$w1P=GW_#iPN}KPsT}u_pt+0U zsIPm{Y_qHkT6;amW!Grv0&_!}Mvw+@4i#cWP3x$7%@CtQu5wQa$2sCclh50p5@#7z9w}y^FIeObL+4yNuNH-If3q-|Lqmj?K>YzHz4hlxHf89JF}`w;0IsL;v-CqP zg;^kxC8wv5%6`GwC~tQ*_K;7wL<+`V zkNvtTI0F=epD|#MI-)Bg{f%N^9aO@Wltc3z; zQj5eD28dr4H`fH05d5oMo*GHVYfiP*`JVj-9iY)aM;uJ=KRazr;1Y-{NYgZxfGlw(8Te4Apu@f@MR(Jce zy)hV65u;(O@D2nw3cNNVD-_20)1;#r#OSE}nYe%8w#u$j_MO?4i_7pZ(4Zl>|B~;k zO-6Ox3B1(>xpYu&wYW+sFF6*2uH<)EWmjHI&Z;CM$GK@rZ3R=wGRp;GmJCn^QWJ#w zySm~rQ~r%0CNdymvnW1!5}XrUqW$BtxEjmW+Sio%{EMNvv_YqqXcn9Cm3QLRo5M?M z=Fm(MLwR;5w_cs>{1V^s)*%`)Z|TtOg0k64EKkR8>S>r#2lrN?JV(v-H0ibcMp`>n z0xq#&Cw{3V40p8v;5LsTNShkL5@HVjV)N_DZ{JR1PqD(*NOD*Hk-1c65DgZ%ll=={ z%X#;oTff5Z0Ym4`s^Usr2s~d! zctor}<@2R7h(QC7QnyW2$&}Pa(BGbWz%S?OVAO*WarZf0E6-b{azPN)MQLH9XR||} z=V=CgpgxFBlbUR?u=teQuamfnEsFBd7bAN#3|usEU86Akdtbk+`Ng_O>>u5IYFH>w zZj$bFHDgUh=&LxuV{)!f;m4r&yki{)p_N;O{&WJdt57LKzJ!l2WyK)(a%)`R{H5LL z$0-sP4F)6_70)pAtZ9VZ1Ix!o2jPo!lCNSc4^6-~k6L#=n)WtSq~WnQFjKY6DC*j@ zcq{#b<1@K(!>3dCYmLn5#Azj(mN%+oPO#0zVYb>j1|7O2DWNDk+3%05W>iVJ z$C}6sh{ek67n`Uusia~s<+UI9-)@gFg(kEfFrSsyDc%Z(1}?>Ch?U*ivHASXBZcxvZ+)JsmRG(@oz*T3G7d)d_-a$1>4yeQm}`wf)G}U0 zy-=l&1sasTIRL12G;K-C6dG2W#>S>-P6iO*+D$DnllOpDWzb%$5sA4K zN=^D!4*9E${Oa`%5GS&<6l$x=B5HCESc^=EeP{HNq?+iZ=H@*>#9LEUd^14FWPP~g z@WNzlRa+Pz#I3EblfJ7k?ZA!A%p&0ck)6gLQVX-RDRkyBt{6G7Rw#AnBv*gKP=i1w-bivas^|4a*vpK+gBAhl z=iTLkGRECn)6~fa$CA1J39&``WaU~~S!lT2*-}E>`&&?;-CbG-=IXtXskzcQW|`9~ zs)ZXli4ZjqH~~?7<2JU`?X)+SDoW4gy??q`|c?HE)-?w z2r<@(!Hg%;^7a-|_H)hE0Bkq!yNg01VC{`xKH5MKq$ z@pD(|6EqHR*zEcq|3IBGg=e-#SM({;QY%jpKajl#1Q-}%4f4u&uQm^VH3-TomKd82 z_IOewp6N)nsw{6krZC(V{wsVg{B2$T*O=4_;B!ZgQmk-m#`>EnE3i!#xo#W_gj8zO zEl#My#%@~U9$>7V&kL^C8ENR=S6WeSTU?{DnNuhuwvT+Ov_tbi_n7J*3U}w zd=lzmKj}_*1O8>^esW60mkBjo4mjG1Y1}?d_3oW`L%R%O&64K}mc|Kj3egwPLhqdm$QO)9Orh4QZ`%&=WceEnRa<+P|CbqgsTQPl?w5u>y`Y@-aW~((f2&qcH`D$_#W!;6E70okU(UENT`>%6~A2$m;EL@_x+VrnExZK{eRE`QIG zJJ4(H&{7>>#crubSViJzM?nrs^-?Xj3=uDGl!n3(S=@V)URz6VTBaHffZ z#v{EE*H6RVk4&D-b9x@I$@yp$!b(v}d%2sD_kh35ugYNB)=nr8sT|sTNqtHR>{<}q zp8);aogHR~)XpvC)HR4%G9dHC0ZD@o+qvCuK32s0g!{p4i(-4Y1xMjD$KwjoYdf8f znvL|IZ89tCQ@s2=Ll{G6Rmlsd>s-oaHgcjb7IiN19ZD(FrweN$EP5<5{g9L=E3U~U znUWxm&lMo#g2ta}t-=0SvE#A&{J1s6D{2^WKD9DPNS|`}_7Zmoof|vC~__4}s}Am7)L9SXkpKEH3|Uc03(z5!H#J zoi<409inyR{Tavm^LL1d2{v3<=Tn3a9+{4e6N0ZDi7#JB-e*MrS`CG)A)(=Yy7WA6 z4F_3kR#jPJ+kW+3<-yuijAv?FuJ!^aI>ex6nGt~)gHK9P*rpI$9uRLAc3j0C_B6)T zx)ac?>SH-jM)*NoG99gnjJ2K#k*mNSPPC5D90rklLczfFW<4M#;T`~KigSAv2n64ye5wg7cLSy#0E|GxxcW?ljiMA5;hY*j0h*gf1?BsY{Wp37*h*YfTV%$pk&=o*d7(W6L-{RLGtB}eS?h|;ugOnZA#U&ii z*o~oIqaJcI4$zE+_P5os=nF zumk?r#)6=73ux2~W<7Wb`6{x^2bJM8ykNkHYE3DOYn*3ADf{Lntos*Zl_=o2PCTbW zA16s2xCHK3?%(srXWyg;gCCG6Y9^ua?3j?z_P+M&_6cxpQs{aVEH+49eaLIssf<=UxS_ie5!2f&qU{X}Rw7KMKj$2%@SXQkU&| ztNtH3p+6f@5A zWJJB^S$jWZh*JMh3X}M6WCbN;Awl13G?3rqq#KMh$f8njVJvkU%ZMFvMIVuWavc}R z`UBqPMBBM;^MVJt+A6i+DFh5=aHETV#+CdiCCKlFEOrfBUVPNWlU(wsP~u5bjF_YI zC&MqaAc%&_Fk?d76vvPi%bE_M0#z?$oq|L_Q#(K0AzNN^Ik?slAg#ymO<<{}YiST~ zZ}XIlm%g|1^wIu^$m~^uD-IT%eUWoK>Ylj%4@Ga6xmVb6P;OT&)^yg_q#LmM)a?tY9PILk99`BQ3KYxK0a zw;WHW*ONwnJcI+L>9Xt)%t-$tK1wf({6A`7h$$B^u<7wrs^rq!c}KW%y5y-r+zqS# zt$C~jCDR>ZWc|uo;b>8#?+a*P8knvt8UDTU2PW5XV(r2{n;OfmH^;u;f2w4b8c3sX zXpP^&cBhfmF1MjTsaZ(T%SOq$G=CK)b+df-j{)2RM$AH%|7P{_1RW3uSj8r>l2yLk zcgm2{0clzPT2c|}%L3}sHb@B?6`?9Qwhg9Hy+=<-B~`#)hl#(VoBdWc(zT4QeM2sT z(obSDx?8_hMHfHjIpry$5(Gc9`$6aN^+f)2vZdyCgD2qxqD=xt3=3$wWYcH;WDP~* z1$Jddc3`~rF$ zfMv=gg61F0aYdzH+1tQT@s+*wnk&o4Va}E8?LAF4}|XX;Vb5jC|;(7 z3%kHA@WAsLpjG8tvk7!X{x*fW0qYS^T!+2cH}|)xXT#z*#|$c6-NF&T>7?)|(GW@( zUbvQ{V7kmzZb%kyZ?HeyOUgsar?Usmn&p4u`PO(*l?HWdv-;BAw__&j84KNLP0UA1 zZmFJv;=+?82?IHX(>bZt&*lRX@+AI^?osw;b~Vp{n!KJW0>ww7AtHQ-4E$1MD2(P` zyN*i|4xXEmSps&9j2mgdeqjM^A%ihYso;y@wcT?ejPHhsZ_(B7F(v{X;)ZuaIJQ8eEW;LCF#e0YZLKY`O9ei7ni-?d#<94w=_*| zhMMMrDa1Q7ap{G$04(jkK92igmMuPYVv6#XEU2x9MZ z4_GwUdcP*`I!dE37M_w)bp{Poxp}W1)_cTRdTPMCecfo;6crMuZtyMZfo8Eo57y|> zu27;|Mql1Mq=9pyfU*$JszREEMQ`qNg8sP5dCCpP!iF?y-(f?u?`cBgU4EU<%M95< zcrRoPhm%>i-YddmGL*bs-qP&a2<7dl(T!PiGBPu3`|70WkVb#{Y7PfINqh*AUIG$% z9zb_<&%b%S7Nkr+fX{iw`S*WA7pp5?e>3rI{_%Y(Q#b>wGgiJ?T-kg0i@i6pO}w|= zs&GdpPGK#@f=CxNZBwExpk%woJ%q0W6asOC>t~&u~9^awU2k+`kYh&=ewVgVj zLBF<6B#s4$0}kFDsrZb`vQhIY>5Q*f!khi z?M#+HPziy#4Shh6=uO-c<@Y}2Bg971I*bOla$NIq&(Oh}!ze6Ky|(W2k3%xdj@W-2 zsKr0-)59{n4e5b2lrnnA|IDOho%G(ucP62+L%m!ZDO zTiiP7o%mCGWzC*81hoDuu1S~&{5 zt90IUva9ba+*eDp?RG5RBCH-Q&-%o;6xZ0+tvyo66QfpN#Vs;lvqGd(y)PGN&t(%D zIVCL!e}B>)WtfKL$6O_`Q1x2nd^B$z53du5lg!R4yG{G)R5@P}KDkM17-I@#w-$KloE_|v7`yc4BcwR=hksDhv7^B+15tL|F#U)f5So$eFbxwYL%)8g> zm+(6mpvn9aD=5M5>-W!$>o5ihh3;1Am0|_`EG60Aen? z?OvXOy@M^uHxsCzOf-4NbAuS{M;amkXG;I2Y(Gn))~9g_R-(tz7DQ<~{Dbp)+~yjj zbONQo#=Te1YgVO;1%*-xlHRn?!vke14F7g#tZm<+laycwfQ;f`%6WK8poOl}cl2p< zZ>w9NZ%8=xj5uKB)r@G5Yb@f#w>#60j*2q!b6HkT( z_jZ95->ya{Nb@tE7LYxvj|G6DE8>J<)E)?VU05))$T8Mr@>1~(`w9R*O zLoVnMu@K7-=T!;m>R7{61r73#g7; z$4!M73+a_+f1^to)9Vt^Zmu=7{puq!N{Bv|u*U97P~<*%&JAWHhz z{@s*Nt7`A&e@LD&~H^ur2dH80&wAJOU0D@ehFApkefmt zdfYvI z8^vl*d32k!b!Ew~5)NxhN#i&exgBs!pa5CDHGjw3AAG7OH2<{NInhm>$sd2G*An@r zSpkqK5KpO2Dt+hDSNzVRf!W&{m+$#eJm52)WmYV{^DKv}S-t@GABXlT$LBMH;TCrM z0*ZK!oHfq;2 z9m--5PEe*}m8u=r(~;iflD*b;d5}}mORn?ByH@{ec!Ova0&jPOwh4=;QE8-=uN0&S zDtcNfnVy#umc42x$88aoC~CNL_?!QL_-iUiZ03YU!PF#T|54gKAS z;b)CMhm}=0IRmY;4~T-V+q|UO3tWpT2V?e3m7(Z_b{@U1-hY+%tg&bh{RO&VlHb0( z$d2TPH=LD+h?1*ftB#XAah>kSUlI&$=wPXv{Or-L;#?{?99T?dvqva9YVoA{WQRZA z(EB1{Rx%|-&?B>_$o^;}BVq6zeYrvP-Io`@!>_!hMit>pX`M*nx>qM5>+UfkRWMha zwq<%#=|pCF6JOkd6l4_$M}7D5_{~s-@iREfNV;;PC)`eAy)htfO-5vz^DegPt%x5y zeIrv-$=(AGW%8yYbu!_=G!})@YQeZ>+!1do$PY6pQlhYH{S6|(Ps=0)d*ZO4R6@&ZY zBQrzaIKxIYbg-PA`1T8~Xtm-gu>d76tdJqaN&hV`t%RNI@gGQ@6!=~gfJOQm~Hcj^UH(zz|#Ec>@Zag zG?WQ-LhhIB0@HL`K>)TV4^%Ta4^#y{3R73>H?&C6ywZ91Zf0BNciJuEL&s0XQbRPf zPjYFW?OV2aY9*{Ky>`n>j6d=}fsc_&j#Q2WQ^E&WjK#}IO;mmvpH@55wY1IAcY-Qf zS9IA!37t(^8SpUN{xK-`J+|$D(j89*+}%*bhc~z(CnTFKN!N3@2*q_>DzeX$zt3J| zWGRg6D`jWCF>iZm;hf7xpiZ>Z9pBi;C4I`>=7J;M{l|vdLFDc7K_6Mfgm^`&c=CM^ z&AqW|KyLe$ScBvs_=v-Y%3dZINf~pj`*0`-WnSeOQJcZa26`M5))lP-@ljI3{&gEq z^Eh-*Dgi@TA3vxE6DPdRAA4-Y>VH{k1q+U z1MP*#GG4%{sbG)=ofT)3AELPF)r)ro$Q8O2>QsMC6Y&qZ$$>sE6BY>YYI(U9LYuJy zg_O{w*-dhsf>LbLm|ISOfKn5=MIH&~ZwW%{jE+H1-78Hw?gc^%2b!Kb1kgZbf&B@n z^J{%|Ceid-Mxb(^0Sn(cM5zEO@h-;JTtVICd$Y_KY694kX|s0vP9~mF>d8pTI^m?V zO4e^E)VXAfHK6{wQ!)(&{v+|8)I+{*lofMn!Q0xcxVrMt+C2{O^dgGR^UQl15}7I> zzNmJIXrfS#bHTL`>P0bO`S&O*!CszHd*370cJgpYz544h@`yWL>h|L|q{?ylOdlR~ z7Wn`ps4FpQr@)Q1)Borb!b|7(htY1M*_xJIIxx>0*`uyE+C(HCCxcqrAQHcL;RD>P zd@GV_z~h(6Ab3y;r5rQVU+P<5m7RDI%4~2C_`Q-dInqEA{DMoObx{Td`UXdQxqBcN^4>1RabUXc{rBdoJ<$3x+KrO9jOs(DHSBgr?mb{D_1s@Kh~o(rIjuh zDfls{+=40SShs-i^-~4^Q{SeG-^b2LtEh}@GDtkT1Uz)1EeoaTdd!;r@wAU85jaKM z_>SZsJhV^m@K}_39IQ^u`5&_V6Kh#vn{RA=ZRC~q-|l{`kL&!`MKouz%5#;8mv5DE zr`gnLOgcxX#zGqePl@xMPgZ~H$uN+O15qxjm^jKe1tZdD0zv4~Te89*uqDY@{F3wGAueI~kH; zkT`VgQvo=!D;7OG_e9}nGOFfH0lJ9LA6A71-!C>Lq) z^!fqgDF$}xRQLx+Q*1=SqW-**)6wOu_Iv1c9C{+|iM*27!q@6HKvcIUK3@(=2tPq+ zpCL?EnBq$GU`d~skp9G3!~8rp_ES*tZc*Yxece z7je|TZ-fI|ly*L>r4?7(I6^>0qZ`99bCHlS@x``evbgZl7#JE6gl>7WN10jaI6)LT z*C$UymeJ=i_$Lv$`42IylosjL>sR@ZT68&Uls36EG_W)?H3tUE>HB>6tiS)`b*(6m z5+;dD4M?(7yVdx%dca9iMiJX$pZk7BU%0?|-D!#jLf)bAD4m^Vd*K?q)Y%d>v!)8Q zkVKvh7ZsoJmi%~x#~-2s=420b4%PuRbqw;(&b5`K+)Ot`KUw@%|omH|wF zeu@*(r-nb|>NJIAky?G};CRu5Riqj**PTnzZvMfZ<+B30ww!kb?v&UqM6dmUNRGs+ zO*Q$R77H!q5LW$b$eAWGSmo!m{5wT<03w)Wj!a|knx7X!N*bYw`Lj)sI-e7iP^D|y z1Nor7W80x)cY^-b(}(&b28dlzrAYPC5s%JsAx*!5V4keIN_>4`P!ntN!j-jTGF3Zx zMvS!Uw%CQf&TPV|x_BvE6vE@%aAJC|%sEXuWbOblWh-FLLzvfNDDICU!skvUlyJ3h% z=VqOLkN*XKe(0+N@O0H{^#E??ko0~18x0f5TJ#Y}Rft8*B}Zj7G254{tGLldNVL9| zIO3GSXDgcSmq4$rhQ-|)IZbjXU%Nn0=H>H;8@)%y2S8&(+wT-w$NyQc@{eL=yaxkj z7>?Fam9t;JsA(H%Coy-008mvOrt zrnpO?MT##tk78v_xHvdykY!`{1j<@;=4+)~HfDOk}>83r;j#ApNQ7K{yZ$5 zoewKZOFNwqW$FbqU-j0Np~T&Xsvz5YvU{>yVn+Q!JaJ&0DmHsYtukQYw~&H1mZK-~ao)dDk1j2lh} zwg|G)hG?~-TMa!iA|awJ6FxGm8FO^$Bui$T48Q zOJpM5FRk{}C0`0?7gd{%HcVvEyl(bVsfVTC2&2bFci$cL_%SO`HiVmSujAjYy%6>; zc65yOxW%O-i7i4N1!hsQz7rHagb4!!D*g~7w7S8sKEAurl+yi`o&TiZi2oyLP;p96 zjuks*#sVpl&&%c2Y}?Q;`OrVGn~ZG_<~_fV#({5pKz5nFbL6cNnJ1U?j66uS_f+o? zi}UDSc$QnldipuR?$u)@ea4w-B>j&s1e3~V3y~XG@_(%+tX)L8nZ_kGQ6~3xpLJs zX_xY8d^k&YK%{eIwPv4wY|On`n?zrB1dNjD7&7Fh^`T+ctBB96)gi_c?+0fRUYIfc z>cZQhGO$W`<7<4XdHqhE1`_F{pXDEh%W5s@TcuYS)!IBpdprcOb8SxOqpTR4hv@GC zT0JCx4j08j);yWgK0RaILtdI=xlVNr{)_3GkkF>ghG^1B*(4!o#OJNhRu|Zb!+u1i z=Op*q)HbQBJn;YzEw*S-wzQd3f=B5EoNH!CK#nW@AmppKm?R)rPX00M<6)G9p7`!O zJ&#mw$FdD-@JMCfy;Q_%$m{r@`0=O{B}`<_&#C(N=D_(WTT#kh0(J~2!p21pI6@mUmO&Z;r8ZMbNm_X=}KDq@x=P9 z$7Xp;&EK9JPR9&{N-l6fPJmmXp`QeNTr|_Ao$&dh+gx?(6>(V_W|oBlSF4Zb*A`Bl`-K}f2{nnq;E!90yf2)UMDaMe~?H{8fG0vt?WOrF*hiY$^~Zj1z{JV&DZ-+cWTAa%*_oXBcy@T@k#t{+pgtYAU(`=AsK?M3%aS%*SdPWk(Z-HOLae^@^r{Nh``3`)Pj?stdZPmEyufKo=(ukJBiXGT1D8*bi8fUN z^*V-AAgN(NmQ~Nxl!ez$L`v^AfF@d1HfUz0#pMh;3EHN^+RuNx z6#xu>rkN1mRl57W#6TYRxaD%`JXJ-|4+w^`bx7(|GPyBE4&Su$&eq{Ao@+btpFH77 z;@Ut|Uw+hjzmVg17pL0 zphM^v(2KFV7AgsIoL9OPR~fI?HtGYgeklS6(j72n)*o30I!;nh>-+Pl6<(OfY~}JL z1!k0G>oL6r?m+4(swQ&uazUk61H>sP$gQC69MnIKe1c58=MZj-@V+oeHjQ0j)8~&Y z7YEVGJ#tP{iE+ysV|Wl>X7mcdr^dYl;3fA?O!b++se2Y>21|*hzLTj$;#Ozw^ut3b zvs?2wj91O45<-uHI-mS>fHc5ym;~e8YSS?%@e_Y_Qw93>8id6mW{EE!;QW0Q$3A!s z_uZ(h_e?ZnXawc{%72;A{RT<-Cj&6MM_uvV)+Jgd`?F+DJgPETI5N39n2dTj!ZCka zIh3ecy&a~zls_ajDOaaJr5-&|&TV+MG&hjxbO!ROYjbWQGE77o8p!aM2hlt!+g0oj za^PclGnK88WZP91(X<&A#3?=j@oz07h#><3IS%-1=<}SS7u?J=0vc=ljn(zqs#tS> zYY+Yx-Zgq^w%F2<30)^i`{xdZ3d+PPD$?yx#}vYXR`IJJvE!+z0_I8iJR2t#EEY!s zUYl==N53`(DAef?A`9Cn1J{k^#jJ8Nluh88zPWvo)df&8x0r-j$?@H%fegjuCheF) zvcvgfX^NgW1bPOmcO#w+wPN+5tq=L+DfQWWqr%8PCshw;03M7T)aZ%4A#KJH{HY0Z zk9}F)r#$C~Fp?L_87}EHER&i0MCNfyW|p(`P|D|r?{@}|z->pe`jp&>p)!k=D5YF0 zF*8)J2N)d~58p%_1vx&^Ft<8}g)sdm)o)Pz97sTu9SG8bs?>6Un@>P5BjKxrD6HmZ zkM$o9`j;VHKw%~#v5-uH);vL@vRUeF?eAZy{}Kxc?!=Vne2!z#Ve7^wlSR!rHX-RT#%>JUw zq-@GMS%cR{ZyeC@7b5D-vd0}c&_hWvrSqN0^Y~nvt^G!`I43i^hu}OrQV;K}RwFXG z&B{7s9fS1C5in1|gV4wmP9uM>ryw5BHe)-m&bYI#?6Xd)XkH3qKaE&(i-ThCYI2>u z`uzd`x{x-M+n{{;NvZI~7uQ$Xmz$Up4b`Vbj^{u>L{;Y;y{6zyW~)?0zEO$g-OhS> zwgicIOnKb_&*I+L#y`)G0}I79?WKbHI5bYUzQD`HXC>mP;)NHGT1xNn2ete9M^Dx}TeM}BUPUWT;ByGZLXh_bI z`1;+B+kzdF^jnnTDLIFD7Y7;S_By$4N+m~YDF!z#W`r7$$+3!}(#f}82&u>$hZopu zMVI<eEemDlFBVBxg)h1(H7 zc4?c{Bd!N&Po5HzS|4s(6zQfs$zuyi-!r8VEdi=#_VmO?({ArrtI!bN5-A=)1i6iNV>xH35qp0MGZZzoQOx>V)_AUO(t`RCOp;hL?nD3cRRtNyFBNn5qtHiGluiuPSXqs+x^1r992D%L*ez@gEjo5~U;Yc<%PIeY6 zezSw3k}NYUlGtiiK;gULW+Ox1sG*LMl;J@%V&|G^EFFK=7UpxeiWc50R87>JX<;Mxbods0flyXHb@1Fbla zsAK}ldDs!cvPhka1TGFwhko+d8$%N)guEv9S9!jy>Yrcf=_IJ?7TPL`p%~O@HF*8I zO}q)I<%>&*rhXEpkrR}}1lF!oD_kwyH$5?$tWi>*b~^kh5L2XDr`^z$-@@%F%IB7z zv|agyJwKTdBWhgu1V|fPqepK%BC0+%j+$w<8hj#!Mwfd7+23q)I+|mM6r=M~t4K-A z*@}94C1s^@cnW+zvShfPO?Y(`ODm=_E;RF8u~||!`{8=!OAyfcw<9`b&m^p{JM`vK z#-^c8C&p0kzLePP;&7!}k8(85qOA%S?dhm8wpBGt6zzUW6k-h^9~~7?;chijFqI7w z(3;*RA?%yN8dVxI;Yyg)xR41oeW6;-#?Icj1MYZdNw9TIt@M&wWiNPLJzV2dwBFlg z)8gsTrs=8=oVI1oCe1KY2{;{<@z1{vC>0bV3`_$_rj#io9clBu?>OUGoie6%2AWFx zPu?L-`=c^W%2YyF76>ZPH^b2q6xOv+H5j&&Pf~HFQ%kY7l&sH3^{o)oS|%pc>5Gl7 zO((wGx@RG(DHnY7NY0&S8##@nQ&w$f6-pILNs(FwRqpioMQ%bQa^e-EX@=eb+k8a$ zySkdRXjLQ$?P!<2%YC=sG94xd1yFjwdE2u}_FFEsDnNyxs?<)jV8pu?g$vQP^F)E&ip6d`G?t!~)_ zKUB0hUT+)*trld1hgxkULe}4xyes)FvFRJC-m;{Qbqz+t-nQ~q^l5LuTn?dDl|(d5 zbdQTko13_3SwNu)nCRQ6L^f`!|*-w@wS6ob$TFguOonN^>^TUU8 zU3VmIp+9uYbdhc0Pg!bnan%joUBbZb9O{8#Abbq(nxOT%jRu`>RA=b2C(FrYvfaO^Z+n>HZk0etV^xseyN0&01y}Q_JZamlru2XN%hG3V?`1ct5n$` z9U|F2jiaCT-#4-D_*M)EKXBi6OZOI`rx(&OJiI)gkY*~C>uM^TNCN?tChzDfkd^yk zW}X0?R;cO?M5(ZQwhp8Byh`2_N>ZEVJuIRrQCfc0Mv2-kilXiCq`EB95l4k?z|6Te zK?=h6bAIu9cHm;HKxW%A{{U+L0P4*&sc^MwBnqqo)lricoI*(mifT^LHlIn%=GLVZ zC?Be()O%=)`DWKp>CkSWvZ+?-%&`}>i%U~1x_hl)b$W$Zs+Z}?w(Gp{z?8fG+Dmzf zZF*)}DBV#*L=?+R#NjRFqTW6>z}VVdXDG@IfL;nEu(tcmK@UK-u1dfGHnrSx@{npBimAUw>n)~I#iPC4|ZVA#A zr9*Z}X%p>kZ)oNm!-n9*PsQHdX>447P{M@zT$Dpjb{mi)*TR{%{{X4p_-z(J-SUsS zGmG!d2YFga?Fu~RqENZJW{0PTYd}<~P*}8Xbc;>gR+s41%S~W)Iy9k`P^T?;0aVcE z`iYwrYiYB5;j5)uwa(K<6_V9N(n%!?#M4R=Q1;sfsTD>|Q)IzvF*<^!bch!Z)32({ z1gTSLf~3mu(XA`hBdE$#x_z5q87ka3iY_H8C2~}?10j7pE8bSr)9U8{_=*e|_r**` zBfA2qWz~~1rxNeHbc{rd(U_G6w2L_tEiEe!s~R9{&JjwaROCfz;{ug!NO2 z7TH!hnpdywh?8?`Pi7LJQKl$>DVFML5GTFAV5%9fKSzUeRi=8yJ0?be5{n^S*4(V4 zu3PSbBmqvN5}xqQc$Vv3H&v(8WTF&cRPi<7iH7m5=p`$h$vb*sDb@-LQmQQ4)8HQH zVbA8K5?hHTq8f|f&%7!_Q2Uj%0e7Ahl86HEw8kTJ)=^fZ7f33To3PR;>Xcd#CAGA( zYXF*#ul>SMp?4tKl)pKfr)VE}w>Np5hJYW6qp34GMjW*Hw$+~d?DJR!N`{p>mBoa_ zQ_@Stu!S=>3s&!|=ZF^koJCa;4Ui`1^6B#4dE0Cb?@PLxG%A3${At2z?4^lq(_S*9 z0`=9VMxCLVo|2e)h%_Z@r>K?oc#XQ}`;F2EvSvOT-$ zqX8tSl{QiuNr7udm*hcAR`-IKh3h=o6qSA~k+mry)hbs0Bx2&~LLXe)MuwFJZYNfO zLLof$D03>)XT9@5)f6HlWGWZDP`Sjm_J!E99mwBqv^rE8jVc$UJ-VpK^Wo@|)TDVv zug5hBwlA|Y6wR~E(cHhJqf`&~ikk(h;i(s;n!j#gTp6ZpDpE2}-JL9-rmx2|)rh^j zJ$@OAokF6?rKvWz-!&^O{{Z~G0~F`Qdun#<+s_pEk`sfZLlHmZ(#;OOfd&Qnrpc?d z{H^my@f9iy5<6RXzqcmp_+ng0oW!V$J^A@!R;mN0Uhk$z)$T|%_h6cdBq=xSjEF85 zzT*HXlxvsbc1;EP?-+w!6Qfn{_fNHB)=7J*#uUDnPrN`i<956{p|4cMIsO%y>u}s! z9V~;^CjS8L$Au|(E}1jCN#$@5sFPN0x%KG%nVzo;#Z#vBj^irnx9o2P!qzHa$9K`D zzHypmFHAZ5;S8e7Fi_)1Qxottd%x`&tGyb&=?#`;p!G61OK)5_mY3O?sr%*WgtWHa zSQCt_-2z_n@Y~NB(%}pO5siXalwVAElT0~s6@1OcIH2R39J*88glyy2mQ)w?s*y6M zS64(`J8yp_iC6VQPDYa^?cwoTtS=YKG;c4<`{j3}BBJpB0M8<#xO%R3@Y^E%QlV<$z*2<(J(BMRfaS zeTbxd;{fZ^V-LdO{Pew{^Z9elRLN%BKoc=ggaf<@wN3GFNrVox+I zV!A%&g$=8~(~gq0H1^&}-s?I{@>IpcVOpBM>Z6y${Q4?(KEVUx0by^kT z5YpD(BGaj9jpu#msfwjbQ)FhFfrjZ8u200*rmaKEb%TBxC00w_V^W|Oyqw#{1Sz+6 z)0k^64zzDZC|bs%9)#WX;TJL{{U{FsD3TSyqRtm;{&pL zV8ke}ihvYNK?JI>Xrd6L;u_hqc8qODAoPW05Rm@i3hCq4O2Y{fX-vjcaCd=7*g6Ok zhQtV)P1;G*G8N&nY9`Y{Ws{4N`9H{{W=Pru{`OtzdBXgHjn+sL9F_4Sno*rRk{M;Y>M=o?T8D zO_6}sl*pJ9VAMWZh3b_0$itNv!IL2!yTO9-5$g-6P7;**ZG_WE(05(drzF$wbt|Zb z+7m9C1wHpyzz`G$mNGjaI-rPR@+hE%OWi1YZ5AbJ*203#U|)7llR66jSxe1sSASLTF{~IdZnAx>0T2h!@^nnr(6wq!sENGq8@$LU3Zp=b zIjO7w(_hxaFU>{mkKO5X^eSZA@U`D-C)hHjJ(7$orL2hR0a74_OGc?H?M%pQqr{tk zIKZmRO)8owlP%_>dKz)Kkg0p#ZR7V7rI0`p)gS8?CoquLtlnun-@&9lO|8X(J|E0zNDdeY=?Ya z8YD1mzypM118Rr=0C5_+ zr}@Q*Y_?8%?ywbUrga%32y&PPslMeW9?8BNUalHNrpV7!g*sudY$M=ayxqDi46j8> zj#Ua#cjBr^?P(II<@qexT4^dmj+*eJ(nBNSZ2i(lDo=8|S(22hf|QLyB!Sba0Fo$Q zkV%dC%A7wDQIQi>RWj6*8K^C6=EB9_oXhEeIuyIaF8^B4eBXjZW%{7C^c zD)Lh5GO|+SZGM;=x+T%?MS zB{6PKY`V2OvVGk+T2R$Cj!rjk#o@j5ryE}Ygw-Ho2%AzapwdGF&@BCE=%}6ur=* z)6MOEa6yH>6O^`n4Ni2U(m`r0sSfI_Ilo;U?{UP9ZK! z4W)pOdI+9{r7A0&G)&NoCa*C*CA+>Zx|Lc)6zK%RZ(W6)%X);3j;90fMlPCAh)}!0 z%O0?%iB{ETE5#!4D3O?6X@Y2O2q}!0qh&rhg)bQwc4AUhpM{$$z{R_MRNScW!8_i| zlInWDc!lXo1r0MOw>X5N%1O1hn8v1^Kzaj9N@aWG7g8jtG?Go~h@N-}R+*8dRh2fJ zJwz?2aP)x1?X|FvxlSmJ5^yWTs~)EbvPq05JEgX9qZQ%rYNVS$O!S7W`3%8xQGQP- z>J?2C&%65#2@{yvn^gvv_@?mN<$%XVuCc6Wi^cC#Mq)*;-PUpsXST0Y^Nqb)hgCo7 zbg9$r9mR9ycDKr{KtPAWG&N}<`jb=pC(m_xu61;PI`mHdVSKAt;Qf-OICfn2Axo*xIXsS8w-woXCw4D@w?wPty zSBPl{8f263n@L6Ol!C1$>7r;`P8ks?PGyV;Ica;z?30|9$kfKN1v*D{CgEyCsx3B2 zw!lh7HwdL|b%lz=hy+28hgWl~me4bxl5QsNwsUn_L7+po2urPB+;0$@n|NmM1JB!^*^lsIi! z_F*@`(s1f2{{XswIL?0czex_MfWHW(*t1x%O8J1R(-*wYMB43`+XyvTh_DhbO)bM* ziZX7j(1?>VN$npsl@a{ZT|pVvi|^TKFh`%g{G=eGOJLvYGb4tfsU`Q zwMRUIR;XM4II!msD^fcrbf;#mg+?xXO4w*3EiPkh$muGY$=Y1sYv$hxN{Q>}v0-U% zXvw5SjU%SU7v@{I{#I?LQKDkz-gqNDUL|$1k4y_D1djI0&hRXwFl+tIAl>DC$4AaV9dsq(>a^+c<2E!naa+Y>pcsuNBl zG$Xj4^utP`X(uhiB)yq*>wpH5@Ue01T@u1^6)Gxi^S0eT41OPtbyF96!6JgRKDXz9 z!CXvQ7CgDb_HTj%MB`ZQ)Ml0x$Rj|^ry`I&wzOM&W?mDApj3dFRMyPYrjlA_H;)m3S|G_pQ6cuZz_F7$K*`4JvM%rb^aElk-L^{WC&nU=r4vA`|Q4=o_COS8JZq@qA!=HQ} z^%A|HBBVG|Em{=dYg1$_Y@#2R&o&|`Wc;DZ^}xcb37EP60Ji3Nc$K{{)Mc7}kFyV` zPUev~i=^$+cg(cHlM!%2#0FrcGKk$y12E=eoY~V+DYFDOatSYFoHSVJ)5Vbga zlxp>jQ8`))om9leuBUYfUf-5V{{YRhs&|40BqWojP}hp_vx_TrYZ9h{Ds|b#I)wDV zh?ycz(ASOyB|aVBo(YV$%9&Fs@DS@lExu%R1*N<;_gSiwO#CFA%YAz=;CiJrnoX}P z-*ZgmhWap*Ls#7z$I%EDog(>o(Z}X&3Sv=n#iCvJhu$L9q;$re+wg@WaPJx+r+Cc< z0DcnH;qZv+_k1bppLw}&c~NLszosoir%gfBL*mj!r%&~0F2ylBxN`T8CN+!4lw@@K zwWti~zM4x;n>FInp8SbLlbp3s>iA7guXKs&8+1&BF#iChuc$}*OT$x6h#lie6ia-P z-MQW2X>b+)0PUa>@c5*(!B30CT)A2KIL&+G_RpqFwH$+H$d<{dv}}aC!hF<@o=WOx z3B#Oe4++w9KehwzrFyEJI<*L5)6KgQzHc9<)hdQcpwl5pf7CV(^2YQ{u%EXpw-f&W z9!y6;(nc;Gko2=s8m4aDl5IlAz9*(%Ob0;|OdpBIh4>}RMKz>ZH)g*#rZpO*%_3+Z zTGYsgWZaY(CQ!7d-)y&Uw_Lr6|G<1pwNegK+6a>(7g3x9yrsmVMjk02o z0ZU@5P>^aun1i`)hUfxJ?r~+wfkv|N!ru}BA=t--WPTU7_XxD}5)!`-*E*|?7uB!9T zJs0WtQL59V4t!D0dAXfmJYuKAYA5=7Wvg~;TmJx>b3N$2s)gc6CSK%|Z^JzK_fA~R z{63F&Zd*=^ew z^UK|f+F;hr?nj_mH?9fVCLMZ;&Kx1-%Q6iLwQ7uuOvyPU(!1TgB%_}-o~oTf8bUIZ zsO{~{)Hs9gg#(}6K7Z8T_Q~BlNP=D!ph(7>i4ZPPw1q@F%Sf8O*h|IwDV+*!604=@@GU{&eAbG( zV{Z{de3m)=Gmqj5bO@!hW+`m#2;4bQ%hxtEM7Ez7w>Q(O#1b#^#pS1NeWIywTv*a1 z6xQ&?oi(Z6bqzoyaru7D`sWYRtArACMxwh#qrm+b(`!^-X3q~#iSKnJT4cO8_m;2i zwj78rsKjmgno{~HYW|r!wC%)N;2kC3)HGTXGFMkyG)R$5!{xRTw+qA`<_%gLn=QAt z@U0~CMRQMvtEQrDEiEVR-a6uyaAfA7%7uQ-_s4RtqzaD$631SL5Qs%;o6FSQlU8X& zt=<~H0I98VB>W<#kPK=xrYWSGQ@gh)%B5bKH1(%061s1(HS^Bm_@q?fHrWz;7ljEX z-qFf2?|#nSBGme42wweK=3*m#L3mMN(&1XlVOes^d^Zk)1=Rssw2C-W)wHF%wJz}s zmMsn-IGU^gO_r5p4fVf0bhim6xGO? z)DdjS6G)9p85WbZoK7`bJUY}m5w=>j7mT^vQAvE`DzOu$1Kd|nycXmMY)Q`1<;yrb z>ZhP-7O8C_Oj7}!O}7!t6Fk1pSXz8N6YvlwDYqUWQsa=2-rMHmrNe+W6se0s?u92B zT6c;3b2U1ANL;9>oz+@T{i|HJ<&9K$s+6E;!#5YG3f)!l?^jTDR6DwBx};vc*!0xp zwDN|kq))p@CuAuK*{y{xxf%t;fdw+A4qOQ$Nlu3@R;fu#n0}cmG%A;fdyfzQ0CC5= z=}uf7Av|rfbQiaTOw*Pv#zd2|v3Yi5+%*-7{?^yb72bOMBmJFh1=x9v0^7e>c7giJVpU znRk|#%S!CXd2vzVNunYOYnq)#y_UajBjM?VI)+S$lOu&;Z`M3d+nPwcQR{)OUIS3& zi&pCGl}1XKlF$QahEPRrdY$*Zqa4h+=20Xfy607{DWWZlXucPs>b6OxzE=n=Xte&> zKfCV4DvqHp>{<9_$K_tWd(PN80mXMErALX*rUQRXO+S`aQWYpsWLgE&vQNa_u|tCo zwo>0!_v12q+omcKt8Jq*vq~?fgD5)9K3)m`0E;O8pHu~^b%BcNm__-K+VmY|sbv`t(qxQjyQ)68u%NGl{2Kg+>(vOhuE`Xr5BG&p^r$-=;@au0zHp zJ6>(f7W0-Sh5auPOpO?rxLng-O*qU&!SOWeCh63LvtmtT*L=vOw)VqSuF(V3_G-<} zHo8DABCzcR8>s&PEYLh{GlbC#NzB4e=aU0moVh347z*SSh;<5{tuJOaA`glkQ5H{h z)XrFOVszZ<-F{Wi&0+YlqzFVQ0(z5Y8}Dof=-<8XZ2S`AwR&X~JWx}!`KgTB6=_ta zp{vV{wo|r4aWv9x^vPl1=zt6zRB;-Xk_7bKsnYbmS6M7b$4&nAX7`z1m+O)bQe>hp zng0OssnjXrbtEZjYV&Q*~c%np3nr>g_AIhIcY3r62weZG>6PdO7{#7}jgJkJ{h_HMj zuiX}bgxSeye)AE9KX!E|Q-%*6Dv4^}NhtQHcwMy~uT0@=KyA`|ur)ZkZ%aZ=Ip~V>PKl-%vM64v^Ll`HJwU)8L$($<#*x;?k&nJU!-qNA#D{wA=Z(F%`l6`YET zJtt0`no1KHP?S9lDYra1P^T2qWCBHWo}`uRle+fcD~hvsY4Y7;qndR80FQpe@P(^O z!q6Imr;|_bwlIPcri6(i60>NMy0kmOShOHZE4ICT&0 zRC67a`Q`(yOlkzlXfTKF!ii1Vdl$Dgl~3DGNr?DTY$_>aQ7S^}>g%pF5rsy~pK`Sj zr6fvaAy;(b6NcxOTBlf9c&OTJwYTm5*t|<74I!w!LHQ2)NOths38{@UdMhD@QdH`F zp1OT70a})k@4NjcrX#y;C{qpU_34`c2*pitO^B7LS?cDQ7`W)#Ptj&3buew(lPt|XoLdt)w!(W64-TFrQ*N}*{*JBl4UU0!hpmk$C(PSDa?sx=R^wL|GR zAbdd&`cS-AW;5HT3cJqe8_O>r3!DEWSndg zS}4*(i`|f3-#ugxxxJ+i94yzD99FSGy@XaL7CXt8~`#l7|W` z@U%mSWYaK){{XF(l288tO<&h+sy3P?%)6Q~xB6lDVCwM%t8m&|OnbpXwuUhge_U~= zQJqay1X8Ht2F^FU<~rzE(r&63sqIh0bs z>zj(HcXvDg02iX<=OSrSrmH!1NL4j0IBUf3Efi+p5p>>~m1PHovIyEfz`sRzgjm#i zTsh?M6>67#P73TvJN43{J!2k#M-NR^t5Gscp)k{UcD1+6H60peEY7v%GS;m;<3UHS zmIS2}MNfWH)^I||rP|Ie)Nktsl%42X!egs<>`3eBnIY9CYOeJ?3J?>N^~{T(@7^S%Q}s_TXWrVz8;+} z4C=>nG4^%Uo?FTD^vn2GI)u}yGw)_FsubIDo77Kf-TExn!jPS41pIcJ$ANXgQ>?voS=@CbV8d2B;STfZ8Lbm)ux1haE!b*5rLiV=TzV2gsD)5tu%|T!D5na zL?>nEkqJb2YOw;2>g5NqezfFCXrV@^%H^eW!fGi}TBFl!ZWp_#M(zC0T~wRYn56cMiXum+ zr%VYW)EhnV)kyHqw*yw7sLWuX4oju-LY+-QF?)P6vkuS3-KQT@YnPrQ2H8L1 z_-8{=k+BzT-^$rl2!ffpzm2E<7nJ1^#NwOoPiu)=%Pc{7b08XexkFA|d#PrXRIcUo z=KA*L4iODSP92cB^sw*j&4U-WreF%R)TGiVY?QvR=OgP-0S(soPVd_{_GbGS?v>xm zGI4rzp}!|9cMCR@N-Bx)!X^Xm-UbOdel2k`zdtCpUvomu1a=XAIJ91>AeT&aD%ccU zCk{+n5)VNod#QIty0#?_8i8=*<2hPM6|C=g>F4X3b$De@RSET5U}zKWfv7NI1fBWB~$Z-xi)!Y84Qnf_lpU}rP#>_F67(AR9MKW2j2BnLSc`f$knyM{LN<|Gj%xv!pGd%-}V;u>5N#F^X#YiuDna`B&uxs97qhOqh>KFHSc`=df;+5G|N-8ThG%8OZN(cFBgI%cM$!Yw&j$9 znK>XIdz1&FxV4S}N}Xyt1p-p7ss2)Y$^5-?4Q*380*06jseS>p&s6#5bV2%lAv#pi z2sJ22ntz6B6-m2;C}iS^oeGV|QFTtyCQ$VpPJ~syoTc_t{?49WyH8GySzX zMG%xI6)C-xLg}_~*D|Ajw@-J{qJMTL5yMdMh>NDBs}Z>0ZA8*jzGLCSl2uU-uN|JM ziV}->YAZKAa5YytrZOZ+sU;g&Uh>qe9X=+Z4JL|6Drw$h!udx$(@ZFvdV_TdI+bH( z{Z`x8Gw}2{dY$4R+}wDI+Dl9;yaO3jWG%b4pE;G3bxXQ-O(P3l{xS8HF}^K+8jI5c zN*pNG`CIciQ+E;4*^-4CAGxd%_}W)4S{nCDX-{ko9A!ubO~Q@;01RFNrxV5S{6Y#$ zgaqEE^vg)2mY-PFzbuXVl|Xs{cS_{UM09dmTp>y!xVx9WOMNrxq_}Pn!-xT2P^tP; z14Nx6Cu4!AEYRKQZ%gL0*Pj*~Ul$zm^OkqV`5DJ~XX;)&_>KUCYl)#IP(o97D}T%D zX8Gq*t_w_57$h-uAKMA_z6y9=BE4#;>(Jstfar21Oc6;o*%QCJF3Gj%p?;sCl{wN! ztmGJ`v}o|RHhw3zS;vPb#JsX<2@y|(mRU6cs8Anx3l&4X{_008AQammZQ~bV6Wu1^v9?)pAS>PNj)ch0HWLN^H4uxaNruMH%tl(aunUhB6@mep<^QS2Zx}D$7Cp25%m~} zvjP

1r)Jx6M`Sk9|kpgcW7O`$Dbk!*QHJ7Y=V!`+}3YfpN35^qkxk>JwfvAN3sF zCiO49S^8#r2-@|itLq!@?D+a;Q{yNihF@!R+I|=5n>df%-u^95wMO>7M+wIps5}l1 zB6n$Q-R;+WY!aUmDbz(!Y;R*^mHBnR)~P~nNwu=;Z)ZMvAo!CesmDyRvD!LfhHTA6ZS?ZF6_Rf$~lzruxcw88|dzM7z4Jp{J<`G=((UZ8vF;Jnj~q zdLiQBc!GmA{{U&tou*>#ZSlvYS-SO5%kfj&YXgryxxEn*DBAXA&9?Uxup;VL8f`xA zs$1L&CPri9h&*78bRq;=sw6RH{RJ*>rb$!*s6h!7EA#lz@2DDwrEhLnm1$N)4N~2x@dJtG)c*jMA5m>NRZ2U*;H<(Tq)mBubyYEAP4GBFB6s(UDT>=*K(gAT{cf!ZT7;9 zDx}$7sp)p`z_$Hc)$+lJy(4-504ylmvlSs93q>({>igYCw#~}H1U*al~m=nO~Ywvm3hg3cW8e+imCqq`4M|59lwEL znvj1^1mEsV`(xXKKc&(S5Yy=T(!39A{*U;s7bTZqc)Uaj1 zAW}R>yE_D5uf2w<{7NS|w$<>ODDc$@5ASbpaeg=&lnCnuF8RxtXkl9Szou;yLWi4g z*_i5?4;4wTUz%(0z%@<~wK?J)oJthD>{yS5WD)v;i_e9<`bocv!&Emfw-FWsQs9Q0 z@bLt_JGlP#n?uRLmR#f2BX@|*jxX^)5DDb%DS>bK%MD|3YR=%>#V z_!@M@{^=C0(5e3bsxkO7ju|DoGQ(H6U}PE{+Y~jP+`=Qb_a_OYm0DrmuNpTxC3sZx zFJ^SLh;!c3DA`o)xV#5+nAukAZ3-{W#f?UWfq|~t=;_2;Yaic;n|OMAKHs|lBSoIH zb=;>G7>il&fG!ycTC0@4!oP2eGzl*9qMyt^^sHKm>wS}yCWh_aM}?QTxe%zidgBcy zOk1xEZT_0`*rV^kH{w@c>A(7h?+Z=eb%ZmZM9CFrvWNw@td?hVl`$r&YDqV6Q@gjl z9=Lzh1E|)dniRb`^V>7jFw$#O>KAeNZ14;;n>7`#DYf8G=a;;)s!}5q-9k-0w5X9M z{Zq^5olN5XB3hcV)TS3ryB=C9p5aodI*R&QUi0ST0W|7Kv(pcdE`t96F?zzy00cz* zQwI=1)l9z+V%65|V+cFIlCd6uP+{jMD{A;Jr52mQqWXhA)bj2iYSGqFW1iN}? zXsM{Dx=_^q;XZu4IRYt@nvtN$ZBKOrw7X2dyV2-y8sX;eL zyTd2pJt$HU-PA#QIP11_)=~Je#7VK=FZl;S2i?UR2_9y92_Q=ik*;wo_5oxHDsaBa=`FUe)ymHwl<{#_lqK7qgNWhy&!~M*isJZw2skP_ z{{YeN1yZG=niWo2Yh+Fvg=SWTkrk;@uijRzRB0Zv4(^z(FF+?L&DP@jIO~_<<&2u- z_LAf031;SBS6h{NJnE!%iK=EL2@ip zNz|oHaOmnGt#UE}jV{Ph66FtTP8B{z^pXY2kqC$Fm;P+}5xVILpr1!s^MuZl}52T&MMXWl7GZtvHV)s0e-B9&j# zbR{sijb6B?sHl=uYMXXKfUoRX)#?p15h)hj3L#?Mx;W*&G&+|JQB~LalGNPVn8Qhb z9N{ZbI&|ArygI3jAdh;g+sD3DD!`B49&wrLaiQGlMAbP{Cnwp&LdwXL5wM2)z zUEui4M!^v_`%el~V4wd0A@8!Jq276%tj=|57=)X0blho%H&=F;&UGp3{{T=`%br*X zONP@?lHw{R+Hu_jQ{wAW4_2K)r%;B!g(4`>xqX}T!2?Lwi8Q@dHEt%DQLW-q#KjM@ z*JX+fTqk&tt}3*#UzS$g6-Ah;a1@nLn$J?5sV|%FCHA%L!&BmErV?me1nRdhwERCj zTD55vWc$GX0C4^}BC+>ns?`&`Cc}8X+PyJKhNhFOR(t4H?D6M1phl8Cszh0BJ#QD~ zoQnJU=Fs)2l1uk&%oFfKxvR{z{#dnmT6ZT}qqKXe#kcpP9-tdWKDkak>{iT0L6z? zOF~Ml?-Nh6*AiY7Nx=RS@W4uZF0^{^1=bF zRaf;d`W7*{1sM$^j6Zl=5=vH<*6!P6y_F=ntc&lv-XIkCb%c6dWTi%hm!y31qcIdl zD7P<`KA|)|xE*xBpDV|nMupm3Y_WA}hOm;5w)x=E1grZmnHuQ77caebWz}*qfEvK1 zOkk?hhy!q-o`9Jth7~(v`Py*Q&yT4?T{>XJwx?(8WjIkPFB0v_X(4UA7jwoH*#$TDck|ir zoE$A`)~iqjC%%{PSwW~VJ8u_-fFcLQQWTV?ZN7f#T5-pNNwwZJ4|KHo``AbH{t2W+ z>bK4q>3&TY-;-jh!qs(UJL0A~W>j^-@ti#>bj^Vs^!v_jWzzUcYl1??8OOb=TEC>Y z$_OV-H1%?AU<-?+kwa~{+q14{8gxnO8bXv?JM~|tro|>Qe4)p#*#vbCBe|muOgDS{ z^HokIu(cYkMD1?bSLp4?j^LO zGwf~OG9P%Wn_edl5?dytHTRRlfjCX{+k6177yz8rC#aRujjulQz7Xh+r~9>EE5-e( z=0?t$X)T|Lb82}`X;G?YlAepU_(P2<4L6}#d)kUdId6dB`?II+CkpzUm$R1BgzqVc zKN5TGeqTHdHtA7BBHGgRX-&RYn{ChIfN`u0lJ2P}ZV~|673L>*gzp-Md6#w5GCEa+ zgpi0dw8E+bj)qq_r#)XnI_`v zch+3vRHVDN1yWLMuDJ^2m6OMz!@7sw4F{!)N>e3eYt-644*KqBU9+UuBM5FcmTl zZE77i@j|`%UiQhgLicq-sCX+dk!7e2B=xD&X!VYJ=9GyHx^+PjM6fk4m9)L{a^U1+ zy=1q#yO%C+z4zZZxKcF#0C)2Vd%Nvz=axfETtzz7X#!MhKimasW}eNvX`K^;y1;Ww!gyNrwqg2Tyv+>g_$EBX@+W!qjM8I+{pLk|}bNO+QBO>1?n;(W--p z6x(1gz)>1gmv|L!O9%Bb3cNpr(KS^!4NN4*^F-ezp9@^Q>VndRW@~P9jl^*LM-juF zCs2G}Xo;l}bh@awNLkm5DTgi@U20g)QRZn@?ZAXt2}3FWrY2GW&Z2W{ZLin_=2d$rD<|U6Q^Zzo?Nh8R}H3s z*9Ri&6(!b1Ou}14MjwKtNq5?8CYvcJLWG-etks_C&3fjFgt*#EgP>ZN)$dHY7?Hht_$hQj$e#!>8yJe<<~58_5RIS{{U$J08ahB93Q*SedL=``ou?WaqBw# zK?716ciJH~g($yU;YWt6PBie;^c_Mc;iOC;?GoFBu7yRpX1y&spRrBa(N2OBILNY*@?8Qi87hldK2Ovg`l zB!#z5_VCT8vR3(z^kRCn@cLBl(MLO3<_5yn=KlcAvgwpW#`pF6<14tMPLVO7Qj>%| z=B<^ZUSy|MfYu6nQ*xlY%F9%VH!x%9?Ug_pm9^I(awys*cw4(EpW@BcY7pr{-(C8y zSz;6wI+|#Ry?O1~Y?93%biJO9{I#ao?Z@Zgl(4_wWHDvTMp7jz%F4w0q{#hs>R3+Dbxs#2p zQB|TsO%v3uyOC=gj+Hh>bt?FX(6MykM_3m(m&<(TMuGPXonH!32T!K#LjyTT&;b70jk6TgPj&grey@;09ifYyrT5e?`byTd0wv64=c zRLl3~CwfuKbi3WH^U0v}tJM`6i6pHK7GmRTm{q4yLoF`klYE4Xh*9gJE}KWgqTk(MOGTUsgE^A(uDsInl@FH`2_@3*CvXfF7QtOk@0J>e$PjiwuS<$)Sx zD2Twt>+tGN?Jq3UQy5Wy_WVhw;$>|yX%@G~RH+&_s-V4h($W!>owsi9uWn?hNRdIQ zn}0KfqH+bQQ?;AumJnq_r9;)txaGCEwU_#5Mv1Vit+w1Ti^psDVH=WJ)eH8_Y@}V@ z8t6{$sHoLV$E%*(kN4}8ONoI|Pj0L4h4aj`S&JXC-%hdkX4<6b%s6ZCYbrB$d{{Fm z877{#lpZ8xG@wP|-34LBP#`C^5o#?hr0m+Ww%5xMJh)m6 zd`D5mz2*M^)0O&u784ZK_HJ@ImlA@WlPBhzZT8yN!ju&RG>9Q;jMY?-E<``E<-@UZK@Sjn0xv{+W|dGVv`-**TCZ zR-n>x3s#iWZ+|WE5h69}1w~PoqKXP9FGM(msjIuI`qfL<^cehvU=bE@X zaN@lS{->G*sa3Z8T(QKVO7ST#=WNbH4KGi>H7%hwS9@kaJIlh2{{WSSmmP2^B`L0W z#A=vw81?IoD12*ghih%8)lSPJsf9!m6Qz6BQET&0HF346_b@L}IND+p>$Z0cegmfj zNxfE6XpUFSGx=!Yh@BE`-88vv<$xh7jD}%zeZ4TW^@x^=c5QknO?7kfz|p5tsEt=p zX`?oWtBuroJS!S5;;Tg@H&HpY5RD3L*(9%&_-b0qRZ=GX5dH7VY`Rov2>$@5Eklxb zyZ(as;SN=401ITJuKxfGrB_$D2UGY(x$N}KHT$#)=%zZ5uXpViSZ}Ta^)(syP3%xH zF4$F5qgmG@bo3_nZEXvd{{SqfNQfzhp%ls8K3naieVHH}J!5Vu^5+7Q(z&?D{{Y9< zG#)x(4(|HbXD97TCd4_>rwUG?KP8$eQ=lnGCuZ9=$^H?Nl>U)KG)dJoH+N9{lAXFd z)*K}u*aW>vCyknU?hMJ0g)mMjry>`2PUMvQ=Ql1Fh^bG+ElPg-=KD8{n_(8)(QWKj z&eGOrYoskdrW^kNQld{(y$WZ|!}(^f{B;KWNQ%&rHyi%|Oh0Z(#BHVeL;nDaDJPAZ zXQxJ;I|!v@tde$C<$pQDME?Mim7K@P&A*w>N%L*r6l(M04rj`+7+C_+qbtIQ7!YdPi{kg2&bGmz1;oT%v-wo zEPVwnzrrzuZ0x6G+x#$Tf~6A^Dxsyg@e7|4T&IY-y(1-rdE3$-;TWVUXJ_c&%{KA2{jJB<7(5+1G^v17M!>ZuWqnQi#>Y&K z)?F4KhoPaBLu#c%{L7I2g?ps=!f!PSbMdvj@0WDpCl~oX*NvXmW1^|P{>N`6ic7XT zV;O^Q?`VI|D48hh%lU^Kx5Q&g+TY*ivAj_>_CII&UkMz#%e&v&zYL)bHs`+c_gk|Y z*b`Qzzv7QvwUa_q~~?@Q_FKMb-DKTqXlp)PTS(KM~1(Y5~o zzE^fUfn8qMi`*50Od_5AeQm1?SgV`?F{mn6lplqQ+8C{mP9gBRD z>6Fy;`NmzE3rbT16V10hlf33(;COGat=nbM_|>y!lFrko^DmY|Ko1E-V@Sf5;j*8b z`K;%|(O~Jc*t1J5OKj@xp1Bxe$AoO~-F$W*?`~2=UZV+fGx7e2|P}lSy^;%2!T)P5kV# zu9%kqDcYmvoaubt(3S6w95VtJ-Z!U~l&5}vlbYe~zU3E=1qI_3|ibtwjzhwGk@>-Rn%R5W8-_2z-@@KA;(>iH*s=pYeD8Dz2 zWg(_oo25*xD{;S13s&czx4WL0AJTB{%msfBg?ytf(@!~8m@0z>P!r|vLbqPtc%92I zAXSrERmg92pXHU4wSJdC&s323SfnjQzPOCM@pOxWOH`du~F?WhBj#$G)j;F z$dlDFrrY&P2UOdtA*6_+VJEbDWTW9Yfma<+=TNledMg*pFmWB_3c_94&vZ-OcXrxs zl`Kb{Q7VX+g~V**&dq%DMH=+`s3J+GQ<@cH(2>a~NurpJjW^fRI*`&WZPEg?t;YR4 zEnA)m5*n;lcHagDQ#owX)Z>;6Ub0-1d6icZ4yfIwXY9yoY8i;bx`^8)xZO1AlWta= zhM-C6CE@6~ywvcOAox1O8uZ#?Tr2s!0f%%-fN8y*- zlVU?yl|8nL^_7?_le8>!XQiP6Y3NDAOUb;)T~(=BQqW6gl760?vcaN?r@2P1pWhRU zblMf@=hF?ueKk-*t5$+;=tjwMQc&JI;kY0HX;dkhl1Y4&&J7^cB=oeJ==u5EHt^N- z(Mcvmnn@^2vo)%GGhPZ|5h_^~*Egx(*_4REj)Wq)_43&+%)2yH9Cd1*8f`+dJ3{Aj z(>Ki>#}+CKse^9Um|j|!&k0nuA(wu<`E&BZ)v2e()e4o1NXDcw zB50AugMO##DR}O&kTlKwl3qzH(!QNgz-FL}Q81->X0YHWD|nCirYG33 zw)BYys2sm$M$xCl)ux7>GBH&Nww0CKMD6A0p1^o|boA*In!Kc+{3+p5ym+!~)6Iv| z_`_z&Q>PnCO*=ocsjc~}+8<3xB+0v9O|NdhhDwz(Pwe@BmU6YX>oUG;8#Ky17Y>q| zYNSH(c)h&7S)eKC_fU-Q2@Jia*yspt5gmWB$CumL{qk*e7z0XNF1{^T%}C0Yx!pqY z#%s&9Tkp>21fOgV%V+Y)pxJX*>i+C?*KPSmKB;#0-Twf5q@s6o4*vjwU13Q#%fCe( zzskl+^0t1f;*-iKYGao7x3V_EXHKRN97;E|;HPf9IEWQv5!x!oaO9 zDEsFU3LMJk${(A;q)O^9_L}?C_GM^4Pi_Fz?RW6`W9ribwE29V{j##4;uJ33Ux(qB z54X$lz**|Eo7*}Xl+s4k{v?0?x{ z4ie;`9raekN zKMZ)gi+uifz(!3jepmc*pq1{se+yxPVW!G=PHpjD45qa`y|@8;O|8~GSj|Z4v+aL& z_(Z)W`ZesboTz)mE~!6`0IbFu@?Y9A!ES{=kIxyXQgE?oXO*+QMvN2<<=S6_ej!nj6XH}m#eI_}qZk6fmvG>op7AA`OZlth0%v5^^z zgrQMARDsU!5+~-oUp&fRO`$#wVG0%eK6Ciy?-wapM55D$)5mMBLp{+X_Jrfx3lB$x zY0QQ6&hejp#0fMoqg72m^dpaqO_+I+By z_+nz%2TrL~nqo$+Ehg8UzBv)CvN};Pt&vrTsc|&OUy32WMw-c>;j8M0q!dh(Q9P_O zh+jq;wL86C1#{7%H{>lpmR+u%N{0o-niMpcwingmxA>Y<^T`nUVUk+NL!Vdp zeDlzvo@M86jwFvi$NS`~q*D4-6qK2fby`=+&TdpFt*I&JDZh(rG-%bdFXziGi7U8X zd*s@YSwN<}G~47k&Bn}|=@Ie5=_y>lo>_VdE-God|EYrz|S;^~%y$zm=7~d1?x88AGXceY<>~ z2RV(IKP8--`z?DP!MyEHv#?*!6>9gLhpp$>(!wwJZ`{Ij0Cw-^Z&V8A3 zJ74ZyxK&GBracmPno)j5?86&Y**r_nh=_ z*UK44@|=8L2MDLW+@UYQV`*Q^yhW;R?ddDwe7;tG+@U&d0pyW zTR)B}BTB<=UhY`K-9L)^uq$U4&V|)aW?4(?{Q2VPexKU`o0hM)W6tYuW6ElOl%wy( zX>Bb(FTFeqY?ZX6y6T^fT5dT%Jg+sC9;58Qxn$!^eBaABkuUSUTK-}G0QhqBz4WHn zIiJG;SDbp~Ye`=lQFBYNsQdDZOXQcNH}JsV5nDud{ut6cu@blPv4vX$w7Ki^`}3n! zn^TI@ijWRq|fu&@xnfTOirq|quU!w<{+P@Q{KsYBjKIwz~$iPUfX9Q z75Sst@wQimZ};an8#wqp3wQpg{&{bfkj1roB0d;BI{B|D7ww#G?7rW@{M@_?R=49n z7{($>-|>DJG$qZa+N0+`b|sU#W9TH;$4|!q)8w+DdPmLR(Y|f3O(?!%o^oPOcAsnc z&ApfeZd4`h{#GZ7_9fYWlX&p<)qbxlL96Nq?fcTeq}*)}*ZX5j>Frr)+O&Qd%WL?5 zhw)>9(+k|cq<@}Pl8dNa9k1D!Eo}Zc=;}S%z!^!s zGS=Q@_hUp&r>tQZq`Q4S{8bBV1mFC>m6fYXdWW;bm&zVAPAnGNuV2Ndl@y9^b=!Z2 zTrKy?U^|f8(W&-ER10b~CCXj?Nn}&m?xH%`eB(JVVBaMd0wM>l_i%bDBZ_~og4##XLYS;5@8mtJwYUUAbc zPQw?x@Au&ZWk`#^qrtfJ`QS2fy!J=oIoju%`79Qd{{Z~T%ZOIp^-16s&2Km+D11sM zZfW5m!g@yi8()r9xux58@PBp!LP~b?{4wd1)c4xlqd5TC%75-(b~>dmR;cWs!vcx2 z=lA7PUEA?R{?0a(tds3`QUJrHy=28 zax-?PwR|yj4d1)xl!ZQJ3XF)9dZRH|Om8o-5MbqJ1zpNO`tBll`%K)H^0$ z;{001w@*A+k6%(l_uvq!15^C{_<~5b`KJCh#%Vr&#Qy;Q0H0!5u-?L@bKB#9TXa^z z_&KTe{n=$vDe#Ky7@uYfP!<7oX;x(p>i+48sFcwBa8 z4!_N%`r3cx%hhsG94vNca=-_Gc{%HRVS1%%+n%Klvi|@W z$89cIOjF+hkQ8$MeDj)Xbf5750DKb#$-gY2@Vm?HUxowg6~3;=e}t@crSl&yepv8| z>3rkkiKg+T`9=q71^u&|&e7@ZmHo4X6vV%geh&dEiUBWNfx2@ zPAe$1FZLzyz(h0Y*@S@|q56|^^Ir^0^&D}&#Qnc~;42o?r2AzT!Ls>ffB?B`Q|Iq5 zm9hLwoc6={ zF>%XgzBu}l+8!eWTl^MZa=Ata5%&8&_(7R>p6k7`5*XW=BMF+ z$WLSs^tWNuT5OWqdMpnU z#*fsvinD55IMe|b7maZ350LYD8XA>P}A8z@jL}a zi*g%sR{sF2U&AdMZagu~&ZqJbS=C!at)~53`MkD3_xZ2K0fwypQeL+Er!1-q=RT0< z=COP|7nEaf&H3PT^lP(FxAJp}!LQ2wakNIweb|POq*k`^fSP?8`8>FMh`-(bcuI>; zIrD4c@pUii_BZoO1GIrF@7VtUvdZ*Kmy-N4ktXIw-!E=c?^X9*v+%%Zewj{f8>`)@ zevM-$Pm;@R_Fxav7q$KvO}9qB+SirmD&oJt`C|*!vTxe{7z1QGB5C@GebJn%M=^eH z@0>A8SGR4xI0mpAU-Z1`^!DWnT|PN-*Sgd$R`_64xwpZ8eCg_v{(CXSk!->F!}d;F zj*-}x;ee9I`j=g?nw9x)>ZjR!?+e}gWQ-6cJ@U4@hu)FmHfYsQCnemHb;B^*ER3 zKYDmsoxi`vGLXXeQ(Y7^#erqLkxZ>G_>}$G zR%FM%qVmc&WpKYf21Qgoa&Dwhw`Aw*}yjiNZ(*AjWsEx)u zC;Q-Xm$UdkipJ9YJd;?Cupa3*>i+;7?&@xlFF5#M6G~j4&l^m0e%uN(`(J)OzO8)L z0WlkEm(LN)&h59$0=IiFY-8ykUy91SbzZRk8A2sd$Ao>W@xWWR3hUJ`!wQgy!j;zl z0OgmZjiNHAw*s29>P_HjfwGp`S)*TB^A!F!%UXFfr+<7t4Xg)fMOih~&P9Dzqou&s zIon4rwFP=)mhSs6>Te>xw%7hTy8i&%{{Z`=n#cB^)^h28h@kp!>FA86%BYExU0Ul4 z2l{oie6mmJ{Cy*&m1p77pzmm0k{jmE`{}>>%l`oB)&BtL&tt~%?0BV1-91|N(C17{ z#BGeYbE-zF3x(keH7T~O*F90y`gB^qmWrQx$gVey;lLQu<}%B~xyHnw%MIM6-;Tfe ze1-o2KC1zApQL%?ZOT2r z&(A7!?umAPXO!yc`%krCTy4L%`@CiirlsGp=M8IeeY1>+OOp0r34>hzEAPVqG8;^; z$MKx##HM5f(&yOzEnsm6R9_6OAlKrS7^*e*S+6`!`9FCl+OQKubN>LCzZ|t3+F$MX zwB9t+dqen~%W^ZdMYB(^{If11wI+vgA2o#y?GU{lw$ zC+oN4l^$Mg zw6}TAp$^tvU*WsJtL?T@Q?h5-hzG6xr|V(%elH5=Vs^cT%jJNqOY;kkiTr2q%F-)k z9hf$p_#0kwvbEJ6;IJE0^;v!5TjzeEa~8gNVyT_QhhbX)WLmx5x%%M|meFrZV=-%^ z>~oi;U~+Gx`?8z##Wu%mL2hJ{ev=R2@Ch&cv$VChvR*nW50TS;o+0hpa@zYOCEI80 zoDRxw1>^O~p12pOw~qtaiK%t@XC%veE|?18xD=c0%MC#0BeM-;p%1e=ON-!gJ$<<7 zOO^+_C$o%k3Zt>FKkb3d66C%a$v%(cji}eXe#~uBx?l&vH%YE@Dg2`dhMQg#OZ-y4 zch1yYkIBRPutZU~3+%J_;8_fE_TZs6e17y_z1|P8thfjHlfdNA`uXMR2JX_z?)to< z0RSqFG>Vq@i1ffA6)=X#MK>3A$X~2ir1&bss9Sm=N`LnU*A*9jWOE|lg>p$*LhQkD zR6?aDnqLge&P=EDxbGVB^p#Hzl4)I8zn&jWU$kD=gW>G-yEIC2UrNuQ@S`-=PIW(P z%oVk=a!tc)cb*`^{CJ?F~*UMq6Xn7yQ)EadPlYaA&U1De8vwV8z zS>`wu(G|F_B_B1Hp{AIT40Ljzkxn|EdD%vC0!2E%<@_T%RNKP;0NlRkhj=+2YXKEa zE8Qj7Q}^R{qV7qq)t97SC!M0wn)6?d1&?(4KE~EQieJ^_N&U++l53E!RX=`djZ5B} zE!D6ogl$l~)(DX++2d&}1y1mPHLx`dlH2F_$Hx_q^uJ{M8_VI^d$zrihB0a2WP}swSQN^{8k0)7>SSPvlf4-eoHx*^sn~* z7&!-Km%{+CPm{{UOHJ3u;g1v70U&xT?Ev~N$ED*jw})4?HSEMZjRv`r5d|rw^ zSPQS0#bwV_FZuJty0eM?dF8QxXXk+O50rL)Y`s+Wzng!40qS3u=aw@0Mger8mJxl^ zNvGwM*jY5hp4V)CF^doRYvA(b#5j*%^T0`6K7Wc?(BoqN0G0d;Eu!>ml5YM_Eo&XM zB==G+U?EbDdBaWfHuGXa)$t4l+FW1+R+ss*+Iyt$>5Zh9Iaz(}qVEEdwf=ZAq&};& ze=A|dYTd`j4n5fIAASZAq(bG)eWF*w_!a%`+|{i;{&)kXFJ!#11WXC+o9PjZkn>9^ zQ`3~wz~a-(<&~y-<>{VytXlr80tou2+nm5&;=5J>R0p`R!vqqP+;&C)^f7X=lRn(- zNj$ezE_f8qJ>=JxbO9+%VCu)aIs9!Xx!dOUF? zT~=Z4RLDx~ctom?JzcmPH2UQ?7!j&Dmy~MTMu^k=$}G(;8geIhk`62d)6VoljLDmAI^zd_6WW*U4rM0N@hzStc!K zmpkR^!cSL``iB5QXvI5AaJ_lJ;HmLNOhiZ%gp{$Lzdec516LIMvm4;3xBi&>Gru3m zxXo7#{xrEf>DH-Lcxgwjhxa#x`bX00Q>)(9SJfkD8$I<5^W^i>4-ZIxAp0{&f?k=N z{!Up-I&G{zEXMEYz@aS5#1d2Gn_ZSc@s(k}FHGT&n(v*~EFo9_0Gmm+<5O1ZQY+^2 zbG;8=W_ivWb=6K>r)SgG+mm!E2`vq#K?t|?hvt;$s(n0dXp`o#o(0iyD-j~<^FZv9 z!D?MGpYv=QoCcJXC-TQdZ+KsxE!Ea^k5PTA0gPN-ei+o>hXz8J?$=|^p$)XlPfJb# za;?|ev6w@#3tq?U%6J0O=0%J2P9M(>+l1iL@-LdfPo(E+UtzCQ+Q8x@`TfrsCf0X& zG4C3lu>R@BPeXw{Q|TXjzzYoS?ecHi7;-Y`o+D}Qy_g9GzdK$_?u_RzeJ}a*#({ZI zmo~5}7cq{^v4Q-}`e$|)UHT`0{tlkpNlMG&{{YJb#5xr9Pw;rDFxsQ-clh8E{Tuc_ zzd5izQ107xDOGMJ?s8IKa@wl7=(txng*! zls!@F%eNIzDxU)Qr;n!B<%1h+Zxf9&AypafT{;%v8HmfumWm1;6!!56J3eswWkr#7qY-jDL)GNyzNHa zmUnc;_*+@b53kK&5eXHeIO+Fp@q!w~O4_F!VEjbo;8%#c#t|V)Z*FwXQ)Pin0OmzG z=g$+m0oh>@bx#Qk03y=##&4s-sOgsJdSGTlE^&)UtHxVsSFRGBH2hUOhX4UhKNEzx zC57>Qr{|SW=({OizuOQ}ixl8qzfA2(`e!hlRs^se!&bfsP0?;{->%+UVmBPGmLkpH z0Nv=k^PgQOuZO<<=lmJy5NDS9_ZX?wrNeh(+YMdg6|7{orujFc#XPdlHJq-ZOM7C<-+qJN2?mA)BKt54{+@Cxur){!_PdeXad0@1)jESxAIkEn+_+rSn?d+JzS^PDcY27R{6pCB(X{&cVSB+_jQZdS4MbC|HPC=s`>_~Q zpULH`@0>d^dDeIUmagfSlb?oLF-bCHK8-)}!ZA~d#`2q372XQ`;{7FOEidf*WBXvp z3T2=^(NpcjbV2?`FRE=|AG;*&j+0yb@aqP9;8f#xWYpmLX4-!41||^iY~oO`wRglV zursE|(=-iYc4}4tx5o?vw@x}d9lGGpeB!`)PiLIvV5xKM!#n{pY2m5WV6?&Y1Dk zi6NVztQI`JNmFYzgHJF9Y7Fw%vk}O|e=@-pWz}FR;2uRaQYSeH#A{C`%OZHJ?}N+Ajqf9V ztK*YDM5y)gFaH2*{w#t${PJh$e)kjGPyQ_JuMTso&{LUhTQ&ED_r7PP&kVNAwFu@` zTT_?MFAC16%jUB}reBo3t1NJ#?_dfl^A!6YQ#Q9vX3Wc90e@_{{G&TeY){*P4e-18=eKE)3HB0 v;*)9ec9ciH%3N_J`I4%n}J3(&ICj=5_u1GBR8uq!(|I|KjOu*H)Qo5$o?nsNJ>KLKWV@v z3GfVX+uO-BgpL9q`)B++kE$0;LZpSEz1L^Im1~ek%3ijWnwrje6Cm{L$@4e5RM* z0Jm;jc}4Y_`U)R_>ejy_0hcp?o7Dd{xb=@$1)LFbG$^zq%=>wfeF2h4H+5FFmI1jaM+hX4v)P)(IfgzX-sk@`i9|<+8v&0%+Um zCKE0k%a+NY6gOZKc)`-LRPLfbSSuu6Byab-q7-_M0jF@Lhxri`GUv5@ri(IIuFIrr z()2HutzBo7c+0gA!Qi+2D-!Mh%1#12is|bJbpr5$q`?MNZODL0LyPW%89VS2A3Zru zINZuk}q^XN@wPx41pmWsgmr;7@c8)5}2Z6)AkYrnh_Lp0; z@u_)?ABHD}jno5XDx;NIU=wosU3r7%GOZYiD)z<*1m$x>oX(Q@6Gf})Ei26q@7q}d z2F2O0uyMnhA>%=CAZ_6@DJ+d5>No2LWHuUHoPP42^;x09i zRXEi}#jUjpf~VQP5StOM32k|9JhzNZscwxnd2IL)`c$T4-WLnx`nz)@LzJ_nK3%0_ z6N&wp!{LR?Rz53oSG4M$vME%?^gP^ zTP9T-d;3uXhYH7?273;xn`&whzw5!YJukiw$j8$a>+5hfiPnjo`Gsx=HKu8rU=2}H zr$9$autwK~m9R5MyOM3(1jBvTz2U5djpQ$9<1o~p|2)Wb&KR`L&TR%RR=H2j_)SaP zZ#R{*zk+nNJ*XFQ%|qrV5Laqc6U#P2ZB5uBR=oRcYO7Sx-Op3DuTM9Ly)z!&4A^ZE zHLZ5Ddi>+@CXra}cb)Ay{a=uQP^Zi~DHNGqpvfqdvREF=4(dzZO)=PPx5xtP@!a67 z0$|&)`zq+0}k64Vv7dAZDRlEdvNz_34hY~$G1`NLL%V){lbt(bf z%ahDrGMJr$fIg=lz2<&E6n5tOB)d^6?gd$?O^8f9zqVc?*5^~HMS2YUJsqqh!&%r> zo3X9deel*V_S& zbXt@TzhC%8<;9PlO0l^Y!L_a1e}{uUgE^0KAHc@l9s(=A%rz-{5Z&k9ks=!xPCe&& zF=%7y+06H|im%f8TzF)yJF;p#W#haqG+WQ|C>~rNl+v!fk+4Iv1`Hz< z^Y={qwzaa1!PrH&g1??WhAyKyVSGk6G-f$e5OEg4)XKAT!fQst2ib4jS4zsTDH=OzCB(;pTwN{(U_&2e4x)gR3oH|NTKNo$q; zxA60l4N8jT^odOC%UnD4Arj$FT3+^b1$mFhwhX!`_2yE`2lfLp@7`Ik_oP5Mc&K@6 zd5+*Q9SmOzD#yGB+r%8d?-n<2L`5Lo#*dnwskc6U9?vW2s1NO)wq1Wa-FcmKRuiEx63WnlQSUwwthWUjV~n{O$+jaxuwha?-yEz7r!b>m(|k?!$7^*XNjY?6I<=`EnWrtuvnH8E(#co&t}SBW|Lfs%HG&i zN?&w<8U10BI;v?iC&C!HnpzG9n`}CVY>5~5x!RYYp894hsGiY#^swM08h<#GPvTz8ABeSDXzqc`Y#$@j4pj@4CWAGUhVXOS18@)nvGD((A<%51KTM zj0b~Ho`-S0n#(xoI4_8$D{qy>8b1{Px#yA;?cN1)sL@U=2Dt`5gc2*J=1@{iY?WKP zt$t98fs(bRV7(Y{eR=4eARojAf&ML4hxouTMa$!l#dQ8@gNx1k@}{?r>l>NtQ!VZ! ziSgSFuES}r4w|n5F9EXYe5LoRI%WnIlin9!xdgZ)%{@11*UE|F8`II#kL8=RANtC{QI_5beRa#}4okJvNx{NE~v4iAot;lwmNan_1OVZ}MU!y@U(srR}g z7`UA`f0rq{<&S2iIVelhKw!MbNl~LNTDNz(!clVJ$~DW1%{C{(Nzu+{O@#t|x7is_ zHoL&q?F4f)g*L9O8S%h1RF3JzxDjF>jYck42eyF|T?TX-5;@a9+~vDD=OARTJK#57 z`K#rgwX&B#(3)7oVg#(KNF9g2p@ON(*X0PjvB>^IY^{e=d$nu8OhRgBY_gqV**$?C z*%pxB{=eyX4MJ;6g=WNI@c~1NIcOQq+V=n}^=@B;{qb*ca8btbdd@`b9C>ES_sw`a zX!c6nS7`2?&Z-f50Plwh-m(hjnN7J>;4UrQ{_ zhftL%2hG5aM)__e?~OkcSo&I4;J_{>KESX)dIs`xYa(@Fu{=SqEJw_p52C@A=s+#^J^wArK4GY0fO4o|$b9+ct-fRTyRxY7ml-0*o|f z>Y5iGDk(_TMmu9u+s;iJwkqpKBoSAv@4S4g*K9+nm0!JTsZ-|xks*bKpq@WGw$rE* z&yXyp*)g+WCu^_9$1EQCm_I7G?kP?TL>!Q4a{`E{J3^R=HU;wh%~gilQRj)^x4lu1 zDEzLgeY_-6g&c{TGovNvUh&=A$m+Ol6R!0$^uE#QYG$}~??qKEmcBFREwMj-R*B7$ zfEf&E%keTDU%)>OHdb(_vz6lTEQCoPXqVk{VYY`P$-IpZBGnsDb~-XKX0ePbKPHS; z516^>IFz^R*C)y!TFXEp)BVy?@X~sQjS(r{;^EfxqPu?~t;I%ip*;650W;>}$~^lb zBgb|TQ$>50eTqRX&iJ~#z>0fN{Q;4n-%o<#x;6Af9``Am2Kv&|Im&_utKKYbs_oT% z(qXf8EqUhF8wi)(`^&}xW81ItB~e%IREe+(BzVM@!*Irr6;WFN@i!@9lKTu9@XT}wpEFfA_mz9`NcQVkPE$d--Y%9bF_^mbT!7Xe)`c}cp+sSF`I?6(+@9y&sRy_3T8WD$IJ_jv65t&E zBW}MH_C8Ez4K0JF-9wf{KV0iivk%mr)>+FMa!uv0PII6xds2aWP3Rcli!3#p&T`!F z_YZV`f_l=`kOnHZep0VN&-+T}LES{+8MfN|>Ehk2**~L|`hElDtzWn6wy0*+pPx0! zHnxU-O?8!x2c7jfczF0uaF(Tq?s-F?6*$5MNj&|(NQLHcw1Lnp!D$y=mlS$U$SOjCh)vVkpY~xXLC;`0cw{uceN%#JA5Aitnxhmqv>z$;VaGkF;Ir zyFc;YTYm@Kp>{jesh@4L)3IEK7}`0;O;?r^(@QJc_WhqcjJcsf-Q;;%_bAb|@x^~7 z|8&1BCWQr;DR9u)#uLTyr+`3Yl`9K6o7#O1@plF6&_AldSlC|t*A%d|6OFPdXm%v@ zVO&szy8V^<>0X-{CapKjKlDGDD)N^+!GK=1Og)~%H5uJmk>LJn=A;qB2b)@z$p+RO ze|3M1b$<}R#4^v0nB6Q{28ZSZqN=)Wc7gW|_VsYiY3sAKEtDv)k@xmTe%~r`gtSwW z9c!G20#ba78fGhwevnifwE{a^rC&SFsp|71Z{=s|(xb zwCLk#vSssPU5}bqi(t0j`gcxG89fY~K%zcI;Y;<)!S3&s0zC6>Axxzi|o3_XokA#YK zVg2W^%t;?zty0Xsd2Aq}2WM*=qj5`DI*g5C%B|u#k5|R^&80O6ekM)!B0wQY#Q?XQ z!rzwwIserke)6u?{0oFxy5bZ|P}bZ=!e}ri8&ZrGes|u+lkAj6SyTSqSZP#9_tME}Z9- z0gP(~sJ)&O;bEicAo~DO5fb$IR`k!aU!r+T z+HtDuw`@MX^tbB&9*d>0Cr*<5l{gV>YpOLFDZ=M*S=9#R!pna>0{)6Mq+mNUT}D1X zEx<>M<3fhHC1U6ef=ia`6m5g8oYwg4>x0I;{Ec zh^E@#qSlHFqiI~R*40gEEbVcF6rUPWh+#OhJmdpq`?xaG$)mt8NlFPXZ7@5#xQCa! zu|900hohw11U4gV=c&r7<`P5Z-wTrvezn%emOa+0H$<36UAvwB%3_Q{n(eNNXz^Vko^zv;IjTvAl!^!-7T zX=n!JS%kmnCSz73t@ZFBVm3HeUQO!Ptez-1q~9_|8UCLmyFD1f;^wv;IL+5|RH!)a z*)LuxvGIQ6v)|22K;7~mTC1$544{~sHE9YOvM^EzWQL_@Y5d9DiffKyM(b?v;D%$o z4CcVc_r?9|*75%>Mir;mTiBtk*?n|;cC#)4%clEJy%vmyn#;|_CJ}FJZ;eaA9`7Ss zg%obquqlkQrR?Usw`vj&M@%26yN?@|a*chh-XLeCnK1J<;|^^SXPSC`!Q%VQ%Q{Ha zmeKvEp5;3o23IPctgnpMyG^+H`*YLtA?EdjqwCYO=%Qx^jTkD|-=d*qe!;<-RyC6! zz6YhA)s{X^w^Ar{adWf4l!(UF3F-wKEQ=NXd9@XXPd~3_?DtPtwv0{cm?d5UZfIIW z=EsT4`St(AeohH9jel|tPAropG6r_g?!?gSnAnn(guUupXiLy~;4ZWo-Y6 z99?;~xK-VD^R)cRed+S9D#CEa{G;H!JaFIw9n(-;YT`_&0cib04mQ3M!>p6G-#)Ah zHo%S7*cf-bvXt9hosWF(ce?SBY4_ysfxz@jrwJ|6R?DjBVCiqat@;@wx!uF;=3Dwtfr!~ai-Ya#TC_2Uk%+{JBcTZl4Otd8h5n;2p1mwcxdMuRLy6*_u z^9DfX9X~GWI6G40qTRvul#isQtfPZ{wN*yxCb+oO6k820x^L7dvBHjWJuBFm=sJ+P z`DH@?fhNoCt6RfY*N7v3|GRY|2)qOsp?C5uE;8P}UI_eBCTwOC@`-B?tgWac1~*h4 zZ1`6COi*q}Xo9Jl{>@?>k`bg+bYVTQGl4)GMHVLsGwI39JxQD)5gmaH$j_J~tXchBlAUZu(y^3vLoF^J&o2?B61RGT^cX}*OkOZBdF z-CUGg8C1bKsP(E%11rsdE4=JzxC>Dz2^6Rg_{~l+p)~*g;O0X(!+j{VhNnRu=+0@u zjxFjuMOBQzanm$Cri*WC`!gAI-#@p@V%+$WlDE3kE9X(vdgc-S<#4&pCgW?_DRf*OMg^P8+^FQ)Kay=J-U#NN}t z472nJ>edjQ&gQE4{w`(IS)&^r-@bA%!!wjyNQCBu>H3=ILn_Nyimp~VW#X3LwxV?{ zl8k2GBTLKvw9Yr) zj7nFKZ3G^bn6j1|-t4laz${T|q1jPNLxSrpmxNpa47Xe_~NEYmG_ z3Gns~$)9A~k+XwzfNtg+G;GxbaydhxNTT^`((RZgb|9SWh=n7KrzyxRQC*#v6)xNm<&{q}v9?_6$){ zZ6>3yERGC+pe=utc*fP3O+WS{PfN-I6*`FHelSIrA`ViDtp`gEC*&u!l{k&!1|F!k z^D5Si4|RZ*K2BfGNwJ@u%$$XXY(y={KSOI}h+YE3`EdvSdb0E*l>@gw?}O<(V=`c- znsDduSoh@Uk3LtncTr9Y*YfW9kt0@>t|+`ejL{j_xNEF*b`1GBmmo58q;2(<%apa! zG@kC6PD?H%o2VuRfz?NMZC6_ug~a`m4;P9rHiNmw1O3LkD%N$%pZxVQyU+F<%-%h! zgvkk;d-*l%{B&Y_YJ2HvPKKR{BX%T80&ZDjH`qL|}4q2!W zc~7ZTNq5<-j737sXI|7!!!}!R7rzNB=pNv{llDvso3)pA38=43^MT8>K(}D4=jm_c zvJzzXCPf45@2sbn5J9_QVzu|#hi>memBJqy=0wP`JlN0^g3zKfr*_I3L_=EbYAqA47 z#9gid>b0fa^N8ZBo=xap=1BDm=hU^9p;?|a9XgXeT%+zO*G2tzt)VqAS_NU&$k$pC zI#Fr`zUFYyFMJ8`_0~4xgvoB^w82R>?Ix=s$vkKIR9OtF;JzDtn={D2Gz)E<6e|~KNG8xY&vf%65{F!*Vu54k zRAsZ$TtebYECC3`fVnWO)f6s|j-M0em1wDO+s-johPjP9OBW;uctT5~jX4@#?ok-loB0+7qpM6k z4h(ELJf_Ecehla*Rsv-}Y17pDE&XJcQQYuQV_)M!S%$@PwkY;ziYCB6SXISFBx z8!G;5;ZiVXM6syKuTG$%@~T~Wax{8X3iSFnM}dlvv-(n*?84^*yziG0Cr`4f!8@N@ zG_}g8)yZaD@zh+90SN1N&NZ zw`WTfr^?;;<5|s(n?W{giO_FA^fF6Ho&#mIm6GzVuyf0mztU8(#32pleOqCER`Ke3 z@4Qm6K-suoHopdhl+KAkG9&zsbYT%5tx(p15G(HCM*7{wvX(RG(`JODqx>b{;_W)} z+o$_1&YKgN^{6N1DXydrnv5(Rn?GMQ(yF%KJU-$tt|lzP<`rH0>!bt3TB6Ej>xVxw z4Ze+WaW&_3v)~x6PPZzqxao=7pYXlT%*O0G>T7h1g7f4~s40&w{TGIwZA>MeH}_UH zYO3A!9-DDd^Qo}N@|fd}@Jk7`lC^3o$PV^2(hs+uQi+{fla|d44jmq;Q}^_cZDV5W z6NJjId0QJ31de>^3oNsMDnkknj_CSxIG&Hv4{Ls|HPS5bTrf^cXwzKP}tR zhR!$WKLa2%YUKQ@g7i|6YHv8X*LaRH(0_JTcF{NEtoXYBN$9}&qXEoJsnhPMBUstQ zETU2OFIZ*(kL5A2-Bz%j%pyE6S>qxLaqh$>q{MLUyE)jc12stRL_zHYYXZBUF zR(Hz~3|@{JKI*nu1AM2~T-MVhO<&`61VffjOL~yYy3=GDnKJDRx4cE6liN!_cQiq$ zM~g}&ru^!$1$;0;XC`PhBf+)Lir#m}G=`$ko+`q};_+o(L^#bNy*_YsIC4SWym<)J zs>i97DIky%Zv7m&IN5QJyd8_iLM79>lt7Wktwjec&y0dZ)hsDEic=@?G`%3kXx@iQLiu;1_(C<2pd*HVlo8_)Ozoi)OQfh(60-f4!mGC z)6-_78(LF`0os{Z;CJ)q?`iHTM()PLr1X~blDBnVY=>$hoRU+VKR^b%IIGavgA_}_ zIK1PD!pqxT=Ks+-c&38PNlPF@p^xa0J>Y3konw$$1aBF(1w)@@XY>fsycj)89!@&v zkEWDm#!3yOmPuI?kHyAMT`EI-*0R~^4RXP3Ds&0L2tPF_#koIrR(8TctZSxi%C8TG zo-g=ud_O&}C9g_EfsuRSj||no@3~@FxkbesNxfW^F@q2k!n*%0-uJIyq2U2&!zgPh z$S`JpHA#GX3lZ4<1)~^TS3ops`yO`)3Bl64oZ&&k;&V8dD!130FCmcC?X2%MmHwUi z^`9&P9zw5bWVzSQyPy6kpQoSW?x=ziEW}I>pT2-=5aD{HVlL5XS+E<6mE=_#AUQ;< z9YqnIT}SK<=!mc=-T)psL!O%IHcJ@;KaX!^Hji zD;g)P*2iF<6`(=fz*ZpEP1d|}u9+V+)ppKYQ)D@7D?RO4T+;?b-*v>U?0UGR2aa~| zV!6?*My_F^>;sNoRMBU7p^y@5+@I#UY0gb4;HM(MGzTRBm}SDmoLh!X#H-D+XceSi*rjA_3RGho7@LcII|tF@R(X=Yi@$ z>wmQ!+in;g|H9@$1F6271Yjhu<{Z@9s;@r8L7DRNhlKlTy)!i5kM4qXHlN`_k&FYPZHV7Vg_H#+BUkCA7i6 zNY`9V*{iWE1|1p|sxU>DP}l%X%e9@=#6P; zb+RKt0$MXp?JvUiuk`gB{fI)O)eRXY?axl6$3$m+$x%LX~QUtD&tyetQ z$VFeX8s=QfTXzX3X5InU>6U86B z*?gLbw)=V*d82 z@@)`KRSNjcq|91*idRw-9YP*@Jc;nR1pK#1g>rSxsLIc1dJFq?-Cu(2x7iG zq(F!B`jqmJTKa)0)0D_x@SUGdcFufJEr0VJ`(wqq_0S%^Go`-87=!$fP~jNeQo`a9 zD*J&#l61W|zpmVmRn&1HKqkSg)TFZZIrFvh>`BWwB(-5-pPEP8Bfb!s``4R(+@`D3 zokM76Y^5)fgsTKFqzV7)OL-aIY)O(g?cZIs5^N3(O_sh537VNmHfJm=&V5N|=i&}K z#`aG@_C>~9`4$L#%}W70IvX1~sd(PSgt`3p7<83yZPHR2u; zt~IOiAy3X^`99E@tq#8nt{)nxwx^IUY@Y@{*Pobx8@}P)GX456L0L_D7M{wEZw-|l zu2m4KL7feP&395hl`m^Y-{rw&2$ zozF+q>$O08wbkqHC7L?E9Q&1&tpUQxgdXPLljri5 z%~t3Qrz(y`+7yXbC%uz5+Cz1MoZR;Z)YIq6)p>!Hk2p*ZSlXU)4{TEj&@V&)s-=lD4_NSif>2;t0Pxrhdbr1iprpYn|zkNXm!73*_H2=z9IP0q3 zti&ZCjF(MvNs#2}zvy=fNWjyFY+QTWx$@Qx`$lEJ5ZT${2H!3bdg9%5GB8U`-_%+0 zcHf|oFuV5iXCHJ$_vd));o&peWd2o~lE;5t3-|6h)FPjXhsE6?+Op#1oIFFZHS?pe z2&v-J;HO`t0zq4c%^DirXHh@KksfVR?n8|-hwGUp*2IV3<^+%G0A#iqmml5d_?3+=0}MTU_mgyW5^nNNR5DF&+655c(04F>eoOV{ z?A#$;4_an|o}7}NSF{1ppBY`o?N)8ooi6w9=(G_zPXGH3Q^sOvISaSZgvnO5zs+rh zV`CMoGf{@`RCc2|Il6JbVr|Q{9~F9PAdMcBRX|Ci7>efp zfi2rlH2swrMSjX#brL=}uz*}yk6l3_L#=?X>`TO(b zA$zW1MFPs4{boBm9c4ysFNZdz{39xs-w^~oErEHcr2ZQAp0VO-xHG{9eQ|j$!!cr_L z;cgy#4mBIgDNpt!fF^T2)BSS>M4XRM;4syGkf#h73BB@QVB;NAUy!KQ;KQF=pcTQANOamTmSWJ_M+6m}?t zfv6mu7MgoD&HMIXwB|7Cm>#&@Ga$xqPmd$TKh8r3CyFkE9vW zn4K11a&9a_yWoAsv|lo#RN^6NRUbK0V`{e6+2@pG9z2-L!l-AjONw507hfnz{?vNq50v&I(`w*X@`ihe33Qvab&vl{y>)bXTyoPd9HdTe>;pEYTg)DX}IHCVWE08=XZovzRqypu?hUuYLmloz1)oC zI8{VLF7O|+pij3dt1A@35wSlbwpcfC+BVX(@j;&{r!8OA@7m*ySbu?5U#0n57m@{X zAH5`h3U1tGthR|i6Fhkd@@xACe7+OX-q)-ezs7tT_2d)1m11}*s{Cbt3VE{W8r}1M zM)-mYedKY9@<>3494MY*|BG)+55H-(T4QL14!ujgZ4hDzxMAe7b&~q5<-+DDY}_az z%psh7S0|L`LztJ)2og6bJaX15f5?*7Vm&2Up)!4=Mh-<-7U2xbO}&@Hbl_Zh&j~Sc`Uqnnm)+hgvgB zKDHh3g9p#*Ra^)(r_wQB+c4mtyO<+c*opEhyndDJY-3QI-qy?)R{lNREtJg9o~NVI zT+GXOK17JP=s!Hj1Lti?$L4|~?JnPo25#>Qs8-=1*>t*)*L99MHKZtl464^HRy3~x z9sz-YBL=2|Wu>O5>4>tOCdY{flSY*tudY1}DP-GMiOn75z$z&>^|QIS)tS^MjQu_O z-G6{|4m{bQdzP1lhi5;+kFJ_3b-qs_djpkTVCi|PX=Z(Sp^dE_FRa5A!d$j!SU`J5 zhvz38{0^Dx0@oq{duvfW(Wz`K7KlGtc~Ue z@TXKf8%7Ih7xKk~*MfA2Hz?Nqq@;cc7|Q-BVJE))TJrE5iL)l*h60^&&zfHUhzIM7 z|4N(<7JtyvmWQ8)Ip!&y3)rf^{jPc!UGwDRm(a5v47OOnNV=}ix9+?WqLj2)a?yCQ zB+jv)=cK*vaZS_IWq!M76L__lEiX-T26SF3kg@BzCV$u6F=i(us=r~kgy~<#{1U*( zP~XZi5x&-Y31~ds<_c`z7iP;|ZeDxO6L0Hyb_OYq1gVgKvE7J1|Lh86l(%pDFi`@L$Opq2hbUGC5-t9aD+%maTTR zViANI_6&`UeqH1g;!vDqjdNc#usf4$DJYshL+CSk%!F2jybrBjoR5Oyp*BlEV9!7s?;w8Z67Vy8poHJ`CtFQbW2!8UfsjnuWN+; zC&iCG?w>OY@ZBc4RtwiptPTFOCl_Hv@W1v^fOz4Kyaep21UPv{@JMg3y@=i{DgYQF zD5R+PlddnD$T6L_Cvx{cSa}fNVee-Z?jl3AF&V=z!Q*|&ZZ@Sy-^zKr@=S4U!X}f= zMotxYrw-AZrRcwBMzhzOc!RRYrE72ARNIO-+m7BJK-P;TsD8c#+&x)>oo%i0tiFz| zf;8_BfMCJx;i_KG5eT6R+-Yj3^TuCj5#bVGb`=)-@)E##5SOVmdH;U1!^_^QwjWx3 z+$`%@>Z&w;(%+jHSgM~3@5`X;<=^ler}-#z33!A-*N7F-!d!=s?3|81(>tr)*aP-v zTDfSKKD`8ZQqx2*L-rq1t_ymWO_tG!{5qn^+YbFS=o#5)9EfO669^B!Uhv&>pM%c=VrAQ$#{Z!O?uvPOR!mmOkBXV-^-A^e z;RiKImH%u+h$oe9nRUtqe17(!p7f{+j=5?Y{H09N0X_{35G0$CT{Z^4sLH*~>Yw>Y zzcUgs%U*2e%7ijMU^DVI_|#_3Ga-1zNzZ!m{|Yo-j>2TMuRl>=bs;ALe3@|WdH9e; zR`SjHDKA^2D7t0@x>i*yb%pL<+ubSg=XVcu$?-mU+}olh@gBQ-lI#vEyNccv8mX$cdehMAVdY6 zz|4W-po7l;L~yE9MPCHDF%~9%I77_|_w)@XBoOti{z<`_6xqGZR`c&kxdNG_BxWO= zz7-ufkwIHJ4K*B!Dg&s~{!O>e-$#k=zbZ5;_kYs)>x;e98p*0Y&^4!BgmSyAo5)2P zY1qCyH?b6aBQu>D^6S=egf5N;zZ!%MF6!yTW;QlKLw{@}Ac#dDueExtd9|J0Qq zy}9_1%o+7YZGphOciDf^Kjf|Cn_1(Y>n=)GuF5~#?_Wd-h2K+D0XvncIC-08H*?T7_}73JV>ydWl$e@%KxWonOo?ryJVhda z%(i7nAL4Z3#Dkwe8kQT#fMbaNysAB88Ru*^Z^`ykepx+=;IH>#K606>tU~vW%O!e) zGfiwhMr`7XX}+*s0)%k~C^0;T7C8Lx)ek)XupW1<&ZUzJcO)Xg2Wm1g_a-;4wUUR* zAq&6qh`b5B-5^`bi_Ggy^+L@$ct2{>vR6I@^lqo|nPaMqTVY><9EEH^2JVCBfAXLE zw3@C(lu?=As3N&}ZpOBHL${`iTkHDg7Z0^t^+s`y*YZ5o_v>Us5W+{-NxTy~suvCi z1{ryBgzIasHAX|BdXud97`?;in(n^W7X;Tnl$+X);iBPzP&Lae*8|(ZANXmXn#9pF zQQ9#MvfvD9jUw`7vRmgIeDdpVGQv8?&eoxcdF+=Kq^t7`JLtr_+$LB3RFbU|O=& z>WpqVwl#-;zv^Uazjnfp@7kE;1#uK2HiGf6jwXYI-D(@AF)tyqyw;Zq|)QX|2%p-Vs}dhFcizHxXmv{V(Q z*=ik^y%K?;?q^ub4Dv5`$(9(9#PE~>WkDL??eJohfF6rv*SVMQgSo9o5#0t76~oL1sC>uGOMq_N z*3A&>bfoOXIkV*VedjzqoOwYw<0T-xk6wb^r|E$h?h-&FYs4Y5>Z6Ad&d$MeoW4}F zqH=Yvk<8f_-b_hbKi@j&iJrdzpJ&iUE_-gH=F^SehnhO91nTzJTNPaM>pix0{I20; z`l_+dUp%sf&i!ep)y===Mqp&^I(8%7D&o+j>%*>UH{m}NJZLKJ$NQ@ zr{aB?>Baq}jZ+Mu>v^sUG=Yg66`(t9-lBpOobq#j?hj%XnVFL8;wjTquon!-drCZP zgt0#yC{R_{8`}Sdw#2zd+zH7}dRS=*Y{XcTJkgVa)8B@ex+H{`x||GY5|_?^WZW=BaX4W^(f*|5{@<_jzxU+ z`?fR1>KANBS?5T>pTYh-6)&N0j{K~qQ29*nZKIt}zYs2LK2M%!f~+1Wiioi;UjkTD zEBiRW#QQBT>H5U2G98y9@EZKk|1^4&CDzmi-6-^d7k0Qt8{>~DV;Z(jtsh}>F(+B) zv7n|&p5OR&wQ4|UQ|*W6rDv6>s*k5XnWdYk?CWnSD+B~AC^kuf<&gLXsK>!Q27sC-=B_u&X>)=fZqu`sO^Cf5fBiN z5Gm47M=w6ss_Oyimm@9e*sVhh5SB>}=6)Y-R+eSWyu%PMB*(af$ZAQdZ}WShUe|3ZT_?^)%B+pI$q$x zwLOnFX%QoZz&D-^)`Q2c`=cUOV4_9VYFCxG7hVawmK`afky+QKMePx8^R-d$DodBX z`@c)6_OlyEwntvTQ{!-tvr3p7p$J=p8;Jxwmf z?|5^@!8{kjLq4(__BoP9Y0|An*Zf92F#$4NHOl9;z6l28-Rhoq<~YUQ1j@?Ex-1LY zK9G`-cCuftw|`fIyWHx!&3P}FmQP_B#pAdR4n@WdX~fNrMilMRRoYrm^;h5+zFpKG zzywPGLX(1}jO3GMN@6hLTl1(s1!rG^H+fN?+F02``v_KDQ9_eK_!ME8Dn%3QW>*E| zHd=Oghn~P1Ep1~w^;6d1XZvCCEJkC_(ydB9f1}q;U z``DM!!=qJNS{^Y@bWnoI2KNd5-=urBc=Iofd`d3Wz0ZWvNxsA{lfa|GnyH43=Rwaz z4DXg&$O__SGSB#v7P zX(6~{*hHzcIZkiw*~|2dK=~67JCZl$)O!B7_kSrG#*}#{sO$QDuY+_XejTw*N1v+; ze}TQ^*P_VA!pN6nOgm4Fu6_~@jvd^va=yp~w_Kn}^VDu2AumIx=QeyY>A5bInnP%{ zdeBKecM~C`p{b`oSk*-P_Tx8mrQMTTUj#5*8v6X!)iBJl9?hFiJ=^vCHUcbRH@B2L zs{WFir}7VwF?xe4e$@EkD&Y)+lyMP2&7O{MbBg}uyRdROkvub9czorqCC@VEsQQ&} zmwqiQ*t0GZ_^BW`7It&6aD2GS^F+CR_Vzu`jO8~Ulu3cv>BD-??&dee0=eVn{jeOV zfkiu1!3T8(<+g_LbEW%ov}=hv{K#r@Fz6X6J{3kq+fE^~9hb!iQF~bGqSbNi;|QiN z0$jeO-Z<2@T@?wcqC0$$kbZU6InK198Ley46)L3E->^ijPmmc*a#=E=+WPtPo?^Qb zTQ^mE%Hhu^&y)|6AKK>^8w2@O<{uuCfP&QE7Y2agtu;KyVgn zsX2^OnCx#>m^Dn6@6_kr!uZv0u(%c5vEJQBo~46v$DMGrc0fmn4^^@yUBY{z^Kr=y zX0EiEUc&LfNw1%4Q%}%rV;z!bnYwyE1>j9M_SUkgUfE7!eW>=|D4?f5NPAgAPa0;- z!HvDZf==nCSZoj;J4;iiTP%4|7uup^>k@!q33Jz!in#V_Bm>`7HjdUyH2xi@;h_U@ zzX$It93~j$ifo4uMV$ZjOKnTp);zgPYa+jMUKWsa;F;@jy0bI>CbQiTcBwT%2VRED zicE;QWUGJU5S`L##Vzh!q6V)tXsp{kKWb1*{~{&1tT*Zt&Riy=`dG4w^OVPLC}GQ8gC z2e*@K*Rsn5PUt1HN`X2+a=r!k*DYd#A-O7ens@*@hGTPd>ic#YQwIoYd=__Uj;mRy~SD}4P2(7jZBv@OOA?=yHk zq_EaL;ciq25JBPdt4Nv6M7y+dZw^#z_;eW0jShgg{6&sI+=j={`^Cg)>?%|JRzoB! zBFxh^o=3T+l6{nAZVt=5%4ZwG=BsiYRz1sO?&*Qnfv5(@ntM~00g*HN@x;m~%`=hP zAf2+$*Bw3~_~=Bmb2QX;b!_=CRL$be7?-(B%w;ag;p0d5<*^&uN-ho_UM#jDw6;OW zhP$}izy?g}t?E1?@?4yv_z&lHsImp(SMZ?9Z=IoX5G^U)q`^hsV#2XGcnT0NE6ICx zlC|0FdHF^n{S0by{wupZqlIrbG{eA>1Lw}`J~Q)^8h&jv^ns?(7_kiv*-6@VboPNMR}P>xSfRAK%e}e4N_1Low_IvSVJvMHl0$3?C){aORnu zkqG?ALaVp;$D(`1r&YIZ*J9cXcft>-A?-Ssxb6~aUANrw-mUhN3S2oOs}mslkZB8Q z7pU2S-$sf4B*d{b-(uNhtLe0SM?qk46=5!NUqbHD1NDNR(sMY3Sy@l!DJ$LR-HhW6 zef3o}MEeTqRIS#4~C@8JIzjeXU_2l?ocf*5~&-k-SG{++Y zKLdNhvi^+$?r*OSRuDKXyDpNMm2=MU)(mQ-I?3E@wHP8BNMPGLr}}+dHjfa63A#+6 zEA%5g`rb&I1zE*84DyTTa|0)u_qqFTgw0j17tR^+c5IGvXv+4-v{Q${cf_(JbzXd9 z+~DI^0i=dU^kW>IJgk+6krj4bD^<2AzCSNITGZ{H`2eTHyahb0wTK^;W)%9Mg}iS8nid~so7We!&Y~;j++tTJvwE(8 zXFV6NKV*gvSXwi*JNt(F@wZx|RAea8fRy7ArB>fO${8L);hiL(Dm$V2XrC%<<5-Jh z+ZGg|l6ta8Rb~B2Y~sWeR!G1Izd6C#&^HXW#{NKZFW{3O4z5&cwWLOCvs4L(n0&)L zZp!<;a$`>Ey49qc#|Z%R2D9&8sS^f#q2` zlD@@HcIlYno~PT7%OITp#I81tqo18C#ug5Pi@$IjUPa&Qgy@B+4;Sj6+Ophuv~7ZT2aC>`B7ZY&1srhKI?l9cQ;j;Q+Gru7TI~< zyi$wqFE6QFtz3d>ef8H{QVknc2=coyx!1u$gEhaLU5wpNBNXWOdCNPT=9p?)XSV+V z`lOxs*zQ(QKW>&qM326v!{37@cm4rxA}8DRD!%^%9AKR#?S|4+-ta0E{TQg6A!78Y zTe#3nnBY|KhqXtGwDBH^=`-IGWvcmT%z!X5KT)sBsV8sIf${7CMRH7QYuKX3Fh&v! zJeDa3b&B*vrKD7C4pGkeWkG7+MG`$QZNPH^WMEnWu=w81aO|R4z$Qs0e{J5XZ@l}gxPRl^j3IB;obm3teCc4y}rOyq(0|rVV9_ zkCh`U_%U)OPxxAG!yJU1GZJ9bdPkWogSDH*5J*t^#-8Vg7w5QUUrQQMw7Mm8%kyZ( z0GqL;He=o))YV%P-wv|E<(s?(zVOQrFS2V9@!Eh=zA!I&11pufcltk05~dD@7wM9} zf+LIh$Zb7E#_NmsLM8AAZa*NHaoG!Lp5`{bRTHR+4F zsjuo1U;9-@+k|LBTEGvr0qbHOhLOB9pBv(wF~X8^$uV_v!E!W;w^jp z^l{pBYYA0*?)i5?amj(e@dBO2SO;*?Vln;)hXnRBe#QA;9?hOkjt0YIY0$HO~Zn6EybE>O3Z-Kb0`TN1iRi_|35e(3cQ`JL8 z>jB`$Xp8XV<8Dg-KGmqBm7Y zeSN#|*w)y5Qngj*e@~jfKedpYnGz3Eu$CMFTJnRunvA+(``S~}N=`4#Ynr`ndRUiLhL{0Ro z`>j`h6}BJFo@ltF816$`j*_?m#wgH)@b50$IP8VTqX*|I5{zd3o(Me_1O?5q2Na)c zEl8mBW7ptNA82-}jLzY?u~%(88*N?~Z$_KzGEr`Wp?40yt+senq3yQMbLX;#K*Ukn zj)vlpUkMgznF;?`b@xugvBKt)Np;h(!HW;7`)OIm*Sg*tl21$d$1EE0RX3HMCTCju1iA=XUts3j!pGniZ$N-zxDqCG?(;>HS)uLw^c?RHuZMLZ)(WmNVrD7$YiJa zX4~Ggu$#8?dcwM%-uI;~o-X%$cZqZI6!HlXYyS=KvBCttZ+uNM?snaipL#!>M0NJk zc2FPzvPhfR3FTJ4Cuihku^4$a!SA?uX^fl6VYjglv7T81kdx##3D1gS#A=n@COulB z32{dcE~)m~*ep7-$xyRx>QEE1noNFP_jixrB!z=UOx_TUGS}bcnmH5H-(+e-{rZ@V>Q}7`s0PMu6YKJq_ zz$@rdo^GLz^poIw5}VQ?Ob>@=w%c`sJ3H#Hg9xgm5BUK*HqMk`)XT)^>DNL~leL4( zSRTrA35oU7N10&Bty#Aud(}(est2_JgqQW}(@oQ^&rJRTRP?VNvYxWB353qrLNYr@ zUAp4DwKP5+gYz^Mk(abYAP2ZFBGA*d9nRKkHE_ zM6p^sYuBs?Rl=fqW5F}R{&;D`p8@<`wT;HzhvQERhl7UIlr5Z0nmj>xys(NXipEj4 zOXd>Dt}leLXaufuf`r#;s^)n1La~^|W>3&UFcs^kj1_Q?geg(hqVu~#SEvNC6j_-p z=Ho@jvba(jHQ_e`an|7v%zof6(K^3vo~ z+y-C7{^dVw&N`Pvz6QTh)ii#BC*?NAXM`oo!6vyqVcN7cNtusHH-7B>ToqTB|F)Dx zhnseJAQK1{-hZTAkL8mDj`zLjIHdd%&{N$}KNqLz#rgQp_Al&n9wGOz)|V||?hz^L z*%|x3&v$iAn!(@s0-@R}*ShBQm+DRIH~ybWL)d3qCPEHLfC$$u5{Z0R_TVCo-`;ExZYeqNXn<;)vjl0 zSAI;q%BT1EZ!5FW&y7z9oJknF%;In{KjI+uAgaQZoMC^O3*(JOhfT&Psd(`LVUkyJ zc^rbGkG(wWs+OIKTTGlx8Ft22our1`mTPNSgtCTA8r`e9UGnyfE;^p%_#2WJ%}b+7 zaY4lehryY~nIe18F9XSod(8}lc%*J+m^fje(r;QGIU~lLd@_lH!$q9A(8X!u_x6BU zV_>N{F!BP8#ficD25qQ;Luc|WwkU?dlq^ZUC)8b3Ed%--`wa>yofN!JQj8dDiE{So z+?sBR7PU9_JW|lsFmx}j(Rh)G`ST4JJ;t)5taAR>6%!Z4deqZM-A+6n1SJV4W&rR78M&sAPtVx)YQzT%G#2I`w zP`81jipa4Tpr|gFV)!)fQ(DZfBaLMLT%x$J((tOT!?ZK)i%VSW2!YjY!ND)^L73l9 zHxCTX5bn7@tk4|xH&%_Tb_@HOa-sKPEKlWvBe2G|fMn5qY}F^0|GoTmeDn6l zKDbEUB-byD%6{xak>JPM>#D~IVJT&aqc00{IM%huM4_zFO=Un#L={`#5OCO@e*vl6L^n$^DT7xI%272ib`mJOEo1Hkp~bcGCATp{>qAqkWF zWYf{$IP>HuCnXS7Fy;L62y&$4(s~VWJ4^iS(>td82F=XjH;^4RBU-CKaToIj(7w|0 zeNEybk&(GIAi_$*vZ=3$g8G6({ZTdOa|ZWI3(8Sk8S9M#<1#+ACIvm5fW@cq*PKZ` z9$ha*%+0nGf9dn+H#;Jp;57Revocp!t}@mVm{*VIx4x`2jAj=yG*Kjvp|6vB$yEGZ zWQuE0pDeoc9cMfX^3x2KahphofG`i$ek|xQ0%0~+xQ2Cz3r^4@M9mPTm|87VX<|W? z`nx#i?CC&HMpAll!OWAd{pAD$Ve$R}44NSu>5P+A9+g{3p!2uGWtcJ}w4!QM6aL`T z+q2=f`aDUV)Ly1Yt@;7n4-@wAwHZWRk5(kN8?HK-phc^Ef*`kfV*&fpk3uwMV)@X} zI%UBv@?4IIbT1am1$J&S;iIZr+_fwuZXl>SU^cc3`-}UvAu+iu=SbTHn*YUgcJSae zaU$(tolj28R+;*z)w>z#Igb4pD7W~>Dcd74VGpkXGLTp)Qu&zdOwmfO&a@6yoXK63)-=EEj$?6dT0NL6u0nena$7Wt@XikieK0%#Zx@@mJ= zL#y{b;px#xCrg52(la#8&)w(>MQ-vjgdu*HLss2H{oLfT*xrUbocJ6NT`Byh*KP)) zh-FcvzZobky;EfP_wA0rnBGw?l>FKcMGDc_r#2{HE}0aek#A$(^K`!^oYT=LSQK1} zj$dpQ_mi!c@i=)9D8??;u1MdoH(umu7w$)U1;nZFWpxGgZLuE(9FSM9vX7NmvZrbm zMOKDuWM`jT@eP?v|0cDYP$LxNFCr!+sW_NvE#V&ImsA`4CCeeL$*Xn@ojH>##Dh-^ z>*)vD{RYjJ4qtlaXiM&h160I$vfq`nMGzh3DT*Zp8oM`Adx0y-r>qj1lU^=jQ8$MV z1n3!-R-Y^Rm+pR34YQ&XSZ+n9-U)3Ab_Oox+$uIYmfboEOaGsLTB%l zv2D2J8Qe5Pt7;QAf~{j)-nf5ef|O~q1x~DYoMidJMs!k$={Gwz zB>iqhfT*wA1zTT~EI;tW_tIo;CAk&#k``&@G}bG7eG~i=ByEOAQ0i}%uiMM?{<$;a za08$I#=Q`N-LP?7m<~RR7$#fkNAtE$enqQ4!}FaD4fM!k^5`D#Qr=2ex2AX2S%x9) zHcX9SpL;zTe28qZ84e@3Y}+|T)D@M7Qn%BDAczPpyJsBZsPwNe?ixA2?2k5%-u+(L zKzVypHnFU<0W4z+!hR*+Lm&_H3rsdbmzn8T|3$!pVu1{6#tqAen;C}=7N1n2R+lL9-E1wT!CiVGMa|7j)~13obvVeH>{qiTMK}H0N27F&Cn{h2GBG{ zeIfoZEJ<$`5EdHpfBFJzrOv#CXpErE9`rMkw5M(cm>;_q)fhUanBwDuL=C;%$v*4d zL{xl!{;1o{=6bNuv+6tTmx4|=0-L#C$mhl^>hQ}$o62Kfxw+yxjb;xwiyroc&wI&y zvCDaZfAktaH}p?YzZHFJ=xYGFUYyMPsIs4yPywshL`4Jtrsd!d*APvngi_N$XH#Qk z!MYR$u%(sR^3{QqtjH>wyI*_7aiSo-d_$kQK0FZUd~NdGc%nSz+5^(OZqy0q;5Jc5XC< zT7(3Pijii<*9y~#&E9=UYrZ4b^E;Wtll{XNT_D7|qSb*Ich@wej?KA$_TVB%`Dbml zR$k!JJxjMy&6%9_ zdyHr6liw@Cn}fn|8w7`+`MTZbC5oQQ?Xc;xVurfxxz*{Gm8imT9LVnozr2=@&%f-N znZkV+A4zmceN&9=0rot*K`?Yhy*lzdJe3)G&(fnCLJ2w_Bx1~wm#Z;fdfw#O zd7p-xb@16Ug=;?5NS2CrGLAG?_t7CjTD;%sk+|d^JnxiFF(7A`Sn&wc{+j?Jj5&~u zlJ+XkVBIJ!U{$fpknU#fdrmR#S~L+yk@%)T72i|tr(IyK^7~>I{MC!k?D2A--8}IZ zrmAn7ax_WEL83Zf_c4vMea55iTrT#Rt8EkC9o)tj5S&1>$K(RT@RBc{VVow8kFs!q zNbnoZ#4$G-&K%a&`F6a9=z#?R(Q^@AL=pCQlh{X_`r+C4&kgLMqR!!^K{$`5jP zs*et$VM*+ziFGvXo46Q|b$FMUiSis;h5OdF-tQkE*N~GQ`iUHS%CsG~Qrl>4*mFTk z@Btq34`A4B`44~rGEW~hJ{aB9d|M0RyAtCF23?t8Yw4~&aw1bVX=0v5CAmtzM-3pH ztNfQB5B5JNR=TOT0QY+iQc>&zCTjRj#J)1?zgRHnhIz3E|J2Zi^FL z)J)DveS};%LCUDhaBXIbGbM!%a!o5JQg@y7?}|wfRrDK%_hf^2KTnCMa~P>9*&x@s zZoI8k9B6yi+)Ueq#iADUT{kAFkSyQul-j?5=(FYB%$afBHzrK2txrAgt_HH&HqXOb zSW(J4AMERc&^PxLhBuqjEfV zDfM<1Gl>wIebi!qR1-4fTTg76_=k5tzb2G>lGT>oMnkSalR^r{4t36FZY^7T%&7K! ztrj;spE^-7+%RlB?)Z;XbSJ8>LHUS&<>Q~s9s4YyQ>6Zt)=}DPrYo(I@5TCSeIi{R z)2&uuXQJ;TW_C8S*;4_wvHRe{SuEbkXNtCg)||Eb5ATCn7@Wr1hl$bb#QblQoC5|h z)>b0&1`}^+6MC*whY!Sxub@O1y{~0lWW?QVAHQ2b1&O=i?#7Fr*x1?yp7ceuP99kRA2Pb` z6`X~{evDbYGkjd<=v`ZQSQ==3l?Hr4($XkmY-R5t>=FE|sDC_|bX3KJIVDRVEH_y_ z>&sVBiaogcXoD@%ITH9}GoqQ0(ZEPCaeD^o59=j&FjfHH}njAsa8g@BR@?g5-^WM^L!i#Mll|@Hi zlMxYI*PZc^RuR?t79po9QXfTo4(PgQu9pWhX=+9dR zI{O)}#cN8Vn~i%|d*`A|)3(a6Gdd_QXCG$qX%09LNb9wInN+3y@!XL2`23GtzZ8QyoV*_qYycK3wpd<9Qy5 zaq3rHe{dR*_B87Q{+S)ZTGfg|H$3tx! zql>kZ(<)cM<2VplaqY6vu7N6|pL9~gK}J&4PW zzxHAm9!`Z5Uh`n(0n60Sdc9`H|3=Y!wfLmL)YFv$uodw zOyWcgR!d|)p^>v|uvs>_fSw{|cXIy-=zvYUIY`hP%A(1RT)``5%Ck+XSDpbAG$uSC zTMzq8S!-xNn6=xL9n;%a$s)&R_RfcUK*Bn5rw@Z>7GW4GO1tvUud*o0KKlg2&AD(| zB0eOpa;q`Gm}HR0-*^G-7?rH4CrU*mD|vC`INxl#lFOSus=v!^lw-A7OWmMKK&j$D zbUrh-W+SY#xTcqlsv+6lowflqGZw$FV;A|Y3O0C>%A66_yP;--&NF$ph6eRbw7u_5&5CCBlTp4gDGQ-Moh_wsT-wHlRV78Ih za`YQQf+LSr?!p`L{G&U=aGO6pn1aSfA9QNZI>vh4?cHEE&5i4gTFQOFF4yL7O3JYP z*#fvun>Cf_gmwU7x@`~yq+xMuY9fzZSA#8x4KLXEXTyUDhKaOSrjH+_vAK~Fm!cE5 zZM?pLa86M~#al=sFQ2omTPBYe8z0lCI*iPd4nbgCORp0bYIefI3+O^0$aiz(X(>{n zy<6!rmgQ!_#82I!E!H3cVw=r1VxIFEN!pZLwoUT$z@t%*6;X4VOutWl17*At@jk(J zoO7D1Ewj0Gs<=qsY!>Ma@ZubOJgSXsG2h53QEh%_hPvmpCZq>kI+Zlib2`hx%K1mx zZ%Q+-c?xgXYOR3xB=JN@jf(P%12LzcsmX3DV@)Xh2fL9(-0%?|Lu8ZHDdA=qlh+r& zbdtVTt=LZ$RbRza0yM5`Bi>fxLVD8bq=WQ~#=)DR{ z@kbNoj8hmiy%n8aG0gOU=^vc zx<=PTG6SWF2c2d!yji3_iJvT`kF%(UL|RKXdlk#_(oVG#1$m!AG&3EdK8=+IvdR18 zngpm-uNcI7%i)>7T}T_QA8&T5~vli*kh1eX4HEafv&BcjdpX>-} zF}hyl`~!S7T+&~&OrZbphauKv&b+wpnqOz$^(XHcAL_?+U5THdB)EvWMaxGDp?#E? zMk2=5WD*c4*v4Ehy5G80R^jRgZJqikP4LD*<;z-HOs#g7=ZG(Y3M~E{=I2(hhm3Te z>9I$88EZh~)@#ExJ;!OP-X>#Hmujbj7#xdWk717R|J^u@g%&}*gvX!2%*UrD9~u9$ zmryFouO(8`ka)^BfIskPz8=+>ANSuh6$t#aad)*qj@`mcA>yOZm>%XdDNBO0?{1YUiyO$UtwG?){Z!3%jGH6I+Q|%mE@9fixIs$hF6JJ&^{+MG zR!Xe9m|XVyW*Zz!N%$>U=+2c&B?{6(q=QKEDD3i)~>uSv&J54PL3_MXMde+DFkGXvlR^G{sTs53C^zn0-En9 jkZo0D!aea$H9;k&X3swGV!e2|xqPrf@$94QznT95$Wqdd literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000002.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000002.jpg new file mode 100644 index 0000000000000000000000000000000000000000..11a2b45dcbe0454bfd1cdb0a9074831c6e380365 GIT binary patch literal 28997 zcmV)|KzzUd*#G_j5KLiqV>U1@GchhPFfsuD+W6ciX37Z)5H9U>wr|HJ^F00094 z0RaI4000000000000RUC0s#g92LS;A0RaI40000000000000310s{mE5C8!L00aUD z1q28L1O)^E1pxp90ud4+F$7UTBC!*31z~}q!I9B26eChX;gW)KGw@=vq7}l@@k4V4 z;v|z(V>F}F!-L~Q7PB=G0RaI30s{a70s{d700000009vKAu$p`Q3GK@A`>#9f&ah& z5eNZj0)PVnA^-vr00R*K|Jncy0|5X65d#qbKLGsE>B|SI`MkteUaUT=jgdilE7oje znv9ylKNqT27-h{EHY&+0DXdGRgH_1}W|FOncpJ*eQa+KN)KOe;;>e}sQxT?|h>jj; zdx@A;8oOF)xR?oZ`4lbCygt6<>Ur6jm!)}hiT0;<*~I?$xpQ#Qz)kR6+CI8`mV+fdbZH zhnj4b%xc0pcmDvyn<$&KIZL8@k5b$7J*kXn47m;nGUAk421Mp6yp;!=>}Xxx)>WcV zIJ+s|6(MX5MWMGjH6m{aIS#eP$R?(JR-9@x6i$UmNpyvu&6@efjsZn4H*O*nC}r?X z3zP+L3Pc|INAG)-hvEmV+OYd>5siWs3sC7ygh-GXF>`R#XQ6T+_ZiV% z&QfWHBNm9r)o0GnJ14zXywU8dVLYxzS}xRS?IHkY*70Lg}EM#FcNi%mmWZqA7V|<4r&rT zn})eq&*NjM5W9r;qBiDP(!4E8b86&ehY&Ui3vFz-F%|%o@)Ey|)=Bd0dq50t&w^=* zigVJw3DEM}&939L#U|VWCe1{c{;(K*sT$7y3*5!(QvszrDYr7(j zmDT5+ZLtw408De&)hb+xIS^EMPEwana+iFG>nl!?Kt{(Ebh~*aRGxOY&B)3(-PExRTbo5*8X4B0)8YX$d(BCM!qUe&SpF3 z0Vg2mr%Wgp+`T5@m22)OP44DE$Wk8ZqZYxx&F(7=M9KoST`w5Ss;zD>SgHeZ-)2U7 z9R5D4EH?Y76Zcc}7>Bafd=$>Al+X-nt^uDQIp!q66 zq3#?#KKl3+ZJdE@`A^&uP7FZ#uI=9JL5*Uwc^@Uy`my!s7?vyYR2`5zdh&{j%=Pq6 zxwvn#JvyK+^5(gA=_|1p6_5y6XLQ7%I$$3>y3sMhc}7t_K!en^gjh~T%@`X%XFFSu z)U#p=f5kqXNKWhMh~`coPNj*M;E_LssTPS$PANvG0)CqVTbEW(cDaLD(gw=9Qn>RQ}KE8+NeQvT0z%(8!e`8+O4bkz0mN8F5PGrxa+6Ay7Q)QW%wH=a3+%EMI96 z$dIhS2!l&j2xZs|sOk#)Y}qqC$x|s~j|9_3K$0H#)KV)B)TPoO6ghzy+QdjJSd2;foN)1@erWDT6m{mO7|2LSayRhJH7R3i zGO>Ke1RW+PmUeSomg5GR6{Ddvr*5)=p`M{xHJWe2cU_zNpiF-Dan#l4FPiiac*;kW zFDKzKiQPv$?n1cq3iQSXB-b@^otaED@W(4Rns7Z9?ojDi<6M+DVg`KkMrv}GXb&%u zS;oAa2W;$&-eI{D`K1WR%gV}K-H(RL6r~Q#O3lR2qE85dVmqq5mSlPe5+v1v3^PhU;jRc*Ylv`=uB%i7h8iu0X5b=0J&ZwX!}k%CWdRvy%2n4{ zEj_i~&UlX9>Hy{;BOCqTD65Rf3DFuS{?{l%LL?FSM2E_)Ufqm%c{uDrh$I4Txp`!d z0@;}+Re#A;yh>f6j7&LX8&YcZPDrbmx4IOJm~P$Ys6 z1jBwLL~EocrH}|bweDVl-wyEiK#UC9(!eK={{SW-@F5C*0@0kXzn}&&m=c$%6_=P) zvaBk@l5wlN#z@Unj$|MWqwQmiaB-ob;xx3pS=4P@IU(U04~ofp5^fT@$ZM{v=BF7t0FX8hKO_+&SP0Z0 zLN;YyYjbi9L;`wQ_g8ugPqrNThf+qEoFv<4r9_P5!orcWjHA3Vk-mh9{a~%XA=%zW zpih-3+i4kYUFT3g-V+t=&HR4oe)#Gu8#agCob~skZ~p*>wwGGLRR}j=u!kegqN(9~MWE{Kb zubh@C6Qb)|nX_OyDEAVr<@s3=p74#sa+gV>0V63(k*;wXth3>}nZS_U(W}J|_a|N6 zNaN{=8$2&xG<~l1(B2w}=^a77UTXH_vyP;Oce{L~+{P@%SiPdcE3G}*+@UcO&jf~l z1!2dfDki-X^WE0mTzN!lo6EwSW5)J*YEb0b_iS!kEJT=`^%{lF&tot7O#(sx0G2P~ zS$C`6pIGq#OOYwPuHtc-26EGmOVRDNdl{E8!XrX872Sq{)7}xWp(e3F<_;N?i@y<; zoT`!j&)|Ar$|r4#dfyez)WCy)5PA|d=u?vL=HogNb5Zz8iQ-#w)3tYS<-n4)$GXMn zqcg`FDX5xDZJL%79se@EIdx*fGZj zCb%GjIN0Z%CBMuXKbXhJ#NV>QBYUP?1mhVVd3z41&W^z_%$p+0UHQ~Nu75P^ZUY=i z4aC8{KBPo5S_Ir2ffZTtNjESn%{U0`%&VG%l5h{G*Lu*M468Jogh_Tsp%KAFkt&xJ zx8_4Y`L040FuS>|KC4OQni6hf7vdG@RmP);TAc4^DYvK8t8K^t1y*EzF<3zwqCgmw zedKJkf1XKTaLYB8kFedJCG+sYP36I64^Zfy^^L^kC;)#D>Q*AoT%u#gn%hDx!eHlP zqH@Ur$>mECxm|Kf7!xdqs`;@~F*!$(;!h>>ZMZW7RrBziMq?x)B1E7^nwPmb*9{Zw zOD|N!*b|OoC&+}r^WGmRH@g!8z%$oNFs14w?2B0Py0r^s277(TnEwEoP}EmSLz1%R z@avUQxhPUDOv*;}r2Ah-ey9urBQH9%$Q2cqnKPh(Woxt1DCwxH>JZPQ2emptjSp0V z5jCHpec22*>Fh{zOv1@M6P95NI7Bkd6BZ`?$oL}?CyNR8jRBd#175OLhUQKP8m^GK zQn&M`p?T9$&-SF5W?<3eR=rS#;vUy4P8YMpb2%yV0~SR%<6Sx&sbs->K~{BJ+e{Mz zG6xm=QmzV+9!n6Bfr#KwwJW7*yV4?o8fU>}$2Ri*D==`@vJob@3m)}4Z(@`{uxxd% zZcdfzj=)dtc23T8(Ser>0Cxc@_9=TW4|8?9n`iFh93pd}EH@cSHZezBJar*BCMcw( zsYo$k^GiKYs^@D#zIL$W;~K@z`;!t|u)-xIO|IqvZ-8$Ybon32OugO@J;!ZR9a{%S>;<0$v=OJ2~f&c^2^x~an zB5)+}_XRzf+e6o_9gWXt5-{F++R7cD&*+(H(q&u%U6}~o^QGARcBC852)!>(prv?^ z-nnCx0QQwd=FHtP?&?#w*Bw6x%`)Js9tX49?lzJxz(|K=0kSf|4g_5tEqXVdlXcnF zcWcG?R`yIXU;rS@%g;!TrRW@g4yxr0>JPVgLlqbtRnJ6MVk?S+33h&JoSZmTEIQ<0 zi&L|HlzP`8J&c{jHd~`);%X;8IRd9loJRPSu{c+vacJAPutl>7$EEp2RA}2DmAOekSJ_ozHi1YT77$CqnykcJ*S{{TWqbm=kXI$t&Gqx?Z4yzh$u zOppHnDgOXc-v%iboRfUcYLD9F8ZpIoyR_RkF)q>qRfeyYT$La4{jB44DECN@nz>%Y ziEw)qb4}s~F(`w>85RVbuJQiVIe2l7QJ<*@*;>a=>Hyqhh7>t{iWKI;m@F2s;QvE-1J!*gR3dg)S$zUm~c zh;BCI0@Ibm#$5dqY^=#*>!%-OTo)xCDiiUP~xgAB~hoMZ|T1^^S?xU z{%Qm_8;ApjX3nI6F0g4l-sVm<5hG+1nk+-VSaDwk#*aq$r1!3fH6SDj=$ieYl7Tr( znF*Xmy-Nq8Kb~vtz{l)S8r;dbiuPu z4;+$A-2~%~%C17GYr+e&F^_fHY2mZ{D`g(4r4MDdwy@M>yr56s=*Df#o7SspfDR#A#)x9nM$Z z)|VGIOh+%0V|5@TFz9o?RHR(PEVa)*^e0`0XF4&l>Jdrp1bl!8zK|mqL6c$4 zMW$+_UFbNlr@|`!+7M`wit^{?r`IJ)d#K9eg!>U&lxEbj%!vhiVnB8{Yt3&C7~hah zySI=75FT0D0b$Y&M=tU6T_-76%saQA+{f3gI@HD-)Tjqw$moy-@-?q=BTmnqEZ)wq zG542TfJJZW>134KxLL(R>aSwdoifOZSgjtELCh$gC8_@ar}I_Xoj?ju{8CiqfARa| zt~0SpY=yRed)&Pi^0Z=nW(-I}YjooTkt(rVP&-gHSazizj}+nv?__5TG6y`eR3*8T-~p@W zOU^lMP7;XE75o(~qk+Ohl8B&Apl52la7w)H?1{6{8u2Kzp4wy?vk96Rm_|m2D=JRw z>SzL(+YX$fbEx~m&NZX6w%Iz~2N)HK$&Wb?+-su-PN= z6~~-ClsMw~saKrwUA$oE5YqrY(w*?{3_G}+l$sznhU3v9LtZ}(R(Jg_l(_&sFZ8cZ zv)u!+W;P8o0j`}9?OV3`vnObWz}+&KgRmw%+D;9an2{{9I;&9JJrj&ePCZCZ zIVsFeM$IkDS+V9B#?;JklSE!xe3NCsY*uQH(|agAJAl-Ud&0dv%MM`1(@tbf8a~1b zuPUT?c+JQ-B@hoJfhg72HQnaqMi;{>?(H)sX6S27=aNz_J|CGJdkAREs^I+PbmwLRkrjSjrsoo zaYzPx4woF0tc-96zu0MXVXUa6v`#=voL3_bhjX}dJ_`7v_=z)nE^-OrEMf- zi!+sb)j^T5lFk^|axuuL{w8B50S4*=tg5qs!enmgL*9#*?0d?1&ea2LcqQNN3(5wX2B(3#K$^H z4eU5TBbr+bPb7Kf7GVP|1*r!bewl777IDjD$?u=u6 z!W1-Gk;ShiAR~rGsbSGN=nVZm=)xlvjTgp*l#$+T$jD+8mn61v1!3;Z*+Mw^)<-@#C*+jziFE7wBmKp1V&M%D z#xWkl^IQI`{XG&)qI|{tQJ*#vkaq|P@I(7TG~DU>Cy;7p zn#9}Irb$)5HxDST=M&m5TR0uhFf}rmr zL9xf9#0=9rEyQsMyDB29^rzLGU2E);6U4(}r$VD0ei;NN_C6mch?8@}7)&X5hX|hxwX!;vcB>L&;cso>ig6fuNJtTptu;ut!_anPvXri> zfjvO+3hbOX3{0*$WqD>&jK4OwGwF0_WkB-fYf_9)+&1hWOxkh+>g5?kXZye`Lvy%{ zBM8V);ssZ#^%L<4N;9fD9K{iz5VY3hIni?o#07()6{0n%5O({yVgbVg-I5;Tbx#5Y zzQyQYYi#HMGSqy;N6AFru;1HZ5Gpb}vevCUXm9P82uJa2@LP%J%JV40VYiS3VvTYK zQ(EQH2vjdiXnLCmh+N|w&O~!dmyCQ|kBG1JcjcC${jE{L;1ka-_7hKexEZSAa`yq9U zI@Xp;uD}7WHSiO*>YqUft~8JE!<<46YN*0*rMCmCQV)6wQ6$3nwmFo;8iK_ zHr4 zG9~v$By-gy(4ceVoQ7=^{a)a!6X+bNZm!m2qAVp5cCuIl2urz|YI(%c1VFkgNUR$I zVEUs^?=nRY%>~ohXXi+$2u`FD`>6=X&+!mFl({>;2y*ER^IDKq*v=?}~fUQ$5!Z5=_ zRhHW^oJSCPmY%af^2tcB=@ipIjZ*Nak~xAAgq#vK55Xwgmw;BJ8axzF6Om>y#BTHN z%qudP!4q7RzY{QYHF2sE^~@~s{{W#)ZFd5Dcwr*Ybtk3_ zW-o$yc`Z-)Vzs>ckRmcDrC@zFv#C3B$-|ECq%!YQ!IN%WOk5nOq!Zkfj6 zkRv`-r3JY~XKL(>O`GW|%p7zEwQ3+{O}8ltXnsnT(sS(bUStrCN-E@FJhk}bpllER za#`|^_LX6>XoKWb4Vag2D$ZF@^eW)NJ^y*C>+5f%|ma(-}_7)Ee_ALi6f!(;ubL28NOyCY?R$lC9r`C$enI^ABIpjs9NMeNKF1{?v0SDU4zQ z%5Jfc0pW~FKU0GE5to*6S&xCnMAOtb7y^DGL#Y^U;(9n_2{q;MU6ZpKN<+AiuGU5+ z{{U}TzqD)r0LA|R^()wcF9gAtZFyht>-}n(e+yYkLny65(I#gjD>W?6xdoYdeXU&h zgSYTK!==nFG(*^zcPcyLHw>(9@RrN#NC3nJh@Y(j*47yl8 zqB&yV2NlOR2eg<35oKeQwztQO#NNfpbTATl0ewn#^I-Uyrd_HULPUH(UX`AUfz5oA zu5P*#=<+9DC2fNq6}UXZXA@3D8EYs=3#}}Q(FR8KUdwO11-Wz&)O0_gZN2ct{5gKN z=uf?BahUMrA@&DQZ_$2|cTLP`G2cj28+R)lAv)6(L9XisUgOeNfo5Ht(@#7l5#Fpj zwrEafEkFYzMMMm*$!8XcuSMcpeUnT^aBeYh&LU)A z;5h-H(}IaP!cY;3(C}L0%VPDyVn8NkIL;6N-7NMk>PNJqjj+hnfaIHC#{^(TX$NPD zw8>C;sEj)1kbRiL4HKQ~{{RH>7C8xKt42Vwd(^$VJ+qC%>m1`7tE^H2>caIcf7^sY zU-^;8_(z97yjMJnjI^8{?Do;;{{RxFZy)U&s-uV-9;)y86sri_jz2-_#HQ}N7k0x~ za)ihx=JtO9L7TUCmG#ArlbxnZ{mlr@W*#J@JYIc8@yC($Yn#7 zc_&YVZenZh=linP^@LS#5hW})?GQ4M_!Qh7g;_d_85E=b8M=TUb6v(iG$}W@h=?#b zW&F}V#`rzYD!r>^ypDW1rC{uPPMm-l=1mW32xoH)yEC5(C==T5BuFf}d76ZeP*lXD z4a;&p8Ae^k(wn#ONaeu$64BW1Se8sZ#M^Qn>R56PhYmPSKfDM}N3-v?B1f1>x-iDj zq-c!NuGw}+yQnQl$1PRbJ{#Ll5IKV?v7cLM>G{sZQuGbn}(4cV3Xa``G7c!J2w zk6Onrsh$$I4oYyDk+}i~rd&N!`zKqAWXRP*C*TdFw{4iqO0bO)5ZD7*Mx?j0!8cT+ zyW6>x90dA|_+|i3)v*K55x;8k_;4GJL}ovdn}>+LGL&FPkRbJxWNBbfm?VU=gc*%B zC@(pV1mve;s~?FHEmp*DT|CosYEg`3cVgF>%o?(ua&BR3r!0kPc>(B~7MR3D$@A4X z+nQO`LUM9|ClSv~)>_>V?;>qUZ75f|`7oU#eBbTD5k*l#IyYA_t9PQv9(#W8$?e=?)pg)f{Mz0J#$^ zf*x_mtr?UuaU9dFXz-Yg-K|KOXB`j`1B6DEt4C$X#_=8;iQ~a=;W8(Q>q{+S%96B6 z7;zsJ@sXCPNot#DcE-M(uaC)QfUulmfvi!_l5-8MVC=DCrooiO08 zQtC%?VoYJq$ckDmOS6_^LWr5#WsB3^n4VE32XFbkJ?QUD2G6HjF#{=m%ZK`9)V%Am z#z9_ykam+l(Le28UQwL>lS^DauEAFuX(=#q}6()(Efi_X34qEa)UPIj?;wedr4 z7nRlk#EM zyRdRigwdDbSncmpzK)1JoJl#-Fl+A6I!GC&fQ%v!I#Wm8PDUB(IP>yO-YA3Qnl&SO z>3B*8sxu^dSofuNdssZ_jEMk5oQNF95LG93ah;JO2_)t$XYfF6j>>1Hc~+mLThjhS zb7a@$oQD89Ueu$Tw*c&_H5|F#S{+qb4#?iNkM^(W_Wp&I`mZ+ZvElMdJse^_R;_Qk zct300rx;%a*Ntq*=`VU*w|}n|pBVpfN}s_z!YhDN$MzgqnyarE3Jlb4-feIT=DU zNdWTAZ|yQ{<%FrTCUNuRipU}Y#(DLl61=xEI03_+T2dQNmbVh2eG1mN9!hhErq%v0 zcwCIHb3X6(t1=xt%fs`$)mrc9(saun2&wm#W!|G8G6$C&&TRl-tObD*4}1D>5;jd=X5&C~ii` zwn^K-)ql%xne$40o5KhncZU>jBQfv-WP1W8?1IGQ5Nj{)91^g47-9@ffmf0~6MyC2 z86YXUKfdtmnpb(@yFScza50tVQ;{|MM5E#WTuKz*205KgE49*md_9@QiF448k_PK( zx7oViXE{p~SdJ+JnIv+E;I1|>k2U9a{{VXKe1VQxShBU=f|71CB%fj=cGP2d=+PI< zc^Xt(*|8?0(v~SkCsC2C=u#VpbCf$7VTVLZ1hWGmZ$qNJm#Rz6UNe79pfJpyoM9Td zN+4a(vsut4y%UZD(vFvxa%;j}`#X-qTz#0ks`|6`uA9@Ic6@jq_83Wdo{0Q&C(_$M z-1sK!Hz!QlF&gJP&QvnNT==Rk%0mGXIdiD5F*Cr0x<0Pn#^2Rf6Mm*ehd&Z}W zuzx&{-Ojb*1rWMe+_>Sw)25?&LVqRde%Rz;oPo}VM?qXd)Hfx!+`?lcg1FX&Gm^)R zaJdM%Sga5ZM#mjiHj=4K&dHP{Jntpz9(bLz@s?QJ#qlHLf#B`1!ExBi5usfWBYtL) z>^)v>`QmVpP{WgqH=y%LcJdBTl!vNy-M?iSURrx;PlFSCa3I*kNYd)+Hbwdst~Mp+cT5jP_HBsu1Zj4$bps_dqpU$fh)4FxIgml{^Yusn1@Al<2qxN?du}T zL+K>?p)@}9`t`4wm;~AbPh)mMtLo?Gxf4bCMr34Yu1j|5=+tQCq4}W*67P9VNQU5H z<*LzlZh9u`UveXEUFGddHZkZ^O))(49^aBIEMvelv>8hhVaBE8vk?)fOhj99rgh5t z)1k~+XqRojy_v{@W!9MNr4E&aP*K?$)asUlmQRvqM&x_SLQT0DlgN@(kH%P#m03gv{Ny&4r^_r!bDW}125voGETSQF@o*y(LE#VYzF;t9qG8*Bo%zVt7a{lfw%xO!Nl}>qdCSaE4>) z%y2XiCO(peN*Zbs!7en783;;T>{rDqHAESjQf^bK0d_Gnm^bw6=7w9(JduRVVk1{q zpEP+wIs7a`K_P1`X8ey3I)cn)ksL}qGjc|&MR-MbgvFN+?*9OA2p%0UTJpkg;#YUX z#fL%8=M&x13+<~|7{mz9Cr>or1L}_5>=U^2_X4{&L?3eoVBy!f0xrd@7aQo0R7g1& z3zNfNO3pQ-iAqCCI-rRj&ni8&oTHCWq5 z+kliCuN~8*ovp`6X`5eOJy*$xRv#Vj|BIIiW)NgmW1Yq4+GK z@;`ZgVKUz0Ch|VdnhA#xRt6aG3PbjhJc=}5ZZI*a1D;~NIi*u=Hr%0d8RyLW)rrPj zIoyfw%`Y?3k>XL~5Eo2au2 zqKHYrn0lvKaEr21u17II`KBrn(EC+PF)3OJF*<6s0s`me zPIAoTe9u~YQz z3ge8}BI^0CJG+Q&=|=AjXQrSPt_;=rLlZLe>kH;#_dp*$va<Wow1~nxk6O)W zyQdaTU`*^*>VYc)jxt4(yJkQ7BGWCjz*iqXR!TI#1gDV%j8FBN8_o?$Fb zNd9ROHRex}*4{la-i0YlM_a4SeYP?t??M+wFO}Iufg9(dUPB_bWKCJR(R6}hNYk1d z2ZGc|yIPbo3pAuQAv4k|pvK6&kJ5lXx?OM{SyB+#OnD|r0~P&va~vrkB_mmqn8Q3q z&5|=kME9}>Ztb0L{{V1V*-VY8p?w*3m5uZaT0F*BzgRA8C-)H^Q@@E@_|Mbrj8bk# z@FRj8oUbiE;*QHuEQt~1J_$@G5anwP9+OdwW6du@PYyUM($V2$Rp^r>{h$H$PzSou zntCDzAbF*B?H1)L&-AC~8`o4bQ3Q=riSCbN%&1wB9$d%GN{l}8KOc&3a^sRCFa~xh zB?-vCSKPH3Tfaxo;xs4LH9ce_RIv@qB00<0eoAYunaV;E zIg?qDm85E!m77%Xd9NiYyAwuI{bTc1{@D-x!~QRlEX63V%B3j(06HWm8s@s|(V3AF zzlu{7j--?eRHAUwb(sSaeAk_E*@5*)dXyNwxRAKGlrTpgwOqKV zJ83%RB4s00%=5HP)?m>jjux|HT4D2zxt=jA&rTWAe{mx#BEZ9+wIDTJIQ1Chx3qgH z=UJ`K;J=?Bw+g7DKlQ8qhB|%a2B2&U2J` zgCcowIVVoy##c)otr_k7y}zLq$Y8AXQsnLEvpvz~W_^E30?ZupS$aK)jP=BH(z&S3 zZv<}^GsB-*N;vzjYUIsnawn{&MW$KZf9721>Sxc-xmBB=bs1KmbuTxLCm;xyGnRaL zs%}B~GYXw$9Em&@xnxsDZ*$U@MErk3`Tgz~j*mVlO*JUybFuARp3-y90T|10ll_w_ZL>@$sf;0$@qg31(Ms1vV-6eNm3_huiz{7`?-?=Nfke(Vk zU$tCSodnGAXRk}#sWrK{5@XMg=u=uEJT*;Cz+gZXeAR3bMja>#arveR#Sl0zJRG4( z=x(q?bZ;Ov^6pbgQ*mP&KyELWxAZRkroS(@^eCP=3s}SoN&VI~#BkxGcX!D7B3UZ2_%la?r$e>$no3Y}WGbUAjfwse+={;zOWcCLWLa{OIoc~jCF($j6( z)mB3*)=A&HCX(@&Y$Qfleg6RPNXhFa4ae8$+ zgpMIuURwDjW@gt&#EZ)@TBnGsvE0T?843dzAJBr{=7hRAUmwtd#=+^aAX{{FujrST zl4ct@;V}`PAZ%x+a=JN69r?4+y#D|f{uRTx?Lio7{Lx?V*P(>Pm!)@g0!OeRM{fH> zTTVvn>V$__Rj_4daKgzp4aA)^Q`vC9ldfYWMq7Q!4hXXBqa%?Jnajw5Np`k-sQ&={ zyWjeA`7KK^n+`?b=Vu!Fx!>qcMucS^nP zauXCR0k4@(ySCm;;gdiea+Gub01;IiT!WF0vL2)+yJqG9i;y~9NE-I3((vu$k?6Pa zFbCTusCZW8lOZO^xaHDr?Dt6nYwh%hXDC?IXU8<2$=J6HCWl5@lU#WBgxvf$dAO)W zt1-}HfcBXHvkwN}$-LOe{{UNG?^@WqT5ojUWf+M-oH58qYgv`ayAtV-L=Hys(M@Z@ zx2$x#8mzvy&b~ruvfA#407@#uIHGwko110W#Niix`7C{pbOrW}kLUMN?CjrnaB#`IV9Lq7IfHsf z$e^}17yke={6D2^cxC?p%>Mun{v^vX>1V+?5-XAxB}c=o*J=I_{7IdRVL1+`Q}G#Gmg=t#6NT zTvLwH4)lzr!y-OKSUw511M7R^ANDMG2?J|C`ltJ1tv|VlfE0D$K5HDU+-gUimxnPI zFk*7M9eVEeo`^n#Y4+l;V#W%FA5R#k)B z`Vk{MV49_#+Aeksa)JG0^Fuxy#b)35rSnX7(!NNA*&L`Qj7pZpf%L=LerpohHvw{Q zT(v(nTMeXqN!d2cYw>~p5}B&vSB7m{e|+>M3{Z?_K)DSk8q5%RuU{>O+p>U*dF7TV z6Q`W5d4C7oNv+e43^rSkAm2H~dbm7e)8<0C%d3s(soAj{uvKNOzG;?V$T`Lueh^aA ze9CguQn}ht)|5@jjiioDwf_L84E>2s!#hbrqSIKflq2$5ZT!w_GXSF(kQiy0VlToH z(+((!ak|L*ubp$l`ak(-f9W)Ud*Zr}c!YoqK^2ks&N0H)fS(KvO< zF(0BuRteB=!)u)~*!_rnklTYYHQB@UWAa0CcX8&m2$<30weV9tgCbEKn|aseA0%?a z0DlR;)Q`ys%7|JsxS=O*X3JMl5zev!Z^NPXqTp(ygu^qH-T58~E$}ms?fZ&vz@>xj zAO8R!`qsaw{?9|a0k|Bw3iT;SPT&b#+{#4Noe=JuwV#ccJFd!&U#%gzx$Wg-Rr!*V zpWbL{kfU)rtKd?dh=Z0(qkc>y92JD$)%p^SIba~m=><|H&$afM)n&&0!EPIhyoQ>n z2EfH=(1tjPTC6&uj2?L|R3-U`#RDM%Ijlrl%4%@3q4=&J;e({2b+h#Tm3(2!%`Y$F zbT>{{^nYkt;LWdl3?E34F*%d2X?ac}L<4kq+DmHWj1E9iToUOX^k!D|Ol}Sl9D+-6 zEMy62IiXyPvRjqNW~^sJmilc&1Vjh`4D6h{h*^gEa*6sutcbsPDtp;1H6RwTD$j8v z0cOo9@OP)5n$+ivHY2VbP=LcA2(v5A5o&-vD4!%f)X0uri@D^1xIGIzYuKPy!JIDc zq#fx213fD-DDdxA_bv%L964rv%g^{s{{ZuI<@Zwl=&J9|oW4IK@{5R$y(Xh^HJ=OX zSdQvW%wm=NJ-{Vo?dr2)BumW_BThX&(BP4X8rEI;>a^DCLNsMFXNjPd6X~oBqDLD} zM5s4*Fq_YwtM1T5(H50t95Lfag5@mr=CMeG2M(Gk+YQt|%-sRPFg?#EC+VAD3_A9M zihmTMMGj*7xu|VY1(f|*gk0yJ`fNwQ@r8$6TI${}2f z0FqI08l)(>8A^rZyNFt6gk9W-_4g=mL}Ho#qt>l&4Pf>6FFgK}K6mX};>~(L29Xqt44PWTtF*4r)_b)7bFRbG;AU}Jt(B-EKXmT zT9mM>%5M;sQX}HIC_|T7`K8L}m2yIE#5KzFOc~;ZUm%m5vDcE&odqVrB4fR2?Bpwg z5GF)g)U{F;HCfX6iGAU`#%N|pjbR5>BY7{oQ2~avUqD`bm(vj-n$RqekhQs+2Rq(MRrv|Cn;i_yU$wDficx#r3gYG0=_k#Yv$q5 zi1HmsbB!ezcO-<GTOI(8NBZXnfDK<@tO*4=J?ys#RFySW_e1datlq_mNvFeIojz=t97h59< z%wG#%NL&x1Rv4sm8gNIFXpJ(_dH(gF&&Z`F zZVgPVwk$5a5^V5YgusqMvPuwYJeScVxgRTEYGLLxLyV**MRPQSW^z5ON0-e5a%5qL z&i#dMQX3W$dxZ+2V2}qG89B^ ztQ@Rx%bKL=xe}XH0!4IyN%kyC%GDcD79d1lHIa%|N9D|{nesf>!ZNI-R)Bq*zzOG~ z#&UMvh=bennAI1B=S@pZ&H%OnXl(1Chsxl zYv$z)GQAfY28Jx_6?tU3*o=u4s*_TbFdmT<3PWOGcaqNN9Mw|@#u^N>_M}-Y&vh&i z5ix3wb2zo{P!*AeUWvILf$k|-y9>w^Tgaa!SU}qzneIJliP$I*dNDc_$aW`@QYQ8} z)+^^(e3WOR4k737R(CsqFwo=9xF~ZsIUXdHt}N607R*{S57*1MD>97mgC#R!?iSDQ zdxEOu@k9h$ZKf7A>V`q)w0@*1%&Wn4=(Pz9IiTjZ9SV?;n9Xu9IPrJnu~fJ*UaG`! z%Otd=n&vPF1h1viIVld@1o{4z)**^?a>*?y6vJ>fgnJQy!ZZuuE)wk@; zIINOR95qK&!&u>Y5qw9@ChTfMxjQY!I?9%+668{TC1r^I`06Q^UMa3(D=~Z%eveVS0$d*>1JNAMIM?#}+`2GKnchhDaJB z&p1U9q1$vEvsiOf7z255Oil(saRDfcxzbVCVIw=V`GQpDQ&YH&E$ZG{{**HkLGCMz zK!AJEiLwZ6+Z>$j7uh3Ik&$nc3i#pPc%^rgtxP+H*`0(`o=pP`E`KaND?n5)__aaT^+iY2)07%bp*cYFoR;t7UC=yExX(R)Q3B+v|!NoP8YXM%>MxG^%#WQ#%gDUAHJWPa{lP2+@2Xn&eLL*$wG?g zi9mhf2JGvOfbyl?zTmoW<(7VG#5CY9(R?CE%GVrnAm&$lXvMNY)DW^x#6|!M)L+E} z+}xp{LmPUaggz1_zq@BtbCft_$TiZN@@y+0(Rexv55 z&wT8@sT=YcCFQw%k+?|U}+>bH>U9H1m!F=wr@=3tGxdJh5#|@nUI&sN% zJ9lJAlWs@KQ8D-+_N?oo4I9w5bkDmbFT-RD&mZZR_YNy#!uO60o0b_|he!inr4^3` z-oj#0ZdN%N!#?qd6m%|+0UXnC0}4PV+WJTH%`vK3ri7$H7nhoEvu-gWsMizoS!C3Y zcf2h~T=91x7ldIOE<|})zWlxkZRgp$3E_+cvZ5hI>~o~v>+<^hlo#HSGKN)k@G0Zi zs-77TSdkzenf}n5kW2VTy}ZAC{{U=La^{ciOz=>ilFOYc@>)MqApmk(MR9Ukj*Dg> zX0kd;POSl&`cT7JqKyJI5^^-=NRcO)M3>aQ#cECxN-J@hmRvZmqSh{G+i7&j@;>At zV`Cz-Fh+cr)1646&SZjw4BUz3YeSL{0Ugs0jt^aZ=|*D}1AOtDLodt^u&ZBcW);%U z$aSwc{H`NiQ^lS^?L|0A{3qPKLoW1UKcYQL#*j{m+b-dR26;v!*tdA=JXIzvp7Oi- zLYMwzU}!ge@~`xi>wA#+9UMNkv5uN`kw(r!mt{QA%k9Dg6yp()vmsq;l6iyw05!kX z3hf5ulCEkNoOT2AN8%FG-LaQX*)Nj2M+F#_wueCXr6If9;3FPq`epK^(&+5S1cR2H z>BXbYSs0)LgoQR{Ma;?glCpU#8P}r34jk6Zu*)?Sy~kxo4c9BxeCGOYbg+TQ1Yy^F zMqnLajN*G1M)#;4O&Q%g@bA zoL-pCxqhscn`dw%h&L+I&RuHh9h`uPLWbrLi3V96d8OwK%~e(REivJHtiPj`;QWjOM!GU`*DInwmM{{W9}s>Ij^xypb3GLU=19BRzmh&BgA84BVF zuB6UVGkDV5U_=SJ&qOoOnULg7FGK!9APD+peyLuvw&2!OT`v%)V4S%X$^0wyHh=MyN z@c?tdACkSH%7ZP)b{zCW1WzEn7T0deGlb^TZULQCMdkaYJG`7F7}O&jasXsFWQ$^) zb>*hqgh0 zVk$pieqDD(s_TzihG#S=%BuX(jI}{KE!2sGMg1Tx!TH8olj@Lx%oNRxig6_w zM1EoM3C5moFpbD_>t1?7=^eS@i2ix4cw=h&S;*FTE=pA`RLF0OIfw(xQa#6l9QTem zO!@$xKxNjYrnj%aV9fL6e3Vxl5}Bpf-_ z=l0L;nu&}Iq6yEUI98@A$bJW!&Ewm0^^8f%a;unKoXSP#0d&z?X~7G-%^kDsee9Sr z@{d?khTx4UE4ti9LU9&&ezFS&;io^6)^x2~ZPM>Mq}7PV&mf}hob1fcg>j|fCW{7IKm zTXEofR)9F(HghDzXXVi-_J`MANF4N4<2J)lJm>@Hp4e&<%3$6`r~P=V*ti8McM@>4gS7V}Xr z7{U>pq#yy#s__~W2bJHFr#P%g8+*?6TEs!ZGb1YIy|0HhcF=Be!fmL5S)Ah@BdK$D z0VxS_B>YwH--GJ%7&iv3r$1I-C7JDAId*ft>g!&#=)P_ca`7EGcca;Qyqq?BpbWb+ zkmDbk+idQq6U#ni3SNp-Vta`8EHelO^yXFWJxMj-d#73sc8up|S>o6CE82o~4>oYf z!z{Hv@E4$I`9?S~5fRBG?D0tVqia$~Qa&79F&M;o9IsS)cpOB?mgX@L7D+oyBwgI;z*)_O{o+(Ch1F(#HGHurcM29-a zi=M8DEhm`KR78y58Y>AOlu9 zmfet?23*2$Ujw>FY6p_m({(Mm$Ux>!ciqVe!3erDsL}(2DPY#}`09xghlg%V-rbp2 za~cjNWj6bw#pt4VZN^Vu<}v>OSIty>5fOaGN={AT&?56gE@Q{7pqT?G9VvX-i7v>> zj-X_MYCn^?4a1&#EaJjpT){$-yG)2rq~)U9T+f;TnHWbQsxVKGIn*92lO)AvbNjgd z_swX^C7QidQ9MkD${GGpeWfbFfpNDlrHUYM%SFEsxxE|E0Uv2x42S_oIP`+)(-+FK z!cvePNZ2xn;a0r#1GABo-I4DquEKJW5S7Y>l1=l;V#bu_I&IhBs^h9kLM_O2i6Z@} z=Sh#kl*x;7&WJ^y4oH_HWHvGP2#r%4cVh4+Kl@!TVjP^KI1MB!xP>fZYDT1J-3>h! zpsOUv8YdxiI9mH>JPx`5{65=DZR7TyHWM3Y;eFfwj-@DR>6tsjFZ6^6&QY8zmMwde zE>NGR`cPYmn}WRhmP>spXZzn28|hMVQNkb|>#`Ge8rY3oJ zlMMY8Be`_`Z)AQ*QhDiXGnWjysawlC{{Y5pjQ}T7ID|dKYIE;BYPW@No!c=e zz(9;dOxbe3EU%{ zf&pCXeIJw2IZ;cv-)ZGaeBCg2>L6vz@LHSrzcEZy#wrU;{b#u*XIk@J+Y{Oz?|4oF z@`OyS0m!x`pt|*6P247_D>)gv6IJx}m7FohMG;E2;L9o}pEVun)W8Pv<=&l>;<|Fb zPm-}~u#wFH_X?i=?YLDr@U+2jNDPhAaA-SGi*V)s5`V-I2?lZ+QTL%U98R8D5{lD! zoRB=Ptr#OMxLcVcBbfBH_lR#Dp573~Dl-`XI2};P7|(f0ZgCrqqW#NKMSML@+2Qtf zZ2ththsOw6dCrLDEho1>bkP2Ye`rUTB01=@^vBVBND)bBOy;{8L|Pjru_-qRnMPj) z*c>shJ6viV3Cn>!OMrsemFTMujd>v05;n7sq81FTGpGe43BE~zy^h0f=HepU1VDLN zj&+NLk5rtda!dAKZQeJxCZby|9dpBrEWGJ-{eSfs1|5?FeAZij-q!tLn=0SLFKPM;yv4xZ)rshULea6k|Jpo#sFT zy*d~K0ny7f=jYR`R=_Zv!oxU0>BvVT%|eteV%E!DnADZe8+u9-kSLCeY9r6)l-S74 zyUSYk1pAkOc@%01!OmjT{9@z=iHGMoM2_yXtUl`=FU_Kh659#3>zDLI$PP3KuUbhn zyOt3NRBFy+JyTJ7R$@8ir|uZRJY*4P(?s#2{91lZ$blINWysnLWgNxll9>sZNLHLx zc9;@wQUF~8ed)!aZ0FK@%0j)V=S-yt=kZBxVTH%9@eg{S>B=ao6_Dt~eFzSb)otA( z7WVC#iJn0Pve3g5&k9-Yi5_X`bVNszRocl?mLbt1^Gr9=2h|Tq`XfUG4z>vPpgtoI z9P=04w^6B9feg4K9Jtct2_Vv8N%1Tr23goQ>Q-8eoN`ri=!}5ATmr|4kG+n3wDl>U z448x6@uA+gdW>&u$%5QTkd$kPuCSWeR9rIS8uq5&Z0@4uK>}EXlaR>y1cNP2w++Zt zMuUi1*7{Z4eL{{{erjyxQ%^Z|FYtvi*+4m&%qtl(%U3Yx3R8E`J+3ZlgFA?1*g5Oz z-k05&<2j82WEPWQ=NUMlX!B2PTn@>}UUwgQp8*o(UQ4?tpxcMEG!1(RS_|>1%D9bC znz)dA{z(_N&EMT({2>1TKk-6mdp5v{Z?rX5O)S{ftj64s&nwLeP{T4fB;&o8XCs6{ zLRk%MDE9sS&`MZn(4aHXQY}~py~`505C;`5KWN=oW@pZ$2+J3lsOmh?#@BlMqQshW z5ta0Y1C{BE3bAUIs~0=e{AqXid7=S`T09pZab=nu$4KwUZ% z7?fD%K;x3lpFmf}vUWKQrw!j)fBj5t_&PCHz zW7Se`c*{_Lke$Q;XmbO-X3c6`9n*6syQvkZnG5q%rN3%ZBu0M$^ILDkyD&#(er9#m z6lFUUoF7klOzq$VugIn2F3_GyxD3@uE)yNa^2_^H9RbN$%O$zjXm3jw@C%Ph031Jr z>BSPFjyF}EthqB<1PQr|J4A85&a=Btq7&QA&!#e!aEU9sZNxFi&RRzyuQg4F`y5>x zzdH97=iX$qMkJC>f1;DevCF0Hxf5`h^NILaS#%OevJK}-I~$$EO~m0fUr+miO*gqR zkrI}4d47digHA*x_eTVMR-)6gmEy?fu2iwQi4Xwl54jPm;Yp`3wAt689;9a-l$#oA zm)o^%LqEwLpqQLQU=dMh8D@vl&ot9Obgwk6r!0OE2mxU6a3v z{7=n#UT2}6^J0FOJLPeYxy9$V`{4Nzglh7uv6&kAp*sme1mczD%<)Mw*`DE;i#Isf z^$5L6XLc%`l)E-34vrBLMmafpCH6gm0U zi@`r0O*;A_PG01D~K#cP!2>&!ZwTqehJj)oM5wd5Cm}z5I0AnFFgI)rUWo0RGNd$t1BMz8;zUzr5 z{fkZHCI>Yg-Q3)e&!ZDyR(~`>;s8d3BzY1~wPzNpLIl;7_Tjt&8S8%Jm)gGS>WCTl zq@UlvIQ~(G=^$QSc&r^2#&&g9=!?>pv=!>1jqIe>ayZ2e2lJZI4@>@Gmm0oYs3!5j zs{4&&@pBkZIV?jdJ#Uk=^hO#AOxCfSLD89X2x9`AoGzcB$J!0A{mOpLE~yv zWjEeD?u16V4hz*kw_(4g5FHtEzO^_0-)bGXc1DXPBo0Rqn9leo#2B4DyR$KjV&Om2 z6Mp5t^1b07!JcNiL`2{15%FG=eZ0ZB8TpFRtkMhnxy(_Ns;`h0=oRf+^@y<0%v+z{n@wl8sh?A+kQuh3&OaPEJuWGU#+xR};5e~gA z0q9hD9ys$w+U#~5@i`(imDO|lL^JhD?A&&EyF+pV%R_tsU1(d;*GY91i>IX*Hv2YZ z91KS@(u}S^`Si@Ixk=r@pSLgogJK3pqeDRw2AV2E@{T0?6sOxW-w}umIgUTV)73LY zTW`b+K#xQV<>VHiZNU&O_;8X_Yy30&$EYs97|zWB8ETZAGdUwO1Jcie+=nHNYXNPz00|3|5i%8C ziLS;yo+~I~F}+ix)b7T~o_bfkS%{@OU?C3D{{W}nti{M)L1UMb2{{pDr00H|(;JbA z=}Y&0fF|(0k8P)FYLh6rTfnCCI}w@KhU=K@&@Q&t>E#SQYL)J z9`xsBmxlhXTj4i$Ft&X<*E}O@CwEbE zlVo3=gXWps?e`nOP;AEQSjcJI3iQ4gu^UG4)_D?7Glq{Kxqk26CmffzZ`1mDdv_Gu zY(^;^7oAsg7wFWjNGZI6O|KR1j~+yGYhlRBiA&f7~Q)Jo7a)3pMf@wEtco90O5@r ztH139EcR1+_81Tan!QI9n?H-#xw~P=Ok&w`<86q&mn8nvhxi+iMC=g&VY}!E;#-%IRRPZF8#^8 zm!rvztGKJPnDZinpx=gWCx@XJ;?S87Xc?y_^ekdIi?38>x1w^4L6<~k(~>48&RDYbTH0KN-4&k>ZN0NmoSSZPCm19GM?_T;35gomUVv}0IY7oB#zQk^xasX)ZK)9m z<)lFPCH~nU?<&(Tns0P?SatIq$7ytKHb#0^$uRcjb7)9N<}(^ziW`&1F-P>OP#<(J zg0T5%_hnGb=KxLn*UTP}(z&I#66r#}2p-f%M29_jD37?F6VdujUwZkY?O1$$+4L)n zNJ=6ggCGe!XcTx=w_;{vdH|yBU6XIPB5Sn6U~>R|mFR3r68@c6mu6U8bd2i_dE}VQ zJGq#7*n)HbdE`nfmD$$_SFubix@H}tQkJJIwM{rDD0D$KIRbSaNli7D3)81}>r{m9R$JlnFMqP|0BV8#BXD%47ubpbO(8rSB8OWJL>s4Cw<*FMSUDh_-47y@5ywJ5b+YUj)x)}u6d=*@ba`sM%%K3F=~nDcMnQa zax^O(xc)Bn-MQ!=-aSEga_BO0Abw-kvd7YYd)yaaj8FPm@Yu#(t$===MgApW$nE<- z;kSz{`!RnSP&Td>vDIf<`C73l64mhl`$FN%kMirrg8u+bxt2}fLF2B8`s(+~kv=-69m^rk0})My1EPKEH=?tKE_x$2Y^hzacj=Klbxh*PYZ=|`Y`K!m zro7;H?EJA}^Q}D!XYJ0*NJhO$>sB2SbZ0ENp0t#q(mmy8ws#pfSL6`MmlCNloix*C zQmd?zljzP}H!!U<2a*^{rYe7S8x`+TyLWFp8%OvFYdt3sgavFdopLKu~ zM%$E*Q75<*+j|$MQVh5UB1HpQ>Q-(~txd0;-ND)bOD;4zq)_7NZp~RLlRgK#nKuZ) zRpON6UELES>_=n4W{4$0-)0DV=c{NE`l=Onr(7a6^3P8362sa@4ELtx1+Wn^U-i#w!X_ z4q9+cc==A=-aNJmHlsa49Cq=Jbc!an2q@e6IX@4h0E`78jaH1ZS?;$o1pMkN&2F5< zmvXhbpGA4cWr@Y#_3?TC0Cyv8LNIF?suoJ?wTKv`-FQW!IEB`v1Yj})40Gl5-P-NN z=hZf5Y!l1mySQ@<0rU42hzb@h%hq*tEt5*?V9l>j}bKWg+_O~{VbkR&s176oM>mA{6d-A!Phv|DgtqgMI#%~ zr16V@0m%|Fl%p!r^FW-3Zv*{5HK2Mb2YFI=eiyrRp2@jR$vM<()nsM5bIutv?aYfqvn{0fNcO6 zrtF9}cNl|SSbgf8cDrZ*BLFK*$?GL#<#~<9IGEjN2>7ZE#$<^Il43Hg4XYiCWAa4h zRz;%ZIUwRh;QP_e6~mUCRvFm49+e0a$A_Y@S3-^xhRD&kHP(mEG#0!MXxyw9XQon^ zBfBVr?Np=Qiy0|k18Beic5(Hy)_jrT+#C912zNvdrO`S%61h1}T0{=@9DSz(J&PTY zCn5mghk@d9!VxNCRcOXf0yAF#zBu?SRM4BU7QsOVT-PNr9(QUc7gBwszbLh^F z%%&rhM{-`r5VaSZF&!wL;tx`LVZY)U(QH5hmXnc&c7}J&M#W6|fIZ|SY4TP&AljSu z@CoQx<#MlaTZZSfkm!MaYQ?a_rswJJDA9o&6rIcRgun-v%?r2OOja{qS#>Q)HR>V! zA4z-E$KDN)1fmZ;JvB)g&TQe7i8UZB-4vvD_03rhl5ohqGs_&0mZ|-rZ?ib6@f}6- z32DZ!$!~59g#bx1pa3M@as!&*E7KBoZxC-@YMXnkaEvhW*7#?oC1J4Lx@)j>au}f~ zkBoo;Fgz!F4CL5|9A^)7is4(!2QKssXlIwjB<2CB0Wla7Ki0Kn}O&y^iJ693Xl6Xs&#)@tr(8yfOJe9)LR0CohCZ_Nu#o z0GR_5u?LtCBzhb}sR~=D>5s4En{RYnpeHPcEVWp-m*u{L{{SIi5+?M?#d!!oR|+_* zy<>0T-X0u~a^wItiezVX`EZ>V3)tpHd0G@@QYOOI(PSKL_L$@wZZ(+C5z6>Z!oe*< z#9l_mDy!auxy$nvPPWUN}_b_(&#Ze_WA%zzA>l&dU(&awi}narbJv=>z^)Hr?=a=lA#9_4ZD}kRBl$h{P_0t@Z_rrS|~J=jKJWT1Ob^` zYQL9W0XAGou$YFK#}l9jKsP; zJ_u0Sy#QeVA`M=Z%fD23?A*^3fW2!vS@6|)?-$V5;fVhL5*!H)06#PYeT&df!CHC4 zCtYxW00KB;r!q*D_fYRlv*CMIc}jWMGLg&5ql9}Fx2AFAKF0GT8AlE0s5kH>JqXRn z)M8|KL=t=GXG_(Mn*?UuW&6Vy_gXjt0;7-`0gxTQD_W7`8`$n#b}|gRLnDeNBm7jI zkB2>S1PE|4IUm&pdQxnzN(NBNkHK0o8LteGqbbCB4A03iEt68KGs}e!jzv2InFMCL zndA`J5wT>P!IylG`e{7|TZhvKj^Hoivz%6hl!Zg@wYBn)cfk>TOD01_C)eb%M7Raq zUQ6qdnfi74HhW9k;Z z?I91M5gv+8iAaorD?%Qp)=1RQ}`+2tWo70tnSYjbZ&dl%FmS;~@>mB+>aOw)@X_6xk2iZ3RIhhfd zsZNG{*h#PWucNf^=$ui62_9xc;7JM1_x}J&r0w~xTU~MuF|}(rw%9S!pe%Z#M}HQ@ z2$SAx+>>{odRcU?b!Ys>UzPxIvwtPmw+>CNKpMzj5$2|ZOEJXi}SDvLzvW zhd+W+RCHL}e-wHyg{TK-Q{0@3vbv#3Ms+>N(A6zMug+bup~I<3-wanNjKmFkU%gd# z_~+B8DgGg${{W{idSg7iJK=knw8Wx8B)Cs1`eWv*{iO1W^ZPHL{`a^l!$jwGPYF&( z4RcLxr7qe0b zWC_EbXl#ES&)WXrnTNeTL3(tW2MO=4rMx6+Y@&X7ca`*!TOGM#OZJhGIfQ04QySE& zGOe~Jvp>9gf-9M{eLB##-@^4M&VLB{b%b9ii>}BvArY9Y@6UqrgyWQ_rXpqWHoZi( zK8wr8Mcad)KZ?6YGNn=vVNrIflIPAk@$nX-?MD|1Dco`D2^2&Q(w!~S@c=dsp1 z_i@yce6nA5js6qyL38R{D>S&|;4fc_n&1aLQ!i+im(W*k4t$I3*$?lSfFtTMH>E^k z_b+D_vdqMJ1?0Rd{{Z@Ie(RtA0Io~dNp!4naBxNe1}NcWw`pubvnzpV&!I>sCm3@H zVDOZ=tiFKWss}{E?l6SDJd|CQ615ivQ9&y_$9j>RL1xxh%1^sOHCu~=iUihTpk%xU zB6xYAmn2Py*LvXkqx->aNy!)R-c{tbB4jMNXovVG)djzyMZ6GN!3d9}A8NuV`c#>e zOt}OraiNxDg{H^SfbNPU1@GchhPFfsuD+W6ciX37Z)5H9U>wr|HJ^D00065 z0RaI3000000000000RUC0RjL92LS;A0RaI40000000000000010s{mE5C8%J0t5pE z1PB8L1O)^F0|EgN0{{{sF$6(TVSy7eaUzkSu?6taa=}s*BSPU~GorHbl0$-X6=Tu{ zQzSIP;-j;ZHS*KLgAoA%00II70RRF60RaF2000010udoG0}??KGEs41B9VdrzyJ{l z0cZk%0|6oc0ucZM5di<%01N{G00I#M5dc2``N-LYwQJKWHTWymBPqJPUEXM7jc)HJ zr@`B(k!GCSO_H5QP35O6&6{-g;Pq-0tJ8{uD>c1os&idVAK zAw-%bDdl>cGu+e7P3b%8!GfU*(Kfo|?;2lbVLIQ`eV+w*SGv=RdEOQDa>;Kfx3&g{ z6!iF6K3}&D&g3Z6m?ctUCZGwXx~910cUxswV^M)-oAeUVk6F?4<;~u0?lmErE;k?U ze+!V@WhWjs_nplNt1xfr-?YRn1F4t;q0S!RwY>46Z-mgm! znqAE|);>41{V&P$xU>dt)cg+a}1x;w%gfM5?-XR)MA=?X876^0JfX6qE2Puo^DhWXQ=1fZ?%3N z#y+*jFYhZ`bNv%v^jqyz+g{0=vv^9yyC|YvQ7+9a8K8Lf!S;W;EnHX4{{XH(VSGyF zyOE$rNL+~1?*ln(SwT!`>YGF66!Fp{Nj4=)Ulb+YuS@DtK1%h6@{DEGLQ|_!jJjr2 z$*7TSIOUfPnqQ=*SiW7fJegvujnArk!iks4{IkDFunb=-B44v|`xB-M`_JEgx12h= zK##eX#{{XwC##vM7 z_7zZ)uYj`jRHkDY@gc@O_`NNq8mAQ9D!g6_g4$iJU!J=WWZ+Y5nn@0=*_Wjju!14X zM*ykOXp@VKt+g^DX^UwoQyhBS?)4?QsK?Wo>Ex~*{Nr}2YfhpT(i)Vhl8#a+NiQO@ zkUgzxdV}oG;bCA)RQ;DMzHf>CuG*P$?%7kr?Pjy+m6!9oDxidwYD}N1P*nuE{=?;( z&cZikx9|5GCjRuab3W}%yR3FBK@%tS~xpjO;w|lQU%XZ|JVBMz_oAQk{ELe+v)ft8j;u`dsDwIM^q0zbc@P zw|rg=zpu>y0Afm1xzDmV+66+WzYF?##rhHt%k6@IwBz+^mWZl83pXf623E0P9p>eW zDx~ofZ(h77lHt<)_PO!4w^c~X4CFc`YS?w?45^kOCM|JIr|nmpllgYdS0`4mzsu6~ zRC)Tj0LE_<$;3k9oLE1ysko=MBCcd=t|da5IgWKR&y-G0Jvi8peNS#_9X|8%Bv@bh ztlQn#uU-*ZAqrxWLMy_CGD$6AM=SHhOq*@Fde}~n%r;eF60JeA;T36$W!X@&`a5YY zwRYI=Hmz|gtWD39<2?4H{Nlf(C>XzBl7XnE=SgGTbzCA21 znCHBDUZ&KGOBdbOx*=%A*sC$4i+qhF2b9Fl=nuJ7h7n?r_DLO5f)5n2eP!E*Q9K@lEg z+kI>cTX9mZO~T}~>1*FxG&YsD62N^bF&Sh7npwVi_*9eNtQQhs(@mzFrnOCRU2oFt z%jQ&tI@^kX44HRCvDGx1SY9mYgU%a#G8bzh+iX{+DipugF>}GL>?NUy@n%`%)|VY$ zeO9cYY*TXFyiLxn&mKA0OV{2}3WUk2Zg7GY_5IJ%o9nMj)jih2sK{ zDRplA)ZL`Xp%mFS2`t-hf=a{HJ+jbqPd#K7t1#$lyDe`MIH ze6WTd>GxVb>~pHc^Ob_1`DCj0#h0nBFW+NdH%xHV+R{ELgLK;0CjCvU`gNCWQ{4Qztj$BApYNv5Q=bkD zT0g~M2Ue|0o+*XHMfd%73-vYj=vE*q6@ipuMwhf_tm^G2wxshR^M?NKuEHjc-mc-k z>{%gt2kQXZ9jrboTZ~M0J)lCT}uB!EVW>lN-E70bymUIF2`tNZU&9;Aa z#qR~C-v{?a%W-z5?V9Y4iRmR#hjJ9e1&pD%Q< z$rTz#vX;nTkfqF?OU~oK@~51>?wtDy=T%ETlwuTA=Ag9DQp)z5TKD2^Asusx!$ zM4H1_1I;mhhdE+ertTJ~;HYEBnqE_rxFQ@57|56b;#^aeE3zG}Pv*&DRd&?K zT5y1j2wUQtBNrT)-9$yTuwVMkxBO~f-&XS8?1)^_ss)rc-BwSWrZl6n&9)7-dbEat zEMuk1lhb*VfFN>dbkMi|#Lm18{IP=p>3Cbza`F$Eu4x@F6 zH=I0$#)%Na?6`gLqo{^FD>+4gNFU}igP4lpBkIXll zc0JvCgNH%H$QDuHDIGM+D`b9exq?Qh)y0>pn5LF|Y2bW#&Y4_Q=?f;3O}5fWX4`rs zxVsNYwJ_ant5v8S(x%}OB4~~2OhmPXnK^OzY$SVZhOJjmE+EZi_I=aF+OJBU+K5n6 z)hz0@9;&azO{kGiA$rC6+V!1dWunkmC1Ef5eU)KkHRwu>Nz*MkR6sDjmxB2@$a+m; zSiK2VQ2xVfJ{Jjk3WrIe0wE=%7SqoyR(e=HQ((ZF#HW)@7qP8wQ8A-HKu6FEMLwc> zk#%-&T?$>@aypf~RSfD8V9WYxV~Jn6xVWGBn-Y%Zk%D}2qJBgsG1Dsq>nf&9h+14H zSg=F6%6~H1nKx36YHRkOs5KXp#wo{KRxb19<_+V(-vg${*y*$E)!0Mdq1aG+72gwI zm?YIPiByELY~~=^$qIQVp0}D?VWU|*5^5NzDQ&4xnw8q6UiVZtM@+p8M~h-eigj(Z z8NVs8kKS&vHWc1pxvnqGMD=`bHr)J;Wf>;4%dehq%~%SFi*mHE70B%NJ^C7zNzSPBqFDR zSt||hH)?>O6on7;>5Iry;@nKWC9Al1xv!0qs6yZ2CO-zOO}CrPyUm1%eAjj{iG@_Z zZAxI)kuwOVtlRs2@8-#;v!>2dbpXzg3gI(|R&yNmN!lBIff~nEx?eDZ(Qtl6V?7!Ly3@72+zZeUQ*$x(=6rM1+?NOEq15uYwxmc zDejme?`qJZOrVn`jF;gA)+aeKa4Iu=EzQpTJUIUVP=H=p8SfKvM@b(;nD`yd70 zR`q@mc?o5cY)cdSDkm)XUG8`D>7*W&F#`ZNv-FPz@z(mE%xJs{rrsWS&Y8K*>VVNK zm+h52)ws7mFR4GSV3vbgjwatyr-);#&2_Nz%$d1!pF7R^3Y!qZi7I9U@;YXk@?16K zu7hejaz};F2F2^DYtqtvQkw)e{{SvJqmH2}BdFp}8RaQ}Wh?oc5zk$1*I#F|%8Mh)jG7)wG7;!HS}nIN_P;v` z8s9K8$W|1zd1dxT!ahrdikHlHbbJ+$IMxqNSksQn9SyC0$MnKf#wNrDd15vq%lT${ zTgz#++Cf%6^uG$kA?}Kg`w;oqF}3-T*Cr2RO0#-kBBh6^xnhBHaF-WtSMI4(M5>UJ zoK)X-SagqA69+0+^x)p?dsT}cWuF7EQ@nz` zH;GtFS9zGbplj-N^hJ5ZovYI2jN75jdW+54rIUBa+IP=4c674shjGW^YpYi&7Zz*v z34b%a#%h0P;;`J>cjq=`6@$0n?s+@odv0{F-IXfD1mqiE?aPNwc5Ua&Y47Q_plYuH z@bW0luWUTk$JAf1K3@x%*;Mp7=X#tE1GuA8Z&0b!Oh{Zb{+e=YC&K!cLhsgm{(x2r zbrfPka9qV9X+CQ_xlQ>hYtq*l`<3oxEN0`RNcj{VfBQK2O_@{Py7?tiPDk_P?&? zwvR0JbN%aHQg(*j{{Zi*HOiN_0(T`)kyEdW_fUS=e^u_D$?i*dQn3Xpcx7nPWiu;W zIDg%7PU^Ylvl)z7Mn-fgNnvI`PvJh}LD>*?0-<&Ew4w20fLy+X;S zl#Fod`z`(5dh~YGPMHcs@RA_|)?yKfM;VQ7K8MT*Y73|W304{`F!tqHrwO#NRHh=E>QUCoLuNFJop3C8T%s0;Q7tMT~RCNNmUzX5Rm*x!#7HWt;+F$w4 zpz!Nv+U^T_bs5p56u?_Dr`{~|NK-SXiI?q!qIq2&=7lE}F{IQ$At+0uVB@t-f_u%* zDW)y|05g1geRmmh>;79=6q!?NFIXOHqYn08JWI*lJ)YFkDs}0{H&9dm04-kVm$Jep z=k{AQwO#FB>f}HLr4_)rkHVrAl)XfSAl_F|1v>6$MKdc?xGx z&^PBIsfC?{i}`Ib z6TtCdN_(;jjb{?jmReqR`)X7PnGbkb;S&VKE^uC!T&7n}H)pw_vKAB<7oL@k6lGBO z4olE04>a1Fjr`{0_SL8h`AnBSPpyM{lH64%ktx-;BvZk)apg?MQ6<>z@%gsnJZhCx z)Im}O{^Y<#;_EjesuQIZ#3-Jr2^`^RHH-+23f1OVvhb@9KTefO$AA>P-l;-dTPBjZ zQKhxm!BmRI^u#}=rDud1Y?|OjmVOH*EymZpz(hf>ue-)Nau)xTa4oIPEK5{G zK;`gCsM8nL9##nMclB$x490lkY@cu4XeV;7KGsFs)E<=&?2n z{jK4CQDocft8Oz%0Bel$7|lvqk*Q2u0VsOKvR{+gzf9LPIg$n|T)V$eE>9CaYAhMs z?>F_EGYS%K!k85Ewn#|UUA7N zONGa@Pg(ff{Y}LNy;!L?`vm*INv$&k-hM4sSv$SC?MPPL(O2ENyN2}3URHLc2p9D-x7x!bHaHOFnISN@M-f1q8Ylzg`93*uJrV>n>@=E1D zCrXk;SnE9GQ(_&k(PgZnUREO9?r9DhVciQty&@|X9Wg>kpHh>UbodSJ%)M_XW& z--mBnwwYu90B_%JT9lfm-&nR$CwaVx>VxWjZp9I2spEcb`3@`GpbJ%JIw|Ru(Nk&V zY+9y})dhiPcC@#QQ=34V5@TqcHHzZT`lt4a-fkVHV%Zbs-SoyKNbP^B9lr;2^m9(& zcHhogvu0H zM_20LIdsy{+I+wBdrFl-O+l#A@lP*ZU&XWMfaLW?MXMDacg7DP0I4t-P+g2kQA~@UY9e!*gDk?trhEuAz#1C`Mx*jsAW{VzV&;T zY+0n<9IbL63+HB~Qx!6NqQ;&)K6YrGdT5%RXD>Bw><=EiC;Q2L_Pu8feW&+Xc5eeW z{LOpQ`mZVU?$)9W&g>q96ifWGabGi}!*f=m(xNxgV?TezTRHx3bN>Jin)cfpqg8g5 zlS>eWAx8tqW~FI^R?B%L|_QGV57t^(*{m?U(`WV7|*hWwn^_J@j_OM)L}ab`R& zhym!ziYNA}CGNPPSEt|Xuv6CzjPpe+-Ec2!S!+y}nvFVNqa1H;shYw`_bxMm5_H^O zoBYn?RuxvFOVbF-A#jPmM~e34` zI~p}-2IiJT*Q~%Nsz$_`eB#42zGJ#(GJ>z|LES9pM7yLXuK{Z1w}DZ`F#$-AjfZ$SWf zX(vvbh{XQ@am=~IS?iM0;2v&l96g5M8%%Ri(dYcH)$Xtc-l^0Oh;5}WLC;p#1;+Z8 z<8eq>)2mPua?qJ1-jdRB$z1wf#TEI3Zmv}-5e^4VQzb}=q-0KHrR4Hf7^-yklzX7{ zAxNXsvt8JS}7863J znqA+zXRU;N%xv~u(hB>57K_zNk^c4g%{AcsDfs@*rwyeJ?ww>UVr_t!q*zv!AyhaQ z5V1tYQ)Ap#qqs#orHq)Snl#C`8JPwc>y0Y_Z+>7^P2g@@k~G}UN2;}heYyuKwMQHc z;~vpGLi<*^-v0m~oM5d;o6tnWiTMb&?|<^zqDfScPl*91o|!x|?aMm?(B1C$1sG_W zN*NV3OhR%*M@Ioavhhy=Zrk10+tF(ZX~eY{TPTLqv%)g6O}{r{sZ{Aw?r73@Maty3 zVy*OpJp+1;(%NeJM9!JLCaq69_49I}P{ZW6U~tFA-9d0uw)Hv(}$ zs)-j0Y^CAPLDj8O;b21A)=9Q4^Uig_keO+OMDrv@v zvPrST)41x@&bF|Yu(Dsquq+!PZ@d~sRDo` zpBEJjQ=FKT7fT2+1FA?sNXh7hHs4~oAlXx)>fDmb9snfqbYw4-WD_&uR^>HIx?@7M z5Lu$cKqTS*ZVa&^?-rP|F3A+fu{^8?SAC#R0SarwmYrd@FZqp; zb+Oa=j`oj+y-f1LZ@d_t5S}IOz8#fT;Jpir@ipWrrr%3#_j@%o=oyw@s%tfzNachx zQlT;z`|J(3tGcK&SSTT7h)$pon@>yuM5*FYn3G6yYZ9Bz<99#^kcugZOT*Qs$l9J# zQ25+sZOvNW)Nek;^y7U?Yg(#J3lV?3lR*i?X~(KKp-uF#lvTD>L9X_bTx<)lUEaXjSvdm8bCo!x{t5Js1f=Kb#Jmm{Q1C%kz%V@Jt64Nle zRo_*purEL_2&}NqxMBrPuL@Jf!Lj*joKG4a4GvAlUawHcWtOY12&vJeG!a69b1>#h z62@+H#zlvP#%-49j}-=D{5a8fX*ZlRCWTKX?nrkPsIUD-=e+TH7=>1oT*?WSWwqUc&F*6LYFzPQx?~6C`HyJ*Qk*tlbnWd!^;qsHDjy2uTe%O zg(wkE!j_3WQd|MAz-4o@zOe$hqb)*Bd_TZ@$GfaltmdNXB|* zl>1$%wpAW&h$YSEu8Az=@@m6tcodXSq=z)omS&+wPsEeuiaK9W-R~LUrbbMZU_y^B zFc@D2tZCBe4%URIsknKiifN2x4h7;wvu)yO&X?6U)lFd(G}CS2rUD~8;V4h4N!Uu! z+}9j0bQ^pbYx;EvPJBJn!H8D^6yqxeedgGz6grUGa3YCzV_^NSw&R}Gz9#AqcMT$S z9ugVNG8gN^*9={t+Xs%R8;2^I$rR+#Im9Vzl5>*M!e)@?X1?3_{{Y+ft7(FF;ICO* z<&~MzyDqP4A&Y3T0cB8b*9T$^6Uah3l7tbLya zY}S9Oe`vlosixfC@n))QbL#EA@U@$Q?Y%_3B{tvI4&Uy(mP+Gjj8zxs@OLG6`qnpq znMF@4dXvVA<4H|N^x<#qbStV}rLq;ssXAq!E}6}zSN8Va={(ZB>2{XeZacPwv@Db5 z{4OWvn|Rz6U>w>=qmD_+U76YC`h4#uZJ7(sB0Z6Ef6Y6ZLr|#FYJZBk3M9L+j^2)R z_B&d(-H)t0QsK?{S`_P48tNs9Os>suTsfZJKh!Q>m+`XfOBZ=OT>FRWwaGf2USkS z(;*h3WgiwZIq=jD&UIDWtuZpx*8&CXoqeKdDZgSDO4)1c_;+#e|2;sbSky zKhu5Th$WjeG~|z>O|kQhEjruH!m4;G)fGRqnC7x?T>RYJp5&DpX|ZB`(Bl$GCs5>w zaE=%iF^4FQb8#cNrpjv25?SRrcG^|xtne-R+PmuN0g=eW@2-)P}ng%RmI^Ii4JQU;W|L`jM~XYw~e09yqc9% z2spv}jrnSaNpUa2wUv9V;Xg=Zm+nd*kn&4}Wfle9)~Qhv5jasSQ%NX zf~grssseH%*HpQy+;1?zP(mpvO(=}sLo(MnmfO6Z@i&BNuJ26Wsjp*6@mV;$qoG-d zj@a9IJS51UHj+UtE*xwEJMGYted)N#V``$$T&f~mxJ}G$i*jk!++gm-P6i5@R8q5D z(x(ggiNoNXi@4eCOi7TGuk}X|i{l_T@kd!7I}YTjS8M+OSOnfkaMW4#brMm{%gVz4 z06#{VxTZuef++SjB;RVVjctK4XHKm(DWo`RPdr53D9@w37t76zs+39^UW$mJxrxIu zFyo!MPi^^HXw@^RQN$4}A`kg-G^bc6FDXb`u+QXa5O<5b3V*fL!Uoh%JU*p^f-F$% zw2*fM6l&22En+%?4YkFhV#+^jhAV=&>d$vpajKE5Lyp+h`MV80v77uUadXQh2{z|R zNu?@cY#l$B?y4^HI-=k85@an*WLh;nY;?9HW$Sncnn=n0k;IeEH6>(K_WM=(0gk8s zWlYx$#B>;GF^MJk?NZ)ubBcxusKNxqVeo;BtQ@Fxw%JjnbC;~A7i&mfjf+VL=_$L9 zt5nUlZT98vxmpW!!}V&708yKzPrTMNXkQHR5VoADylywtS$>r@`&7ApZc8@x;FW@- zv8xkVIs%+FY6>#YOAc_;SeU1xcVk|T+d5;6;X#+}c{v=EK`9oJB%4^Q{{WexT+qF} zM=ycnX=gY`0O!Xhm6o>8WlViXQk$3QEIAA>QovPb%QrgMM-KCIj}X)}`6rX;$40t2 zPUS|X`4E=lxd2|G6Q@d zsXoG@rM?(^-H0|geO0kF_-U_G$^PcDeK551&0Ut@=;cC(aEg6dkrHYe#Pf&6Nhz>B zeU{poBgcdJO_0=1bTDHAT^UCv?}KYVa!!R)COC0c2M%(=6xm|MDKTPNTzFC3Zc1Ni zo5?B9H=B+qW%RweTWV8|tEkuQrlCya@k5bogb6DLcQp6O4kpmX6{<^+q<*Cfgk-HC z*w;Qnp|sn`qg*ks@V-6a5A=ea6`WyGOB0rvm#CN8mmF=1HFwm9I*1pAy7&(U5tNds zh{9(mN?(PyM7G~QXZfl~>Dn}&I6-GaZ8t(jG0N@x0| z1j#=R6(o!8C3eSUMe2KqeQASOba3&eo@t_vmhE~wLL5X@DYAxPTHe zq9T(RtRXEg*x+4M-#Hh_-dJu>zUk8uXjAT&rM?nkn9Abjc-rq}Sfr|xS-I5CVZ^1) z^Lk6LYOeZ=Sa69o9t=V{W-l33aIs)6L zMVrJe6;kgZtznlFRqiDyNxj2(dIq3&F6J}BxEOR z=AKsMD@zu4q^N4O+NyJ(sJ=J3l`7S*{aT-BzBb(->5p&4-nBH$?VDbp%bqsTWbJbC zT#J`UT2YJX%H0c$q2pI6O>23kJq4Pp+ieYA>x8Z<)_yCtQmrdZYA))7Op|sd;`nm9 z8)XL5(&)deeV+>xQYfv8qw6oH3j)#=0`%sSML6E)*(ko{*P^lk9gQ z6%{cYTxa8N(5RX)IKts)rgN`OcaCFHZR5hKl3$3v6ZP90H#?GF$o~MoKV6e~^77l_ zstRfMkpBRY<*!G=82LdCy?@=Ei*L)97J00#tTL1&lxg4ez-GjOTyDG|&hXz|41SWv$=Eqg9b zxB6>{oe1;B3e_MF{a`e*BWy!}wgf0=4v!^zoKo}_!kGf7#K6%8U?=KWNjV|aB&RnK zI@-1NwQANr>@i0HYI22~war;}G;r3GRWn35Sh2*2URh#V##*-tST~fBRc)AyX=Kr7 z>=v)l?DJR>p}U}Sf}jw+QfQR&YdtlKlyy(?fX$b9sg@?wXN$Zjm#cBk;!A2&)c2HJ z>4vR0{{YJ_*twm_RoqY)Oi7XoNwlQNu}zcW+im9cy$G6{?a8bpLgD55RPvAZ=i6-T z>ki=Q(mFeG7RZjHMvDL@MGYGnLY#UUoooSHcoL#h=SWZ9KwXy3#2I5Mh=~whCKxtJ7qJ#OcjZNTc@{ z>a$*%Pw^{LcBoXTjCUaB6e@KFJ)peYPH$Ulb5m|VrH-JefmG>>aF$q;sg&yB>z&1l zqGb-T2Dy+jP8S9brXG=$mP@3ZVCnHS|AMT;nU1?%2>&_M&xg!4#!E;Y^t+ zZBBwS5Uc?}K3*h87~X{26@-+aLdOzreOXLp7av-Q8dV)iFBCZM21k;Db76zq*TzVx z7A3Ar?zs<_(W)t(N~cPgIdrj*5zk5VEio2cBRgMdLbfDG%mi~qO}0(Eu#Oy)YgY6e z;pt}G?km+VNSa9wX+9c8ZO2yIdbCvu5 zITb!F@Ko?UbWm0CY2!9i+Bevq~TerRim~@1`47WX0lDUo?6Bnql(Sz zWgOidxPh1e)J||x$VKG{UNa`w?k3`NooyQ1npKuyNv%D3docT5=kpXx8>>W&o{Vac zMzblS#jaXKmF=j~6zeBXH35qeL?`Nn3B4ghS>j5=VYS=t1(hdRC8<e2}+`%PT(H<-|`FKqA8DyAf{` zekQbB)~`ud$TvG(@S3p85XfjT5g4Uq($R~rJ_wK@@^~`tx~;D zMLI5+XoLfvz8-vDZN_farM0V6*x_Qhkz|**Hu@=U+zKyjoI2GTQq?C=Y3TJK!bK9A za4_d5V=eaOo=GXo4|XK8)5I-%BEY4(RFYLIR@~Gd+GgD`;iGLglTzo0rCMdZu3nVD z2QNr~u9~$kwYzKxgvPGr(T#h&1B{bX54u5ydCaG(u^#jlTy$uKG13&GhpNRdRg>4s z!ICCIPPHAq1=J1HpI3xRqv9}AoHvyj1hAE&1cdZdDt>oohRJY+ffC{{>CU=dB}&}W;4WLtvU0;UeWTB0Hvb1lbqE*QqC z&T9B7ZyL!3FnCjD03wm-^H`Y26**W{6t?RT?n;}C!i#vJxF91=1+yw$us3nq9DK#1Rb-tloMU$ttc%SbR5yF~k z89I4RdE}#&k-ATES=AazdR98=fPy6`kb^|U^FT{Z4)I#1(}<%zUIm7jzCziy{?mGX z+sTgDgB41K>VDD1-g;Aix-xH7n^DH~LY~yt5y;UZOva(gOEJMsv1xI{xbm>p-lYOf z3a$?e6I6gt(At*CmMq*X^3w5NZ%9sRZW?ppZPg%saFb4WG}^i$tWYk!I%5RI5e+30 z@`a*qTz216+9H=t7UBsg7ARx+u(N)ovdwwe6>VsMV@E`j1sJyioVaiF!VmqexEAbW zJHk+~0Ly4;v?DK8l+JNEH^TD6bUu-$sbIt{iWE;4shgJhYo=F{qL-?*O2+X<5Ek2~ zmT1l)@bERqrGX8+=X^DPZ5_TZNeeO_@S4r1h57}q5VDPI zs=nf^U>c|DC_1J`n-rAd!%SFA%UMsA(AKCTl}fa*&4(`~yQw9_v6T@~=!ptI?Yx3pMB#e7dIqRrw>+o-S!b&rjwNO7dR z=GZ?2?D*WPv*@YTM@FKND#=GbQ00DA(p!D~dSN6XQHq^XMxVMB&r*4t{{X7r!ux4L zldHO_)~Knzt(7-C=I2RJlud2-HK)UgCi)S4KC6_h5+b;~9`RpY$M3esMxi2JnurKX zfiB6-IhDgJAZ|eEQ*#eSxiF8pP3aC%P-OWE{_Tpaeo~0#>kwXE7N3;Evr#EIc`PG6 zJr6%>_u6g0%TPr4QpM$nU%N|`SrIB&O<}02QW9K$sfK+0Gi_IzaAM!9&`m77V}-e# z^Dkwuykx4=A{P}4ZzZb65aGAQtydRqRZPYdww^Vb{iH4DKZaF+1*r0N8rV;9LSdxe zrLK+4biIeD+&)fxt$NA+W|NfsZ?A`2*6G2Tf(?^mEtXlN^6i@iN@#)}e3R*VCg0Po zjml|ftXtZejnLark$4!A%@C0&Ye(s7-KYrZ;ggwuresez;%u~xyKQ<+iCK~57k#Pg ziG8&cbWL}MII0R{H0orY8TEwe$d#6k)OP6FG(=F5AVARPi54AaE<7!5t!V@PCCC^_g&=s)dYdphm=qP?JsT>39)2{(UbCW;2-@6ovYQ3u*os zBTp>tt+D={M6&%-aj$Ldl->O*esBt?tBBw<0<@$ws>*_n0Mv2Un|a!FcI8yDBVLx7 z67M84)nY)Wic+6t%Tgw|w6lGYY>~DxFDt*LPtGFu70Dz5ed9^ZT850;*Tl%0kHavz zOuGzjjZ+dy3-Fm!3_5FwLp2%YGuGs&kQFQ@G|357B*;XXnvY3NEUnX7Ag2yxVIC?u zUHwz@N4kq-)&a#+3jIVq+83PRVFeGA%yP8^oob0XD6?5&sGMo5CWnWq6t%J;<-?-F7AiWXiV|OkB(0OA$S|@*mRW^4dGRC|oAY0uFk6@sM4!UMYupH0K^j|QhVZV1vW+M;FFCMxYqUg21-db{{RXx zGJR!;uDMQJX8XG!XekS#&J%1j)5TL6G^I{ja94ey>sG%pQ8KK;nhE~^L>cG&paY|c zmz2t}7UXH6!AW8mc*r5L=LsdQnq{rk8e|m+yV|Ik4stevOmlaGNpnC z#CWEmWGIn3kVV0`P^K>KCaRGtj-hUYi%a!gGL_bWtX|q)`(f5GA-2&cxA*-`g!M^ly5|!vx z7lmtx%bvcMmGBY_)f_Pm85pek6+wm1D7p$mxO9;T(hxQKNd)2aizNuBhIwl5GCEPe=QlYrc2T1B|6Q3n)I%qLN9lHh*-ijKmg=EF& zrfiyw;^I1{=W|t-5Mew?kwQr*XH~qRaqPUh-nwqJK&gd`Dqso7>O##S9+_g!UFMJ@ zstHyyso_aBEx2%zlGm0{^qbvRSSlB&MZBAJQMSC>HpZ^82?|9HHJmV$TIF#B zqqG{hoj&fa6)zo5>s~_eHq*-@*6bQwthl(zG?U=EUaEI=^8=xp;57;({veZ6u1h^SSZVjPN}>|nlW$br7Sf$Z)Z2*V z7tXHWT#?cYUa7_2yz|24)+F-GVQ|HJV_DNx6+0wm25^WIvWwc(L2`SbmGIis52}mCNTohJY8ir4Zg6f4rfNG;2!(b z;ZqMtY|vs(75Lg4%>Ct|cBOP2l7BWigu#3{^4(xe&zXB&h*4nx6pqPMj{wY;>x+ zco822>L)Q0(hNDSQsVBNB+-RRrAaK<9x6$;;le;kxN8!LSv>4isXE1g9Slpuh@6=x zr>w9MXt1T4;b3B;xTHd*1|h_iA$YL4c${5dUP5Q9yk5)1?-$YvdE87*%i4~J$6MCe$}le_ddAh~!0< zXSs2j%loSOs?AxV#j3h3Z)PzUg4J4$t0P{OVo!y%q0(Ky8-nO07_;RUY?B(RjE}~| zo+WkJbfI$V`JHFn{yX=mA` z?G993{@1yMh>D2=dg3S)li@g4e4U1wcf!Pp1|X=ln8!1uHkN8RUlFI=^i-py6XViv zIA3{Dywyio<))OXj~Y*mYnGFF3YN~bGhNkig&N`+?1Ia->CoIgz?Y%IYa+?6)oe1C z30b8N-Fgwbk=;;4+8B5MpFm*0QiDw_)0CkU@Y2cLZwm|y#PeyoN@q-&OvY|fYVUl1 zqO*q0XQR>m_Q!feK=EfX?tw1+8&zpA%Z76N^nI5_QlqCKMx$-D;chzRZ8CUXe0W}a z@(~zK=Ve7xI=fFn(={;CX{@JQ)wxS^Ou_0nQpv4y^S$nC6HUyglswcuRhw&AmfDl_ zrG+R;G;fmQtFjmT)gF29U*mm7y(45iRPf=IyhI1;bm>ldfWPl2YvH8JI#jy=N8qjm zXjBXaT1`T4SIJo!)6L@K_~{Ao^R^z|hZ;irNLR2J+WTSK{D8j=bEtAk$I_zN|b&k-hayYcHdRn zRpGe;kHm-~IUz+eWA1kZRGTVgg}HdLa=y-4du%M5pF6OAq~dseCh+O*m23+EJUEsH znoFxFi1^)wJ=pkw8kxk1J&=jbgL9p)O00roq{sH1;L5QHrA*mOWVza+XSM`2DHy3u zHZ47L&MaDxX4`x(VsPKL={B*|RO1*_30dM6qg9&ozNfmT>Sp3ruMFt@_86c^cabag zN$`BQ>*Zx8yLsA_YtoW!tm@^?GR^#Ks--+B<}#M57GF*_5+Mx(Xj>mpwkk#c052z{ zsYkjOS(sCBU$_xMK(bP-gHUjcJh||+X⪻;?=)kb*?YsEXU6LmByMLe$^QTqZQtzF{(@ATDv0orsdyDJ zTCAK7Yb5UvZA{==3F3aKCgx=c2*!DGi?E*TG=7ot;qg{L@q4cOR()d?Xdzj-6C{X1 zcSxIXr6OYqzrx{gy#7Z5l&Q9xa?HwkCY?MjFDEahlX7VQ#;C?UwoFZ?wc&4idO}6F zY2k{#xOhQR5Pf_|U$R~5mO5n=P0^o8GzLxKjbMxkxs{~3(OX!W!>^Z@8xAb$<@@v3 zJe2F#;d=clesH*rN0^b54B@Drik1RTOyR_3dM@&_X8VOB=^8^t#7BuXut;?= zEGQC9bd={Ml-`r3Co@oEF9j+Gn6b;$o*i*(I!<=cFIFVVe$jn-*#el7Tnq8Q5lm#{ z#Is!W(>$*Jk)O>UNsTr&Dbyh`b2tr{PZfvIT~S@9QIx8!0;;;^E1}-l<_R z!d?3jX%Gt|g(T8KA|msg*(~MR)jH6-qBaedA@oxcX~RSzP-BYAge7~Yx7?G&`hpT6 zai)Hus-xf8Z)nf1M3-=d=EyVjp`z+bapIO~O4I9`)COOvAeumB-#iR-L53E6R#9!9 z-n|XG#_7zPffpXEDZfb}>*498v|lcwyR`eXkSt75;wI93!;Hhq6Nq;A#jje}c&yeM zWF-A9KPNv6HTg93I{Nzh;8=oP;e%GgygQo!iw(5yHxs>f&P*SP-76*>PWRltYV@ee z-Zx}gT~`J*9Q62ItuJnOw4EWsV{Y2R;#SS_-z~>1tc%sJ3muRh)|fw2SC+H8V#TSu zt8fpEkyzF0=saPM=AU)YySc67Kx~D?~J(mQC3yrg+kD$G$21H40ag zUe{qyx+VuAO(lS}Q1JWa_f2}f1Nt&5IJD@~p3{xe=-q|U`lglXJ@~~CggRf0A z{3SV){4twuHMR?FL~%b?FB`oG{qKM!$Et0yNug^yM!gNaG?_Yt^36Hno~iq7klIir z4r;@k54>4vmO)_4FHCW~QvFpZmP>`v?Z%BCO-i$cDSp?_v$paBy3Z9cjF+T^@n6pJ z>tfq2(f*>O*b^)|E<77*U%Q3f8+PCwN<>RIO}>`cVq;B8$Q1R;Eh?(94NuczI!n># z!-MB#65I+0+|*+WKrisIvo%|ozF7*(_*g1b^b;y|MX1zT%6*rWx(yp+3pCV4h14ji zPd6_b?09~wH9XfHZaLPb_W%%zvtXqoX2uIZhv+{G4**i*&LPXMS<(uOf@$$MV#>);j6CVDyux=@V zG|D|cyDa0%?-=7rkmk9p=Z%y~s794fMKbi0%v1Za~AtBUz;@ zoriNB+0QJZHTtdwDbS+OG4Cm?3K*VkJo7JCx)qJ>XoG=ul&I7i!>8b0+dVumu+9_- zFDQ43Y~Fk>h3e4&RMMzPkp(u%3xwL!!uV&b*}I5sjd#N=5TCZAj>TfG-=jS$55m|U zKKlSv+m%!?R+>sl;>5V+q|dtUfLCbbsY21q*RzM3T`!x}tkPNX*7}fOGg)Va!qivs zTrLfeZUv|MM-M?LczF2Qmik)6JA^eKcakRDkm-klnV&4}vTefb_ueZP=^AS+C6{N> zT^K`JoljL3mob8mw1vLTH;T@!OgJdiCM}fUah6tUWr}=W-#qV`vda`@9#HN9jd387 zk%>j=ES|i*yj9`uX~j->WpJ3w&)sLCQl=PelakHku<(wScT|f~ZY(4F;B%giB0dqV zo}FrGc$Z$6s?|Mo^e;)TQkcwhT%GM8Q1AzbfR8UL1-8`{K+9Pt50h>E_IVLvu_WG> zGr;SFfGt?KL&3Q8J#viieb(R8=oxv+Dv8f&YuRI94~JjeZGV~D6r3$=XGVy+eulkT ziDu`PQ)!`fih#eNdBdqq6b|r*bxKwxf6?nt54jc!KVr$8FG( z8|Rgoj983WER}_xLOr*zsVOLko2)I0ysOOzB2zrb-)2^Z-bvPc>pyvoyIPwNB%W$g zw02+W4=R!F8wq+q$kcwpAA@$aWrb90i~j)PowmP9z8T>D7Al#eN<2`x#v6h>N&aLT z>ixxTQ#{mkwM-*)lJKf=+YZ^I5-Y4;yNEvgLahO)m+B@8e-=q$E|Z>oZYykU`f%YP z??l9<5-$_U@rmtMDE(B=Ow!jl?64gLSrt)eY650YTtOou`oyI5&Ms+46+&B}OtCKL zmsxM%t%595g4a&~koL7PwV^mvEEx z=btij?!406x?&*1DAshIi8p0Bc#|oI{x?h0OkqY&K9P8$ zJYKB5KV8MzMx93*CHLC&YOGcA^>>Ar423E3%ELo>Ny>7x&n{_mumyT!P~@^AC!>tV z+ie!@AZ0NZ8TzzHno=%SPE^On-&HA?%5w{jP4MFutwRhjEOT#);J%Bt(GSxxs1wa_ z=#_(QUH3whX$e%zqjUGW2Hv7n%5#QM6VvOu+tmR9Ot7{^s_U%jbxc+?Lere(d!3L; zB$j+5URDZ~DX5ZMAxNlyj9wDxnbXK+cv{+gxm}^3#XZ?QOB3s!Jin zyp~QCkk;MPPO2(&&AP3&Nj@5M&V9F45R$?nPhJ8)R(-Y#1})WcIn>KIZ_3PKD<(-i z+)jO#QzC)9!j?&YrXTj$PL(pQK9%4m()96MqBY@mCF@Y~YFCxudN@n^jtA2uFp^qE4wqFVw;`{A#(iCsgD0Ee$DNT>_S9Y$omB}Z1S_uq zSxwh;{{S#ZYBlItxZ>Wv9J=d!9~WZc?+A;&s=0vsrTz9+YldE`%_PSp9ae3#ORbqw zH>GFca-A`cDa9_wR-bsYoZ8c+<7!MvhAdffeEH`->kbMvRV4fBZxT-noSu13iX!M~ zRYZTN6P9ljWzz@Mbw!h+KTa<6r`Xkos`tqQnnfl$V#_D5r%rc@!CM2C1lYgb1KF0Y z%Tl8iOH9fi`A;?FW2&)(&s84Y9Ihikt6_0ZC~3a%HI7qx5!@Xl@Cs8ZNLL+oo9(qs z$PKL4sx(#O?j zedJXvmSPHeNqu($?bK&I-BMTyP8Sn;sZ$FKXq06bT5z1tmFA(HI;p1LH7|?S%<4Q~ zNx3vh$1h8|fI8ugMrk9@o=VQQG|7vVs#@H^vgRlsG6e6eJ7Ojz*bV` zLdu%Nu(Eki3_AT+Zm1w1R!fS{KDS$nu^D<(MAlxczUg*H+>)Y5Ddtc1j&cQ@M-H|E z#vS9AFeaxz3&QR8*LF;%F-EJUs9oDUf|`t3cVg0*>91R2pwM&$F%V}`o@rr?J2bc( z??MAco^YO%mkn(l-Fj}B3nl7mQd0?M64O^Cl@McuY08LbS7f|AZEB}gi^5nX!+7wg z4sQ)@2Zdc=ytY3CqtjJxK`sGbQ#-Jfdkm$Ip z5t3x#&X*c~C+7Y(-7f}=ukQF*x zfGKFA@s(XFhxM0l!Q6rBt{s~S?l?nU#b*4)UUS&K<$3tj{*uo)>3KSiRpd`A8!Sq0 zG$tv!n7oc|!CJ7=Hi~kN*;8~#N9j+~TT-8n&K1P=swe1A>a@Gi7Ta;y?xjdnaX#CN z*ymoa{73!a`t7sa(Wr@uOtl$36PK!29k!^0a(<+jd8>rAPmRgd9*(RkxSv%v5r%bo z5d1HSq@^u5SiJMJ3vmTn6yZ46hj-=D;)UKn&9wO!_hQ_nNi{!-S_qNV1JfEc+Y{2> z6upgo{{U5xR;9G>?Kt*Whtwp$>67-jvVwJs5WG_g zGtH@5r^Afj6;{%1x8~-=##F1MozT#ki`SN1BP^YVbyS9l@pwxv4KrLk;_@#)VzKW_ z^j=(}{MTX}qgj5Mgeeb%QiYr0W(wyXr@|x4veBfSNpqm_=-zr;BBWCKtf6iQ%9+Nb zS-iBwo?T@tO0M3R)09hs!b)GYLibyQDxftGq?7jiRNDK7sZ+z5Hn7Tv8NRkac6bF< zSSi0y=EWAgAAif5L(-3eCOQo=phMtR8-!n05Jb?a;HHpl^Hi9Ebb>CAX} zOSjs@31_d;Ej`;5qmMo|2d77FR2S*+9PymztJe(nc3zz=zwHD)Q#os^#hQrPYYz8S^n73yd$rjyeayWl*;R;ZH$GG3^&cq zl@{nAr~qxHDq6#yx<|TnnTwLCLXkOTTTHz{s%tqzlt*jjX;jX*bZI=FSM2L<6VJ-k zBDE02%4J~?fPkbvDZyIEfhl8QZ_AA zVzA*-2)r$SFz27L$5w#=!mPh@<~5bLe_J(>B_>3DP0;_bkAGCi8e_*=jySf zW2aZV0V;&253ALuw?2`R=N9|hwrQ&G!j@G1S!Knv_~$z1Yt?JM1wZ36%Ggsg^v-ON zUD1&L09qks`g-us)uqsFuvN;ofWK}RETL#mEVXa4%&b$2v9ZnfV;<7}7F9jHYDxm2 z67aNgCE{76k$hq;8j7&8%yntL6FMh`dfcXhv|g^=eC=w@h*Z#IvWWG0*wT8*2|6X&hx7yhRyl}bJQdCjvreW%Zrgk!5vA^LU76U|IzF_oQ-e(;Y1 zN;Gu^Xxqy-?AG(RlmMJ8^vj>O-DYf*H(`fPvF53);m$EnPE*DC`BDllWUHj80s9^!jkU?c2Z_tuy}sZ%AK#f~Hk~l!6&gES$sUn!O6GQ~T#plYSqn+VZX=Id)m+gw zmV~AA#Nzk1GFxOneCc3dsCi8)iN4H?v1pln&awN7qsC!qS$VST5P=Y(<<)001iOBvaj?g8gZ9sCea_XbQ@=7O zjT~#jI$XN>O#c9@{g3{uMeRGXYZT8&``yi>6t_5OcX)eetKl7OdXLiXcs^4_Py-`2 z`a)XA6z!4ucGjf+n)1Y7_giVFnh#Ajx#AyntSWIXx9m@8wx(v4H5ij=wo6?6P5v8f zXlyW~scqhCN?_lFb9r;Q?WVNyr&rUbgkzqNzU$2qb=^k~SN)~@Z`9NPZQub&SwcdT zc|yEwJp;HV^Zd(R%2e)q8hNtX?gtJUH5&mePXT&nGIEx2W%V-#WA=i5^f_Fdl996Jr*tOlC&vwpB{1~TzpdyT-2 zR)__P0gOzrB!%bch$)?JX;vl=)j2eu9OFFZ;!LC~E@eso0CvGzbnC+zrd?&P{x^!D zxMmcEh9aFw!%JQXC{}TCHZvwWy}Q@P?-s9Ux|BC1QNHq+Xt=e*{-51rskX0Do&u>& z&T|DG;G1C0&nv8ME)3G!ZmEB*O|A(}Yl4;&dvv!eMmjXJ=gvaOMb4H=Zex{#JAJKM zw2^nBDY}rQyyhoPX}G(4Dxm#U0F#_8ku;-=Onwu3*)+Dmhg4<9tNX30jzfl%$6KN< zp3b16V^)*$XP#?Sf<3m_1!k9nFXntd+jI@2`rg3dF58BHrZwsfR*IWguq@&l&86wj zTct;EId`<)PbhM__!kN1ceOTxs>}MbK5M6y>65S(4Yg`Nm}nI!z}M=OT>7=HvWqd}OHC#NT^NbC}OPH1u8vYgtuhHEAaK;{O1)J@#TI+s^bE zwR2+3E5d>Y*B4K*rQkM|>e3eL52#aFmsXQ$Oy;xCOKoORWM@3II~mZq6Q)iVD8vs*1^>YJsj?64IzR;ZgE@5hE)ESi@1XOq@16}HtX zf=uaTNzboe^0&uW*M;>4NmN;{)F%%onqPI1H(+NGL}xi{w0cTImRv4^jehqZc|>wa zBFCpc4sW*fh=!dZd=Rc56y55Z&1+$U@4XB3)+&ONMEI6dl3&8;_SJaRrpJ#*&A_)F z+q(Y%>F55ie|hk;6a6=rxmYJol}dO;9U@F)_j_E5y&$Ssqr1qE@zp7Rw@lZ<_8O0s zi6)g?+x54Te}r|(XJlA~Qmhp*(omv3ZzlO6vQ!$xK(W&qC{5yW;r{?C;pKHtGFQ5% zh0~3D#U96H3a;3Rdh2mmeKYk(zV(HG7q1c=Ps0uS?3G(!x;4jJ92a(K7Mc4j2vl&W z?-H2MH2&JUrAdNFYU0vnoNAVj6HnQACF<5}pjcIa!jV^^E!M;0l!A08l(OS>N{Xhi ziOP7j7>xeYyV{Z}DW~}TGJGeLamvqC zl6ZdVZ_A z{{SZLg%hnZ<0^mscj;|i^ZgsTslC#FrBCb+!r4`+3Q5!i)~S+D*xl!qB6~fq zDEz$~pZ-|=R%%qK(i}9vK}^7kUj-JuUGCi*f}Y3qMc3!)!onpzWA43cFYPUt;oG^i zk8((5CGWNA*Qj`<+|?zM;o8sA?k2NH*OivBJTE(ssPnzy46e=rB6(Rq)QeMY_3XDl z)SIk|WPn`XpId@G?K*5n3#m+Ax=L3&RFhZcgD;p<{{V|m;x}=fvv{vzxdl&dlFR=9 z?P*plB3+hU*)l?>ovB&kk-WMr4LV2HFKvt}6+D-f;5{Hqt#f4A|PBEpFjW3N$C2#7lv-j8uq|P(S zpS-GNdV$e#O^@0B_wcymct)YNqtM$~{^3^Go=SL%_Y<@O^|6W>6;WcWy6JUNr9aG5OAc7&@m`XFp1&Jm?e?Ne^s>v-jJ`VF*57B7?q)ObpVxXq zwM<53=^VJn*LR3onae!qTOZ6RQ$vUm46()PZM2tHW&qihI7o3xJsb<0~I)Y{Ml4wV+=uUIDada`qLN9}h46rhvH z-f@xuqZKq?q6Mv6)X@`D6#cuDHLb>vwCUeWdNY)(tXfM0hYvNSOuk@1k_wobhiRQQMV7g zmboU9UkkNbnKqNtKd$5&2!E$fO;8Gc<#F43qe`)g1vts#daR}+Oq%6%7;4jBxlN9y z>YNl5wu;^wjMgTWYqG}oE{MmgS1{Z|;`7m1N7CJQRKirHJabBUdHGptZz*NzCE1^` zwC>KKszjT_u$)e7)y4`}mAO)^=a!4zm9|JlS7i(-lT)5gkGWZ9u^2@f@THffwc&;J z+55snu1(AH=XM1yX2CMDk7O)?K+=7$d8ncSfP>C4`}6%bl(@D^RCT6*Fo2Qes2LC~Zc4EvmH)Ck!{@ z&!&Fc2j(XW2GWq@$c4q!8l&7d4{p}8H3k+@uADgDV{$HCB(WyckgT4Q>T6FcIaDTG zVXmDW(>FF!q8o}dh>Q^|Or=x@lU$SG8Cfa!#Y|7SdFQHGb4m2G8~r+wt`^eg(&K4Y zDnc?Wr#YPmDE|NpA=2;O6g1$tNeUe~;#@2POLCoKSQEzE9AM`r-cs^O%+GDLT<|%V z%U|}zq*W1(LC#^z$udbq#JHTNEVr|L_dUcpX%)^aA_ zXT7C)mt_X1R-!b}l`@tg>1{{(g)J=TD$Y)uofNs#?D4g?j%ly7m$}~d$i_hBdL?kt zLgU4Bu*FJsh((do6^%mmwx2mb{nw>T@PA%M6l9>Z>!x~Zm4=~`c(qCA$|vJDx@U|V zopbHv`q>*)kJF02k0=gv!eRR_H7GHnZB=hmc!ED`;)}}4a|z5oI(59&f24V@2Todc z&BfBtWGz-tD9B$wI}@RahC1(7;Rcb5{-Vz>edlk8Tod%+n3VWwiE)j5?-j8OntC>} z)-#m#%aznLG)dDI>bIQuR!Vuf={&ezd!UwxsyWD@kf!~Qh4m;P%bVlZrRk(3=6;-P ztW>8?op73mg^S6=&JkM21<~F2bdr*(RSti-DN)Z3otKKfx|Ec;!c95y)02Gfra%*M zs%~`E>a2{@rA}cLPW1`=UsarHQfOW= z)!13nrANAcq_M_DE_04N`Ciacg(6II^x{4%Ega2?y-H=>rb({xOf{r+yqN_$LK2s# zx#EvzwiQTr=1}SSaF!S&Qzp5T_Dbt-P^Gq1{2AhUnMN31oXR-maWtxhBYc#{r!_G8 z<@emGsZ41B8)GGufh71hFP9vy2GtFc@#gO| z_f%NOw;o0Dh_UXggWG3>^nt}ZSG%3&ReQVZkME7@{mbrGoNe)U&rYW5@UdP+-md^p zaw94nt6OtjRTPrZ&Uce*6Z5c@^!UXKS8RUYv;VD+;H3s8y8kvEmvi7Jg?yi$7hTJvY6oZPFXYM%>6uZekERhmt{R<2bIU5Gs{ zK4Wc@utpzs?6WJ#**QsE5klS5fAwknM&}+~m*-oJQeD9 z2;-C~^OZh#n&k>}X;U`W2GI*yb5U}DHt|PonLMo$-qOXPdo_I_w|aY-2~_GsaGYhq z_B_?=NbUA?l)OH#x!R-Npu{pD-xM!BB~QZ7x>f1`3ZYa~=OX5F$|IGQpLi42MoB>Tpt&O_B{Hl->QN{T9kY@J7m2}CK4aP)mFrLlKR z!K_Q471rd5i;UKAmu-jg`)MpLqs#XapDiy`XwXVX4a6vA9}82MISeS}7nVpCsd?XS zcGMjrfQGOka@!WjWu(ciCxwZHo`I>9GN!Po1RwKO0`3Efd~^Mu`y%FcyZ@z z)}gpRS9XUa+r&J)=bsCauTOK{NKvTUq|@}2W=o86Q`Rn(4c1?!S4F73eB#K!4&}pC zhfN(oPeq=&EcA=1+%xXRkf(NDAE!NY{ug#U`=?dY7G?sCVu*8@!dWY^+r*D}vDfrlU($ zIwZLAcV?&!Nnj?higHcn$iJFq75XpveF0c-3`*3SZ_XX$wi{GUx**>iSekh}+Hb4Y z+cbkgw|HN4rw=ptdbMuEV@wb@5|J^@vQ5IxtmiBA{0Lb*~>PbR-${4(cys8rSuK+Ym-^q1`uiOJurL7&qryf5kZTV|ePY;{Zp zfRtm;2$X2_g?p(>a5z{|GAu<43yi>)E}ngF(P1cg$8WOx1ZUHo-H$djw{;CRHFAlR z`Y4y>ZPeS>?gUDipR5@{3-qO$j87Bjv$>_5YXxK zdSydThe^`RBrb@}AU$4mhI?KapF5Xp>%NBpXxgj=;&yjnFB82(U>3Gv;bMB(1?g>}s8MF2`@25- zRH~$|=v60+DATuFQG1V368dRi8f;YF+nWB%E7AF0K))7oCv*0IWY} z>##C;#rax^_q3n=P=57(kj;#{fiaqbxf=N1=Lu73;sNQ4<6tXe4hlohKaH$zUAtJ= zyx*PHwKC$>;pCm_n|_Y5d8@Ovd~2+(%SUMiQ*)pC)8`i9`cIrMy<8KhL0fQO)^((# zh>IeL>&GiHl-?LpY+Huuw7j|4X>FA~x8`a$i@l_UWZ*F+APf@ZnX&enf0GQ=eEm|D!u`T_Y~ z)M`4cO}3#)LQBYS>(+F-{{WS2$&nQ&6O+VBeT_}k(lV`PvnJf(BfOs~~EONU&qHR^X%4y3F?BumXH&z-hu=JvZS zFbJHSu`P*Z&+_v5Yn{QFe-*Our!rfE#HQr{}Xp+SfI&!z6sp7t#+jAF4 zaM@N(W)aqMpZ4mlLHS1IDeo!zUXo1|xz;#vg?@`)W!7TzB^+~!%jvDnnnI7tYOhgL z6yF2{*R{3qR$uvG#&sHeeK~eqyXky|w<=rZcXZM$zFA1}I+Y_QB9iqd&p(BY{G)6~ ziydU;JevE(+{O}RDNi`%<$R)e63%X=C}enOThnxE!h_SwHBx0%DAR$0eww=0)z z_9^|mb}k^-08?u>g(m#$n(Oy`VZVLr6fIl1w9sO*%}19HXFJ{- z;A(h2JfCitw&$K|Gjnxx9DNnkJJV0jexFRQu)Z2t(>d~s=DRiWPqy`>^w&G-Yl`jD zUwxHX{W)I<{{S4d_Fqga=i_~pWv64F+YaHyeb-w3I9|Qo7cSf75z<{7Lr}kh0{FxUAKg%FpUM>LQo^8LHN#Zt;cR$&@V9l z_}f)wk-6cOtx`L+_UV85M)Nod?iE5*gj^OX0y=oTUwzJeue$|Onij^LN}?0?+t;Q` z-*Jkc{-Kh2Q^Gwl6M5+nmu{EnHdH&RMLsHqRUW)vTIuPDT<+~;GRRY&TsV6#MTa#n zUfnEahBJvuZl@?@raEgE8(c2F?pJ7ISmEr~hraA(Mwg`ZYTzCSbLmr@X5Yf?hpE0C zy4@Q{Nt4qr;}^W)a%vlxYrlyASIarC}TWh zlWG%uovc4H4Z2i&x@>$*@Fr8qJYRLf(XMoAfwCdMf2)j{V;|oAIhGZPih0iqXNJse z!&3cvamnzqXrdQ9_Wt3#K z`f$1o^~tZKt0YeZ{wvDh%1Lm~%O~40ducV-v3>W9ABFZd?>Vl1Tz}imeI16owWr?H zc}4RPTE{&j-+eKzePE?~n)Y6#NvTeGrSQ!^rj{6RxG%rqcZ4|(oH*yl_}(FDpSv!) z{nxgi2R?5vgxF^aNvW6R@oO)2-d{x@<9zbq^z7lsJg(D4mT9+=DW9j6g=F%}Kc|)U z7M$hu>#u9d%EjfDB(&iBCKc0MzMYOT_Sj>>pQqWH_QPY#ik zb4%f$WM1)f{{RkGoJtl^o_v0u%LG7}@0eb14O)T!&unn4_`IT$mIJypIW|*{{Un1{{S1O zI>OiKt?xI(E^%Q{L9ncr8>0SxvcAcnY1@Yw; zUKfW4*>m8bx-D*(j`3LVlJdA2MwN!KG*NjEQ|+^oUFh~*gFV|+#k{5Ic--Nguqev4 zJu-~lxlv;rKZicshNyVA6ozH@p+41h5|#Mc?-AiyCd6F=GF~JX-$)EuSkCzTXE^4-%yUj8)z5mNAl(Ev0+uKQeU!* zRZmAOT7=V1QznyKowTE>N!6oE_hTnnOC3gHdO=0{Hs#tIDmbd0XT$_Krtvw6(h`X4 zZoDfj^^46|#YdA-$$M{A8W3tUQRjO?f?Ln-x-_0A(K)kA!>3EMvx^=ky6Nw|xt9@P z#2BZOb_p)&Ifu#r0545!wH}$p;1kAU!TPY%FDE01U3FPl z_gjTRkyX(OTJ>pg^Se8VdcWX3&h&O5TTQY=W)0LVQ-zGCEioU_zgDIC*j z=gYF%+Pdv-Dw3Z0lNRwz5_&fD^>cQ!+2+aS+imve%Kek4j)MMSjq>_=TcurtLsL>- zm&q1Sg4at^2HgfIekjQgE4|A6&J;S*1U}6<-Cb9#`8`!qiBdxy&+cwh?+Bk{FFTZJ z3^?g4g1e_4^o`+H$xaBMM(ez4jKpUXxsTX1hO)>5N1^-9ES3 ze9+FMOWE+cj;-{U^!QkEWIE^N7evM~m&N#g1k_#3Im0pL{scgkLtVge${u8a9ykT$Fk_tETylfy29`x zDQK5#&&#)qceID_XT{>>+iHN|oZn;D+wjh8-p_+yPRk3y91`|FjqFXi!dPFk_+As` ze8yi6zfAr%8^lH8N%s90w(;za9h|sdX1Vs~@y_~E`g8Z#R44cG`{we&lxOtqy@_Io zU;MtUI^Lo3e-&Ybnpw2+T1D#)Tz#+ZyL@4_Jh>~ZI_K=L8WA53U7C4bmmWU#=cVxi z<9w9oub0E8oLF`!mzG_h9IoRM#jJ7b?7moeMsF=)dH(7IUFI9_HaO!|FMcF8^;eth^?UN~^!JyuUn#qoD(kK*=TaSykKih5;rlj$8Iz?fDk ze$9TK+op$~$N1kN^SodvaQ;7fZo)Q_^;)>>X^UlAvFOZRqM zjWH2XuRG03OwD;&u%}U#=-hiQfl_hpwvt`!{WP@0hn>2-Y1|tn2HpAB{{X%Z_>}%C z{S$KU%u;x_LH__0f4zFg>o?+Wa!`%^HMqUiSKRB75U*pm_Z2|tn-?peJnj~8^ILxi^k6&lVkgTgxLt81Ee95pu^fS?&7@jj1_lGe+$eg6P; zO}WCUr^C{OTrR=N@~=%ncR(rY68zq}Udz-&(>oH%=$i#Elf1TJ<}af|?Ym+yLfV}d zEV%jDMtiQf+vB3r@jk5V2#`T8JTq=Cvb#c-aOZ{HI9R8O%5u)an?O!@M+JCpp(zn9 zXOi=>r1C730vz+6w`|(VVOUaZgAN(aVdW0t{&Ji7Zs1EO?$qT!)ExV6Tw9mMTz?gD zpF0=%3eA5exvZ)U>mK7dbwsB?cRPAyE5S@eyAtZHf5;coKD+aKZRyb+PIwT^thl*Z z+aL2y&;EAf{{Yxu_g}H&Nk5d+OAmGF)-e{m=M>oLZ7J=sRAYyW&T)J4&!k*`{L^qw zJKe;6vij~nb-g-|H&Nve6K(6~X1%XlPyCF>zuq+oH%}?a`2H8@_164n$>?IHMAG$4 zJ~YTD&nZ@}0nNMv!J~@3)C6BW7Kqag8YOk=vl%n-K?=7jy zUawbGt%ngj!FylDS@QPS+J`CYlk4YTNkytteJ*7?9whjshu!=yOUR*d!b{?;Hp+T@ zENov6H>v5{ePL+0H4o{=?=Erl_N&jo;;d|?ELn`qsg(Xc>jKJ8U3mR7y)pj)8^XfY zP5nMMv0A0<;QYRc*kz017nz)+57U?U-rJ9-(cemT8NBARoW7jzO>%dPQuA5!RvSwf z738&DpMGnt^90s^PP}um!dd&&PqTyeU85(gpIsmL`OSs*ar>KGoUY8DzWKg>p4Qj4 zemBj(@w}{EEH@3Q*S1$@@VZYwchWwqsN$O!(&@IpZZ4}Q@n0Ux3t~FHx&HtcpT6vI z@MxE}>dUh7iO)VU#e1~#y<<(rp6`Ush2nZO9Qi(+(!65x`=$Hr+4O6z=K#F(UQMrs zg(ud1>+iZDMzfdYImgqj>5Jh0=l*4L>``@xlGC<2 zbiNbflkIZ7zMI;r)GUXm4`Wd52kL3Up4-YKmU(%o_}L0e_KJKP_{*P-^JkQiW#s+& zy9~?d*H6CbCe*At@R**ttL!7Im(M)C(Kba7HSLWT$KPRCaQ*E(4tlOPbKB-WRIA2m z!qI7*>&44Y-+hdi?VinFlt;0<2x?WN-d4=b$L+jbmmzwwhhIhM1?>t~kh2#zY*JQN zt&HpkdApm#yNFcFH)@xP&RV;rPZOROprf|a^pBGGSSqM8Urj9Q!n~EnVw;6Gie}3f zW%W-BzS|7XW!8-jmMV`sI#vqWuPeyvZd&uVKXSAwgje)z(zE}|bJ(m}_1Qjaf<puI9-mlgqwIoYUV|ttwDe{!+_uGSs^wXxeY6MXo}D&X1SBi`|eleZ}N;7IYDC2sil5xem)10g~H^i}BMcRIuu;49K*NL^4-*%ddUb#AS>JYeU znKjX+t%4_BqpCqo;X?I<<*igcPb*L6CAqD)AXPGKWnmo?PtRLpXeC%tJlT4i=G)Ip zaQ2LW>z~t=^Zh+?{{Rj8dt+WAhpIl;#x4HoOhtmd2eqpSIBs;wL-JW%wGaw`FBq(r zQnJ6Hj#DYUzJ6HQQSOplE*`V>AKHH#^z~_gkBDDtxQ~}@-tTsvA4r^&jIx&+K32;q zjNM#uxm`Us$@Zvrl)8d1^EvzOQnuM4<(kipvA3kDh^|U6t2kHka?0hF{{Y+T%p}8K z=&dTwsI>ASdG-EolgovWJzkbdVpQn*PY254sn%n=0bUZtizS>V7br<>76{c zu_HXxJsRuu<9B&=JH`CV_j6Z_PUr zFPP~3^3TYV^ROePI;PXQ&-W?oV@65vYe`%`+TY*VXA(!0$J6Ppg<<7+c-IFW+twDI zMAyD=Q238%{p(+4&@69__Af5m-W2$!@%npiu*Vo*Po}m;V>#mZebT*GUZvOR&1oG| zyNC4C;g@1pQkHn1XZYm1U50VSsm^(?;q_Ss&m3K6KG*bDjAC5hwLfep?Y!Y83D&kOY@()$|7r4ntJo?$xYJ`n0U_eSLWgnJWu(d|oH#b$ih38`k-q z;^17hc2UT07-hT3O3LU7sW$o973ME$;L{&W-%-7nhVFva=-q;+sJkrYvUeX^t{j%X z3k+7RfW;2Ec3r*0qB1@#(w-8vw4P?2+fCbsFKnfXFVuXqVcdSbUlSil{uKWJbN*<{ zqIRdsK3+AOtx-sW^sWT?zKh=OS8-nvFdTFLzG)a5YeV5y(1k&OS`)#UJmfvkjC4{OAs9I=40J7m8 zsC-+v*{zyY$ON-SFB1MtO~?6ix#()*zE%Y%RFq0WyGs#$*D&ujwMJGm#3{a(OBF+b zY1|p22!k~-o6Y^#vh>HKQl1O%vQU15>ld1>dKVV#DV9#@XH9v#-brb8lnhvBNVNW9 zOg8l)c$Nukfsz-+aCY3wiJOl3zT>&4PLT}vveTG`1TG>oo3I<@;3G(W%DGPzD}~uq z^bZLizU2h($DeOkj{gA3P2RvOCz6yd6c*B zzfAs~G?$A^uO;Q5#_*sE3UatRAtY%_j!kfPrDq;_EZ(=Zig2O262$O7O-1ZA%~_MM zT#;(f$1m}+Qfe@`>CX1$yzEcXSy)pBoZiHrA9a&k!xZvaKaGcCSEU$OX?wmM!@0?KfgC;tF)%i5KJ9vAe7_p8f?s!z`+?V0!XUJKK$ z=M(2(eH0A+lFfS}f3w?t50dO{{9ATBQT1^C9g%kGO3i=rUf&Y-m4{99`h7CGfqotu z=P7+%v0-h_wgtmKN%v&#h{!&Z?N>>@KXdn9yz^bIJ?XBR9?2M2X>X=yI@vVk;UP6bZ?8o z@2F)qynydF7yz*A@VH=XrMqu&GIPy*ZFLA--lt5g2N_(R+hq4u(D(!QK1-)tLvAdI zM?3WUWPy)A;e7JjrZQ~K;-_>>LI)(FXp}6K#*%*9C1aY8Jm&KByvVt7wVDyQFPRop z7ejvgo~FH1etYIC)%Je;H`^91;X*02Je|38 z)`mO6rcS7{{Zq-z2P@m&gvyLRip=^UrlaoYAf>lI*9Cu5VMtql6bO7bcw$|jn8e*e4eS*Re`*N1{DcJ;-5a-@2xmW zn#+=~m+85Ai`S*?WKny0rk$EBOS}}*^@EXrU!CkKI+?^0@h__QKvtPuRHqFR)mSvM z8BqE3yf72tcS@ozkBYstlduXBqvG`sJ}aZLG@fej68``j!dNuZm4+el_WgInKZjmi z9k8LTC0-gyN|w0C?XU-?AmT$`P zSdQ6f95Bi89vzoXc}0L#CUJ@9&dQbr472jPnY_&FVYw5?*J{bG+sRUEFl^T@E3@ys z;d*%xrMg#4!T_T-nTVqV{c=ekQr z9Y2n$!#KHV&1SutuG{EOhidw-PrKoXKR>^ZYO(rr5M@Y^KX=(P>TWkCn{=*sAz@Y=fe_nBq}zD;Hum!WdRN(7i2VQcc+itah|0ekW!{ zrHI$5-j$$a*QUaY3U+#r2wY}4mv=)o%4PGku$4kWIkIUb&9{~HN|?{xey`=fF8sAV z_OL}#%_7tV>6ZrebEcVGyd)z-ZZsikP8)xBlDDS=u#_TQFF5 z?6q{L-;%SUfj*+{Dun$l+)vC!cVu6QkHuVoXBQjuR{{&3FLN8#EZz=$yrO}wb(g_d z>JvK6DVD@kbZYNlU!+|sC-nAPJ0MjzWPh|)UucuO{ax@}7}0yMUxJcf{<2f;xet}l zMnzSv76A7$e+52w4Z1iw>~$cfmz!~Zvt>om+tU}W9Q@vQtI|%)Dq`B3*&&w~n7p## zd2AR=t+9Ak9u_bjH*LePs);MZYE_o3HY=rslsgM;>v|HK#9veKlH{y3${qOM{7p<< zeZ562hzJ_t0_8tHTfbnni}C*e_<&O0kNgn+`QGrltYJ&qfjZGY)Oq+!j`K8Bk|EV+ZW2|89W?~JVfU*HE5I}Nv# z-D0w~H5M_;@eeO60Xq%ihxcBQCF&mDy&}WqeKBBgl5#6PA@yFoUKy3By;o(dSiD1% z=f?IGG4a7uo6xK*obYPLj}4HUbBia%SYm7mj&D;MBX0OiiWG`9Krc-s$aQ(bP|MjD|Pm#BO$_1_m4@304Q zyrT7nHOs2w=B$KRtW$;OU`$;U{w0%LtM9xOSNor`7r|Y5U1H$U?fa|>6{jZbjRA4B zFpt^kcxQpdf!LDjuMRj_2B%xH5k8K=as*tR@@BGY66<8b!&&oJuSqKarVYY*@_Ao2 zL#^nBc{?1C{{S-N>@+M#_cpkzAk?SAJ=Q3uTKXTm+oj)g=j?V67uVOjjzEZ%{5*g?c;6xj+xae8~!zqPs5@hQIWS%kR4awRcZ(RelvXNAG=p z7QITdA~rf-)gSj>7l^N;6*=-vqOG~5P}U?QM3d2y(pMbs)^4gpYx!;u{{Ro_AMd$+ z-;BJkj(^F#P={hp8twR5U4iR$XrkuJ!TpzE4J^+`_Fa*hTs*EX=2@lCu2Po|D~$P; zowum9ow?x>PmDiv`|e|D`Z<5O`Q)wyG=4+(-0=9%x8kl=VLDrX%x_&bq*A2M?5S}2 zHj zMdMeK{-Rf><5x%hW#wVqZmGw5S#Z66=Xj_6La?vh+kS8V01!}QbgIEWOBcx`S$Q_e z>#h1<-2Cn@=YRWe^8Wz&0`-q<<%8d~D^@X2iFfoP=R=!v3dkQTZim+RA-XU;hBE+WB{h zU#OwtzCKSTSarkE~*U2|qZ5u8*GHd?;#;@uB z0J7otz88G3^Jj9CkNkGe!GxE;{d~Z&tcvG;OI)f6(RdQ43>WSsWK5p0(#uYF6T0y~ zJ8R^h`@{bLhF=wa?~f1Ht7C|CuqwDcCBoZUTxEY7ytHvuZ^AC>YabzA`??+*CkqN& zd`r}A`CoTb={SckNpoyd(!;p%1(HU5B_o$US8fXUw7ia%P}+xg>g;#NdmR;k(rMmm z45v0v-F#YDbvnJ`!h9CIX83~+mbE_}ymkOZ7qnT}p0{gT1|AYtF)R)>T%GJ}55-`7 zNq93{LFw4<0xn}r{hco9>tudA47x%?$)lt@jJ|4UD#!(W6|-mI}v!7(~X6~Yn|r{9}DbxkF$mC IX?J1&*=5Q$qW}N^ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000004.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/doom/000004.jpg new file mode 100644 index 0000000000000000000000000000000000000000..03314994b6bbe9a7cfcb9e3eb72c6e9ca992820f GIT binary patch literal 18463 zcmYhi1yEaE)HNKm1q#KXxVyU*Ee?fHf(LhZD@B59aVYK*2=4CgZpGb-6k9x|#l+K0fQy5dn}eH+2k^QMkO91X`xfmj8pi(& z%y*doZ(w0z;o#uFXG}~iENpBX99%qH_y!Ih9zH%kF)&APWFML_$D7 z0Q}#9gp2|}L_mE5-~4a<-+cf80TKQlHUI$$0U7lT5;_VZD*U4W1h@elTxxD)poxnV zPf%?>N-rKwf(Gv_zO-wB=@sn`q;4+piyHx-%r0GDVZB+>H7fmlKhFQyhJP~v0U2%t zJ`ivaak;6bkW6TJfYJ#r$at?k0Gzi7)`)>f2s8jh8~_py;B^h~4t_Tx4ty;ExC((B z9>X-*ZZx!Y+Y%MSqSQwa-)bfiH*Of%sM?v~5g>LXi6itoQp&wKpUIOxz z(mAkd%oR=Taz?Qb3pdnY3|f=o;|XWX5Uo;74{8G$=_9?xnY@GSsUO>pG#R7l(tbT= z=SYFwHa&hCKE%4>7QZ}ps+`gk^M*kqVRgFdOWxC{^DQ1o-+T%ttk;{-A3+k-H=?p* zBKj+rCJJ*XiEWH6c$Ft2`29!MWW!)JF0xR3v2D#8aFq0kE_e~i7{hz06V)(|onqP5 z+K0~b5fR@Ld~dL9D(kqexsJw3_p4gzAXoqU$2E_?GSh2T=jdj1QsdbI(|HK8_X!lD zVNEZC)TuYTD9hIpDhRS^?66cok9)#QO(l{Nrl6}qMl7-|n5N3Hh^#+wU~az$CrEiz z$il|w^ex`RB~B0;(Dy{xtSV0OLBY_#*728g{hzosU!ZGWYAbcioV)`I2eGyKO&B^wYSrc6H?Ra?AP4U@#+9!a{~ra0GQhp&N^=N5<(F*Q?@~ ziHI$q;$GRKdYJ$977h84$E-A3vu$sDHbW+CYZ}O9)%%@T z9b53q^7;ony+XXM+D#;X&{b9iD(#}Wc3u6$+xX#k?yy+CP*@?WPL6qI?Iaa~7bzzt zl#FixjETUo^$+7YtQX9cP6p~a`E7nfSC8jrrA;&rsb}j;Gfd_Yf~iTB3Sk~|WV9e7 z+C}XaT3m`Q5r9x+O>tTA3vHoLbSM;xMnwFXZg&rDFhi)86X9U42|hpllFX>cFWP2_ zpmAAfK*pm*sP(7`ii9!&*nojV)=kiK!U-Rew{r<}*#{ppwn@K6Njd&!OJD+sy&A<% zC{F+`*o9E3vQ3)lt2+o}{;HPzqS07$kkK1$5tCH3=Ag#6UZ<;4F=a}!S{y%Ti$Z>=zY95uS#9?0#;=Uz!8iQc)|{2|i9nS#|n!51t#o zU(bn&mJ~~U`Yr&oZ+9gm%`{fo?lHv6(79TRI9IuWK9qQaqpwuxCf$~eFf)P3gfYb* z0TEB@_IfTtEtf<_)zWqhY>NCJOMdgKY6zJ|5eH(LlnbSaD$dy!-1J{tJO;c1GPOJ^ z`_&}EF0xOVr^Ou*ixtLB!;xd-9Wa^7Y*=>pim^mHwJr?F*Tc35Ev81vgHGn;XnZwBNL&`PQmJZF(9;b5a z6hvX=Zi$u8m-%SR_*ra?QN7?TpUNP4iO|Kdl9E^tCWcF z?^cDrp5;*HyZS<^Xc-*PsLoQ~DJD5{ z233N7X3$TcDk7i{|M~93&_pFW47?-n{Bxf;MWXmm)(_&%;PhKQ%x{DC1d&_@h55W% zC9cnmwg?V&*`ZDP)7yaa16fu11-|~&^_|pA^c#>`14j&HqIv>7JCM3}7V&8| zhNayn0J9#(43X}DKqxz=`a9fTyUdWo*b^&~6gdz`laxaTq)C9h8`HEh#A2<32n;db zjRO#)XdYX{)_UuKKtbWu1ub}x0@_aZqZ!3cp_Dgc(0@7&Bz5EQW(^65k~XeJnBl7dEiXHycF0l18(|f=OWbBBbxcSApd>1 zJ^{=Tfi{hRQ{stpe~2M--?C2xU73}-p0)e_x2}a|JdAYhU-k~b?o(VB`O^#Q`)$Vg ze02fkU82~&U>jvTlZ9DLopp0u!A*_U%pCuxL@bL*w>KAFh^5409my15{;8O$R+XfD zq?}r?ZurifE0|%Vja5=ECCW3CCh`j4XW|xQ@`>QC`N~nPB>D=#Wqs)CRQB|Efu6_)R@_A4wT7#VWAYde!i3X61|A^Raww>jj^1j3l~#rc zHI(^7k;)^roZ658L;9XcJK~m+8Q%KB8^mowH$g53nXtca#6v6B2Lr`tj#;zfGT!OW z*E-((eMHcUnH~_JnkDh4uH`U9oIuEou-7#`i!B88H2&vh)e7LQF$xOJ zV=|q9B+C+0kG3iyr#m7b^JR}~m3u}f2UgI{zHO;utWiHrX&~IiNuD5aWVOT8*9&xY zPOfw4n^(}m!DI7Oi{kQK&0wcu4UXA_n?_V&0rz@ z8h7U9!C1ICbmV7RTcDuC+w~2N=u+_?AP~qJ{z3tP)CFV>e&%EFD|0M-05>I=yV!wbz)0eL)4eJ; z|EC3j^-Zh%2u7?@D$>}muHPgmI{hV`vO38^Z@=@SiFT;1d7*3az_k+Wg4W81BqR@bEz9MK=1-#B*~MZpZFNEQIdV#v*nV3({>8e)gPMsC|ZxQFCFRrSjG_@K0zX^*nA>g9qA~vRy>lF?bHx}h+sY8!i5yCE{H+pE7@DdxGrQ!O@}1#!Z&D2{a$6)_>mUmG z-lsA78^ujh!HM`V%lC=v*ocX3Cpxxi;+3c=)Eh8oiO+p5@}jBR0jX9W*@oCM(~3QT zO5GPc`=G8y$>1Z`U;4f?)d|!5OrQ01U4vO+x~wn-0d2%$b!;2B^99Yclu2$|jDH6J z@r}b74wy&`c7dnZm5Zh^>_D8#4u(J^*JWxh48H-=G%?MI&GUZ`ArSQmH4ZPjJ`F6| z`VNN0|J<^4VLASV7BZ5LMkr7>oNpvH*tP$9zutrx1k#dc3tdB5)K*t$>2qO(lN+{K zb*qQZ2Xi>_ZWJdQZPHRCP$&$&=tJLDfe5~@MV&~n#XER43k~OX1~|Hfhh%riT$BTU%Lbf(^``|U204s z3Wf4L7!-&Kcd37V!0H|~8c?X2wN)aq5n_unK*zE*LT>DNLpU?D?Zo0AGqyV z;s`!xZ~rq zxXn4Hr0WIH$vMX=;I@SdlCmsi3syDH=Rz~3Q?JTAw2dHx?&uQi9xyym$^xXa)yCvB zW)8aDwrEiQF=~BjX06Fl`P@cR)Gjq@hNaA55CBK8|dL05|Cu zG-Y=Ko|aNxB)qTnncNI3frRrnlr#S^drG9Fu(l#VJQKq7+ABn%V!*LW&H` za-|)BU*00&+vc5OTSD6vvSvuE0uW4F=rPu+45kgJ)H25z;8C2S zQ%0R5a~sv`B9y2FQatwnf%HLg#8idv5T|9}ai^#MN=A;IqlX|D<%pmK6)H5I{gEk{ zgD=G|52!g`U?)wUB3Y(wpKPJd{e!^G&Zhy3BrD+pPfoZk)aJOz+|BUt8;gf9;}3-W z*Z~|PP@F&VSK)h^3r9?(lsD2G;MC$D!|#*D>FFJ3ED|E*u*q=AmDH9JQZ-#R;e{4iJ%RC-}DLCEGL;@h+i<*oCO& z^W8S&N?m^nvl8Jti%k>HO5YD;P^9>rFx&c($Jt6uU&|4bi2P{s6Ej1@n4zLLbU4{< zr2fF-P5QQ~YX4`98S}hg_M_s0w}lbNo6HBZbE$h@P8m2xe(Wy9a`O}%85|jMUEgas|HGSclK%q-sNs&^Ol;Czz%-Sbs;~Q5i7>hU_$_(sYI+Ir{yo6>7twemo#jomuaTL(qCrRm<*`iRM8uj1Gb9Vgz!z$lFL*6{ zTT$Vd1JM-9y3+qC%dPjwB{b;;zrpi=O2XY_hFQ!& zD?R}&P-HyC60o5 zx`Nq3G?Q&ng@Fg^|BZBP+nnwp#w9;Zj^eR9y9f%5A@DT*UU!*VxNpF6^O^TSu%ryr zJIX6tlWb#H>#dYg1B}ONO`Hdsd*yZ{V8Kb?49Ev4&VsWBhjU68s9Aj%ew`WMXY3M5b zjiY3(dhedgICAjASkDcHC&@|If~Vrw@X6EsV_X^G#xL%YoxrMm*Fn4VDOXEXrutBn zeT{3+w`yO#Q3a_+SZw*cDr>@3!OB82^$KUWFLBA*l>B_f2d@CPJNFPT!^3QWHbfI! z%KD#wE=7hfT$2)&L<&tq&(BJHR!?NEzpz=mcrHTY4aT!gU_aGuMjeTtmyh++-@kJp z$Iv<#N&mT7GmjKfRPvEIyk$E%r-4+fv(6>486l7Aki%xB9#7c@d^-8Hyd2ps_9WtG zls*sJ2;lzf*ap=sQhi-eqij%Y6=@fSM$MQkvxl9gNUA$iO|CVV>5;amY?rWIN&AET zoxGx+N3Z*I3 z6M>5dh-vq9_cY$!6lVsNn%qv#2!hUABPA8S9To>gnRMsS@eCO|rOz<<1Y886asQuC zuMPr%5N=+6cRZpe0BgPu{d-`hXDV~Oq5!jc=p^1JJPg965iO;Q(EJd|>M$)kSwYlA zX=o#h50W53og@JrT~4?xugoT!|BS(P;oLV|rHWk=_tsea3Dl(FnI%d(R^52O z!kg1pU2}VTp zdqvt6w(XPcu04`L&7nQUQZ;9GvQIBFMxRUqD$>BsQS( zK}XW7Qy>mFm%#Rd{P!Qm=xE~8ukMQX<|$Z?wXXmTOBokR`xNVurX$U@cK-t^19C>H z5Vh?Fr?U1$KCU!!1QN{tDFHT1{XeRYGI=Fi#5PdyVL{|adi%zy{Ni4A_BmIy!>{$V z^qiFRG=zx^A~!5Ht_w8hf`vE-qJ%d+I-+0WaniKPaDOqU(U;`K!r9?MRQN>yo&vcZ zfO$!4T$V^VY&Yz$Ua#hp!7D(Q3R%k-*VOK7S==%%ViV)3F9%n1W*7HexRfHuMP6># z*}}Hz*T-X~%3ffiy+O#HrZz-%IX@LAb{wCOgTgM+5geoLrcXss-BXc@Xfc5=U56`G zM8hcy0)|Dg!{xY8YMNwOsWdM>Qiuo{+*lD0<+zk_dRE@(wO~RKEL&iF0h{whiB3>TNQO2oo!L zN`=L;;yY+c5M*eJS>~Dic$eun;kI9MD8~H|)g*a@B)6%I03bokcyDGw=+^)j+bm!u zVu|c7bTiMVz6FRXDdMAV9LBbh=$`fPUIr_g8Wn`rQ!bEkVm(sP%L3nwg>oRGLOj+S z##C10esXpgJz5uIj4ZVYjsEn?FLnJQa7B!I8HKEy`^V)nzqCMKKmuiLTs}_=)8aDh zV?^c>PydT;51jkv;NqwQ?7H;?3z?USDbcaORk zTjwhv!cZ_sP1dTGV6{ub#~uGnwOy~6Hc`GVwpqNbjjU+TgyY+rY34b(L$7pKwSLZw zWZ*9w*l|r)s6~C9p~i=dnJ;T-rEQgrrk~BU-WP8qFb~a*k|ZYbrD?k^5Q`?#)FNS) ziAMvQ`|)8;&@;p;K8+c+`DlCfvX1e{$VCwm>91U8T0TXaNTHZJI<$K`a19sj>6g4W zW{W{uS|B{&54ml#^(shj5>QOw7m=~AG%lAg){j-8EDp|s473T#PgHyRr(3AussoMG z|1RjsEHUiV8whW~-iXOx{DUNrS!kgRNYd@*1<*h`?7FCD?0DS1@dB*D4qiz-4*WJ#4!p)s`?4UAO+9*^=80yaI?o0$(fK;$X0oMrjDr+1H~vSB zUZ8@ouc;5W!amYTLwCB9fa6?qlB5aLqyM-{+xOrH zTM6DC^7IKG{D?34CSy6Ae0}V@Ph*yI|5fJ6Zgg^`V}&`{LSnse24y`OF{;>FT`JPcJhTB*5ex-H-mY`4iTkOl1| zRM%|(acpQ+|7~Y|Bo~I()w8kV;OXtm;phx&G_XuCY;YPRXc79n1zCvDq3*Q0!_9XsR6>N#*qP{?Ab3#G zJ8ZzBICv=(!cinU^c=NTndAwUDj|AI<{2>1H%~}olz7^PH{tKm8O-_qr)@w@+lbk1aYDggg=0y?e8;B37V#D zy7Fyc4w}4dW=AOBllw0`C?PFvjvQP$K%65gyCBZ?QP>{tLblkw)51jfJ`&Sm{q&^1R7LHziFb5OYF=S1V}J>tOEKD*uH( zlfOKLr7izOoBMB%JyKT%C2Q{bBkT|J9*8#cLFNPtQV4|uQa|f7#u+J(W7cjSi=_=} zn@&Le(=HaDcGmR+QvXOlvRv3=uF4&2Sc>&4ActrV`xP*P-;?Lotr3CmWZf(qC~=8c zM4ycDnMhqUn;6~HuG0CPWz$_tS>-MJ&Ux-y6YqQMnkb|lqp~$>Hj1N9H@9ERgP^}4 zHQXr@p_By)35CCEnCvT#XE2#HQ0N@zT*x;@ZNjBDmRWk47JZlO+w-wBGmVRVl8nL#l%L zO-kDS6%cNjE7Edmu>O&9*XJ|Jr~Aq<>YW&*%m9{$lEO(Hlf+5;42_oCl1_A1UeS6r zfu0S=ITXTHrNr^EzqIkW8S9{97@u|Z)bwGbh{?H!v8y(QQakf;o>=p7; z_Q8SOZ%P5&5i7|B3s&N-WCh(_(YBxjk}fgW@>7BA z<4L$8xpz1;G&+Dz8fKUA!)#3|A4dfPH4i=OKxF~_R1>dfvYQ5_m;M+`XdIi{Q7IQPg>xC(xxNPVyv=ENgyozv? z#l9fZmNpjyFix5=B<37j9aQ4-zUv_6e`cu^ID9qOwjFzC3X`LSNNe2LJYjInIekY5 z!@`)o&Fy3n1R`j;mY1|-I>UDXfq>|P6t=ihQO0E7!V#8m%*}5=1TZ|~YkNWSkT&l;)8K))#$x>$CEXXDh*{t%`IX@~s0W zE$pv3_xzJr=W`<-ibAtu6W*3$Z-?TQybc7)?4tvTt7Z9gi3XHSqTl&(i5#azy){6% z+LOrdMiEP*@b&U;lEc5`mrZrT-$ucYnU%VU?-|mZALF1i(nIi3Q;Z)MFx`3!Aia?I zliJ9AK(YQQ8(&qyk$qwDB-~`rX>>I&1-zH+b@XvR9mZB2qRcO`iU9+*$hD z54XMBkAC=8P*a9%0!2Vmk zRBpZTA;VFdNPzDmFP2e1@uHH&R~pMYwSkzO3&*M=wRpwi(&g=XUq{y%f6ei4d4hBd z(S05nOQ+RJ6Im^w=Azr|ge)G&kbsoA;c~O${kW13XB7IFE#u;()8=*tTtCdFy!ht* z3W!AeA-tP9zbeUU#QXWwkskcV#BZeT{=8aODplEq@QduzTeazw2_?NlxrZp-o2X6B z?nRdKRL9(tqgTLgz*VULeZ)G`xd&79jo9)X!z*Afpx0&9e!z4-KLtj*F^q63OQag- z3e;y8WK-pgSkp*b#^-7|_`Y=hXPn%!LQW~4rTM1ypXj@LQf2;bC37olA@Sz@!XJ>i z!Gt+qk@LlReZHf=9B3!EzHon51D12(vZ|gudrm{%vZ}t31YwqR0q@~&(dVXoi|Zcc zsBw`^zU568qtw5V6@CT$3($WdAR$sBH>&*L)T=HA);x+r8>D!jJ?b>idtroh;o)zM zte%iBp5TSErWw63Y$jlCGPC^+J@v2E%HvjIxFfB#Dx=DHc${Z0@0q`{1P46!+$aj0 zn2}Q-Szbixo;}AuJAiGR8=(4-Hf8B#X^^Is2Rylj(=>HZKjd^rG9mGqi2YPYsEQ+k zHvoIJmvleZsp3{@{Hb?Rk=apjg7HSJdGT)*S!a;#qL07Lg0S~{jud`B6Mi_+o7Ajy z0mXJ7#)aaS>^|5(vLEZ7of*{c*qv(kEic4ChvaSuQkqaS?vr$L5Sn)rLF_{ptTq;! zR%1<+veh4utJs$QK1#$?#{@$8BbR)Nep6Jid5D?ly|BIl?Col?4FGe#xEF7(ZMwAm zR~85OBb}%h5XJV1EnntO#`r2Q?Snf?+yF5h+tx{W%u+mcNvY!tO9^p1pEXMcC*~&; zRNwk&Y$aKoDeplqU4tM|9JNkY!d~o>r~GXdOF{ZTgs#H5F+Ot~AfycW^DE%~Dk%_4 z0ywB-S=`LO_FgH>2ELc@jmUqPS#bnc)D)X&R9b9+M(GvMk5W46NnA`%(CN6GQXXA$ z^EXnt`fIkHnLo*xziOgl5p7Mz2zyz5r4jJ&>Cw1SOD2 z-uf6wUNX)d_OEtE^SEVE@;)nnPDI35&+d`%uVrT;$#IZPOV`fw9ON)H^N!a0dYo37 zJg94z<7v^1>pm?C2QrYWzZKN5iMKP!K44tP60vAM2j-2{WH5`|yP!y+PVeoQQ_XkH zja0c0XqQmALt$iW6Km5X3#pkcnR==^miBk!whVf(cLI;nmlUr=WA7$-H&yEzd^Ln9ArjZ`;{|KZBL5koffN834BOsi3u+;vI5gXJL)B;=I4mwGl&h%OKU- zuphyd#Y^TZAo-aHHgSd7IY`$ucyN?W7E9nphFT*{@*C&+d)NNjG}vVK8d_~0qM2>8 zeWO&ZdgCooOnLJiBIaY?N^vUoE%lkOF_$ROK#0@g#-RJI@I~txTg~b~u{wOoI-`qV zJB~Ga15SfR0_4v|pKbr|%I-^j?;h1dkRqoPrO7@lpoKr&PYSKs6H6~gdqN;JCk#d8 zlwY{^0Q+EDtW6tEzK~Mx4__<;;7-1(#{!}&8rXKkC$v{Y=*~8^d*FSb`3m^M{V(9j z>`%%o;LR(b`(^7{xoVgbA2BofXZ>+=zOE>)lrnDZngimNl3uljMXPpgexp+|Lbn%v zPNiKsZrKacU32wzb>IbU^>%%22lbB!vlB1;jO0vKzSz2>=3v;S~xXA#uv-&?t zW`wvPp|28r2;vkZWb+H;8o@VkBo;H?h;C<00i1Qwg41$xPSy6kdd+&%1DY9clI`L_ zAdueY+9GT@!8>YF(-n^arL!ub#`Zp!2HYS|{(WvGA)kH9r}r18!L_qf4yv5wH07(` zcP6m@{CrZ9%|m4E99_-AFuJ3B!q4|cF&;n0i3|Lm*&3cEjiveuxVir6Y@t;j*GRmt zj?#iO4UL#3Ww)NUD}TGe49!-CA?uLje6vnkBPt^=$BV-+t^8DumgSIJkPjDQwMnbn z!cXIi>G@q)K{Ci4*lnt2B;Tc^%aLcbGRx|Skw2jQl$wbqAS1u4QWiKpSCQ)TTx7CX z=5AN&m+duqwj!Ys3}LEK=EoSn&t5H;P`kjG51;LAb22n)3|Jg}3^3FBV^#_s`xu9n zPaIVt{47=9QS2J{!g%sa?%M#H|FT-|dSAVLejL6?iUu9sEj^t$xx%NW&rCdjGc^0s z{EC;83%6W_@fF)q${^gQQi%eStgW*A0viOV;W2K#T^RxxrEBQ$9y-JJ6nD)~ls_TtxUkmDvacc9bbm(LVES7;!`RuCUG9 zO0D#qFXj$Pf7TMLc1SfW+>hooB-km2m!4CEORT-L2j!v7dt?oNCTgB!(hfDxeSE+e z6r}1p%jh^7Z%JznRli}-za(+~gA;6HotIoXr|7a{<7B02+Lp|Az0CZXwoU2=hBOS1 zh3Fb-HPy>$j^wBoi3hdip_m8?x(~ z?zO6kDAg6V2HP$+$nVfIB1s9p+Q8jggN>v4qLAE6Q*stBU+TjA*nD%q{&(7*D80;$ zQIr7HP3RN9Yrzk1{vlszBiA!`UADC_k5Srp7sAZr-Hf!K0QYjuV(a-bFPb_@BnPLm zyV5}y7~<^w6$X$?(VU#)Ql0Jz*E3kMFWX&hg4tDV|MGe-9Y?;UDU@~PkX*H^FFQ5g7%$1ipIgH9hJC&eK)YRIxXf~(o7P@Bekv||S0lUkw^j8F` z29%#W7Id0_pjJa}f0oSNq$PC?pJUAH)@f&6nEi>Pv?)D>x#?VfMAgw)<_Ikpnx-ES zGVpY#R;4FDPFvS_CNr*}<>;1D7Wi7DDtUF7T|xT^XtF}nveEJi@GO)1QnXH?fl;&(g@YEv6SxwFY{GP+8Er6^qlR?$ilq1E8>p%rKUy)95aM}bKkh-oPVR9 z(X@5->5xNEGBPYk(vO*?*`xe6I*?^x_N!GWg#e8J#k5jWZaXYu13whrVWIcQ_n3|( zbWp4JQ{DHRSHPhJen0P*W;F8Y$$8E6PqOm?>=R$DX1h`9bCa8Lzzq^4bf7OoBxVO# zGerKtayFaD``4q?yBdSVyMHckL0wjq6ZyLaP-GnPY1hjYBG3o$v>Ozhi)!)#I+bAe!M zgss077aGHX+3?@dJAZhrEXuQmEM&;?vxWBHz%FB#)#U^M(_>m(%;mVE`=ZrSd2|H= zHk`AXmH4yq!@J`aXf?OZvE4E&Ywl1&AP4*7N`$$EypMHF;-BEE5eUg13!L$Qh!9${ zh9^5H_dMbP5BQBKZQgLq?%;JTvW_s-jC|A1zF@t5PqO+XeTUFohBc2rA$I0;j(&R) zH?M0&B9SJIe>8!(X}Ikf7i!^PC_sO|@Rnv3msLG&iDegM-eHMtH2M*dJ#p|4Gt za0_+|n<@FQ=8r<=SiG$ehgVJIx8Hy4zm@xt23?%fvu0pB>~T3YC<0tPvo11M9< zDDeM4^Ixa+f7O<^HpC>`q@Bwt6K8>AtyC|-*pJ+A0@pNJwW7@|fn+c-etPdZF9#ymP5@%XE~3aS=qC?L$TPN&|G zar^n8*xY=L4-565d#gfLFt09T85gbWJ;^8iq2EN};F%yhY#X57w|khq3QUH+Yu2VE za2fsVYAl}NhA1a?8lu(l)ho-dh`H{x!wvxyJNLVQWq_8+{Gb#a;}xL9FXbS=`8!L! z{L`lHLYW9|4-x*pDxt?xG_9jELj;b39)_1zS@lg@8DF`kC{8sD|Gh2ckpI$4Ps!d5 zmWQENqrtnWlGK-;YPS27u@MCiFCfKYGl)Hjz`iZ;SoS+fd9IeP0N1C8yF4fr``u1F%GIx0;TU##?~a2nGoLYq=gmGDQ^Xan*&MUT6~l-vnmD z!~1BvywYblU);_BdbG>xE2Ec?oE^ODfP~+qTC?(AA!jT{oyre?U&HXH{g_v<_@&1^ zwE-OAJS_EQJ2d@UI=PBpcPk@T2GwXp-5mG>%!%e&FZQVkU)NyyfZ{ql9eK1rl^Q&guO_{O`MUZD-E+z2zWJMA>;L zEEZG^Ht$YQr0e7qaei8)D!}Z_H9AkhcZK{-;-x4ovB=5%Puq=@VMih>b{Tf+vhrxk z^4|L`%c80yRJ*|Pm;^MLD=JZrbk-SEDU2jFYm-}J zM54}gBnbKC?8=6}@y--(=L9V^GVIVO1$r%74i(#KuNlu_?+Uu<#RIM9dy&m-Eabe} zt3nTM4;OwNz=O{`DVJ!Pf}s)j;g2;;5yY)`cXzrBi*;x3$YpINYpg@#6Za~c^SSwd zu~Jq{=hJoxx=C(Ef4IRZeiywMGE{M`D7ld_o=d$JWLkC=%}#Zq z1q#Dm4G!n)Q9A5$5@_3)?NA#I?+wU3HfCmP@MEWGj=bAb@~a;nl_YQ z*3x1V4)ddW9rTGEGPazF5v5ou_vr7*We;6jj|Rr@sQ3yM1h3|94V+X0QYPL%)Kfmk zz0jZgXgHp$&zkq0@i!`e9!6i91OK(ITgurcQN1LDf`7+yD+VM_i2Neu3ebI_C(oSf zk{NgaD<=H-khXs^daVk%qV3faV7@n)=Kl`a?SYWJ^%8&iU>eJ zZ6YnQQ)d5Ij?ic+YwX%&E+PRV-aPEa2 z?>f!}8~?!=bc?lh^%l}|{*r$SPQDs)-qkcM+MT!zZOr;)fsGRa3PRWK4P!Cv>nxHi za9C(a_u_;5=9f@3>wlW2%)<*Iw%u zFFwkYo8=Ky>VOyag4?$bg(vN`RJvBm29e* zv+6hItX(vGvMQ`x=+v{wm0tl|1qTp=#?S09q-M7n&;mDIvriPQs;=rHG|F!&Q^t4P z4J-PcU@)Y`R?DxSZoI?ztZIRbC_8?3;QTXMNFh4;iDE6Q!9cS{XEv{qIt#Of@~H3a z&uqE0v--_a2O@*|W!7WHf@-SJZdKeZXFtyk}cWK5=sBE%V8c12Om;?a!3FvX{*=*%?0*q2a_zz#avex)4L*_ax71fgy7(CGe*-yd8A7F$dL= zT_DpT2Qpy-UWHX|EKR}Af1|~8=3Au$Pq>9@DE<44F&Vfw5ARy#kB_Jea< zedcVJks<)GDd^sR(?6H|Zq<{Le5O8Iz$W(Fm=7)NbNHBW_9@F>FPpC1pZO4*S(FRm&vw2dI?s(#*WJpxBu?rs$)!a1VdiTz|=VX1OW2K&&UBa~_uh2E9 zkT9o0={H_RvU}iae%E7_a8qG^2d?9OvWaTX?(r)CsB4?gD=gCS$LkBWrrV01eU^54 zhJ>&)DP%!dh&MjdH12(#cX7TX$LhBoBL+W@{q%JdRBh0J-51@L$xQk~aqBbp;CId< zSpP`SAwIX+6{Ui@^;ic`g#GLf!_rq|>;mavy9DL^&`&8(d}jvPjb8M&pYGh`MbS+9 zaXK7iBMVX+2vCv;9iKp_UlK&eqzB_NudGA|kTeK@dTB15Rv99yxSwcSA7)XNA=V@M z_3E|64LCgobs>6vbZ8}}HB%CrzKq-R>Xu=vZTrUBik*_NknyM$B6j70wqC7U735em zKI&0578y%HK(dI!5HiGH#c$3v%j~CGJ)tSk$Ro zob3!PX@$)DFRXmpC^ed|_}Dmnd+5*9Drb2xV6Va9@rM4$EzxXLC{yjkmDY$dZg(vY zb%oHS-mAci6=*tQ{*GS#ShKG=!&SyLU>SHYuJ$H0r2TJ)k_h{u@XXN zs3k!lkUfZ;k+dl%g{Clf=%Tb3)mHANaaTf72ZW?PV&@-sw^)-99-ZcEfdE4(YP(R! z`9saIkn%rl@VCF+b~GXRBoPp`%K3U2l%w}uf0>t z^}D_OS2C9A6p{IFCF!d`|5L;!gg868xqfD&Mb{X7TrrzT4+GJ~n7N;&N!Y@_;z}5U zP5K^5_)>RTc8X}FGVFHEtfdNdPEQ&Bg)4a$=wuCVlCjog0P;45EjH&)!}dIpks|I` zt#cZbZCd8{Zd|%2Hs@d^W?aGoW%gYe{vU}u|5>Tv+Q!>$pK{B1>EbV1=6!OGGB0H9W;?Z7b zS@%g+?bTcOo8X1-zv(k1&@YY;W5(l0ehoXt3Na|00E)c=&f90`_1#0r z(aNlk$<^%dP*Nh(-QMMh;C$o6B0%o^9M?ZE``_Tx0+Od)- zerlh73@>{9BG7R?_M=wo*(fIkWt-xha9o8deu;L2sW)cQ$gSkZG`F8q7q-|L`NAQa zZAU}=mwLfehHliEX`Dcf!=DEXJN43;juuHxd-7(|t%v-T5|OT-3BVEH0Q7Ogm8hLb zayC58o7iZbKrWvoWUApI@u!51Kx+*W>^N+OT~d5vj)q9`zKoebFnYflzwCX!G1ua@ zkUn(-UHf|_yFo?SzK$&iQsI%2m-v5E`Y{Xla<)T1gH)S92RE)hq&3WCTcCrNtbe`| zxdFJ7zVt=Oj?5*SEh~4*H6!2#2)S(9xMY2-9k2Lq$ED43x$ZtP6I=i3!Pej5%*SPn zvh~!PN~(_sJ?nql2lQBcYZDz`<-DWH(?9f_2zS2;6iCXTAAs*0+rpsK2> z4?#ew1fp17vas4aHxH1=F>X5BOnEOY;(yH08V?=EZOvZk%?=beU)CtRo^K%NW|B?D zOH9_&O7+}~*x_}O@GzjK)hWio9(yTfpgd5<<903}wBl%MQ%yxju!lI4TZQ`TWjP1n zTe3&YexPnUw5zC&%b$@+Ab6#N<^uW=aZN(;AgJC+Pxh8jW^&QZ8-EnH@c{YFAiJ3q zG}UtR`8h;3+UgHKrCK}H_KBAoS7&>&C_h>rUmZVNevlV)9O$9i;u2pdm*q$d!EGye>rROoD}1KAj*xa@ zp>5hiw&=~Knh;gSypz;Tx83qkd3^D3q?r3(6*$9Kffs@e>R3ylMehgSa%sUU8KcMF za%S6f;z_8#~I}hXfOPs{`yNZEMlDs~;NHep2V$(7Ft#5zv5Tcg>>)zs&&29emQZyGO-m#ji6G z1HH9TIhfjad{a-7FfN(DSK^g(i8zDIE}Tqq?&>N^u`!lwtUcb)yG_e8il(6j(6G~W zJudEDt@z|LWZG`fq=S@WIGeO5)Qa4W779@r{3Qi2VfY~;zXT;<(|Cd{0uylv%29&Q zgpP1PRtl=Bsw%3gfNH9$z}$R67QjM}h$dB^KuZeA1SS?;g17>zs;Vl%TUa$xjw-6F zBSRIXkhHZ1s&uUfp{l4LewEu{0IaE@x`k>IwN|O4+nWInwOlH-WwoMTHW;Jt4B*{k-}LoPWC;R3=;B zx1T{%kKh|pflVNwI;qrRZ@&EX~4*>+07JrT zz5Pz6sEPSqFMkrz5Qfw+U~cS@4WaC0ab?|1jo7U#6-2FS0ju#aThzeQG= z-WN~3A4j~SFwAZMhaK1rDesiV#)mqj<4dSD3)FDIyM7C~juH3Vx!-QE*ZkgO`C40D z)OhL@Pea&q%}d-3!^~Yv{{TTv_z}_dg{g*tC*ca3fyJCaQHfd`{{YXK^MCUBnY6=4)kyYy zw!@=0S>YQ=B`KTZTkP+A{fpK3`RP|NxYGs8XpX?8laOZ`wFMLAA+3x*1SD7BgoCxB z*nX5XRG`%aAeEr9kX2PxQB_q`0jlj*cdno;d_gjnkBBBzf~>m*rI%o+3aYAtqf+D) mZOT#0AgmaXl~658F0Dy|z*>WxH|;*U7W?;gS%UTdvJ%~zPM`=++9B`K!D%@0wfS1FZceq zuYXLE0ptM~7#NrsnArboxH!1~ z-|+D82nh-QwYa!=cmxE5gdif&KL;TZ5iv0_IXM{_H8l+b1KZybKrR4)gbV}%0sm*n zNT@&*02=x~=l|IM`2heR5;6)Z0RV`Mgo1>QiVj3YM**S&fk*&CdSnznS&)@>T8n2m z0}+2I>hi*O00;Ts9KwHnX~5&pW1lVb0(Xcb&uCXI z^sduhF$KHLS|~u@=9EW#+f}PMkb7`b04LYvNBLKd3b~>F(a?j#P6!rymj03LT$2N0 zwgyYl@CV1peV`(eMibLl1MhNXFbcE6%9xArx^mr1o7Xlj&L3KZO~cAJ@5VrEH=72* zB$*uEl9`vXnqn@+OS7Y!JXF(W!?oj#Khqx$8nJVu!-xOSA46Yco_O;$k9C!y_P>FmRRWu=JU3U1go1Xe~3;ii}`2b^Dwf`#Gfyl>@oG9lS zon(q|s%5w#Jm-N#j! zao=0`S9s;+#!#(_oa>OX5Xb0cr7;VBWYbB~iZovk?HUTgbqIyMrKrFw2t-uotfP{& zS;IcGuiFfYqx>-S7qG3G)Mv-u-{+TxNdmH60C;EUm~Pjnd6ml!9%^p|EcfH87LYe> zs1OpAjzmCywOL(vqGr}N9%PoK`=y5E)8P=5L685CRZaI~htGTON=OcsN_8*c+w=O06=L?q6?+n;UCY*8EUOI1oed;9^^a z4bixd$7x31_~BK>>F@M~Xt6!NCt8SB_sgq4dt>fM-n4hIr9NBN(+)|6fM9AkOVVjH zbJ7!0(W~I`{mV5Q7XDs$RHB@>veDS)6Dl$Z?+C7@5{Iouw?;@VklNbWgDWJzBM-o& zdorU67g6AB2esAc1vu{wAILgXCfDRe^-uu?Q_H`{pA~Ng)6?I3I1B6(=6s#+$%e(q z0I$Q9Nuu^X!>1JgD=40j0fOIae^aj6DB`UcF9*n1if{aWipJW{mR$cr?ry!O}|~aKaUj6V-@+eV+Sa14@2Bl z?8KvLO=mS0loh%($UyOMw=J*?gI?;T%pa5eY#m1dW-Zh=zCoD0g)^yJ1n25o1}bvD z2*AIyNHnD^`S%Iq4NbW))uKL}F~gEd*{2Pf$vJ@Fbx*XO-290^j~I!$l>z$2n30k? z5|^N|sum%XxgbB0+ZHCsRXdvnp~M#I%0u=fV4nKypp_9fPVp!}7ARFwB7nGApuTS@I0gcV^w?c3RNfKjjQSv_j(=lmh}8 z24#T~Nt$`>ZgqU4NUs>1v9~j1!kk^OC+|p*L-ybJSoog9P0sDUL}o`%3LbRjOMw zhRq)70Yvs0_pISeQOKU>$+ZYf2yo;~f%;Tu@2p;sd@uRFF+p7U*0gOb$higYh=H2##}SVWAV7@Y+;9u{pvD?ET7fbQ_4OFo71$^I< z{`1UttPJfcG zUmZ5U>t#uviT(np7+q7Nqd5wdU4#`%^3R6dkqXQq2frLUacD1R-K-klQ+*S=dtUjz z`sN49fpErOK#I2o{{&SRi|NFC)%rA~*pjv$k%LbEdeK_i+}KOb^2$3z z`;_IS7J<&J#x?S2#n0ls){jMy#W(}TLPbgVb1@L(CYllj2e%;0(ZT04P50|oB7?i6+2q^Dw;i`9q!CuxqO_mR@IK{Xd*;HDMZamzpVVnb(tx z<4Q>~-hlVU{?S@*8M~Gi2|C>Z{h`{FfWs12i! zn)tbj`+vF-L=xUJJtzf=m~3p-%*9UhJ!oDNiGXRPa$qau(zFi=!+JM3uS`pIcYr` z8Ky#-zk98AJ21D%>yCMsT86aYfkM2`pY%p~+4%Nh2{-t3 zrvxoW#>Ec~=vCyXiU1-p%-_cg)?6mKZUi!rmHcGc^jof~p}A?x%Qv{_Hg85N=y3b& zRVwHX1TU><-M2I#S9h5ewmy>4XFYN4;a<2 zrtfnx2$}I(KWJQix6LLn$EKFYV@j`YxjfgE-LZZ)=azR*N`9TwK*g&W4-Te!JcVb0 z6d38>&}v${)g+`FdYr@Etr;y?`XqW)F@E|`#p`R;)O!id7R+`L_2U(J{MzX+X~EJ; zXBMXwKa-N1$xAZ})rb~Jjn_CE%kqu+>^@B_o&mC4OR?&&Wl)1Xw3Z#Lp`lN<;ebLu#(i7nq`D7!TzH`)| zKN2L*pe9i(=&~N|1drmAuiBLIz@DQ{?8zz6x5_xJ(w=@EeL3PagJ%sIE% zH89~P(%#(LRcnv2D|g6d4r9F+XsKQ?p-$`y6H05b@VO@=@^P1o9dlFEgV9ltMO#*g zGR{Z2TDP3jyl5IK=b97KbP?@7jj;#PX6vqEUj;UBNoh54SJ!Q7tg^HmwKcrqt3*=x z>DMtCxb8pQ=c3;yS?7t~EBsZnR6vm?=B;?1I z6&iUxI5dcW;M~Z4&4?9%*tQw$yRz^~oXNYhPc_CXTo4z+`~z#7-{-~hU#&2P*x{M! zuxO4EOV&sMbwRsN1M3rWD#88A2jrx+Y-=L#9lDsjCgJYLl`BOB4wjvkYR))yWlg}p z06yuv^e^Gy!PT^!T{xVP?=twx94ne7Q^Ig10Mf_+t*+H_xHF~n7e`|AvitN6{81H2 z(JXIUOs8_H!B6R+anr*Vt1PCx?Qql~72mCL38&_ky@`lYRKSFZ_QvSD%X>SXAcG!@ zT~E0^gMK|@S!EeKg|~ob-5{tbyoYyk6Ef0fLHwpUyc5R2Xhyx1=rW}2_|!iM&-ehr zd=ZuSN$1Q}mgOasrsc!w$fi#(gH^Q%8RyT@QZyAPF#B&Ue5q83X&ah48?b1*F0b)m z;v_!n#rty;DBg4HT9WxbViw_~C@G1HJ8frP@x2t~PLMnbl_FP~Qf#P4;7@Ca43(`K zxgdp~x19jb8$LFDCOJ!&Z~-E4(ctW6i}0Z$jJZ%OR?E`7*@`V7%Ws1LN*vN})|_hJ zt)ab9QZS@yGkXgLE z4`L4TIvA>#Vn1t0cP&8%|4@WzL@R$|vA(32YP35BO!`F~Bs8+zCCKDyLnt5tdPr@9 zahoHEA@vsNl{z1E#jk1`y1CV3yKmLi3ugo((IC*(pj}*eRgMR8^z67c?2V@z?KVKHhlD~-^n18&(NU(t zK=vh@vx_u-*klLk@(U7)(563&iyZ^|*N2DWF7A5Ss{P_$;j3T7qp4i;^15>`Z6P$q z)#$O}_A|V;auG5TFU$>cod2K;Yt3&71{;1@O$?V@{m(8~np=#VnxY>; z8LjkVq*Q~~P`zLHNU<3tQ>R~BF{_d(aEwiwP`L#Oh=n4i{q8gDtZfg`s(O;gt0#w} zmlm4djNnqps*O#1zRnU~TrZ~L^P%K_Fe{V;r5)JxBK&U9?-u+av@lJC8bY~}U5*|W zb<0YBeW z$&)PBn#OP5ZTt1E^oM7aZGy0$x6e(7$BaO8(y%~PVv9s1Wt)pzi8B|o(}iGy#WBYZ zj)se_{9yTn{z_~_)2)F?o=nZ9Ut4G28K!dZh`Wc~rx0ms)1WJ`#8|=aX0~_SF+$c$ zC=l9T6mbI{^=sm+zsP$Wxyu0QGDi@sU;QF(rEEXeAjEucQV9G_-7a(YwLw_|v-FJMSkn&LLF#m4GW zK1F>#o(uBKsJV)33IT(VfLKNzU*Mbcled@g-1{>i+bKfa`OPk zThoMQql*|RKlI>Ez`U2BV4?K$94vBTi{zqwrch;(#X+lLRP3Bv6^%H0=X0OaB(jU_ zMhS)#*c+gXx4vtI#a=j*>l!x$_smTt`V$-<`CIQc$ZfkHTtEi$&7?J!9C})nVls#i z(N|GeSNhKyRpg5ObTf+%8ulirFnYvVE?vC_7QqZjxeum4>iW=Q<)0oWo`piGpzbBH z9p7J>OW}87_7?q>p*6t~m6J_L_~e=ita$*H%+lG6@GCr*RAYWhje};nrOmT?R`~YQ zd<#`nX^#$w-|jB0h(PM4SdJRq>d)L{L)!mvn#JP3%QlxM|3hR(JpLven{Icqmzz#_ zF&_!~7ohU-E#)yCwH=cK-_Yj`Yb7#J6Y6w4=K3S_!S=zd53+I+T3V+MR`d$o!1ADi zK?kT4bZBS4HKa-Xz%XJ2^{iY<(U0G&C9jXdjneDG=?EefLv7lry?Cw+;0ZN@B zKs)%oW9;qTcDJ>R0Dy4&_Rp6r2ukD`tcdVuz@o38PM6y?OyBd)7i5nUJr%t=0dS-- zr7;=Y?~JbBHV&WmP+JNlGFnJ#v+ZnH^HByPnwen|eV=DHD+rs&J|dv%wa*B>>QZSIvJj(=IJpEM zK?ntIV5n)BAWiAmHzbLLa(HPD%#Q&k^319N2e6`GAhEQh8DK#rcjr4s%2u}ww7XNW zVitR4G+BS(`!PA53MRYj;u==&+(2po#7Pq?v@ySft9?%yREP{o%G$(oXhO-jxHtsZ zOOd~2tjEIjJ2K}h0rS#CxNrAaBoJ%oGxbxIWeOyk-j~gdk5{opy4K~?)apqpU!vKx z&DzuSG_sBl4`Im~EO=47owY1}jzq|hWVUKOD(-08(LvQGx7#DGTG&z`#;a7JOFLQ| zQ8X0PHFMR(PM5mc9eHEwJ6{eB3a`GjIvurE_V1&4?ArNuJb08RYSA@8GP0Q|7&|xk ziuLN1Iw3Ia-(d#{(V#|rDS?s9b&IlM9QEA$1fu!F!MOg!S{c|HEO{+;W5c>kMUMg= z`}=vvJY5Z6J;2WB%HI5PtG_ZDB~7cudm`?kzhoD_+PaiYTvl|=?<15L?X)Ayu$O#2 zm+Jf_)Lh)pebZ3NFr!;m7eDOYq$O!PgtU!8w>=kfj#Y&8K}%P|?yx`F^i$2fE4pYk z@@yK?%e0kAeYIZ^gLhbcTuM0uZw~4~S>gP`XIWq47{W;`i^o$yUDc`Lvrn{T!k@9b)T?!`Sd;q{;V)mogWj|@S zT#Z^ADhi&?*F6{Hd~H z)KU2ZNIPLolYpoQ2?G|+Zy{2K9nMqTS^|*S_ct{G)maz^CtnKUx&lVn*ctj$L~5+L z}9=e7HVZK8-38u z--sDP!pXH*FuzuEI>XuCY%-Ol%PHv0kqkg)U)%+H%XAN_;kJyibitdf=A89C0o+EKe4x8u7siX+VM zvzM=tZ5<*iNIAxJJ>(@5b|r6LmauASFT6I_I*09dL_^-ln(i2;fTG1SvI3@oS|XP4 zUfa96+Z7U)NAe5Lg0kbiqj?rW=f40Q**{MBI-Whfz+U&_pFWEa9x{Qm2x>nP zbr;)aY~1b2dHnKtNJ~{26m#R3FgjN4z%V#4MCVVjX59Fhmrk`A&3<7^rV;`t&AeB1bbhUe7QD0*{ ze@KyCWnY`G6I~J^VP^Va3mI3vEtw99VA0%IyPHKg=5)_EyKacsygYjg>6b?Nz_S3C z-CmWIaQ^q0eA5FF7M%zYH6m@u2Tel6qMiO4f}tY*vZjibnxd0CT?Dbw!OuO*NWO)#QgM-v{Y} zy>kjGwV{(uy$oH*mmIFPo0iqg^X)(TXujtsZGfFi??$*R(r=I_sun@uue3&E>+ELYwxNqg(3+K-&aGu zTX@|P#^V6FsT`c-n7yy6?O-C5Hp^t_2VEI@3~S{A{8Rneii^r1X#*i&YNdBvt-mLU zJCQ?{K9drQMY*007s(}vRUtQ?=D9F>pj6ovAxAHB0Ts-D@}}h7G0u9ot5CMPUkY-1 zC)Qp6xnGtIsy@jMz^J`I;GPF0u77{{6$ZGZB4L)!94BibKhJRkgGx#?l`#?gjc#DI zkz!|@#ehf4&2HihBlH~dP*1XTPvK=Z%yjmQ6DVTqQ!NcmX(XY?W^WcUJnN4Pfs{Jy zM7D<`qVewNIV+Uc(_oT6LEM(Bd z1VL0g^C{OT4`A#g36AY`!ATN-0iD|Y71Syb!b~b!X#NyYz-}=PzaIv7IjQw;ZxGl? z{q@Sfec$6&F?vMKU|x8^N4JJHvGmV>8Jr?Ju=0AZtbdTV-FIP7QC}qfI4SrSAR~F% zoJ;M1wHqgx0>Z(+m#qC2`qIG)k|s3++_x|(9=L+79P~5~)dAb0SiO4vJPBNt9#Jc` z#_xj-SVMRGj@n!e2vVV0vVv=498E*3he<{?NdCeX0T}O33FOewW}YIE+8u z5{&wMb-oVDj5fZmXSdVVkYP46`KA7a-arw8k0~&+%D3@dW^u~pSs__!yuz_B@6u|AP+~v*?w_8zqUF~u zXsj98A|vXeJ~_e#gAi}Y(A0wOdz=$)FVh-AOAp$j$rj@V{N((8v)0w=!!DqNzsfTr z-H7n|`AK}ekGYi-`6|&vjV(428;&H{(wm(R2L|j0?khRbmVup97@ zVC~AZpEwm~&UC7%Og}ZnpTkSB6pVbXo8o+~-+Y9bhyfvoWaWKB)m}PyMojfLM{y~A zO4XC47>h7%q@3g2+&bKryT_8N3Zh0)xkyYc5EdZWKt4i24Fjba|D%}x&dBlGdPg&P z=tFp(ywJmWZ_GevbtHzWzVdFQAc_WHVul+)Dhn3ia-d_6#8o}y@AY9U@LXBgL*&XB zF57W1ks*7Lll-;&AGI{=P3yFAfVbsOZY`F_S1BAcKElLjv|O>Od<58Qhcpay?Zqs;{F0uKT+1$vR17Pbxls_Qfo>D3IC&A(~cOv1e9~L9Pd&Fa;SEB9W%*$h!4`l7;48!5tD#Ka3;ly zc(z#3qoLOu?JUwr&KHP7D~}0-!%f2U2oS~(B$KyAemB~2D&<~pV>1{U8V~SR8kAKi zlh;pe96vC~y3#pw7rmeSgiyd1!Ks)PN-9DLuBOzI)^t%RsU;B@RHAESO{iG!lEU>@ z#qzizl)X8J^v}L z`%cV%(zR!VX%)|24#hr$@637?SLdW**XtDp>>w8F;v z>Mxdp|GM5m`(qeq>_CWJyrVX@&veG)Q6agJvqgokiZ8MmCkk`Qsqr^I)jt$X;A*V* zwgz@mzD&8|DZGZ5A8Hge&Xer6=kbz{;;TI+}2jR<)Y+tV}z4O!d|z@Oc+Kj*4f72-RQ*w8ZxEu>=Kk~gk)qhy;sZ7+95XKY zOlm{M7KQ?Sn0TP<%h((iReX#25N47`!k}qKl_5PYZ9lua2SE8ni5G8mTyH8zqEpV_ z@;+LUn~S0?>7zG%trEdM=*_Ii#HXRyE@4XgMQhyKI8r zU*S&jPl0e(YeYjXV4A7x+!3q+_chCxrA#)ovn~o4P!M@UXlCXX ze{%Qvyg1a%?uB$ySC@pC@FFGS`_^Z{)72)wRj*Cg5|j#Hx2>pRU=b(2F)?>8BUm%Q zYJ?S%jC^MLp*4#vc#rSua8V_;N?IM$h)uGI>nzT%AO>u^YMnl|xUL{E8RYKtlTk#= zHU&t%mCzsobmqMvz$v`5bjLfIr(gL7gtychR7%$_0eOPBvUVgulAmornbMwiNU1!~ z1Na4vSH-6?cmQ3wT`E!vQV{Z}r=5Pf{fZam2)aBg2burm1PCeXx9sZWuW&j4s!ch9 z0;mdHOns`z_RKs1I7e(;*^`N*~-y{xe`%f5qYgoI0jPpEx@llR+33z^3DSLezN z!6mG)Ee^+J|8E_~0akwq(|YPOWHprdG8-A$zQ~en z(ex zM;erL8G1;LnLnqSNpZW+Sl*;$hgR-*=Up^yUa+2_{0u!KFw|B`sS`NV52ig(=pY!doWBO>No>P5@~x$w7ox@7k_Rr8_q5c`|mq0U=rp0=I^ z0`Gddgy(4{QSs}KU|BSW%9`+o3PWC3o%HO7F%kCrfL#7<HgwYd*MkjVM2?*(X(dH?Y*u~ym|79@WG!eM#SX$vU@&9%j) zMc*F&)ohqs^A}ejezqqlGX$6?ygytP$So_u2?ST6nM;hh3F)GLj4ur&mB)(pmjPZy zgOd5bkOu8Pe=EJacEQqQr+}yxRn=|f7M?g#<1E#)_e)6@jW5qPzFQv)RQPU0vl0S6 z6B{$%Rg*Bk>~L^rU1ph{OhSSpwJn`TBobwGT>a@{Z(62XEq;3EahMU&=ETO5*EsNn zmXGV{!z(-98s6n%uW+@T?*+vR?N)#S%j$vDTr6{aX&91^*P)^yGj#;-o}S*`Hv`o~ z>deM27DR)@{oU-pXTf5V=gbKJzJAZs_qZb+C5U3A^Y=`C-B>l)jdPB ztH767x_E67Cm(zxE#O|io%a_pM!yDjhtHrAaryde>fH=^CMx(!Y6#aNf0pSYqqZyqg6=QD#iEsr0GXz>}3(Pk+q zGSsKsiF35Vt=x))5G!EzHs|gX-~3pI^XzG`{5D~?fJ(x8!N;g z>CH#b@5Ej8#6+1aHa=E@YCEz+DM2Ng z>}kU+^jGa!*f}`xeVKT=2>ZW)djB4M=Bb@XT?kw=7>ZC(F<}3W964=s1(?N;OC8I= zx}JD=&RwgB2!T}-%rLM!4fTA9$ZJY7BizHrm)!4*Iwp#E9bR?Q;U6Du3*K$QQacnFk$J+MRwr6OD>wOI=kP?fhipBGZZ(EL$g?1)Zg{fQkpY0!!f4r9X<@ zt+X>LQLQ8aOtNKtdjrJ@CIqsDQHMV)Zm<)<&VA$-O#SOW~hk_y@?(sRZqX_Zh%-W6b~wlz_zdm_Tc3Pi?7@@xt7UswGIa63SIN#?U#)}9f4+IG z%!FB*tP|XP<*oAyIp}6(Iu=oB48s+vZb~3h7fJeK9g&#m+w;JkCcB zmI&A@>(B*et|_}d2mC8>C+MVzoJxL;CO5G@I5agxK~^t$(HOuTvCpRVqur&#)5_D| ztSOu&qGli&66lB<8k<^GeU_ha|A zH5d$p7xlzK+Q^-i;rZQ92Rtdmc2p}+6fG_Pqum8ntK}mr9hZbuZk~koZeQi#yy1

}^uQ7JU>(Bf- z&Rb%Kb>c4LX6b4ob6wqj!_e4<57CO}VhvZr@;aEwZ*EOhI&RU*CsBC>$#SFON^b8l^^qD;`N3)W{#vMeH4D8N{QdtfGFNq74vn zFHBDGr7s zoz&kg{32DXv|O@@Q7&u0fCz6g$XaiTQ93ul5LP{OE8-y*&Uo8iT9UoOtQS|)&#&~Y z7+PuNpYa0ybn22yWpSNvipS+e<7ZSqvLupGgw`0=frY7vFF4Y^zD*|Nl`HUt!)?MAw<7!#nM#Kd@+)c1hbg zHdJyvsYk?SD+9If<5x=g+KoV-x4WBn2UiDg>s#U- zGm}vps+>P%Q<+!E`ae1tcXtbM1a3R&Rf}*045P`&-H&x1*j045zR?WQqD$Q1tN zv0__`ifa)37a&BKx+mlnkXYB_WxuDY^_hK;I>LCo@>`A)UBEl_B|+#z4SspN3TuQw z*pcKhU^q8FOw*I;n_@?xlc^JkG;tJ{#=5q z#r)VPj1I^dKDHKbFUT$VSw|f(7}J6<0~e9bz8gx7;X3~$F0u}7nAkK_{l17%PhKhWDjJudQ?>q_KEhZd@DH$8_XcN{a)ZQ zl5Kv&xN4zBTXH5-9T}LlR)%4V!l~C@YFTkND!VThBjD=rS~_Dpw35XVcf6X#k%rck zATuRS&R{^y8Oj9n_q$76G&yOUUA)XJI_(SekNJ6_b zQBacrhOS^R4#X0|RJ!oqlhFK#m_8^|_dR_i8zcAoef2$OM*JsQnYi;pgZTH3WIBar zJES7t?@d+|?I@if^r+?QzW}10j9PSQ-~%t1SqlBJV>3PA_IJht@l)-=9}lULFO#ek z)myhugCTz3wbn3KXg}p>3DME0iPpT&R(IGXZ~*&tuyL}I;KaFbN9R{e%&deOUnOl} zylK8{Wwj5V-&`nf4_4?R%#B?V8^*{oFIFc%He8~RGhtTQ;Zn8jugSnA3&1JnSPG_P zPrq*{5Bbaz*VX&-<00|QhFz2X+Lx)cDvkf zFZa&AaO_9MooL_K9d$+jkv_kv{vkVmxlb`lk?b*XwRC}CZt@t#O~t$2gC5AI)|NlA zO~o%y^5G?xjS;FUmFK#0N1o}iOC_tS_!ccC?+a4Q>zz_(-B5)pOj;f_w5&gb2SD^@B ziWSDdJF6HzU5N98i({+^LJsS0=$LWx&@ifEtSN*Ejh_0?I-5qpU7ERRh0VMexcX4y z$juX0_B_&sz~=GZeLV@u@G~qv?XT7HCumV`_zMyLZn)c^g7=K{-jU6~-7d%?D4^M8 z^NTCEFMNg@vU9;MnLq*|sw)IrhK5R_}nw1L;! z{CxCNggf-$h4H7zXa4WIJlaXQIp+qB@8TJVi`Ksz{Ofwl-M}wTL@leD z?fCr#6D5-EM?L;Uu}-O~NX0hNKhwC|d0}D5dBzdhWy3yVDTZ3t`4|VJvacyeQ zq1r0C4XwYNY_d&Y{s@ASN=n%wFws|iF$=>oNF<6YJ5!^WFbP0N%9h_p$E>smV0;QN zYD~gaZat=>9!yW(bamu&%%}*HU3v%17q)%Xe_F+7iMKYo5rpZ%&|6o1U_!{eS=0~tR-HhR9~dO z1c`i%03RniqSMiln;3=M=Ee@Ea!VkeB}@(ZNHqn12LOGu1Fkp^La}!ZJ6|P z`WU7Pw;Jf>>bYPl&#-mg@eBo$k`Yk0N18WlHZm4y`#M_)9Ep3mLmTbzrB?S8>)BK^ zrWA#|ESNuJ)s*~As#5*iJp0jBH;1}UT7|7vvZ1lLejt)8WJMOO1s=!s!!MO2engX- zdi|;=Mbw6dx>9`QmB?xQ(geRF)*T6rtW@x#oTg5~4DqLT(jsR<1G}c*D3ve2eiY9i zq!7pYzKFnM=<)vqqPbjs^DA=a)`=7qBL=s~)ge&6O=~GY9zASL^x5}Qj`vK6U$Y7x zjhUD*631$>YHvu0UbLTxt$6z%a%uu~e9zDu%?yx4pHkY0$;jO_%sl?xF%T)xtd(Xn;MPMyoxm-o=)~Js?wz>Egcxm0;L?}U36Mz|b6@hVR zp5?$VJ}k5hPU&5dBhJ{mzzINnf)h6=1Y;{;>P!13$mu;gXeRc+>yoO3d(;%azFGPi zO|)QhHXJF_7S$7Ij#wg^p+~FkzZX$&^V?8(O48BbWNMB}1TVM)l|Ra_k*Bj!Bk!wi zJ3y(zXE*2M^K4)VoyuosP?mwrUqBI%7}_KgRw;1hwJwF7Z#xo!3P(~)vGK6uB*KhY zo3w;wPej;6VhJSMWwC9A;Gx$>Ab~WUk=82NlTJ1MAgB}CF5TTW&{_&`tam1hG?k9H zQ-8hYE*@t_*8p(@xHl^}5_z&^Nhg2XP0^O7YYzzl1dxEU937h>L6chK& z-0Ja6{K%@c45i5igtO`GCMpRq1}d4AW|QuUg~K^3**Hs`?0V-)CATt^HKm5Z?Wu`M zvu?jEIy(z&h0yP%P%P?VCU3={o}X4Q?TO_?#wO;ycaG`vSK)8Ezt}uZur+M>{Bt0OPUKHyX=kbJMf!fFq_uGTzx8I z?(f;eDX4CQ&VLBh2waXCDc(=0h5zzYr2M5Ny{pXI9Y$YZ#&2~oTQ5oXq$I@f;S9m8q*wth1vYYUBN0Vhe1@ka`ubJ- zmQ!|2khw00OA(zu$u>v8+J+kx`^MYvvnovnX~x$zMYtoXCm z22Wk@o<6s~2;6oT&;4UtuH@4z#-Di~aztY98Go>x?M-H7jPxL+5bpz}4c3`G)qZSO zF|j3aH{l5o9Ytgw`yT#e-YPrblSUljqQ~9t|MVoEk?nbc-6j1lZux);MU}U}X;%?p zH?LTx+ImNZ(l08_Vz&U}{PWW+=GEr3z*+8{T~u!H+Q2<*d+?s8h|7>*~g|}+A zSQq|r@gdq9>g|(qTquw+N35Aq$RDF?;Ul3}=Gb<Hv!wW%H+LFQL z%G0sq*z+RkPlZJAKR2u^ci6!;vdAK>2;%Gzsu%~Bg@-TN>uVq>+;cA~UZYd)mhAi2AV$@O)iN(YOU72ga-@uSHd~acD@%eIfUS z8pu4@LMXAeCR)8l21z9&<5ta4lV>f^bh3w>E5rv7^gO}DzaNNS>)re1`df;epO(ZQ z)w%bJ>4z+5z(Y4D=WJcDocEx6b8%-N`p=5$RCOsO^s{k{UW?2Y`W#*$`ImS-GFGRa z=&Qx?zv!0;+5<}?^-)QcrP1-dI4gUxR(E<+aT-qI2&2CfUcH5+n^wY3q^dbge4ac~ z>SH}P2OzT|Tvh6-Up`$~R*15U>T5mkI{a4?4@J}9?_ogkhDNJ?g?;8Ll_RvhAuxp& z3$T=h9z9IFF=E%^N_mGZh_*1;dsd(V08+K_DhYTn3H`=73(XcmxJ^bt?Eu71;W4Zi zs}`wRIG1n6U9jEqt=6zlW1k$earhL>sxkl$haxEBaG-dZRg(J1b8gT;6W@~w3m^>L9NkN%7II+C28x{+;U7>Lx)wQn>A*R?=x}-BCnnUE7WsTC; zY&eD)I(+$nlpxD_hQvr=Lbn2wn+*jp^MmCqjSDzcAzCopPOJ?BF!-KqR+S>HmsSd) zH4f9ru?53z+OTc~u&9IF)^}Ch&ua@H9?->bZ%-u#VKDfl78nwbIbJmobS9LRdVgjFS+$hk>6;HybP2X~Ddnp;GeXfwEG;VsUgMZ{NGv{Hqc_Q2V6f5@ za#x6R{z-J9JVRRMoZ6vam_8H9h1D+ihiU>CLjy~)i4xYMJh_K{G~{G38m-nqA8Vw9 zld2WVPm?$yd3aCZId?Txx&D=Pu&NpsGKepTT1{)jW*`O ziqHWjB-1xNgi>s+OIAT8`Fu^)UiX+;{{Wk22Ot7y*wx8Dad`)h`J^+7mzxwenBe1E zWf>aAnL)$wpa4d@u zSfWTQnXnK(CUqvE>l47h0 zu-jQl?&977Z*70Lx3v1x>77`rD5?>g2Fcs!%sGbTxv0&R^8nhpw844`1WsW49(AT` zk!@=B-wD@3!jy$1QmJ$X?Ov&+G9xV>MS_ZqwP4k^#Vxp|qp7e^P-z=#mKUD~V9g7# zWK=PftDP7U0AgLasAUE-Z?=*cWD6*vY`@eQL~6*Sf+`+=6}+mRUnUM06jofreUy}$ zYLbhT%Y*M7qe`cf2Mf@KO~bW}DHk{%Pqvh>Ceaczc9Jq!@7KAYR8ic5!HN!^A?+Q^ zK3FwD?q-rQqhSg)@RP2GwTGF&{$jrG)IGOu?V!!ZSXs778I;FkEXy$BWwHd8jF$5* z1F{A%$k2I9^DnG(pvLc@7If@V8$zTwLRbdpR#sbzDKC+L9J3fMxr3npy*yJ$crbi zbw5vpu$(p?-Jy~a2Y#o#)uc2&Y(_;H+L)J?)SRCc0NIN*Fa_heKs}10Ko36A=5{>< zu> zEjLh=jE-L;#AHT=nI01_jhe30*tKTX*(g!u&2VbO;B9F57%qe2!O5Dj6-I4?8p8r6 z2$r`J6ep7!6%6384Yh6@U0s(~2bR6VBHcr9mH|feo$1UyiwV4ubsn$g9yuvxw6c>U zEkg?m4WC{}^(R&kSRL~iQ&Td*%|#w{NT|#*1GTb%RYL`}c6q%NF9AZ6P|a@R4!DIN z?oL{UL@6s2Z@4Htiwd@7EH%VFX>Dm*M>f?3f-7)jw{hNVMKEZ?LX$NN3Pl?QkjXok zV>XErf&t~tx-J9ZSlF&P7Gj-pEK9Z*n5f9GtCmh;g|crCO2i&~*wb}LF><3r^JP*K za7eEZ!O9x$jMxEPgZPunQ1&Nf(MAXq3%AC}(b7RC3^4Mj^>;HdplZnL3S_sN8+&=Y z{Tc(iw-advN~C*u2T}ke&;cayaJY<5?iD!%(LJ2|OS9FrjO*I)-1RH9?=Cv$sP?{? z_~TTN{qMRx-}_8In7Avu;`ESGofBHlrh)kEEr}xHg)>iY*!|8w6R`%`OcR`P<-%V+ z>vtWxrzp%$Al|z}DH9)8g7>U;eM=$_h_KzNyBPt#VI8!2tfy_%edN8j5)_fz@u^XW zPu@%1N|%Yap3up7y0n>A?oHI4_qzkiz*IalqaEAHvXXV69i^KRpqmNpDm>gh`@Ut= zTjEr&Y<0U5xBIINyj{c*BGm!@vOpaPmVbGvL|fDy{W1^L$%{X;!lKPLE5~yqQ>S!9 z>H~=Pq1kTP`~5U|B1+Z2HvO@pVMn?l%|#q;03E7X5LB;XT$?&6iV0M*x>?%8vS^)4 zwI2&ykb4*o--%_FS;H0{vRPp6V5KG6Sq@DQz>I43GBK;v7!)z1)?1SdX!S57qm$(d zhi4+dHqVIgY`HCepLuO?z$2JoAgtmy9jQ$7QGxAxFqSP9Z@G5zJhcU{%TN}sLWw9K z)QS*6jbY%h5pq($Oob_0B`EO|&^f%G^Y^J$rCP8*M-E6N8456})e1!KA*tchFR9i|FP9aMAJb1r(e&WT%3qb`i5N#wOYD!2duI*AINq$@X=NTv4 z_CB?XRoj3k)JcKdb%6V^zI^`x#P_>#BHh3iy{NbV30YL83wB*blb&9Ym&=iv$p@P+AI=u)~ zj>a2>JAoPM$SO(noSG>+OHK04JH!@H;qz&VJ^&EGb8Q7JK&x(ouObk`E0>a}uo-8A&19k+kkb4_ham zE(jxP`FxmYvwqPVE_fCFRh}c{*6_AZfns<(3!Tz)c9L`~%|gCv4PlWf++ph!AuX+% zvE2};=tAt#MHoY3q=?cNmStBz5IORG(xQ&M0JN%?bmN~F7?YEd#ytTr&WF<0R^ ze+&Dondrf;wARPUXXVN2qo~*aO0$IFa{|L6;siE;?8IE6L^IrW{<81<}1N+;QM}^N_Gj8ht0ITZq*syQHhqW=Jv3PO9`rtRydl+uq?yH@tjtis}LcKmQ*kM{QmiA2on$(1y0 zyW8C+A{Z1@Fh}Y$0SpTD_l5;I%T)T^tTj$-2)56p7uvAPaK+B>%CgepWlxwyd(RT9 z7FL9MuOy)ydjBW)>szTOC(8k9pcL*-{(-5kF zG@cqiS6A}#_WuA+7Q7lQf~5)!=sCk{V#TGPq$n5{23l8_2kGv6x6DCGstVSSITix47##<2 zbfl2Lg%p}O2;PCsgnSTa^d(@mQb;gZ;DaHCt|8kB7c9&v^Ss?d3U@H+^9oXCqJ@b` zO6YxA0?g=uSYuMIBf_&hTc+6~CQ}}j072D<$6)vlrz?n2CI(J5Nt>$s%IkZQi9(Mg ztEjtvIK^66K_v1&MgXFsNvxO35E?7MnR;whDok<@7ykf|KJa?cY>~L?VT)$mKp)hf zqs;M|kO`|>stE>JwRk<~e6*zl)UYR~{Dac6T1v4u{_*G^V1AnF;`$IW8gg;I?!%i{ zYnYV`&z6aw6(pX?fDGmfGEVGF^4#*Rm#c?gY;-6+9`CiX(w0Yx;oUmG5GCaxsR~au z$Fw<+AVAT^h><1QjbZntxC1s-jG@#!b{GX#%1q|+FV+K*QHOx|?=aJBSGl@_yJ^+2 zVVH<+fg)Po(Xa(59~rPK&VMv^hTy{OFhh*BPI$))9UB zsIy%mDJ0}1p1>SXy?*D4e6jdwYtBgaIc|h^zljRMsU{%Z{{VsGx9cbp(O>(R&%E!b z5^moy&1C-o#P7=B!*v@_qrkqdXv|h6S2syq5=@$tJnCPm!B|zRLJrjp-kv1CD2sk= z+i1^nk~&la`bHlPHr4rm)1T*2>i+;fAFY9fTGoOt5LA;KVkl1&1?h&Dt@$J^KgTi6 zmcqr<1#|WuT0S`C`i}Wy`KC&xqK#dm*-D+n04L1%+Vd&J-3qDA0xrZshj2IlRa8)buCB!Qc?-G@)() z4wM)o#8t3Zim|Oku|X^fZbnTF#T?7z+bZD7t2Cuq0@`zL<?4&J(1*#XzT+xUYE7wJ_#JeL5cl6Dm=FJ~ZZX59l~-ExZmJ%0ZZtPvTJ(JU&i4-8ch=wr<=>O~v?j&!rSF zMueK4?Pci1-k%HOm9^6t%)5r=O$R3paPC^Z@#}II6-W;RRQ8uvun`JZ-dFaEl~Q)w zRAmwDaf;0S<$q|o8;YE=b(qGdlI*EzNP3i5(j`-_UshJ&HlQ5?ja9&LK>q+x`Z;V= zTer8H^Wjs7Pu?AF;>}JY$8kNbChC`r#t=M1cb9J}jD>iby%+EKR+qEYyI?)e{mW~0 z(4`N{@ptL*%sqT3wLOWpoIVoOl;n~piGwC9te*E7VJC7YTAt^>6?CmN{6p&SrC+Fy z!tokRVNtn6=qujjVTTNlEd!a&oyLemw?*43-G@%tCCUQuqN|s(_n^Gi#4z7wKc){J zwx*?(ACjND^k|B(%X#u*0A#xxnj% z2g|p|Ua7i@$6$@iCYm3o4sXoc2F>k?mN*x;LXmr;?_IW+sNd!x)y90%I;r`*<+9%9y^$ieHE=)dW~ z=bz@vp}oU{Hx_QJ7=g1f{Ilbe_X$iT=fmNVhe$$10YN}Io|RBP4{m?-%Wt98*Z%-K zbi;~J0QNk7@48m2$o?oJZ3Q<;EispTD(`uH+@)K*RT38niH}mfN3uPyr(coBvihIg z3$+OhX-pPsKCVp9a(rlb)F89lQvm`(h%VG5L%Yd>6!;koGFtE~gR3%xzFGv!S%Slo z3J(I^ZHU~^f;g}W)444zsHDMXh6{oWh8lM=iieij+?Hf0r0`%Xpo)x{hSeZLWevg} zK(8rqa$scU%;`W779RsFEt-l9Pl1LULjscwxZ?K0vI=buYc9* zRTx_j@|Hl-E8f88AoE=HcQd+xz(FOM0j!Lft%|VoKJdlPGs)W?ObMCYydhH&NXt68 zYYU`5H}7R39`m|WaS9~clnvXF7SZY|0X5TUJY1$4^ngxp9k{jEo~m#yl4amc#&oIq zwDs?85E~5GL9T3}!V3>Q^+HQ2TaLus@Rr6dS$KrSF0sy8EL|3@s&!7@my+tGLRhY- z?S#KvoIE*SUXufPcksx#vI>b@@r9W7J)z;i;n#=5qT0wMJF1+jqIKav%q#I@qOo4(E4HBa)szkKe1%n)zNFcMdR!lhy z(6?X_FG#3iNo#i3@=i^S8m8vc!G>@f5JNiceJ|fZ*obt&g88<}p^t9oVCAb(+I2 zBJB79?Rz0!!S+D1Jz2QEe?koFGc(X{6{FzXl#j1S_wsY=!WGkMY z_5o7QbJD_$!_-Qh_i_S2P|ZsNppTo-o1SJRTok1C$J)n${Iq@l0IFYM{{TII?Jo7d zK3~;!oHb(q057~b+#R86!bZ8C*9D6754<0vwdf9z@1H(#w?2INf|WxNRJORr!#S$n zyLpF@dfv2V!$xkMhfE`^*5H@CiE;LSGU!|8GF+8k1jdi5jPi7#+CDQyvsqW^+%e$XS?q% z{uuhy1_p zu*jU>EiRMlWVpj6)NVlcU#(>B}dTLR-4gfI_ygstSn> z*P9a}!iZG8sWKiHuHU3~GOQ>P@LdbT-c7{iFIdfo8|Y7IVYhlQ7Lu`_#D)|TF{2(2 z@(12iB3Nh-ja?gYt5Y=h5-g> WT9FRNsoL05OU1@GchhPFfsuD+W6ciX37Z)5H9U>wr|HJ^900065 z0RaI3000000000000RUC0RjL92LS;A0RaI40000000000000010s{mE5C8!K0s{pF z1qB8J1_cEH0RaL45g`K-F$6(jB2j^np#>9hu|m-@!4y*A@FTK<(qeKmqLLK`b3;=! z@zayyW5VNuvm~R#@(}?60RaF50RaI40RaF2000015dsn+F#|y|6Hy}nzyJ{l0cZk% z0|6oc0ucZM5di<%01N{G00I#M5dc2``3{|TF`gx5&vFs&3xz>+_ zGGkXZ4>I51Px*(%sf-ht?Q(k!Vm#ktJlJuknC*DRCouGSF^uzi1gOC*IMMf$+Vq8^ z5aUkWp@A2`Bzipgh8{JSZTVvg(>R7o9FF%AwcIC#OFo3VjOt$#leURUeFw4m61$+) zOX^FWv#1_gKU3XI@osX()7OR+664p!4|gx6NF&S#sA$}}K^Ro#6=YSmia6Y5^7R)2 zNd`IxWM)+5qchRkl@k2m*6N4Iv%V6O>F$<9Yc&c`1yv>w3VscZYH2C+XJj#`gh~rn z&%19WM?vs-RpO+nMMD%*Azw;7{{T%buSEHA{NMasb)E~Los;Zmy>{uth|Y0OMjK{V zCH9{j@6_I!^xIIbXJ+8!Gzv(^GqOA1o(s5;a2Ov~cC~t>OfEllKTuve=UbgUa_=X9 z=c}uVK(~q_V_=%NEiR(+*ZQmB!%6{J8$b=UUnF5F5!2fF<~O?xrASfW`Mcj^Ae>pW z6Li~Lg#qk+b5ohbp66LhVhThjHiok93rh-D6*Kh^@QqO5sQe<%CbGiSZ$9TLfo_4v<+b>-^ImTw&XaXZ#_l zuD|98jkvqAdY-9oqxve@x4lUn!OPxGEIg9@wwq9N;V_b_&T;fEXlne+MPDga!m6eZ z)y}?<+5k;9&(-00eDn|MGw7tW#;5>*VrD*Jw6zthH+2YD2ZZ6m4C0(3M0$3YmDI_? zNjbwqR;3VPf(Ny)=3i14pcEv;0qb0t5q9GKbz&T;Sa%VfGL`m-C-V+m60r^w6zZik z6g??NBWF;VApF&@p?ahhNFe0y6cUmtU%}_jO-WFoCo!?v)S+5&oXL?%%)F@ZqK{Ga zX=n$TacMvE)jdQPwCa$tgks>%M%|4qCahV!wZVe2UGGy0=AFX|{{ZaQUF%p$N~#hL zc|xPplP%ps0v43a9inIzH9VMOPIwVC;Ro39aHs5u2aAGhBmNoL-tlVk1+-#Vpxf&k zJze>>{>Ox8gOQ;LgEO*a=t`>cp)ra^Mt7aLy7SJ`!kgAqKe(RPSgij5%nuVM_`0Er*O<&VwP zy%6oVnDd|M`mgza;z#j3D?_VDpD+AK{wL|NLk-)b%P%^(^H+4&u>8AukBiN#OF%Ls ztC7^|a3CBrD(Ab=$DL*`cl=+2={_*Z^w+K#N@u9}w$tId%n2gaV0*~^Cne#wMy|nL zEHJVoJb?7({Tgfp+DMs!?0CI5BgT&l)Zr^Q69*z|{t!odwD@i1d)E6^{4de!cxw|YvgaF9$De*>DguegbTj>G&<3)L>+Glsl};@(v8h|8KAw_^!Fj%5D; z32UDWz7-g)_N(|@?YNTla11I|(2_^2JndYFKlB}5r{rCchdfGH&(hiC!k_kgR&HVQ zEN%Cfv;P3L@vhF!5u6ZZm`fo|&2>$?}YS19A%hSy`JkRid z>T2q=o>hgMbCod0-%|ZBK$?kgo+)`4muzRj=qL-IH{)c8pn88MGCJ>0*< z$9urBsIToA?DGEr`i;Q7tf;++?RRK{z2QG+zpc=V8TCoD&gD z@sDG9d^OG*SFTLcIj5_$Ku7yMC;p@V0O$Vz)MihCX~KU}^3+G&!jT-KiGsrLkxi|X zoGC5sJrKF!`g>{+yPc{27DV`cqLOi3?H|JW;gtxlqi))`3)MhV;g(87s^IFM;c@pg z)>Cyh!pSNjDFlf%r?t&v;Em1+RM*&7{^h^ha%v@2;TCP#3sJfcDNjG?b^e*9f)u-e zM#4XUxh-AQTu2kbe+I4N6d91?RzOdO=@OykuIi)&l1h8H0~zqm6sb2ZUP*#e1Xq?w zGb1ed>=uDF?R9loc{pwo|oaebeKx+kppUffH0v-rxtXNt7$Tj zsZ_aF(JP)AZ#lWzKZTD=;nvQvDL9}ZR}T=HF5DUFsKfKhxq0u2pD7HWu%9l`c_l;s+l%-0KIQBZaNkw@$w7PWu6jZ3P zx@E+_{8k0&bZNT@wrf7}{{W1!1z}lxnO-gN4x;YqG?whCM&48JE8@4p5cTecx;J-2rYWLDGU6w$Z#peLm+Z+@Nh@5{UwqS`0h*wm zp3lLfqgEX<2lK3Vmo;#E`h5+*_&BnGjeWB$YSZa)j5fdpKY2v^OV+O>`i3-_GI#eo zsI54%lc)s%;Z?Aiu5-m+LxGuQS9(j_;Bn$K-viIENCQKwh74y9l?ZvO_HQ7R&9uZ04Lt0W!2*9En|-= z-EjMf=Lcrj#aLra&;-1if*;-hCKRFvYo!D=w1#t>^O_Cr>v zP#f2l^%MU9c5r`I=k%m^t>^>v@uT$ZfKsAH0TjU$jqQLCEwA*@C;)^eDJLjtk;H*t zl}DtZvxp~#MnLDdFk;=SVMKX$$a-;hYRYvlW>my351=6uPo^L|Ua;VH6Z zYK%4JWJM*)8M8%}!j}Pp(2~h;aF;6U1z_`HIlPk$K6dA3AX3OPcnPa-$^n5z%&5c3$QVBWcqS8zW^=e!t`r(A!{KeCrN767Z|)b{De!WS+r zLaHa!y09W$FNGwWgR6H^IlX@hB8UezC7ekh8oXRkQi);huazOjDI!?S8bJqC41> zsu1^LMlPRMC@8lLwFtM+eHf6&q!h}gmFKnN>a8M6AfaQnF%VfYQ?*4Haxcnwr~OsX zKbQ6c^nh{8fbm3udg8g1{=jZeAqBwLCcRBmK&M__OL)5ngUz zt1qUc;mzp@UUJh5x*v2z8?Po#YB~*|L3aS7PYM(0z&H!(rjQYPnadE4#t%B{p+jD^{ z!jPQsjQt!X#7+BSZX~O{>Uv7_7w%)V!y;xkL29jT$SAg%Ge1@ZGfgQ{2ID=CK$9nN zW5qs^4244s4S~vwNf?t_QM$3gJgeH!wo-8O#3#|nE9F<9FEm(%3T&Zqf(MAJ`bYl& z=H0dOgNmkPb#eCz9w7rWgm)#_Q};0{;-HWy�g^*uoTer<1kK%R`<_BhkdAZf8Ll zhgP{s1yQ@Xn4}CRZ2MSNsyv&O9`@b&ce=I$bhc|eJ&d<4+X*E?Q&OQYGu*bM5NQY7Nm{qB&*%0A>IgY=enAoDF;GlN3*Q{2CfFHiz}E+Spp9OIBe& z0z*SVbpp;6C?w-nduNkol=h&NBu3^Gf}%D^<>9_{J1X%`V80Rb3+GngYZgUQ&GOld~ZFi&#=6g<0Nk@6$B55eggB`F`N zbs)-z2r8h6%#7I%mc^^5jb%6xl%$v>k(nusu!aNFY7&eIkWM5JC+^N?D$O5V)*(va z1e1p`w`@gcqdHNiw;)?Wq@;sz1*$+ZhZan@kOu;AfRP(wMpD5Po4A1TmnAD$3K(aO zXNoo#V$Q1N#So+?W)iHDF+qx){jYcL6x_2XVu3uQhu&n~0M-&t+_@OvECO`ukh_;i z!lY0NK~h2M8Tvgg)v3awwVF>6;!X+!5JAk$mv>BAP#ON5f0-malg6HEN@qO$uurCJ zB(7<+!Bn>ts0&nd#7sSl8X^cbGZa=eQ_vDb`mrr1AFI-Szej~@PkL5H9h}+j+~7Su ztxIDG==QTIA>Gg*yhH=GkA6Z%(pZuz8$`FjGLRU2)${HQ<*ajbHrp(3Pd!D7gzoNq z<8cQfU4?KUylP=KRnyuYF=4A6`=aHtkRoH+<@$O=-#C&ZpU}A5+z~rmrPSd}v@alB ziP-w`=hRI2HmNrB%NR-hQg(gBxwI_ZPAsQm^BN@icE}%odxM;`qva01*PiozHP6M} zy@>=zFKb;4g^cWnn5rd=y*#TYXwqk6;#6r@*3}a7R}1Rk=qRhSpsJzlk20NE5-^B* z6cAIM);WbTEk=!5GF%2GOEtBUPevrJ-U1GFj>cU&!a}#k$0&L*q#j}BfWO&7tAtN{J`?e+J!LBxeJIY19Y`+CkX1nTOTBw9! zGh9QllIo&sLstdbf@tDA>G1-k82Lcncxc*Qu@(R!Nt~lR+&Xx#r!2vG<|TS_#Ahe9 z94HO}dWG6t$yVurevFFo8|5&tgp*JsXH^$4JSwB-CChhaHIKB ztTE~XXlV5-3An3aC{ZGDB+PCEtF$-6==!ZnKF*+%IEPWhHhMly&}>>D%@CeYaDEWU zZ^ALjxpv*KV-LJfs3kGp*}37Gdo~dlRqxMRcEB_fC*A=vC^k(uQI$jl6CU9`jvr2?AfY8cF`Qe~6y9l@s*}`R3B;3{ zfE>B<`n+ROeCo)AKnEx*o!PbZvQ((40+Y;VKF6DKP@;CE82#mk0|`{6J$u@^r4;G{ z#Xt$BQ=HeavIGQrFt?u>nAdY$O_vNl)jwr>MJXZiRcn zF(PX-4_0s7z*LKBSQf_?#K4m$lzLU`6GXa_r6guTZXV1GS1u(Ng6A=C7~N6$GyRt7 zUM1Z?G4i8|ihw~m4tc5O$~54hr3q58T*RefIb8IC*vQA3=NvZDwZf55IAT)z0Ltn5Hm-Pm(|gTL$nt^+8O-7r26h*N}zZdy(Ho_e6a~7AP_jeP!k$6q9D|SZk0-A^XK3tbhP=7VvD~zfl4|&=Q8LuLR z`WGB3-Cc|K5+dEjWw5Ya)7-I#8cK0YRBX>8`b=V^v^adTQS%>Qb0JA7=m%hMdJ9Tj zl@LeHMSTO493u6C{D^hH{vVDd4Ie6A^HteCd$vJv_C0){;Ji^J&$}u?)px6 za)87B?0p=K*nu9e&*<-*a9|8SP~{b>w1~#B(oeC=E~KVXVI`8!%_oylI;&R!!HCu- zAWTl$_cstm5ZR41r4ZD(tW^Bu&wAyt!d96ZBu`|^jXIkNRaQF*_6H=UjcYh%w*d+! z1zce96a$D`Q9O?b?W-cS7?CtiBb0}6G;5boX^2tg#-SWQNy}w6^M@NrCz*OpvxJ$3 z6KlEUkvmckYxBsxI-6@zfEJQXXabp2V(r-oO4a=-D)u%J;QE!MA0>6#RFs--BoWx` zyu`AeyLl_NR+A&2R-&~SZLL|zQbe7w-#!niO|C1p!g&lzBD|mu_Z6+t`DgpHMCk#@BtVT?N}=5Yil37#Pwq`SQsk}!!f+G0CP;H$sKa_zJu zFk=z)07p5R=NDzcb4CVNxK^PS(Y0fC>E3p=BSVYU~rnnICsx zIpr*>zK}D=U6t8V#ijENq@^iUNFeOQnvpp`2J!DxaVRhF6KZ{fyntovF6>vQc3%gc9S6Y z!+>Z-s|vXwT#{1?6F`}mGAe14WeOz<1jLB+lNmh!0H*47H`KdnDM>U^=nnupYuFsk zp9`~UF`qJH=AiW7)$3MDfYSMw;uDC}7|24l@|PiJ!g$K=T#QOl!kUm2W~#^@h0UVM zETkbR2~BAfo`dS^2_GEZSH!)oHJb`5LO>)LCQ6MWB%G)y%kxbmm2>G|Yu*|jAwQju zu*7Cg*pb}g$iPVwY0oUbJzs2}bmV82?pxa!V3@B;cfn#d(r8bpr{}em=k4ana;Z5u z`y}E%ji3y@Fu*}U7zm+l*A1Xi)! zTxH9uxNEWRo_*^Uro1cuonP-2`zdayH9RN#3;zIke`PJphNb4u{+g4~b!1q8C6#$R zp@wjl90jsf4=YR{mtYVO^u#U=q&8D>&`x4V{N(7QfJ-6C;@)MDLsB<&OS%Zhi94bw;_sd=jfA06 z5QLQ;Nd-ju0ar`WYLrw>rKBb&7YVN*s3)V;$j#cl8gVGkDeq>yl$8z5{U?HIl$5Nh z+5r@)D(eC}+bi%;kDl=~k^*4*0!j8cdy6@*Jhfq>6r}B;b>&>U3R21bqb^HP;mbt% zR_-YYNm7(W1WsfCGOt%P;M#F++AglzYLbyLHCBC+*RMu>Q!2gY){l`PK0k{WCy|Ai-L~WRkT|k(Uf6o}R2$?V&0-Qj~|asDyg8>j1aa z_?1?t1_Mea0|C_BnIy*`)1gvM$g4hK&m3PPCi%FXwBF(TKqtzf+w?9$IE?_Kiq|SE zPxHrddW_$U+wlbtv)Yv(lDMEGWCS-Zuk{8R%45C2Sy=@pC=>}igL!(}NnZ$3@bjR~ zcGtP<5>%=|DnOXX_q(|ul_bFhU6T(wYl*yZh#P^J-Lduc#kBw{CCaG#!E3fHCTp;7 zdeU7*{?kf@&O0g=l*xyo#v|}tzJqSwd zt#Xh^KBqAd4BU>FL@)iJca%~S zbtqI1n8^eJa)1Xs+bXxEv=p0mrk^sQN}yI#K_u+t?6_v5Q}(@93>v!}at7Vx0Q-ZW3F?cWm) zgf$n4x}=-hLaIb5awc(n4O1XJr;V^L3DNGHAH68LCx<#}m5QHT> zfBdj(+0FpWE6*SDl>PBQ`)7Lk6V`HFe1Xqv~^>JE`AqJa6v5OL>H-0Q? z;XG77bZmR!ntmO?1{wEin%087;D=!1R>8PLW$k)^V{GLFdp5elXVYdwp~_fiVF`$? z=-*dS!fKf3euGv$OQF#?>Sv(QyRK(q?QX;2_=(Ipe-w-ipM)yEl4X7FIv0yyp;@G zvh@vA-U&buWyRjq;z<~DjjvqIOVNa^Dq~|MtLMx}Bi@FVnMBKG-L3}EDOWmz5p=AD=y72zgIpDb19_*eX%ZMYRymP{n zy!<8C*uhp6D2O$I`3`s>VkGThUJOMvnaQYazvwT8{a-NbZ)}!4)>%;7q3F11Ept>Y{yUtOhpG9Hg*c#yINYLVI~*p01V zfEe+R72NVciR2iuNp^Y+B!Vf|)v%Re3jUHB>B1s5)Uv^^^p?@PH*kvdaULQGO)>`* zY{6 z8+=GUwcST{ucFA?QvyCa10JtaZBQo^1C?`!WI`B?u_;ll3!leh;Fj3R0PI8@*hHo5HQBK4zlm{Y4Rh0}_uFB%hXdB>J-3)%haboJR^}37$bXXNPO63pWAe z6bK)@eXZ<*>5-$bFx#>*5O~+bLVzYj;yGqaGgw+>QfrP6 zR}~m8VH%5M(O0LbkEnW5hB9V^SFRxH-j3$B?ZA(h%~8b^lz;_t6)tASgQXl-SBXR? zg)|hMxt#V#s~J&=#u7qg&po3F$hOzvH?Q5cwWwQ22RfB-ryw8!6IpYJvWs8@BoA*> zwcO#a5@1xOlm1ckWx`r-4Y8yct%*_1O3aK?Ow+nR4ct4(nf%cG1E8uD1`xSHT_DEn{ zN1x}p^pbY&{B}M9Gf1GC7^hc40mW2QKM zSrMC*@lv%WC>WRG8nyob#ECTlBQ3;b5}Q{DIS_>qeGECg#qq2yN>WI~ zYrU*cNOI`#y;_zKQV}L4N=y&(l^*9IZ@{ic9n_@%04hp^dXk@GdBX8tWLhIPqhKy1 znsR=*9tmIs<6Xs(oI+BrfE5q(HqoHb+B4-N_oYLzWPs=W6T#w$6C!O1>ziX^?{zE)xV&AE?5jY9F6cNCdu|)^8)|bQ+Ce zQf(zLh{U2`3Co$-at?mqhg{PUjbM~lgc|MI?vE4|5G)-7o& zxp65Ze=2ERj~3fnnTAlTQu%iyYyjhQx01YgR22~|i=W$9qMm+MhDG!lsV@uEO)&Sk z)LkNClJ=P@8{XL*u7&SRr4p$;zO7~az;lM~nHX_S@2gQ}@Dd`eG?}~(^Yvv+3(Lio zW3`D^v8bG?W3|CvWNIj*w=L;;bk@}VUo%njRPfnU}1YuQO%)PcX)+*ra5 zMHnr1s8bcBaF$hkuNLSF4Iw$6RBS0g6S*uX1S)2QJ#q`!X9DYl}MH8f0(=8i;oG7)Gh_0){c z6jYJ2r;+56-qkHSO*Ok{B;hb;e3|X+mXfcPT1vSCBh!CJHtOVUcC^+f%0%J9F)pTq z<`b0XKpkndDaM<2aY2_B5g`r^_7ktj$zp=60V)NfCNYCF&Ew znc~l&_`{G!krvJ&#Aanc5k8zhQryB!nk#tvyuk(y4$Y1=i}Oy@^&g(+!E#MKfnD?+ zj^~%>+UgQTN7co2#@>fF!1M19jjy_Azwz&RDN<<%p^?~VIugThBO)nCs0Y*Fmy>Qb z;>tJuPhfdUtB{@275-Amh*yXR%qS3j0TKz^z-1&8L-2aSVya4~V9&16*z~KZ0|@E) zns!Sb0;4MY^!6aCy1Tkz>isZ0Hd;&oU0uT9x$YjT;P%13+ySaPeTj{Q@1L5{10=^0C2|1Jc;dhDjR3l=^qDp=>~^k z+1vOqqJ>BTn@Nvp66)G8*WQ{(;AC#2xd7%l4U$gkPh-49W}1)AM?B)WUn-zRYoJ&nU=qjGvFV-8O!xcRFTGWvqG|2{} zp62>2phpN}8TD?MO=LQojG$Y$@}(&`&^dK#5fsEvbAu#vXE)LlK_(wQ{{TDx0Irj} zKZ5z#RUjG}cD-f8ln(y@Ym7J4PETnjUc7NUj4u7v`^Tj{j6ji{Uc<$~lS#FoVy94) zh=O}QETa09G6TJ)63m)AW*RDqA%rguZ)Hj5*Ivh|5v5$s`Rl(SbIDAH=Ps;Z&E8^l zTCkyRYoFr8yGppcTMT-J3}LXFhsvF>Uh#;K(9NK@+lT7p3Y1WN53$2&ucp|zZW5Wp z6!mOf@jGE9DN2-TB)}2Ro6VZkqJdW+ghEnhV19Wz+lu)>9q)Ga7Y)Sui&+AAW`c)f zSb3*N1wpfFlF-xK#+nVVWDMcoQU3`FkO<1bj_Rm@ze&<81`vKCUJBzHY)t&A(e0Qr5P z>zkgE`-pNC0}odtq$UsY;hy&xl^Mgyoi@~V`tN$Iru8P!Nx-145)2v5i%luClljcJ zk2yeL%3QXYU!Z~O?Jj04NL*5QJp&c=i~)Sq1a`cic4*brYJeC5Ng#^kGCvobXyv{I zrQ8)2Rjc;PC?u&>`=Y1e-t%i)jzUr*b0)cRcMV98pCw=kqJ3O?ZlX$YCV5SI^6KOD z7jCZ*D>!E$U{fZpmet#r#k*80JAk52)0axLiK$DvvV(;wCwa@dtSi(krC7ugg8(_5 zH-Zax@oF#BsEjDt`^$oNj6GYLy*84{2a|0g2;wy)a>Qy}@=VvB(RhBRQ-x@@VQ@l^ z3UHj%F-9TICCPuHTTQ9Ga18iu@bv_W`eAYwwK%K<$&`(miB~qRA~f=I^A7q(Zf$E0 z>AuINfnvxjq$uVI5k)Fg2H^pNQE*~VBZ!iH8rDpx8OovSP^x10r{do9F=kSlIA0S; z0<%>rdqbJDVJY+z>f-!j3I{WnXm)){1^q{irFqd|;E7q@sl2Rxj zW=Aq%KCV*NgDL>LqMgaAx|2cJT_m*@RH>Y+^m2BzcWz-+5w=gk%-Pe~um%(+e)ek0 zGd~7Z38;Csxd>U&Td+!j#-l$lr$3jT`EMmih=P&_s(Y$C9!z_|@<25Q5dQ!Wa1fZ! zv{ZJ94>*G!ECxcl8XnxeuH>I|vbdf^r1K83FKj4&b9WDcVUVG-g!0+YAEVe&8TOB= zO5Iq%7q%rw%mk*^o zvg+Z+3m}cd`9Qe?q7nAy;L z9bnjfV{d$9^0^US%6n+}2_vq_j^1oM{hKJo(YJvkV^YG7BRrcbG;*``R1Vthg77q} z=O7w$ESp-OnSFdvQpJF?<46y9H@0A38EtH9U>x2`)UBjAGATZoeSsR>zVRZ;d zNhc7@KPihDhv;vAabKN|4OYTXNSbYZc}}9yXe)bd2%r%RIwgGOyFRSSx^aL6)yk2Z z)}Rh@)(9AZZ?B7EAD1VSn~RP~N^%TQ#7{CyWp&@xBGX3RBZ77OyD}^=MCq9zqM}iABRdVIgI}m}insFTIp9uABr&?8g zDJB!OBC|qjU9Rt&Ff!U)+PMNvjU}54z=W0XlAch2$Wg-S0)Eq}xg0@eHw9t^ z%4$lJr&%A%`$5F%5phQ81Cc%M%jqs2yQ&C5taq1JwH|f6RW;jAjxSY<79Ei+1!_ez z+uH1m>T^d_l6QSyUtj5g{LES_*sTHu43c?4F4bvwQz->RoFO1d$eE_*_|}VS2njn? z1JYTwAYlS~8MN9Z!hFxu7;-t{{PS`)^w#a<4CjcCV=vTJpHFpYfj~^uk>Y%eN8Qy@h1A5eaf!jb}Lcz>%S``LEG zFyZt$az2M5#Eg8Y*k3`e6Nhym{^{W?2&g+AYMh>Ns1d{Kgc5(qYuHO5xRnitIcL;n zI))Y&l~R+w9VsW5q-;)0|uV#JEI7Z5C<~uU&4CKemnBFQr zA@EG(ehdU7Gu`mp-04dPBiutVx6E~)!d_EoVwoO_%es5R0HWF3LCLY06wC9rz3$8; zaUiGqOpmHwdi;MqSXLz`F6Y5XKQc$C5#AtnhtG|3ujilO!=`>!SMHySqOvm(@T*vd zE&0b|5&>Tu9Ho>@& zwT2z}3ZbLWJF%PK9--uFUXoadA?*%sfZzFtqvQh*uYAMP;>v|#L>%hqs;Ileh26TW zOWn`}I{SpgJ~%}KCH=3E2#r&nzj$k(YIo^ zt?ycWu65|=;$q*2s_BBEZYz|Yz{tG~n@S>7xPl2W+#6wKB*R&2{Y>+R#_e|Wjn^j! zm$GN*)GNT8LC-HuZuv=GBZpuxE6;B&tqY2@f}oh?O831^n6aH1Ss0BtyP!FHO2sZZ z!IUWS_y*3N;NqR%S9ei^{l$M#_ORbx6ds<}#{M;|Mi?fuy~OIT72`-cj z#g&d+o!nL=!&^=b6PcZr&tsPV08Z5{Gvz4)V3qK99xlMw)k!rI=3P$;RReZ?uGE6D zDl_YwoYVMap>YQCu~bM&gvTO&Vmk|-LbRGoWj2r}DFR6xip-KamoIDL@(J**N{9B! z0NZniq#iD@4orADEL%eET`I-_gn*bdq>^Vn&AmpZ;^$zNs?~@lWW+gbtCBWRYQ++P zF`P+{Q&tkYXfmQFINHg5eqn+05F@p(Yfo%ko2uE)1p4N8WZ)__;`-7Ol~AT}MAY&U z+|(*i2KqSbNlqld00JA=N=iZA*lh<+j5m|l*j$2$O^b(Yv@M_4PbfaZ+PFVWT|KVY z0j|^pX=4nPf!srm@ZCaGZ%^`2e)A@(((a%@CINstI`H2I)nZcaMWm`A6UR)6PQg=| z#XBCPn4i+x+(kr+d?k-hq15VV zj~z#7JTU2PVYqzEKuSP_ikX8lYqkTFLT#iFl!W?c`QFQENnfkwTIECBUUj)W25IwU z_r5PeQ?{JOMbb@R_q|3FLr=T2ql&qW;nomfhhzn1A<*F8&4ATzOo|`QZ#c-JyLLAQ z6$&vf-0YSEg(L!+j$0k#N~!XmN9gb^s^7`}Q(nWH(8F_th*vL%HXFjA?+`h>4>s5H zj$3*&6vTiQgf(eY%Cm%J1|9Hun>}HYuLyP4s5LXXWdjr!&icZOsOz_@=O+S zx!mk51`5%)7qyu;0=IhS(dP-+=zw!zY32C56$z}ci9w}z8iqi$8Xg0_RFl`{6V@1R zdXD!-zmLZ^#Uu&La}^z<>c&Z6Kx1~#U)~>3^-Z)L-4s6t0QHJ8ncE*$7D1XPu*(@G z>&#HuJH!Lje5bM33igTQMicFh_7#d=u??fI#DWy_1v@)vd)@F3>1b%Uok=e_w_rsBpbY+HGGZ3X>3O3;Mq3HYFeL{#F`GIM5OR&& z$>k4MGI>Nh$k`w;sL7XCZUAA2882T!fvp~hFBU;k!(!-&Y-{Nwn}NH5C?G~B=t|O` zngj9}!94f9^5xmf2R7Phr0yY$L=(nU{ITB7qf()e%dyM!pA#BGiyX=Mey?f+%#+Q; zY4vv#j|?VbF)guvQ&klg&c>xNs5tQ9*w82mC6QMQz!hRnn^ARf^2zs%Q*aMhXo$Fk zpoy>G`pw)ppiH-|cy*-4HK|psaD%&Pa;eiW z5!7c6Cgrl5iUy=8gX#XxUx1X1b(*Nc zlR>&%*1qlpF1SF6?G7))FJRExAy`iwbN=F;>aO{gRw{8slj|=XZRx&!LWfiWgh^&n z&D$akN!a2-Nn8gbYRCQV9$t<&!V++GTSO}twGhpc9q zxQr^{GfddUprgd_ZY$kqu!6c7mRO^NQ7>0PRg|Pq;;*G!pJ2GWB_Jm{sOQuE&HFk_ z5HSo&8O+CHjYx>n+fld<5>hi(!6K68m7;(J$SF_;Q?*g)g@YPJ^K8u~70a91NL3DO3S@5%;#$d4t#1vWLaG>I2*|z5x0V+1aRFkf zpJ?lw)3yP`w5ANLJpJMEVM-0kiiXOD(7`}xe6@4feOVh8(kOFx#EwKf`Itx-AWlm6 zPhE_#K#@?&$Wl({uy@goNlL0HUnOBQRTylcD)_NrQZnylDVm4Db3(%dc!$-R)sTY^ zEHo7K7B;ca?dkZukvVoSIZWinQbVAwbY{gY1er}zaGqo8GJD4nj0JB=!g`OC_c~N0 zNuyQ`85q(aDn}y(4@t@2 zn2eT^lh57!9q{iDv&<40bPsgm^fOKGBrI%o<%9B=-;=PbsLzOV+xhCl&uRW+=-Vx8 zDJfAWlRC0yXl3RfOVKX0C(m-r52E$R!eXWKtf?JTmyp>RGd5Gx5lvFb) zwdF%uBtAr>J|-CP64TNLeDU;qLP>_T>#0ep*z&Iu!T>@h01|on4~y0wSe3Q&huXW= z?BOd~Q`WtLz0L_5tIN4nK&=G?5bINH-6*bDr0u952Yn=6?NSt?P;qal*}OP#h;pg0 z3?(5-CV~X}Gi67FYVN_nErKR)7|7*>O2PF^JxaxvqpXUz5s5@4AgGf70Wx_AeuG8e zmMy58t5wljMg;YOPxye`=1_E+K^D@4^YXxx9M03~UOK+rIi8+xPvH8y!f*>f?YYC! zAgX#pbNwwEbrh*(akOxbc}Xs4>lCk%APuuq?P%RniptgMdOmjQ;Y*fMoG_3xW0DV} z(zPCLz&h2?tD&1D#G1iu3)tnYZh@UC*vgGkP0Nd~ynU`GR0J$%Q|*3)+XndqZ$_aAPV>`iE~`+K5}7+F7&L?nVO!2jD?!v>Fp{iTXawvTwfcVqvwISf za&DCpoZ5e8Hdk6k%=2>gG#ZP@95@Fx@a^RVUCl|To8?46=G(KTxqARYRI@=rK~1M2 z)z4b+tvU>@ZXNVY^TIuZka;;Ul%xOv0Av7wJssBkQ1<*GR5w-l9!BpF|=)E3dA7vVUP=>B)wH7++wKBhX~6e?+lIZpfA)t%nFtORj{fW9fj)I zlZ9por1N=Fit-&^26eBiA&aOeVFabN6c&hL^7%5YD#RY7G5XNDj%*7@l^A0Z^p$gA z*xwW-Ziueu&~_FSM-CMU3_vc=_ROd5AERb@yeT}GVHZNPp=QDrpKAsHx@~j{?PpNH zm=ubn^mxXpt1=XoZL8mwmG1U1%2!5Ilq!S*)wqS)JjXqck#U1-wY4yPtN|dMp!c#N zdnN5eDuhn`y{rUMMq+oo*J0@PF(A_}_DKrPg@k(4wW<^&*Cx^si*-(KN=j9jmw|5p z;XMukO&+r0Ob9Q=-i$tBS>gd(@*}Gs&!2N3K@kHNwT6Tm1JLI9_A;Ax)Hq5{FleR% zi;1W*KI*=Ohe$cSVV{(|6(mtyC%d;NtS4g*%QW?LYqnmoMs;~0AC!chj3N-1tAJoV zJ*)!!+fVJGxZCge|sc^2!xaM{c$VdXTYgr0w zn?INdfYv3by+VZH;r6tK;!#u_nnzb@2CBvVB7Y9|7PZ{Es*+^>XzW_&FG=KaTH?AT zFe_72@Ez|~uFfK*HASTuO|+?MIRt{Emmmpj{fAL(B-~B_xQQZr1xHssC$d_*pxma! zz1z2nAQE6u32&#v?5N^dQjA6!f!$vB>#Pkfn#vM{qTVDYFPPJ5`Hgv&>#SMMyw;b( z^*7^#%1+fPnNaZj-xn;81G8!k%Zj_5K2(vu2|51&F!XQSzEGS%4bw60mk+7bT)0Lx z6WlumA3e3dQ7(aFPMj!8Q_l=dKJm=;Xk65oqq`Nf53jVi^{zv)7ORS0YA1HeZVy3F zD8P-ZsZa_YquTX~cTBJ`74E}FS5_8PN*{yDl=1|*h*t0k!vZsPWpi;T*;#tjd%are zShlIpa|tsN%9VjKKx^1oxooKL)Q;fBwF8rj?BeQ>dDZ!CYLZRVlUzr*zf_$)`-w=1 z5GtxLBf#thsdtuc6)2o3l%xV@Jjv`1&rI2`J*2mC<+Py*QjhY9@5%aYCW7J+k#~nV z0;fHvCns$PLSYF?K-iD~>=b1K6I#^g1Gl8p+v5mMKfHK}^qjf?^zp@tvh)mx8e6xJ z4%cihVT$D{$k)6ebCv*7iGn7LyK4(b=e6~?!#fM)1g2qyEevTAV4p^HR5h0|BVz`j z0Z4c!lNGy?$^#>_5ME~qdr&5$V;DiXJY5-FR8)9FqJ>DsUq`fzCK*^?loL^t5|Ydm zQu83N3RQ&0#>Vi(Qu^G?FxdfFu_+2kuSRkm3!du4k7EEtI?a|a?@Pf>(88dmsjo5c zb}-x$lLu-|9fOLX!Q|@*CPxartSlVDo#nhb2NA-$5w~G_DK^b=J6Lf_Rk?>A8Y-ik z+WD|aG=@hXnvKwv;F=y`j@m`D#3ph)nN%kt%H&-s%PpYPJ5`H!YK{eR_fMxL0(Kf6 zZo;-O8A9zCP-IHMXBQ4i0YiJr?S)k(hdyihi$-;27XtI64R2vmf~X%YShzuXN>no) zj0&4$t6*7DYZ$$u=F(O32v8e2HazZP7%H)jhM;!#vB6G8M2B{0gI-YoGdH!|E>6G=?+ z8TLD@wTbXmT8p^=k;%s|Q*~bSe@#CYvr}>C?q0WPGjTx_+G>3U(){xEmdDBpyN7*y zngkURNt4XxUtoKZMs1m-)1b}FS98qa9-sp=>Y^LhwX-7Fo#vZ% z2AXdzOO+@G&uwmUlv=IXw}l}oNcU0z?hqVKsY2*LiX@ZT$#U{kAXUd|7XVf2MuEM; zSyQOOgoKKL1eSvCLnX5y7q9CZ?K5)WQd)Nr=0KQbOsVGQ02|q%?TiQs83zibIaJ(A zP$s3rmm?yisLVEP*j8l2R@$O^H?b0f==5l*6O&>g4I4z+26IzAv|heN4imRy$F;)O z&RtloqY}W0-J6y=G|0wD^lzaEw1`4LADynMB|2$XN{$WDS1D&rV%2csJk?gE(HC)V z1-m2(Wi^Uelql&YL+RMyyAV`CpU?f@Vkx*o7&ulA_GRHz78p{AFc{KdQ7(-O$q~fB zs-2F|WW(vinG9)BN~e-Nco3K8DNdib@c zTInfDhzDr^mIp$%7lInP9YECDN{L8W1gRj@0HP))VY;AsFjZp>Lo!z)L3#+pW1AVa zbn1d;YRF+Kkim5%aiGE49+FwaZDmq0-$o(x4BkP51+8}$ok4cyHvA6e+LVjQ7Z7W- ze0Q+8myv7-$yUZ@12kYht)(O=HPvcpz^GOr^rTBj73tN@CCDK`Y4u>SCf@M$2fU7< zl%g0)N@^HUPCBQA`(8nlg;XJ!84@#jfV}>0Y$%WzJjI~bi9ESGSY5MRwja*dkdn4X zRb{f2;S9pSP##&R6$fh!kYPZBy_K-|yke`oTNqhLSYpG6H?p0f@Kn&S5m7XJi=Y@J zhDC}|s;D6IVA%mduB7tKiprJ@WH+-3ny{q*066N|NG702b_OT0Zw*jLJ5#YtHmn3l z;8+wRZ?4w4Q9nI*y%ou*!d($N*l8;a>9`sy3fLbdMi@Gvo0M)9E>ws&|62&O30k(JD;Y+B8FT7ms4G? zZMCoC$Apv7&e-t9P~B92Ds#*49J>pO6eaqis2v0C)yU-_%2_#{_OQ>JT-3|fKG(|| zGP|Y?VaJV4w5=j>8OT+udf`$CD$S&x0u{;bMs+-mNw}y32;a5iP_at5t2dPiN7T^6o0M8@?|Wh*nN31yhqu z#P&G2jnbejt8QM`BWFc&?cJzPGw_!wZq>ayaN9^o z?>?t8$b(ShQU)~kt^1*8nycNDU}XA)yJah=oJr(m>?%?ot|nu18z47n1+W*10gCcY z5MNq6EEq$~!r_nz64cZlAVVpo2_f=~2gx{)leyrCVM3*z0ZbMFn=t^e(m;p|D-ct#ZZ5&Y47U<4+X@>409;xUg(+DEo~a2{q-_gGB!5yt`nD|C^Hv4R zmr;-VYOt&&DS~m5K_WqbQ$vLEde(TJn%Vm$yGdRU5^r2WRJ9y1RHfpQshY-b)A{p0 z*Prz3ZXmpmC-qRTD+u|s&tzn0d-OUrTh-e0LWEL9WGVrJAaiku@joIg8)zssD#OL; zifB4GT|S{=;9RXG3kkdCllp>TO3q3^Do`gQE#DjPCG8nF`Tmoude8X}u znFk^=Bh~FTqc`SvWLj1kWl6@J?`><7XyY8%7x_y_zbN8OLD|XeD#XIoWagaSQcAO-W%CQQ0dtTn1cUCbsmt@Z=H-Q1QBpXO z?E3Y2l9Po@*6tJ@VCEPJHzJ~p#Ywz4vm0Msw1sioAWysb+SkHUy@v3^q8(x`=2%?A zp@l{j!g;+L9D1#|lsL z1{`h3l$H^2pw*ZkyV&TYD=@=JxXb1-aFNOM5s_fX@@ymk6KyF55|{!rpdXoOYV}B6 zvV_T)uUYkAx}=&*Sy?$E1ob@LUcy?TlW?4$pHK@pJSJ(+~t8`OH2WTb~9ViSws>so%J1`7GWnQQ2FapFtLFUngH`+ zvh{TEI@5>*^kK>zPl8`6t7K&i(6a$R!o1m&^02~$yaa7zQHMfEU<;f{hm>XL_i#Wg zRw|=Bm>(4(hTxGckdUSuf`l+GoPv)>YisFMRR-Y_!Dd8~_@U$&U}w&fW#}-%@@3QA zMX;c{Qk{FWY)aCTq33r(NZ%QF%%J=~ffSD;8YG6G>VrDp9PGdtb78j~)6E@hoPo(|ZZS zbO5D}6U{1--qG@O_;FX}jgNY%)D(p0Z2>(~k#x%Sei47nwyQ)|&Jl`cLdu8$YzYt? z!}9+C?BOJ8Qgcs~52y!~(@_X4>kJtbdInffQf`bZq5Su}LAIz?T-~h`U9U;F6fj)qSZ@r8 zNf7WAguW?NlS-xT6L3giJBmdkZ*v}5J5{qUHA@Z)=KFYKU2=IrA%U^Ip(RpKV7x$9 z;_?YWDyH|+yIJuhu)tDRQAvBm#%(1-6{834 zyaka((6BVTT*B`r5@m&@L{XWEB3}|wyf7{hr;{nnNO(#6r=uWPIn&jKVJh7Fh;IzI zQeH@g5O$JEkE3B}0W$GmN{^#9paj`fd3!^jS<P&sCP0rPbF@~gHYrIyx${m_yT2|n_; zN3f{+fe+YP(P``fg%W=9s;TF@qy{V0Rv*n?ypI%Clfo>Zm0bN^K!`~78+y}opR|Ob zFqDLWkN^SP#ceqOnQoL}4qL5cfx`vS5!JBQyYqB$TSXIBnX92z&KO{_<{50!9ZMbE zjMnIu7$UOxC`o!=Mi`KibE6AUmco8kFaT^JBMh($?46c1a!PzBO-7i~4(hSP2M1 zQebWs*v0k**C*%no+EO|nN}!-kO9bTnoW5(hmY#Ok2PQUcK-l{`yM;>qu{_N2Ob&4 z^Co5^?&R{V5;VbARW+)CBbhXevv@u`t5prcfBSa+4kKCOH*YA+nndA9?gAWJ*H?D^ zPWT~DpRFL#O0-A+0MYtrqwyXQV66&Y-uI4Yi9$L*QKi3dsM}bYVhW%UmKt`yCw5+| z)%nZfaVD@?00D_<+Q^eHEv<~kCR0W=39AiA*!YQJPHfB;QdwvcH-#$Vwrk1RA$%7E9&9P# zJKKeJ?*oWI*nS@d3X~KQj8Yiz)$s&2Z7KvEtTPUo%6SG7A%MyO-RSHXp>-q|N;1Pz zLpoF%b$SG_jkK&PK9032DzvVi(CH5De{A$-5y%7(c|m(#qT!&YqpU!X8Pjb%r@Xy- zT|tX0HOtkCB5^N{Gb1BhqLq$?kO{o#$Q~OW~Tm zTpM=I0z|D)lxNKR(YFkc{g#`-ZJ^@1uQ2`8@T0$o_c^z6@z-8W^TsfnX|^FL2^@fj zx6a;MWVq_BWbm8OzrC5F%hNoc6zxM8xTLQkVo*Uiy^ zkO(ZSYM02JtTLf09*lrd^gChO7>bz{lK>LH*79yA=49Ord>XnC2bwI&N6i28M=#N&VL1h-= zQff$)?T`pL%S*)W>MrS5YW&xTR=gczc0&FbUP_%4<-w)dnfw36j+>7YoV z<|x_H+7^%oP$Ax+fFH^0yL)|Zc$1H1?OgK{yioeP0|{}cx<;)DNut8BZ-7X0M>px8 z9&7Z*VbnaLjIN*yP7@*M8Dt_u0S4Nj?O~#*c~ly^%rTY&f((x5DQe<1NW!m}XZ%Cp z&`xfQ+E`M5JG~eh<+Ei`vtj1)c!PG@)2g=wRCgX>tgC)eh5=0-fSG%VV?6od!1U)580F|!7r8ykN!rlcMlt`k6&P)nGEq*&n#GRd;STJ!W zh2gS`iPWcHSlRI0!n70^EHg04^9DG5FvDQOI3a)#b9FdP7#x76xe#-fK&cEwRe>#< z)v<&crGY3pz0@$$RLLPs40z7g22x|u;|n4#431Al0wfj-6!K$bAcG;?1qKl)eRAE) zh$%uzC-b%gb#^Vr`ILF`11KG>lp_jUXA!nMH7TvtVn1~>9keI8$)O2A0U!W5 zJv((-FP}bTbLY>Pbi=8al>$q_IA?PV-ZKHxMFR#RrFCs1HbZGpNQRUT%AKvI8{%!9%r~P@2vT>c~QtSs2ZcHp-pg!pw}A#ay-m zXu_7!sD_+Dhg5W$h93|mI!lopKqI2m5_-X=_q%^Mo$W6c)eA^XyffyU<}lz=4pCXj z#2dqql-f?@faP;=niY%*-UF0MpOe4z_5Oe65Z35ZdTCgvUXCCAHu_8h9aa1<)<{#V zA?u&lu4mNi^5?6#^XTxBYNe8SO*_)%biN;FO(6*tGH9Ci0NjAgkw>1tUb_CZ$K}sI zBY*%n%~jD*2eXaqHm>j;W(+Z3X#-N`a}?*70d(N{Qj9iCQm=3 zmC;`Fmw6!Ypra1#Sf%>o$dE~_GBuS$EXh=dK?BX$Sjw*6@W^18;V(o^Pj?Y72vkt8 z$Vp{@UA`fL(z^?kle>Q6$~x3w-JG4t)91ATYO5@(jLFVMT3wi{URy zt6g-XBUXf&ZXiQi*nrFnDB!WJRfnOI5{_kpmkBCw+7OT&dE#HGa)OL3U zlOqdsjGl;iNvkNWYk~`9g6P`A2^(A=j;2MG`Iggz7XpH&`mIqkc)KY5OLZm|q(*=edx^z{9FtR`rT7_ih#=D9ttnO#AXx*ux z?<@&w<_P7rwzyE4Vz`zhs|^@hlEag4HZ`GuGkxX7UoxdrLcmO^hrBFAVJr&)O>35( zf`~HCc!LB2iZgAZ=TN}y78`e40E1QmQdrwru_-JUs!WE>yTA5ghn3n? zVoL_e1?8=3{b5a(9Pm@LJ-%QgE`f$pCEz9t0zP8}?pRx*VbY8+lPok;^PA_Si7X0{ z0fgqtqed1KXw@}eR>I5%I|CKQLrG5$N01p-U_2p}vqZJklRDH0h6OOqw93PEtz=cS zD8ruzM7sY$}$MSFj?BG989LgLk-NCt)2nt3=0rTAV|&mZ}z6jJl!?HixQBu z9I(Z*6gSTqn(}_5BK)`eMoP6-gqRjV17IaJmF2MZPhohd-A8IQ3N9UvL zcMfUpEF$0^B1froS95Bi%xlSaH>9|oO|W@}`qrrV<)v_$sF%JYmse32B^7>;!8Gbk zwcaE}>^MM~7z)9~Nc4$tSJN-mcxr*s*@1==hiAGfNwm!wsZC)e~x^`VK=g*%o=g*%o<_m_b z?Q6O8U0?-{ zv}SrbxXRtLM^^=1+I@9`w=#Zj#`Nl1tzHsF0);^NaH>Hjee6&@AEaB1@h|#Yoj+(j zbM=n?Jc=z;7Q_>BH%qi-*!TYc5Pw76VjH$VAxg+>=H42MXh#yng^CRag3kDV-x&HtM z)OohehydEG%~TYn69P7RxSem$)V90}{;KEnd|$8l^GB%^4ST7BEtVVEJS(lLXwDjz zv*_ClHb_X4Z1`-1w z{SQb8+!G~~gTLfLkMW-BPOZjhVTwJ{W zS^fX5frCql$1X^vh_A~bWSvg)&<~&_#&N<8$HUM1zC=KD}S5M zb!aF#nbz#K*)cPP>UIil#Y<{T1`;j@m3Yg$`Q+a-izS=`BHU<_x#^vidXDAxd$kb@6_`g}7PVafB+o;kxVWQBe{{rlxE z2BX}_-vJ344)R3%V~}$WAJI#n-v%Wt2WSid`$>yTw|W9ZW|LQdJ@KoqvvIhDsxaFN zj_zK=V-t#CI`mTu2d`t)Of&jS1RZ`Zf!UIK#q zoOH)Wed5MXldQ1$ypp6nqd`B5EV#hyyo5XqJ5Zq3fMx@<8(Yt7xN%V=ChFHdi&2!{ zrO7!T8=(Y1rvM||&yH2GWmK5AU#C8`Eb^G)g10%;EZ_0_f%73>LLLBwl&^SreI#=O zC3j$qbC<6GD6-Dxl|7SQlo~%ZtK6hEcUA1=iL<1r__G4_BxUbqJDBK5-AoVATfh*i z9-iCzM~;h~p)vwLeS+>resPtDT~N`4=>hPBsDRwSowvVi7)iJW7ydSc#o?vJsz;@G zK5vyKZa6!;Hk^$)uEUeeS(P$KJVnxwgD%d=RKLqCPXcKoG*)dfpXh;04(U-*l8)>DlBA5y!KcXI|;8AsG^U>&rLXj7ZeW1=OpOkJZ=4wWtMpVB3szFx+vpaG0PP-(wwjEUhA*#Yri0 zu{a?opDn7fUZid&O7yzga1_d>;%cg1!rtE#w4zA=0Dyk*k7mhrbF}!aqQIr60}$%x zMNsMI^PFmAl8+WiT-s~a(xZ835!Z$TQzM%F^ z6r4~=fl7fG&jib=xBqM^yo_TzT|b?jo6DPb)g;i%|GX5Wg3*d>t;k!i*d}BJs}%SX zGTDoJ+>CyVCL=&ZmI^)@CrJ$=n(fyn?&c6a+Ahr@C^#X=%ld6lNdfpa`LRY2x$9r8 zk?fqEe;58utE?jPS!kQ0lA;*G@2@bm6cPU^BBq|ZgIDK~N|J9Xwcr|Yk(5NJx=ztT zys0*ocM1(~S&(^S$D&n$uT`m9SA;Ngn)xTk%Y&10e8|Pg7Sdadx^wZ=vpds5iLEY! zK;jRdoS>;(k*jrR@dyV}R7k)b{=8^uQwhyGh?LA%i~4%F0}Wjl^HAA16exk~Z!79< zQ7c5B5H>aTYUTyR5`1N(jDLzYsn#U$wtnpX{1&XhcyV^-z6Pd0rXYc3aK3yFAu@tn zJDt8PegF)^@dO_Lu>*5!2N|yjxLp4pAyUJci-DIH_8p-hL}v!sSgysSSbvxWe#2U)kVdRxGDJ5 zydl&W|6z%PK9_B>=Cq0oHNv~;!5Y;$23)ENia$oBE0#6 zh3Kii_0K14B{aC)G|8I}jZ$+0Nty$r(5M{efMWWyF717J*7lHc2N#5CQHmck$9Zfbu8(Vdzf9#u9 zilec$J-Bb}2)G0}e^!d!oEB>{;;h0YcYq<+*FbGU8UuxR?&q75i9J+3PKWFSY+Lf{ z6BM{434=U<1$8#R$DX$>qa^B-KiJyEq?W?Msvc6USHD}g~l^aOJeM9jSA;7 z{2`=JbtyuY>OQn;sgwaj^5a5yEo@wV*Yto2s?(Uwl)qUT5`edJbVmS=?Jw1@c?3Y= z#jA(HdpGED_#s{4MHW(oS$F4K2;zo67dL~;iGae(njKM+ECAv&2_;AZHrD=-0e@4` zqpH$)Lwp)2r>OZM-kANxP}mAWCl|8zQIuMKk3RL?+|Qi<;=~ws|9DOK<=-10aTYhX z5V)?ZLf#y>KLAJ$nqHs0NsvEqY=*iXF;zRm!?Yd%?pMhlqJ0s`Mh)b*>kj~BR2cQy zxs$rCXZ8e3%L8B_b&GFeE*ukX+4HA>MV`FL%^I;-tcBHlO_cQ0qMmeNujs11!}Dxu zv@0Y&iuGkw(mZ1r%h`_#{>|QOM#`cGKyuHkEqT(9Z+gN+s)~L?MuEEF%UqOGIWT$P zciUeb|NZ%J=a%)Jd=Br=iwD5P17NrDrUMDC$tC=+jg56qDex_0hRD6xAdR(=@LJJT zt2}9|U)OfgRTo{8HdVqGQw+07(POf7+Ot|uyNz@0xsU&@h}o!r+?2ffGF1A7lh!P7 zZ?jVS`SR6hiwCttWaHv=9Quam=PQkH_ZurJc>aqtIexPzv4@KdU%qF&JK@iGBN+Cf z!ZeX2ZSyKrIz2N zz}V=u?b&V1um4%I$g8Exs=lB7jw}bgNnoD>zn%O5cPG+Ji>Z46C@NLn))f6Mvz-qS zg_mP=MhrjPe!q-Fx}1KhJfoUUuSF^KA{6q;p=9<}u6lj+8X;-ms7U zv|nURz;$?wv-`0!416&+fA#gJ4Q|?rp66+cyr^6i4xzPOIO77)L~Bx707o zRG2}&4e!UM=sU&^ZSAwaPmV_9Wrr@EQ|&I;7Y*Q5%X2*hvf?m!bibs5_g}s- zP#zrl?nd!0NVJwE>f%7QV05%3EN12TP?}a@0y6+*zzuRh%#2OSMpn6-fPwf}=Bto~ z_;T6DGKdF2^;AcK!X${?*;5U94354L!w`wG(LRyQqd&P(fjl$Q+QCti9{>BJN%8@} zZ8PUlvC`bxo^0xwk{P^EF?QfyRlm*6P~Ulu zj?dVpS~W@_c4VfxZ^4vq1WJm6n=sr`@410CeU1-Ja;ck#y6%eL9}wYi9t(mVFJ13S zE>-W$=8B6&8?8VIL^p;OKXo>zSc|vfxPG?O1GzbULk&x`bL7oZnRxqHWlAWbWh6zM zBmx}xY1Qfh&>K$xWLA%_27`7yGc;T&R-G+{M?9PCHXwE^W=cfIFdY?vGF@zGAbZmH2q!7inc>jO4BzC#Fj%uIy_F_|p3=)PhVD5neTh;9 zc&u1TpfD+ptc%r+h4kX-SE9$)31~tUVrt`tWQ)F5?%xSWCxy)16RW00&gL!|`3;D` zClWAec(W8yjio|%enqU@t2cVSPni)~4mfLKLB~s%Q@@SRZ0beEeVf4C=!Q(%x0T__@2MnW_5CQANIbOM3?R7-+)< zvY-J^>T3mamCw?;VNr-uY7Pw+=2GsDK^af4;2k(Dxt{?5ZrX`hgE-!B>a4T`hb=^$ zkPJL@{SV8RYP#7ZDE-^l%^CR-m`F!eIb58MKq_+3g76qJ^vUP@r=9Yb-nLnNZ5?DL znbSti9lm2@OEqw`;q$Gj>gjj8Mn1NNpI2h*KzbocLY@YE$&`v}%D#fwNDihFpjJy? zjM}n#V3z^2P1$pW;TYYi4jn)p5rNug%DBFUU>RYbI!X;(GR7jZS0a)yomdK&z{l*e z|8d`S$|^P1h38-U0BH1V>feJ`#)duaBy zg}rL%&r3?E9|?N9dN4j!4t?XrI&ktc*>>mN#bi4ASIe5>BZ-8u7?5z^;n+xC$}!{{ zVDF`LwT|>R$wLCTa9(sPrTJFIq8Tp)zBcJ>0byikxP0!;r915KLEUM-Bu%E(j!Pl} zI-ktPNm*IA3|+yG^p{}sJA}a^;q_$7kWG-@%9I~ofq7x+tyO+dt(+I-;iq}>-vurA zFRyvOCR`=z%iyD!P>&Rulxy4+7JBd?pC@^=SYGEiDJ)^wJa-&I+5@_jH%46vd)Mln z={%eSY6ycE1Dy#`OcG{8pU0P;-e+^a^I6{2W>yi zAQq~BLtK_s6i=DU?VOGGI-tn0RQH5Cp5*S*nO@iQ0U*v|p}Lf8c$~qV+Z(Msn_5|| zC4j*%jfH8FbJ$M-UkfCus*tvgE}UDn;8m&(CH*bkl%@UMes+^JpMs-9q+BewXBHOW zf27a{=0}052$v|Sn;#Xhq_acuaY7o%z19Ml^Vm&W9$mvBC3Cy9wiHf+5 zV=~Rw)G_oDs_sPsiB~2NN@*s}YjKMZ_87)Jrdq=AWWt zoXu~Q${aZGmwDd`m^X}8N$uxV>(2fq=;-cNZZ9l7)%BuB<_iEJ=yH|M44QdAkT}Ne zxnnTscyF({bO-7CY6ndYTN4fK1n<-5znv2Rk#=sJ=e{`q{8=5x-3gVIwTMn-!07gF z7s~guA75AYwqYI*&U!6NIc>gf^xy)K63Bj32=Q6~nOYzoKkxv!t|YF8UFZr|>=htT z5q?0#KN`aa^2UbS_|^N__U0RD23LW;j?h3N-Wg)IJMASR_sMxUhtWraHkqZ`q9oCF z`h$5%#~z@we5KZGP4ORl%Wcq_XZ6MygaQPZpdY%_?Pe}y6d_dhyJ>5ff)`hEwpnaG z5uGy`1=0A?A|{H;cYtld3U)q~Et*j>PI+X^q3l2HHM#gh1nmV_H>h2AlO9PFnKR)4T zXDP7mPRnI9j&S8rz)GRD7g&8vy&0bkX5lsMs_IyZ7_k-Ep z=7&bERo(8z@PT1H2wEkFD1>n+zp$rG-*!EkCPz9!L(XLpdyOzye+}4*eyYeJOo^Rt zK*VHjD$Oe}aYXjOi^_n@jVJ^vLxEXGcxX%LlTX#bFxdBVV>Zah-W3EaH;8okv}CCW zmq6=C{+;~~d$o)kGor}Bw)-K+DiERP#^i4>nCtIMtuaPS^mVF3=_!-=1EA&A`%X#$ zTopBrQkj>G^F7j0-A%t$jwZ~%z9>QJ=V2K$YM4jxmWSzfQ`1!nN0;#x*$~wfBmq)| zxwGnk!7`mXgdvL)?mSei+cOUZ(y-31zIH?+RrF`03YXJF)E@wMp!7vEQ*>O*F-%nv z(NIO}*fRVtRV_uF~X>-#fzxCkXsaDB`P{9~xi9Y?XmNgjY*^p^+s;2&J|A$O_ zC|^dxd$Qr8kUDjDs$j@$VF5T7{)0}9H$?nRHZsxs`BpZ28rA45f|u@E>8jPxusjq=ia7HW;HxsShZ zm_OWq>cV#SL2=Fx*stU|9Ex}8ddhNIvY_NnoyhkV72Hz6b-Z~>Wf_rRR&&2htjji# zLz4BYunv`Jo-ziPDjZL6v{ATtf+#2sD=*S=Qbxo~jMVA~Ly& ziGtHz8;A!zAeqdfayydCf0^diqdaz(9O>g1ycJ(LRvrLho#7=!qSb=U#a+(tt_Kgh zW4=a*TMHXAzt1xXvMljGj^-7iW|(?$O5nRfNLDzIJf>uDDA!<7*qp6^n>h#oc>Ap1 z%lcj%_IR%#$Mtj1@+6cv~B>bgzOb$lgPxqR` zRD%~jRS22%=AB!i9k5ro$6Oe{qVx+DUL^q_6vaIkKD2su*0C(2wbD z!!4er-+1AcY${+C^v}a1b>y0hqZR8hTH%wMUk$t3Y=`lGSZWLjW8j2}ac8Zy(er3j zF^CFB@D59}-!#SFuk%8-hM`A?M>{fJ=*C3QQM8dS9g@uZ!vHH~pQu`Y74H1h|r zA=||>;eJxwF!Pxr2Rl+fcz9;7ULJg{KaZqK8@Hansi z52k`w?Oht|V;rk`+$EmXRq;_s0;uwb`$Z$v#g>&l9{{@6!Zfs!z{zk@$%$jf;@=m( zvFoNj$G)<%*nYT4{^w%n?gY*N??|f5@XYAMvtmGcx7fa#XTFqnC2BNK7S~*anBJ9W zF`Fr@{v$R1!lY|6%!4}n_q{14L!gtnh=H?$%J(a32Is;xS6KL0K!AeOi;4u;s0a^e z5>Jvew&!va!vKpYl49Ou2g9b*KNOgY;2xWr+b97E12TYLNa&v`mmJJCx2{uz4@v$V zJmw9KOxBYK#xS|mToo2s>=+(yh45;fScf%$8On=Bx=J^D+xP|p^iCQWz2&!H#hN2$ zh#Gvjd^d+f+8ZS=o}1V%&)vcnqopA|?Z-+sMKANJE1$`7EhgZ@;rI;8pDY++^4e!n zxNv-Q%WYuS*(LF3pXmm2xx%74%@VqCeiFTpMU$G)kkugDC0^G@s&kAV!+fYa3$EQ2h}6R(4!H+27QD*#h(rkEP`Z_(URqYTVuYN=r~(Jg64Z}hM}Z=~jjWgs>$??S3^(lq6gH)cg0E!d0`Lo67N)+=uFP6* z{+xj1m;3fXJxDnIGk?-l(EqV4ZCXPKQEG z{;P^Z{d6yQ=2qp`q=3eS!Rz)#%yvdKG*SZ%;;32t(HXaLP+xox)=LRhfXOsYhq*lf zD1I7j%@dm(k~Zt3BaBB)JIhkc6h%fQ3TOuwud;sW^Ku+|R4efGa{}3-ETS28J}zH% z`SgB``_%AWV&K9N`x_BT2XYT%w#4%rO zcV2)syzWQWCl87JL`feHQ-mE8Fv+q6Z*P8pWF=5`AxGI9-b*o#g+eb zA6YM41lx=Wg*pFAOF}mhr#u%x-1qkSyW~tM>d%b+Q#h2$AhjY|eL{ACGxefE-Ne&2B9Ww2OpLige|zdN!zvk3iTt35)cL^_-`)d*CN4Um9+qy6##7}WVH zSHT=MRMn}01&UOiPyKo{xJCtB`_CMgN-=z0g!DuBSMJ|$-J+96hf8G=k-~e-*!xEn zv_|@%H=argM(H0r9{`UE0^#};4q_5dsx-Ei$9gRDztnDRa-lhPhlW)~|7pz!(62-8 ztYkSV@uJAa-y24qow1d;J30rV?DfDuuY()ZB0u0{2rlzKE^O zI>Qja$!S&4>;oS`G`xAa`sg=QX!*BxqKJ6-UtX^s-8IhWY^}^!A8UQ7NAfSmS|53y z(`9PuTFAMH1K~we7k)A(3%0WvG1e3lo4Oi_hxFm;&$R3cwb@vLBHj79Z??{Kqj~@;ps^yCQqPnmOeomE)OXAqD z)ppFn!&0hA9By@L*QrKTPE(ON`}m)%f2d!q-@2|F$))2tJv@=|A#SCWhStxBjX+m| zPc&x)-0Pv1ZVaIJ)`Wg$aobps!WiM zJ;iP1cd1=~k^nY7#*?N1%aVE;7{%9SSx(Fm_oQel@LHx zT%+(62);wWxo=`O$2LsmzzzM$)y^Qt4P7YI+ha;H7agu%+-2_HI(wT9qmffmICDsl z`})Rnk>|;zmOi13%*})twfgA%ASo+h?jP=%z%wS|MY2j zX+k71-p|*}U2cJ~n0i&=v;1|dio)wVF$6+wQ^S4p%1S^}W5o6qc4&@@aB{4&>)F}3 zo@vg+InmxGFD1S+73Tw>axOd`$l;2C@P1S2!|NM{_$`Lym5boYeTb{%&JG^OLsb>^ zP2wXkbm~Q?WVRFSJx_>HC5mfo70Q7Y4A&Y zyBicqBE9w~{^$@NN4N#Ogh>bvGTTJx1_GlK!8kJ8cqk8a)15SrW=}jlHU^uR^5V4o zh3Hk_xM4X|3WJCA`&~BM;yGOO7*Nf&7@?Pgx3|0YVPWSn^iuAeED??@lZEbWjJS-G z{M9`gj7aP;X}BeP0H6~IhBG3NrM?$deJOC5PO-N=h1lWUEnPVkpOP`%@h(k2r1$GH zLw-}xA-l0&^?2)qaG~{dSA^`sQ?;#Lv>O2r3x{Hqq%>|>b%z%(fFT@?^_iM{kL3xc z!{~{iUQ*p^JI+vwLTV%Rcrvt6x<7OTi7{$`npA6c`Z1$b~K zk0nLgVl0&6E6IPEr`LB&*sN8$0sv;Zdoyh7!uh?X)T=y+-d_{~n{)Cg$;;;T1SS>#QYHjL~r(GkK4Vi(Cfk!Ctf+n}R5 zR?oXop+PFlR6uXdON%$aM`-`+#$uF&z2hV^SMH}5vnKd77M+SyF9r!iH|XJ)WcQl> zw9~~r&wAdb7Idyz3Mrf zYW#@p%&axnGaI9E+r)^TDj%t<%dF-}hzUFH_wib6EKg?)1a;j=S(cu2w{TLF#HaW6 zpLth9+IJ?c|7uH3M9^uHWL6IIorN;J0U5HD*KrfF(j-8z=@1M98X(sJKbw9BLVG_eM^J(p7(bGj_^|&H8EK?KQT;0@ zW5X$?L^0y1D{%LSuB?mft^B)XUf{Y53FmhF>@Vg!D;*vJTs)HYdJ{B^bU!r)D+f!u z^%!_i{s)&{%~kwS`O~^HFV!)G{x$e|KQ7!MwnpfVQ%~y>Jrp7L=r66cHUWpwVaJBT zhRkie=)2^yy$t{#B=De|Ze>50bueO@_)9xf_V861^ z^5nnIit{`&|FZU$a7vN&${0Kn$`5ebiJ!hJw;*^z!z%h@nOGQ3t^k#0)A}k#9xEgu zZnU{mQyL*>_qHrEP`g4aU4*{(;+%$rKp=O@H|msXEi(!`!n&WdlRNu6Ymvq!z&JU8;HAgTxxys+-oakeO$OgIbrcQWDYaXY39P$TH4(Vs$@1diNp&Xa#h zs4*%tykE#?r4>}hHg8Fa?Qn2Kg}HZc^9)M(@N++I_I$`AVY4Z%JjGa2ARQ^v*iT$! zSeV(WKuA^M%SBl(a1^V_rvjo(fo(s!Gpu#JdWq-B7lfNRQD1~_RO3))ZH%dc=W`Q- z!MRnJryU zW3O=-q=2C($&%g_?NV*|tMW_u%ilE|o!tH}+gG={|Egq&uSoio651hctTB6c&a^6& zO@m!NF&b>IE`qbWx2LhwQ~z5j0;I#_GtLNT}&9M^-iqtU*hQO z{l@{*hH^xI3s55g?~?jo8J6_fUvy=M8~=6d7hN444E+!cq=^$%uIT-qLqE@dp@Y79 zW5i=^!DUl%OGE9`Yv1XLO|NA$ZKphE)ka?>C6yxcb~%f}GSV;1Rm2T*Mjs!8Xp`a? z{*)Dd;>PxBe%HxRkV&K1&ueD#^MHbuBrBiaf|}USguKYtLi|$`3lWlf>C)WX95X`= zR@zQV4tB3oeMdh<@QV=K4T{f*-7Lj(@!}VrQ1B@hGuQlzDs2|VFuqP;3)Iav>wue~KppCo5srI}St2i-)KlGpCS8%{oGhaIU@J zAgv|xC%fGZ5u%9FE{Bi^>Z36O;_TH?H|m%ER5Zj}4V%UiM49d)XE@)$^ig{mA;fnR zYo9jau){Avm?s(MLvf&)FpU=*4*U+pT84 z>~|iLsvQPu+746;Ps`olZ8K`E1qm#88nN$i_`7byeDNkv*wTnf9i^U-Q>!Z}xv~;{ z$@wWzqM$#{sRto?S7xPxBtEw_4OT?jS{LQba!OnjeOiiQ+-H6HMyyk-Xu?0m4wS3* zEAS6f0w?G7#llox9L&^1SWoNeD~Aq}HIqyM1~ zCVy=W8XcRYa-sKJjw5{(7q<7vb=vcZ(oVGma_i;af}tjm+;rpRtw&xsEdDJQ^qcls znmXxg)s+Y`zaGTVyfX!7-T|6&<);_Iof`hB&W+6P_qd?USJ3!;3`9dIB2@Kddkb!p zWIU_sd(key^uyncPCuGI^*JJF3 z-n;Ak^8(q6&8Jl+ky;s`*OCviB3KCk@H&S*k# z6+(y1GNg$4-=k7mm5|Q#Rr}rdC601tq8d85zQoGm7b#fa3g zvWu^^hR<-$pBMh9!1_J_wn9lWxm&w?5-G}JaSg^J-|S!7wo92kM%G$LsTd_T))PZV zD90JAE$Um3OWpzz#=V)7#J_sFG4tZ%OTO8=#nqlP9y2&ZIluYW*^SRNr9ULCF!GH$ z4e)b(dwE?Ei$o8JQn+4(@c@cr7#S4`)89fsB0K>A{8(TK9c ztAgi7i^fWZF{J>FcX4dO_;^fFSI%_kO&n<$h1O+Wxf$ca(!TsZ_5PEWdUi*)6f=S_ z@GNfm;4jPET8&Od4cv&T(+__#Q2A{h8GV)D?O1y^t=)R^c<3!eIC&Na}2s9>uC8#O{D%(2foiz2LbZ34=9xD}BhF|5@r zImMjW@3yw-Qza=Jv|34us~q}WU|lUUCn+wX1>xsmPlqGDSv5TyiIudX-D)!B%I=>H z2L5EYLTzj_51AmQRsIo>6k|P3Bea&bwb|Wcr-G_IxerOCxE4H2vJ;ao2|6RweB1f= z0|78d-01SxGixti`Ms~#6sG?%VzCz!LudX>qYRDu7$>E^z6EWE0UN8)f59=@_@i@$ zUeWSEYjzAI@W;MnV<3|$hV)bcK5;ry0%I(a3S&K@!I37rJ6qZUvCM1Ozbm2*UEeAf z+VQ!?+vwH+{UPs|ZB#1LDJFk689=82OwP=2*Zb3UC#oP$#=*M-zbK90V6ZFTJhdgM z8nZT4EV)a2UGC{3GfdYg7SiW@GOSZvEgbjvB|{n^zQS9{xNT8MFq$mSDREFHzWjgSWbDo9mCO7Ht(ITZV|gE#_~`f@DXQ^k>qfGC^tX=dGi zJC8UH&S_fw@9O}t<9B1zBhY^O0|4k0BA!;xpfCas95Tm|e8T);?_Gp-{Ab@){31`* z=>zHEz#6P_t& z>wT%hn(=428UyhRxlf+I9z2vn`+LSHMLQA=+tVmQxdD=U8b#p^UbhSk-n9CgBO z-b!Dep)$=70IUNij3PU@!$EMk?X0ljUb8VeUDz{XNAD}Ev-ixPku9*xPc#SNO4+3T zeJ`xxBH7l!mxgunR==S6CEup2?u?PDJ1-&RL+2j=sg5*5VAJX)zAQ}DY!RX>7ZQJTblX9b=b%7}yUl02CI+)n^rJM> zXxhurus-?qC&RLjwtYXP-kcF71Q*i3x6mr9If~H^3hbnN$Atjx*GdJM6}S(KsH9|s zWN@Uqgc13kr?S!vm?TKe3F-a)w&U83(+TtP!hF$idQ#qY*24y9-JPo8hT`?jW9u28BK#de6RyLb4r>LbP(&0Brr2Gr7LpnNLS66Tqi7>anbH)Ph9u?Cb2b)wu7+p! z%}pe|WSR!fB1j2?iM>r|`lP!R4`JC9|BXu|QZOYq)_{i^#FQyamUf)|^0y&9frP%j z3+K-_%cxm zSd97JvhZn&KOISc?^p+@Sx#Myt(k;EaTjdAEyI+VVjUIw01&t!>|dF5#LLbSs@fwg ziA{fsFh#$m4$;GykwBSmp48$`+tHkGkS~?%6&wEZPaO4I*ez*N^)Gu^%HJXM+)j{Q z`Cv23MBX>Izb@B7{*gz2Ie6 zibsz1l65zVN@(KK_u*P+ZpWy=%ZoHvk$n! zDNJRt73cp<9cQEq86SWP)*1PT&`NB%jNowK8==$J!So3IQePjJv|ygS?R856di?w% z#J+Z*l@7%JkMNCM$Jtc{=rcTRjf3n@p=OK1O? zx4}9`QkG|!o`79fhrTfnk-1y7Hl5);+SgxJr~v{s`o~u-l8UEl*hO;Bl|Ej1N5xl) z#*$8}4(2st7o3*Dy`Jn{w9SmwL=d#%f~hs>tqf!0pBZA51ZqWUIYx7sEUVew=*vM= z=$!=|vWhvi_gW3xvm{0VSwN^ZEpK~a-9a}$5SQ7PU7V12*sp-0N93UJGikPqD$gQS zu10%eqYb(wm}YIUmMl=F02!7n#tCUP;<%-dNo%PYEHTd}u`ng!vHu-$ZU0yiFgsmm zPsnrGLWy80i4e5T;E@a&>Wmzoy|n(QtDNYJCd-5XNbAy1n-imrSrnWL$mtEH)Z11Eg%hJbci$WD+Tn@~bbzS2>>u+h;Kvv@XK z_G-mMI!4U2uLsz}YgZ{cK*(819YRp@8r0DQ3A^;5Qx(3kD%F#4dNU4_tIP1=1u-4QAz&r;hUHIxL zf5^R@Ux326?Wu#Nd)1dw?&!Nd2}g-5tBREzLdo6lCj{Ia#UaWbr(0M2G35PcML_~E zeSOk{lAG6$1tcl&(|UN*jjw!Z(oKUK>A`=$iWH^4N|3Y}4D5CTF9}+x$_^0h;obEH z^tqn&f2Shxx+Hv0_H0(CI=^eH!vCuDuzMGmTqd!5^@g_AGk$Z8xlNsy0AiTM}MdU71VgVIdIx z^aOwS&{0&ODHU9ZfLmNhbpEc!^B~G_4FO3ZqY*+;WH1yMocUEzt7Ux(xtvc!Aom`P) z!sR&W6nl=dKHL;TGg)M)_b)PJya>ura!P?3bhv_e4FgdF0KUnBRl*(b*!WXnI*O&l z024i^D8;oX*J5@Sy{gl9krc~HDUqIjrECVOQJQp#Dtf9mJ|5Vb&zVy>m@+qY;c6_R zvXu))J-P7t{ZtP{4ecV?_hP2&*iL?3mHykc5npQ@cnL!zRNpEdn1UKEg{X8WjSOEd zikjMV3$U>(#^^*9q4gVt`17J`keqpV(Q`W^96c7QlYx7ikul#xEOQNh^b;?Ig9MqC z5}BA{zBf><%Y2{HQQ^`0Na7g*Ena&ujQE+sE^VOjTH-wVoWvP7{Ge|PkW>{22s_t zHe#{1(Z`zHVV|urmW|1v`fOvU_8#LavJHel3^ppAKr-rrUvcNZdLZ{jp-EtJHdImf zUnaZRinmJt!gY;?j6FWg`jWMAO(_=4^Q12(U3Gkz5OMHrF}#~c@y7MjD`*;>UL;x_ zVIWcq$|_am^tiDqqoTKCnD`j8MV$?nMdt`jr zkNgo1ul~0)KOjgjeI_Zczc;cRNrcA4z?qe}Cj~&L*7?l2mxm;(&SSQ~n{)di!lEc7 zdMwsj5%2XZJIHT|xT^$D zW~v22*)ZR-DIM<|yhGB2t1ud1!&H(OQ^T5k#K*I#RxGRL^R-%Y6V0z_ZFz{_n+MC2 zW+OAgvN;<1y$c}K1A2ap3omnNjVG4&)L^FCO=7#_+*OR@9QS~s{{Y1Lv%D-238{Fb z%r+_5+z1R5Y-mZEGCIQ&831DagC2^;yoeeg}- zMLVWk>tv|ah9vRwZE(ZVm{zZE2uLQC`mm2}VKX^0n`779lR&toCQ zw9Z?bgpFv*v;qRq)jUk0gw(-%iW{p6Sk4s&3<)97aZ^@+!z4{uP)VWjZ4`G5I>D%O zaZQk#r0*IRB3hU^Lj<>YriPx`ADCJs#DQp0k3*ARJ#FxhHARUJ?SOB@ozMZV0K^I~1JuD#W56mkY3 z&DaHmWT{roYBsqkxCo(!gkj7`V^~Ld2kOF35Oel;zMWuVLBu&`H;0z-6wH8j9en=) z2I*RobqA9O=~>pC+<<4BAzByTNGJ0OA3dx~v+ zz;vIhp*T$?Cu)9UC#5Gxn2yE{Cr}s7JSyIHyu<;a@=$~0lc8f))LkjK{a9>mVK|13 z&9XbRP@;nc3i95|!~~H7Bpku>5*YBMVuUwBRcsOo2Uz)xm^_UPy4SUn#)PY% zbzWpp_)B5gtGjYP%W{YOS#DuiNJ-2yDJw>4^BFLeZ43~jVLnCBZYQ6y_M>LNuv}>) zikrvGJqaXANb;*}F5rs#QG()7Y!}JrXr!JfQ0Mg!+=MT>W0L|rEI7+>Nemij#f{uj z*&u4cF&0qA9DZ%efMAymUST1j3%afmATtqiIEambgiC@*a#5y-V`M2P$PN0ugE*KJ zxod@3(xO~2WwE^pjMzh1h-X5w_hMfVmey^RgC+oAbuAiHyzK5=6yfIc2u@Cwsx^&) zg4n=`kv~1`k_nMb&N-ClN2oPnEh~mps$5p-Q4BKw02h-IXc<*nv?5q0Bek#`+TnPS zC4q1P^umM|G^t5dYRyG3h!kxoiIC(14MPgiR<_rK!O4LxmMkg)T$5H6&}ZbxlWi8d zn{krU@Axsnws5Frm)^%2ID{=G9Q9I?WWCl?YFy4Jqi_cJfjb_R z;ihi(YShzUWC;O)z*_WywYOw7E!1E%NhV2PVJS=p9FncfxWEo5Qr31b)hr}zV#cNS zT%`1>xtyOzEfx`MRy>$Q3MwD~%h^R#@kAv?Hnufj!wf`itBR9_P~?E)C_?GAffT5E z6;5wR1T`pNote2+A0YIiGNX}gAdK>(j$%EN0fLI=@pi(8JsTv;)&RD+I8@%?G}YUv#3xYgtC*;B z(U27@3m^spfo*Ugim;_BBaSi~I0tq3W03ToUOLaY=kLhp5|gf+xfK*5c{qhwlA zxTapF+G4E8{@z(?z-2PRK}m8HuuTL!IWWX3Xnb3c!7yBes3H*xCDcrIx-4St0xMC zbK2?T0(0UR$84dxlJ~kOY>iL{;c8(Ml{asT#Mn z#9Q1VT>Fqo@^vTS0JVX0bxaG+Tw7EFfLi~l7Ts}MCA5sP!9L7)q|Bw zk8&AA7CVWC0+v)PD0|jOz2TCwCy>6I>nX`2FlCqt`MkoYW;ljyYq&^;1I9*gF(p)(FG-P5!G(d2Ajq4P zg$d`Y5L0rQu1pFNr14u+jNr73cTW()y`q)-+Qu;?(8H2~i^3N}pWnozy%Zd>n( z=E6V_WD*;8vECESpi)*SUg0Au?_s1NRZAWMDhX;90jPtwMnYgq4DA#sYRa4|$j0_M zDWOeR9Y`UgbSQzf;?$kYEV}Dj>cWvBhXGDhXLW#kKq#sf6Q>5;6DGSK3(oWZ@B#;>4n!>@C zb7^ehOhL|Elpx&VUxQUd!$BUVAGI0uO%Pbb)fkp=*RSXbI33iE60NB+)jNaMdIp*5z9qa__ zxjY6~I4=x5b%n`^YY@mpw785(rg|_9yJBZHWnm|qk7$${>ddOugUAsN6q&0DmRoA! zs{|p(P{B~-oapGbZqP79s3|YgHx00qo7WJPEk_6i zM&MOdB$boVr^gL8O|mYasU;Z@oLPc0tmm=!x_+e>kYb7vDKZq2c`6KZ z%8#0#h@!|ARcQq|o0w%=V-_$r2#3b$dW(s=m19~;E(G1}jAKbKrE^L_DoHsU-m}J4 z#@uZG0A9{Qgc`WZP6H{)kSQtwNg+Vg5l}(MhR@Aro{8B|tW?#qlMF=~iZ=>6nQ>T) zinl|P(`YY=2gq&tI$*K!Ajn`uUHz*bZIy3U$?UHPRTZBf2fzy4jK^z9&TeKCG!ZEAE#838Sd8F zhbI!uC288lOdywL10fArSSr{6>^DS904+r&?kEhmaFPzde?jm!f_uyrVhsFKn#qC!F`c~lRBa# z7ME=cgPx8WZWPe4nh9|(S`&-;GA8suITpP|{{Yk5RnN;F0dURzylS!;EBR z#c6je8dMy?$@D+Bd{Dx@*h6{=45rymY-B?URWA~{=JBo;9R$L|MI@!gO~Rbqki0^3 zV69XRjh$c!Gge5#-Qg0#;tw$fumv6z>|pQB*o@)=DthcaJZ2}*GjwSp-?(W;1u4-_iuP*lMaLnX+}F{;U)PC`Tg!XquBq^gZp zm@Uzh2}?*)Pl#}mV23u@kmFxODn`Dn2@FaS-4hL#oE0n~uGEQn)7(|55L~R7VB3R<0Jn7+piz}Kl@ZZ zohDy9<0=QByZ{HepZ=m=u>R4UyiqOm=;}IqX4nvfB`O@?S4`MHK%w{ezdh1?M&*DF zW;jRy&ew(JfbF?U0fe=&)&S)b+keGfxOuw7e$3Kiq5vyqu+YGc&0p+t4 z>Kc$?+-4!NWq_&~0;yM}GRDDF(F($;)HxU^C{*~qmH@PA zT(&-1gcmZ!=tdnzOd=I2D%l4nOP4_vW9k*fJ6MvErWgX3mP|w_ERBWFh+(@2dj=Oc zMCRBTaZ=Qt+$N&w+8h}L335loJ} zoF9oQ{oa{K7+~R=82}ml+gbWhtI3m6YEIH~E>+%B8X?(+AaSLrWW z2u$Ni^AC~Gbmy$<#W+RDQLJin^4jKEddo>Ix_WPebf9#$kLEyE_=!I^)VtFFBodMh z2+TP>J)vnJ4Fo%Uz<(m;a?5Y8juK&{PMvu1O$g8H=EzK!%(S(BuG$emtXTQLBtEdW z0s3c$mi0mzvdkuoL_@Db5~2xhnQo8>dJ#q&hz7v@36p-SqNs}xQ0t1v0!!etVIXPgI zz+{17A{3W#R=K$%3soQ~hJ!5%bqri+qhO*LL<>0<)HP`!hKV&{A)=5TmZ5uCXjv3B zY(QR52$nK1uy6=$)Lh!xLQ^4tS7|G#p<%+bb8-_P)sl>ez=G9CNoOdQH6fG`m7r3g zx^HNBPqWk&CYidlC3-M8T(Kz$mLZ8P@dgshq2iPSs3gk5PSPWLkl7tePJta3z(f&5?c5=uA4Qr9xsu zA|h7{8^^1d`GQ|bB#<`-QiRE?LLiXN#Jt!ed{k)lIj|~nY{!j}g=7h+ zt{7euM)ximp=g8^RIm{^jGDP=hzu7%292#2K@?$+HdP4>LpD<3WkQ`&gfWURkkvws zfumAr!BD&>lLy3Chork7xCHqvK_}SJmL>CkE!6f_k!uY0r#bH6QVvmB-WW;va#D+F z))E8iioPF3cYt#N^-{Ox&&$<^gdVfu>FO#4VxBvTBjGAg<64jVzxrQOr1*WFm~bQlOqvY*fO9|rkO!Q(Gt28(@#Bxhj4zqh zU5C+9pcm(Q#mI9Bx^mq_J4-*PLgb7pnr8x}d5U?zSX7lH5&M8XB43l~&J|+VUsIv# zRd(g%PYMc3L}XWpo_W7ic^xuz}R2rn@L z-GGdww*@NAvl7v$VM+_wDQGSbV6#lHz_gs`*nq-Az@j%WG`pJ!E#QM8gA@jUL1YUW zwt!d-NCk;j+=eo!IWSdJmTOrsIVlhrEzKdQ5cOsni8WyEy6~{)gdJR*gRFHdf-v40 zdASG>D#L$>h~~;2to)pjn~!W&$Y4^aVkTZaVH|)9nPAE?$-=vvH%NQK&5TDDJ%zwEVUtlK!YraT+pyZq~;!@Cho8W+aY-fEC^l82MMUyeolJ@#Dy>- zRF%{=aKcq?tjM|z4H!D|Vijgkkj5xjWEpG}Mr?u|AQluB%mfs-N~pnz1&o54hoO60 z=D>V?SzhOh~P4$Jh&jeL1huwU0|vb7)X&0 zLkffz5Mv20N)?DSxQVE25r-y0(77<#LReBDvME|Jk?u{BCF+9)Ns#WK<#iU)=d7#q&)+uzMOdp*w}gZD57PWKK=hVK7SsS_1d5aa$~d z5K>zuPZMw~Mz3&HlEe34bFadHc z3OMdfzUX19XmUH7siDa3U^yhR1~lf%z-wT+-7M>1x!f!vl|$;Jf&m#ChsRcLt84mb z{t&>e7n632$$(WkgI zFv*4liFV>?FH_zxSKVL$VVf}Hxe3TH3bA6blES*qqq>|w>_0;1TKFaYqf!k8Xe<8!`MF&det#$GdOZBY z;huLN4!m>P&Ah_D_P107_Llzu#IO9P2dnr;RnsPl#A%d~{{Utm(v{qNUcc$;`Qgi+ zS600oyclDTTmJyBn%ox+vR)4J3MD0(jai096`@Ei-C!9ZkV1nR zvuuU|yKVs?t-HigRdScyFQ2Oph%zKTMadTx{7UtGOI^VEx4Y{8BkIXrevM$+Pnh?2 zsSs1Si=qCKcdP!UeyjHRvcyMm6xG*[{k2}]" + + # Format Mouse Description + if m1 == "stop" and m2 == "stop": + m_desc = "Static" + elif m1 == m2: + m_desc = f"Hold [{m1}]" + else: + m_desc = f"Switch [{m1}]->[{m2}]" + + return f"{k_desc} + {m_desc}" + +# ========================================== +# Main Generation Logic +# ========================================== + +configs = [] +readme_content = [] + +# Group 1: Constant Keyboard, No Mouse (0-7) +keys_basic = ['W', 'S', 'A', 'D', 'WA', 'WD', 'SA', 'SD'] +for k in keys_basic: + configs.append(((k, k), ("stop", "stop"))) + +# Group 2: No Keyboard, Constant Mouse (8-15) +mouse_basic = ['up', 'down', 'left', 'right', 'up_right', 'up_left', 'down_right', 'down_left'] +for m in mouse_basic: + configs.append((("", ""), (m, m))) + +# Group 3: Split Keyboard, No Mouse (16-23) +split_keys = [ + ('W', 'S'), ('S', 'W'), + ('A', 'D'), ('D', 'A'), + ('W', 'A'), ('W', 'D'), + ('S', 'A'), ('S', 'D') +] +for k1, k2 in split_keys: + configs.append(((k1, k2), ("stop", "stop"))) + +# Group 4: No Keyboard, Split Mouse (24-31) +split_mouse = [ + ('left', 'right'), ('right', 'left'), + ('up', 'down'), ('down', 'up'), + ('up_left', 'up_right'), ('up_right', 'up_left'), + ('left', 'up'), ('right', 'down') +] +for m1, m2 in split_mouse: + configs.append((("", ""), (m1, m2))) + +# Group 5: Constant Keyboard + Constant Mouse (32-47) +combo_keys = ['W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD', 'W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD'] +combo_mice = ['left', 'left', 'right', 'right', 'up', 'up', 'down', 'down', 'up_left', 'up_left', 'up_right', 'up_right', 'down_left', 'down_right', 'right', 'left'] +for i in range(16): + configs.append(((combo_keys[i], combo_keys[i]), (combo_mice[i], combo_mice[i]))) + +# Group 6: Constant Keyboard, Split Mouse (48-55) +complex_1_keys = ['W'] * 8 +complex_1_mice = [ + ('left', 'right'), ('right', 'left'), + ('up', 'down'), ('down', 'up'), + ('left', 'up'), ('right', 'up'), + ('left', 'down'), ('right', 'down') +] +for i in range(8): + configs.append(((complex_1_keys[i], complex_1_keys[i]), complex_1_mice[i])) + +# Group 7: Split Keyboard, Constant Mouse (56-63) +complex_2_keys = [ + ('W', 'S'), ('S', 'W'), + ('A', 'D'), ('D', 'A'), + ('W', 'A'), ('W', 'D'), + ('S', 'A'), ('S', 'D') +] +complex_2_mouse = 'up' +for k1, k2 in complex_2_keys: + configs.append(((k1, k2), (complex_2_mouse, complex_2_mouse))) + + +# Execution +print(f"Preparing to generate {len(configs)} action files...") + +for i, (key_seq, mouse_seq) in enumerate(configs): + if i >= 16: break + + # Generate Data + kb_arr, ms_arr = generate_sequence(key_seq, mouse_seq) + filename = save_action(i, kb_arr, ms_arr) + + # Generate Description for README + description = generate_description(key_seq, mouse_seq) + readme_entry = f"{i:02d}. {description}" + readme_content.append(readme_entry) + + print(f"Generated {filename} -> {description}") + +# Write README +readme_path = os.path.join(BASE_OUTPUT_DIR, 'README.md') +with open(readme_path, 'w', encoding='utf-8') as f: + f.write(f"Total Files: {len(readme_content)}\n\n") + for line in readme_content: + f.write(line + '\n') + +print(f"\nProcessing complete.") +print(f"64 .npy files generated in {VIDEO_OUTPUT_DIR}") +print(f"Manifest saved to {readme_path}") \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000000.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000000.jpg new file mode 100644 index 0000000000000000000000000000000000000000..b243cf10ab6a4873d99bab3e92549d0022f4296b GIT binary patch literal 31367 zcmc$F1yEc~)9&IP+#MEo3l<0tySTe2xVt;SU4rZ4EbbcI-8D!^a1EM3F7NyO|99WN z>RQ#kRrhS2J+rejeY$6!?laxf=g;b&9RLQymE<64TmGnrD_U~>&%@*6MUJhR*2A7KPRzEyP%GUhks>8AX((XlbSNG5z^4V z2jq2KY&0JTbNdL9;==~OzS2cNK>jx$Ft9k-)Ld|;;;Qhtg*0={YU)XS^_O_u!B@1q zW)cWI$&!D*0I-o^{9uFNU}yla*Z??ez@K#hIvmXFkJs{hz+(|x7@6ue6|ABegJRZ6 z6+c%%a;LoP&l=KC%>#PKVk87<uqrN8 z2jpQkF9bISf^9c8S}`#(+S)zd<`J-bO-3;~+xxEUJ~lTo4u5kLsIu#&|BfxS5QPLC z99{k$6!3myldXy$TC2T2=?{zx=-p`2Bao3+^AKwE|7%zE3kOG8QWOXu2Ce?(3Kht$V zJIC<5!Ox{1gr_N|MsMbN95)4U1CygV9Tw54)k;_;(h#GtIj>%(cD!sY4sN%5(ykDP zicQMoa(sy2_Y)yy{;Tu9@bN^NLx^A*s<`kJzS-__TP0?ys3r%}lS@i1wK3{pSu>!E zC-yA5e5+|?AW^fXwwRWPOta0dR|TRA@)rGjCO!Y=$av224bDZAgHHQLc>*srwUkg; z8Tgbi)56FQP{{9fuUZR+wH138Wq}GE6c97M7LEoUOg}RGd|9PwkoWmN36=cr>STx} z&@|iX?~;O=6JTK}(9s7aGkXZIWBc{c%s;C}C)c{I$IPR*IQH#mD&`X&gZ>oNulT%a6!qx=4?XJct95e@+|7$om zJ^ktKJ|BuCxC}wA5WnfGA`?7>>+(a_c~qptmSL5{STuz`iOib9V9c7&o z5Eqw9#*xVk@jqF(-PCSq!wYCwD)uwnJu5n=W1@q{LRY2wQFFhsJ-!2dX_$SM=g*euhX4q81$Qo23-DS{-uaLqDH}+2jeD>*%FlEshuFL*>rnj3$ zQ=uoZSgiDuK7>ln<41!4f>A?^XY-yQi>%PnQ0ht#XZ%qTJeCep;y+<}x}P+l-(!Ed zcaDcoC$kG;Rs-R12E&;w?U;=Gvt4mD%;P^yl+_6!;2+r16Z%5=k3k}cuS5F()*057 z@Q9_UZu26vl&#Z~9GYu_|3pE$p>4N;Y*_D!98|5N;8>Wie#4|WH-PG6Ty59j^`58{Lf9D%cOpU{#NU-#k`2%Qlys;Ng3I}Ox zf>zDCkr+tFJO2RoZf2f{rDH}|0(^RB$4v#=bOqLzHx){Rs_YlHF8q{rhhMDs1gcP* zZN}~Pi8QLEZ9;1(j?Mh!ub=@7nL#nu+a*|-x9`ZiVwR;+sp1M{@)uo8ipvV1WhCk& zzK>hcuIa@W!g<-@cef{R!<#nc`Gxmc*WMuU3~7C#gs_y<`vUP_xY& zV}qSM^F*QKa+wkij8l&teJQ5J5D-cRb^ihAl3mS)0lyWc_OxVec_VB@JaM%`{{Yf% z@|5S49#yfQWubG>Oz28}R$F2!Mfe}Ufj{`X>JS(>`3F#vR?xUJ^`;Jg6Oe3Aeps4C{Avs%v(%=7sQvwoXZ)qW!;g8AC4d_5%<1)$$3wh+&x%cPfS3? zQckURcEII%!e^aapyCfe+r5BaXg`6VKqz#Y_ocWv8aNF9_&PNjVRAi!=a{Q2Z6~m| z*~_Sk$jnB-Do$Ar)l1*N^oNf56_iA zpkScK#r0R>hn)90y`7F{f(aJC-hoc6pAwi)ZjC4WEccvVTwcz;{=4x^(|3{V)2lZv z`Q_KO(mnLIRoZOSI_ho()nYYpOx1Fsdf44h9RF^9alJzPkpAwK3DJd&JIBUBq4j_w zqF?f{dNqt&Db8Apjp6ujJ0hy&R;dVt^Je#I=)|>w%oieti&5>^(NSyZ@t%*ivA^Xh z{$`?v=Ze*P;#Z4*8Q1Ao%SiX8XDKz2+7bPBXR~{sy^u7<7IBpVD!g69kOB-!)?e!J z^$7?!U$ywmP_%vLiIPn&unv0D+hzvqe{h^{z3A<~#~FE*+H&2MV(QK-LSK1N%wX>! zCVf79la5iXh5>}8np~|4Ows#Xdf)7|O+ODc?2cj?h%{;8GUY~LlH(EAO?l%@nc8*| zg-Y0F^9O)G)A{j5dB2zR(@l;4Lj0$)lFy3?xRWn)dw&4xPkD~(Nf88ii&9di0|mtI z&~)3H*$hA86pJw{WT9fBzPJ6IKN7%?@x$O(DQRz-z=;^HX9Qn-^Gv5>VdX2t?vT{T zlv&K9y8E{X?W$l0gUZ1j{TO)2v!_1xzLL=mtYHT6;%ZP&HK(-OW>98I^+49YMf~rB zxbiPyla?25u{O1v+IqNP_|Fl7stx6>1uE&)5?wj+o4gvDS+X^CpM~P_mRE|GZcnp0}4+qdCyu#)mNVuuob7b!Ka;a?qN6%*{;#nySa~Qn4So&Z03flGB zW(XY`Gs3EaO;$^zQ-zd>McXf>1Yl*P??4$oc<4m$+w>L}F{mr}lK-kef{n$&t#b zXB_CAFtFUyx%qtW@CWdp;x+zD10s*@hZ;sTerw5h!sG`4!6z z1I79wCa_w~HbIDkrzu&3g6Y5~WG-w>%YUQ&-?Z(1DVeYJ%wWB$RHT01t7O?F+EM(; zskI$hC44C4+ozN=#16l-NIe~aU?9Pt0&U_2NU=YnL`uqGWP-% zVyHcm27qWTUGjspU2?ER@OCEcX8}UHEhR>k_;H*uCguy+Fr39*A^#ijduvw6tEyFM zfTOnE=i_@df42??Fkt?zD*F5HaU)(w>L1elpBRXm|A$1_{=(efrmE$?nv>%3j%V$^ zOxWM`y?@!P)&B{rf7Smr*;n~7e_{C_ll`u$_7$!FQQ!HO4`KeV`FQtl z(6PSWZ{YF2*fZHz%ls`sdmsP#J*tqVQ{Yqf9{|%;;JKIbtMot4nm_;M?&t`Bc>OT0 z(mu%JVA&MBYQJ~PpLivD`In*YZCbf=tjlZ3ub@y}IsOAk_nH~{`x$U?Jwy7RMxO8a zKP>o_wYJ}A@w}R#3!`|c((PF$KGMX_FY_Keer5A(OlkAJ+}+Y0zLef$OfxQwub7`d z2pCy6$5E>k_)7g0EskqTY|03w(C}4d&MziD7MTjrUbQY0iqIY8_qJm-rBzqT zujcV|y?&@$a+`4bT{!wpi2WqlSJ)QesL`#$1VBR-j)n5jWvvLzvObWxvrp6JAQ#p% zD~WHqUM#^=Q6kfm4PLc~qZ6oC38_#ay_2KzXqEVNfRbWp+3+#>paMP=g(MF72x`fg z-ZjwZNEtrqS?$Vh{zNB%qgt_ zPyUqoSih%Wk&9T?#s8CB$S7KAnWrq`oL6iJ&zUzT3~|*9PaQZs~c& zg(L9yd`~(&Q_Hi9oz_M_+T6bce7!SZjHj)n=QIGv@8Bgdn0R~TPj+EM#(x|lqN5f7 ziwo{Seh`OJ^(q;xa8UjM)Z|l+y-%82$S?Vopmq|m^IZGoYO&c{Ac~1#QNn=;S1JW4 zV`eb0jT^hPxA+Hb`qXAmhd@}A)5hnb0@6`*s)kw}_s)g!Wob{qJPk<7QtQy!QT^f+Rp& zf0tI*&jj_vQU|tUZ5aT~bq~@ubgxfaG1fsM_je!tt0g62M=T;@n78K8W^2G4(5yco z85GwX9RaKwZ6_wbkg)lD$?q&up_$Tyl%ef4mh4L3+0{WQhNU#{Eiho^{aB;NN4{lg z%$0q1(H|(zxIy6<(S?tGr0tpU^U*)t{URiNtCF@!>k;{YYwcgf6@{dJ9(p#GSZjS! zu+Gzm>&PKrmKfq;PDPdeaHzB9K)-&Ejcy}F^jVY1j<l`A^s z^0xc_`LoYbi4hW(td0fmlm-iY!ZaC=KXF)76t)_e5E{L{{2+1Z z{-&t&os4Oh-;m{P5NZgzwjsLDFJouVV<3ijwMQu4mTXm4c4T&ttpc|&0BMVZc`=-b zDB`a9&@zj_yBMsM#)hWYkfty$Cw_zt!sn^}{+ydI1-sQnc*Z6#7ps;Ojewexlxybs z9x14@SDHaLflc2NQN7jbwkM9<`p~$H#TcpgM{rJ)X+_>y*^+?cBpJrxaE-Wzj8G`} zK81<$nkqM8mfE^42$HB)NEL@1h-O3tnWFMjPG4cLTsM$~oA_p|i!FfXtY1?A2l)XY z4}bD`|Ku8%URmK&RQ;R#TF_lkS^iRV{N$31|GWOB=AbS}A$lXJVitj@)TQjW&Dh4t zKt~B}3{^EKjf|O|1xA(DtN7NfecD(bL0A;0?}1ExE{DhH6vS`newXUR{q9FMTs@h_Atzyc_h1P2UIcq^5)FReb^fd0LE#>jtYWjXN!0(zt5(4IW4^ zbG3)2c+QC$u`0<&wFjA!mrv~?m~x?XdI zI^x7xue2!dpb^npahS@;!t{6WJHrp5P}12mUl;7Jc?~?fYYP zAEUx+PwOeO`L*9P%(061f@RWlw58n;G3N+F%Yc=o3i%UatJ{!PSm3#~v7t}oer-+N z8_|xF#J>$uZEZD-v9X~EnZNro%wM1Mh&WcgCbPkpdNQcL}HAiV#( zro}5Xn?uatHC`f2{CX)>W3sHYajR~@nHFNlXCc9Qc|(v0 zsib4Z$p?HunNEv{^XxxT5kI0Z!Z@!Y5AqPd7by8Q?L$8AWgv3(ly{sMg!hpJQl^vW zi*BDg!*TkH>E-%Zq9+O>8>{8nK>uSkuGmqY9mZvq84!Jvu3^EocH3l}&>JEoKyB?& z8&U+55#2``P>9quhP~Y41ewue0%j~CS4Dn*4r<1Oq0(Z^NC+KFy@L60laA?z zI!y%_&21E6|M^4l;)#rNRnl;`RzIwAn|L-!@*q2fVlp+Hey1F<;y(1;MRZQQE6fxL zM>8>3wm%LlyG6j+DUA_6ey`YyWOry5z3arDMOgTop(gPDz>MYRD!U0#X}81JVT&LL z>@(sq9YH<*tg(dz!-;4XDv&^R{gRN}B~PtWYJIc)Q_*oIxcWoGxR|!Z!EAC@g13<0 z>Uk~l0j(jr^n2fo#*ke#_aMxOP|eG9C{{gB*Rw(b;~+-NyHrLF1A1{G zY<4atCbA)o6vaFZR|Dhq>_fQ=)0V zzE_Z&T_Gub5T5%&@8wEv+Y=j$QduRZoy#RB2KL%R2d7ogvUjEKBxI>k~?W1 zFHe|;j3d~76N0>8Cs8S4$i4|?Gol6t;d~S!30;F&Or(8dR*l8&!tfom3octc8uN@I2lXl@th0mJ2T zxCI%L$BYNi0Mol@^a|?^7#=1GOX?*#NUwlzsVx^5K*pGSKzlGL11Lq(w;_te`c;ar z+4N!B)-0uUGSm8KLZH@zVHW?fa*OtpRs;b{5I`mkMI){h{v!uM@8qWOF-*Hn|0$C= zGcwv>koB@q5{+;7Aw>Ra;bhSv(^8{yRSB3BE$vpCHtUl&|D(OC_H^tr+As!}vUPs5 zQ@L$+QG*<yghsA7DY*fYW6WTfvZU0?&H_d{};JrP*R_Q~@He!CJ%(7gelWl~#_fRq{c=wQ( z8g7+_S}fIX>*j9nTvcJtyKuDV!wCC!5Rn3>a6FM zEPM%>NuYK+Q<~gfBBKEkgIljvi#!Oo1P(q)0g*fIpVkHbGo)!3j& z=+?9-j{-xRsKI>l2`vrL)_Rnz`SqGHgv|B@QfOFQ&}6;w|}9S685eD>}yW5>EFlEYTKyG6ge86g)Zi;YR^OCsYHq^nUH|2DX9_ zi7MV)QJ;y4Gd$Cle|e?zxYb?~y7Ea&Z7Ke4_*`Cwk!y1RQIO zT5}dD2*g!6gfx}^NwH}goE+HI7#Rn_R$aHQdp8=Zy`a9`1ev*_=b5$h#fvvQg2jP{ ztCO8t*u719qoY333di5mSB>)Bqz2kN$`#o{cmk~)+f zidg$6FRiO3!z!Whg=)9fa_lEYq?lC*c%^ zEOnz+m%^n52@+khv@QH8={2U&BSEyxzB3>*J(Z9GMJaD@d7t}zrDIP=(ccDTD+cW*zS~OQFB2?If>0IV3G)l3QlTVj7BEQ zwv}F*)hda6-zeeXQ!c!Un!DEN2BDjsYmFGttY)56}%;^06FQR6)0;aD~3wOA_l_Qc9I0qLp? z)M9$UxfdENVvyc@|8{R6u&MvhFmSwCuR%+``cY(Tn7ACowUU=ZJ zD#!ZlkL|qVg~C>%2};(w7LesPZ=Wpky9+zjdwFepW3hH?>$Ws^jbywO9X`Cm!c`rC zJdTK+vm1_0f`z8BydW6hIm~Z1GDm!y{x-L5ou=Z*Li&_7ngTg^xNN$Yx3@2(5p``$ zyHlIYJkf;k;h=G*IuYZSqB3O9itWqp7p=jW!gw#^x?h$^XB3$6Lr=c_-znF8-TkT=!V^YhaFU+YO5Vdc7=v z{<>Ft{VP`b)xU&G_Q+cqp04}b*VGp}0qmdD)S>?CY<3c&ea{iP7``SGrEwN~%9u(muO<-`6(R@qnE09Q@J0$_9W4IAi| zExcOksvNBc4=;l;&K3Swo`76f!7h>{)X&`uzZgwbORH_a!U{5PjeF;vUQ5kFwGR)8 z>$mM5t2RS;l&Q4tuicr}O6kyA^#SDI|FqrB7HoIemvlgkH5LzMy)gNmlW|wQ1 zW}C_u9MvE0CB+X-(ibu;X&=ms;yW;e%`-AQ(i>X}&o} z)_7#Av$A}6v82hDY@=*@iV?gtevIBWMCERSQx5$3@}1RIaZE1UF7vW>>aYp+93b;G zl=(EFX>m?KHZor;i_ZyauaKa{;u6B}9`>hxyEXkP70{*xOM#ecDJ2YfH1l%~1l%fP zY^1i4ZfX2^VDWIFJ9v@WlRXs@0xq>gRrY~58>n6onMy?s1Iy^y!AK%BP((# zidD?odq!(Xx87`!}ev_~LBU933{R#dTYhu=eNN1m@` zo(G}eXGIYOyeH^s#9#P&18>b+P4Wj|)n`+=+LaUtvM8aL>J7P zAoo^v62<79It=g3k^jft^F>+H5?)`?XiEuT-wISVCEe2CCc;8paeY*QNCK}A#USy6 zHfp5c8G7WG3Iw5t-@56cq@U6<4vB@T-u6tE?&P2VDdp1O3@WgAEYP#;?Jyl5Jx1V$ zjdVK)O3zNDw!T+@snT%Xcf6dZQ1#V}{A|6+A3JVgceVz);$=%y4bvpW;924;Jbj&~ zTy!!1O`CRpKCiQHXWz5w{I19H51@E@)h}M?>EvASuW!JsPuKKU6e!4f`Me2Mw8cH> zRX0seS)lo*A=t~8r=0NaNy`9!{hlUa6in!riH@wkP+#<4v80|B_fY)hnZLqQ>qM|Z zqbJ&oq!yPaVgcTR#P(TTOmiLXn0wVelgZw1pjU)Rw$##)DDW;CPCi$&TKOTFbG>dJ zg?S=Apq0999=WBp>R?=Q4?_<)>n4IQyr-V*9A9Kx*7enHT(Zi%spnL5(~x9a((nX5 zbQ@;|#Y^H?NOCJG8LwC_ZhnE^5-sUXX_ThniRvag;QFRqqv=Oncr6z*RM&J}b<Mk|^9hdwzbW^dAenwVmAB03yS|t%wPY%H5Tb1pQku!AfE<3mSuV-1&2GHZ z5_252ih&Z_j2hRjuGYtBy%Z~)$m1_z^YxAP6iMmtwRq!S13c5lW0X^!D#E>BoJ8(O zIDg)$QaUrmVNGGSNT!Ljh z+nx59eouIYYxaZ{<4N~6M9rB_4b_>kN+L6bBpC4CJZU*T+eXbZJ34x^r|#Xk4Zdzk z+Lg!+bayr2>PO^>A6s1=jL<~*CM&?{gr3l;4yNO;AMIkfANzlWPkq7t#$)P1+6L>M zg4U+j1{a}w*(1pe;HaK`SQ0~7wqxIQC_q41wTLjC5<`WnBc@(;D8M2AWWc^_Q-FjC zz3h3tB0i0Uw-9ImC{44`PWe>Ds9OjOIO>+Z5D+>w4Ae~oKW4hkl`@oAsr@v07P|8p znfjf1A;1pjI#Yb|l$A8eO_a(xJD0uzUzF6J9Gy#IP9c8Ou0at&(Ekx#(q*M_=O?wb z7>tHUFXExd+-#qx%5AE`9{?-S$4!YM%<|tE$@SFV{hiIq0<4jW6wXxl5c{B~CSs7^xJ8D(n6B(={V?3tcMPz)x zX)oQ2WhR46$2XK!5}n%!k!O$dM`|!?6*iK0n~a^KXZ48D&sf%C(B;(HoIFKDEzG3E2JFaU~#zS ziq9VAcs8C)*Pg$${W){jJ-sIphb)|@VW8zMkj$`nQ=j%8CQu=5Y<7qR*2SBXjeMTb zKY;t&l`2r}H8r4%zbFy0F=CHgW3DJWK0pd=>O#oL+b>gW_iZ_;Bdv6W%HH(VZLI<0 zxXh*~>kmbAqf!e?lp=kWsg-8Q8U}0QVuo20w>CBRwHC=8C&TB&R*LdPXyu?@pyD{C zfR&U#YuZa?#MjE`y;ao+b;MDUW?D`Oddda+hWHCdSU(AgLcPYpC_{#(oH9x9M~XHN z$!b9~g|_f+Bk>AE&%M`(Z;qR^m5zVEeOAbtP0*KSXd=mHT~!ebR!2bid7AsxNjNN|X zc3O&2rmsEd`mv}Tqr3Z?(xv~L!0*jYpsMbH8XQ`iQT3Z0R~FqAdk) zzLt0-+l+EB+w}e_or>rf_K%N%(#a#L(Aso$)ILU-9>3mOgIS^OSNLD0N^*Jww#&u= zsD_Z3ssATWU6pEBN~cDl@sKaC`k-yFX;RnPzqWji^j*q^qBVT|jN{fSXR}P(N zQG~u@>57`0DZ_)N<3RB3Go|ihTg=u(C7hV`-uxgDpUw8hXfungt-L@sEJ!G#euT$d zJveDa%Fq~Ov3rc(QHKiozC8=b@KCy1P9i}>D)g6jBMc=rBQE)FRI{(OGWNRTv4}36 zpPr6`GWuopdI!V1O7X1-Ee#zmIJ>y;0K{y88iDC;OZ7Qf&;l?vQpV5>9;B+f9*~B2 z6_Bu}3PJ=%1w)Ah3GVMYJ$5C;NRVNDC_Hxz+p<3QZ?sx#tHwtbJ; z^mea%&kxz{Lvg1EqN^+Nz-xx0eFf5<{ILT%O0<+n3|QO2gt95btRD4NLEtBZ$%fdp ztm^v@6u+_b&y(}vES)_Gm0$;z<+}bV>9<#*y>-ej9O`z=Ay8@bJHO41;MAtP9l~m; zWVHrjvO@5MH9v^M(HXhXX@>XN1Wb$mMP=)eT)K*j$q6si3A{I42U!-m%H{ki;JMw(2&?3%7K8hse z`E>jx^GMJq$HxILiv(qzqRGrSVuih~MJI;zyM=L85iO)43WN%d59wO^4JRC;yqxl!ZPs>#2 z6+=@x_Z1z4f1F6KK_>LSBK7+03eqz1~UyO9=w8JmsPr;wEKp&)7j zpH&MnzWHoQ^v5ZI18>opzz61$!xiK2>`3eHv>Q1{H|eI;QsZ^C@TxlGfYO5%LJ%#w zm&6@ScRxz>suBDp-6jWSKw2tn@fJX1ItZyIzGJxugYW$58BQbHFu-#|bZhrHZk!OS zv$8X?goz0Y7e|v`YKqYU?U8}-v2}yrk(K7jUf+mO!d{OPg*S~3oH2vTCm1KfV5h&+ z>f0ytWA(Y>EDv+_ZoY@AJdNTtl{4MF8YdBf52)Ga6qw_mJ$yw_P4g8m+&}}tKoL^N zo2g0|i)t5$GHfzI!M=lQm>S^wgTW6+{*!-+N!C{Y=Y5K?AsYCu3veoR1j_1c+}=^Jvi^WK*^t?MXy90?)toKLa$M|?%OZyB z+HGBlfE-WpYGe#Jk)}YpLr`Aa_huf(D!$zQ7l5$EVlz(k<{Q>aUYD>TqmhdM+p+?ds z=I=OI&4##AWxL}p+UkQ1-|%FR0+XKkuY7u1c>2ap{?%nMJ1BUT^ake#&L!=VWf~}R zi{h?8e!*W{b^Zp}XBX7f{ZdMAXKfv37mj%k`V8gwza`uxet#nT7G|9ZlPYzxhDe z2!?xE;ujz3J0`Z2DQ7p@zTGHc3&m|T9(kwO)i^Ce;DB+R`$K0EPBMI0C4q7pzI8%P z89+72QI&%n{hl{uFGMXfLy=n|*}9u1!pWIw-&wm&3FrCZ<5f%T216U9LZy<4n~PdC zsXSmN>i5bSzSB5?{ccCKfCJ4oun7NaQONuUPg|~(%ZcV-;0-%&Wf=kaJZB%g*s|`d z>iA$eR!Wfqw(!F0{b#NUtF>m^GTXI`Hgmrj=r6R5JFjVcl)Vf3uQ-|D_L8@cN#vIR zuv7|>J7nNa`KE@B9WT7*eUv7w?=b4T=PYsjr{(98k;}#ULBpn$!+jVjRA~(mACBbF z!jCKvn191BK+3H)>4`f2V8SC&+gY4RsZt0%f27TLT6DHG$Vj?#41rpP<~x=Q?G|JR z7$+@xYuwVEeU^t;biDFW%xr)YB{*Xd)-|;KPBv>k{g)`tKr1bO>3-(%`Bju+Q_a_P zghgyl$zbi)2vl|odK@ukup-LV{Lac5iywK7W{8tQN(_OdV2N4gdz7=7n7taal(59t z1bx*RtHmD(2NjdLM(h`#cA-2=RW~0eHsuW4TTlSNKFaH{Uo6|NPns_=zdtt9?{cOh zlWj5Qgu&?sB8~)Gz)uo2-f3I2*$PJ+ml@b?F#hqIQ+*4l zj!$Eo)Gw1O)LpqV-)R4K(b=$1La1#F3oNBxz)#`edv!{wD9W&X_RBRKNVChNJBp%g zi*)u2rK_oe0T~H}B>C;>=f{_M$Bi}#*m}>fpx5kZeuv^U^Uz*jA`o&WXb|kIXcEL9 zXXyW}ps37TK? z7t^>EZC8b;INdc+6&SI^5euRxRfI3gtX=E8At{F?m`c7KVEe0ecx{zbF3 z5v)@U(W6?dg|&vyN&(S5e6>rU*Us;CdC4@E{F%WPpY$I zB#Xt3QndIWtm?FeHct$-g1YoeZH_nfT=9a&O;7LZ0tJku4>s`{kczs!78V9NQr+OO<9T?C@<7^E=t} z6^M8HkUmnL_wRmHBboEJTVdEww%?x+aY`y^`h$V9RL~DW0D<7CU~3{wN!X(N<n7(%k|U}RIP+4*+`mkpPsk0SKaN|&8F}ZANNPix#=ssaE5Bm zM>norMI~134);e9CKZ7w;1@`809}a|Evm_KN0TiWBsz=`D21?J+QCf+jLKW@I?7lO znbr?gM$dkY0Cb{|A9&(HLhBZl^k7eOABh&MPGa!J#uHe01JEYBrTG0%9MoCXhMqQwhK0xyG$EhJ@*LEyd+f}*@*b5uEz zO}$a3YLaCXoYAJV%|7z^HeiA+X9lj zFEq)qt_n3ROkmv8q8b#Jm$vXNMrQXiRhFWG1X7$_iW)UwXp>rfaBWsPhC=$;wX~&`Rfb_xK z;@8LJHYj?yhGIM3JoMOroTAQPv@L}|AdG%pRn7K_ZeOn+_;UbOi)|AynN*B1F@(~O z*nH@<5ao_k=Uu0Phd{mRSt}v(y+8+Rf0$28beGODMzrvQJb#{m$*m-^DfjERaIDOp~ zs>8m%yYgv`8XG$d;;|sOG)$^Q{@FaBj8IkJm>GaPq^P9HQ*3J`v8d^c)d7c^$n&vz zLX4>ui-p`KHwCy1M|jMJS(@IJ@(?pRTjf!#j#oO5Otr#QCi!7Q0S5?ck<{old;0oT zgQo2j?)ukTt%|Q8+KCBQEnm>!QC&D7ZI4R)f}%yuqCLwmgCNfcD@NyOSk+$OQ);=; zOhx89!&;C?kN)ZmQ4*>~T<%xu?4EnrdZBniY_%1hZ~>vV-_X$~16C4bcpz)uFg){_*UcbZ6|wesF3lfzGRlpRW<8e><(&jMKMTNWh%%Nf!JlFw3Iu4H1Bl} z>9xwC@|}--X@cCdc%%K~I86Ej*jJ6?{F1lX!8|jOM2JuDZN`ncU1bxDa!8ZF4RuCs zp_ass0x+Q4Ra(g0?3{HHDU?@;LCo-qN;_a=moU7gD@)v3SqD6I056hV!Fsz0^Z4Ol-0kzhv4SkAT;9*5Pqx}xfOZt1vPWc=%!g$$9&M&4tK%dQ zct5l%S;lC5RAPnMi-fQ)J|U9ij-@cC2G!7~o_DC)H}`(3giln7c@O7MoiREZpcL4$ zOa*PqV+uHkfvl%t`6B2R!RK;2@dsdeHw+J-w8d5G6Xlg;j#Hp7VfG0Ov7a-#yh=l3 zV)&NzSd!Ubpb0(LGm}LtFvy$(4ICtWIM@h}F??9IzP=Y6Fk4TQ+7(ty5>=hm`2zrF zfCveh78T%HI?kqU7DJ94s9|A0f>T=F=If22i$7RnE2eHp6vqtA<#D zL%F(|48siMG^mJh0Ju<>&y^R#tPY~*_N+Y(G>=*JFse?>$0X=#qKb^<( z*jLJP^0B}4{45;_@W)B{1BmKKzuvch0b0v2O%%o2%D_uyTyGu_e%Vq;ozzOB&nKU& zQ8EtMWi5;7a15)EDnq7`!+K+u^|PC1e~9qYMbbA#9w8oTZCpe-;z-KwiEHf0PVP~m z@a=kIx2CcVNy$HX>g%`pz-axmK|;SsjbVIxwQ?#VnvRTlCF#!;G!FFQYC$R@YC&#` zW5U7(7`*-mOf=-JR*6!<>vOr>JLGT2e|~%xYDq92yX$YJ;Lx}9PWbr;Fm(L1ZM!TX zSK&hRX&J58it)&RJQjvVasg%aunAiIZg~9zeQbknN-@e}19ct286cXAPR8AbP-PG& z(_m95el_1YDlCd=APbw=6vQwobyZ9AKv- zGaOwZh*~GLoIN22o{k4E5dJ zk1WU{SOFYdi9+k1qepQ2hU+aRq!H50U zSJp1W!r>;d8bUVy`mL(_62+0jwsI8QqHxRK!WhZ8K|wZd(^Tq1-+#3`y<6EDe%$o6 zKA2n%n>)~MoZ84xX#;+N0j4x^F(~p`2qc40>BUO;mE)LF1p_Uv1GNWSrT1$5q3m>w zCkz_b>6$r-kXbR}2I6XaXZrn=cNeG6Guo?rCB7O*m#uoy;vAo!YGX z%`>uYRjB8sdksJB*5{kz(9gXcobH_{n+#$(`KCJ@G7hV}o_e~6G&!_hR|Rn`tTBK% z5V<4G1hrm;kc^S8-`D(uukYtJZ-0MP?{}sMA_sEEoy#$8e~(IK9(j{P^zDRMgnRzr z7b+YDzA5kQNcV6+^ixqe1+cq(M~cD>k`c=-%Dq*5+__4SQ28SF(K%8RR?}%R_>(Z- z17}sd4xIq3pY5eD>t#g)2tHW~^zqv|FgYyQopbMM*V$9xtu*^Rwi^PCl^HJ5h%9ev zp*S~y3PViOPgd!vBb8bRAOt60^@~pGO)`S{LzvB&P=glTL`Yp__eX|bed)eJZ@m%e zoI@nUM}$nTZ&hnC`iq{haJM@;$fUXEQ43Jjbyf|+W61bYY_lz#%)sesO#CP;NQ64Y zIATo79#o4KdrGg%LIm53L67+R9a;MIhq1P%T2wg@6(jl%kt&W-DpWVEZ*7@T%6=T^ zIHPEgAbL;-(NmkVO7oP6EG+XvhEM&%Tg|24U+p)PZGxig#)u1c4^xtU&Gf);Jo&=W zx3PcQGU($ak7C4)sFIX4Q<}8%lYR<9rz0Lt{s$oRVd^Q#5hdGyAySpTl&{2X+LQC8 zW!!lbK1nX=lPiqKJa-(ix09XDL{#>j|I%2C!>$;D7=ovIQ|c&g7hvOOniP}U2{_d< zpa)7BsFym4kAf(J!{AZBa}&@kjK7XF^wqESbaz#bng|i{x)Gj{fTLewZYf7HpJ({y z{Vkp5!6wzzM?u^7JlQmkYjA2wU6gL8&K8RKh}M0vDyW$$P~mq(3t`C7%VK9U<2A)G zP2B|fB+6Fk5BFXP6&aV9yq3}s7L~$zaD3<3%rMkDJ=v+RHue#F=R&lJ<}HbDtTptMxbf z;-0q}$>G)|GUuN&{aoA*j#7{0$~?_qQ=Kmy6HKGzYs##zIWj6F+_z*a-aeS)5M+M6 zfxBbRP1*6c42Lr^(5^@m>_ExQwp zY`ZHUayIEiMaW`_jJj3ix}72&dzEg&LzH-R$quZa$B@uaoM33QLV+6=TEF10pRc-U zZ}&NCe!SFXd=`lxyLklq-vZh-CCXw;tV*`E8UpmH=4v#TuU#F{w=H(Zdp3$)9E@pb z(->xkz|<~Lw=v|!hKt+in5GlRoV|#_-zRu#v6cHOV52%Y=LZ)z(rw@vLY}~YQ zD&>`FRc&G^#9*4Fk+({FnkZ!?^AJYTRnP=`-eo73ZUEMY!DrlP%#3NaR%hwN?~hi; zS97Xh$1XqRK-nN}Tp-~zz@g%-GH1pe04nAIOhBZOpn?10wk%fr{b-VNs<(d)8rijt zJlQh@r7>Dod|9~8a;JT-kX73_N@V?#XBHl{BX5hzjkIE>RQuH7Q_) za@RBC@pvvATpfV~!5;$iQtWm(9OZ2CcJT|tp(BI1tjz57QS}yzZG|MpAz9hj^rt~( zRm5}|fm2?FWm*baLR3ToU}X^i*L%*doy>^TZ1VN9Bw;nE^78R$tA$I%a~XVzY@>oF z%eAl%_H(C}?<}7bs#3Iplu(&rNW{U+Ai+CE@vps>15y+ZRDvidL3x+Np%}mfsRI!j zQb6~C(#OIipf3~((nkLPm>mYMMax*s*SwGc6SVkhb!G{bffIAu3#Js|+cloR&2Ndn zcT652y(59M2@p&X1y$WxW<{G;=Lpwf@{Wh4QQV5E8<7UP_w8I#RdCSd47P4vyr||Z zK-3-JRtEijPImyE10s_d4=>+Ft6VYS6{~(L%!XG9Hq@ylderJN*c^Fo3cZ zqj8uy?(&~|!oDpQymJ^xIpnEn`ulnudvp`=WtmjI0e?+vm=ypKqzH`;Mm<`Wtw}~( zz8e8?OXCx&K_`7ZV6dJ!N~Q$QN&219HdUv$;!2(uhUW7XY$yUTBNBpDwqSxMI3f1! zs~1#P`g>^sDG-FhKoBH>p3|9e$-l%%`#MO;Q9n2!__>ta$Py$FVC&_hQ>yoG9J9mR zuoS9}ZTrBC2=1T|23tJ6&Z^CX$14~GJ>aqrcqpAyYUQnT`Oaa@7RF^S+{XhH76=~0TgD!kLc)YqKNmrw-IqBMR$+h!H;Nz zU1{(h=e;#=Iw(`hf1%aPM_DJQ%ePm=FH=FO&MVSckh%g&u$oMR+eXlZ^;HLb55c?f zL#^dChZS*gW7RoiudWytn?kao;rpG!P<2j`){X6MtgHY##!TVwd$*Q;$K80Tzki9& zdUORus+IS_@nPhZ^;C?vAA`%@NfbUK%P4zLu%%1`Jt5bS`_lgauaPE;X9{3Kf>M&9 z1W?Y|=sJc~B~u__q>Xi`8c;wqAV@DF5LE;#4|J##p#pQ5$c~rGsp0{Fb~>?AB$$vi z&=HDg9(KnxVBqhXGhIdM_f^)W(w(NUIrzBsBKBxsVfT2sOJAs3t8~FhT6@HT1mt3L zBW&(_=JF`|q>M_&Dh)b*Z+cXjlR>pAf_+a~e@MM;gr8rzvaRrAcSQc@ zHSA*QaZJH=SxM0E*}dgTgo>Ws(mmfdy>_@|8Rjm0K2JX)#-i!*0g6j+xGRqi!&+5d z>Z7J(PLCFKT)L29dj@-QoVy-0lp;RYqbABGO| z3Xb9MWyV@Fi%RHWMwCVnX!^s|qe7+u%jQu`h;grav@Z-hL`Ys*yo=D4Qm>N6qSyv6 zsZ$zz+Bd>>H~^U6afhAUL6!nb7(@_SI_GGu8mpr{wv4 z1~7>_KpB(J_T3omrx5QYDFkn%`rPm~0bwh*cOICn`074PmXW(ijqnD$+{;BV7$P>344-@MAdzs*1{B8gQS11!i)a#ZghcAVLlM?C0w9Ju_MF+ zJ?=*Dc~?hV93wgpx-ALvp=pbMjR%VP=Qn|TvBZ>_>)%9q_kL(sV)3h^N#ctAM(LkMj zD*N2-><|?!T{030CQ& z``HqVMM;SxqflVJAz@a8Qwox6kQ+~gw@!Bl(pD4Dqpz$m0INkDB{&s?Wig&1b zoHRO(A5{t=Das_}K0gV0abbdXLFpYXKy@of)X#y6_H6}Q#UzmlFlYvJ`lq4!Yu8k=3bn3+52y)s2v%HS zSv7l%7Rr)3+JPP4QPVxoHNG0gm$v=O1S}HWidFglePsUfM{o-#ich5a1%3>%k;}iZvOyObi~KTSp}>D zfx^_KWJG69CMYo;kj^369-T;jk6J6*C5)NwJk=%T8>0I_$+Jd05btG zKEDT|-v?<5lRh(bo%wl0P(4@*QHvhQ3evIe zn!wY0*Du+!?qe36_cH=+WF#Lg?_C!~D0rq1#iFg_9Z}K?0@Z}nd@}X3J%#smKh>a% zwxnnQ^71Sg_JcxAL7mz7S8F0xq|6wuls-Y9n?s2hqcITPD`{WZ9m-xrE!BxTj8MC2 zdc^IV^5}aPdxy@zo{|2SQEtNG0k(44?)W{gv6EjIzj3PC)tR1=)BBwqqZqBhoQIBW zISWYckfC3$e5k|UO$d#YK0Z+oEEw&jEU@e)60|CVRN{l^&$^`UeNvE|GO^q_k&w!u zi6BVWA;#lEj#bixK{;o$*6>_VQJT>0`TU#dku5LwC-DqJyb8O}tUxFQ888N36v^`E zP^Jwu5i>~3wvkZaEhu8t{C2(e?x`Y#bRBv!aYcpGd8|yAZQSnn7cnZ?BeOcm@K3lsao~#n&Ib^&re{1mIwPU#vuQVKK*GF2 zGXzj{0(Cl3^*3y;LeK%-^)jaIi$GzKodR>|HE}6E4GL3swVDouB_iCRAyBZP6@w#x zNB~7x7jDR)N&p?;r2`=<`b!#~@Pwuat6$YxNTE4u0W|*rlfSFAQC$Es1w?5RS!*4p zSCI*ELUhw?gW?8_?I>t#uIc#K*5qzn;a63HOl(eC4PI~kDKG+uM(4HmGk&JVMROaW zTCoJEA2&4)rNw)|^AhTB+X@=%vGerzeIHo)v5pe?`Z3Z+xVpoSXseGDdW4)jLJx<< z)m{5=2AGG%l}N%8qY_g)gN?=(xlYfMt)GVmR*!NZDynj-yf=6bZuR?C( zx`Px%_8og(b@xi6arL|M;rx-i9zVd!P)f)KiBeKLBtsI2Nm_{SR}YG$v^Mc1tbQp! z2=)A0-J4|_bUl*6i`H442?y6#HDyBOxXmCW?l@c(#q~CTM`8W$PRhQ@gRCUSLybj& z%dL^AW)eh#@}W2)=6gQ4bqW{{OMGa+1dLleUPl;ff8CSQ@O|&xwg9AL!EJGEZIYsz znX3S{kO-1uteTpTIsu{2w=2h4n#(FHXSbN>L2ek{F^t|&+vHrXP4kFK%M`pJt;-d~A8Ih04U7V-50`7fa^jLi8QgPwaLdJZ#wv-Jjowo9jklKzUmSohjKTdFV^0)|gyOdV zW;J>4aM#pL2W!rhUAAgGL)w@dQ>XOFPC^%3YGJu<@=|Fu%Wv-Zz9C79Q+Ud~KBh~T z?xHF@x`Cw+W_(3^`^p(@k0)YwT-f^g>svEuZO>URB{5vs9B1cQ{; zcD&@#GM=2bd<_2pV7*2Y*nP#P=&UbF{{YDR-qk{dKr*aKnXFWrR)ItW>n<-Am`oq) z>$~dHM-jf}(4z-0#!tzQ`!^3%!(Yo8-J>p{{XAg z`!)ym#A|#N)hJ^C0u>4dNFYe2V3-~+UAqWKv{i1ss1J z-m|sOb1dnNF#<(=wRn`C5l1e?cIe2Hf+<&;O;u*Q$jcN`Blks_(7QL zSIMt0h1#BsC*<_6V;@&x7OR<4qsX*E>*O=~oW33xcViYF3|3?QE!DAyQ)4g<1646% zd~Dc!FStWz#tF2JDTfO+b{0fa=#LW1nM!;B3puiKz1v}oi-jAVz&5gr zZSWX$XxY|M(X^UVC5_+R3$UfugU4S^!Ps_@Yh5xbD-^biq|UMvOEXwY`C%n zS1~O1K;*4T*9|9l*T-wDu@-m>h}#s?5^L*VT8i$@pcJV1kxsx{W5wOw>pC@3d3KU> zDHxas7JAf~xjtR5RlDZZtRQM4x_n*^_g8$&Dk!MaX=$9AuOt~tpak!uWJbVxN%t|= z=${j>$(b^xZC3q0O&A7Bu!0Pq9U1Y>y+jO-uQqT%2Xr4$X6sUid_2Hz@{_x59+cZk zqNJUJ-t%B65_B7A%$ssCfT8~YUfGufnYfxLQZ)xplM-O-N>G}7o{Q;Pt=32c9S26b z=JFt!rdtfpOTiekp6z>$wN3_m@uMX1qnn~w9z{*v}Fn9eng+v?`6@P?q70Yn|(0!i;>)zc#6F<~NY;4jLPAboHYqndnC+fqzaN61zn6v5(p%10A_Y-D)*J*#2{rr zV`0{G*P*T%i8NNDqLms@X|#!DTt`}$Lhly71jZ$O5&aaEZt-76> zdcLQW@xYJn!R?ovKCcyGAyGMsYao30Jr-=6#3TsCASm@@RLMKFiKb6M;1ji7qvZ8l zw}2oSI0-ThYtb7bG;PgpCF(8TBR#$r+R~F=xJrT5A|#WLjVT(nUj_u%L5U)r;XUF_ zBMxRDxgF6JQjBI!bZ(@Jnu(lKk;9||GY43jyy}skQ1h!#E|fZnPgI@8AnuJ`Tti^W zbw2+9lb^lAbD*h3Q2=ZJ4w`%3PucztSj`)CQ4|oSblV#lFfU>3bn_Ff9aLoI*aC|TeNA~`>3zl$)x>0tfWCz+ zXgy5)&Cc^SGxRV7uRnvm=20tVN*Ev^C6fSpc}-PpCpQ%VQZ1)2+nAh-1_wp9fFy&x zt`b|(FOsGK%nE$!nwDu(NDKm|tld_Dvihz->df2n+VQPlW<$3w;TT4z)bnEjICh^= zcNmozyRD>^wNDFaJ0$wddW$a-k#f?`VJRN4USH^oezmw6f!g)q=FdZqk}~qSw{2q6 z2%YhlkHPlk&i&vt?)t-3l_7fR-Jb|*rGN^ykb5Da94Y1NslrQKc#(wX1K|FC;Kvb z8PF%MLNY&F%#q4&!3=s-`c!z4?ILXx3}86d%uBmOz<-r-V{6w%A2Lf)y58F5QZOiaxSxQsF; zHjiJ$=|MuYo`X+Zys#;$+9&4pP9Es)=4KDBu0}lKQIaW;ufRmT30@h5{T$d(b@XLx7Kt0Wn>b19ikRw=@!B21&*wT*lmCPWRBiZ*N8?qSAK02NRoqkRmruTci6OdX`a8D&k5M0H7-J3gxV z*nNGT=J>vm)_e+d9=ftA2gK!;PV;U8wlg@q;f_ zypQ2J@7O%EY31dsw>2L;k7l$>|6=r!TOB}+&|^1i5lphLYrsSD#E>%5Buvq+5SjF~0B!N{ z-tA%xfDTnLopZnE$P!oojwv8Od z6J4Bi6)S`Y08C8)AULi(39CJCf3)bBMyyRMpRhZ&4DOpV^K9Z5%)xpQY$m5)gLN+x<{_hCGL#Y?Qe~NXFgwf@OiOesmX-`zKmBydLW+C@J(LS+&2n|?AYzwK`FKd`T4!$h6j3$lH+04 zyf#kuw$x(S89|4>?VzNH>ZJtvbg|e#N(#>L(fi!o){`$2QPV!Q5pNZxoq`8TOP4X> zPGrx6bgK=ugXtfD0kbhutX7FVV^0Z|gF!q00PINkx1d1s+#ST25-R{p47DAXnAJ{4 zv5_*pRUQmIiL-8wV@@Y~cAISjrH=z8)tVW(|gWX5cXO|@-sNMziDnI5->@`A4&QV0jV*`t<4 zrD4PhK-f)w4;K|G!g@rqIf0u9k(k-1V={2b1_W&-=0{r&tjSgr5uIe0We2vPCXqR3 z>!T-9ushU}Khjt*j8z1W;e+vKLV~CPft5bD1b`z2C}U0jEy3HQ`E-YFq$;#Y%0Zt~ zOjNARFu|eetFSiU1YE;1Fvi{cD9=Rq&`NEk+9ML?w~)4j5kqZvYeKLek^fDm_-GA5XE8Zjk7Vx-kx z8)u=9W`+ea^?~tx=%!Jb-#M@i7(%m?S9f=yFD6*EcWh?`3g5(5OJSZcN-(n;|3V5Oz>SOZHyN{Ig~)-{T_TASpJ4 z;Eh399b>e0dS9(tLJklB0ssPl2%rFHA@%9#Um1J1jK%EVX7Jn!4=pmI*0`#u{D$G; z=3Z-O48G!hsVLyEDMZ%D8nGMJ(5NQ6s~s#DcxMbCgQZze!86HwOY+AD#>(t}>8_I` zq)oLS;zWp@sg3gfW9<)QalXw_7wsG~0_sHCS3;tl2MOH%`ulR-yQwI+l&vZdq^JcY zLwGbV8R)6ygC!rDojd#=)cvdZM#|>z0!5{3%27PCw+gCLI}C}~AI-R)68``RaNECW zR@5?7S9l=dP~Hqa8R@uZ?*l$w?eCAY-_ZHc^8v90o{)9O_&#$mVW=i%mEF+Q@}4_( zFalyiQwKA;K^c8*7MPxhik*BR)a8e54sOAccKEOX;==@{plWiAsq=Q))|i+$qh@0& zH*16_&(B{i?tpwy4Au;RT5Kn4A1=C`X^+>gVMxsW{|Pla5S-!*681 zK-mvPG7UOjzz$z4w7$s5K7duFSOUS@>t{j~y(O6z0P8&xl#Pxq0;0%HwL9A`;73l! zop(u~r^SsZD$%QP0yPf3)G81Z!Cjah8=gQcnK>gK1#L7+pPxlv&KY3RW;q*>{P#ii1y! z%Z>yGqD96RN$gC7)q`oJ7wl*8q_A?QS?lFKBvWZ$xP%=sq49D9Wuh6fj<`+-^GhyGvh~V zd7IpG0;o0-{a2+Z3f8P$U`j$j!fTwJ zIw9z`sns*TH=G(k0(Qk41o-r5=%ZA3jv%Z#L5Fcc6sB#Gk^m{67u@AUPh&)yQoJPc z*C2|FKk5$>`-V0dOidY)cM1Oh!~X!8wfsUQ7*4T8NOAzsqa(crb)5MS z(tUBIdsqQ66%9^RHG|+p4%VN-p|U1})h(%0-ljkgfempA8QUU0XXNBS)80OnV8utP z;`?0Z-y5s*d`iMJBW(P|WvZ?S21d##LE39j=x6AT826E#1oAH!9IJ$PkMJ&m zQ=k-7SvRBSqZPg<{Ax&%<^1&2JKJIQhYTYZ^#)6~c-rx3GeVN3hTE=Z$P9zmp2sK22G@;W%ADSIL zFY6ziI8(&?C0Ffr_*DzL@My8%2wvXn_!_vY>CB#w-M#QnOB^yC6=Zrxe5Lt>-@lKn zs#M8UBWstn#EAiN4&Ke#&Rvf=X}F3|aW{kI`=5Ngwy~u_QR?}(*LmGq{9F(bF(Vdz zt&*eOr6QB4JLAZ-Hq{d_nXcaoH7P0yuUSz^5CPQ7kp{)w?)@5??9|ne$M=GCm@%`; zk0?$XfdVyEdOveyC?^pe#KOYOSS~1L{?P$ftz-TLJCZ zDypfwH@ztd5GWoFV%3{`O3cenNW^H`D^4!YX!jV)+)-tt=m2IWSSGwUZk(8J6`*UO}hmn_nDI6SE1-@@-_TJb8#6OR1VjFq(Ydj zrwyIu)hnyHf;96W_`KMHt(eTmV;b&g*4V-b-dJZj(wVUb)cCa}R3(Z-*@DnanwZ*k za&Z7j88aiT-qSakAiZd|wXujU6f3GaUwL;jFgi;D@u0-VL3VT z1Xmkx3Fan9AOoL!vVgdyT-TsO^;A%<6O?*i4~z46t#VcnqC#sCsrPF7H068^bnPq| z#N8Y8$=+d|H4qC_rr4ge2!lXRV|!{&H0|xx-I2deC+PkYjVj={Y$<|lS|)1;0HE6& zeNBRjMMbww3MiANf1Yff0n(a!z+N}FYxOms>;-p zrA9|h6d^)(zthOQ>2|-BescC;R~2WBKm=Y+1&}_PAPJ^vps&$!Jg>mUH`C81e4z`h z=jB?x(~2gWz{DQW!46u{oWFT_ZrZ_gCf#5sX&su4+t{vKDO!^tsVW^@N2~^e zIhtoL8!3FraE$W7+V&-U0r8KERV?TPQfJDJBG(daoGRTyK6$xIcdgv&z$YUel4yP} zDMiG>QKZloAdO&5#b@i)*u(_)0PiI!2@-oL^J)P|$a!v`+}h+OgTFB|Nj)P?u1Nua zf*{hgIrPzuDZ^8#CCGOtbXR$4gebu$4%!$k+zfIU6tZ-qS6+n+4Qj~PAe|ThaI+@K zI??Jyu%4DyswB(F-qBzqIV_92G*4^H*c_>JA&>^_D4=0?6Q+UZAxg;hIRJwCl4%Y9 z08w{use=jF>#P7jHEIw~!`#OB%#`L>02Ci>^}}nKFkBy1Gl# zOxs6zC$R{^H?+5IDtsNftTfHp*a0+H>2o|USpNWL-tcg?Ii#rdQ|mbon|o`+at2_} zLvje4Z!Z+=0*?*5e971aTisS(FIrQ|w;AWs;G#J|fSIN*+|v+E#SMx1GI1)!cGwwv ztMGYM^p;+&Vq~5IVX7x+E=!aIl1|nYry1_-3|KH4!_H4}&#Rq+gfhOMv|8)X(TM3} z$9X+nItR(+#`PlMpoWBZty8XVS2Y`bN94v(AnL(~3(qh$Qson`WgWn6>zn0`PebkX z^=q}81OZbtBsWS~?IbNSMAf>I5@DAET0>>%(Kju|zM|Wcbc4~feBMNnoJs1JnGmkI zBV>6Nd+Eh*pn~$-PLgAJ{qJbxS*a;l9oVkd%r_IPH0sL$Ni6_NY6u!? z`HHawFxyd1nLQV7AyI(WLI^D0Nq3Vwn4`T@8!F9!tO6Tdx7^dbJ$qaL3aY(W&>By; zy;jx~zW9-|Nu$V(d)Z`a8H$9_fORmEQYH;x)CLRIQF_r%s?{b$1LKy_FGlm~XqCnz zgc-|YHjMJ??srdFCJ#vUGuu+GD%pY7Ov%t@9H$$?b4N88M$lvxZ_tSvbUapUUFIUF zBV}i!N7Gu8xz*HGCAhzDao24phE2{*t7%QBw;@Xi7-dVDtW*L48oxe=ZvOyyuk98v zu$g_a$r%QGDhYYiwH?l|19LVsXx)Gk-mgQ@`Hi1oPwsTC65iW*8A62NRaBkyjH9)x?d^fOlj_xi z0;-KiZ8~UQF)1)8t*{L~4dl@7&{MQ%-=jM2tD%{RHwh$|1bKOFda;#)Aea&>q027T z9~$_9Pb~}P5)xvVt3tpsKEt{+Ql;5ND%kstds@AAxN^FxgW|vjj&;fccb$$&7J@_` z_x7sFl%xm{9a7SPB*V$OkS+uafbb73%Y#IL(+pQH;?9g%hSy1P8P?5`lB%*`EC4U9 z%5VuJF|ZCiPh-lxOSqOSy>AQLovdOr-q0QI8<4%!J`N#k$<#3_F{s|9;^<*ToOcS3 zR7Z(xj}5SQGw`a!gSQd)JILQgJ?8%aMZT;R+~ZJ{-M9d$FJ~F_Jblkk0)U)Ej`ceW zGDl8q+=L4@z#0MCez)hXO0Ukz`8lhXF_H#=$BV>-gasX>lkRFe_Ksg5cXfI$FBY0O z>#Nl5ZkxT}HYU9^cVS#Tv~`wuf$w?kYmzSud#!Uo`{3i#-ao3e+ zgtN_35UYwD%QnJLl_fG&e&w|FmXH7>nbFICFZ*NZaB_EXTnYGbOrMuk_WXYT0B3O% z{(laA&&Q(m@&V^xs1x?9{{U0GEp~8zO`f;*AGQAg+5Z6kR|Ee5)i~?CClkY8Rh!%` z;IxFPK?xi{0aFsL8CQ}G4G9IqbFXX9z5Z>_^*>MfwQ9HRyWt8_1`$N5Nzl`(Mu5@J zb>A^kV&$$W!tpcz0Q8ss>5G?t{kIqX7l`|(5-!~QYv1Ot{ZG*QDN9m#j7w7by=5s; z+`F`x$+*T6!@bx#OuW1Q0PXm{@Z3k;I-7UD+wp(lZ~fDWMeE8)`&-%r{o?-s)ako> z@o#^azx6f!f9=>+B;ZBgz~aBc@jrCYcJ7#tAB+D03xDpJeXzAA?U0ozAd9nwMKcSh zRE@3=nRoGTf0)1ZHzf$Wce@Fso3^Xg#D1HLHJ|;z5%*2Wx}!g*;>PdOaR>hZ)bFyC zr6&F*EcIg&l&J1q(-AwI5-!~+2b*?aAMY3brZ_^$_p$O7B+uLb0Jh@)0K;(~cH|Sq zaS#XTxQ!$K0Bgm4+w$DYxm>;eV*dct=Y6oXqUC^!BTtbBr#FDmME;keK6+(+Fv{{Yp0x8nZ*!r%L+28*{;2io4KAJQ-V zOt|lUE#LE3{-(dL{ksdKoP~1#0QUS)R^QU`A9Ti>cVw>VnLV(jC3Zqq zq=>!MQj;*cbx7LaxtCM@E${O#f2p}b5q9r(5lQK7m8OQr7ycKC`=`~e;=iEc{{X`A zKXk+FrIm~1sY;A7iAq#=F3e&lXQ}!7cdtO3mvj}zyvO#O^z`hXdmk?Vpqb^}6&5(Z z@Z3+`H|NS!l9h!eSWYCV3Q2%qrD+F4vo{K>i~L)^=CA!uRdnKb!UV?ExWf+PAd8*=K20cVFnyDB1K~&^br7=Z(Bcw17CB&=km(swNekPZ$XWq$y2`bP(RzHgJHW|AoD{CekIywM& zOVAOvtyzO$cSdrVVP+aS7!@Ur>uRfe9EFUI%VLrmHUqw~w#3jAyf+ng{9C0H zyhd=R(UVvi44$kykKuzfCv1Ih&f8uEg={tG&H6BABfK|l?WcqD`H0gy@%foB7*i>U z!aROYwZxhnGJHItx$j2fvtL%Dp|YZ}9U32Z&GV8}+!Cl7_eqYz`V?lQD@fK~RFYzI z)NSo1dqZWQTOzYHh|{8{;N%3(oAlSSH_i2dKz^^8mV7vp2v=CXT^LPkIpwV?2CNV~ zSiRbOHSv5Y?&J1w`lCaA4+qIgzK5v#%HG3M90=_M2 zLS*<2t=wAFRBH1|;8iH7$z%^ZGBkRWJIiyd^t`vV@MJ*AadD=uM#`~UPe;Oz4ar+k zO@kZ$F51A}QM4DC-XizV9xgE{D^N(OCIjP__HslidBi(i)NKgc0|onL)yxFU1pF)O zm#2i7R`_?01N8~sk?|~+wLJ{nYrIR~D-P* zJpr4D)p%I+{{WYvJVzcs{@s870Kdb(7jLLK2f_Oh(DjfbF zQKz~31o1XzKWFJEVP2bcEU@Alje`2*(f?W^DI z_%HGY&90WvJwLX8ys^=H{{Xlg3j+PU#N2|rJSyU26|${Np0bR1H2t}M{l5kNK>50y zFCaZ%vbwkb0OS7vORvr+ME?L}F{j)7QaZMHMD4?C4A6?Q~C(NFcedZ6CF<{hQzY$L0S3ZeRZZZ^3_%K5fQ})&G_WT$51Lnv*zr`b~XN6AOHi~A1R|2>+raHNUW0~q%A6ppQ~6+hXv zhG=!dp`@5#{kMDlzXkq4`Ln0pO8Hz3xLnnLQ^xG)tNw|8XZGd)0QUSB`2*(Vcy)fW zg5o%ckEwo)GG0hyzCOM>8aT|I7+H~{Wsb~& zA%&XGrPvJXGMHS9uQKiIdfK#6f${B9L3r9Bi!ae|w>g|V=97mxj*ib4=Ny*<)>lAy zYpdz`Vnm6hVGu!ysjX)-A5*O2eOqJmL(;-)u6H!w48bD*K(a5S8$DP!$1Go!+%Ij literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000000_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000000_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..d831eb718402a92f2d12a81dede670f49bb6885f GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5TJetG}vP|b5tA>E~DWv8V<-g zWHdccH$5PSE%n(`fHi0jYi@pNaVpkomdQ2%J50W-rM1*l19a JEKLLwdH~_5xDNmT literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000001.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000001.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ecd25d2dbf5f21841e67e9cd244f31929d9e77b1 GIT binary patch literal 31098 zcmc$^byQqUvo|`p1ZM_^;2H?-?lVCLclY3K!GpWI1a~L6LvVM8;7*VP%jJ3A@0@$i z`u@FZ-JUfwy?1w4S5?=qdiSpSyZm<>fCZ5TO9NnF0RWh{58&@B08`G$&4inkg@c8S zl@;)J4IlwPMn!#x`VRfyf`y6o{}vn^91sZf_KSsug98MDK=}ChZ#Ull2nh+v$Vf=2 zsHmBkIQ|X*ngIYR8ZiN5aAL3 z=>rSErhvm?gEtaWb_|HerMyaj=Ck+K0cQ|)5XI+auQ~8ksEiXO0zpoy_0$DPea^d_ z4KAkNXiT`U0kE(DICvCTq_;W$Wda5k8;1f;jP2@;6g)H@H-E+mNO@h?yQ9J`?u0;P z99WPb@wW$njSS-h>kkJ*34p~0z+nUat^&~DVBRj?>LP$Y3Cu-EK=&|^K!MXv&gYof z!>k(m=k@oqjcq#GHehiac7kMBwZ;xtSM>}ngZ#A>0jhJS1gNqyWJ71lO}+I!?>UR- z$xUQu)s&FFvhP-vRny7Mn?1dFQCTCR;$z25=JObojR&e#=F=T{rEn)Oc0gXzQiK6WHqJm zioU5UJIC*T(y3Znbw~R@(iZ(At&OWc$N%cE2$-Bb%&DPoU)BA!u#eZG2`BI=$E%~@ z(&`^^v^Na@KOO$1{mAW1Z|*eFgBJhlKA`}Hfd?{;YQb`i1~>X9`M#&rrImwHndmPO zp7Zmg#G*$(58 zPG4d6+W!al26c)vE^g6x25k-{%F6|IYfW&->XhmfkRZvhG^4+Olp~iPYGY%azD)DP zc+r8m>U4OJ#1{&G&+7l768s}O%SLqR0qXq? z9Ziyu)gV*4;yCE|!PbqgnNc0KFqz(K8LXs4IxWhl8gu*==J0=sRru7^PN({zLeXwF zH7vQ*xGyfyp^)q-F@5{Dw$!~>ih=pBHT)mrzRg%z z>AcOvXI^eGbUTqn!Kq{b-z9mE3V!;jGsAL_t?^{bFD@Q~poh}ta3yKKG3bx*uW{jj zUpiz-pPzYnjddDqv_}8YSb)WR0y5ofvD_`CB3Bbd@%^;OLZiKpYfI1w{*IA@Yj;Jr zNC9SJQ%XDN5j6OpJlw6;6j>@?yHyeIUgWIW?Lrfj%o!wux_OwkATv6@+a-3s+Hqk6((#`)|=3U%~EN5*zDh?2IyI`x}jq?Mbd!2JAQ>sf0seMVyg*6y=2Xt0qh zgU2ciDU)g644yXAu=Ax7wbznrU*L}!bbGSV02VHFu|5_{iJ|>(X&PLA%;|I$S`Dt# zGeV;K5>yKlsEuIZ4p|Z1{>-k-4d@hwwj3;r%a67Eb{d|5K|vl`z@=x(WAxEr!{y z_W%FkZ$6>;(1!?3D~Rw{D4ix^w^;Gx!WwrU{%5^D3dX_U+v&0EzW`wOPSA+OEh&Ey z%oFDygGYUpUM^^Hs^XDV(JB|Gj+c|8txA#25>-pZ!umoD(Y9!Cp=sR4?PclKqwVYB zJRyNZtx5I(qrtA(7DC|-Iz$;ui3eo8F{NB z;6`W5o{^A7>4kjTp3$jGw|=MjbaT7$1vZL0Wg@Y6d9K7y>pOSX7H_GZ02J8~it8uW zsM+{e4t>30Z&!PW&$ma6B+h&~>V(9#^kA2KLO%K;x?z5_z*8A3hgYEUfS=eDXp4(X z>jkNr(?umerZM@0onnfgSkJoVGyP5M5l=L(9L*w+m6HkV8!eX13dJ4pUjR)x9M-!8@#_ZjO_6GYjAxk<%|#C?o7|m>{j%HSq!&EG`Q_h%ASdza`x62cop^%M z9OC-rKVD5fTu&)+75x5bueL=AAxgNx+m&Vl^e?obM;TRa35#W@bHTcU;zc;DrNwAE zD}p(7!A&$@8MkQkw+{&pHawo-s7~5WZNArecRGqelGdOQfB#AvUa?hxW5b=iYm^!O z)Gtu9WC}rw$=)Yz8}d!s?cO*Vp;*12jr*0%=n<{57Vf(LZ<6-M z0y5X#pbIj$Tu;Y&N565`T-tSDxDO1FrVVhP#;Ml`hk*He2jA7j+)I~N=wATC)<%`ZHRnH7=BZhmhxaAl zJ==Vi%7zwvKWS8o%0C5Ip$<@PG^UqM(1SOxUS!l&i-v& zX|w#o{Zf9kJgDe78Q-VL-040y=bA5SeHQ_e+2wkY*?+2LpY0i{v%gO(^Rw z^#hzdJ_WeIin?G7Yd(3-<_i4+p4<#-M8m*VKG(WYDOv)tp0V&iM<-5|yL>mAdfXP;h^+*Lv0= z`^Qph3YT7ajxWeu1h#VP_EVaet)Wq6l4a6?V6KtD$L{}|iM2Y!@6ha$OVyHr?lz+;;w{-Q(wq+?1tRo4)|!Yq3MChg4FTWY@=ZKstStKBE4~qJ{uj5v|6t5h!=U`XTi!I| znmai6M?-`;7hLnk6w-fqQaim9`EN!Z9&EIp4m8Yi4SC-t- z)<5=sZz2rL-L1dP#`5MR{+-S1iA-(jr2F5Nf980@m;L|2SMH~p^)K}Qtn8g&Ma|;t zxlMO?$xZbao!pC(dBLZDHJdA6-923XfS+5ksoymm=lb+<<9+F00IuSjS6YAD5ry-g zZ9l##A?roB#(e>EV(yZKKNjco?&PVeQs!aR*zL`R7M>d3aFtu-mvVaS`V{+L0%|Rt zt2{^;JnPZ_1%Rf;yr1%GURg5V^IQ;u3a|&Rl115vn|g* zO8;s7U=V@H@sAA&eyUk}Lt4w`>^}m=bhkYYzf@nswmf{%T^v%d`jT-Sf2hE?A?K{qsgx8Z2_e z{9RKz7imT1A!DC8j&p@reTOOzu2yb3W~q35STR~gQt!njS>H7i3bj}e4!Pp;xoj#z zKkp8sc9=M%xbaKF#ngPwv8(#+5o){LO`n}spdl`4f#`@Vh|a$Te{zYy`< zs0#%%VhOp6J(;Zcw0{tH0?~QcHXI4)+liBtm-M``H1u5k|NZK>^{_2fHzEqTHm|ak zB4*}aQfN5jV17?3Vx{=igiqJTuW#*kBW09Fr{{Virsw(tg;~%)a`|IHY^ZD*SU~c- zZbwuc67R@i%!Aaclu=clLfh$lLJQ@JR*8;n1%fLl*~1KgS`?G;M14=TCy+T9@z~?; zhUfEYX*H?>MP@lvCEfCOk{#*?a7cjxrhRB(GwrHM9Lz3W(H9abt9uF4uL5||{Yxa^ zy<^97>_fUrzGAl6cmCgl;*k25qn1(oe~Y(v4S9w;KobHqF)bnA?B*7VsmbQuW@Xn~%3z=c4f~+~{ONU2{W(QaO4y((_T4}wXQ1U+0H5P^ z+I?F*Nm>Sij^rAu7J@iMR&abWTxSo10ohR3wSb_rd&gO{%snz6Z*V|&Sv3R4M;++1&tGO#oSlea+-tU~!WD5_0=G4@WLuKIXkiWDFODDD+(l3!pw zvWN(h7un}X_*JLUDw@@Q0WI9~CoI86`2$oer9iT;jn%t=eg;?{LU^8GLhM`?L1a-n zJVIi6OpWM?u!XJW4sfR@ZJzi%x?2!eZ03wxFaw1+I+BYQPfRMs3Ii1<)SOMDxDF$! zh;*Vl(4nLgarLpQ?Faq&LQr0*x~~I_Ai7g(itA%0KJ8?|BOM`mDKoqGa_#!Xq<4T% zvKXhpZ}Z!{A9UZsZ#(g-iw%5kHXa1Syp+5X#veg7o!s7>kI)sDd3 zsiucmDJPstSsG>KjTrVL>p9@aBmpQc?^lGd9yfB zl)r!?2wdUVWvK1h++V;W{WE0lj@64O^0{HI!)NWi>huXHaN`!2knEh&q7$vxYc>g* z8Kx|^3j>ZPu~A5!ZE7rfvA}PjHQl4>6oHvoyfPuE-~4m$>SYV2TzGOv^)*a}%^$ZQ z*0QStn?9V^DbEfTli9025#Uo{M`h`1N8z}CCT6N`yT#UXU4+r|#;8Se9a)6IU#JbR zWtUC(6EosD$y-n4;`&h!MV`Z_rS;PG%+9l@w6w==IaPN)FbIhUKc6}wumL9hs#ewi|Y>yi$jUADOj zByomn`1H&M65LbutmDR=NNvj+F)IrTopHt?l~j}@qptNs@$Za}*E3C0S0_mJW3Tak zKfap^Z)vGG_o;e2oaRcX&LYQgu=eXceX&&Db$Z|}zrozey<`F;#gzH7Qutb&MWzh`+{gxG8f?M&plZW_=l#NG?) zrDGMs7@8s>Rl<=P$eZ`LWlwaZmb@+*V}9n(<#i7)^jO`b`p|H#2vY)N#E5lYp&C-U z@bS^eod|~g`oLa9G)X2*l{gs~6M^bKs)HH^`bM<6i%2~c9X&^j?$$1&swW*C9FVOm zm-JA2TShL5$%tTmR0Agm%leS{lLNRwot1~9 zY&cS+7*sagH)e!Kg%(rNJMEz{Mck44*4Zw`Q-amo6gaR*stvORIz6Hpw{jeT1e$R3 ziYGCeO<--G5y~okNmaZxJ^2ub8W;ilYm1O9b@Xfo^w4c|JE2y=T?MLME|}XSx8GWT zgzw(U&Cq&q5~oYV4S#4tEk#9P8=GrA5r-Ns*+>ku!_kmKu@x~@tV|bhK zZbT@<%MGbnz7Q(|H|sZ=YAJkayvoTn`vn)U2wU&a1i=?h75u%_9|0~rtvqOe3Xc+1 z2H>6(uh+496+jSayyw{%L$;K&C3?O16RKFTcoij{ez`rvm(0k{L_)>Zbby2Dox8d( z1%Nf?_socu2%r|-Nae|%`GZSIxGu-7`=P!$m+xQ;>ZrzqXv*;{Z33*`!MvjX-xqbCIgNA z!p4&x(Y}eZd%&{El7p3~a2iPq68spP?l~1wj@$xWvNU5zSkOYCkpbqM!L;(`Y zhIY5v6R?8h$d1Hl)Hu^Ny@6h~xc!Py#xylO3}u{ovDI*t$fv@m;WDZDXR!R3DG?p4 zDt8)CICNh?iZJIkVkA5Uxc|$hp9_WE5Cil?*5Et01oV&J#CQ+-;M6I7xpPXqG|PzMOg|{ zf43cm=-8unI$PfY1q?iRZ8t9huhW`EKGf?|fUksPMd0%MGSg19K%4XO z{wS08r=plN?;1T9Dzbhr+|L2wrXD|6LKS#QfB*dD^!p23o{ZMJvv2LpSSNj<+C3mf z!C;(G^?Wu+j$+Q;!FG;GJxeS9UA=7E&(ja)0+@P+gczM()7V~ji)e4pW4#1-OPX&! zNRz$8>lE?rfvW-pTB^|vd7mX7ey2uAva=)9mr!*eWz)i9mp(@=5SbvU1E~GV&aSU9*{+j282@$ugOr-mT%Sdh~W|jB`cD=#6k)!X82sV z%*%a#kwKCDeQd31oeC|QCU|ewx8)Rwu9uJI{K)o|8F;1jxjO?Z_kZZX(1?NKhZuX$ zLBqp!!Of{6qyu%>lon?!Yw8TZZ~m8rTn^0whr@_bl}GoeN~nOHEZ(q~D_F^)#jZ*G zuhmx6M2UpiFI>x;5Lh7MS(IeV*ToAFT+6vuY=( z$%HsGnVp<;W_&&PkHA2jj6o9<1}5kLc`=4-sSrATr)h8)oi8DG0yCw8_ajXRa&v%X zvR-)F#(?R~ZuDI+3d*Ke#=r@_f$DpN<90?0PVC?c8>EK4U&L;kBnW(T+`P&VwT<$6 z7XUZ^ zq|&3K<(k*AZ2~e5X8jTxUbF>IGG z`q%7gT$E$Gnb2iT{`7HEs;yS5$1wpNKZT0a>lEIkM!y!h?dKUcynd%^1>D=M59a)Li<)mej8gd7mYqV4 zQrKDglLXK;2~HC$Yu_Tsd+O}0cO#3z(xh)O0;GfS6ZY}2KdyoyQZQ5|rze#K%hF=z zjsUB1SOOGfY-~;JBjFW4V0MK}6VaMs2;+B$9rls4k9q2=H|w<7Xlr1oE3z6D!QhXB zm%9q&c3W_MI1&{OEhVmy??=f>`afT(ukShI%tq9c@Ct_{V?kb6`5C&n251?ksJ~C0 zWlKr1C$_h@^YWFvf-;y;BCss%E}F-};}wZ-j_)~{NSv?5moqE)0`|@x*u(OIbroHV zgOYu`GlNxqyG~>fv8%1BUA8@DCh!6(OFhOY4}#vIqf({DU@G1^)I?=L(@2k_yUVr# z744~t6Ai9bU-t6VPmwV8kTz$WW|LKu5}PJ8fZqtp`V=SXNW<>JvuA|+dWBYM5S>9W zE9EwOgJjfkjz(sckTT^FeIbm5I7-HicNpR!x1`sL>w!gyoP^P{f_UuA2!yeu_C=Ox z>I09hm!od2g#j`cCT9g3+uos-qDLwGi6K&qanjI@a^0k)jD2yhJhn+1S}@&*@!U`E z6#5%?cYnb-OYO6qJh8R&eQU0+)X6{n!?)T2=Jk)SeIC;tRkR>0)_;ZK1kY;Y+NlM7 z^_#!VS>2(dXr~jPrUT9DFnLI=6>I{FOO|JClZ@NGr4{;Q&wXnYDsST`*3D|tU~@-& z&2Y5;Dv}G}E089$6JvMwrY4c0kfBHX#{JF@XdED33Y92~K;Q?XT=FAyF&ciKz@g!F zKj%1e?W6>4AbFOdTVsF4SX2rt_b&X1Q@^E{v!vn(MhukGpyU@Ds~Qb1 zlvH|cDVQ&0a*y>!NC#CACAjw>b{&0{YRuOR1fxWS!)U29Ao9~3V2R?$nbM|DbtLR< zHkX0PkxTQ5HGzK;`jX0V;f*F`aC-;*d1fwfU^vI{y|3ZG2fN~S%8USMst6T*SZy@5 z8(r!jxNMx2ww{Dh%J4BhD~oq#c+nMBLOZC{&f-p-#*8@0fP%VH0ys=C9N&qdacAHx z?&g$_v=WkVYGx85H27|5c{#)@4Vp3QF-1D%@OF4)cva-)=4wn6=&{P0Jp%LC_*~!&k znyf#ub{ad>Xq5CTOiZf)8X)N0%ZWB zT#r7H#gQK|LjCOC;vcf1VuH?1j2ON8Nat`-B1t>)&zRHekN!j^%eov4c%L+LQD%^M|>HTM^QgRY$`I%D*sFA?gMTAb@LBC63poGsDiY z+A_rJkEnk3>P3c86OqAon*A)yk)q9Qwh(+kf2WTlHx<)7n-FMTWeU9UQu@6|X+`jL zi9AYpiCX$lARZk7SW5q6tYUKn0*w!g@K_#h3XVIZ0^-Q-w6E{y7Y!p@Wt|59-adpO zH=G(q<%MHA@HlpY)TV(*wDYfr%&8GnD~5}vV0~mpHYaU_`8Uq)f1g^qUJr+ZXaFkQ zSqN&XzmH6J0JwYW^T5*Rb(8bYOfdvdW@@OJMK3B^foEGxP++dPVa1uJri6%cIh^C`LAsu`z9P1i=>E?UGKk+IU$M!y+-a@Gin!3Z%9Z~gm=7e48e{+aRvYr zio~i|$8T%1B#r%v?7Ta?JUED?F*nDt62#OWZ87eKr4a|Z z?^9KrAGso7x?H6)Ek=kcz;W>gP?NTb-m7cRSy{^!v1xLMZL?1s6R{TlV1W!ZLA=*~ z6w9MZBN!Rv#khFL;eIulojxZ7#38<{DI9XGsCEA86YkIT-)0Tk95eauOcnDNFyH4) z4~aAiT3LI&CB(#fz2elxke(o~`69oY;IZggV-Xo>Dv$D;BDkzck+8pUvUhdYVyC zuz{kUXB6x&GISs|x+vt#wO<4b@^P}v*T#;f+#q5j&n zkTd)(3*J&UVv1$6#9&A7`irjwf9kWi0y4U@~!b9j_lO9OVaHr>UO(|E=&|k z@TiAL2d^B=F|p>_Ru?Hg)q`%_56mWE7x6P(uG*?Q7ax73gn}iC7fh(gi2{*NYItV) zpp-Ch+3xma(HEPIH?0N=e(6b#>l_D|w^vvp#a)d{mN7c%_%3B_j>bx~>|qal`4D(I zbOvqq9Ng5+RaMhWnfK4ttI2O~-8bpqf7JMf2aIQgs-ih!j|ojJ%=~Kl)|MNm)J&Jq`tFOlC6M-~Tj354+bP8`*hi^lYM zE>)m}iWF^8?zcNoVzwDn`23o$O2-wFuCL0nvk2GERHDw*fd5lglAq6zo&DbC;p_Vl zZSRw2cMx5wjzbR94=?Az;S0zNsw#cYq&%EiAFgDaMalYhI{|JOqiC$cvPlq`@Nn@j zoj=rmDUuqIwrelFaJ&dHlCU1XN8dY3*=?+mK3UQ*QkEfYayK5^(~P0={@^tv+TX2= zqFyJbm62Y45RqD5{RO4+k99uWYX5rG$hs4tsEE&KmC*FjXdLfJC4`!icmK$ou`e!) zfnjXuvMJnMF#ya!zAM0;VQ-fCi9m)7hpIUGE*5|E>Yl=UrXQ6K_-6Y6)g#OwQSD zZL;0WS|1S2XE{BG$s@gjN3=|L+PS5 zJP>Li@+8x|K8;coR)L%myW$H)JD%2;ep`3cfWZ(6H$ov>1ahKXm}BUW69a@Olx!{v ztMBH|EvYFpbs0zFbj0(zwY3L3lsV}Ou(TX4&7WJ&?qHCb`XMhKe7tTm-E*LQRRu3Z zuL>6&F38KHi|pT+6j;+w9WQ^2E`5_C3V$`3$uSbtD7T|mM8Yb?Se8!dY@-Ufrx)GR zq>GD{b)->Og*xP%U6VLd>O6rMP_z|2%^@;;@eQ;i_0nLih&?5a9s&%|*3w?&l>YAc zPP!5zOc@>JGEDXTeFg;tw03l`5Epeg&=Rj8vv&3tR9j%B4kG&s^9{#;GIQ%lo>8%X z_|vA?wUBO~%6HyX8S+Zl7buuRT|CD4x}_!E+5?qmXb^ z6FH<{J?1P3x)v1%g_U6YHv|p|JR8Zro6R1_}xeM1cL4=MR=N^lwE3Ll|Y!O=vTWxWT^kNF; zIVbZ@WzE8%(7FSV5ivN;IWT>%uJwG0;ZBJ1%bjplf){cFupleWd4)?2D3H2N@ z$0|yjg+$v}Z#?TV2?;Df^Ek4w;!Lq8Pmtwk6OqYzm|&!{*SYZJR$Ilu^4k10LOijb zN*3}Lz@nJr6%?-V$A@kam4h}Jvn4<=y+*bNHIyzP(R&=%X)7%@)?ZcmFF^SPf2!TP z#d4azJzbUgLp#;SoqJmM#j!|LYEjjsroL7daPDgXtl*j1Cc%#6yYtwtgxTHSNM z{}chpTZ$%`%r|2N(Q^A_lw8T&ZRiKre9)0qD*$hWk+t<}*g;`u#RQQrmWs2{!^QF9 zd2GOe415Qkb#~uS`Vy&3j1pho5)ty+Ejz@{#T9N4_?bQsr-iIjn1+qZ%ZN?fGE?AV zQkz?MsN z7*`{Vu70sbFJGT5E0BzPJhr+-x$PeuG{kgaCET?G9L>6MzLFv(ZgVZ86i;#|0Y?c^ znz|h9Ca<4eCpcp4T1Fqf`Eb;v1VrXWIzJWKQB|cTuMy&iA1C>wzVd3ew5>^1Oz{pC zSO9WS>}G%ORT+m6RR?xtCj}5u#D4DTu6G9WR0Ro-fdEjM zOUP#=MmkN?kD}=5xIws>^VJKZG-A5K6preP9f1NPia@AH(}c}{5Ls@Ve4Xmj(%i;u7L`D(22GMKz z({q3euef|W<16(H%Dy%&Y2ClY(I1CV{*8hC69@bE+pB>UphhG319!29_B0|TsRTF| z`BQHRt9rZmX1`W|rLOToejYrLunOCDS{3kLWA%l63`%`XByJmH!`hO{B*k$Yb;+l^ z5j`xuSL~b_hEUgJUEK8is5&1GZmJEWGV!h;Ax#P5wevKRj$wjZaV-ytr!ZDpiv~)l z2^r%4T!f|u#OE3AP07J8c&9gh5|1x6WyjZ>ttAaE)d$3s+xW0hS_cNrXg0#A7WO6) zojS{;HPU=+O$y<(k7d@VWHd90`xgx%ujko&OTd(|BucPE3=%R;KS z=kwBd11P=7pO-aMuE~=UZMnw@I3P>t?-h>lVG7L22@?37jml^M_a+^dwN_Cmx3`_& zYL6myD+(FlMJvNVPlRrzalFFpEWp)^o><=ZR%?mugM&bGba%|G3rG64Q+5Yy=I3m> z81jUiuYpaxvLhs%c@)+s)kC%e_hAadFt&Pij5tiQXurQgn5oyRBxeNbcTyFWI_Kmz zb(Um)RLR#Ix_$Ekh;R$gsQaa>mHIm|Qj2X&uLTGaC%!gcQnAu{IbpKW`CBQGIeqPw zArxvF?t$+zZ~Y`1ha)KlZ)=KxB6Rounl!oO6Xiy`#>haH>W@SP`M&@S?@@*Hp8)W) zV&`pRu&&at#0E#qnyU=Iomf(4_jB#$z(N?yuuHn{&ZCmtcfgzRR%-^v;+mhr26QKe znJBS>-jOi0JJjWO;~cWz6+7l#&DwOT#RuLp##|%4UwCngePa8&soty2Br;Z>m z^=u-Nv7jpNm+T*5?ygzvAXKIF!QZkw2PHOm1$GUb+9UglRat0y>Bxw&WlkGo{v@Jg zGIP5`%sTRgS;ml-9gPA}n=|!5hY&(mL@cVxv`}~B_Ja=1`edtxU=o>`Bis2ZO|gq0 z;u4Rw){45e`tYuquwpl#_qL_Uyc5OHqQCfncr1M zGthfDjBm*uXiw{EDRg+|ML9F5vwJw|-tA?(HMtIM zHp+r+{ihT}!K$lDS^MIxS9HviCoT^xMRbUgujEvIu;Z$~3bGo0^L?sYS%UilD z=IyQqGQG(y>6TY{J&l!3iAcL6Dk9^ z``B9w^}{2$a9nJjDF6fXezo7VGHWJ&-i9AnHv~tVbFQx@o{)(I#fgz}jh)$Xf2Dpn z&r;?Z?PxQQw3<(bg*BY2G@=TFLoo}sMyO9Jm=Tu@2$z=jPTa1WiliAK^Yr}zhf0nDK7*ENZAVTR31T|r^1nKhMykU(i%X)D}kquqp|U4$xqB&sQoTvM;S33%+aA*q`IZhqWN{- zT<3dIG3L(1G39aAo~31T#R-ipUym|GbmgEcq|r#jVv`A;L?;0yr>g~;O`&J*{KQ^_ zl@Z4qs%GnURRbgPtTz>)04{RQw8Uz8~16*8;;z-4cwe^i;ntSNaKy9=hp z^)Qmj%Ch)soxYcIxJI_N6&v(GHQG7;G}fLwg1PykUA3c@6bSF;Uxrh4OGvN-{u1=* zZjC>i`BX-ErLsuuorbHH>O-_)JRDEkxwgHn%TL8rn%dj3$Lg;e{q`-*b|{F;P}MrZ zQF9sDFLWmcUQNh)^H$I(J_>6v4@_Y>YVGtqL}6(!xv(jJ{{B$K>EwFvFJOX$5wgH8 zHljmR^P+w-ZzLr&8}wC$VP|7puFFUU%DU0B(9okw%MB)gi@&Y6i5$RiKKYCL!}Uel8(*RW|@?2r4U5M z6lf+X^fI$$8c%-J_QaN13;)qqMv^yQ#BJMEEpjcgBJ8TF>3Xktmbhjjzbm%IZI@f}y+IaNNSOuaJXoU;n7AR(tcmzG z{smwJdCjOD^Um3~rDb4;B__#cZ4J$rMGbk6rMv~gOfU0ja~?$0D!c^|^RIkEl^+&Y z<`Ar^2G7)_wjtt%i%N})MmM&;*YkSueh4|F5gPg)DczDH>2!_jYAtu4ylR*YsW3~C z5t(r0Djcouqd&b)m1-*A#csC0M=Djc{N|vok3d8sX@so=MkSwfNWFRTvFyOZn4<2hAP&}_UK>Wq7()AD;eAjvL(K0 zx_j5#aE>I&!pakjHM_zaQf0li$mVgC(l1*6nBBRgwj_7Y_X-#K!JQ?MLe2_ z3T&5Iyz>__Bz@j&<%)Nl5S)??>l@~z2{+zR4b=L`NDldd`rckA)iN9jpN#LeNuLmp z!;pA&P5Bb_B7CPimRm{8$9+>Q2}7({wtD=`QauY4SiUbdN4Nfn_URA>e1J(u4#)TpyUem1j|UNb+|r|HKo2_ygv z2#-+9&->EX^GwD3?0YFd{E63Nn2qktq+3a zv74zcVeBQGvL5a~{Y|Tj5MOutoVXxJ;JnE70 z(cp{4r27=EY_4|Sqf6xbNeljAiR^J2HU<(NJFL4RPolya&*J_3X1?c)Vr_F~^YeM;@m@vQhHLcVnEAUwC1_-FzBFq$`B41W-2riv+}Y`>iQ|g<_S`r=KVjA4 zZ9j~I>Bw0Hwk@dxg(hiwG3%=)x^V|%T4t?JW}8CqFHp+%Lv9l95Yh|0BI6coCKQX( z=tgh@_2#}UGfHmiFB9IB7$A0swr;*Urn5>a*g=5^atv&fCccPYCjSspQlULcg#l#1(7#r|djP6%E&>!ugYv zZM)MgY!V0#7IBzM2cOjN<}ZN!WRV$_p?4FJgSMeH)mA$*ksg@@tY$66P#5C;_L6L2 zt==?%6%xYpJ=*JrLBXXAX2D6;!`|ucWrU7&zf4pIUbeDaZ7Gq9-yzae52PAFf2W#) zG*G@?Q3q$aH@{nDyA9J5$B^~YOM+3iqEgdrZ0Uil6BSjUxS`9dVMkhYb~OxYVreZk z0ZG7LDGbv4b-6m@bzj`ari)Axz$F~P^61WLSKnW|f*b{89F)+=u-Gz}B zEoBqPirx2?)B9=cIJA&fCI5=!}f;P=$v?FCGQuRPla=4AC zdjbnMvgR-A;hqv%m)Bj<#zvp_quH^(ZG0~fD@B2;MpB7y+F+6@K(kkVH<|ktrUGs?35m6)Yu}Sb>lidU)kYUyamf|YD7CCdJSDGL~cFd-IPilVVL#6Quq?zHL?8Rg?oMJv&^)ARWhbl2lc zu*31rm#i!Lg6(b3a2+-CIoF|y-O+^9S$QsY)>|StpGD=5;8)5vHrV>Bq8h2*QU~E# zWc#Bjzy~nZTI}s0bM?+Ev6%c&_fHU)JXBpJs75#=`Jj|EAqryNZjf@*s(Y6dOJc6u zM)CqHqZBYm`yxt$B^AN4uYxIt+B;v!FaWdY$`X>OCQBUX9QAM*Z#$$csRQ^7cjmUSyT_P+q>Pl-C(SNDP&6U?m_#u3t#WoswFE!y>s6ktdgYJNZ0 zG0iQD>I_(UMe}H)vYIECl=VgS=tz;b<~2PuodT~)x54fafO?J_+Tk$d18O-pp^~jS zmW!&rJb>hi>W+2lr;>NZ9)LD#`HQa1;4QNIB%dFI~ zQ_LO&loRr7`xK|DQV^4oUpf_xxRXRrQ67XzeL)EKyG~7=W=Hrwu~Z?1Y*%)Gkrlq~ zPvIJ35N1Hqbx$I88x-`j#j2{1)^>T&5j834q2jeeI0!SF;at%V)5zv-!|JD}t4=>k zpY-U#dXgay)GN00ktsvjQG9>RE3b_R-pz3m4#hV>q@`7Fj9|Ec$g+eihaW@eN36Qr z0B3Me>+L*qIEvQW(M-;7mu0oV2B3WRr}rWL$T|qFX!u~#hYZ8*THfrgsxZQ^c-j|U zlx8-s$mncgvYQsP{}=GH+u+BoLIL$5&K#Pe&+fVbxJTcVf{nJ1Os+921KB|Bj%G&|PP>_cF{O1Ff zF}9(^iPlSo!P~y0(;32sW;hu{_8WoAf#=>!*IQvaI9{OzNG=D4I%Ea+2M7%W@6YMC z%n+H5kH)HcXGtea?=7D6>o=A9$QOm4%O7oC7mjgv)vI$nC0VZ=r2x%Y~@Jw8bt17h>A!#SJ5!zgsXf= zk7F?~#`%Jac9RAHAkIR65YIlQ)r4F)N@Qh(P$&R+c#PXSJ|{joOPENcR<3WPiy(_= zKmbxH)I4)=xnjc3GMOTH`cDQ#jQm%ZPfHy}M-OL@b1i^3!s&&3G3oiaCdv^+AvNXg za9YqaS3^QAqUy13lj?w-yv<$H=F88U8no$BB}qW&a7y&4^?NzkHt=NKv1LiZ690H$z! zLn96l6O^AslRa!~LIRO_K~Q)}kOxuUJA-NtBZC6!KoSHV^&+y^lqD%Kkrm?lIzoYZ z%2b7g_NV~j5$g#21e)hi+WVxI?VCX4ttniEB&Z0H6Pl^fSI+{^Yk@j-OoYe=B#r$Z z=heKsQ#R7HN$(Q0uXxM(X(Og+;uIxNkN`b1^G#i0c2dA%tTD4Z~sHH~? z9u8L16EY{3!R%q`G`DUq5>%Xr9NPPCtV<6cfKSC)C)XWH_J^ZYW+<`LeovL~TA%6b z`y5V>QRbxd{N3P(c+(6!$7|wk`RRH!D4$s`mv2O}Cha+4@%o&84&k{UhmA7ng42X~ ziv&BzTjCMtr1b*KV{ zXN5~b^~e-Br-XIq4mPyvP0~^Z5|RaSuN?eUkgX+^EGb#R>fAg|ZPZ6m1Wvq75O`)j zLCt!LxN#=hqi->_vgV4);Ev=P#zwOTm3lK|#k@F3@HhM0`*tAvc^reUR{;^@C}ila zD&ka0;TfJSgGZ;&54YOOLAHt{CO9YTX}Ih~JUbBK9A1;YLrSh70%Il!)TZ(5#v?Sy z9o$JSD9VRlzDd9hNr@HrN4<-F-k<;wZYn^Yb=gKjG`6j%DpexB&K{O#AssIFZp^*k zCp1`EMQ&W3@M++5OXwcWezH>MAnbDPF-~daCb}{q)7W4|t_kXRZJ_=U9CB+a5kX$B zvx^BgwF*=O4=TQc?sliEzZ|?pggtiBaDpSJ^>J|Q6XzBhuHW-eBWYCia&YbS7;#`s zWxi?CyMG6yb+x0qb<)Gje1g?u`u!5|T5?8XoSv#+p6<9WV_3m_t+{{^YNxXjHvvHUg5ExySI=cE&`^1yFy)T5PFKQpmJ` zVxH;y9>NLBDtNCZi8!*%^67G@m>zE45CQ1V-qMh-1BrDin4c6r*y?$T zgTk6(Rj)Y<7m~9K7-5$?mya#!!Kx#|?E5M$7VankFDj_(yagirI(*RE9 zwcG$uZI45O6$+U(?++d`Yz}?vdZM`)aL5}o38CHzJx`#E7Z5wj5G3S5BoCbj7D`tJ zxp$Qwk-@6rAt|q{NE4lN$-Guol+hrj4|Y|xS%L3--9__hNdX8ZfX;8KS0s=qkO<&U z>U`x{?N*f6c@rt)XjH#YarENH|D|SS%p6u(gs)Luy>xOfuu$q^Ap#z(U#}QO`G7J;qlbzLN934 zBh2odU63RmhLNwfe+r*4*t~xM^f@O-e`i~-E*#6j4)N1Ewc6Y2*R2QJ;L5K$Q0{hJlaaJ z3M)Oa?tLpdWg_N0Q@}y_zN80X%b2nJ5}qT+h;Uuh+Y)9z&6`^Q9z8zB4f|6jAi<0# z43c1H9t+7)niuhwxG42}+HPvo9n4;V>?7REIy5CQDN(7A4K~b7a`ZUZ(b!5Zi^x^N zq5QIyloTEyK?<&ONXSiIpJG?{K0Zw<8($qsoYma8pn`kYf&3zQNcKMNp2ETjN;fC8 zAY^MksDY7Y+S6K6ETY}OR{$wD075_q@Ti){DJLdfuc&Iior=P400PuXd#X&3CvnV1 z&Mq;=SuYLDPPT6N&-OX6ks_OXeA=+2@eTK`rt0ySlT!(SJrT}#wG|Absu=RyeHOnm zRAl1hlcm4oq@W=Jn>>5o;~V^ENo=v%1%q)Ur+7)iec3l7;rmQV3pVO-g#fO&GCGde zE=!Yn2ot;ZXVa4g=9L#fB}E5(3QsCNoxp)`_Fk^yxC-$tsIz+NaSLl8d7^pqjr%Ia zD$jSI)vex#3}|hj_m)>&fa}YTLvr-il$Z&Fx|7D0=bnD|(6qZ~x|J;nxR7N?k^vkf zR!cuqsnXp-J1sb}Qxc#|D`810s52eujOJ;0+a4#(@`}=2t{f2&yhnn`wWc5iD5nr0 z(l-+ojohcHo>#EeXU4|@K^b~tyc@*B8=<&>+B!UZoDxNH@s~W_0*%8j1CVi2xDOuq z^tdD{!jXy-nGE;FP0Pu;nxu)WZ|i*?(C`i#Bq65p0R)_cbKuvFTO1amq^KR?ssdVd zp=mg22j0fkk5Kat8YM!MIDmSO+R49#X^sWf<}IfibL(irRc>>s9A7^00W-n3>6a;W zK5Bx8PwIL#m6uEurVJK+gDB-j75ZFXGxKcGtSBr4lpep-$G8rQRnaI{Cf5~I5a9^t zMqN$?0Gz$;j=c&A1i*Uju_-IJHqhL=FbSpVyl{qCun2BzH zoEh=#qn60@eX#dr4@a}wUVxuk4+sqbVh(0@xSlyrA7u-^t$Ytpx!IH5@|XmKZr#@N z7I7Z$aC(u<#@|ItjxE&Y&==LpPgm-CO=w_m&)f2GksNxXu3RcA*bSjhW8QrZAtYtb z!FY4X4cooOOOfFjc;?s?naZ4V752G`n4Gxp_ZGff=>+@7rO0w3c<=Ug7J!V5PSmaB>>7;$3iK9<1TazFqIy(uKjiP^P-ra10T1ex(KrIFbjKF@P1 zhge0bfCIWc4Qn_po+mD6CN;(4ypS>(^wI96Ux$Tye^abGi&5QsM_D*6q$K#3;UGii z+gX6iO1QaeG9K`td$E*^_xRn((d^G_45*Cv1J>P)rw&;il?`qXRth-wbHS|&R^!xL zAoWT1c-qhqgbu$R=O=JttBE^~S@`j}y8~Tm7p-~0fUS5+prd|aQa#S8y@RVdu<~C8 zdh3|GilN+N<&zTa)m%_fB6x@$d!2c5EkU!@HJgdI>ll7|*;0rAs!t63ac3Ph#akz| zKB*)tmj0SE1#J&6uP6mYfD45;yVXHHUF$_Pg@_O7i|sRgEVl z4z`7AZJ|)2L+9!2qV>E+MQKtWZ}hg6O9e{yFPg8 zFBhw}5jduq0%xL(4~I}$5MU%md6cO_w2>kdcGHVD;Ovy-Qb-1T+KJy2y_E&bSYo!Z zntQ_VF(7SyV{=YzBqqv}aVNa4(r5f7F&qG|68Uy4b|8T#e;=3YFw4b0pB^aklD@B+ z@ge2A^e2}by6YRa@aEQIW1k4;NZtDKaL;2%L?AkuK?GJZm_GdaTQHXzhS&_$>-%H; z)P%?4!GxY=B0gJ~2UTPWC~~jKm5cC^vJUz7Fz?tGJKdHeQz=w$&I(NDUGAcz62&L$6o9#)Ac)<+dzPBs1eppEG*E*Ik9-Mc;0KtB&iWk8*0njd z@ba?oE$?;K_}~8ko3oU4iWOC<=b7|98qvDixlh>1y`$}j#c-#N5sHzRN#8qiVXNt@ zk%sK?l*hS5J>wWkE?;V`{{V7N!tIp&otV1Ht!koDBVkXrjvpU$&9-?TwfFSjL(?Y0 zu4?bh;ltQ2+@eYKlC(+4_fVop;P}Vss5x9-5=asNJjfI8FBs}+Hqy6W1AbZ{HEUK@ zf{w=|>en^0KoDkli5+4{_LNw(8=6(bQ+DUl93fa$n{WrnA~|8w;RbbE4=l6 zvdVD4pEPwft2ddL_Yy=n03A@H00P;=aC3*!-1&;`u6Ba@B}1*b05a1lGWNy0cGOC9 zIp^(UsCP)%)|--%+yiYHUljjil!JJ~NLNVNeD0a0IP%8iRKl31OhTR<^nk((XqST7wQB$<}^ zV?CCJ5iYc)wnIWQvz_D9M?wN?pQq;L76#scT}qRXn&(P0Ff6Cd9RT}WqytI!dxOFv znGL(W<;tdkRQ{=G1mZkNY)B+hho-)f=ydSRY)BE(;*sXee^8%sbOeY{6iE&Wh!KW* zXRc4Z*d*>e{zk58<~cM587xCr|L=B-6b0&}S^br&T*IQ@)kW<;xweIwW&uvB#ELg}!}03lnX&nVo>qVl9|sG_|2ZQp;?KX%>A#OGNw*183$4$ zS1BdQ1okLQMlVG^PFN`ep}_|)1(7FxCwJ?YiGZHOo$iZdC}ODl8DFa^x_(|N8xXN- z+I>(fKJJQ3U{tGjt?r!t?g~4Pj)$Go<2N8ZDv&^*kUuT#Tk1+IDos+F8R8>co5Ppwmy{{WlsAc(IM_IqgYS^SyiwkI}o zE$Gv1%1I3D8J;2|$2P~0H!CL-V8+4;k^y`0w$I@mOqVSEKIUR#wj~;T0tQ*b*u%Sb zEwI>1gq_(_Ve!!JEKY1zM3Eig_EV|uB#rz2PkW`@2|KG3<8WvRViC@L2bUj1iHPB| z#5y9rfjr*O?;ov}0`VV5j!u$y4-obpU=hZ4xnh^EGHeQDJ+BCl8~PrF4t1m19*I}FppUPZtp2rxzAfyp6Ajv^WnxohV`+&c}vV;W|K}V2J2>|ae84GavKg!DQ<8{ z&AKYwlOQ7$>4<_TMsfGm88XT+(9jXst#?C z1Y(gqDet41LR4iZpr4TTyeTJiZ6oH;7^N7Glc6|Zl>Y$L_B{r(Q|@xacb%vH3!(iyL84`Yyx_ny0Z!b zVh=2(^ZQ&x1GVW0nnE_sNW_w@`@hA!*NehEUo>}kOHAQ#H$Pg6b#+`c{+Za|*8KCnTovHX$~yOvH4Lof(O?YZ*lC@i{q# zDmTZCu1GQ`g?DAsM-7;_WUbpJGf!9nz0T)!ZKU(+aR#%+uTyl9Z*C%T6-keCkqj3m zea)DOjQn$a;GxBom6~ty#f<@O1!57rIi)7FoVti7cE>dO$ult{qB^%^=7o1>j1Ga9 zx*8B7)gyrto|I8#l)2-W@d)vU=P`VMe3~jgZO_gx5Mb zM7!N0Co|s5#T^2tXSIM=v!4DOUP<8|K29dP7z4o>aL+FlF2>28sAvR@lsy|;F~vRn zsn8k%Nx(Q$IXrUW@R;TE-^Y7esw52lVy9yVQK>cz!LDX9*_LjUTOC=@P#!aJ_daOX z+s+A}93K2_rBH)i-(z?gSXt5}fk8bi1Fg0U6sQrL{WCPQ zD$;8lxOKf0Ih3Jid}}Nn<*w!6MBBUp;6$YV0MkA)d?2B-etrwGI(!#+5vP^%bbc=I zp$S*H%i~v>Na+1tb@Tv+=G3afmq%1Jn7xCjPK9`HuTU8W= z4w5~A$r@{S2tDzM2_j&3iKOhqA0ooJV9`ldDr6HXBxe9*BoQ=numX%8)?%-y9_7XV z0EnM_x%^Gx_~nQYtC@Vg&R-95H+;EK*V{w!duej zsftmS1VyV!6Gy$xz%gP^c~j|R1z1Ma3{87f{rY2zL4I(~A@;2qm+%?GzIfc9EnY+=Dbklqg~; zm3XV^grx)41ngmH^B^)5l9d8eBbXmIjK8ruiloVaQ^TRj0j?oXw}3cFV>AX-N4_U2U-|1mqxDxFmqOm#_ ztDA;qJ;quPf!{nPyi`8Y-)%k83=;wl$@?4%fe9HzgZ?D~_oL8+R7iT&>5egzKqTzr zvFg{}IA>#!7`A?bh;IU=~Ya%>efisay$9^jsjlOq&v z0LPkrp~EC#P{RrsI7skVmkK)|PG=Q%l@sXV-aO?=r!P%f?W8K-cuYhw3iP z7?1}A#L9;=!;e$Jbq6JYY`c9Kar_kFL>{}McT9kOG znJ1<6Q@!@bK_!?Um-T8E@gYc6yQ%IJ+(`;f%qFwKf*&nc6_zO!rtQ!4_td_Pp!%|K z>a2pH>fBFu>NzVuG%_|nko`yQR<}S(?4OOX1ouQ;Px`INorN=%Kx2j#d(ERLrMold zp+l~YWk8z6Xmfo;=~90Rc|)JUp(K)#UmKoJQYCqa7qS6CaC zadsz%Q$R5(1u6Ao-yEC@L8vq69S#-!S^8T7jHaMd-Iq%M=O<33Zv(H`&rXKkYPvco z?7Sv(tE-Vj%CxrV5^fR3e$Q)?RgE(dux?`#H@4bPGUP_x6l%hhq{;6#sFer~V@j>( zs5Srsj;PD0ICLbaz;uQ!2Qd0ptfnNWbQjLFi8C<_&;V(PhLu}DVl8;L0OCthig%Wh z5hHN!Y9&g{wS`G-Y+{WZ9IhB+6_;oNKqcygGDJ084Oj|JZe+OR*D%-xK!?UnvP7CW zT)w-M$Z+G$l9NHn=&c>vP^-PCI;NZWIa~FU&GxNXLInj*_s+9t`H97lQ0JSihS+F6 z$|TGddXy0W!11w=zvZS92Pe-`v1jRg^39xD>ZUZV*)=g0R)K)+h`~-EED{@309-k2R zNbJ*vI|?YkwW??UED99dJYbH$v65v=02(T1p^tMsCvYB!1JLZ~WQ|wIW#f}#NC$5A zyI{dGV|#EIfbRntM0?ybf^x+xC!S=H&9o>O6cN($fI$FDY>^3?4p}%Ll<}l+-q?;Z zK=HN238e_*yXM>o!vi6kh9PS(m2Z&Zg@AxgPi|or%p~S2#X^n{DahB+4c0|}p0K$G zBO7LVGIF7rCInKR@3E**L{DB#f->E_;Vhy_4Z|VAN4BmvXr22j;5H-xN{FUh+*Ae? zXO~B%pcQX+dpvlT$Je9_(uu9$^*X#ZtXsrY1xoxV2hKT1OzeKM=^u9v zxg$}1SZxz2N(jer0TVR{9-0~l@5Y+Wl_|Go+HIf=hrH%V=K3`BT&caoqtE>#=sJBC z6E19k1ozQ~D0O>2kZN7c7tOktH$xK-DLV(}6GU zu23R4?%eX0lxOhBKQp9)Cn@*6p){0;q`$CzU+$KZS&H_T{#Nc}57{r8e*xN>e@pph zv+3H;r(ICg+fqSL!n&JloUo_}rwS3gJ}K$S00fY7$?aT1(3GUrB~(-CZFMk-*{i!7oaEmx}v4x+5M z#}Hyn5w()|sZqub*V$x|Q-&H5dmzG6fMs4+aKq+Ix`|MRE`zykSdf#%I-^R_iE7|Y zp~kH@R#Y0pnK)J|`JsxwCKB|%dt#PV@_J!5dRd^WT5w@UE*x8wFqZ**CfazzidI%6 zy=7F5pq-A!NQZ)jy4ohMN--$LPa~iuWSYwt5SBuSh;T}+rU4HYjtuR+*Bxmyfk(sp=6&>1gE6c>lm9t7Q|p8 z5-Otv> z`TIkfK!7{I(TvMc>?UyTrr0g018^NKB%C-KQWF`O-NC`eJqp5VKplR^ENLn+9ZuWD zhd)D&aaVJJ4@lpb+e|M}>qh4_0uT-=7Y()OX^f7cW3vg0&V6hYZS;KG^%NZk*ukG3 zD(!A7{I0OfK$)z!RsgMOLr$qUpsOff6%=S(t7k5@L69Bd=>(feJiNr^h!bk_QeE6V)z;F4%-iJsD?M{eG4i z1P}!QGs`{hc@901|kO?oc0V`?6=~ z`cwY^o&NyOarZ9Ng8cX0lm7tm^t|r>0P|dafn*Hds}JhB5klQCoXZ_|qt#%Q73O8p ztYOp~bwmh2>0zlwMoA^$VCwa{T?#Q@x{G6hGNpf|BfNSTT_ktVqLCoJg{%H`Hou=} zKJbK9y=L9yD&5p2XgQ@qLFg=Y%zSFfCeN9D;rqX1LXV5n0)!Q%H!bFr z%))|+kR|~Y&G=XYVEMn$TDu6=>2BCoR}xcQ%8u#iFkitgT2dCInF>iMGBP7}W$uoDX-?AH3=yh6XzOhFv~LH{=9p0fM})(;r@Y51#9g6_ zTT62CX%5iMO$cWSus~;~sSa3)L*QH(ZYXUpy)gLNk^IxN2T(WB-V7*TQgH{%WQ(CG z4@RulRye3-&rq;jl%lNNN=mX~R8%8{3_+u+Q`MsaS{N)D*;zL2Z&Pi%VKryj)2iv5 zhaVf+X6YQWX;&^`K$6c^oaw+xGGcmPcHN{|5Rf*P%+?S{jjnFZDIjrj68E(TwvuFiY{gg4z7;(gZ6&meip1#BJ5^Bl6=We{Ndq}P;#m57YbaMd zJtmE|(h+E@eL{V&L%Bwxfvy4;A=Rf1#DTa$=T{gv0+w=+2S#PBN>N<#2in*&Qz|XO z8U|Zg38G>gvx;vMG^HENa&EQ^Zy~1$5XkSKWMJ)ZK|nJ}Z&HCNNN^G>NZ|C=tq5SV z2BnxnV{;A)lQ^XCZ$-=~!mPO&4qos%`m-mZJuFcIJwIz@^D8lw`x^7Fo3-u8(p#56 z;AGIARra?KVn{O3f)4oM44GCOr>EH9gHkt`4feuSoCc&EM~2+nA{0G4Tr?qfpe$|> zat5i502OAzk|^$%vf&Y?(NuS9bpZ z9>**Jz{FQw#a(QIoL4~35*0r-qiAO_AdKk_SUc$?$r~hfhJgCt(Dewn^sP8ZllbKdgV^zay?O zN(uC4b)PR+uhQs~iv851f^%8_07^%A^a^o*KKh5=%}wb>kv)QP#;7+By(s9!co*VV zaaUASwH2vZl@e5WlbfO(&v&o;*L8A@B0Dts>#EMspH)3LpNPGB<@J@_TCOPOR%Eq3 zY{|CFewXQgc(vb6gag~!AKhFegpcSsrt&{W=jq>E(EUsSZC#z+_HIC3 zJl)D1w`UK${{S?}Ndy=ovM7FU-2E@_W~WPr-mY~P^2hz~9La_9Fn_6(Xrz6GHx!+r zl$(iL;b`^xU#KGHeUlhzPzrOurUN>6-Mv zr>5ynZ_oYX(Op(8CT{2n%c>9@79hte14OCxEYhGD8iG`j&A1wAzqk?PIilLG7z6Bm4!~Q3nemg_&iVvJne~}; zO#paH6?Z2mJH|@tghtjYno?G%NFra6516zo>X!ZQi`bYAO!&I!h==C;kut3Tc*a5CD4{K6t7)d$@etf zx=PG+HYIq7oP{2VQaW^_PWgn}xKNcx8R;xRiAomw_Eos7105}L0?Fc^Yon#JE~KPD z$&LvI4(c~9?Nk1*V}Pk8jhJdnD(WPa;xJhrsB6F?TMA0za&c!Jl1>ub0aSP4rXYYS z-n)2XCm>9WBng&4Hq*YWG@$MTvJ#okKTC!L^yyJ23?%K^*rXG>Sold;W~nke%fLYE z^oDH%g>%;3na?f_Z~%36Bb4>I14+zqdox+@T?k?gC==7{Z%!D3V6lP3asiM;{wg># z7=TcF-mks1U{a0&19MrJNTChMU9{9?}u z&~gpwiHY&j(<-o%YA@+-3$|`5+Q#(I z^u}X|G=E0=XWrzhE$h>UVlP`1{{R|AdQ&`u`HQWZTSw^&cWvKDxov4$6_S!df#gKJ zS`%>#wy#xbDLjaeYyAHJq577u>C45ZxKk&*jL^UJCnL;38;kmf(*E@6HMmizjV;}$ z`ARz%_<1Uv33Abz;-LkpP}VO?o(QI_~77@L`5aU zz{`GlJ*Xj3pW*OcS9b;hkuc>1Zk`{kJcH_y?VhK&vi40m(STtGGBe%a%Yn8`0Z7j_ z!xK}##@Aw60CpLnEXFOt-sbf~BI4`CC?_{V3%5|o$%Ci36r~Oj&Uw6nm}lErSdobHP3cT{xwbeLmbb!!Nm+7)(k$*S2Q zXggY>VW#fc-a-pQp%fb*j@EPM>DuMBH206KS7{`rC@CK zx@?uaBTuD9oKQ*Lrb6xn^J3oAgm+M6@!|7mCwY1AG~9AEfU5rhIPe*odnVW<=65!y z3YCf~DE8&dG~VcfH%N$UvY?BJJ}MfjK{>!GJq$G5Dq2q9Lv>(!6|AX^a1tPIU>!lZ zbr@)zL#6bruNCBt)vnkscMo_~iwU#dSFOi}&$WYfQng{ujN6wcgD1|pa{{uLCOYfv z>jzDx%#rPu+>`^5z7;?VPve#zc=TxXdhJNHrnTCsHqGUZ(#@ch1*IgFU%72QveEzs zYRS3xd+SAEUkAH(t3Z9)r_uT`KNGFn^xZ%|+iBl}MeFdAZ(cG#i*tYYpG0XS-yWvX z#PU^=+wyvW_j~IYr0B-T)o6X1rXHtBsMFl|b!ji-KnqYpQ>Y#JiB_3cQZh6qV{_w+ zw}F#>^?&~WTc4!;N8YumA6vcSY7Yqt(qxGI<>cP#1=I}AN;Pb zz5f8?U;MWNV^pgAHn0BL{{ZE%&D*p4Tf9&6SNc!C^o6Ntzlmy8V2nyCj^*AZxZsmZ z?$4kk<+hGmG}?jdX|;L(0JguBf91A|I*@1KwP1hswI7!Gh`o4(lX~%yKeV~S{Z3u2 zEq#oAnkPqz@`b=pQaIk*pZPpba_wbXu7i0uKM z!&g83ZBOMs7io5XYj^ua{*&(gU28?lAu3WqP3gj-neYDqOf;jW)&1F5Ag(w=|%qipL7ce40hUq6S`SN{Oi+u7U-rk`0H4x3T=O`lm>k#f*Vl#v&t zB}z=f{{W=zd@j=H{`U9wmp@6T2u0Jy>Fh=Wy@v1meO!O_bw8Bo?FBv`R6)=FS5Z&@ z08^LJl%*!=Qk0|Jj7lnw<mDAAXE@2V{H@RI%P$edI#74&; zdGDyG_UVZo(LH7I#IyD0Zl<%`AMq9lm+8a$u7c4gFy<|i*boxzp-|qGORcHNurwqF z-~j=xu`QH9rNNZoQ9Bq&JI5j+!G#iat;$#jTE@Fk*{4iylV=bi3aCN;ne}rt6f$2( z1f~r-z3-7hH_XT+L&T@=sRQ^RAjXVLa= z_OJ{P}8K&@ss} z^;_L{gzk(|@CnkWi=%9hYv^9oCeVXZ!Sif^1zW`Xni`aT<0lmU%b1j)*jtHPwP4#FK}iptwc0ZcTwjGZc{4^|mBMN&=N7)XwJnbu%wOYGIyFkHj8jJzejg49~F@X4Q&`; zoZoKEj5f-Tta8{NGjr_f7D7aCWQP!PKoW9%(T4d_t3wUizu6v-dybtWvImJW+0WWP ztLh*GDjaVdBye*6@PDa|tjsyua7Q+JPogO&QLfB?{OkPx0Bxs!64=BNH$Z;7=z4`| z^_rC>C{*bbk`^%}gMUw$m^_UiqZ+GI{{Z~|0H%N0+5H#MR*|jXZExIs$j&K#tM76D z0Q;BcdOfewCsk@k{D;#&?5uu^@gHCMweRy4-BqnM#oa0X1 zalB4%(ft@0)BzR4{{RKo=0DD_f49+pU_Nb$`u?dNt*t6&JT|b+2(A?g%;xSIbtRch zf+&H>E?Mtu{{TF_{@X?Ufcd-rcYFQ5i~9ldbLBo_JvY{)wzE7hZT|2+0Elp3*L61_ zuFjQlmMcY3CQM>cj)x!f%l`oFv|rc{o2lvCw}|y0N+S-V{6Fwr>3(%N4ny45pRekX z>eXGW%(x!96Xr^dIaHvjat;{s(r(j||RP;IXY5xE_z5d%n{epgO{{Wrd{{U~I{=j_N z@}Dq0H`b%JwO!Z0;}3`dgAEJ%_NwQvYf8A7#cLEvlj|t_oPW-*f49+pU_NfAr*hs0 zsQOQG$Zh+WhKM>?)UqB7xn|@)6;G#Uh4qO z-A}7`{{V5USN{N#vHt)?x95M(?|-+^e_%dut)MLnVK|qlQ7|d{H10Cx!lNwr z(K6f=BMb~CYA16d2nj04&>5B*Jy7Sp;LO9L00v}l4r{hx(2fFSp*0h|w#tuqPS=wR(Kdja}df7Nsho36U`85Y#!)aH!aG#ck`HwMdovjj2dZ%|{q z9^zDJUAf7S;^3p6K@PJP7>70qqA#zZxiXr0)ytO`$7N|x^OMcxsIDx`^f{18BMB5T zNe*g^yP%OYFg^8onkr8RiQYJG-u#mzE0;GQ4R`V5@Ao+&MOcA3e44ZLwgj@0YhE8~ y!!+5`_Vhg~By_`qYaG}E4L}n-L#eJ9s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk^Dac9q**$ZOPnkLTigI5(Wd~BiNG$iSP3|UZ;EKLLw FdH_X&@1Xzy literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000002.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000002.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1b1f91939f673272ab8da6a00d63ba4a3f75bbfb GIT binary patch literal 32192 zcmd42WmFtnw>H{naQEQeO>mds0lM+V-9jL^JHdmyyIXK~NN@`-!J&Z=T!RJq*n7Y4 zIqx~&xa0o0Kkn+$)vLPKdge3dd}eixRaMVFo;LwF5E-xx01h4ifO~lXo}U2N3MOu5 zd>m}tY@8e%faeu}6aWPc4IK>~^Y4a(jq~3P4-b!!knm-TgM)`hKtM=HOicXp#mfgN zDJdl-IXN92Jqru>b3dR70Dwn;duf1wH$((@02~tXD-<}u|NP;F3OoWL5;6_|4gnqk z0p%4EG9nT@G91ziOI#WRJPA%!6GX@OfU~-MTKt4*E~Kq*P_b?2lYSZ{;$dmi37mtZqo06aVZ5grv04)HH@FD&5Taq(z45hPThCXNC4XYu*d z1hjRCT;H}N)y_@n0u#8OzXEVk;C$fy5#VS6@VEd3T)^`(00RN;W#^?Y2DnRLEq+Dd z5k@E|z29Jizsa@!HKIrrq~BRshpDT7SF*Fyd;`dH)%J?w~19P5XPS z9Q^_>3#gJbT2kv8_4f4b_gmYew+r=E)ybne1UD~oKMvJus_Gw_MPcVIlg{3L^({;R z@2Wm7?H)hPtmYsQHI-WA{sYi$K3QcLZ|!=W-6t~k-^6s+j4}S_0DtcBw%-*qYV``rn8&_KTtOvBgW zyjvUymC&RQV(RPM$3PhJ{M(U@zrE-&&2>6+i~eoY?ob+bp~`Q&91zTc%YsuZQ6R0r z@eCOApd>|W^K`E%_K~Qw`~kc+b+jxOTPAsx@t;BZ^mz~Io6hE}t5KN2SSq`2BQn7} zh=u{I5}#D!YVK%wuzBP#)8{by@QCuX!+J#25n4+~UoN%W3kmc;3xt~~@=)Nw ze*5#Uj1c?$oy|w4v-_hQW*seTJ6LL1a+zswT%bb{6-r|Hk6V4(hi=-L@A01q<=>O2 z4y%w>l6?24`_C-j*LbWqy>}=KYcRdR3ju0DgM*8Kbl+^vX={z+Yh|4JZeX&oDuR6G zfxcx`h*}68!FL3}=06JgS75<|)!$EDeze&QbRbMZ#pRMsAj*w}Fx6!v+6}~xETNja zRUt{q7$gJKc84Qri~peXKF{B!?CFJB2kGw)cDJ8v%PL%*R8Yr-hU_GT#R8Zo(};O^;^pk{-f9bs+&J_ z-(Jeqab1Zka+NT11X4)mST^-_b>44HpP`!Uc!r%u6Eo2f)L{-$(e>A)*BjVP`|>^o zZm073`~H;SgZ)pJC9e zgv@*}U6kHDdOb?6uw$8ygDI5GZ!7uB!aq zKoM-7SBG@e*=ca4f31yjBl?LMCQyOaWH;*JV_YoXuU;m!bymVu>UKnJdSMgi*GudlusjJUefdD&yyB`R4eIIgi|K(LpypdYDOtffY&`SgAhnoHAWc1FZ zih`Y-xA+Wv4M3HfrnyiY3%X#M(K-@j+#OJ%p?|vQF=_4lfH^{+xt%^n5!R~&0fG-6 zYD2Zxt*)5mRzN(@fc#-ri;Kb;vHF{)@#S2aQ<9&@{S{aQUN0WS=Hl;`@NJ9L-E)(B zMr1G^itB#opqte?68@>5xFxr%^Z40Re@Hp*Mt7tg2P`SbNtHo1>vV9{oU8EE$H``* z94A$!T@5FIU-U=znk-}`S5o#*>`?>WPl7un5jU8<9w4az`-W=iM6@wVisXW7KjUdR zf?Ij{ zF{ewVGtYn>yG2gbxwb(^Vw@zlEvQPdzCu7GSyUn?eWGrP;;Gm9s$!FFhc}*9D9$$< z(>@h*Ao9n``E%CHmLzD>3J%(rVokAITk8&2?uWd0AR@VEKuAikTWF`pxBe%laZwu+ zP;W+9SXf`)k3MMh03n4sBu4jSHvRe*+sRSH45-pVuEv%CI=)*HW!Li4> z?_Nvv_-g3)O;H{5@>YK0YT!ttX9v0BI?=BVb8Uh+adigOFo}WyGXCY8P{Q{*C~feg zNwywPhvYwoGYS)TQMvB-M$vzisAk%3?4AMg=RF_)hU_WXcn0WQ-(kPDFN$F&JY{2# zl2iN>@DZ(_cC9hJY##WEKXhG02byGen`|+x+MBU_@0;`J-o0_v^ZGhVtl0Tv?J0Hp z(_-jVweN~(s^6pi%8rO_cY??g%pTNhL+A$tu|SMfWd*}eKZhPX1B!1qnzkNUa@|h< z0_gk;VCiaQ?bQ(_-v*eRk~kVBz*WQeliV1Iz8C~)gXZO5+8A}Vc0{%p8`7`IrnEtf z3-=MQgFhXKF8&w7|EBQcF9rgUH?f3{S}7Iv6(d9^w@^-carp3(bP3;z?g#WOEpz5B zmZn{tj$*eNZXl9CxY%kTwqJ?Bzu4_Qat=P0V(;t!Y4+v2x!EEjIKO>jr=jXpP}Tpq+=fY>FbsRv2}^H1{*$zaUSqhW^{( z$5#j4QiCh|>~Dv#_Bva~BK~r39Rlq^vx?87q{8sC;WM~tq4>01WzZFlj*3oV8(7IG z8^-t{WBk}Rc0Y#ySileL_Y%Yx%q+2@o(_vy`zxw1f2XnYhI~SJ-3sQ16-nb3-$#$` zR4Lz2*DnWOfM{dgcKS&aHdSm?)nul?xGW=9=AWjzo4WXvw*6@~^eXjV+IW$h{9m3w z10?Qi;?A4z5`hWJtFjI|4$-cy!czAhEy{~YYqD? z_t^73q0v|gm#dykMpge(+20KRt04Z3^pfs>je7CpA1(gNT=Hax#mwmyRm-r))L+p~ zaJ~N*Tgcv8$0PgTW6eKVW_$7V&;M=jMb-bWNH2ch{8e24B7XA>SiF6yr{X)2$eKmJ zms0yXoGpIEt$(TOB^08$evf|sFJU!G9BzMfziRnQ7=OF3_y36ExbZ(wUMgs^c;nY6 z(Z5yXJkA87#g;6rFXKkU^`Z$Ii>}1~>;6oo6MP~p{jjcigFvznO@S(BPn333 zU($HdgtBM;4s=H$*is@s90VMT??oe!HC))i049If%=jXgH<97eJbL+aq#EA%U?I@7 zi%i%dy)?`vHK>XtXfkWMy)EAyIbI}aVQaC9BKz?;ax!a3zrCVXqk0STJ($!+u=7=o z%4mx(vkW=DNKPp&B4V`b2L}G6EHFLwK~4&gHv`LV$vfKl9S)zOur}e|D0SPAp>k&E5EsA|dXk0OU zF%7DsT=)VvWr@a=hOJdn(pR8>aV&+zN+xpWJqpiPFl_;*o0T9q4Bstx60W76my531V%?O5xA@@qXgCmegUtWll zdmB}pHxXob98GOPZ&6r5hkD=_`gnPZ#1h(F{F;TjSVL4NA_KmF5b|F7*iQ@Y0fjn? zS*~20V8Y2S4_;CnL89;;dulZ)Z>}*?+djq3)2IQ1Rn#lq_50#M)_Vgd*AzqwXIKAs zi~PC>6kdmbplZ8X4A;X-c&xVeuhfT`m9>9uNDaIWH4$GnZgdncQ@GgLq-v*dy{^%d zIHW0@dk+DT)ec8ILNwU=n;l3YxPeDRL)!3bHm-R@zw0Uc06|r02EE%jREQ#^j31eR z9Kg7kgHsP%oichnS{72QF!nOdo(6Thpq%ad+Uj6^^)SiuGFnepKb&Hl-`kA$@uQs_ zWs*rMq=09=yy6L;=M-*&P|xOYOkmSx(i1ySJ=ljBS5v)e+Qc#O=`}N@6sR z69SEc%4X!DIEjnRF_8)$sBcYHf!0um5-HkIpSckZYryZqSJq{!s0aEc6rb2Ys*FL3 zGv~m;vb3Df1F5uSSZxBRhb!V6ZW9Pn)Qv&G89WSzP-Zlx#Pv}~b-sl4prA!1k7tG& zpKkS=74!8eE?orjD4m$ zBo;fdpeacYRT6?a)vJ__FYFd>yrBULHdAyvS!&R#XtM>A5VWI>Fm_l^&DgIR)DxAD zt5{D>!+Y%>*lT1F^%gU_Mhs54`Ex?MXfmxvfJ5ORR34=bE$)IJl%=3%{U)bX)7jJo zy=p(Ma=CYOm2ecNQ1ND!BJc{ty)iJC+AIVNPyIWU%$qm$4UJIHYp+y%Bqxv{Me^Q+ z%qG;gw&)_<)l@uau`AJK4UJ^)ahgsf?ddD>2mQ!UY|&td3F}CE0cL%fw3cEbH~#uI z!9dwz+ZN?aXxYcQfW3*5HA$6pFc^&%r8@3-5?5u^TgtqK^&`nIq zrPc(ealX*K_~u{9i_*mgn?7q_dLRNTh@OcVOx>T9RN^JAmfn>yiE1W$>n!x}$tv<4 zeN>^R^mSg{!GuH?BZZyCcMtTh&G5ga07qh#C`DyF4*K?db4|qIYjhD32hunF!VkCCB&Dr-G)KHIJHOn>>Qu_yUg!TXj|uLp)yqHP4P#cT zT?^SEszfsJ#*Et;7iUKwwhJ|>U{)8#oVr2h#87`_(t`fO>Z=mo4XSOZaa%tDBl>${ z0jJOBfV3B+Bd*@qNwaY6an~Iewu7jWWkRu4=?korpL5kS!E(ZU5wX*ktx;}aK+rE? zsv+pnTaA!kWk4-QC=3wIBXkr-c8F~o`8kMamN9!LI%z)0Zqq(H&~ynL*ty~M%H z=I6Z5TBxYKnR&ONjY3y5QHErSI}^I1fyY(ul7pmj6XHBOi_PR6PbG>YZ+-6lsdm0# z>f4kvt3rz+_AxR?_+`_BQR4DPrpQ?DfM8Ez*{!VDD*iI~grX|h19@yxg+ z1|)NpDoIzey_EHzNS74-tfqU7#rt>0F(>s?!6W%`g*v19&j5mYj4c}7Sr^-ElK!v9 z6!LP!4;GKTDS;tmVgBA-c3_Q85XlFb4({cJ80tuw1%f!|Nu3`<{qJQ4KM*hT5Fi!p zZ)M%ItpZh`=cZ|+OsGoXjFyy3l6J+00Cgl5{k!YkLR`kOQ9{FGn);en#-(J^(!kAG zBUCI)TEwa5vgP*1q%vxM@zTa(rA|QyTX12+p3ILvCWH7ph?A^cv-#Pki|XervWi&COfiZ z6dpW0Xoxwv@3t3d$6ih;vV9}pNJjDA*gIf9Dgr6+2Au_=l{Cr)6RVHzuTGqY zJ31Xg3bGj}z@Z}~IbaoIkXoN@%iP!S4O()jo2admcm4EkM zAejDw;0Tgx)e)fF!Wblx_zNRgIGCHn1iAN`R6wd#D$7KH3VoIJ)^N)n=NA=vs}9bQ zV80M<$O+M&UO$b>4Z8IZAE$>w@ABarq%C)Jv#K9By*KEI#F4Ng2R;qIuvXR;1xOf^ z8hkk7fg7(&jW|m0HC0T@gV9N53%)Hs+h`IV;}SyuN2trEtn*Gj{!aRBDtzDeO2}`g7zI2=a_leK(-YZc4GQ zj#1FheSghwFuO~)n0Xi(-kTr1=(aS&?-k~OJHGZ2Tyx* zJ^q^M=F_vM%|vzjdEM(;JG#$A;!tqk;ca6OD!t3^&#D2ybPD&K zuQYPJ1iP>H*iv;McQq=Rj+Bamkw_t!>QLR%@tXQAc%s!b*lE2M5=X%*B>wGIF+7$gqy% z>R+ypY)KH!L5t@8un@kbu8XK8i?d*C4;1Nql9OS;`^f82EFCRn<*!{#c5ck^<;X=d z$LS#w!K+=bE>lXnl8khrr`)DLF{O%jCU`MEutqSE&vZ~kJP{9A!rFuEpMkH4Z9qPk zub!40Ij*f>cT#~2nJ=y`8#b3k_rNe+S(IE(WMB}sHZU{<>OcY=qLbj=0$9(PIwtd) zf{Sh?f)=f^WHPZD&YfQo)m4#;(DhW?MHH6WgwK&-Vl*YIobvcwF$dM`n&&0s@J3UV zzkkPR4c63{>)#)T#SG_8rxeBUm|}proDKyAcq|s40fA;`4=T@qr|_30oJZUT{MDRm ztqy_4Rq9OZ;MJG&>3CR>q*~drpIejH2Cgl?Z5faSP!!YohpOrR8So6?@$BdscCK!} zSAZN=*P{LVeO_`rx2#ZhW$SHqU>7CfFSux6O!(J(yHbfMYO9qVJ?q_^E?7klB$OEU z(-MB|lUV4}n;Q%KfhR)7U*`mju!LOT2V)Z&b3^vAtxt9Cwky|j_@nxR5M3Sw1CL|| zNeM(-WLf7$W3%dRu}4x8I)lZ!(e{ z7#GpvITF+y$`FcPR$J*&M)}2Z*V`)=PxN)eq2crkUsMxwW^OJ1WLn==8V;C{Gey=jR6eoUnrI(s^<+cN^8^y6-kgfqEI9emy~ z$MB*NiHY2Ed^+2DTrttQ4Z-mj=U{-ki>3b=yWNP8#VI<|JOX{^03EQIT}w~t%UzcM z3-1QjLBIc`vaP8?Q^1Vi8UrfEz2iZ#zLQVRhm`myonq$}?Y9%Oi`ehibx&&Gl`yDD z7{Sob`G-nH$#CZcP<63Cp41Ss?` zBQ(h-gopFoo8aT}P^>Te_~mJs?j{L9F2@ILExq7L5l57%3`2j`Q9 zXMjcYGeBu^qg!KRd#5N%^i(%^D)R@JRE|g;iL2l2qt~O%=re%f^btwnzOd%!1NM1W z4pZ2>jt3zRc~?+p#jU@;vB`xReHvNluNpsm?}GUzW=FRaQ)p5mNEng6KLecS(vr_$ zC4YX0_Le&$DT%&Z77dEZWDIXKtbD$Qhe8!xSCmVZTL61}>~$2~em6l2JFbX=#1r9z z@fzhg>Lux{LDizGdI1iV!Y!3zr}&u( zn2lnmv3atE3u#5CPAs9Ptmq8pbPYS*d&D%_LRR6(T#XOB>DGo9f7?o%|k!*;`aD2TAwWr9O&Sk!dU3{=|yIgydsa8Zi znMuZd3kR+eR98B#kxwLf;>WQihx0buO{kxr;p>+shb?sg&&URiiHS4{J>1b#O};JB zY?yv~L3$nDj2 z&GJ@4oSlBP2?&R!nZoqlw)i1-7Y%C6D#P3cU3cuE@VC;kh3y79pNS5Qu0ocxE+l-^} zHteNu*P8>t*{G!|e^R0TNkxCSam!)}d<`hZSyE&&i_;3;1_Xp9#b5-Lu_;kCjKqN?BH3_LTcQ>CGbjq!V3cbxzo9-x1HUCHq zq+zo0Ny{;qXGUTC^7$|!AXREW5-A9P zsRWwqyBO>8^-HR8i7*jbTtf0NLr$Bz=gg_Lr`_*3vK*;o zx5c97?IX4lP54sZmn^{?X@BytkfNi~ZX)BChDf^odP@HA`{klGyB>{PYg}VCJ^5xh z`R5L!y#Sj8FiFduTv_Q)ga&J&$~JrZ*uy;DmX?Es6#RO|Six?gX3$Z~XC5Yp@=CfSj!~O|HW>b4?~ic$PebgW<4&<2 zfltB#Mx>r{^|mA(cFLa>=)=DRyk@K-@{4?2Gd*DSSP1#1u@%O+-T-7CH046Jd~zT9 zvwH4BSWTea7)64onU185`?)DdXu&%3(&5Ulfqq`?=bKl*y4i1I*H+nh?sClg12*9_ zkg;1R4L|#qdMLxF+|P+R=hs_|M<@*jO2?x|C~3u9i0|&?I{k^mdb_fA5JDTJBy`Uq zO%z@fzWR66XR$rluv~XaRyVY>d?4s@wnxRtKQ$0a2=!if#yGli%BOou0UZzK=;b2dh`m68y#yLB-)BODR#-t^K zxl$5PQXvr@y=L;P?75$(_vA}L@G5Sq;Xj4|uc7h!zyt-bP{Q4Ts;W9UEstjyW+=B( z8}a9phZa1;MiG`tc)8XNeq2|?VlY%Q1XD5~{a4|=!G_mztAJi44ANUwqFzt{l?`#C z{UDu1%OZP;!xE`VJ*e7XZ)RG9WC;kY1mdG6CXGyTDJ^F4ljG;Ni$?iH)i~T#t<;&k z3RNQi@RQC}3HhzRw%#N`=0^C`2FjOr^2DA4IdFk_~Z9R zlet*E9g)P2@t$l292Ug$*3MiZ&@Tn*ik!>|%1SP-ZGYGt`=qz{e#9v)H{oz*e`})O zA9X2wjd$SI%`G+K-LmoE-u|X~5zRKZN_&DHiPWEW3BbYjr!NZTBchD^>B6^?pRF1u zu?!y$cgZPUV;0ZjB{+%odN?bh82$@5lyrW-KTF8YMimznafr<=H#p2VQ$#SNG=!ts zM1dio-&cl4>cZXnnku3*i9rI`&*+A4xc`={0}bh%Fph3+HQ!NEy`UB!9#V$vZ}*z? z!JvZABQkK^zkht<;Ri7r3aynpQR4{S*hfQR+PE&+vAUR~*YTfgbwCX8a`5DwevbT5 zX2iLUQk~|l7WdN$qFS|-&++l^p8Q{T(q{J=yw)2XRR{SL5dX)%Gj^fM)&&dNrI)DV&W!JTM}>P13B zT#{17SNhA!RF?SMa$|-m)t^*ZTfXLQ)tDl#a#r!%&AEP$tAMD$dv0F#V`q|WkP!I@ z^cblnoV;O_$hx__mZFw^qKs&?(ggw=SC{!T^Xa5t=Vq zEBsKb*&^BHVI=L)j2EVYwvn@_)#q@4h?q2d{W*XBI_6fJ6t%Unf($_TC3Lhh?oBen zCZiK~MZd#BEn=_whZ5@S9#NaM2i8tFW*i?;*4YkcPiHC**<@>;5-#$XBwgfpjl(6G zUc7h2V_kh~wc(fQ)>M9`3?XaUYrL}YsS+rt$d`GhqQC7bz!e@Y40lt;&#q_V=>ub@B!E#F} zL*Lu_ifBVT3XJhATt8u_MY^-G!w(E+Usd08qov+`lEOR}-qAzt#iI>kj&?tO17AaJ zOae`h4_B>n`FyC)km(v&#jJ42Io&rfpAfDj^Nz*vfK$_RBWK2Gl1nqCH~|k&`Si+{ zs$yJgrqF&=X&SckJKRFAInqR18V_}kO*`2LUsG6cp~kQi0>$At6mFwa)U$oLW$|H{ zhaPwAXt6Glne3|r|1sKYf0q`YIP>AKT4*v7xppdDzcd=Ah2700Ou&`Cm>Q@z6bVcO zmG~YNpCe8Xl*O^siXdhzQSnC4^EAPiCjBe=8CHB}pxAG#l=E6DrOV zW7A{}T#yakZA$CI9lp3iz9M<}qFeM-joRO29Dqtv3=EM4ZR?6AuQDTP#>M4{@EkeT zvaRZ`K|U69=lEoocezk)qHSPW9mN%xX$w6Arq*tG(sQ|cvB*kw#+uu#?xo57H`EL) z7T;)SUlMW4Eqpmj6FD}sdw4)*t6yQ9LWSRdjVi5WM}DV!qTAy1VVdl=R_Sv=X{I#~ zj6gPc(V?t58oc#=C1@PRMvoyWo{V9X5sW19(3s7MsjKD-!Zu5F50)W=r{kuqo1%%l z+Y5kmWi&@o{N4Y&a)%vEgm@h?RU9!?&G;!iOqkeB4MvEd46$&CzP$`*2mz3@uo&%yuEFt_n^h{YG%0 zPfo*Y{zhzZY5QdS0ql%#9duER6l4>eG^!3|`U905&pvJ@qS$&gVzOrXU>V`Tu?pABhJ7I97C-%82+WTHTSD}8~G z1N*Z`xK|k$Z>jb*goGaZMD@3gxkMkRwZ05qq{AIKb6!fYzni^uUH&At{SGAH@yF=5 z$OK+|u+07b)J&umBXZ|tjr)X#!*ZZ2HInmJN0eZH^`M!Ae4RttprGgs21?`bt%X=i zw?6--a07=~!O*D?@L6UCNmiRQ&8>gaVbjaY;IL;DVF1KKMIH}qp3V( ziB~h5EjZr5OUe=~&TFX<)Vf6&O=8T(F8+4L6iNL4 z%a7)4hvQwrZ!XCV`YZuo(bY-ETlAokUl_fNq|U&i#w;93cxkb?X$bm18~|Zc$m^0Q z(um2JT`neT3}8ktUi(&u6}jb^q_P8SxtLVOgeRbTEhn8d4R5g?M_F1#^PSa9bh;Y$ zR_nWSamF8VKrl3%>4_`&bgwMeu%$?$NGKW;o&^*F5cm@Dvi& zxB>6Z0V(|Sz_vw#e=HijT+!DQ@;kF-S}zh3+J=rJ8b?y*>G0rRX`dxeD5Ranx}8!L z-FcXp9gKpe&{?zvQe-U;4Fs1b;X|nK=99wDRlp^#)7jy48bzm*hV_d-ml1E9Dh5}k zDi!x}iEgE7pz2?25qvx_gWu^vORr*}zIGBiP%xM|o0VVC-y7uR2swjaj5oBcGQg7g^a1E4l zybRO@+UVJzLb+m&xu2m#nC%kFrc9n0C}Ug`2z0nb@|P`&Q~4kjw4y|cc8uby%CiIY z(K+8dsJoS5-c<|zU4ExVrcmI4+EsK%dBex=Rr2rp`vWd!HSt5x@{%pE5UlW$$|oakTrfyJ14dpAp@k&=bn!9M)P_}V#d0`Psa-DOIzrC!Av<6p zKF0ZHK)q4HRc~8rh)c{V%+}X=@^Yl?{5nL$r}k@8pAqN_2inn{TXW6Y4KL&yZ+@wX z@V)+KD5ps%=SGb3Ol9_i6Ytj&{!Mn}#+aKz{|?b*y{c~0s%{geC@?{@GK=h8@RT0; zQKN&Vnxw*w>r!>}=+<#k@MbX&j)K==^~_HgOF#;HWDvEZQ&`6D9nU=jV&lTE;!{*5 zX03;qa`idn9^eE_FEt21dPG|8Y=hy?dZyzY^Z|Wc5wOarv~7-Nz`%wOJ;mAN1+<+x z1P!a{eKA-OfJzp9-s&vR2K;E&G`yR+s*R_*cCkL*;wUedkw7oY#CH8BH7*ZY5I=)# zUQC1X3VRp6{ZOp_osM-}_moo%Jv$b?e}VMigwA4eU%2859EdonJe?;$H}aQmYx4Uq zngzU%22km5h%;6|xwTHI%mefT+1!VBj!XF;S_yNN<1Wi3dmUG4ILGl1H6IwoR}m znK+$KHc85DQ@JWUoxOfq9jajHkCvU4uVo_XCxOefBnxv*cpvZF4#)3qYSx+@if|E} zwe?I(o&ikOM9LQK(1*Qkc3-nJA&-ilk820^ees6mvEcBxmM1*3H5T5N`)&Pf4&}1d zF)Idt#(D|9Shr$sw zKj?20#9<-H)LWAHMQP0E!QsgSFLdzuzSC=N+~ob;8mzt52q!#I*?NK9l86qTWwO&p zImahGIk~eOcz|$0IhLEUaqhCSy_qC>L;)3PxpE{nBfIv zX$Wa#;eHN>K!u`mU^J=2Z&qb_IxnH+2~34G&ZIs{6y7C*eUt-5hKw-(b8p&GOVJ;9 ze5u-H%;$VEUzFc|^49nH!`OO!G}as|hHm3azg5f(oE>YMJX@$*0RrQyoast6XH}=i zT0x?IJ8^B+%eXnFJ$#@`TZKq6-lwRrVL)k4tmcrWPG<|WzJ(*U#+~TYr#3Ot^#Ijct91hHj=D41}dT3~N zVf*f!oo(_nXG7cj#%SlVqRwR|ShNP)Z0(0wP7F7JwbgE(V6v@j*Kc?W_2@#gaX8sR zv)O^iPr<3r0HYsn_gzRaul=EN%Gmv7;(_c8X#149Goz9MdX^XQ_K}hrP{n>0C3)bn zDNsOyMg_Q7_Z?BTY2(aSyGK=ucUrNEE}>VRWp?24iZg!Tc2` zq4^Cy>)X8;3m&*PXza{*T{c-4(~4)U#>_kZ2e{SF)CboJnv^29t6S$sXd4`q<2!2Z z2|`T31PXiZ!t7YZ(%c_j*R95uU+5k1^|j6Kb&}Bku6dwtlHA6TU`n(9@qMtKemQ-% zZ)tehz8oIHvo^KpkeK>%hajmqtc*5m&0C*$F`91iUzeigYeX{4FZEKeTHm#0}`5Vp{Z}W-H-YD4drV01&H#R&Z zfqdwU-WMH`X?c5jSi41Sg-?{aRYr-6=RL|SPW)6`^}+hMMfe-{Xz)y?G5YGq(ya=$ zNq-)m8i$DVe$lseOm6d4l|01Tn1V*K_)9D3gA7xj>OpQIC#Ofk`nyv%N%c$1hrcNJ`fA2f@$; z$-nKA@oA_GIKm8S^ua zWR7|C=lpDGXSDB~$YVu5jA^nHe9+_%2jeDvBR@V%Dhq%Igv|t(F%`s6O91G<;?O3z z9mv>GHS_7A>{>zZ}%6vY%wuq( z9b24ADwm+fb6#3ZZ~Aci^6VfYsy}U0yjrR}$bj9U{MD8uH4U|JOp2$wO>0JX~yh_*jkEPGWO z5qpN`2cg$rL$|8m9M!~%tE1AnMN>tS>e2Z;(1Z!Fj9quW^-lk|382HvEORE5N`t@c z7z+@b#kiKIK1(G^$SY@JUM$K@mO5Yxsm%xunDEx?brbAQ35Mx&3D5;~f6D2?uK3Vv zo29}jjW?R+^;4}*u$fgmA;F>(YwC`M_yoSTA&b$}x}$IA&pSz$MVUD#X!-n-uBFlcF3w5Y?Wo`YWQ_C+1fN%;1lnCCY@;Yo7+cBS7CO}C&I4`bDk z7A;uPhok~VNGoJa(eMq3j-*m^{AfNex$HEnx~cy@Fwt#(jYxQ?yxZ8;%y4V2UZf}@ zEo%ITaQ`FBcf)BpLNlZQCLX7hH|PGTO5Mwd_G^wCr5uNwYvGJ!2o)iVMja*n#eL?$ z#RrER*btFcC>vJaH!_m3#By62M(4Dp%R+#@#E8^W$!2;r$o+SbAlLW@>Wq&KRlbT= z?Q!)oYo6^7nP)5eMjE3 zl068p|Kv9sc68LRL-!Ck&vsfVa2_p_F+Eg z&Y42HlJpVZdFLt`kN0c^2pqnSDt{{B6-*sfCP@%65;IE3W?#wpd`z)~KCr92TabmVlrg^3em#VPHHVVcpTBBE)#^to`A2V;o#m#$W+Pm}cTu*iI1*=slkh+#=`tDBMUJvnmH4RU+~D zyo~#i-H!5mw=UG8{SKvHHdt1z`2}X~2UErRyo^kS8J9cv=wHG84a13sgZ`{RxXk%g zhUJBp5OnQ86SN}Jw>8Bk92}JdQ7ViXz=>`S;eD{uLKQ+WTOL{T=oBtFZN*FHpsF1T$SoU1Z zhw!)pAV7Q>%fafPS_Z!{eALW<1VN)9Z0akONXFMh3bN4r_vO5?*1H7km=4GIB3H)v zz46@%n?(>yvpnkqhaohTS0JTiuVMtDxkS3Rui=YqD`q*IX1ESzWoX7`V8tvtWR1vQ zc9=2WQ{Lq1h54IEO2if+l#va%ro;v5@SZNjfESylhPEJ%m2fKfqzM>Lu>`g1=~G`k z+sU12T?Otiq@dx@^8>(-9SJ0GVogvF(Fe~B&~c5R_yoy0rqJP=?V~{pErI~pNK@R^ zxGp|Fq3$O;?K`=jwtTEr((wccCg~hbjt(!$QndrEyH}p)T1F=j=QK_Y} zN_z60O>5sTQ`-Sz41g1t!_A4}Sl7E*8l;CkJ{r?+Jy)8Eh7pV2Q~o-uqPxU$A2*^z zEJfYB6)NYPo&}-eDPSIpg*8wv8|8|;n^BJXmf;hVFclta(K^4sm2qpIP5F6`*p?P5 z*;L9TMdCObYmAk$quml*0vA_9U^i>`!xRh7T&9*~(Zbot&W%35-^!z+MS%f@$%JX) zAl?f$AFgd8mNYeSY^LL{$YWFY?aNG^!VGffX)#RJ5WM%j1lCw0@2E>|tx4>@4E@|S zS2GsZaCG)0nX2<@`?AEQAvk@;*$Glwy~m~;mz7H=Q7ZM_kfnG>fI0s z@+U!#CR})Pk~bZu%Wih(_e4ZRPhE8-1^a$onTlsXQEZ~;!CR*}w&L$ZcKVtG>c7XQ zT)ORH$5i8PLM}w&3qV!r+>t#(Bq4+%DOmSwX36b zhgMBH?q@PHl_j%vscvf~^&b{%psv!i@Y?&2-cFmZrS1}W$Mml#E;9ofD+>!XlfS`d z)4o-|J=lqs4GT17sNtSwXklc&onJA7<&Dey}X z@-(wXigk`4NlfW1pS7@4&UG}AWqX=%_xa7dkC5JSC;d$#>u`D9e{Y^$%4TKYBZLqL zHQ^BMJIqHJAfUuaklyF5E|I%#=EM!)fo!{yhG!Sl2!gX_GCKnn@Bsa|sJLV34_)}- z8mo7kdEhZcX9_3ZSmvYwJ3CQec(nkzvk>=j)X4Pn~S)`ot1GeD&te*VO+ujqv3v|h*7$;w80HuzaLSW}y> z4_y@U#?sYdMrvv27S&6?oAxr}a*yxY5s1TSIc{W!>*gXy6*99CanE1=lsOV7gz4%w zC1<2#%+B5b*r}s65Kp{3^d%j$`Q+{z<}41f<6jr*q|S#QC-BtLjL_@pUYy(on-d{6g>v-wsM#Stn?xOA7tKyU7)SUbW?;dOw?O zB4Q15o!L69kkl#|ffBqbJcGHGmCJ~+Z|YYwV73O6ExIOJl}hZZh%-qV4#@R4d1)rI zN4`=v7X*tF;qa3*)YF)JH?oG65$x-}3Fa;UVG#1XolmWy4@nX-%UuZN=5Zuf_w)VP z`HuFfh&+aVm#QL@s3Wo?)X*F|Kwh|^TF!aw>3WV8cy3a-@TRnZsp*&6vufSC0Gzrs zitD3s39dx=x?`;GdSWjhBad(0z*So7tLP@m6Od#I3iR%A(aV&XIGZVfpm_IJjR<99 zt#xq*b#leYzm|VS$P=*7V~`iFU2C<{PC??=gc+oCbsA425b;)H+M@s_SgPWzra@5z zZJKkjFL6jx2}Q*MHwb*C+*QIS#CbjsSVGMs3p+PBv?)FtgdR_keE$H%-7+!qLVgTz zi>myj^LIX-o)86M%o*gYy=pNus6$-DDDhDwt~YWq3a+fwl@R?RZNP{T8%tQSl#*jt z7}p{c(*FPxjc&!&NjYgIr(KlFA5zy2n`UF`+x1ZAlyxl8OuECCEkt$5dfd6OV{fQf zOXrk%tBi3Qi!SQO1{k2;Mp7i}ux^e)NBJ(C^4}SLJxCjP=Yt=KUbx4Ho;%UR6$nm4gG#Di<@)X z9eV1?O4kuXqmf1-R7W?gvctrjf07goj6otfo5jm2)(6RssYt^dkxeD5%+HrzysV+P zDd(dIU64ee`r3S}iL7q@UeAt1pz>~vy-jy6tPgL0T^X3Gay)DxN_7D_onkgVb>)&s zttdJ=96ZIUg*PZD;F9NzN&70rD+!{AqvvV3Tkp$s12AgZQwad__xd+IOE1(i1!>noC z!(Hwqs0lUF39lfR*gy#L3Bn>!ni_tad)vxTaIXj|Nk{W@}RO#U4g3aO82U z{vlPxCZ-8Bfu|_n0i2tcl13!^T4$+&qb!7@8T#geXL(kTauDfQw{E(eAam)+1tqIb zfj?`=9b7^|Awn?Fv5y^V=_K;8uB#p@1w{Ka07=|4)$*}}mtLJ7i*K~A4Rbw6vb_Fm zprQml##K!@+OUv%{DZ*2W+#<#mzcuX>)iB# z?Fr}yyqBs{rZhWMQ{L$~MPfG3uhirXOq{y&JTgroF{zgune8>J3ej1n2@O(@qBw4iJD1) zw9j#J3sOHYQQx5MU>W9}00WcOKzR{a2X@}qiB2$15Kh2Aj)3BY6qyMek{ATc#Du`z z@uzcM5;P;XZR-AZJPK=c4!EaAvg zbzFe8+#^z{J6|&4@r=ONzqB^pRD)Xe_%SUNV$ziC(eD_W?ZVh#)ZsBEb;t@wd*~Ag zm`UF??|g}qG3=xbbLuCt=UJq0iZ!IdfS=3@DMJzhGl5;&&bsB$ z)GZ(lCcAsv$Yv!&f93BL=7#jI8iG?&U;qS@x;2v5<%F5}^7`5Q0JM%G&Lk|W&bNf7 z;!*^tYLjh5!Pmqn+)|U*yfqMMI}CR9=W7z7Q=CEicDAp|Tnk}f3_jA*qIPJ6>CDrv zQCAiJ03szGC>i%|dk15Xh$Ly&8j6k}e=R9m>d+e>HoQaX18qiacQ}NhcyKCCz%wzo zC&S{6vb;emLa8%Jn$PL0xT`5EIvht7ph7`G1XScZ=WF9))`MJ!V{N-DxIqRVX4eiW zFMOhntwTef!lbQd!?Lj-nILb?^zCq{LP$uO>z`j+B?$@rTNUEIrZR%5>CAS$@hUno#j0WyQ&6!R+Erybq_VFpU?|)YVE8Uv{B| zX+GH3K5<@`17!;|EKFfhItiH_dgj`aoI)wn)6dGv4-cZ~s^QiJ46!b7_mos{{2A%G zq#Y!5M_yQRUltLnYq7hLFT`4-iRE2Mk~y2wxHG3+jGx9SWDE$7yYplNYT<$_*K2s6 zAjc)ZN>n5W)()e$mur@)BzO;6wjLNqgvsnRH82;KH#^~5DW@jM&oC zT$61x`MP~f09N#30<`PM*+iKsAZ_YCZb}&RC6)w0wnnO*hxP3bJR3rL`t!KLg$X2V zpVzVCibZ<-{!9X3gyt2qA;DU0{}iLJ9DLxl{5kSU@tQz~859Q6MUSM&L(#kTlFuKyFAN zbGe}*U-s#pe@3!F(w#0quPH8omCI-YSl!kf5~RV@^T%sv_-05+CuyGMwVXzyuVVn6 zw1_5rSqz;}T79B9B)tOwoFonE^*M-9stMSCQsf8`PiF}I;a-U@2}xOiRCfYioon9W zf&HY9?XNqa7->SDy(N!vV?YdVKCe?^?kDny+0;H_uF#rjlg`M-%2|s@^9^P@3~BRu z5(<8`N!?fvP>oE%q3xfOg5%6k2SMH@J0D5tMN%{nMiZ>loHr#S45$)vtvh~2gaoN3 zyLwqOPn8_A9ob};c<>OcgMWo7h>mSJLlGuvPVZB2)B_zbF6AV}UtdUivkmM)+N{{S#Y(s}+3g8u-R6A2WR z@1Y}O^SSBB`jBRzH#@qDZH z0@O-?J}42ILQ@|x-1>No45d-ss~#k#YfoDhmaPi4XdqMOds$$+07I0PH)M$^Q74*p zq!X8294dgWKtJj0E##W@?Q)Q2MyKWLV2lbh*L#$R>N*`DWO-a%fnLFN#!tn7N)KJn zI`z@+Dtw+uIj{=#%NQndspaQ!P+%M3jP2MQ1Mnk@g9aE_`V%b06gCk`FCy#?xP;s zhAy9*ei-~E%KNr;VcGcpqUWQ$9kB7ck~yVHBZrMd5^yck-9H#PI1@Fj`}9 zGwhtYhSO^0Ji5(9Zkg8(j(g0U9CK0!Uk0TJ3K7V~R{*h}W9laQf@Gmo)axBP8IFcM~9yf%_@Zy5YmdG%euk)*)}x$P_xx|+y- zm0T(faHn}|{3WxX?rm-&{BEH4p|2GlVlSNK99(1#Pcdnd@zxL=ZUaII=3JD?w8+#QHv9t@AM+O{UB5RVOvH`HW50r*pF!Nfk*t@!B=m~+ zONN!!cb>;F0-%W$-b;cAT=r>|dK_)2#&zqlyniqeH6o(3-|J$gl0A8D1%o*k_+l>_ z_*LRfBVSRM*x-{1Azmd;Iph(aFEiW81OudJ)VCnvR&l*U9JhBEX9Xl6U!Xp-G4NFg;cgq}b|_da^3@mO(CtaB4SpwB@H zT#-_wp8EGR-?b_iGX}95Jz~+MTH+d&RUqdFmUr%}c@u&J*1hG76X2<>F!FUh&FM2g zyiasfmEy=Y5+`}4*Ifc)WD*Q)=h_{PwP1?!IbPFAa-3axH?Rox`8(X@US}SStOC7` zy+g;ndgQ<(8N7k&a``-T1TgY~=cf0s7zWG2=T{!NxdJfKt3g_cWI7wLIcI~nurZgx zW&`eNo7y|Z%EZq%<*L~l)2?n|T8$5PV13mgD#?hx7;Jh*cd4{@t56Yy0}>}#G_I5f zYXair`&pOJUkteLksI>nVJn1QYU|GQxSC^wA#Mt)dh@fb-dYT5JF?;h?9|$p%OjFp?+g&ih_6avPTkAMFm1zJ+LdDEcpRkNb=C|~adHYp#bE{udaVTK zCwHz~=q3!#Mbaa}o>wbL1}V_zbvS)hV1d<&;;1C=)0vu;5CT@N?~rOtfuuZ~;m8wJ zwJ7*hdT2R=gx#~D6Szs~Br-x*hH(UUo@ii7)T&7vO(&s?kQ#8Rs@I&d5Q82N2J-4G ztwKluZ8T~Dc;P(%0IPjKG>d>iGsrb~6tt!SBY8igc}jwPsVoHoPNkqU70ySJt-w}r zOf+H^x9%FZD(SicQk*=Mp5WLNt!w8aCuBe0( zI@P-3xUj2XDb~L~li2i< zq#}GNRLqE+;l+I^!B_ec}2N6IS zlhFmd3Sy4>yAYCOo``iZ$O$tn99Wy?sG%5zQZ~zy)H;mND z*P+iDh@hPIHC`tfI?T8Q6VbB0q3T?esC06ThFDOGgS zpgIEyAfY;Vjd@|nVFuVgNsdr`ZbczPZm#{>hX@HN-Pdb?9s+R+W;VQCu>=jI_xhU> zF&b{`Pt~w0MFd8pvB-gxF&kndH6BL=ECQ&kY0yW>xT;jKo)cY+5QQtgH@ZL9NAwRb$Wx_M`%?^yNvm)yO-@{+QMc6Utwa)PaE$@>YbrXb2JrCQ7>cS-&CMo9h;8U) zvEpKtVLmVclcazmYE|lDD%XTzSOn`P0*X$69eFgD(L#boCG>4MbTMIYZUB&Eg&(9V zWXESir6y>ndi#P4#CTK{>)-HhNw&PlC+OwmDOhC8sDUI@N3~5Fg{e`22}st0VuXUP z)!t8uSR(Y7t2I^m{kdbsUqy z0o1hq6+?Gj@4i)TDx`)w_?{dJl6eQL2Jya3g((Rk>U9l^7)-HLwzVYa62amD_PN_o zsSS57rXyA@O?LkPkRn2f5v%le)r9JBFtDNc}DErL{W80BOj98wFGa_-2Ztg=QJ z5*XqqD(z)>h!qAmLxU1T3_qXb^&GJ8n4<}S4r&R;5}giiNgS#vASj-Z9F3S&Nfg@F z0&N*Jh}g=XGTjs;=%0^!fgDaIi;bt`8j?ahI`U-MenrE${F;oP5RH)c;At)$h%Vk4 z6^{1k0Y!?EaK$+@E#)wfKnJzM5a=XK9$$t|OEYf|PH@+VVS+qaZslbq9uQA9*O@0V zJ4Axql4~s*EP!+^*p9y+lsf>V#Kwx#zXzI6QP-u* zlA7kgGMRPtJJ+S(k1MgT&t6k5K-zBW%JZ)$bI$XhCg(^ecV1u7fKeHI``k!Q=ded- zU5>%!({mHc1B%lTyy;_MW zObIDhgo!{Ctii7An*sn(%5~88RpL?&$*2J9Ss-%lVD64(#Uv!C6(0&gDkfxPHy!%4 zsD;5X3K>*%?0tKd#rtIu;!?HxKsmG{gGshhtz?7HDHAYGrRy2d)g9G_js?OAAW0KQ zlaojp#WJ4P=s12DB^cKvq!i&vgsWgxE6XtEt`#>?Z3sXFe~EAj$WT^t?seiVE4)u} za_>1MYEq>}T;;7PDlL~S!)CJEQmP^58KZ%%UZfQm4VNt{&=Z?w#yC;7SNV46%o@IE znGpRS3uG-K;-iz->T@n8t2IX3ia@MOmB~#>Jb{^az%c;8D^jwD&hK`4zCKV^vUR>SyzQ1S5ET4Br0nL znzsl7vm4qX3nloT#i5*?jXEC0oQxPsKpOnR%a4`t8=^%GcTT+4EKCl5DC#! z9GjU1c+znU>6l}sK_pC9XIeKT#CSv#&GK@<9Bkj$HuREGL~@Gg^7=3+MsQ9+;80Z8 zW}1GEfNBykVLE1hT9(uy3GruH-YEstnG_Kl`u04j*RJ;h9HKE2n(}YtYBD2fc_0H* z(Bwq{Cb?_+IRaqeAyA{9d2_f5B~nFWbn*uR1RCet*ydt}bnRdh#S*OM4KxI5@TQUk z?Iw*0RTYX(P}z*o4L)oGbQRUmcZZbAJ-)iSC_44#U=rw$jJp|W%gW=Z?%7w9(E~OC zE#&LH?z>$Y?Hb7MtGG&|G0WLnw?JLA>)Tg^$jP$O7jN~?Q!42{8tPy@ap_Z zGXO=w$eATuu#$H_r=bY6l%WX;N<;*J00UqHCP>ls_zRBUb@c5C)Yd zw0~^(hq-t!d3dfP%ej@*cvb%DLWeDO7xRyK`-hD8aKsQV+62wSdnp@MHBT%Nz4K3I zQ0qjVUp;^Xw^xRuGV8hLvPAx!8bial z{Eu@0!2w3S?Vpf=UFUaRm!I1`{{RM@2ThHDMlgU9O!u1dIUlIT_9Q`*t6^0l6`yWx z0Q@ou6{*qD5xnilG-Ej!8#5m~&FeN!v8tzu@$& z2n6C$iuDWuR?Ke9?Jok4i9)7i4FlPeIv(#+!&;ubd7A<%B~nSkk&vUBRe)re%jzUN z!Vkh66Q6c{ceV#P7-El2L^N5LG%y?AN$U@>We5@~cI@2UaREpmomy?L4GW6aPa~3n zE!x4ZudBjd!b(-4)0Keq7%XBs_bwGM)JtkifLiWXN1Yh&#}d9m!2s`aDr`WufhEjd zwlbFoo(qhP&iTh{%aXH+T>+~B*j<2XN|l>t6PaP3F(L&z?sYcSx|8#2y8xBL6b_`S z9IyeKlpLEGkvh1ND&;I&i8xgo7z;2(T#Q2}KL|p|)PTnpRH)=9;LqWbpCuttIEQwR zW4~@VQ5y+T{iE!qh)AKIzzXihUQ3eaOfwcyvkC?z;(&L30SGo1)J9F97?C`!VK~MX@)i?@ zLPvXnkV;84B)7t?nw@rJ^Z=AjaQ4|sgsQC)I3vse3c2D3!L#J{+R~i^iCIdCh?3w( z9{7ukr|16w=nVl?sE+1G6D3L`Qb)88W(ZHzpM!M2(x;08*$KAweQD z&B+50l*S`5%*a3{Cqi{%B&ei@)C2jb9&#i#LZu{#hr|;mznTDjQfW{I!oj+n7gb3S zGcCjr%swHY>2E@W&;22Wk``)VwKzuIa&3TJ*icZeL>3rO5;dnoAS<6eVWDAG$uu~+ z(ZT?vu9_UW8`1$f3Z1W>g&j6Bkq}5V>6Yk2Mj0kG8GKy?kPS>~Pw4QC6cRO)UguOr zX#-xlI07)DD@dH>#Q10;nbo(#qa?=aI2v4$MD^%&1R#JEBDFnr4y7h%KD(VstW6B= zp1T~hognDI7zETSuSjt-T{`r-Yo#M~I`lgtH;vw+luWOcN$4;@#zy$yS0ii++IbFOh>8k-36^cpK_wHzS z?WVgJTard24kE4A)r7utaritjwT_E;k0T*#2@O`4s5!YTB_jxiI@eeXTl`6qgmj&* zR==4M)494O=9=bgsQewmp;ll>NMbE`%nclN9&Y4tItWeFBxyU|dHV-@KusM36&aD3 z$;4Y^#VYVwfl}aAheiPc2r{EhjlO*#B(cG(ME7yB~AqpZKlS)N1cG}>Q-{wfkpd>MAYzWM`CKO=hwU6duqE4-;KwhRC z27$7%x)yvSfMalQO7VzQIvkT(r-zZ|0uXSKna`}U5TZ1V zWRB*QBqviqWHw?>E{Zn5B|seX%4T|XdPOwv4mkG>Dk80*fRjN@GmcjR5_~`mGryB~ zb0RQ^S=>z>CIW{1wB_9BP}MV~b*}aRwNOr*9%9;dbGBhoO=NujElNO5WD?zhsF{h5 zT6XB!tpzu2=5Ylp6d@-Lb#V&tz~q>(V_+}|ff||t*XHsQGee=%d`7_cT9d5x+~%A* z&;mB-Y0Sl_AA8}*{5`MzG=IX@q2h7j%-FU60H%-lT~PqPSCOq1tydC_Sj)Ojlw(Y` zT8LWsg{VNFXcH}CmGHR#0Df>h1f@&!E`61{tNqz0rn>_R(Ds2!NswPc&k&PEcz5J| zT`$4=ulSRK_l^`3_|Fx6D6P-b76XiAy3az!D>l|fTE1Po>$kjYJ!sAI zk#qy0*uxhIAmMQ<8}>eb6T?{I&~4wgn~B(=q1q0h?*?3360$X`8YdSs&E3CqbnTA%1c(5-ZbRS3~;`@h=KEo&eng#bI7%{3eR|DE{mD2{V{6 zNSEgBM+}H!;BRsYjZbehIFL!%(H+i7MCd23b)&=#3M6ircNIXVtGVs5CNnkU?s^d< zfvDAhM^brS6!$&Wl%U-6rf0vu)W9?vl4GYX?&l2h=yK5IJ$c-tla;VBN5ex=m#yhC zbdFkjp2CeMr07S@&@jl<cPYuGaiB#x`s=h;MXWQXC_)_SR-utOa zP>_^}13>y5k6}dO@iu&;N~bN%K>M6d6(`$cDk}xUF&p zH#`tF&Q1X5sFalrnK_I~33I*_AzO8Ek%T69y#O=CZ6YAcnhLZzvAjMKpZ7K)K&*vK zMGHV>=`p*Xdr(qjK%*!nM1ic*nK4$J3nMxPmmmnnRiM?BnFGb4)_VaDo+Jvg>YxQh zAg5H2er{MAO{6GIz(%G_9m1_E<1B#zWbOmzU!A&@+ED~(XP0F)V|b@oGl~E@y_VW} znkZT66~`upC^}|)*q@DB)j~DdCa;zPjZU1*-Wem!20I-%M7Os{?T=&{%)JyYDfY%1 z9gX(W5)?TL=P)9YW+2RL-@979f~riE>I_D^o+v31kU5GN%SOZu!lkw10Vi(dfc9hvuAJmKj?l{fE0hia6c_4ZFwj zHgtuP<|J%N&dh?OY=`5)ogTJX_x}KQ``d%}nkLP_Rh_j`;(1dhc~c{M>%Yy2TJ0?> zwxuCLprpYEaT-5=?$6@uw&LF$pEWvpsUOTAyqy)wAo8_Fsz=sSbd6a_Awomq@cdPt zD%iPZ;%*@3lRW!54OvqS6D_qZG7nM`v?(eQl>(Ivh$0B)e-Gk#bHU%eV4Ni+q!J^W zI3lm@2);(O8(+@<0E&22CyKMh3a08x2pVHfgwr&i`d=^XM$(alp@&n*#F`oGa2E(s ztZ#Dj4H5@k^l>F8O1=^boG9@X*DU+qrsR?UBWXSDL?nPtlh-$(5F$cmb=kIz0IFM1 zQ6NwWI!lBCNd)UBV@<+?0xPk+Ur zC=;Y_a`6&C@M)zK zb%&&KHrf7Npo=&Y16R>SpD%_Wj}q(Bbmn2*Nf&|C3b)?BiBL3%ele^pd22G6lscT_v*($B@(_ zRV0Q@$(mN0u{5J3D8vURF^QvlL8i22Ab^(wG$~p|IU9w%T0>D=Vll>H$cU2?1D&gL zG9D#i3X&zr@da7{9Dw`^nivh2C@AyAf_FFt`E^xV5zW7;+6e4uNhX_U$l?i7mUPjE z0+K~Dd+;}wk?bZyl50lvl>s13HK^iHCA2MYQjtSK%NWbS9$DM&J5#(*P<5pFhB^FSpu53zX)DU+cZ@`uXB2?H@m zbr2okKp0YIGT$l+2_gxoV=5&u4)#J!C}N??g|HcjR74%STGo@6x#s7LW5p5x(OI{G zZzsdyCXnYr9Q3MlsM1*oS;bXSGRdGEDpj&PLp|-EmN4OC7``&jJ4WClZB{c-&z3cQ za4A_(8dUAg`sDb+MjsFmPDS~yD~DCP!~jA~Qyt@!J+|dvYc|`L1|nflbbTHTBN~ZJ z4vw(FPY$^g;qUC1*Rg3pS zD(KDg9HAD7A$XE!o?Z)S7L;lZLq`*JDM%+>p}>-$MiU!c@$>**H{YCfWc|agWPtS)gBg1~>X^CB}7{m3}h}e6~L5+;^I@qXbTZ@2mV8$TTlEp@> zmc&#yWc!-yH)&^l8KdC zT$^X2d?$E=wf#rkpTt}rykQGJhq_1jYk-v>>|FumFn2tq8;us~Ncpe_If1hyzL4CI zN{DJWG(ECU9Q&s^h|prF)KPI^B(44KNJ+1Mu*r+bNA zyg≻-pTTugu)yE-%_9H_vrkks~rxMC3;%PW&@Z37(|#4p)IGR;D`gzFn*5MSCgd zePB|dAO&dhri|KaQZqv{15G_|5w>B`QxHJh=4fEg!O12QP)P@uTxkBHoK4-M#TY28RWSxI$2(bb`aTZ3+QZGS$T_sDz1(x4w1j!o#=wf-PV7+dX}Ly^Mu6FVP1+SlVLtt!a<$1Gups!6+h0ygW~T+OvwzN!4C zo8^H3?XKb%f$O-zo~PU>Qt~_W1S(^(AWI zO4K(8i$S=R9fduumjF!0rHYj+3ie2KLKr+cGf{{STSvVZ(8ZYzK~a&c^rb6PfUak6I@ySRXN ze>;aeUxY>L*i-!L)c~L7ulAm+4i$Npm7eeXv)+*(f;%b1@E`vGcEL8+X8P>$95wW< z2}+?U{73;)5_nU6BTW=ReqYSJubq4STc5Q3C-F*(J^6egMFR?Gl_@$JdnYC+!KbRO zTKk3*pTwf#lufQ5iz6uEc!=+`!}9p0Yr^q=`+NTYbouJJc7Mga{;K}~Y4`omwJkUC zElP|riAq#>F6oG!&qWwKzV7-oYb3ZYP&HcOGU2$t{k8u9yLA@$l>Y$u+*tnr{eC0v z-z^uf5GM8NfPcha?KvB_kNw-<>aX^i{{YYa-o#)#MHh%4^4wnk0Qp`c?wz)q$TZ>j zy8i(BxBls~-wRSMSqV~;BJZ-LCSiZtL*aS1e$w=R>M!=0C_*ltFJQuV5$7dj{vV1n zm*IGix^h>XXZ~M{2mb)^c#pbr_fpEm`BIhafr&~~crN`#@DY0U0sPz30RI4pui9gT zDL=bD!C*(DMcep5R}aOIGW;(Q_e@)iJ zh+UACDIklpg+)3q`$%&bZRXZ2wbPGq@F93Ff8%j~`)(udo}a`hj8-_RY5xE##C_A^ zo0q=Y^#1_rFZQ3H_fm?-5s6Dw1jZ#5M}qIvOL2zWnqPOYU@!}r@cdu?*NFS3+jx+p z{EjXU{{XARebe)4yLSp^?b#q1^B4O~$z9y;-Twelf3)STzx(?J1Fdxzv0_D@FMs^6 z5%aXLbN5|pU$UDFXgL*XLr z*dNWkFe~OPfAuwj2`PQq-!Fi`bOo#b0Qme~TJYROIjudmx~V=Slo%;eRFwlm1Sufs zLDBP7a_~Rm-Tweq=j}eK$NbM0p_dSU;6pIQrq$(I(3%{Hoo8`v!zd}7Fk0=VYu?)e zHHXf~GXlIaDYeZ?D5qT<)S-X?u5sAnyI8_Xs0i1irJS#rk0r%VoW52tWpRQyo1=Rf zDKFHI@m@-0M#d#cA;Fcaf{{8fK<_2Ug5X#( zZneD%+Q1!qh`D^T?>p1}h4NDN)lZH`u(OYCw$!))4@Bfhju1p^y{inScyV zEvI{yvK*kKdL-Nn0wE`v?jgUQPP+(vL@jp6F{eA|TfCKwS1ZV4puTqlmQZov5|E-a zFi$5lo`{Y6^!RLGFO0O1K3VgzSH~>-2_18tDZ8btGUP=wuB2c%K!{^pTxqN8@d+47 zet4Rf(J0i9QZC|AB*1+W!D6yO_KimE-nSkRAgrB_uJdk<5#FPBi(^tfE`qK#I)gZtj6QOUEK z5Y6_T6p&In=-`>TdPak>hiymskD$7J>A#Bq0RFkee(3FfjT~LMAMzgT{{Y!o{g>gr{qJkz-YU9naO+*vB}yx_ z6;P#$Nu=ks)FYE!MQYAM^*oQi?pY%Er{(_ug6s2N=U2b+_%HMat;rt#;;qgbTqh2qDJF>VKk^Hg99NK@*?|jTq!>-=sX zlK1}ra6T8ZDAd1x@iz!7v%;<>G1|ovWX2^JuE!7ayWjZy7y1L%?t6Ci1J!$@*}wkX zzeU{Nnna{0kUBm6#UgOp;ZwH_tTRF@g+ocBSNYxV{B8^V0qb)1?kQg|44Dx7p84@y zv-`Kd`;W|jonHR{$Kb!v9=9{^{wm~SwZg6@FunzX;*spy;Zr){wT5U#aHvkFRzqr}os5zU+Z=(gRf;|Lcz^iZ z5Bdr9y#D|@z5f7@!GEAVZhS7VJy*UX&a?ZszvB;pL}=2le{okM9j+C6)mE_@l25dw z6S>3u?)Uyb1^$5bx}M$3Q68t=L&3y#aKjg#v_!Nz$)aGLT<{sfdzgvYusNpUkdX&B$kwJ|fQzFE%XeK~ zsTf4bP@v>_X+&d+QXH*SX$LuIN0s6bJnMw*p60O2<0-`5s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuW*wM_Soe{#UbG`8V-cR0b3Y>#74yl zhaI-uhb`Pd>PE#!!xK5ZjHU-#riUrO8nlNsH@~zv6>Bxj1gwUoI-N3@kfUUjJsJYE N3jt7)EKLLwdH~ZE$zK2f literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000003.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000003.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a38873c2d52eeccab845e9a8e919641827569640 GIT binary patch literal 32308 zcmc$FbyOSe)^Bhr8njrEV8Nj{6e$`!KyWBh9Ev*>cMVY7-QBHt(c%sT3KS@=g#x|2 z?|aVq&iC(Kcdh$mjbvu;-?QzvXU$~tXX(!-01qM~F9Se81OO18KY%|^09-|5H&Z@N zb{=*vPENp|Rlr*SIws}|%oo^y4?JAF{~16akcf!rd5wn$1QHSw5s{FPJRf-8ynOkR zii(1Qjt<1a!t-YU&|CpA`TW62kMB=Y2829h|KQjnFBa zNSM}TtGY7w;zwnT<%8j?nff+89UFN`;A_%kM9t1lS65iNj#2OGeNO2gfISof0W<4W zcpfKL%T5Z9Ww&?bpL#nlYBlPg8oaMsZV$;tz75cZGtqMyrd>FF45^3l=tM2D7jm>h z9Ev2>!RfsVoOhkqyjvx?*SaGt)xwK&qh7$!j%hnxSKmcH*|<}CufvG*-1LNq^r+t~9+&ZwkPugW=$^Zv)05tB$|`Y9_R2VA z|1W*j(OuhJ+ah$iDH9D%*$6+en)>F-jpjx`_8ALM22Phe`$p;DX>=|dg}9}}BBkWF z0@Q<0cOxtNV`tmXiWT8iyQcdY-$wQ=?5`CkGHfXRy=)istK8~WGt$HXn# zNdF3r=U<`GU4QrgRNb$m^mwTwOk zT+gqAzr8&{yIwtQQ1S5>%dTsx7z42vH-!HA%p;0JH&M&1*{p*PqcxRJEv+bEIq*Qd z*@b%+jlajqe|bZ4q(0XsEOTG8E}))zqQ-Bx5)jOS&w^JZkuRml`3ErOBxnZS-QDO; z-ouWP=~XEph?16nQvEEl__y;0{{-%epIe7T@b$XZA&JxO*g_RT2rndD9;yMdO`5E% zT+}o>dY|IO0b16~bHwa;uKNYqA*0tT$ zfM&}eUqC(Su(70GavEjG*i@ittv9V@|0>hQglps-7}HCLJ4pZzpn?pfXxaarL%+nc z`=iXM`pT}3NJCm!UKFMzMqj)qt#V0F;3!AOxUNn;M_-w35T-B+LV9mZB?9H&CF%Zm z-Z|!7V}8P9G|X9-y3!<*D#|1!p-Bl}n(Gs^IpsF#Y-{h&>8@dss!1keD{2t+5IVvM zBtXFbo?fVfzrA>H%#Ce{@*a*TRe=fhrKf_%0k_$9Qk>Z<9&Ze2Xk-zqgVzN-X!(w| zKO_F5e!_b{R^0U3`BBAbu+bd*OJhDB+a8hGMvLWkDV?~cIEGV7r9Px#lwpO-EYX3Y zlVN>RrBD)XVv@%-`B}v1zjJcETwVU9;M%Q&dMNs|NRf?{{}K@?@Fjk@7mbVpq*4?mj$6>@w>NAuS#DhPJ)cm(0H?oWtET*Ds-3x5{KL}Fh8a|6fTz#jE*Dt|CQ|jPg)Jj3LC%RyD5`7(#345tan8B6fFh@ z4Yo9vBo5(_I%Cn-+z&M#tx0*{J)N98a#eeMYCRwIZ@n|D z*18~YiD}FcZGW+Ik8u-sBAUF0=uooy&vBnHq3Ou2X^fk%Cqflb5`n>rUcWI4<3Hpl0`0v(fT&|efRi2&K&2u zNd%{Exx*GcEDTrt19;W*f@?U9hNB7$mXz!~Yo)=Q(HzUCpU;p?GzlMgyyiQa<13llRbc>%N>p_yNMiFX zI(R!cxR^nd<1vm#3+n>NnR*rb?TA0dnevU)=>1oLCVgo2NNa_)5sgTJ&D29DX2S`J`L^*p*yZXb=mceQZ^P%Zzs)`@Y(;z$sQ zZThNbFc`H~5s9f$yaLxiMlojY!#P8m$Zq_qJ*4LIG_5@!!K_iSunX$lc$IT{LV!4< z@*K*p+Z4ZOZ`s?I5zd%C5qfEzg9G$0S~ZfGwFsz9UZ}w<)j)~5pOj9TCim>h=U12D zodTiASlZTDMKtP4<{kQAO-DK1+JeNe#ORSks^Xg_{q{h~Dg@*>#Y?Z~fwu5;O4H0o z*+cKyjRX{JH8uSGt$zGA|0428XheC0+Om49+Pz}W*3pK^L`Sb1Fe6;@Gg#k07kBzG z=G*McZ$)Fjwcb=+)hy1#l_X{-Y|XD<&pydry7BNkwMegmKNkif=7FOnu?f+qyBD?1 zE2C&gsCA$*)evQ1kHU<6+BdGc$v*&vAATtu$NKjzAB!$~6jE-gef2)t)I9N@A98rB zG=}Rh>X=_fVDvb!(P#(jH7D|t6;U7ih?-`Ap)niK%3DkUbv?tHf*;kl{Er-2_xe3e z&WGzyDU**3Mk{UCw!aNNp7+G5-ZuW0yR_i(gb}2GAK@CXq&~}nW)q*Sy!&ql(Wf)J zj}E zJ7@r{0ooUI=ofLS9R`E~#X>25{Dr~&H1kyTH}3n#qDQ?qt46gCi>2NxzD0|zfkxVU z)!>VHH%(94^$my&C>)^nde)4D*S!YuqM|;@FnQ%9++e1|3LlyfhR=AHB>+QzuF)cQwMW*!RnZd5RRg*?xVXh@1Je6x8!hHE>%y7&xhAppZ|^?PGaJ$K%1z;@ zFo=C3XTNUtdTi3|@CFhCi8)tMv=w7j6(@Mpw)e#T$z95|{r`af$H*`%*AqIPb@R%v z2h(QkGu$JHYZbtA^j#pJG=PHa#I@URd#cE&s>xK5VMRtv+doz9M+(P3zBcw$xZfKI zU$uR-`Uv~lym%zWdq{O9tWe{)v(Ut54UI07Dw7G6=bb)(PNdIYiF^)t&wpw&$F63{ z&t^{a_|3Cdo{UCSPpSVMOU1v6RkUvwjcT7nWl!J!RWtI7KmVtkduHzb#g!bJnx&6adH**uDc2wVg`;ra z?AzV*>nY&W>ZZFS!S(+1Kc1id$9Rj?pML;x`&2hMHiw>QM%7QB{^K~?Uo3eJ?EexW z`Ag)h@4xEoZ;^k+Ca&b~(!KDwHLCj;O55LQw)ls)&A!b&rn}FTQ?g>j82^_ww*Mhi z^S=qT^7C&(xvMnB(f+*tHnV(MH1=!lFU~joi}TOntGl{>)q!KPbHUzY@W}EXE+_wO z`CnZ2weSCRjq65Pbm8f(<7_*%jb`41>uueqrl!31(rpAIT245hx%iy(`=yq5w<;l; z?Zc}WyDV!Pk?`-15ViD@^{GqBl}$@oyXf;&$niXFCVCFnsd0X}7)RTFr@T_v;^KX# zKvf!Qn5S);1Q>M6d)HKxRBp8zRUs0OCigqzmEo(Y(H9G}ZoH8Q72i}op!VHg$FI6$ zH_#c}`|Mvdedj@Re?y1)DJc;_hEe;%V*Lwv1rivTUl&?!$C$l6B1Ux~)bKfuA6nJp z4KffHSH=*rs#%%yFCqw`f9r6IIOXHIPo@45ou#x=mXc)^2K06d{qA%m89~eUb8mxt zQF0qoi3@x5;NpY5q;fnTEg>z+D-4z{20t8m9SuDVYSRFEM)gh#u5-uRf=&VT5OtxG zI~T6k4`Gp$Q9+$tYNg)vXe|?Y7y~ASmkOJb&4uAbQnJaUhU{^1s{YW|AeWq|9pFa` zhUB?3kCqResz3tBQAtqw;CmS&v`G1Vi@A(5t;NrMVW3`$!MQxhfRy-RbC2^Dx4EBU zlmSz$fjbb6gkkAKNKoXE0NgToCy`X@xQiRK zHOHbuBjYi@=J5qBnS|Q2y^}iPTJ~XZTA}rbR#8Lgm7Jnf{IY!+S}4s$ocBD&^Cb#E z9j&S)^OdYyp7~hj8k9I~@Gzk!NfnVMu;csUY*D{3PmpDg5QA7zP5v4q2V%~Q_vHV? zIL|b^pz}IV_E=T9sn5|nzvlQ}tniUd$J)xzXcaCJw-owvci1`pNL1+eq!;J?)lWcw zjeRYS(EUF}Mv&Q{i?xgIQNzx?;gf2>Bi<=y+m5?v(rt~a$^FLq8yCN>Rp?+yz_-+Gf zuP<)9y==a%IwuHfE9$8l`SWZ~n~&}U@@YY@8f|m*2T>T_4K!cqaj$>Topxb#RK4wI zoeX6gORISt961t4@?%UfR zT`NmiUkjAUobF6vPLKLTjG`!4g4y~LJZ!%rjTynrB`%HA|B6rgiRClFgVyVx0Sp~X zd~@%D!c>*$9JzKLhJAyXGFcd8(qm~~RVa-J$jyYh^>F7bS->N5)s#HI0qGytJyA%%*;Z9jQ8B(^B zPgDdh>0c=4t3;%JXd4xmA;cp5)?+$Yw;sdZA&`JZ(^zUmWo$Dl2N)Ij^ksHS-(Ww zQvKXOJYYpQ>(XS!=$CnlNtTs#*NCK%U|TZHotuaD-lHFSX9&JvEetk5++bHN%@U=e zD;bO}+)8-(sn9qi$QWP6R9AiGhf7Vgknl!ivLw-dGj*4?t8G0PyZr+j)=t{q`J%Ht zMd`AP%(sMF;^YZtu5*ex6N0xFZBc1@&8M_n)bOwOwV}fy??12dEV0d$Pi*1b9<`8tW0zl>ofWpd zRvFGI8D}@QDm!g`-G#~|;zxCRfdh#%C3L=G<~P;Eo}=4sJ$3=_&m5`K>? zMsy`h+B0E9B%mgH3>8Fjq$OGD`ua&;+{UsB?0rmE5>gnlUr8$5K!uj0VLsc^vjar7J=pOs#;Y(XgOtk(fkR z={6Up3n)Gii|iHkeDwM8_6}P@;O?}YfEGpC=EsmK*=u@q2%?Gq1Pb32)u-9BJ1UNt z`zZ#2g|4=Z8Mg9FEBX0dx4EF;hY6omJbY+lX=))Q|GjSWX!p}*)QBNyYp9gKF#V2_ zw5(6Lj-ivcY~Q{6`Dt^Dgw5xI{g4kO%Wat$uz01pcge9(H`2TV_fE=XQuWh2AF__$ zSsO_E&bUz&=ZULDNy0y_(Mj;&Dg_d11e=EQA!?x-8f8I%9ED^up=j_}CD6Ys@qn_p zbAmV}z}U>B@Adg@ZN{1j<|ht2H4Fx~OsfhD4|H3|aTNr{ek`SB;`(GT$0kPygaaZt zEk2^@-I7UX(OKG1-z;q;yq&oVf^U#f7Gp}JYqbc&52pySM%gr@7Utf;qR14Grg^^R zh4^GowbIp2&ly@q*nXW(HR7ymufO2&BqvD_(6nX9uaA{qaSPmW=qXWe^q8Z2(W*mg z(6_;5jQZRJl+he`@XmtX)_(bXf?5|D{I+)FdB>Cr&%^Ddz$*tHQe%`bOh=^vrG5TyRU)l4zb!p2!MaW>64bN^Q3YDBhpQijSr(_at; z$=J8=OW2ZgIMb`={_x8;)Pxdg%Qtl2*9K2Dnxj=O>jR!@CHt10YXyRbLEq9(R;;&2 z0_?RvpQ;aM0=!OWJ5|{R`$r;DUoRwf1;8J)<^@;Fo7ur%&<8h+PLB07rGXl4g9dcQ z9>1rQ>uAp!7W!+qH5Wx;0gR$a`(1Xjy!U$6t*+_VV9gASg^4f(r>*@JtJ$K<9+oj z5N=`VsXVI~vCBeJkK&e8SgN}Cp~u!IbGX>j^qUr5tyB@q6}#0+YmByWn9~>6aw59bC>4k^pzv8& zO8SI12xHnw_>71J1x0I%OhNU=Tbz||o!^CEfgQjk_A{9C)?3Sx=)k^-b=vmo*{TLlUy&E^8TKb#tQQB0qdCr)i6xnG;Ghc@=~?%=zxp^4wutV={N2 zT1ij5%Jr#)SP+uCbRU?h$;t1An;L8-O1?5zW!1zM4VUf>kp|g}|B{cU%Nhn9eH+ZN zB#JmLC_YwWRuvtv6GkUI@K#~68ZH=Y-o~;Xl<#l7nW={9#pJj@uuB};)}}G@L*1#a z&2fHFUy>Xe4T#~oe!GJiTeWlPbV3Qm%Q0QF$)y!zJS?A9l2HJ;63B??E4(PbSJ+A~ zF%e9TsSKdC|E%`^UX4^Obn;c5VY)p;pC1wkg(Cz9fW8X(7=4-lRZ(pZ-g<-~104eF zzk(bF*F)sUjTlOf`V|;6&i(+r2@`ID7S?qWgMZY&3qtnGpX?ao2Vwb3ORP5J_KaF{ zHziiyaGd-B1UtX%+>kkbnBQFocZHB{M?j=y1sTnUg}O7>*ZB#dz(9nZ2+&td+m$EjtOw17+fyT@&SCfW=6mG6>u^aTPIx70AwwejwU`N+ zr|^f*+A5{4J=^*@n(n5506?wqq!nEZDcPbv^TY~*ijO2_&bJyNC0E8wgRzgRmh#** zbof9MB>}MzW?3wM0Uxle;N^_zZV^M;YY(Kraf^ zx96V}1buWgE1J3R@bZN}HzgwnOV<`KxX#3F?O;KQ17CV#O^uk>-=4W=O$@B{QY#Y( z#h3nwwJ`GJr?1!dh{KUR#4o5B*L#qk&nbBRMah@1WBTzMn++taI2FQ$m#F9=sc_+W zI*f_QJhD4uY(}(^36@KxVTNijgm%2jWAkPD18_2T`C8Wgge69HJ>jY2_3>iB=Udg` z>9C+p>K}k^?Ea{Coh`HTTW zI~=`_LCDblmgYSI*_H-k0n#PV%u0Q8F&MmuhY6L6n8N|*`Mb&cIV(J7VKO^%*9UK_ zk&EN2B%8)g#DM}nU}^v~A-M?S{OHBYWL?oLT+raD_^eJuMSPy}QnF(m3>{FCKW`eM zM{2|q@wyfbN(sIufykut%yFDDGJ4I%s4}4j4d6sB#gXC1jtWG=Y1flXFpYU^jpqwB zzPJh4-HM#iyh=ejHyJsB8mR`spy^^B4owO|7yk~xjrYNjyq5H`ktLZiFVgt~C=>?) zyXT$Csp*KbPx)jBLE)09zkGvVeM#LzSxpIOY46})6AxjkfMe`AFKZ^#63NZDg$^*R z$kNk@^dEhI{DcT3WpF1|>K?7Mfg-Z%f5{M3Hr>MW;V2^CEDMl5Q(mUU49LxRksb(W z@RKyIK|$z5$7P=Joe*jjyW?njV|tEHSN`*~C`dCzWNIS}VMiwzg*h38nUGan5@I>| zg<69lSF$V1-VA8|i_HP>dBg#ji(r5YFjYq6pc55hQpy}B=+v6K7~G_<;zcb#6~5jx zvgY(&mc0Z2Ml~dw1sZ%$7qML1<<2+iDQ%^FB@vI!LYNZ-KRIQV*U(D}MDZ-Osxq!K zeVsQ7!M5g-$DbygUtn+6O);ztXiX+6^{=Es)I>D-ZHMC$x9x@bzR>z`twn)sQ|AENtH#uA33NXjjusVJ;c11;?Vw4bf*xR z85n~A2~8@)@;MGRv4|BmMSX7 zY*k;d4%t@jRpfbdQ|E;rQ~%~T_E-SUFLQWlZ?eT4JhxqtqOBW1Bew#^MVfpaAY!PQ zB2blS5;^eBEsK|T`zHx^(w7G7Ll;PRjA?I`Jf31N)sF*MV}QC+hUj4LT5ZUSs9h;l zq?(7};TJ_T!|L~dJZTnAVEI|r#dvM4AcI*1cKDdP7RFH7G-qI?e!4BS3g)LbETioZ zWqJ$*ZE+2&J}=5LSl=2`8B0>+CE55v0i|5&w-S4-3T6!EA$kZfQqp5ldGKnr6Bda{ zP%CZu8d)%@Dyc{u(%?ePS~!#8^1a~JKdz+!}W#5@y75ziP6x-A3$`&7p7Bn z`)vdS{Z!p{8NyhH)U`+6xWP8t(a)MqyW5QT$ly-#EgjC^1;2%$+l!EV!^{+14Ei`AEmWCw53NKmJ#280G$YV|%uRkaWO(Ak^!+ryJX5062%{Wdtf+KhQN zP9SbAl`f5Sc>S?OL`TzC;%(ZkUQXmit#AcP8v_nKJ^n;6yH95R{qKs9rZSR5M|pVx zKR`Oz)uzV2U+X~f!RobMVtLDPYft7 z3m#BOE)LdBljB-8kBaR3>RfF5MT*l+)%zM_>48e4Udd`GHPS($OszE*$tAXV1FxF{ zrTp=#%?X_oLI07DSE3iZ`TiFUT(<0iH|LSqQC>}3dI!9Y4a|o+Cc)8mBhmsb_ZnV> z_b4Vu*Jd;!m}5dsf)z*5j1|QI8RZSsjJhK#Oq8RG@~gN^YHC$)ZViu;ti;F}Dz76R zOcyQj?DKyBE10sQ5$Hx9VPChgV(nLyI&OMTI(G9tVV{-hks1=41;oBOR=IWTN5pP$ zXg|$;@91iIx1DzVYJi`+%f?uz!*kVISqVV^2rn}jBuzFcX64$Sc{M%_My`m`0tL#3 ze#v&58(EbQQs23|HY;H%w=E2gr`K>=5c*MB?C6h--pQi_fm?ba zJIXi$e=L0U)U?#8i^`XLtwe!sBA02xFg^rq(-O9_Bh)5nf5$Se-xs`1PeeVFO<%n| z8zD^wlp|9s@xgHaj5lpr9UY%r#m5CI-R|?ijiOyorst*RZGHEFuEPIQc&-(6UMPq5 z2JZ2YDz_3KLM2yX-5tQzw_++2id!ic*#fuAu#v3u)2HQ2zqB-QwP9AKQ&h!l;=O#u zUrD{X14b;t50K3AN1)3w;1?p}3nai)1SB&NCX7}mXZMztS)LN;MW_xq996MUR2%0N zlpA#N4P=5#2SK#fN|uUuVI8Rw5LhLXJbIs{2JRk6;zy;p4Q*(?#M8M*V==Y(l_rDI zYa&r{(xp+m@?{hWRLKwkmpWPMf?H@`68j!e=>d(DQ;drewX^?4-Uge%yg!BfXKGDQ zwwU0hP0b_1ej=kx6C(W1?j31@uJtzuCySr}EQ-R*Q1o`h&7eyD746akI9`kmh)SV=#+G*0emD-5LSDm$R#TLK z(&|LpKQRoK3xU}e-{g)rHi8}fxt ziF;&{k$p1VwLT|bRM9TKP-6AzyVUHk5EjD9_WPTQ3hi*@N%fI zc=MRWH6>J62drO8dkbgi7DC9 zu)(y_q9b~K+YY4ebi#@kfTph^x_iAJoIYH%YLp>2PGt4&%3+^8jjzT;erNNT94YU5MuT3Tbfx3DpY! z8n1p(ptFGa9`eBFU4lyl|KRmb4NqFe#88{ZGWY8W5DN0J%HRkK840uGyTU}Oq?l~% z1$<2hf#`PCSY62`3em^XDJQp>`gnAP+#|GMmA3NZ)qM>*jAT5VrUl-W;}CNm3`QUn zAu>jH!ot@JC3}nS;s9c71u3xAnusT`Lhr>BShSk)TflEh!+=gK7yH{GH4pJd31)+2VASVnX@p35G4~=01D>2_;xU2;vlJ()pXb8*gO7{5g}aB2LlO}9Y}SEI z!KniAS(%)tzdrSvZ>thHX()eHdaI25VQ*!!#eUQpn3Qc?g7K@F*(%W6`sXW_gKc6! zW>M*2P?8IZkqA#soWkP4G$LN6^QgdrkW6vr)F?}H`HY)Zzl~%AOkGmuOt;M=Ah*Go z!I++?KYSefM;Ky%cZCe0YO-W_FxlnNW$2tygVQF4IP8*9*|CIcWrvRJQzsqSc$wM( z@hQ#?Y(S_|PbZ%@9TcD&z6(+$qx7E~{&sN{f++bV9!yP@VeVzQ1A(%Hml)tu))nA; zRssgf#rPRI_)a%(Vk_IEuOg2i^=?UMN0H*2RVF!U3b(K682lv&!^`-t53rI94QA3H zk}yRpZnU$nZ7KK8hN`i`o=v_hUOhEoPY{*7KY+^UACbQv5N<^>{5gJq$$9(U{rA$r zSpGU;DbW&x$CqYzby=1*Jl>1nx|2q4BR!uk3@1Mpv9pXy>4&&~tZTEMAD58Q2#@9& zTCp&Y!y-u9q0$126Od{e3a1^`=iNDa*#59D>s@~DKy7xLRHyMmsg_GoFVR=9jbWBJ(vlSi8w!we(-|hSC5^o@A?I${5*?GEeK@biMLxx{j@sOy^4@vMw07dfT|F!Eb{}Sv za`_Bk#Z&g!yoRYkBWX$%FBL61;19rc%f&$2!AznVGZ%)lMNl)ceJD+%u_~97MKM3# zG|;!hrfn@h_0YY*)aaiIC7Lcgdbl`HKP4}-koxG5xW9&`6fE-YGkWFy8&{{&Mrc|u zEjqkl)$mi4vB9=R(j*F~_s6QlHbp8e%jfPwiRIZP!Kj-L-&$en zTuN3`eH~Ws@B=$nziT_YUC@vAI!-A{(C_$5Mi8dk*%61%9n>|!>=~GBM5e(H=-+YF#sF)J!}bT1}A6s``& zc4IXW-jNrUelL^Tq9hN|jvS3f0+8-ZCgagI$W_?oiG+_?YvmcZ1e|%bSL=`irc*Pk zrpyuI_EZKdu3m<~<>Xe?&r)(ct)AX+3`Bi4OX{M=*QR1u(jgxqldp~|U~~Y<-SLNi z#p*&arB7K9bbZfJy@EiEfSfd1iouz(if#}RQiwreEx2`f>rt;E0ZlSX zxBJDzelbUoFEq+Qs;Y!jVus`Qif>?5l0Hn+gPT{qo3lm}4Lxct0`-%ogz}wm$w>LmPhAI3wCGTW&9{j=2^GQ2@2tWs z^Gl2wb!^L_ujxTsoD3aajLjj|m0CNalD=dkk@mc|Hof(mnb|(5ti~E1f@W)wc+7C* z7z~5}lOJGE+Kiqv`TzppOF>IXB%dAAs{HA=kw;Q_=(|~O-}x8e0h6Q{4X!TeYWu(H zlR|cq*%PD~>$W(K<^`*BShJB(44F>@ZtTClu| zQIZG{!BW#2sY|XEpdw1+ePI3h7xoPrZl<#iFa*-l%&Np62iPnD~cX+$6Q=FdS#cL*g zsMOF+C-I{9jlaX@=Mh^pOTvp;_1;f`rI4iOHv@z@Gx*r`n8LZSG|MlxoNGh5iexo7 zIXYGH%kX}OV4jQ)6w3=2EH$OBu)=Z;GJqqAZzQ*JUO@Kb32~-6(xaOiU1lZaGt`Y8yDuJ zrsvWX|Mmk`xC}9ilxN_z;RYockkoAm5Af z-0(wpw06IYIzd`ooB|WeguUH9Nk8OvN=QJYtzMy$U5-BF;&?Ub865Rev-*p4os=uQ zd0XZJY-ZIPO9K%M+{|uO*sTqAn7Cd#RE%vXeF4MkP|d)a0c z>aP-{0&cy!&8h+1c|45Iyi-ise%Uc(i}zh2+=`@%pTAe6wXzjKh$H%IO93^svO&6n zGF-^#oJww1gRz>uzLI|cmnm-muxX!CNGKH-)YMe-cUqK;I3_9|!P%yMnk1sCVfDHsDjVC&JbB5nLHl2BIx zW#)5HZbq9=ogLuT0`m>zBNS=X=XqykgiTBP^-j#FN@zFdX3HUe8%1W3hHG2`uw>AN0|K z?#A=6v=^%M@4^c696n0n`SVeS{gsE~L1(1|;FH3+Pe zaG={ONvd9F`!v0wkCCYB;%MO)xHjy$_YzhgI9f-qX*Gc{N#W1_-X4(?OB~w_Gbd=h zvOJnRGl8q#1lx7FAT^dl3I~@}2KyH+m(B|_Io%i(=6W+sJ1+9{9+@Z?k+v(h)0t55 z{m5(_)mVgK0}{w1qIkF~@i*XBNarAO(d!eao0>qk@rl}%{6c$-XLH?%YGQBrsvvA~? zqoq2^YGbIumDXwk%hj~KzTovt`l-W$1?$!y06kg}!x3d@$8UvE4W$_ZYIG$Quo%;7 zyFqh)hIujpBAFdxVi7vG{v{o@$r%n5iotDh5>$fEs<{qz^I}GdZBXXHz>4Af)x@kh zg|9sufV4e7&4=blf-;S%jatuKw&*;#Q${@6F_@|vuk`NxxZsoIz!2Vh(QZGstQpO> zFm03a5i^)6Fr2!aYE@0BId#8=)L@xLx`6>)8>fPZPug<>t+Js)cRN7u( zFU(gv4}f29`Ek$Y(%36HUOLKcHVlVjPZgz!Z`wWrAh2W3u@RqC@K)!ICa>dRhu6v% z&B0%(`!-14wS5~$S25P@!K6_C0|(-2W_KXG=%t89o7ApdnT$Lwb5z9YkDJErPe;l` z1B};9uTni4-=Bo&x2xS1T*PEzrISRxR3&uumm@Zu0fr&EZ zkG0??E>vNPSTsBF93li$^A+A{*Lg@UQu$eN^YYJmuoP^5qu0%Qo1V8?Ax9B@wwN7b z)h%8wE7PrfyG$mq()21(OA%p|w^AaVYjAMTA$w*2s*=P60(W~&E@hcpy}C{>EsLm0 ze@&JN<_RTFBC75t662$lSHi`KOE{u}i?7HkKD5}|&NKAHjU*Ex(SsZp*W0dH5=OJk z0@I2!^c-1{Fwt_Fe^5F-j$8#<4@WAvUf_F51|=4rEvEZ0ou|l%xRk5YCW4Vwk7s-+ zs#4CmZ^hpw2X{jKDvb)Qs+qXQl~-@c!jo}?D8{6Z~I*PMALow zSLPT9(SPvN$S$2_##~up2Ttsy^(nW@Ea>~7{Q;2gn4g#&S`x}~(HZ72bhy{BX|~o7 zoS1@#-u(d-dmfy7NwPC)z{goWYZl)Dt-3rYOC}okU}y!a2IH^4EEAC>7C$!q@Mv$3 zXJ9GLHl8FjYQg_j+3JRi+K|?&9F&9pi_?)MFh_<%(-3X%Cnnu$9L!78KxyFMz*8i; zD?hO@Tg;Srne(b*Oi~;N=QFK;$XjbWwvS1}ue!<b%qryvWD{g^rcuVbumROO4^7u1Z28Z@LYD(z+6=xppVj z5}cpwZZ3WWCra}9Rw2+J$nU4D@<&K0$NOYg>p!NEEBJ1kd?!Occ)xP;JC`qB7Wh5B zTS7>tstksQ(>F?H92Os(+lx&0JKZrHg9tmKJ(YT23iZv4)EPDHh^^{HdYmzI128iD zw?njoz73gAdXFH~%!DCMW~a39bCF;_@`_BP7By&EKHg?2N(5tGb_HOTBG3(K*yl|N6N972$QvaSsWBL{^ChYTGoJ7>hKmbB-PQ`Z zc%<8&gA>1Jw35wj*BWQgQaf|MU-wPM9w%eOdfm{G0v|sogwUYkQy7^5Bm*48~mLeNadVM_p zt6L+c*0aQdXLJXo94hjJWn&YFb!uy=4BH(W(}ffRP2U^xxXo2I)0%T8j=Nbkha{xb zW=Pc#0{wUjID^ZOKV(PxkUN!`!9Zzf-i%J_>k)+`!ZJ;FLYX0hnh?4co-1w<*Gia4 zl1sRdOi7`S-Ytl~bO}4N$0>DC%x~{x7gHcoLS$m@(l>R!-!{tJyF>1czpuX@Lb`fO z9~C4}QT;R@W;!6iI?W51A<&cr{E4Rb^nFjgOb!||%R+2zI7__p0jhr_6)kp2oQ2))0%vrpQ6 zGB(pB^gGculCwaS-G%4pivqdv4YjL)62QpImuleBbcDJr3l{5*(SyO%{8r4(R&C1I z2LsXw=IZ#@371Y;d-crYSJL&bgNSbH?1|Kv6=NxEP6{SErA{t*92Ioo0J?nWxzmw2 zU0<-;k%3CVKIIWh^CCA3JDkn<;}~mb1>(&XtF!sGONfK@fK0*Ao0}bQd)B#DiDwh< z!}~Q3e6qNdiB1+AGSK>r7id=siSdD1gWmc&&z7bsba~vZ;J{IBfnmBBa?896dJpfdRl^-S`nY>)Zq;WjZ7q3Xouj?%%ayyHItMy#{S;o zV3UtDNwyi=AkZ>@-L}sHKcym#KFDd}Wi!{&rw@6Hg#OS zUpsGh>qh|@J4gcADlD;OZAaBVzI7^>)><*!WA(yeP7SCk3(TC^f&zGS{nR?lKf4{7 zlnKUza&*Oc0RTuVa+}5!2;Co!`sC+KpTDWury0WdshpHqB=wZ~bczy5!HFO)0NI|k z9P?g+wYJEgwxQwV6K(x%V3G=p?zt`LHB$_~sO3uT9a`}&ICf2zxXB!#i6`t*_q#7? za^o8V-y2`iHHs;3$8!da#u9bPR$ihIf%$2r8>n1O2Gjlv;$Gl1N|Sax%{Ar#buB{3 z6yZ&g2^+s1UK^S9j_s<19IV84pY&w{KQP}a&wOPJFWT%Vm+E4?Jt=7m4`(QZ{1cP_ z@>OOJ9k>aF!Lwt-7&k6Fm#p=Rb+P+#S}`xrNF3jMB2&^k#;e58nkag&s)c6JmA4oL=}9pZ zFqqi$=z)u68Z(se>}MA&o-cR~zhUwYq}OR~Jgz=}bCZ@+#AgoT^=?n)4v9|o(iIbk zM*vCpLy}I#X&x=H#G{%4dGtpC%@0XLRVToGu0(Yx9;xY6d1~01I-D{5(CQKa#WH+e z)Iy99Cv!k*2G4o(C8bK%f`f}{zh(&lS9vF&Mhce%&uV-@X)7r!G7chm0|5QPb}}MB z*IH9w62`WY2~MkNk4IXzf`2}`MIpg z3sDPB%Ew~xS7|gQ1965uvg32;!mzE19%Jl=DBeXzHcZM2U=+o4k8@?R6NKzmt~Vlp za`!~ynhI?xa%oWRQe*O_l#1!(<*E}@#*q*`Y4O`=ZF0Q5#@<>f)g=H)Vpl&cUU zg)3C7d)ZcUsYZd5rh~NV+^5KR>tI`%bg1i#cK`%^y0Q~x3E0?@3~!xv=53?N$(gpy z1c9J>iFNb0ZLV9gO<{R-Pmbr%ek}ol8f8_PtoeY-o;|y8eC^=Dq8jmq&c7z)TUh}l18$9aF*t|&`&U3rEJs3e7_ zw{9#|FDL3(_*3y~Qp!Q3ZPrJdj8)0jy&o4!C>$ffl0aqdY$0TzVpWYNlsV}6j)t*Kqfj~7g6fRoP9f0xJEd#M?d0Ocj!XF6V42y$rs9eG45KoCI%M;G#pzk356RoXwRTX9K#~t7Lug1QBl(pY4>uEklBhf8 zpqU0M9*^*vxunAc9}v&g&WZW_LtpCFc72{~3`V-`?W3};3iFquf(#Mn6mwT(f$|>) z)Lyv~VEAw2vwMo=&WDR7E2K%0KN>dRhk_)MF56M6ME2)y?`&#ehiCSx!ZnOTi7}+? z>7yYTZ`373MF8=d(uaJ`n;vaf5je$vaO;*+fZ-JyFi@2mhGMQL{f>ozn(TT1f?Xk>*UNqN0tic#SteEXb6uwGFPn$0K$xs|vD>Xdsqp?~)!Ly@AT1`Bd zsHbnSl!Ly!{flho2~9b=HF~@;B49w_QLozalUhX^E)cXR3UX$fyKi|t#0ruFYOSL5 z1&NC4N63a6I1#6A479}i-E0Fb+7{OcCkzfrB-aqqYoIdrgu+PHi3#Tb2Wu0i_oT?w z%>dfrfly-<@S1Jh*lnq0Nli@hKoUuWj3b{$lpvu~B-7_tGn7@d-LZ`;D(`K8psQc?q0NL6$?S8!xuhn}$4~ z^1QZuIh?lFpz2ZEF?P^!sY(zMoMI_8tu|A^ncX3Dis4KXLXvAd`faYS3A&XJN(m9+ z+8c32V;0lVr%b%tW^YmP$Q?)tj3r~y5O?b{wuXY63ejAJxO9NGY=EMuQf<+tbdIP2GTQjAotRpRU07quR2^oY$RQ9P64 z$)Tt-yGQ5e7ah@Gonyg}3WkdD)YFH(!+|eF@Ft3JYf-a^-Z&XzD$SX$5yYLl`dm7$ zT%Sz#@pJ;Tm?ZJ$kCSQ*aTm8Pf=r^ye=>SL$H}liH<4*ZO=?I~>TuQ|(qvbJ?D1$h zWhWA%fQ{I(X@o_72CdhKVDSkz^5bDX5%YLW%L_h)4n3ph<1R{06#x%B#Bm!N?WzYI zj_~-FwEqC~O(5Dzr4h+a813d+6G%t^ZTs8}y<{k-S)h(;GCReAIJ<3mxUwFUxrrB& zr9^bStWP_QzpzAoWbj%|#LSSq7vPj-qP zchwG5hn1*-gzyfKiUC@mE}*83Rs#FU8#QJq;yO35xB%KuE)5&jfQ3q5NRk^0WsRc= zPLAxPx-oALpm1%7)#PB^m?n6)ASF8qdrFh`nm)nWXXMq0;)pLP!?eR@5_Al-TEaF< zn*elRPMDkd1jR==dQuM6bPB!%{{SrZ-sZ$y2?LU3@E;QB4xD3RL!#Zov@{0>8H)4h z8L1Y46$I`tHqekiXafA9;bjs^Korr)i*Z0C!K(mZ@ zxG^gb)JM6LB$XhIB>cVXJ8KH2)Allr>7-Po51D$M@Kq9`;e$>ouKhf4vhl(U?4(BC zi1)bZBHt+M3I}*4y7kUHYUQIHWojbK5>EpjTv+OosBIB|?>_dS+qc^29$IzaG(vtm zenpfbDLM;+_|q(t+O_zU1)%T6xO1_4^luuEg;lHIgB136?8c_Mk54oL@RgOf4`JIzC^ zOm@N3YYb@$Ku7Zvp;YUpymVw6a%gH#Dx5SY+&o&zT{ii|zJ(A9K|1*DC&8;>id@Vt z(lx^XD3e)KNZdA5lcerpoL0yv!7>}U%`(36|?zAZgz!aK-|@K<(DviS?YweDDYSXs;_7xD zEh-ljXp=Mvr=I^ty&^et;X2J29h0GVAb;x;>bWo(RBrU&`@!fpqxDJOpPTcgCbS5fYu3a36Rjm3^PG3rgqWrV;V$wvI24IQaH2%(xd_D zeBWrTjlI%9lMe|BGy$O<1OWJz@~Wfb@O|!9PbfEwNR2Axfb_2z*auAXG+anbU=`!R z^(ZdMWQ| zx}NL^I}gdxyBly+Qil|diu^;Uw>v?a_LB_(e5mJS1tk=6&V6YO5-3n3nmpf3rZ@0v zQ7Rw`KH&-URX;ZeEwF_gF-ScxK*-rX4G2<>qIt`41gk(t4P}5(NYNmkGuSc#r1EEL z9aF$3m^a&3CjklPa>OSvk%cOzzO>Bl*`pCX9$xl-V%#9^gYIA?_>Z5PrLe+9H2eJG z8!ib4b$3<r~$-;>b@bhfX^5IS?92A38ITN!MnhY3;p&p@l^=&8NrW$+n@xw{C_@ zUxiT_Gw57SjWN^LaF?uxYb@ZzyyUV{Y9v@@NkCNe72*^=#?sb~-)powyc& z>!Y{Y?Dpq&SOx9c9n+=vw`%;KHum_i3*qJB`C08RuF`&exUdd82k(2tDY7S>_Z*4R zH0|zS3+>vky|jBje-Y2Wp{=kw=~obFOglZdJ~RfBqDk6y8=E5#*}u02nJMkqwFPOg z4Soy&i21|hMF8#Q=w5`8prq^1(D{zIx*$>*2XJotUJ5`4Dn^Z=R0cUipWyhYJ8Qsp zt9Ax3rDUd(QWL2(4L!m!D2VhTH_oSrH;8Qj2S_AF{%+FRY^5fF?b`{!2uN7OqZNfH zdR5>m0})wkXi&BgRVz(@GU;Ad3wL>$`Mq*1O#Gun>$Wo=i^4`^^~%pb$?hpI$CK$H>j<@FgfUX6n@y)Cd3)Cu|Ky@lEaatb&Bd1w~S4T4q(DED*1KtcOOu~)Y$cMLbEO|8A+TMg8UZR0eJS18y&)<(2J3d@%8fbMDxzUE`7{9L zQAeOjeC||sdA{IK)U+dE1zTS5#*%rO^xpO^T-bmIF=z~OgV0^w0gndFYF}CdiA}|y zNtXZyu$Uh`2AI%Q$D78kic1 zdyHBU&9$Q4cEf&}2P6)*fi(^I06KxV8nJ~rUXkGB5>S-PRF0P-6UvTAJo5I0lAufh z$3-9EXeudKER?Slm54hh?sCLi$xIL@O|B^?jK|H&ClD2^@vDmv3<#%7q%ef4WY7XR z%PlJ@5Ko4TFghy@z!;(J+lv|lmvu?jX|wNmPfI0myJkS0o1u+rN6c4;NOFWW<|~$= zB&Ay`hP?B&`NdqhwrN_PjqlSWnWaYB?CrI}XYKcQB|0jz!0mcMu8m-m+!ZTX8Zz?X zN?x<}uyK)6cOESKtd$ZbA?qxWw4)S>gvr;pZVlp)HXS>B-Aw@+k_T;MQLzeyq^No_ z`B}(Vfw=7WZh7V792Sji@`(GqR*_}1}7zO(_ z*P^brKpIr+_q>#Z&e7rTU>WV&ubV#SN87J$9;2V$z&Lb#rq3?X;P=e{^XcdEd4>UF zcaLwi_q0*)53{Go(e%%kfyKHw^=H2mV3j_zqdQDGnx5@Hb0>sPdif{6F zo5Qhql?^&NeA&1Ugp&iI;GjniBbnhKr15>kM4i4`Gy=&f)JUGkYv@rV9R|qT9PNgp zzVh%xU^5hy)DAJWnmS=;(F1vE!aE;}um;}S^ko4}w8Xsxpka}=PVL$?a)!~re>A?- z5TjF5U5)pFqnANck%Sq6_qB^SG&LM5-;Q7zAQ?SA=X=AY02*sGYd&CkV3Np1zK5Jg zW79cDv?jbXh8VXySqQQTmY58F{xwFp(T2wwarBufXU+zUQqdj1X01nF&3^}JcP3smo@;NQB>)mdg|Ml zt|>puJ_qe*XAB`bWZ~wQo0G*vaR9MxE3XeW+rdIW(v)n;grgpWKzO*Le(WGY)52Wd zV=cQRpsNHxBu>5#+fi1dgJsK#LPfdIRj$bINfcm>3`F`k_yFSWg(B$?Qc0LQ+@xH# zvsj8zi+{YtRt%F3uHX=WbobG|-3gis)EfD;309CJWHuy%iNm98V^VYk9r8Xe2|`Ns zBD^%|;ObY15J;}+FF{I808aB#e2Zfml*Elj!}jfJ%-Qek;U|?+l^#ALRzNZ}8^fvv zBV}qZnelUCx;(hbt3rNeegVnl?DusFQQZ{c)Uur0 z2T7=n{{SblsY*%TPX}P=b{%E!2~pbuf#N}saVE4Fn{g7Ph@wZAz2w0{ILX>iL2U$1 zln91^$pGsjdu)fnBT42U{q2aUsQw-&eRn$mQ$PXjECCpyLtS;)_LXvJSw7C{lKLQ~ z2;mzk&9Ukcih#g7BVU^U%pQ$n=3fkod?m1x%26h>8)+|`RIW)H(`J&u7ZJ@P&*cw7 zIa56SsiUj6L572WxuE0F(zDJt$Gxx%ksyIdjx~*qYEVrkN%wK)XbPdwaBRh(PL-}9 zw~GMYA+Fz^?~J`GO+Y?=e+DoL6rHBM6n#p8RK;PS8VE^GJtw!D<{%PlGfH-cW<`DS z@zL1ut8@VvgkvTMtz97{G~1-@=-AR|Hc9DxGEOS*;qqV`jd;7;r<-fc#VA2~F@}`r z-cGFnbg2`}>C?}HvN}B91vG39nI2ub9(-S9^yoh4+IHOIydMFVAW~i>?jJ&Ci~}O!*dF`EpShTeVIxO{FJN#XX)4sH zwgskYOTtl!x49chWgr{hr$7r@l2&S40umb3YaNR$4FrJjW?($x15({@_NNMJe1jMZ zK>+g2I%?1TO}&7h4~ysMfjp(hN7}_8B#@aSiF1?UDhM~FidJsRTFoKi(xFFtr4%qk zT*Gxh^LR8Ul-wtaW@R{T;vDTC-Y>R~WgnDvsNQ_znT1tUWg^LiuVO&$?_;KyJYme@ z0>A_FYwu;`Zv|b*88?!vYPPMuf#7&_w%NFhDz$QvFiyq_QH&{EH0|Q$`oRHNRNjtI z!zXd4&CAQZ`7;^Pb=vr1a$Kb@*&8OizW2mfD9ju_iVuK|JQ;aQmhNCyiUSV*4D}*! z9g>m9WP|Q`xl?C6<(v8LA~R`>Dgq>sdw96rq53TCvD_HdBq_kzNX8qBv}8$Gl-x-W zB6ech-(3QrmF>|1SHH&RMh zfgq6{d|`)}1SVrb$nUrVo?4?xdMY;QvV2-by2MfKTm#f$K|w>&KPRb##t;ia0VWE_ z=41?d4L64ma5rsG6bf-pm56RtvxIpui2y*WCed2bKvV?mBUd6aE*vCn^AyxFP&pIQ ze&#wY?K(G#Ay`hJ0lQ>;*udr-Xe7}ihW?jg0#6r|f)I934I4?sLNx?Nx@gb}T9t0G zN=Jve;F*%8P{+UE$%0T*Lmi7kt>~Sb3jpRdVHkA=0o>h-0zy+&f?{I4W0*803c?e% zdr_`xBvmKHfEOg;2aRi|GpQ+(kID|LX_KU#0d(9VMjC`*oS6!e$T4K*!|Qin6g^>Toj6boD(%iiuy_Ij zi9A9VVTAZ!3o0lDij zm&@lZ8+nUJx`I6wq$qSwhk?{Sm85|+$o$u9F``!|-KM;EIXI9Ah@R73AG7Ok!1$)` zR<5GIr@8V)!_flgN}taZGZ`y})?(7EsU{Sa0z3x&gj0j&o|8c1i};7oOYILIadE27 zp*kjallC!{pHhb;cIkOwWMP*ga!Ee^#o|CsNEGd>fL><#+IkN+Ge$M5RRPRhtu|WI zlRX0S7E`K+%=rlUF%_Rc;(P~&zb6W%2m(!7l0=QJy`t8SiRmrog~*ytwe41Jf==FT zUz|)~1S-1r8<|!wl=4aFYEYtOr&|+IG4B)468Rh&(QqrA^L)0qi0!|2j|O5Si-{De zCr%HNrEMS%hDa%z*IGQLeJBKQWr1y+0(1lBUQ$vd$u*eS;MkQ;Awo}+&vylM#A2H( z+OWVnrIiG7aMrx;m0$@52fXt%;8H6yNdSHAKm?JrnVsKjG5|C$7J=}f86^9fQ*ma1 zLns3rs`I^y69Z%H7U>V78UblRC7PRfUjl&G)^ib{a_xwBbXEDGtya>GcSY5x>;swa z(k)Ph;vhB3&ce+Y2BbZZrD>xq2<#-b3o1~h845>sB$4kg9q<|kUm!3tyjA4QwfiK( zK;B+bN{1*zt5y)@+X0JNRSOc~2@eF4*1Cq$8WuT58kBIU?FJO_nB>9Y^hmAU3(azp zH+$IOB?*(ZuekXfb~k2(p(JTQ^haKn&%nl7*Lt<6{!nYYx`mY`V}Yk{7Hy&}SV$VG zLFe1x?olWUR+XO~4c_L@@kZ=gS9QW`&dIU+e4X2B;-Efb&8sn;MUqqLDwNnKizh03 z%V4BL>(4cK%IM^rn^ycVkp(Ntk&2*@L~PvW(y~`<@sZ*u+!)!*7_9|Pir1ay#mPcH zmH_C{!+4dP@_30G$M$L*L4In`#X5-K=ckJW2?<f$LM1G%Dor)o;x1Y}>XMN> zqRjvzOvdIyRFINprLC9{^~MI-uZxme0&N)Aw`p~D!s=L1M@V8G=s+@mPsY{w*M< z4jrZJN+3+j*r1fG%^Ql$`b`O)L2O7HT(BSBL<6I6IHgH5AD3S?AsCDfg2)LdPGj@{2Snayj3b3pL0PPB|7hWT8bd(O8|Vr zf{Z})g+{LzE#er6!v|mvs|n*DFMHAz5Ef(~MP0OL3<`?PX-(DWd5S?2Iy~Erpl%o= zO*?co`f87ZU@W7Rk9~A}1wwZ5iKE%2E6?{kSQRxa7_>%|8|vHXQ28Bw&x3^y!^g)) z`&O5EX|{k=yN3z@uSWs$Y(lCHWa-Duxz4WYjK+0Xqr11YYS8}x3V%0{-u;JG{u#Gm zce%K@y_Scdll;f?EB^pay1z;Q0sw&eVWD};$aE*G`nM~Pw-%GiK9bE=cTT~$Kytzw zQm9MnRVsanC9u%=ZWRqJh+0i0vWAUVS^lia^_{NoHE9<(80DjSR4P6E8SvW-+=pkv zI_L04cPI_qiT!1vUhI$Cq<;!TFZ(tASMzmZNreKE4ib_unbMeq1V;gFa8VZV^Cr?auw@TK*Q!^-d+>8H*!J z*3xJ{of4v+dX4YP_ivrHxCglT5eC}M?w6kJN$gzTcd_Ky*WA`3ea1$mK#0(aQ|@IV z2pGZAk*zy;HLBEC?s!NkBV=~-ZtUJOQVc*`%6L|E^NO>Rm_CV(+E0s)$Ap@S@H(~< z3P7ma5F~WG+-xPLnJ$GPdK&aSl$98h&THcQ`fe)G9*_}E+g!E0+E1o;&i9jxIi&{> zq{yb5`Ks9{Qo)^?e0b5ULaRtS`d)w%X)xf((YR&+h#Je!r4l{gYhn|MPLbg9AgVU) z6=(>t$s8JqmqJ@oov(6?p%Pg*UBm!crQj;s_99c&&jezv9BooD)GKz`B1i6p`Em+kjpm2S*n#(a+WTaG*dvm%HHU?M< zDlpLm>}O$kxU4DOzyMT-twU1HTF|{i&}%Sjs=#1ue9Ya)g(H9`r{MOxed5G)ws#}F z3fKeDEUfC&QMj17mBC>L^J3+6{5|ia@&Msa27Sc^BM$1Z)o5W%bhfiPcBDtGSU~c7 zw<;iX{Cu5Vm0VT?jZC?F8AMqUVM>mG)S8O%GMv;%e83&#^LU3p)r%d`NZAL+?s4ZR zFI-W@2$QCz%GxNfLIeT9<2fd&X3TYFou&T(L3hAim5wQ=JBRA{T2_TngCmW8=P0EW zFv>gzY=Fy(J({ zC}dv3n}!%BR$=WCg;_`>iG3J?w3@YHHucwwBHh{m&YwppEYsq$X2d+g>LDpgZ45 zF|XLr8(D}LN=H5?+$a%TTwcLSQYVy7?Tu<6>B-!{I&h6E0-c7hsO1n%JZHm4fRj@8 zqET+RYqa*GKwL=+F|Bms8&W72BN5bTPG+P+P)-q3GS{rc^soX>8p#HJJ9o8#ixhM2 zc!P^WErn`L6qudAXlMl?M}~s4Sl;G3kpBSx08c5Qn}f8tcuFXpR5ncsQb&T3Rk7=%kQA@N5bw zG8idB_Q>={N;IPO#ZiZAHZ)x~zNmt!8(4GGLAM0gi}77&_+KTR#d65~HPa?gKW3x+ z$Pg>UM-wm8g^@TXJBez5_iV~v=r0&J!l_q2j&&hBzUvGNUSL(I)s3dWkOZ7fFhkWawZn8Qm4X?ChBB< zXfPw~ZyPlCBuAW?Lcmg_XaE#6;m-Clc7#Y(IM29E9HB{6sF^g4-KhFt;gtv*9OGhh zlLSft(g5Qh4Oo?QhuzKp6Ss!{FC;0U!ZQcFS=Dm&S^t7yXro==;MO$@!GniyFUml$qgP~7B6 zEh^uUy~kKW?&bsxQRLm23QfhK zRH*Q4Cef6m7W3S8TQX7>fm0xZ&-Xo>*j5b5kV}rXAT48=9)KI5F&?G3e^iCF%w9uBKbxjqZJ)39pXlk(&!LM&@dPfSk};Hyc><#O#>AxQ(y(0 zK~2=2Bjm#b26k+fgphP6&5dk#cN2_KJf6#8o5WSI)Rd=XSyzH^0)wfJ-)j&Easf%Z zKb7p1MGq^b=G{RZ4T-i>RSst?M^c?7oEC%y1yU$H9&dilcHk3)5?(OKwxxhUB}&;G z9K66$QbNJyd!G<+nUPbjjkiz?;Nd{Q!lIcm+r_jXqP%8dCw|WdgpjOTH#F#)M)36m zMADmE7*s7b@unbZ*NiAi=y{zicjVzF$AW&={YnAG1b&g#|k#J1t;psp% z2gEXRXZ|+CkM(Wza)YS<0435=-1`;W)nOV=zXpOztE(Bq=lb^{)#4R%ytqHpx5`jI z&G}Q(XLI$AwvZXNdK={{Y0f%-p8e{58G_JhRTaNA_X)T}+cxt!2>rqoX^rPa)QM&aGZN zz0#TVka<%UV@5r ze+ufq#2MbLgoSt3NBT=7!2U&Qo@%Fa`lnX)-b<{&7S)38+gBhip034ngE6RRyqW4z za%fxJZr0Isr@bit*oYWk+6|nMPDAgb4ht}tdjO`ri1<7c zv(L3^`3X7_Aw~@D4QNj&00`dq2X4VSx{HJ;*MDuTfbT+a7$->CO&tjq(g+4;kGocw zN|jYin(cizQ?jsNc9$SN;8fJ>zp!bD{N1;=j8_OJP6xLaQ!rCbbN4U=Nla*b{GO>* zMRj8 zWWFY)(ixU*eq+$qz%2>GQ>kwQqkfE_0N5BeZeujidAzqIkQc0=0W;#%yvQ_6*K3Y1 zGTMMpLX6Dqaf{@b;LbG;I{L;eJ}%+8NgYbH?M6CoVp(cd)}gTPd!%&fCAlkX!6$PZ zpseFN4SRFB9I7!V5&YEX+Vy}U%7Nf(@@EsSvNe>n0&$Zv*>yJWDHgF-dwq^AIfdIn zLUz`6vFuzX_>G2qDqJ}6zJr?#o4e>ejhjfah($O1%ZBS4T%;?Gr+9Oqg#db3xujyZ z%w#@v+L6l{ILW>|PdJ-=nuZu0sB@^d4m&cQ>hb@_V0*@oLH0%Y)qeFX#pkRo451Q^;HJC$5fP>4KUu@syCM1W4R$fRpMwy*+$8^clWF;~IK*b|qv#yl8y)7~vnTr$=C08PuIyIIte4J)~c zR=_j@d;?Xxc$N@NDx%wg0!pN6e#WvuD&@+A5n378(5Xu1ut$UGSSN|ne)eB>mDwwq zB56+AH?c%^G^cK?i)2AbI<*B#c_|&apAbkjdNB_MO$iT2Xx5ZG9k@~gRG6-!fQG~% zk_d?2^ANJFH*3A^LMoF@gKO9XDgf2wC;(;+bkZBl0wW4SO!B#i4M!MFbeNM+TFS7f zD9v;&lBFbx=-7N5&np^^^Un0e{*v{Lktc zJpNa#E#u4GO~JH&WX{DT)O+?n&g_i7V)j<(yJq68ENG;XIu8-V8Xc`sYE2)a-^~92 z7T6YV+s#6Wx*)_xD}J`xrX%=U`m^zHRvUTb;Xh1~$v?z_QO=$^2kCm-FK4X!wMs!v z(1^WlNivdCAn0q!#3PHxQa2K%Qb)>c<`aL?AsoVo?PZE4lrzA1>9zD(LJ|~&A_lra zo@YeiJ1HaVeqEer4Qa=wtzjwVkwR)G+}bfACTlgFW#v*xfjfI0TL63o0wfvNW6&Mh z092O30n(#xlxu@Di5qwhlGp`XQ9PjnDLV-oyh2UFN~f8t2Y3Q>5NENY!3KFEA*ukb z@gfrtM>pFKiUgTd`}`V2&X8L+(k3|l#gqhaE#a*=My9WLhRH8Njkkdgr7-oQf=rcR z62+>FvJGED09CX}(H904`sUPGb23_P44e{?_O3Jr){GqLRKxvsYAgvWWix zgEapD;&LDV0IkQa&SxuqR<7l*rD+LK2uRFPQxdGIuOu31NH5Nt_qFe@f4g({pQ!v) zvW~pI5Tzhs5)!2+Lr+BPWkIIt<=2}XZG4U!ico`NH~#=jnfj)Lf)D;5Hxd39Gxcr* zyt(wZzuaH%IsU3rl$-dJwR%31iTsyzlj72h8?Vdb(0CBsOF#iOGk^5CpQ>Z4yeIpN z-~BFQ>YN&G-6>D)ZqyI;7yHilw|He&k3hnST)J;t zxhF51ivIxFvmaFTnHN@|QF{H zl7DM={l)(Co9lJ07i7gH>Rr?Rt?&02v-?hOs+3eOl_^T1V-kspMb0GlhDH#ROY-<0 z0~l^h{+~Dh085zqrrU)|(=VG5{{X_~KB@WCUAm-#Zr+js{_%gj_7!yh0IPR*?HBvZ z!>9iMuYkabbfWS}{{V;0$NvC}nEI!zD=`^-;(z`&VE+L2-n!jtLhS^pNd#V^RHV!- zdZ&AU?=BDZZvOzebN8E+G~(&v?xA2r(M8g){625~mofEB%Su=fY-aa=!sb4ytE!6B zFBK_DBM+pdDmfQ)#V^k$_0j?FZi4{Nc(GXiAjhYtSNUhr=ok^{3Ar*d`L2wvXX@S% zq%10mN|IqgNKzD&0KrOtIvx%Wyu0+5f81a1ITgUnXCq(K7#lqVIKd1ptX8?#`V^v; zVV74h02FKs!jw|KK6VloVxmjZB+VXpNC;RmtA#j0J-58(7ykecawE`D@-5*?sEym0 zx9zXGxu>AG@GM)d;5>m*FHbLEKUCDOO@iMVJ>dbcFp(|d-xpIt+QFMZ6Awi%387?` z#tQPtDVCy=8U!u%((u^Lxh$7iX1cv)RxS_^r)<{RzljE&I`A$m7p71ilHK%NO=t~Z z8V=jpB`SbIW6oSIPT&NB0q1gRxS|4KQOCE%`5KYKHjaIUt~3+8HAaR9HGc5yhz3mU z^}Gw=Aa?9r0jU+F9hI)>(yxMsX&;yuL=hwj?Z&UN0*Q`e+F4*Mqnnt}`L=p0NEAFq zo_>0~j4IjC>Qgm>V^LNN+LA`u?Ze>K5>j^8!LmY?^6f2)D_KYa0~Od&6>`~1%6J%M z+qM&3to*_fWl>mhWkz1wikQOP!n3gS)8_k^0;QPw+oC|tsHq?T#qT%!z^=NtwV_Jx$@pV3nNz}A6S8jJ`-=JVyRL@C$zD^|bJ7dvN z-1XC09a>6~92JDisThrU%flkyqQj`lQ_4I8O4W%(a4_8`l)%f?WwJIJu?lNih9G&H zZhA$Jcl*>0go$kl@e8z-D(E7CCS~`Og7kRxNoqbZ&4Ear0E(&TR1MX>|z zCGAFqN-zg#8{Zf?l^*jB#Lx#DvjZ6AWbCX1v9K(bhE1RfdRt&+QeQBW6lFSDVkSC_ zMeOXK{{XMdY5xH3%fAh5LKD3q`+lp;D=(V7pp+_dt4RwOl0md*Q+Kf3hZDA79dK59nS>c1BM0QK9`>VMvTkr}&kKj1pE{))%$zYo{_ZGZg5cU5aG zad%LKD6ZZrg)CA{CvP*-Fx^=0+9IE#>XKsqn?K~b{HOWl{{ZmWFZKt;u|0p9MB%lS zPw?8qG$OcECs$C@rnBZy#EC+Ahso`S{&jgY8ElvP1LFJscYpjoOZ|cIa`dy@4^h`2 z-dUa(zw-lvEOqtnX609BD!9`XtXi2ql8kgbf1Y3e01uM?0APGw&raRg9;d3k8~*^= zP-{JCK><^3Dy8APBG1;upMK;{o;+`%1pe1F;U zU+gF1@A=*D@cA$H2gT>@_>A>EaUJEJYv1{Sz?eIW9bJ8zyBO_dR}&bmVu>}*e z`msQ#ppAxJQ-%mjJJ^ZrOIfxORFX;F%0h6VK}iO5A|;NJVU?TYTDDS1D%QK7Bizi+ z5R{Ho&qG83NioyH8zbOvTBVyy*#iumAivW zcLofsv$2TehoU*1A&7PBYs(5kl#UenF~}jD9#+r*7ss_~@YsSZtR-NbCa=({C?sh@ zACx;Esmqzm1wfb;d>^v5W>N^`;S&VyrDKe=#}0{mR;k}Dr<7{=nFbS!6hI@1(d9px z1PR>uf>H!Noo8)ZKq`{jlBx)vAA7<;lJ_(}#M>E|OsYg~KIc&+j#s_s^D`3e3ly^& GKmXb0HHIMo literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000003_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000003_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..851709245ed443e3cfa04c253efbeb10005fef07 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%T;T8VvzTLI77<8WpD`R7drW zh5#f4M)T1?hQkzK4cfz+n_pU-inSVM0#?IPolY4{!z;O-J#%I+$VWq!9?G!+40Bs( P*kh$ZNwPE%Naz6orl9i@ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000004.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000004.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d59f69235db2b1f08f503ccdbe12fb87afe4666b GIT binary patch literal 29376 zcmd42bx>T*zdks)gy0t3-Q9!3OmKGs1a}+U-Q69A;O?H_!GgO4f@^{Y%TDsX_ulWV z{r$7GRa<+gnx2{N)8`}4^K|!g^LzRCHvlF`7AOmVfdv3ypnrhhF8~Y$6R;T{2OBpV zCkF@M_ZmP7fQ*Xz3iZ|NzZWJ3=KoyS*w}b@c+gi&Ol)jiTs%Ai0s`n4(BjRTH)Lca zB-GS2%*@=s2LWvW04y8~^aA|%1qTZcgMf&H3ixLhSo>M~ABnb!MM^F)!^Jg`#nTBL_{5eEi>TOBWpLoHuDcn*njj0;N zO*F33Ap>qDJeQ&UALePX0I;wCcvuv8SXemdBcXP{z+z!ja>C)Ls==ECN&F~EnrXm= z_NdOaE%_4x&(x*(_a^`rGK?>5ARG)802T`XhXwe(3P6M242uPQ{}%8onY9!N*E14N zn96OVwsu6&by0i5?;PVu*Py1l1yY1Uga!&?+A$LDs{{m?L!? z6#7wBF+!t2~|z|rKqC0W}@XlZ`T2Rr<6#a_?dZ^?fo;lSZJfa>M0)Y`e4qU!8yv)Fm0OvT@7G_Jeqp{xBx%;liAF0(1O zRx8S%Z*cRPT8X2A(8NFX_w!?ko6lm5c*hYh!{9k>9 z@4ut9q|2#%k~}x)@Lnk7_*kKe!UPJ9OU_zYopjSu8rJnFzEn)pBfzX`)o7vp07tdS zX0HiJ{v=ZNH^RDqqfce51q)c+51Awdb|45iIBbLAEDj!))ZlQq#q?tLM*4rfas^P}mAxvfK=bk-6M z2bogLAV-cWx|=RJJP9J$O2E5gqXuWDuM9a>L&=JtTz=$u1c}X{0;8fsM{)o^eg88A zwa~cBkjJ!GZ4tQK&Us*`CWml>Xh=Idj1H^Q`i54?Bl%rA{GBRfs8&@t0B zbfDF?s{p-HK^=_o>}bMMfCA&6i2MtTQ-WzQb_=A&gr9w-C0!#JSxs`jWc#P26;BnL zo2>X|V~j_oE6Lc|)nY9purAi*Rv|i=5Gmnb`T8q0=LbpYZGQ6OG3_cEX~bbP6+jyK zet4v*_h|pwYH=7a^&gpJwfp~j_1Lc%#jk`sge{1?R}pj^ul+;V=d@YYko6EIc#v?( zR^NEIY=IpQSm|zJ7q#|U9+?m?KCLFLK24eMu@o}CsS=FRf@O^flFUv5Z*)3i)26H{ zvH;a5Ew9-vtG`-kx3RF#JXX?YO|tX55_U==!PjI7a}WE+~N8&ciD&g>~EyrUavT2AFQ-Lt4_r^?)jTP z=kR%FAn>-egx~xIv<`{Ye>_+3ds~v1b~L_d|H`(NZ5uPql#%nv2uJj5LeAvI)W!h; zEIci>bi@(foiPu_Z-8^!Ue~StTdk&ocNu%eD?QZnv~L7iD;*79lysiEX?E*BJ}EPe zoSZ!`P1feV=#bF{Se*K)eD&kg--os)*WNV#;UC^@*qc#cHpuy?ekUh}^lQao$l#G* z<9TH&nmAnyY9xX7rnkgnUEcf7S7#>ygWE0N2Z8fz}oD!M`$Y9eS4n3ZIuo z+o$)vwBD-UI0|291!M(Se68~O@*BX5TsYV_IO%kH*Wg zZr;ye$`AVVj$6C`dD?B)Gt$u|8{Zbt>s+MEuY2-D>wHzR{^OB^4#2bijiP)Kk~7Bi zv;O2Wol(J%>*=!C*#XAGKMuESy|kV8%HFj;$nUa4nqmxxm`-z|TT62pe-aygzO^O} zg=^_QkhiEsKrl$G>V62vUEGyrq6be3h%GI2Z~q3Uv;O0bfLjMNvd_bQwZ8!p555t^ zj*uQDQz30pdk=he&sqjXwHzCu3MO7?8M25tI-j??sM#Btt6$N>)j&|GQCk!p2Ij+- zR#V^K{5Xo$So$ZAK*POqoAbvx68qldt92&_iz@=mRJLt3l~MzRph#9(P?;LA=3R?V zscT1$O$+#zrVgDlczGzia7+blPPgh9nk+xMzuZL0J-^*HgiZ4vgrj=j`)pT% zS>HJx|KjjZZrh*rr~KIdfT`)sd*lyjze_>GCT0t1@*0HSYR6Bk)@ZY*v*87)f$Oj1 zp(@Sq3u>BR*jBXn?)~cDZEShG>2buh1;zHr@ut&nK*b+)+dV4&fiC56#JiIpURPP& zRVd?W{%RUpsr^{gjBAb`*v`H2W#+hM-J@% zcy4UF__pS;=x6K0yRXi}9kaUQ;m2v7DPD}AC&o(IC^B-i05rahBZITYzsO=o))}zF zGvQ*wiMFmDKlXzni#6e|X6XKljN3f3f4L4NB7()$oF}u2-Kr*Gn}c|eE)0;Ya%ygtbM%sk6nC!!uhvnsLRkGUjIVp5i&Lo8E3W8 zn=gH%7Nr&kpKMzaa})4K&Hi!ezixe*s}9irlliqHcF~a(GiV#A)SCU3ii&dOR)_0D}zVA zKaN8o3ss>1=N++(97A*af9ME`SBZa&PB**xXMFerNvIxg{gp#LzmHH^c`W>Hf1Q#4 zeY^q6M`+Z?pO(a+qW)(z`3JyIDV=uyo6o(>XXXEP=I&2yp+mqQ4_6(b?1#$nAFZq! zIqrp;lk#7FJ^zDs_ew!y{P0Akd-B)sf2sIi zL8ALRP{GBaXUt!=cB%`fG^<+jf@d`w=m%D6ak?WjhEdi;_fAbpF^pJ-?awLtnD&w< z7t@tOXX%G9Wk;Z{J_~R$KarWHJ20d&(Auaul$UY>3Scg8ySKt&g!vZH9M5f34NQAk3oMeF zNQ&rvbw9Kg)6U6gqdGTOTnTPj!x27y?E6JJ+7%s7raZ=#KKs+|8LlNE=+2|mr6V@4 zy4Cbj=Gqe*Z#&h05OP?IK&|)I@LuY~XREJe5qxJ7L;H2`yHm#LK>@$Rripw}`TA*W zC4b8Lj0a1RM%H%Qjkj57qd#Om-KsaITqn_o;bfhU;0$Z|S2i}^;V>(t*=*P((8@Aa zYNU=_A}h~o@AH-Yb-L`{NxHfp$zCTTo%+Xm4FD%zh+^$MMW6&cDe~$L8@D!o!C{4- zSF#!5X`&h7XE?%P0B7v`{}%-s;V=hGtQp{M;r$7bGZtV*_1qpc4CX6#6AjhV*{&|o=ZCwf{^0v@ z1D=SNLlck={=i2|*HD&|&O(oZORxISthlR)s#hq%BsKv|cDo3$ZOCi8^Po>@DChw1 z)Zz6fqkKvYO~#AdA#{cTkkkJ>;ji60r`t=*u~i6!%t%TTib*#cP8jvLlAdI+Rpv8C zt4(-SD*GI+Z0PG`{|3+(<>(|#|1!!reA=nknON0*+e}E`pkip0KUGIN(~;BaK+B)V zI?`C2*i{Y`P&nmD!uSSutYdd4UN}x;*7EigzzS{IOvczs79bs2zuvZd7A%QLd6t_UmnAfaXb(;4YXiqhxRI#+mXFY;d@bu)XAt2<6*JpDjm`;tt zp$}HyQLVO0e0Sy+FZoHPREGhTI})zRGGeY}9$De#fYY8jz?i;t*QRwVj$}cvJ*fWX z%o>!z;};C4B0eNZ<{h#+QJ(W@`X$b$85D&+cB%)T>|WQ?fK2#aekZ_{DR!PQ)qX=| z9%rO>mw zrdJ?)__enUyISxX0h017Jq1c9H)>eaGAU0TxC?78)6_*ie4b)E>4>&Iw3CzPNdr6OPQ3oMSCjPS<*< zzS~5Q-pO8UqM&eHBcX9u$d)ljF_pN`SWlMOQ$xgAl{36-g%I+qYtj21y>A8IMPUW= z{wO_ag9D6;!f6!3lk95oH#H44ad<`^ipX-I(amqZ1b~a{J{$6flrJDo|KrPtD^5Q{ z*S8X!PafQJdDNK(SI4cK@``l;5?V3Z0IQ6(!-Vdt1BC8N2ur_%qJ&!(F>umEd42X-qmWpw6e zA``iT$}gFk($bfzmy5n~6TSUE;&AzFe(|dnsjB#~?|e0L7Q!#tCh4ooF|X%9?*e=4 z!6(wzg91+PqD&^ZbFk%iYoFbw?cMhn46T*Ah?5%wtazDGOd;TPF}VHKIyMHS|fH%0=D(1_QM$p{+3Nc zDr5c%}dwJ?2=Oio}$12*nP>~_X#y|r3X z1IAv`rb6NL#(Z^%Y$cz_<#7$+69+^MIkSR9C|u{qW`}<~XvPUfH_lY9Xc##zp}RQw z^=BBVuS4h+(A8*5?F8qlc?*>jIFBvrXnX~s+)#6EF|0myZAe;6g0MMA$eYx+Px0W) z7}SU5ih3SaLq8;f$Vegk$-@Doz?BslRmO7laznLYr%^`m!^Q~n2m&@520=`_cFb{t z&U$$%T`K!<3%8H+SN4@Veg2v)Ne6uuH4R!Chm+BV)+-8RjqzU>8wk3FUAKa3xwlxZ zC`j4jK#FhNYR>eeh@+YQGf<*{-jl>2d*&@^F=c1k>kHir8jNm>v*^DT4p1{Op08uCU&0|G-A|k7$ zS@9#^c@syPZmjo61;u}iMt?X`{hn~7+6p=_K_`6PpEx^Lu}jndSK;Y{km_lk>s@XL zfeEqt=!s!pbmUwe_4_eL)m^(ME~M&T=?FV3HGD3cVtw3p{l=?memVI@PO&l=D2yyHtg$2CUUdh?-L9pRd{Ii zC|eBo7^a#pb{1gdc9u(tgxMG5rFoEj-qc%cVMF0m=sWd7c8RbSeh`kxFTdEyg zK%l4aoMmAe8n(VJS+*Vf&2k!^t$G9Ucj?<#J4MWD8x&6a#U@}-9*LDE97xMA*{)`*fvZ}zB z{m&~K^MbVtZ>>0`Bw|`k{aLL+O^BM(uaPKU6qoRWIl{1rt3HqNdO`ODiaTW*l(9Hf zlnAc{uq33<;9-4en{qWR|cs{ zob%)gV4vw1*Kd*AV|(Td#ITb^z6D{7nF>Jq({@SvX!U~8hS1{2;5Kj9HuLA6^rb;q z=?$Vobv+C}t`<3h5vjsVLt51!9}$-kOYy1uH77g;AR$aB96*DtBSi^W`jndqSI5VD|@>uxTrR*{?*f!@!?t&ybnS$HS zQ6PWeM{~ecnc4SY5QhM#t>un&qvPPs&|Rhg(35LCSpm~P0#uodk(5#QuJP3Y{}btX z>CktLLEhFialK$*)HWN(FzuSD&I3JDB2%AS=`>Ys2`wv1F^d^#B;idg9N|sepEc#{ zHoJ8W(y#r)mFEuQYoD@&saxa^bcxier@R9Zvz?uMFpMSOb#^ynm~|3| z4`oZ(N*7Hg!8mrB>MQD-Kdtl-v{z_42a~t;zhW0R8X-lOlNvL;|k**&E_CW!H)yc6PV)LB;;tYhALo>eyMAPNxT$hs>}>S~7v~A|wnW*X z`zZl0BOA|-U6Xm#YV*>gJFPf+eF!ptm)yj?(nKb99lh#_;nA>gnI?~PkLo&x~pzOw@(CN87f{j2_p71W>S^M|rE-)~Jt>YZHQLD_ar4z)km!I}ukPLEK>(zZfhmNqP zPa3nVb!Tt0f6U3<;BW9qRTB@ATGE<;qzF=iWI*m}%l4>Z` z$S~Eh>#F)Ns;Y4Ta%V#Lt2h?R+CL|LCG~bkeEDUil1VCO>)yR+4u|T18@ia(W*9%m zDqZHis+rkY`5TZ~GUl6&P!00nh)I_1vJ5awN5JM2w5xmtMh{va&Y=Nt;%jshWNki-{9&fyao%B=TL&T z^X-XVzIRk0<&fb{ny7CseJA6V>_mho$EJT?crSM8k&ds1Ktjt~VcyL{V+#L?$iYOu z+R;QXG0AXk{ew?;TSq3{YsyU7)bPQX%*d@S9r~nDQ=t$MTg69`N-L$nPapGr^-_@s zhKqSk1dn-Dns@qZ%m|6SkotU-V5QF&`Jm?)njEp`(;U7Z)PseY`R1%WanV_Ab-uTc zvm__MFctw(0Eh_N%rJ%VqnAJd@ zdKHLB*8)@k2S1}rU7&Km;=6*u$9+$+^CO!S$g@Jc_jHn5{1Lt3pl1Lc_fFfL@28)> zp-+H)-Q*XSbdSv|!M%ny+EQ}&#h&T_B2ng9L05ZVpYJyq8kQaDM?D2n@FD}l%V%rf zGYAW&GYyf3#@nLf4F^$+StV9m%dC#(+Z)1R&O5>olFUzBZ!nDB0T#Kcs{rY|q?KOk zL*M;MY4NCx9DnLkrGN0s$*(h7Y^2=i2*3=Tnfsvc4GB!<2w}?ncHvJ0Bb|Tn)r>@) z$PEZ$<}x7bvH$RHB8IG?0qsEhga8&;QAM~wRXr?f29IV&3~-T8sTSz5dKHDQTG`+- z;IYH{bJurRq@KRY-zR3!ZOWX)i%8oG^*OUmDL+FoRz;qZTf^B7ny#v0QEyD>qP=0} zS5J)!_0GEedS(c#IROu2Q~dKTWF^UJQCUwsCYDa+14c&HX5ln=yQM_RtBR)fz9=z^Su@0?DQ#1sps z!n>R5Y$Zr#A#?Ihfhkz@{x8}*xsoXH=p34g)KWbw^G}CNhkEi^waeFsvs%Se)?H>D zny@Lg@!2&7il)-Gi`87#SEG7^=(5)d$nl**jVFf-h4>etx~ zssIRKP8VA3$dXg_7ZsOZjrH2SqJyBpL-4G{3~D*BQNJe39(-JppU3tLdCK)=96)^k z<>ST#76P`@chzJJRJ%RT)MR;{P4E0JeKr?%AjZTRvgb3V+oAqOeqU@XBnMX~90G?DNs7yCI zYv*otGk-9*aMoTm{B|{4ENqtC>sOAKV}J2#n#*!~eYZ4~g2|86qL@8K#3Rit%4MB{iLakdmmL3H_&G1@BnHDuT804QOwA z`}*>LR^2>a)|OPu3GBB%N|{GD`!#!W8J%5HNC%YInH|SFWK;;Aj@E~4xmx;MN)FFA zx_a_I@eEK@RVbwyzSamxaJl&#qM#xTA$oe-7*`(lhK#$VUCNq3cMD6IreL*0@ZNH- zRL@7peHHE5Z0PEH!{`rgbVxVcduK2Ey4lb93k@8NG9u@*^*%a*%G+7xp`&h7SQYvZ zDUxpq6sqy2XfKVGNyS;>K|>%VCK=s%^YT>a&202gX8ny zlQ+PbJ>1mOJ8wM@2Ej>Om+ft0vS;84JEM6(t4P%Ks67zjj?9Zv5*AqH{7SNZ=K41P zF||^y{F`)p6{A~L(HTqg+rdK2*G^F<-Cfpg;hiWaJi}idbH2oeLpqE~9Rl%uM+*-_?2=iy&1jGe`3vH@uI}WMk=U40JXW=T3{ADcFR7!V0cp zr$CRBgZ=HvEMm$@9Z67lB?ao?2CX&RLgJ@l1Z*p+kod9F%{_Z3_1!44 z>7>sUn!GD2I&AxjOt~OUI^@LnLlknp14BfI7SeHrQhMnvAaUhpG#dN3oP6R8M4FT{ zTi(p?10kl+b2pZ%U_1ThG`z8~h`l(#H*lsK+Q(Hzl=AYo*oy>V^aD)K9~G%Xyp#nU zr19XGG%jN~_f)29e#ye8mZ3!lio?I*{?sx|g*aOky*o20Ac2I53TMSbFD0ZVmA0-m zFQp2X=0=&Ac6&$}~es?n60&We20JFY9v^8T|9k2)k0rI$MG^D$8K6e3b zn2Hm#2hwZG;(-Xp=G8fs6#0X@Tz8G*+i?tJGR-xrc4CVqLc>&3Mi@T) zd7nZw$i?Df#)3RRvb+$4FO{-b{ekjisk*+H-5?*@;DewPWjd}b;QHP7CEt;1SOf za2T&B+{Nxbj@9QfE{$8-x;Ez#x9a7b-U_ti2dVfbT-m+iM9I1mOMZCg1X345{7Kbt z>xb&pT%?m)aYgU{LkeiH7F2im;9*Z|+DNLg-0#ahx=EF~(}h}r%~6@Nxu%6cQP%=HY!U*I*-H5m zP1(d7uU~;ify5X|*Mf|6>z<)MM28ccXl%SM1(=b`lc&oQ8DT|q+GWX?QnV>eoZONs z-8j1)y#hzE;qyHBziXBR1DSAQ-mHn*{`mM*l#}3+HU;xQ7 zUXSQ?@ng{>=fbNaZiSd#P{dV^ghoYUE2lAbRDTgj)1CeeV1MT=saVpW&+y?A*>?HE zA;T-vIzh}xYKw~lw)X|tu~R+04%WqZRA5;3vXsf^RP)pp3ecFZjB5ME#rkQ=SBjv> za#=QShj$qIpJb9X3Z zZ-n)53#=Sekq*#PMs2gKjow_Pr(ue38MEdXoWnXW1X3`>Q9H!LtCfrq6-BD_%cU9R z)!J%KP>l}^^^}v zCLsc}@O74O9hHw#Zrra}7&Jt~*V#>~B_vG}H_*Zx5Kzs)_$C$LU?H5 zhab2x4>h=mOF}5CP;oF!I>`oD7^If%BcMda3vDDeNn8j+4LAAw`7NJ<2=>M#L;KY` zV&Q@N2e}z7Jy?C33tJS%#pFB8%{ha3U|G~G5uVYKFH5hDe3D`9VWiVU`1fyzOlNN3 zsGVP9Sz>nYxDDGkTXhH|MC*!i$?=2|_~+7h!#YTKrPvru79b2bRbdw>tZklFcDxRc9d%&OYs$c3>y66=sJU*F?oDf1L3XRea%EtcZ2 z3N|q-Kl;(`8|r}aXz%j`w9%)+2H;vM=t)T&)(PbbiuQGZOkzg+Tb znn`go&;68#pOdbgLAfMYdLO-ev|TFjuDS|X_zIcWS)8-{>fH7~2~hFVJuX{#Hm96$ z-Xe$RvTSIz**2T$N+-3{f3?3={GJm)k52`E$ov+LjbN5#Td^zjOO5UNN!GALGT7!w zpONp$FFvrI6Sjqo8tVtUrSZ&+UMY09JG#roESL9%R$9Pz{&B5P`}@JPjpU~TD%rfp z9yTUbCQXJKl6W?|`=g4Adtor^i>@w6Tj+HO=+_NfOs6Cz-Kuy8JQfG4r)-Iwi5D-Z z$eJlC%oG)FRGB~@W4n%~wm3BCDAR+rDqmNRa|g1(t8gxMmv7;4@k}zIdU`J}6nZEMuH=Z@OgGMJIrcTuhD2jg8=kwZW`8@;y*v^6n7I27%;HR*#dvF4)I{iE3U0#| z$9YGI++EnRj#M)wjjEDZL388g+BH+IovZ%>ieo9b;SS_&CmH+Vf;ih#RTg9o_0DBw3?Ot$Xm0X1uh$`Qaa!*)@w z4V-ypXqurvxVRjzjk0}#PeI|7!s>Vl9K&*DHJ6ZZwI`#nu;T0c`Yj1{rP}3o;L&m; zC%SJ&3kWb_7LO|=_1!9)H&?UGejqV4=S$h0Iwxp1jAhd?Ziu`(tO?6Btj2KS#?sL2 zzkAgblu5FK$~-{1{OJyk6Fs_qFKqeKURY+Bi~+k-mw?gsYg1A@r=I@PZM6PHisDv$ z6R=a$D^3KwA5E{C(E5o6akZhn_;S68WRQi{Q6^4E6=6=W{upjRKVI#gu$`})_n$Bwv&8mm?u4a16PHOdGKyBbs}N`$(WWxxIU zu04~U$wZ}=I{%vK_$M{1jta}%-`ro~blB7MFRjM%tpouoW{f*S)uTrGoxV9)Xrh zAYq|XVgRznxUNtM>lDcE^VyHtbhhS85}dN4fEDShF=kbipM=#er|5g3P_=e_ne8btUWO7a3y_i`X=QmC2b*J_aHeQozqsWqvT9khTgc(^pee}8 zl)3qx_vQZbqBKsQGEL`IJQdE;mqi6Gi;p{ni@;gn#{-+wHA!%l%}2K-9aenIs!9mT zrZB(AK03J-NFM%8o^MH7m6lAEVZIj;c5(r-srr-w2o;p>%Loh9I0WF(Od>Ak>fqJ{ zVx9YE6eH^Y%u?&pRvgG|RDo=O)I{E+$ng}vpTD0|oV=eYt$aa;QJ_<=`uhI6CVKWu z?ZQ=tgnWa=0}KswZgLO9YnIicRL@l33jN5%Y-Z{MSq?aBP zZaWZSA393gxeI@2$mJ-X1)5wyeqn$C-&7qncdgS07}OD z!&9Bfjj_!(@BX+J$S9+b4f4nS*Rs`-+5{f-;SBI9)A1Uz2#DrG1ON0pUFL&Dmg0ZhXHb@wVSNV1P|GKDi-d++ZRX1j32j4wm1h zLBA8o;3VY|$F}0{m392cB}hznvv$Y`4>wuPatX)2E6RUY#-HEkvT2YbXl7T2g4q?N z=4-IaXuW7kJuq~iSd2q75oGLhbtklA{H?C-7pv^oUi02NH(Aq5R=VS)XkQQY$u&&6 zDqR#Jbua(gg*A^vHhn!dT1`|5RNxLW&D*gCOo6wy(|w+JxL(lg(5ekObto!#pZ*3YAKfh!R4;E=w6&WmcN zF7Fe{TfPGoY>uy_d>}v_n+NR09Px4#q25npCElI4ob@HS30A_j@9JZ$nczovNZH(# zUXgpqX}cP(MF(jGxyv+5tjLCZS#3HK;UgYb6sPvSt*QW zSioDapHMmN3eBq7LlOny=4tS>FfuW6fkUmd>)3=F@yy88W0DTY08g;GHMnN=00^V9 z=-Hd^#Ar=O5E{~CFCem&&N_78SQA@Wy+id)po&whVE=5JE)2+1it6d8ZFaENP$5Ru zBVTt+S~pOF^>e*;j}UKOyT(A&)e@~`Lbb^`pa&MckX-(mIO#)SNE{Qy%EPOZ0Lz&f z=2g3lt&ni*nfOmu(8sygeajn*<=k)l{jjEB3g(29*lb*kc2||5Ul)!GA^4^2MbZj0tLQDy| z&Y-lI5yHZ~O%?WRiy73G`s6h(WJ0GK-2-8;ZY|8X2Ffa+Zv%`HoiPUL}wm9N|~CS ztfXvf+BJ+Hhv++I7`wcP!Ip#(uTJp7{=LhYi~ ziaaxOUJI{iSxV$W_ev9?x}CJRYGqq?vI6v6s&>Hxd1UFz(>PYdXZ7I{Kd=}7JtpEe z*)9*1yu=g3u_7yw3M^^>cvhebb^}uUy7|sm-uh+vqItV^wjR%vcucAHy#pJtVbV|_ zHH56AXkFSC)H|;$NePsn29@+0Xl9O=x+05-S1B=KDA=dPH3dy2xeKTuVoTGRXntcl zUwi7;bPJ__f5x+9)@IAr>$IVSL6UT4WrKzz#kSMfo{ZG6gIb8YEnL=<;L1eCE~Y!u zp8CD5L;4G~3P2c4K<3JV4Uwq8wCoK|T4S%YH_}6-qQ`G8e1(Sy{#d7^Q-^|GimH() zGNG-gQ)Wcv+T;5`vfi?jfC^nNN?^87AXp1~@a@m}2tk<6)}lX!uE~nnFc1wMBPexR z?YXi$(=zp^B{IO!A*N>0y>=@xX>+;{OHISXNWcXWk3kAzW_{W>+7u@4Hi+-4vRh+x zKb3w{V0yrd11(l6AYb{{vCs98X2tUV7^T0$wvYS`a0yiQxi@J}FgL|(hNW;=c>j}Y z;hs4oAuxoq)$Bby1e39OF6MacM89ECHRIx*;I-JmwGB5P8|^!pHCvP=*x-_>S0L^V zpA4_mbzp~%qD8Pj#?=|bqJ|+fCVkdQKI>yAlRtGxmlp zA?;19qF2>z)1B92hsJltpE$cDp*p+#2uj0PLzeG7z7 zz3$qQ3zEpHs_QWHj?ADd{_|L>o#MtQPEMM^5&VmRO{Pj*^e)-W-+*_u_AHoq+oEn! z3zs{M_4`^VT3ZRW5B`eAv))3-H5T@km=Idu*_ntO`c+1%o!I*-3ab~J@+_qF)#7GI zS-_i<^}a2aPHC^+ zL(khR_My9Cv zq-~(k&$#o-%cqV0lt&nepZG_|J4TKt`iUW4Nb^r&s za~rkfY#;hF@of)A{$aXaYkiJ2PQBn&(QF3v%z_nbddDXp{kLd-4i^ivC|ch~jm-$r zi==_$1I@)H6S+den|Es82o#Rc7v`^GXn2_74L$K|n@H2YthS>wauOSUPOPWzSI*Kd zj8K~VZi`3fF|(#*ZnG@BzON<8ji2-(mWqDj$*{Yn%chlt)E5Oa>Z;C|GB2qy9r$zl|ds>$%77h%eafPnTr8Pokd&e$_knwSu(RA#?qs1F?s zJI{}M63*lvtnZxbg8>u20WIf#@3hj3B5Ta_Jlil>FQz?V1G(PIA?f<#pdd(}PRCu- zE(-r({Nf^)snU&a?Y897LtQ(3O?5NyQn;oj?J{t!qevHwA~ZfYh$+7u(Day)KAiiA zX?7~dV6G`%6()WH|1;@nRVGZYi%ejvNpGa%y7uZSeD;GW%3#16@7|>ipX;kwko>nq z-Z@)N{?YV{l0&Xl$#Z>8SJwS?Vi6ETTs z$z-ke**D8k)nD4~!%z!si3s}YudYLo3Q#^)veVh=mB>McFOh$3ujyrt1Ojj3cR^9$ zWJYpgkc`rHD+$E?q~u%d-k1-&Ecj@ix^MFW&fIDD3H=A-s%4$th24;1e#SunvFm~y zdwU1k-uGghefnv=?fUJ}YcisvrP(iuwg*SP7{lc@@xK8+jWKPDj28J$i!dfuon81E z6>5?nscUmKVodNqBLmR81eCLUzuyPau`0FOiWzx#T5@K>dNm+2>#VhU5?)&TUjf-I zCejzr)G5B+^Svq1*Kd~|#R75pMfgGj}QlKCqeq?>ILVP3?+0O!3$o=q*skw!`eJ0?rbS) zCgsAKcyU4Tkpt7lg z1WVI6tw&8QOV=`}AnS`{gq{%;a!?m2q#hmZ{Jfp(NlH>dopn9C zc^_;#=Gs*_MZR%M=tGNINT!Unq|j(}sXq|s&E*9ZiBY5=7_a-$-tQ$gxa`)gm@N~c zW@$n0Ze44ue_#Iq%uyegGOe{`EiW1Z!oP3jh!h~}d0M*5%h_7-W8XNuBD(V5ETJ$ii1m^|)%Vndgry-DOiTo;!%gB0 z@{JuE$jsq5NdwPj?X5kza_m{eyitB~`W$0y8Y>rTUo=XgHHz%?zLL|LRDNW}v;3SA zf`wzCrA+?-dOMsrxMeKAWyLDz-SSC~8079Qqd??Lher_%lPwO*H!I4}5_dUT)!CK- zT>!{-D)wG5WVy^0iFx?5u_Lvn=8)@D>|hPKaNQu7rtjv~@r{sn(Q4ce3Uu5Xp*sXV zcjANq5_W6Uxu-H!unxVCJk1psi~w|WrHZ(sHGJ$xdwGu|qqI7=J*Lzj2qmK7@pgOL zp?C>W!dq=#73_5v;xu!%l9ZQJN-dNN^)e%KdJ+(rPRCsf!ozVzuMwKHnTai7Dmnw$ z=We$X#Y5uzMiWW~gHz4iw6SudW9wzoQ95jKx`MTp4F?_$b{C5mX4a>8OgEiEetb(^ z+>b|RXf6}9)8bsxqo#v?hr8VQ&I5Tno_DzTl60QXeGWz`1Xp1?cf$aVCo1gQ*L#c^ z>ke@rrkt?A0CqDKdiP0qMC}8$PE-?6d%X+-1*oKv1oh=^JUc1yai9dNcK-l~ljw9Z z9NfnS5fjO%cy5gZ9=-0G?g^>beg(Q3R$0d>HPNpYBt zLKTHYI|#y)I+3p3rg<8%Sll>nDDWh9I+>eky@3Wn0!ba6nmqhLQY8NCNi^Q26wi~( zm)3=nP^u2e!|>Ky&c&a2vC-Dg?_YN=j%vtQ5+u*2NuKy|H}Apxttx9exZBMYm=?+I z_qzPKmvb<5asVW(@cnsR)Y-RaVNAr$I1{!t6Wyb&b2PNXpots*07j3-(n3u2@@w;B zxt0!5E4|JW;oZlLXRJPkFAM=dAZ*zWh4}lPpyWT^)v)Jo57;|DnQg{R1B}_%GHcgX ztElGz6%*Hy@or~s(+xON(jKQbdwA~j29zW&9FEp#KuisJdeQmS zX6~pNk1%84(wxocjAvI|IS+>;(iH^?_0!P8R5o=Ld=JH41~Ke)&D)r&quKoE*A0gX zN24pZsR^f(TC+BiJ+4RRt*S1WVz^nbphP{8F&3D2T=1 z=K6stLSZ5X?VI=3ov)Xd%X3P_jGaU3Pp9KA32TvzyEHczl;-)Ok2B3X-ifI@D4JUPiAmDzlbz z2&m^S`4BlBWi21JArdEL=Mr zk20FCEO#{<~Cu!02viWODNF-00K8Hl=b8Sz1dXy8Z>D)?CMi2*( z50#k&sW3J%W;ghjtXt=SP#^$d-cD5mQcJCrCe^YOKXE<5+c%Z401M6Cl(R<+DD&_( zFW_HVWa}@NbN8!m_GOF{J42v4YP_3hP}}cn$tbU!+3fmU)0gMGQvlP@>>|axAb~ZB zB+iaz^7cnQ^897*>R2cgg09eK{>-v)>J$$0BjO8MaFJC>*(v}Jn3t(u6B0Wh_lJqX zN)D<#fP2G|Aeh&&Q{3`I)Qapi^fj$OQb%P6D?l|wpzY=T9tC3dlt&0B;30@k5i_pP zLHH@CsYoQ^WdQg-sXl(@l^tshYreBX1jeF{qSpk2^mEExVJRZuxx0U##h|RO+B2sQ z+|s2&Ngk*p&u4qA3YVuI!YVuA+KF+4V^we=W6S&PVpgfs^!MoDaV0(zRZbb?B=?tL zDZmO>>c^Jnr5!Nat$7eOfn6vD-Lq5h1@&x6wx1`-xKoOTL<;fJc^_86!)YLY7yMj~ zC;lYwWV^7wNmh1OWhY{*EOfcyQC%ua{o;EM=A$!&j(7~P08sahWS?B)%r{3P5hoQ`LdD^o3tTfgd z;xc%QixN?R_J40EBlrS^Bp;zsqij?`KUl1XxU|6A{ z*m{mvi?$UYfMU8pq=eE(QtJL9(a~eH7cPAy`$N;Ux?(YTM4wdPfM|sxoY+kaH%MmFabJspMxaipz~|=YmRAa z@29D6EtJdNN`kK@(H_83RB=$IR7Zs)3XGX2fiIz#Mtd#}V#XjUF{Uby+%T9@lR^7p z9ZbqHsZ>~6)D=Dz4-kzj04jglBTDMgh`|FxA_md4YVcU)ntJ$3Ixiky$u3o{b>g4> zuc31>(l>M1MdE@|rC~TmA&S1#%ulion;$n40XkIEy~RjO&=KXMjvm*#Zw+{Um)BY# zVNyqTZqZ({*>9UtlaCf3-G(Lx>59784#Fe-wH${>(aUC$mc&!!HOxc83TnL za>_Qd{eQDbJ-n(ig7L)h_x>?n-lFeY%aWdGt`D@8iBfq2W}E|cTis!8FQ=K*c{*ZUrYDgrX#nW# z(WM}uNenjw86@sT7hUO5E(}90TlQNt4R-W$CXQIne>M$YPb+BjU~Dj7rE0k?*sL9r`t38 zoC={hw>sGIr0*tOBwS+O(7B9aSNjX6d0rv5P?sd3R{#j1CkZo1c&N4&B1lm{;sd&) zqB~wJT2xkGnh5T8K*9=>rXfB+C9c(+NK%#Ij~5-`w;mLTsE|)94b*UwD#B-Kl0cn= z{(v+rs-*1%4lx3I+*fejSn~!(gB1gclt56LApCbWRhXbHr0-I7>>mhyYic+Wk=?GZ zic}X(+IGH$qX|f?jcM<5Gn;t6u$6+N;`22ffYq#7wqk^AAI+{w)6@DnB~;tzKcmYu za;JslMOk}5im$u%bLK)osh>8}DiN=F_q`q($6>8FYRemzA_X7`thCG(gIGik*lxP1 zC<)p{KbPpvWGhm7k0&G*=Tp4LAfJn2GK0kk{{XTo-YlXGAo-UCS&Ag=inn7OD#ARc z;(jfORDszof42l1+5l z4**Z->D;w(3Nj*mRQ76PvZI6Ur$VYql8`|D5K}&J15J-FAl{kMJ)MN;G$6pK17T5l zVo8~%lhEN$6HdPe97a8&?fwSr>m8q~iOAg>UY8zbxVHN}iQ2Mn&J{rl1CNT64r;S{ zfUH58tYI8ye9|{)?B#8$&{!g-fBrP?pzuOpVcs#EOZ zFg=}BCZanksnV@Tv5E;LB0_gTHU2HfwfY|rgFY&Q97sG`lV;c^1t4vv6zF%!Gq#R+ zfsZ4d{XS>KSurCJFrqi=L>>;-PT57cRC>|bq2f02cXo->5#K^}rzCE2r>)0jBS$Vx zlIGq)Kt{UMb>8JPG1rTUN&QCG5}2MMn;^I|B&`nUk>BS=x1~Y=nUEy#1F%l6ML^Gx zuV?7k5qYbSI%%z5f(eZPnvXwtXapNp$TUtZ@dvr#vuwzKJXoY`Fa&v1PA$-d;-25o zH585Q*wI+nN|oZmQ;WnVKeCasCw`k4GPYoNY64QIBAUwe*Jo~y>;e@aDC(UafD&;) z3Mo)Aw`}iqiIsiduZt@Z;#32qRHL=y+K*1lX5w0n)`ycV(5!R|p3n`jkThyZ2^Dy& z$Qm9;(;PH%%{z6Nyu}mR>C4>3>4*-ASvy{Gr=g^7i=wV!1*0wdyFD)K(Zk5GQd372 zD-(s)3`VX2c!(R!FnSZ0VFnUd5Fj-#UdD%IjCE`V9V8f2akv>lWsF8Oc95Zk@=D8k zO#=ePUKrOds)T6fyQxOr#_K3eEoA_J^DLuP> zR}%?RO6n&cD`~P6NdU1{O~3{$G27{8By6{rKS4`cm5v6m0RGEWyGmXwbK)~kAnpDg zbRkKh83S!&KXP*xG&=xkNYLMS$9ON%oWkZXoJC;2w+ zGw|*Z@DP$f!(ZHK=niE#DnfgI{DYB{lyP4GO(LcAgpH%OZ*0CjwPIfBRB}(7>KKns z!`F+ar*U#rm+Sc@x2X3^49AyUyBUysHe07lvPy{`kQ4=3N(S1dpjnA_;uz$Tj1Z1n2YnUyC5*TSH9NpX^cJ1X!h zLYL~`_^@PEWjE1oQbLj!N!cJ*&V{Kq7tLL&30F7lh^klVcMzndX;#%@?uR6+Y)1Y& zyItE}drDHF3a|zHYl@1K{iPpEZNy5Bkr;VuXO-Hfrn|dZNef5=#7PA8@_E~_TPuqx zPlX^r-TF3*Y24UTJ|H89Bi)p4$+2)ZOVL-|=>Cb$bc`El=I!WiED)Z`N{^KBA^OX| zbZr|Wn7podZMZxN>`0XV0NQ#-ZgN*cqvrk0ce8mM&PL`7_SB7Fk>OYSRqUR^=<>X? z5h@HmbsUaf9V0+V*|n||Aj|^-wA$woW^`0fe6$2gHy<|`BAY%gT2N>Vo<3Yzv4Ahd z=hxi#5uG)r>Pv9c5Fkf_ZC2o35jYteUqX+SjIz0AAKRO0p1wW=o{NF-sEI7NQ+6 zlC2+7)ue=iYIi%KIZ;7#h-&&-b(GozZCWmm5sn`Q)KSE;79_g|m6uxqQQX^!xd>96 z*_5|=d$6}_cx@uy=L?9yDxf|twjy+K-gCC);xh>bf;#!OK-yZwxskPk7y?G2$u|?~ zwp&@;9+Z=*Xhbnqf!YdIq=!=i#)@!PIt5E!z*#p)paH`W+5@f$ZHZ$+MT3Zs3=cM> zoF*2oPkc5a%v6@eVb}z@U?_wrN^vz24TQs8Hn9OT6&eb0c6`fkzk8iWguHpltA%qfk+!2R3u(sh7+xK zxn>kO!T!)Y%e1RP$}y&KQo8G{8va0q%P2?$kt4ENFD@<5kcu-JiT&ND`9rIiB7z90 zr+cXOjb_nJBMu%X((mIsX*(Sfe=vFZH{zO8((XpvnxfHB5w@Ox5l20PTI>2xz1Eoc zQaug9aRFVLxT67yI}dwR=?YJ^6~seqr&C`nUGU<(O2T3`5m=Aa$P0x}keU7#$R!~9 z)7`7Gpao-FDiO%bWThB(>I@CNtOAH3KnakLuJs29@U1)Ge65$j#zIAC2#+G+&8$T? z_qszEgpbKP%DbdF+fpEnvai#IuB4C?G75AJ$qDCm{jypMFJiy}JK{ana0*H%PZfNg z*vsC`=55-vgLKJ&CrCBn#OVFjL+-ydhURq2ghixiI4xUvgz}e5ZOcGB-cuu*&fLLa z1wcS5G(vpOqgU@bH}2+NDtXJ~U%4{`ep-Bi_Pco>ps<8m0T$8_k)l!{2XbFrHWYQt z3FzF0q}8=LzLJIBv1c)C+q{HaNz^C>d|!d-AG#z1b0DeP%-kS+t3L&N&*<%`dCC!% zs7(ClcHPS>HtyS9wF5X*2qTnB!%QmF6Uk6FzrW`?J1No#!>K>q-5?*7eqJ}K& z;6ODLoq@v^5TdUXcvG~E+aZBKD4nJb;mE)?F!0+~n^~^3?`1}ffEsG!j^C`4+b+G# z!L{unTjW*KUEHKZV#mOmBv;_PZGpA?#wD$a4Np(`9fg7Ak$$1u8D-As728m;I0`x31Qk5R!!_ML5Fds_$UNr8j`ph=Hq+LQc@qqb zE16z=U3RyH2*bO*)~c0A>Kg$jC~4a1k9FLcU=gX?v9&Nb_WWqjlRP7Xc$wOJ1CtS> zua{6Yi7-Ii;~?;kw4;s1837~0em!o$1nOOsqh-5#Ih6#|kRLGSNcKlBxztkB zvr@wFo}dtPGT4{Iwz7~2eH&NOqgoaNiFM3SJeG^9DS{Yaar#rBa$P2s6{niJ6uoW& zzqTS2JjCuH{FkJ<12@s0H7B05Kh?G*0lwhX9kiXV?ufjVP|?LvQAd>aqb|l#`N2xA zkO7g|8n~5`pp=v1GrLaL^l!Z#50_-F4Vxd>xD9m`as|)g%A`p@ZVsN5S$9 z`-DYc(lKw|?L)k%#>*#)5xR0pY@#kT-E8xnAAU=FuzCs~)bE+skv?se6C#W`Fpg1U0K zu3&7qDo(QcD7F$HJW)y53qv{SwjG4q#iMvgJDsGZDbc4QMW)zDYzYxi@k)mtaBgx2 zsVExH_h}5Z0~YTF%*n7W?dfoiy{GOssCX=YK(QA&;F_ADUX zvXld~n!aQeTT+g(1N%o;=rYKR0BjzHPb772DuV7TT2D7-Hr3L)fc@Vi0!3x2F=7NDpiv6#;U9iX9xn zLRLq$5RM~Y`<9cU*=rteF#kV~=7Z4Y3E~@p-aNWz=E1X%oCh#mNOiM=rjG?8+ddn(7^K5Coap^jZwKl}_3Q z(QTIdQe*{O1LmKL^b5F=w0d99e(K-2Iwc7d9Och5;(nzj zOYVxOrPyi1W!2s%K51M(tMjOQZ0sP{04LHZInuKmW{)) zcRVQb3HFHd!2DisrSsY8=dY^?{{T%s6n3O*z^YFc5af@%Wpi14!c;9@w{)V42~t#3 z<^#Rh{{S^vzjrNgT(Y%WT1`(>%TqSmtP_&Sw$RN;)?v0Q2aFL_YeoTaAv>Eehgm2Bv~7~W7*dy6UL!{`;#vVk zJKe-xwvnVd(Ab8 z+mj>z0NTiZ{-;!6G3-cWH-FfTZ;;;M9)P3q& z?eyjFg%k`TLR6&aOnWC|r5Ha4m!4z8O$Hh#$@A!+K_@8^C)TJW+C8<${F)2!q1>abgF^=4xSo`RBYd=3i1YQ3Cr_KKW%Vs{w zpr-aI{5Eg?TK@pqzAA5Bf%&&|`bF;l0CUrF?2pd9{v!VXyxLo9rRY2-V50e0BQ2UE zf8nzmzw(;yTg)ht_-x!pzlOi|ZqH1uXt`h|N=PE?;ZaVD{t`b&=lR>m{aexh0GPUe z+V{OB7)8^?=u}pk9dl~DgVxfr`dxAO2S|fA~(fL@cQnu_x%wD~UZ8t6-^=^;-#s2_#s@wkn?dkAN1a?(za4=ZSjpOjSpR#r` zrscEav73+nS2BJUnmS!&7c7LWNd#Sxl_@g|r?N)(#0$7T)xG{D&+j^LgmP|O)$kZk zMv|#Gtmgj!<#Rt|)|9C#2b#_(PM?L${CSbw^P>qO@luqdNW`Tp_FdBvJD;CT+n@q& z-jW2KaeKeq8o>%r@~8Uv3@GmA1;q~qjNzy;_-xHNZ90PqQj-cvN}z?MNFcyCN>%~T zg6JEUc3ivuCC~3W*FXEL>=^v@cK8z0QNr48*X#H~!$`82Ct%%E$+fvLhiI&ghq=@f zhf&=lubZ=281(Df@ni_iTgPfaB}v;GHrK9?fJ6=lSL;Bh|-2a6P3xDTBv522dr{*TEL6?9!+sc*!o;z7bYOZdIK&HHu@ zfpuxLR7pTEl1Ko|Mk;S<8ouSC8&QswXgSGh^m#*JYUDdAQ zj-^KtNXek zE{prql+IzdnEwF7UEWXSDDYACe2**m>lkR6`dW+%hw#3P`@x;)*g#i3Xn&-$LH@;R zpN9waevRp@mr01Uyf-3ErO|ZV>kYCTn&dZ5XdrpV2!aY{!}^aT^-*$i zboO`0>g16!HIGK=VU7Z8PvpUnF}KnBh9|>aPKJ*{Qlds713|aFsW*IbE5J{sp%y}@ zuGgwmj`t38Zzl%>}inTpfS|n=u%hi`yKI3YAosCl>%ht|S*>>tI%L zg66k?v<86isfR}fz?;~5md1oH3X8alKvdNEvLVW7;jkEBY<1+9BJhTQ1K41CT@1n; zVadft8xY%gBuT5b6o}Lw*UQ@axkM)pD67swIvo*n7GpUk2lVW~8~*^#%jo<9eT4Z3 z^h~AWmcManN)%33;2~+mk`4U6qeJ9q?2O&1kLmpx{{Sav`>&xaP+ojZ6Qky}pEv4$ z_MiU%?q8o*()}VccI1EHdNXJKj>hr)FT-^AOW*#YtE#n@xVy+olvi&RLY65eleOy@ zYOlb%w!6+(Il=lai6uh#r|mWW0A<(Y{{XGt{{W`Re}F%{*p8mfTg8UfRWq&|SZ0J* z3WVxiLvEUOQzU^w zz1jJX%YR(G{{T&r{{R4c+f~!qn~{#rQF_KJSfWgwRCRK;+KbOpM~%h$4vF?N9B}^t z!FBmR>vzBD@?YQ&dvNLe(mk75RL;0GEITC*Jj*-MvP7&WaSj{h#1dV$3KIb=Wck{9iFT^T7L_aP4Zuk8@OZ)-veN2vFd6Zwi zxC&j;RTLh=cgu-G3?-i-OGBnHDu#o~T-7*|brD_1z3M7N?+(r61VIG%d!IRx1g$to zfG3F+^L&nyK!mTcL*Qx1`XNM}k1iZ`?gOEb<*tm2sSYlg<&&hjVuqWZ6+qnLAEvY^ zk1Kg4t;jYMYny|ZV9gr4j+3~Z0|{ohxjbOek3lIQfi1f`Nh$oBZruoW5rMQGD%-ing@qQb_DO z#{$(Bq+Upv+R?$kDPBTF9A4)Ewy?0kXa;B zJi3tdt7vOViMfYnz0IjLJSVLh0=3#yMlG)_l%~-q+6JCfc1Mp4y0<(EOaO5TgQsJz qzy$>Zwz^LjX4xFs-W473@Ms7I%H~cPis?so(nJ%!UXhL=pa0qHg=u^M literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000004_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000004_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..773ad4a1a8fbed48d66acb62a19e1375f6c31b90 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfm|ogUE5 zgYjWB%x)MTMh|y!bpOHlFd97^M#W(v0HH^E_|Vq>Q-C#S4{L6IX>lsnYM2RF4NG-8 pWiT-?oIP{K7By-{dALFV7*=~ha-%f%aKIIAAaPKVEKLLwdH|hDJ(mCg literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000005.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000005.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1b89d901c1a596834f7ce45df7feb54d87bf55e8 GIT binary patch literal 35341 zcmce-Wmp_rw>8>01PK=0HE3{mcXvr}Xx!bM;O_1k+}+(8*Wm6D94>qB^UC+@+#lb) z)z9jxr>c6+xyG1ttg2OA?@RB$0mxEfl41ZbZ~y@8;{$kq1ALazcQ#~Wpl7CMWMBZi zuL6Vtu%AA`e}YH+dqMt;{GSCC74^%PFCX8?$f&63=wH5IVq$*m_*h|MV-pbI;gOP( z(ar_%BOl>FKte$y1Hd3YHlSgkp}?SE zAi!b30N^MP#Hfsdijewtez|CuB(XI;aY{@xP*+0A7+WA{ds1`*|M*!I<}Z$9b+y6? zc}60=4lMnm`P<}y*C+sRFfd3MXlRJP{QVUL7z#KlBQb=&;HBaWBwCGKuAfru7D*2j zy0Xv}Qyi(m7yI{a017OaC%7*J7|Dlo00au)eFcC3@pmLB0Diy?g|Y?G*DGXt$FMOI zw^IBn>W_4HPwz*a&id^xJBjE#-8yl-*~0oZ=JnN~+YVRMKIt>w%R+ z8P18k_a35r3&oumW`&!T)^msRhD#700;jSVs7qZkcjWwAR-?gp?P{f}xvv4*K~<>B zjvGjJA(f`7*<#;?CoCX~SU2&DRCiUyJVvaf?D&-Vh6q>Y?$^)IpQW;%JMuclyw_a3 z?p?g7D{OjgTcIhi>e3veI@E(`TjZ=9%!^7LTeT9s?uEM}PLtynm@)=Y`n!P2n=bXr zXVf0&e}3K}E;{A6&RAg0#PlDwMa?$91AbZ8mn;#h`g6DymH$K*oCcu$KTa?MWERSMZOq8mmtM53dsA&2$Ym!q*OpW?yMpr9a@yWBo#aJrAj@n~uDu`Vv~5 z-)`AyXGiC<>)zQ*Ejc#ejpt8MV9lv3YQ58VqV04k+kV>iYgaa3?vvHs-tMcRs?6y7 zr;Q$5GjaiT>;Kgqni`uK7cQ~3@_7#Jc4ykO`Xhl}{Lti3hn(d+3UvM_>zy{nc^bY3 z=Pp+kDjcXT1UcGF$`>KPe%s$PX8(dSl>$qO$o^=<3B#VAxu|Ce1Q|@mVi~pwJ%V9(Ck@_*KnM&S4 zW7P5X!UF?)FK7$P(tv2hzopUUhYX>7xpWP0jYBmiOH1E&uXKR#r9ar{uB!|YlE>ft zQ2qHkbse+&F7jM*MOQnxqP$7IZ-v7P!}U&klE|4F2(1$?U!Z51{k%L;ChCjw@1FQQ zSOWMcl7DkdQ}z^Tt|E5K{*#5qhSnidwZhCoYnBInL+cwf94@2(MV z;gCi<9@Ni?Q+~UE1d|5|GzupqS}OmJ8!*Emz&_i zyR_B6q7i-t_RY2zD$eKio2}`6BBEAesY0<>z6T}9A(-j6#sNUffVmz%2TA?E%{YrY z9}!3@;R-J+Khb|7}CrW#4-4q$Rz%JSb?wu0%^ZV^$=BNI-J<|JNe_|3YFuBwny| z2f7bKZ_sl~lLf2}+pHCF^F$S>+>56ptJcf`>e`u^*MH;ELnIoO)IpQTV|7`tZH>UEsL$;530-N zcIJ+I&ap~Klgl8y&Z+2RKo%jqfx9|Z>#u(znU`Oczg~EJXzOZF{xfIn)mV|R@Le@w z<1yPkMSXlGWxwpe3es!wB(Y=`eN#rYm$(=O7{!OiD@p3HaeUF>ArghN)^@|IwJ80or*xmqi_ePxeY*Iz17YQJ74gE@9bJ2rChMHH)+Rqj$OjbvbfTWO z_-%zBE#=jwFVv~FL${bd?U?^Bl!RA&{$=Jb=n2Edy_>fBxt70Ifs$pN2>*$l{}1RN zpTKuOV|4#MzxL%ezU41<;8;;>^j~OA|H*y^zWv+f-_bnSAvLO(*YAOJlP+E;J1@TV zQlD=_j(7c;TYLvVyheUZ=M z`tfY`+;y4t4ruR!EpH1&bK(Cp->J>144i(QcCz2RKI52M%#Hs2qvRdXZyoUZLHVa& zm;CcoVPDv_GBO_D0ezaZtwp2*cGUCi8J~E0_Zz64fQ(DpD%I@(9u0)XU5S9|py`j$ z1p|~_AkB0>0MmZ7@qCfp`~ar13yuG-3yJKm_Ep-382=rhv`O$3<)q1qFFVDdz0J}I zak;_<3!^HPt*}uB1EUgi+|d?{u~8sK!)}slEA5kXKh?kf`hoo4Te)Zae`X$1-vO0> zLGz>3YL~q3xb2QBk$}dvjDvG_fwQo;yT=luA5boSoPLL3oH%udAGwsaDBW>^CPP3> z{id_YkLL2H%zMQrnf))zAK4MzKlLw8^S;PlxRwmo+`*>}apfB!2%aQ<`9v2;^W1irk_1qym!+YyE=RCP0;>O7(JGLuxz{E0&P#Sb{^W`lU(ES zIz;sO|4VmJ*mdt+*T&i~w?6ysCpApCF#O&+5%HO8YK<;CP$Qc+HF>nqt6PWo9Oy$! z?(@HeAnr5gQWjNlubQl8R;=DT6P5?Su&SN?q*Z>`{W^cEH9GsG8~eHJhitymw03Cx zn;b*=|AvsvQ`W<1W={F`>8jKj5NK$R1hm9j5xG)2d^L3~VlU>LbTaRav64q%uA^A8 z?MF5sLI3r?g(*>bM|YiH)-oT)nzqo@yIOAq6VgYx1Dohw-nm$GtZHFkco`DG(&LOq zvejI)JYLBEzgZ0E@{(Ph`g58`D4AX-P)y(dBE%!&OHU-$Po1s4S;iaXWC3z!>(iKt z-mpasA+S$)2hiJOhy5+?|1mDUZTj{v*-G&U-#FkH%aG1+&A8TR%^vgki~YaEjsHKJ zgkjlFn<2<2#!pEb;)QSu*=%-*ZKxia&Rf35eXWcXI+L54o1DkE5u}&)xx)7w>@RHV;*PApgkoJ7DL0V_tqmaeCsm3yIu` zYQqk$YV+Fr9q{nn^ij9*FWFshd}kj+Re_^}7jK4esn|`Ztn)h{f&CpYxZd&MpziOp zSN}SD-jUhsTVH5v!na*Jst+^4A7)m6eH=*gmf!5cb^c4Ibn!N&?}d(M+P9T;yLMln zubO^*2uw-z8`h@AA*+iQ-<|xbto`HE1HU7_WtS2s*`n&Bh1ZJ|!Cx@j>~;KuVS~?K zqtbbEi1xR+C`H!!H|_t7Z5xjde~)cH{!M7?;T`sJb0GN^;ktTx2) z*_Ae+I48d~h~2E^sqy>QHWD15=xXkB@r0lQJi?bkRtgywMHY}2EU+jI@X;VyK(c_p z8nrd+(F}dRAv&NHdfuYn|I_yT%00`NBz!z`a+6AtqjI{j;1V}{KL6#SqbVSa z3OeiM&xCrXfcJF@@tn0r_hqYZUP3JbtrwkN!wFVc z*aN8f>abG}b$&6GU5Xv2wAX1YghRPXT5n{?R`85wjaE9C2;9-GfBCLe3N0(*eLc9h zt?;8=#rZo*4{Op=rkSQJ`T0C7hg@g&JT+BWI%!cU4zN-|^JihVi3(2JSU|Wy5ZI(c z9gT6^1_gbt;_4V~au}tGkJ*ustc{#j!Nfd8QJRGkf-DON`(~L^e)_FRn$HQ>+Bax>0Gm`{oJ-?A6}Hdru{I8+b;#qO#OxlEdtzvMdXM zX=N2@Ley!%BE}Co!TzfutEqV~u`TCiOps58XS11Ghi!W34NX7ItP02J$Y0W>2rkTRSOrwN@b zs}P$SMOq3hEdn;2N&FR?5(zI?QAWvl(JUPYMvSgjx&tp9)%^FY>^A+8&x4InHmyLJb}I4%J=XLb`UO-6R26t?uk znv=aT$)dX+FX}u!Nm-Uf$r#W_x<4KoknDDM_QJNqOK@eDW0PN3$eHh=+EyIH@ajgpOLIbMAuoFrj`8kPVeU zwiayJ76_tE=3K2G_Ij!NNIEuDz7jx~`AxDh>k4L+7(eT`1n>h07^G)1_=m z_U<;9n{$VsXxFk1K1I^g(nmnG^vMi|rx201Zt6ZcVX7FG;W>_D)T6Ou0trwdH?G?i zI9CnimQX*Vtl5dA$5#v?aa!&01s#K?_Gg3&sZBDits@woQdHy8<2RaQZ(KHk&I$No zIZ5frf=hytWjfyg`R$$Y0Y@$4&OueZRs+=?P}u{P#?$~sCv|kq5O>fh11n=TrHesvzdZ{foo=R$9I6# z8xc3?==^T^GxMvCETWxRDxDu1ij~^3kL~by)u@%Za^-XC?XkF7jobe7i)6df29077 zRg2#)Qvo&rcQR_emCB>(N|bWCbQLiAo4XFtB9cAja$*G4(sV-U;L*Lzd`_&f`W`7$ z+&1mz*F>VE&Fe-=IFErH{pcB&MY`^U*rreD@rB9+mki5iGk9~>UDR&bP!t%dagk-w zA9j0j)*yG{ZG7s~qvHa<3d>H5VHf$Ti}Xryo{ssxkgx{0=zU~MqwmYaHtzt9Uw1Hk z@fo5#t}DGzb;x)CGM$4y^6=*(r6N^Ut`2_9HLVfLKgj#70z|QfkXSsb$zkWK`&TWh ztY16vk&nQZ{92$*nP9%0t~Mxs58_Jk8|>b}6Ze+>m3Tn4uiQJPUOH#0isU;|sc{co z-@FT%6|^A$(rJ+2w(5z5&bCm==*F^rR-{Y_H>qs#K|XI=SC(n2zQ?<-Hy9lf*vW~H z+tt~mKd!B#%4}Fu&aD)eOA~sUnuh_gdkRF&=F%0xRX1!n2d}J3=-?A>%f2S2JT-8u zu`x2RMt;UeoJyd*X&KK*JDuCepub=#rkg9cOIYM?esyu!#Ifs0Zwo|VCD_EoNS8XS zu!d!7Et$w?m1U()+Ay(BYC!xL?&AJS4#07MMZP#@+yiRox90Qg=t~DF1M00GFpFd8sv(F#1z>A>`u$0s?*61p4komg5W=Q zQvUJEa0KGGH*l;-#L7C8cV92>7%$dJAF~$LL2z@rrGlC;vus9C!^GTeN8p`1sB=iM zjWrN$7!!^auT;DGvQLQ2ev zm2qsct8-T14xs8t3!LGSMRa#s43>BayXCjypl1PVCh48A;)!4fGPFff$+&y%VaVmr zqtK1-ca{jFykO}=l=7+k?^`O(Ci=G^QrW)qw!H&Tyu_UfmmAJphj9YX(uaq$unGX- zcRwoIj#=2b4Obbc3CHnxx{|xsQAl%Wsd*y=cj>`iG3RGUN>XzAxW){E2!L%DVEcS4m0l0n%~W zVekstjNLdM6o-s=MMWN_U01lEE{onVVQeQ*oljUz%;>5{VDf?qdf)DtEL*WIIzxJv zT?w>QjQjNnROSGehRfu}rr3G`J&J@aNPMi@1Q} z#xLi0rdx|6Rarp{wRzbtO{EU6&33S}U%gozC3_4U9u8&RBdO{eWzNW3kY!=52BQxP zrap*PZN~o1hc5axd?=VA6JbhnsffN=_(ehIPq6hR>);#Vm>$~+&euEspB69=W}sf? zIGZu!yVj6<0!{wg&3G#N$6xCDhcRh4>hA!wi=Zrw=|P!L!(qhbE1fHa>=)5H>L{oN zZTT0C!NJZ;J3eFS=j3mtYA?<8N_%xI9$962a1_vM3HUq5xLGIf0Jb(9ujOP9x@4zs z#O0kA+#~pv>YcDeagwpITUkQrXjiBn_cuC;uQTreysk(7ylpFas?PjH-Me={k&H9w zrNZaAGfeT=2j!KwvZm~;62bplzuyq+zF@&afF*_N%RDR+2Ip&e>gyEm6J~P%x_zq0 zi#Lpo@7#_}?GFA05dXa%Au5Up}r&Di5(ji~@Di~PPP;s&iR&b4qAP$+R?6&d!eCDpK zXtm6st~*=hO@(*S#dAGh0_R+d7LuAdCmC}s2X;^{4?Mn+ zzGc9|U>L0)Y?{a6%LPfP3Xesy*Q{C-fH=igr7lJeZfaMQfj%edb!VJ)obJ>EU)%o- zgi#)CD(BxZD$b?t|F$LINa4gqTO)VX(x`_|YiP2>{;WW4eNjj>emd_-4^v~*VomWmoO9Nu z@|VRhX6+DT9w35b`*Aqa%N-nD&m1v-al;+-ISiufWPpPNzT-%Ka$b)Fw6 zC~hmTCtP#@J5D)qoqAA69JC5cQ__zKP8ZZjhE11mQgBYil6)QbRjIq>D>c0oOurrH zWbPo-5-19imR!#{iVj$`d5VKetVp}a?Iid~k#?%alYrQo-&R-xeE41^TzyG&lgxNy zkvgThd6XdqRx-q@YvqtE8q%Ns&_-~%1Yy_fNjeg$OOZm#t~b5|3_`!8&~FchueK<{ zhq8U$4$qQ9s}fIBwB9GeH~4MEv-dsU+{`7+_zoM(xAl}igs-s8I*|tFf@S4$n)U!L zW7v#x7(LL8^^aZkZ4Qp(5dv8jH1mm;u}OQ`yyHgb_*kZfOIE|(w@Sq)D>0AVr9zC~ZY5x0^p{(X5+snOz)UZ`@wic8{z_J$`*R<>hpo#lpx4o!zp~U2aAepd;?Sf?gjDTZ-v*f?}&rN6WA} za5`86v6rQAeJ=mFslSsO8Aw57)90h=H#wm2MXZEF7V*VMHH+H%j4KHJXRC*+yqvAn z8ste7_;2I)x4b z){X?n65#+Z=vG(m;6+v4@yVhD19QM)ZYD9}b-RslooU@lgiclVP0r`5cU|by8l0&*W#SyLrr@I!dNn8u|2lq^=VziY>^UnxZTXr(eBH{M)Hk(o( zj0@Ze#5IL)>t*#5>I@HfdhqZWWFrBIb(%Bxcz9Rw>Iu<$#uU{e4%_+a`4Vs$G=5K? zuZXM&&+b>Eon#E_JZkt%#akt(H1^fMA7G}^G$l=(l&j2db}jWbV4)tVH(%S%on0p-$bpNva4`TcvzUS!}8x zV28qax>EJWoLLdR=qfH%_O&usSh#jLA9oy{WcCanoD2<9xjHx&$QNY$dr<-nJ7=RC ztp$NKg9ZSVg0~W{Y%TLKi&Tb;+s0WLCwdL1nD`?wUCb@zL=)=z4;!0UF!S$>vpVV_ zpX_eAx1~5-#tVabr)&9VQ%pbQ-c{GGp~SgEJ6lv3eC`CHS@CwQk36puQW`KuJ~>YS z&O>Jk{DDo1JBm%AfNn#mdIvbZkrm2C;csW5jS2p)R)PptY>*>})2aYC^e2R_Hrm+iDF4$C0{~i4q=c#rr-#gDk`c?Ly9~bXGrI zbFwMr=2#~SW88qjXQ)f5UKuh&d62m=bIU@Ve|-LBop+QkjyyK0!DD3R&1jSO8rznr z9wW&uep3aft6%$7Ou)9d32=qqMD!R+ROJIX=)e4AMWY*tP;mCjX{Grl=W|+hq41>X zoglrzes6!d~l-v^K3?9&w~Lr=!vPV<88D;VJP_eO0;gRj>WI zLfKKUWa%0=TeS@9tBecYA={s$qT7%%axGS}&1u(kmiup=TM`H?n9y;PE`D>_rZSz} z(&njAl$qLE$WiJ-V2! zmkXKDECgK^l16sr1@z%5&o5lSwuT7@gq^ob3Y;?|^g9;AV0yzw*Z|M#5haD_iBd+H z4+S~U(Uv>eKEF(8;hH!;HPaXb^PC~`=3c8r`@7x=x+TBFW(t08KhLQR5@%3}aJBq^AYxL4K67(G^{Ya@R3fbt&R}Ey@dy$Te5hk8 zcASZ$j)|*5Uq^g2ZbMV4<4ksXQeIlxlOYF|iLtU8GWEEe2dD=IO${lJAgsJ`9wFlSE6CyM!G(% z^moRfJ+If-{rD6x5;o7XjnPU*O?AqTr&SQ?>m~fI6)SqN%&r-JoThooKC?|p>xETO z45l*+32Ab)f=hDC*)m@O++FP~$3^7q_RBVHHQcesI9kuRB%%s6@L%E;l=J zngRiA-Q%#nG1S)cP~rek2ueWFrl3kW$#x406mZ+k4Uu&zYLpd+k+Yh_XwNgIZ~1bV zz{JL*Nr;x1inxv}7F;5>50uQBMN10~O)2AeRBfk@^&>PaK9|!H*Kypnq8^}w!vukV zckq>Siy&PnN#cc)_NJi+7piWxfk)BIm&n`y>L^kTjKf-XerfP z5;{0d=3r^0%h)%v`&L*|G?~Ln*@*>-&_=E1H+PlMkf$*`Gtj6#+{gx0uV1HgJ7!AT^}qz9cHB*VRL2jHI$tQx)B0(PdIw=uj^K7dQPAaB&1Lc5~T1*z>~3oKubzihbGZ zEYV6C!1m&z1ctCv@keX2p1sT`@C*EyOw-6V=%K+rCaf@`Bo@%Yuv;np6XlXnLZ*HO zyndpq8nS@O<37%OgW^z6SG_%Y2gq=&ZZmp+f^TDoBbtaP3F_%WjWxsx{5@E9U{{L^ z6^YqJ&X&@*=3^;~-%4Rn z%15mZNA$8{a!y_@2mIoXLrdRA*S3)4wmJ!v`h}kEmwMC#;XjWX_71SiqbwA+GLHlz zSZGsBBk7WzG+<<#FoftXPZ~Mqs1naJKR6R5OFc*`d!*anQlDuvSYq^yzqTWzG!l(J z-gzd8tO1jxy79&i*Rd32+aDUK|>W<#dKXK*88V6C{q8j20O24lEq(8I(Y!Q;FmZI)%Sz!}xZI}az& zD;Pqfw_9+3QjR0vkT^Z$%n@YdvUE)GS5PQ5XU+z zegdZwQLZ(et65oS(LLwKXp=c>X!M=`1~Rw3eZkC@7HpYaTVPtuogfG?XCg|+5OgH; z={vj)>bV=jJ}FPw!EV_iNBSm-yvp!zeF<>U{ztr_F@y3IoZdqSUDFW=bH~Ri@byL> z$1(wZhI7=lA#BOSdnuNqkcLG@;IO%u=JMAN2*9e@JK)VfTr0&R2UqlzEi)6Yz!E!XBAHJ6e_!8hqkuGrngP*@$d0(Vl|VX}?5?*;ey zffslMjT4sy@v`_I4q2b^u^)vj`%{=xCWTsi%^FeOd66B!I{T?PQzwr)|jue z?@PD&lWZ43;8>uCWSQ_Ax#Efgo<1h7XxLa6nE~o7J#oNSVkyg5vXX?fYvX0$jnY}l z>}XVbON1<4eLy0J9({!xQASg5l4b5|v^PcY5!n+eby2DEDYeoD)`8#7#cZ5QU2uoY zjY?s9;9eZ}uq{P?7#X6M)R9Ym*4&0*pdFCblVm1^-S+`GQ!-Yv? zt*I}Z-5uoDURkNs!wqee(%>1O-PCs|#DFGecH_E*f=#1v#sXP&jVESBGQDr?4iyD+ z6m^&E-`Z8bcMt8E)dc#Tc8`T3hW36N^Z%$#|NTiCgce-}9c+L4J@p4!wn%zq3Piwn zIwwntX*K=vNHyu8IvGUf2{lUEh3yt7)tM0hdQ5Qd)Fyt!Lj6J+?_e26ykbAlY|lu_`ZTg#+@zn~z%WDFz|P*+|Spxv;(V*Dq*p)R3b7ed>6wK+!%8_xvxBl5-$BB zhVk8N;qE99*;qz?lf2ls>ku*ZT7R+;z)#n&0P6*Uw9^su>oFU31Ve|Yl>gnF!hD1sDEERfydMeOUoBb-zwnX_-cc_^xD|v)=Uu(|U zZ+i@j9v&_coh;=PP{SqLTj1P3xHbnDRdk^eYKS`!>-AVkwyqEE`w zS7^e`X{kJXq@oy1LRU1I^*H>AMF%a9kM6ACOv%E7N@4?e3{&fw-Wmc~6zj~AcAj0dP4YLk{#1GPpP_-K8w%2rmR=>nXGWjt8fQ>O2f;=dJW2IQ7D-4$ zS1o+W4$3fS#Aa}9P^R7iah=2%8P|Mt%<)gu%ydm(o!utZoF%3LEn7>$pE<85XLz;2 zv>X3$hQ%LKX0DI?u-UkcXb#xqlz<0ewxslW92_N>uGPJYPlw0`^}WvyN+;><21vEzESC5SyZ&G`)@#@DJ~2~1?CwwEil-)ImI zv{yEg(w-5x`&NfEk#6uze z;VKj^<;^sRSpV6)=5}^h%s4i|_k4MLiE%CiEpBr~cZE#=JhctaBNnw{R9G+5hQb|mH#E#GuyK@F7CZ0 zNwIuIxHDmR@2`zV3_c!;7NA2qU|-OfUgr{?3(aHIn$f#|(!|KQYN|s{$%m?C&N>Ze z=IrWBqYU-n=Z~BG#gwyI7ju;C$E+e_pB7+>xmwEVj8aL$s(?aaO&*varlVTQk66FT zd4D8sNWM48M6xO*EZT>&Ki1tmdZ3DjdLV4MuwSV@xC;}RkuiQT$9ir;1Vjxi2a$0h%&-p4A3zjS6vx013d~Dqiba8 zSf3^=a8oHC4HtY_C;I`&?)24y>?(9u`c^H2<&JjYLs}4KUqQ^!Nb{yJs5mlL@lI-M zh`iL}K!@F3UI|zRII`3Ibl}-y*KsC$-%fkm z9umrplw95u{e#w(3M$lI<%%U+sgoFFlTy3vDI&4oDbX|NskcvtjhTKN6It*uOc(mB z4a6cXef52i6!g^w(TR)BtS$W*X7q&H;JBk#?W<9UYXd-Bi?+H64Nq_f(r$)U4V{~f zFJHf}w-1If#&5`6{?|Le#hDT%az$kL7Cfy?TC*+?PBpaTWq_8qNYoe>Y}) z_R*nK;&%^9R9Jt?Y|(}(e>o?0jM%Wni!{l4Q?@3$ZS_AkTtTaKM=XSrzibNx^ac zA=nA@y~V8_k5zFkZ-w?u-`rr}EH{nwoJ9fw6i6G`_pKosVoNA4&U(dUJ-m0q6*jo& zC*oRu?fMqG;70ZeqSD|T&Xm~oXP{vjr#x4gltrp|E0`bLqme5B$-X^n69-W%J;?q{ zrf8AP%=!^hj+hjvXznv!=nXBQmPbuE*&Bg ziGc|u;uY;m2`e!xSQ}%eH;#}(V|T444pk`CL_Z=E`;{=+NquAkiIc#jAs!imHz_OB z10I8J0J8(O=Uh)BRas1_3R}4WsXj^#(VSt*$t}5ggT`Qo|8BfjntPmdhUZgaeqOr< z_zrWBl!=sI8Yv_*DYW(8<^*bD&%VOyRokOfo$m4qE4xB>0#cT3C>)TqODS|E(&-my ztyDSi47M1Kb&Lk^AQ?9IeH8u$vi`nS5rAyuhpe4?_*mf~U_t^$X>;O3E?hRY5ZG=n zT)i8y{u@^?>m?R|fQel!6~nZ9nyaCK`gOMC+R1U-hIEa}q}mW)c2?Id!f(H#cg2zn zV`N-rWnuiQhD^(WvX%SXceODB^3*Helc%}kwlO^h*mM`B95xZ&o86zJQb)I zS{^oYLNyf!E9E9asE)CTD)|O1cotmEMw&81nM^SpEXXlGISPj!DF8jm1`^@!lLqSP zde`RR(-cBjK1nCl zo-JgW$YC(oat+r@5Ta+*Cd6CCXm44gqy|if5(tRmHfZ+3K!k=8_=jmF;(W;EL5ww| zXHV4E*cB3kuk|HqaYnEWkF*tAVgb@3PB;Dwz5aUz+>7OJ_!<|g#ueSsF< zxA9p6ud8#Y^%%Rpgj|f&#_tWyf@D{%x%9v&Q5N1GI`ETdUdntt^)oAT`_{M zco7+ZAAHXk61D+}N&c2n(}E}ioywjMM26iASTH31)2@Zg?b13BB-7SR0d+>Bfs0XY zQYa{Su*r;tG3TpUulQ6N=dPM7TzUY}!S533jR#^4sP~4BJmGTPdSG>NbS0RIKzOBa zZfbc%rr&UM4SDubvoy}ukjN8hd)JBqgrX6H!-!bop_UP`z|E~-pJ=c0%&sa5q|#`n z?JSH=3IF@}S-ClCdMD=4FuezRWKwdz1_q6DKvu!0!St^naAphl1lcEv^xHJN^Z9dR zEuu^o(1>8g;JgiRIc?IR`uDeRQ=%gj2$6izO9R#mGu5i@cL2O3>dsGQ2sj~ImPyr= z)j*V_II?mB2R(fz5Irt=o=#bwH@- z^I^q~Jc9T;U?q4)@(R&$V)v-Z>;S@^f}ij9MHxBuO z^!)LDB|OZw$<4N3jyxLa@HXf$VzY?{tM5h5YDI_9jbm$||M3?WqFHuYx6G}}Fddq& z%HV5T5Kgyhxng%VZid*`QCBVJ!{C)~hy+BeOf&S)*%*~1?a@0l4rLhnH#AfWsk>we=0U6W$n6y zc>`n#+vTjVL|Azij+0FTQe5BA)uRh17i(2p0}Hv09hO5SJ3e+}Z{C;L8-_a73LfYc zZ_alB)fmk<6&Y*q)zQg_v@w`o)5Z1mi}m z>iBZhSJHARODl+AS8L>l*9L$4uW(3L0SFXZl>We|6EwI)qXJ|HUbC&}UL@K-m4HP#K~VEeUS+~;vD*(?T7I?-V0Jyg=D^{P;JxM(m~Oq3D0RWE&!I=)h~ zc=#i5W=cSW#3=^*p~)kTuYzWk9Q^cf*6}OPZ0g8T(lQA&Ug0?->HFC5Z>fXM9i%B~ z;4R01D&O!+iGe2Iq!%Ws3R#&=63g4R7LB`A_i>_JZtUE^tN|UU{;h|OPloI+jI{#& zPGsZ^7>SEE*f}eafRFc(%(=Dwa@a*t!@ujCny?uaq*)VsOpwHD32-+!^k7(52k!qt zr5ejD_DYsQ+wFAM|2WuW_VkaYQlZ$k^)Q@Y*bU=m zoP=#>U@Llc4+{UDMS!K>QpwXm^gvJ@=;@BehK14hMQGER*DT-Ra|os%j($Vq=3GAF zYmrsK#Y{zzR%vNq48^mTeK0QD@wrNqb##V6O<&+ql@vb^&H58aST3`Gj^g~%j>~jw zD|58PE`@xiI+F|C+{OtltclddZUX&J+sJ^woZ7RJ>!*V#-ZMMAYTCiM!3&ip3_@{r zSAmZ|^yXret>d~7Jv^VPbT48HX&P9%xdaZ8RUqP__sLci(tl=J#coQXmtt7o${ozH z@QvBvLV7e0KHJU@1dc5fy#q?)2M@pZp_?P?@L{E|te+zXbWfSLmCfChJr7Vc zM1&Z0TsIYse(q(f;7BJdnq(@YXO4P7qnYXn{PNlRj|!x0NIt9HBrmsnOt9PiwT$a& zC#8Bj4E{tyssjR$Z43@=Ke|o%R11p)~tDBC-VGrds1+t z*Kb4pfB`|*OQkZ%WEH?AAUlqIfOy4Sb)q^F$8_pU;ENkQ5U`Ifw7&qJifJjCZuz@F zy<)}$mMtQm6~h;)UoKRF6tNn;JZTou_C9YddW6dfYrkb7hD$1V_I}0z&MbZgmCIOj zZY#ge85~DC466niDwwWmL9F<(acg^iaihxnEPBX#tF8~Kp?x~z1CF`bB<%zN_#%JTZJAAwm7=$3w>8E)m0SHai+_V9;MP3 zEhO0x5z~b4j*7shZ{f1-&Zp9b5FsWYb5jg=+##)J=)M7+b zu-kMwlHZp{t926!%EdX@K1mxfp@S{GMq4SE3zXTVSjs8&0%Q;3`3&_bY|?FSBH+Z! zXBT<#l0A|84(C#cV)`XVFdXTANvrvw{1tHf(8V#vBSAVkhu=t*3c4j16NWl?oZ7|3 z`)YHO;JBH8+a8_*bWSdRLI^rDeU0TDcmJsP_2`DCMcVOL|f9%YLN!07kz|(Nn zLf9{nn%JF4{n2%f3is<{(no z)({+UtO%A!3Ev0JL(~^86488XC8|o-kUB5M1*zfZLD0d%s}mP|`sA~0qe`$3egT2y z5F>bR@GUQ1*`Zh~*CV_JVvAD(+raVwlR-tfB`F#Vsfmy`zCo3hOqK?foH?-DP>_iU z+fk@A*#3h|N%Kgig|@z<)j_AFn%ko`9f}1%wA16p0ZCiF3}5^5zMuZ<7t`=a24L}jB+dS+<&fYS0Q5I*ild}Ddvq_wuE z0%Kn~MkE-?71#GOS{GBmx7WmN3zaGsvnJ|)102{f@#YYWTat|}n)gfkmQ$HMJ2~6Y z)3S3Jl^0jL?HQULSL*`}=r)Nqv8l}Xs*aLDqPa~2i_N^vcQx4621b5Ln zfSB8WH_(p=H=EibGr8e)d`x?TU3fA)I9sNEBeL13F2&k8rPCZv$=ZSmB~nwYf9m(_ zEt*u=_BtuF#-u>DqYipZq^hpO>FNYVS|K{uk$(FYd7&Uw#=X7EDKB*n&RW+4l=bC1 zE$UIgoe>*?6Eum};sQnp^!?Of>eyB0sCh;<;$- zS{t(Z;_CHnQw}N={4NCgI{5t5XFZN^s^-AT0gpiP*~uJ#-tSGJv8|)255rhTw!6sP zy+fOb=mXS<)^7MNWzG{Mn;u?zYKHJcvcT;f!UJhIee3twt~`sgyTVGv^7Jd^e&K7p zB6%#4P(ilT7Pi35O>iIt0Ma!IK%_hUs^XC6-#?Ij)U?UtM|a7D)6jeL{csj{hdBaj;HQS&);q5?J7zSS|fn7%OcVK=rh zl-Y#M?|b|w_0>gS9MNiA2+nr;^FtPC%3$%+sRK$hk_1nGl8qiS;rqt8AQy)+sPmm1I`RT>9)*}*&yI? zFmByQjQt@)?U)E5!Zo_Mt&PzlAvj=E%}RsA0i=l>^UBk)9pn}`9}#E;r(wMGsLeQJ zhY8q-*N)BP>WBu%gwkWoxVocYj3>YV*8FqmP}x8AK&;4;!G_sb=gjJfGUHNUR6UqH3v<8nt;WzD8M-veb?>UymW)kp zghBQ7Da0)>HE6P+4>lPza2lK2=0X={=2@9|PO+S6u=J@}xu$#~1b3X#+9H|7@ke`4 ztx|0bR~m%p?$352=?+P1B-p z#DXcf>U;7iRT_LJst2k{n}~rfY);;1Z%@e$-aZ=Xvy!PaiMjRzXe54{Iu$!dwD?cL zEwoN{%@-M^hJN-}9%ykSnau1SS15!-Q>V*-p^46Hxzcd*9mUi}R}E}o9!glSF9S2}=d4|ca~t09W#4;u^}<#%`Cm+ ztD?$fHj!}lklu6TaHW}+^6|Yre@zT(xqF{)h@B;*v`++9?(T3W3-;uXym&~diMWPi zJh&oK-b;gxN{}uh!qLdAq`O#rek2z)te>$ z#h_EK1|;FbNe^a1OqxB7unpPo3?xilo@%QM|_fvN;n&0HA!u7 z?FYIu&t{TRg+Oal0lu*@%Vga;uN_yf?RF9AQ*{z*5D!QZ$6uNb@JDDIEy7jU*RsyM z+cTz!=8K-w;ke3wRD`kCw(=o0?WXK*7^ZD+)n27$x!9ly$Xi9HcXL{0qbqRg*>ngP zE(P+KPElBB@Zd?t(uCSsnf-oB;*0w->t!PU03f3X>~ zi5Rr?$oZ-m+3P^B!-zXPmv{McQBR5YZ>313yVQOw#%EnRK5F$?XqY#H!=X7s!tNlL z8#f*Nkep@K86y`0(g&gy1hJ&_CL3JJxZ<{9DpBm4PvgSlrXV)y6SDhKZ4m7)cyQ@N zeHysDLkz@BeA4qFsn_7kRCmQLE;D1Dm0I$()GD}09u|w`8T0B{P?pgHYo!I#h95>4 z!-deC(AF3i{dw#`VIWT>C9ViAQ~rqUbM|@>F_AOtvNK?m$ixUiSd8bs7!K!Hk*O?yZNHwaUJ@hKGbKInM7ly;~}R; zsYy6R`+E=@A}{Fmtuu)9TkTyJV#$*i7a(o$pG%ZJ$UHa_K?FcM50u({30spDUwD~H ztTs5KHRhS|^gdYVG8TBH{_yWo;PPhcls1;u5@kf4?SF}0JwhoW6X8BAp zq7Ex})d?cHAQ@A`09+I4N5;{H;YGU|4jlDT0H+xVIc=2e;7>@is>Mc$w0zasQLb~O zz|jqq9QKxmV}CF`B`VURS9)uy)^BDR1YvV?zJC@=H+N|+l$V#W8MOfuBHU2jSQiNg zH8Xaj{{V8U0~Or^pji`e7+zdJy9^yg!cG!;2y`WCe1z9;5c_4fG*9jb4<<6$r&6HM z$5(92agpUYB)ZQP&NmB0I9yK8Bq*Ni%CRpESCR(saR4LX#H4?57cl`Mc4UG^7K`h6 z!RA4W<40NkAHOQdS6{pLWlM0b@F18};;$rfp{bm<3FwV>Nr`Nn5E zN|!n zZHubrWP*2UU_tQ_mH2$`(4#blyAA&UPW)2IqyxSk3FwV}OBx$5(%0MYpI4i8uG2B{u(y8P0G;PxHaw5~+`~a65iSIC` zH+OC(ytv39jhW}lE~U43qBi6$iy|Xus@GV`3vPlg!Zk`wE||k`b3A9)Z0(>ZjSPYg!Ku%yGKrM}oFI;OT1OTW1>AUi z`p_wE-H|YH7g*Rwqi9n*Fa(l(_=r7b-lZC;!x1sT#QL~os#&Ei&FBqEH3i8L*Xswp zl6h_e;kkECXOL5i+;b0({1ov*o)BCN^SsY4@TzQ@TC%Zn`^3)4+q-*7k~HJ4lWgS` zo70@j;pO7fB|Oq*6EbXOX<#*)mj3{8GM3{b5qT4(kA9S- z+FhGKkYY}pyX7c?l?ci9KJ$+sd*oGzWCQ`$b{mgJtwU{@y4w@q9v5sph59)wHm&4r z`#a{Ob~Hn>mYB4XCfjeA)atC-JGjZG+CM+nY9ODs61c3(4Wj(vm7P3dBzyu3ZUy z1_&(>d7YWZen?KCP17 z!6#Wi?-DhGl<-;0hsamEg!p2O4u3V7~ zCU#I6rg3{uP13a0>`JKq*%SMI&6vadQINl+netLMXsp2M#%|&UQGD&wWwa<#44RFX z9SJs`N=-^sOqR3*8w5{BN|#smCfwiVSuqztQcpXZYShB#5XNh%@5v}P zcHP5#Ll(CuylIjq3N6D78tFOllhUSMl2uQA;#GallPI{c&oO# zrr@D+7~P|o{dR0r?6+fzX@lv@L!9i2*=1UE z;$R54=z(o)2b!A{+mmm?Jh(AA+Y)X5{ku$Xsn6<qcjC2%pWc-gnV3$e z#30%iQQsnf6xwkDd-NtwJD>_gkBtnm_@fhm>PMIbZGOn9H7h!!VZ78n$Pc_7-1H2PFD+9ZiCC&ef*PwNnm6P%ylW$H<}_zrqh%_`Gk}l8v|{U&hNTBG1$OlTu7Pg;EJ$B zwuKGynWIS7FD-X~Lo72=X;&3#RKf?sMTk0Z3j5J74PeV&jFcVDMHVlEGDM#N=knH_XBCX_ROnWL;1lo@Ju8^y z2UcCL{-AVQA)=EkqYWx4gHhW6^D7)n+0#Cxn42aS!(8*?69Kt!RK^)hxM`nBQR_JP zLo%Cdb)-zq`fQMbzODv_oboC&T{hc`rCHz9hegK)pk87VfGqhG!2sftIV(NJIfnJ2 zi4&^Bh?(!IAPdxSYg${JV2`~gtMyUp)F8F32F)0O_qFLp5(gQTL!8CUilqVGq>9JY zso}tOc%*B^6AgcLx4wD8$35T^;vECj5P!OUAUs-d8Y+(CM5gL|Ne8zdb);Ab+Ikdc zReMN~E+;T-z;3zpE}PE2%Z^nfs>LW5+(AAqkF}saMiL52)fGw@K<#7|Mi8 zjC$l`4>(fUMXa}k5fU@f=KF2V5lEPuUtd&ZvUsH0)QN3sLN!OdghQu05qSeoJz)&? z&lY@JWE_cXbA+Q;c5Cw_PrH=OhDc9f5eJBa85!A9lkRFN^4FHxUb zlICF&Dou=AJkHMWsA>)-B>k#8x2A@NVRRE?7M#59QH%bCj_(4ZtI2@#I`9KNDpP6H zKl+^f^5+A|0+(mn9|b`1cStsy<(>H{ZX%mpV2=l6e3ZNqBu-d~(k@URj5s%I3xWH&5|o+FfW4C^;(4*! zHHjXbB|)!2@^C%EF=r-r3CLdBMrBl2ssjmWIHMtGbb)=Q_Rh+cH@7$L>#ki&l$5P{ zP2K({G`-#(r$RFva#G#8GO3v%yv*hE;yKwwrC7Mgpav)efCTKmxgx)LsNel3Ux$6D zb3f2I^pZ~LPYJ^3G~Nh4P4d^Iq-W!6@Q1jQk(Di*+ub7p2fX}{UA+un@>)+Trle}u zrhJLicVhsWgpwqUnQ-PZ7g7xhE+2Rc{o^DK5)yddb?o}pnZ7m9Fdu95NveUWp05DT z+^J#(rP+W+BY#erEwF3cY@$7xUJ$J9Z-6VQK5(z9NYVy(iV-KYTt zqBiqBdU+KX?i3DBaL>p`qgdL3{p~yA4)4KOZ?wP5tCrNFy&>}x9l7rMgFnrF`s`F2 zrtJ|A_K6;zlppS>y3%{}p?`frq>mR@%^xW>b=Go0!&vE-|}wX}ZyGVAcrSc$$}YHSe#R4E!dBUTsy$HI%?20J4bm| zEZ!;M#OB@jWDPIlh?vjKDKPj&@j*HCpIBLTFiaR7Fft1}S`n1!63O10CkV7p3FDGs z`_H;8=QRHSq+g?ooaf_VlaL@YZdW}>VqDhD=o~oR(CD-m2r;2Yk_;G`{pC+|6d&(u zf?zaXV4i+Uca}Z1WZrLSmYky9Tord(E_;iJi6k`M&VuH;;s~2U+9`uuG(ebLYb!S^ zs^l!J);Y%!Hrc>*EluSHYte;3?+HuD;P0F@bN?xD$ZVnU+vy=^{KRk>s!LN2(bbDQg-m^eXW(WQ~v;NKc~o`B!nnNh{BX;rStTxzkjh* zv9{PYu#Ez@lCc{JJQheD$LOSz&_R?x5Y}(1*>2U2dm}jV2@I|hb&JX;!Ta#-u_7& zwvb?4oRI}t@kEHiEqO(TQp;keYSKl`_$v0T+EPp2RuIq~vlfeHQbcQv; z(&8gS&d30wTnlq2;he4aqpd-89AVdyO`rjrh?AaIQ?a;VI|9K0f!Uklj&@BrDL5fA zlWD!7(nN( z?__QCAkH?H#2Z0@_e_rtTr7wVk&qmbEZ5T1K_|&62?Sxx7V$V=ke?6ugK^SS8lO)! zFqXOq&zQ)>YqKGgrBNcLA(hpy4$zafM|{(?)x|2B0@@Vm5DB&Li1;m#^F0e{P{$KL zdM~%XS}Q}OhiWWA%a2mZbqlc++kBGGR5Rvc4Ru%#Qy6YU<;Xt`_^F$yZZB zeO(7XLWN3%VQxoZ8s|A$M;;4byHvyrqY2MVK%C|zfiOVTbT$G*Zva{}D{&o~Mp?u` zJ#eR}$?CQvf78oi4B5v+;szQfV&{s3Re`pN>LhZ^1EDLUG6_3iTY?L>YE?5}$~qHO z?;6YzXj#+l{3K7s7LI9k(`h&UqyS?#gXjomg*r@zi#g5OV9tF=a|h4Tl}<^jrBNVL zXmNg)h&hJes00uo9)bYV?a5uvfB?P{9q9+utKzwXeER-!kqL-3VU=7Y=IFl#Y(yJ8 z4xf-ytL|^C{9wV^>B#`@b-@6D;lRP<41Ctgs$)^IbZdY@rB_Fb%60Xt_O+tVHs_G( z)h_Pz#R&vWx(TrdJ6Oj;lTQf{=s1hSMupR^kZ!Yet;D+M^qknQUJcIKnfa~l>tsy# z*4~xTb_{ESu(aq#$p$UoG&WLbW;2_E1NIQn0}jc7Jk;Bz>J-trk!ilB89CuDGZun6 z^Gwz883qSyJq|GgR88OlAecT3vVv|l<;Pr*AP@n_&y%m|31ChD>(qTClUyJXp3Qa@ z7;o3{HN11r%}q2#T8(YMAYMPlPoS(VYru31hqTD}T>}8fm=T`3s=%bFBY%4zchl=l zyVmey`fVVbBHHQqwyENSaR3<5_JG#QVU8qX$6 zIkCI6q6z^i_Hp`2^uoBNJA*2%iddW|1ZL51a8&V3i^Ouz(5hSB!_cPCrGONh!CX@Z zkC*6Jb~NTdaa4*U@WjXdq?Dw09Z z(d$>9Ap#k{+CFK=4gf|I9?VKBjVygwbwyxePVzLA3u)viSuXF4S9aZ}G#2A|x|2$qusGfZeF$$M*Hs3#Eb$}3HWM3Ekk_~3Z%$C3YHG6#;mc--#qO!$ z26|9hNV6E%R1;wbD3iq!UfQScSVkMiQU~=IV%t^+xr7#KVIr^%P#ANvFM@W2GftCm z98Pg@4L13Ok2Rd=i@Q>CSKa9sJL~ zT3FGM(!1jX% z+GX~j3%p|tZh940w6F_&IRt0C3XP5)loyvh?SmIQ{uF>aKs4G+_=r4}8c3+^TWCWP z=XP!+VZ5Ds_@ca=n%N-UGmW_<0P!3^#f16XEp>y7t&1-HNYK#I&}SD?hzTz(&kn&U zz!aC^4$z#=ewkElfj{J=Rd9Ru#+jiO+bA7gg+ntGL<^Z!aWf#nh`wJnM|OhYpxocQ zDhYF+h!No=TkmLu2@zer)TZcgWW?H0#`YZ9jO8b~nWoW|bDPgTvyn zy@2t0aIG3h)zYcfCRW}m!HZ76VuClS?EsQbU#FK^>rEPY5Yk234q9#ElB({3)La_U zT`epl6v?}!M$$mo0GLfzPT?m;;sbZ;ranW|wzl~~SaGaKknp-tgP)ct7SUI?zBarze+-52hXEvr0?1H&QI%HkexYjxs zWWv{(hz^oL+S@_vvd=Lk26ZI05u6_3O~WIEx?;k7T~d==`hunq*0?$DZ8L8di)`$K zv`8cfA1f7B;ilpDjuLuI03)GhnV5^)xu#!FJb|A)<`j?b+B0uo=%v)%dr(GGna

  • Ql0&G*vdtEf1FeP8 zoz9Ab4Z7yui>K4wC3N>Qp#I^8(ly$p0f)rhFQ&!H#v|c)1~vP~=DN5Bo*Rrz@bJXZ z!!?~$*)nv&Q2RN}q&jC^WgwEVi<&*k)gU%r0SJvQ9763k&*IL)D9S27y*KItbH?K z5qwhJKU-ZSe_6Z97>I9e1md(8WgL>3?HyP%Ho+Oq#o+5;han8Rbj%y_&0g%pGv}ui zoXWe<7@)Xf!)u!x>FG)b5p{4tIB-FXC<2(9 z_DS@tCT*WY`j(c**(#nLXV8E$9CahB(F+J8XRT>#J>%%~ghsnYHhbn(04bT1*DAnj z+-Lnis9OYlLONUOx+Ws)4x;lOWTXWmrvNsXHdO@s$JqdRk`pZqyNMlV)~H<9X5-R* zP?!WH>FPf~?=Uu!?CuL4?cLPw z;?t(N^n@2wEj|=N?$y2{8m0uV?U?3*T^hgv3}M^JRMY?f3L&823B)=QLkn?Dz?cOl znr+1{B)8I0FyUjQx>R=7xl~AYfFFN`6MNzKnX)6+fE$cRb{|T&#$*WLq6H-16-N9} zCi9vQaVEfl_jQt}UhSBdo7Dqg9O?ttT1(0MeUsQmH)D zpz6o5z8TMr!PTSGgfpc(kZ?OoKbxf5DHZs*#t$^*x2}t z4JY+Zuxm)mr=KJHq7%`PN&|CkEsf^LgzG1zG;#JNR5i12^C1g+79xa2axv{3l_J~ff%WYf;_e0 zkBbvSc1L|u09dg+LDZ`P7*8Dg!V?>UadaZXZs`~Al3-MbekmIj$B6K>th`2h9RCn&woZpV)nET%St=tXV&l<`B7D|bpv zaCQ-Fj3pK^t-@zHFsfW|VFFCvXCw&>q|#tp%>Ai3VkESQGTFQhk#(W0X#l`+^NxfD zJve zblY~p&nZB_>2paXvEQvPph+F9ENF2*7FUuul%%HuoiTLmpn-3-rK{e0y&_OQx~4K|Jk&E_ zN=@xtsZfiii$d>jiughst2`S(=s<1SCxVAj{v&kNB8hktjYqpuw;P6Jj%ajT3s$=3 zQhLXTO_1{zLUAYwmwlt!~on~B~j(<2UwJxZVwf+~O^ zJORL+w^akDExX8cyGIjcwHWrtiX~O9!NDINTG5?QR8b5QixDdHK$DDeR3HntT;&A3 z$O25tp%QChbO!iy^((Tec-VANo+2>AiO+{xEi%wHoe~*(wCcOWNQ-yLM{PTmF|Nj9 zXM!tPh~?jzQzt!jMcasNrDNaJjM`SylAqakfQTvDKm2}6J4QB{5tOpAQbht=iY>Ot zp^;*xgB0z@(zA6kTVZ!-Z5NEBo|{5WZ6kYf>FQFN`PNa0jJj^7StXdlTMm%}z0_v{ z){TEsrW&8qaMTQxImOxEJDA|;QUPey!W1=kE9PmZ)tcYb1AX71Z&iY0_DI9I0tMImo-WIy3a7JhY`EJUZl%| zxJMb3z^kN)^`I6#%?8K5)hbL#w|*-yi(HBF{{YH>0O4>|p3|VdmW`WX5S<51{;-8p zwEL|FGL*ovE)?TbFyamKP#;m_;9(3Qk^$E;$S2JuA@|N8;0fyEg2ME@B3?53(eg|0XY2z?u4(=0+;j!ZZGgw8;2DH8% z5pARGR%_jfh6G3l(`6qFi6&xgvtrtAm)bz^oVI+^YXMbr?iN{SDqU&3<1Q*)^7U|yY5XPyog1neRw zP`vkf-RsbIWnPo%JRgO^Ew^C@YWfO}s+-p5yYIBiE7NU8@fRP;a|e>rtd;z593MV| z;vnb;QB0F2_Aq0L@4lt{BU+F(HzV6K@ZAES|HI>8Ovb-S->=w z0pbOnGN;sVC4^WD=%V2-u1L<#*MdM(v2_qyNHdqoZ^49{-z|ESs5J?38z5~_7|~F= zqr5ExIn;2^Vg_2RLekdeO}QO>lb1+tFYz*=I$H2Yi^YoE^IF+;LIKW7s1Ts&4r@lF z$oVaf&hhaHJP6H1sN#D!i`7B)T$fV5GyR3Ro(}3DJx=@ z_e)BK8uw^bQfrp5WZZBC`qW1mak}2k+ZDS#r8|nIh-{Tgplbl{nyags$SB0&$x`O& zStU{PKQeP_7Ym`k8;^!aQNvzzQW;QE5t&s8b|J`C2QWjG#SYWNN*I1SqK7*mrK5GE zgHgC%KvS?gv&-)Kl@4-qO0TTm#M|y4LYW#ldSKMNk&B#A^6=T9}Hm^#uUKeT~$ zI}y>V125~8a0ZhqT}FLED`?a@DJFmeKV&HlYlX=r5$PXihG1hY0LbS zc*6rw)9ICLL>6)fsbhRV&TxGxzyriidMU3k0!w*<6spgbj(;>Ya0d>Hcg&$ZxXhuC zfv1Ad{*-_{H(od(8knF2=``YWbG?=5(=2ht26h=h3_F^*B)Dusb}VPYQyrspDFGq; z2^%#}!zUsm?_1VZ-OUds4q2aEss^_MZ~rh-8uTcWeu7_rT4J509p^GgqC2Ex=ac@o)UOwZ0<&I{>+^jei+fis}4G;jx3k6!A5n9_Vsi9I}Ntjj& zG^rrjXjBp?{7Pa>?Hf(NlO6aaAVYR}M!QAY4u1fMZ4Dlgz8FKq&QPHW6Qhvm{^$()&zfN&A8QM}ARAk`ko42Q; zQWSqIbo!SgO$aMII8%AV?5+AMG)#k-&qMP>q~8p?FjRmCHraw=$TW+o_?#`}>Qh?( z0FL>S7AhlNTT!Al8Oi{?d`tsjy|(JE>Rp`^O6L@jz}u-7p$>C(kCY))$eq0&v;dmg zRRIPj(5Ww`=8`@VQiq6I3=a8~hT}odw6p{O?+zLGCBaFc>?An41BW%GjXDmG(Kd)2 z5CA3{#; z>SPp;84_h8UCaP&r@vCXbDm3ZzrIL}@fao&Y~dRKXuK3=XJF&WX#iT~7R?qjz<#Po zG9>z?5VrZ>$zx%+E)45EOIq_eNn7ND1o5C6Du|WLo^*g z0p^MuBFw+~W1ZC>A}0w8F%1fV7;y)9iR!w{O(s5Gu&?4{uYjI{N6BFmF4sFM0}UpT zi?b#>=L*lJ+N~K|Zj;4O2e{>L@-P;HCOjtn3P2N^E@gQC-=);U;!{iAWl0oA^QDGYWt=4V~iDL?&8Wd-U0EHyrS^aJm4faQn@j zhGA?2pNFj&w7Ia-2;>$yu>#Z0013?i0f-&|tPAE8;RHSWju;hX&qHYzcIm|ZKPD<^n*N8Sy`*ziDx_jh?nUp$7 zWud^ezz}l;NM2j~N%Xo~m3lkH28wK|OI*=rhm4?PBxC@KFG>$1G@JE}*Zc3M*QPG^ zx3s7H?D1Qemh`N1DOFO%CyLi_X(B)_Y)z*2@5u(9CISnIyoo2~s`mDW5xD{{`XbXT zc<}>hxL#hON@ByFs`MHkGo&arAKE(z=u%lUfH_TwR0N>|5ykMATG<6p1xJ7`*90o= zEy4xlLcJP))5K&sr`C`Kc`Z0*+coB+8|Br1TCGZqdxwSz1S&1*!FFrwCQaS@{IJt7ksgoDXU zH4cb$+lYa6I8tZYQfvre&N``7qyvc`z%~UQr9cW36c(v*98_}KEf2)3^8lwZ7XVml z4N3H>xxohsCQt&0X=P5ut~iO#{%9h~_?&V;$wLbSAUMg*Xr)l-qP^vg!?Hp{6xl*R zCE2M0iw2kCqOz$B7a0#i+(fAy;0v-cOk2rVzZHVV?-npN`V<;YE_Te-w2W$Q89F*q zgG5Ys=7!p1Y9<)UR!rn~vaNY0!Vzu`-b(-`!M!N0ZV$W(*r1ZwMjJHr!o4`ui9(k= z#Ew}a3QQT9JT8zJTybq4XyL@$Bq1Q>WpeUtETxDxB5Vr}6Fo|}#N-boIO9B#*-RY_ zYG&yHw~vLb>nlsZ8xt-VpfSJ=re~p1*xqde`HBFL=9e;B=^T`C;S+x(R?&OJ2Q)_0 zZzVxap%=L3k1#9YC%HvB98<~`62`X)9zjxz@Xn)(0Jy0J(d3vBcDgzhOWBG`M?yPY zP1eKYqzL3oBHWb&!?U^CnUV3ILZHmTU=WtL#GfQqv@pmq#Ycmf5{TAQb=>305C&8N zqof}p3a@EnK=Aq9P=>YEE{!D~n^^fKPeP@D$WGnq)y=&k$`iL5jdasEj&Z72FM|wj z5T0ZbI#X3NQmLUOt#cgGWDzo@?-d1S2WM#6sZ#1vX%j6`dp5M@ovA9S?MIk8O(oF- zo~vBK*LaV9d8rOI1-6AQg9S=w6p{(gKBA?wGCmex4f!0n=DkmM-YUO#p(p+?x^jqB>8MYYO2`? zCQj$+f5m&&=vz;EAKg-jl47fW)P5ns*WwqbZeLQ}KBUZhy+G})$?Ttbru{h`3i;%s z(XCT=czq|jn}$wlhYpaLqG!@5t*A#8<)2IbFCr=L?Zu~V!PfQ+`Lo$xjYoK+ywepZ z)NMu_oYNzhJ)qnqX@x?&UIaw!P*&HHdV?ecW{OUzgxk{mU+O7n9mvs3UeDNG!GXs&$Q zVzoA$Qcb(;2sMCXj4b2>RHj}5oV{vOoGC;Tew7`>c5RcG1F2TkJS6f(_khj^+)tXg zc+ThyVQbG%r5&Zl!GhX>bb!3E!fxJBJqSRJEwthErbOQmx7}8x{gOQWLOjq1Cxyjn zuvW$v#wFU%M5A6>WGnWrIg~1*8=8p(l%NGnTu@wOtrVV7#T}(Q21GRrGC{{t=kFdL zVwYB!1VC4$dpo+{Gzg40dbz}NH%G$_;|f8p9bgI($p9ypxTVlo&;;m{d?R32?E-bu zDrzToyAor7bDVaWM8--EAdta$iBp5}9SS;86qwOEpk>7xV=?BgOJfKEI4H8L%uX;A zFskR=7+X18j}?WMkQrw-O2EfB+QDYd1{JOU03z;6-HALE{{W=H_luSXyK0dd+8`_e z+W_LM2|A)f@~|6FAQD0&LfbSWP9bK15F$Aw8!1d{fSDtK)1e8W;P_!s;&kFOnvfKC zdK<>|RMVkJ)!l*BNEIm44P#u^l3iY{Cj3um01|GcpQQatZ@D{=-oBp}<-RL@A;eYR z03ZJVF5Uk*6ZA~=UO{tx<8YL%cK9%)QZMmHC{_Ai* z{{R-F)HQc{uA^6W-Wv9`t}SB*@FCM4E!EYU4YF|ug5`T!xti6#+CTl}(*FPz63?!0 z0j_gK7TKcf9$*0u96%Roh|Nv?3>T+ApI@Gw!SAg{2*d+cigM*5|CL|sT11Pk^w)| zbaeQU)5}Reo81M3bs&)2QQm?70I}~rp2+~3g`e`bl7H-**X=HQw5dPyD)a3h{_snm zS99A^h-;kCMOnh)+427Xi3n6s(|0a!ylp+E9Bzx}T9 z`$&@L{{WM`{{ZbbpW0mOYSk@i?JAN$Kl2~{^4xWGsaO4@{{Y@f&R_n$>EIR-(i*h` zU~c=!{{a16=k}E`RQ~{n-tYea((gaCgY|bkl_-X}&5>F|T;|L_{7)W)i;AHC0Mx7g z(xt!oCsth);q!S%0I-GFalc``gZ}_!-hXKxo1EJkJJ&I~Z$Gr7eJ#bpu`Y9uhA}zL zBf(eRCt^USR<05KTEfEpqsE`ihnGYr^J6;JuOP6FqO0*`yYFX@_PfvRA=9qM7dOPx z9fY;aa|=uZ1UNM0LAcd(=A}=?tNzjd0PiKXzANv7(fz}SxA-wdifp!~RA2B1Ody2e zKJehLhKtRK*dao1>DDYhWG*+@Tj1^5V7fM1Uilpk!vX&UPaj z^GJK{@m{4vPNhdZtTng%vn1L;JJP1{{UU{6K_CWT)+BbP6)vf0}u-f z<#!ZGkf>gPB{T#F##T+$7N!K{iGv|(231eo_xSX37SndvKvFQ>N1k){G zdiT>miuY~n4L#~NdYNPY0MU0zCkOhi#dgvNkYr~eeAbt~oAurE>7)ibw5P2qmuLEK zE$Ft;RGKP}Yu@M$=XR?|{q~sW_fJEhuOD+TytfbzX&{4Zk1)L>>7QTRzNHKf>}viU zLIFR>a(AbK=03fS%&JDvPTn&ajb%*!z=(awhgV7eSRdX_nac!l`q z=~!?PA*I?I{Pla`VQQc`(Yo0+4Xv}NCu0Z5hpwu0!6kenjp||23)f`G= zH70_WXsT)wfKvr186~a;~h5nm14*XJBqU_C;rv6Gv_)-Q06?t*FxK_ddD(|pNK%G|#Ch1Y`c{dic zDJ`PFGMuwh9>GqVu-XV&wqspP6M-i*cGPPbaVGo_LqMmCRcrvX9jhjD7X^-&5hr4^ zTH}y_sL^FB;c0{c5SKOFil2eSvpkd;#7QWn#54&?L@?LMw5b>#0&OP%M8@a}K#t$) z960cec%!&eXc!516&jmEs$CojbRc4{(hv2w$+!OiM+ zeVgSuANLQYY5xG(Q2zjRai3rMy>ayyRW!GEzOx!!Ur!aoo-KP$yq1*FekIJYD_%>W z`b@=e0Q<7&epI_do{4{q~&Uk4S&4|7i3CydbE*f$$tehi| zR>V8xqkq>c{{X}8U+f3XSN^wK{vUGx0AN0AK9k-DQ}v5gqsp)TB18d1ch~o)5M4)g zaTD=UF3idJoMWM9{c^wjKIQ(veAboa%~yfc{V(9&Hh6#V>U6GswrhC>9DRR!WZ|Q_ zH*OkOX2f3$hSfv$y2CBsavZ1+nolZdN^X&pXA)fc!J4#U-WuQdiP5Y@Y2IJBKTYy3x2m)fA;&A`vLP={{URC{{RoU zf3P1l9W5YrUsnUQqsq7baFFk%lwG6t|>gbp*rHF|#V-t!W{cgYDZh!0t&0TqO zRpUC(rTjJT?*9Pfs=4M53wZN7rD}j;eRmTA2f)mxR98iu zx~1+SyM@Yi9ZVLMG>x%z2gx%jLH25b>s@e}1YuzqUdiJqcrOi@*h&a-l|xb3vsMK5 zG_vD9OOg7w{{Xvj+8Se`7h1=X1g=l&-~Q;2@oL+b{{VhNLP#p;Xp7M_MS|4YjR6BN zG{PXKwOzR@CIOn;EFgt?dl4ZpBBr%s23A(8EkMhT`%?8ks&6~XY0$4zv<)W%-*>j( z4lP<^U11?C1~0KibgHRoiQ!&4eT4s}4Z8sZI0|p%IdM=GV8#|zZAE%lWmQ#>^^g#f($c2ot2kbx?3BQG> k{Ug#WRfYcPZ=89`+V*Q)00fwkZ_yH-QnaC4dx0PS**2d3sQ>@~ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000005_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000005_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..6cb07b083db3a19c6e01de65375327242cc257b6 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m#?0X39G&Yn557vz3g&_FY7p=J+B RnpWXLtht~hS(*qW^Z;knSRnua literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000006.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000006.jpg new file mode 100644 index 0000000000000000000000000000000000000000..65f7bffc22effc50e955b9af9309fd83a77969d4 GIT binary patch literal 32213 zcmdqJWmH_vwl>>#~FThYljr+Imo)l?|jY2-&Fh&^y05UQF6&V8q6&dA8-%~4)2#~p6q7X`|Cl;8! zM?E2`o*>qscZEo;b9bGhxiOfBk|cfk(+MEJL<&F-K|!JiAQJ#k2mpT;060&_BNIIR z76+UI4HR%o&hbIs(px2(qPQ@bBp)6;O>%`lNj`*&@m z-%_@@6MQ=ecSfsxzV$Ypm#uJ4y>2uRFkqD; zahYdtu-L`?Z21v837yWkr*b1i1?rPt3q>PP1zEpoB&s>s~#{dg=z<;wBfbxJ#2}k79zgFHVbfDDv80v|lQGkdDmo_yd@95wHYp zZLPGYZaj~X?NaR_Dq{pce(F^IyL;@YGG38nQ#zHMKWlQ^I?xV5WWuC@a1bHCf(^et z$+E66xz$R{_cvPa45cdPAycWJaAN#g`LvT1kytnW@;B)}rQNcxhqbKmI~dl$ibhFf zSkK<6rZNxlj`vC?qZLg=A9XEXJNhv9gS11TsVo8dU^TTDY7&B)39KKH{r_$gBq+}) zIhs@}qQO;hN@0r6R$fvFf|mjwIdRaHx$%ind?RzC#s0}m&Ly8mm#ghP#^1W0wDg#2 zraYFVSFhTiFq~Bf5*7wyW%%xBMgP?_u~YW9$K5HzxpuT#ie1lwx`GCbLGuR+QpCS# zx6rY^_LysO>T5w6fJn%v+A65_7H+F=7|65|9+^!C_xC`7EW29cS;eZTvJXqT#I64b zg}aJi?!6pksy_htrD{j=4{q!%Vb6TR_4q3mgi0|7Nz{-HUm;TLPwf_M4-vya2 zcdy?|!H8q*)RiWOxoZ60b;{lR4f6J7b5!JIE9_fRuhvcmRGf|yqhIpH5fBzIAV0>?lleuG^See-K-1*1JyQ_zoc zZYaJ~5-E(9Gm}U%RnNWoHTq?WL7PV8iH>uBovVU!z@}=qr6&Hr+T!gx*kjNRHS1eu z0V%|FCuu^HfVRjePNv|$qp!ao^`FuH@1*d5dvWS9+4C_0UdnIiJO>d)uZ31m9?sqR zzJ}8oMx(s8unnCmyrA{@18@!7;}@&AG|9gkM3!a1v~XN4^><|dEi{(bBr58^+ca4} z8%6aYh2S7SSL8+M<08&Mv32$Y)7!^c?DSpLVF5Q`K z$D7ps{s-WVcg(rJJaZycUs#67tb1mF_U?b39FUs!d`yU0|V$fZ5Mj z_^76GEllLsNj&h!-6@G{!^R#L#T_?;{9l|sR`e)7c-FE}`fSZN z%t*;PD(7nPTY@+ek@RbGGbtPeMyy6Wm+fIsO$Ds{0XSL&P2wFJ-qr^#^gqB7e1o;G zIu4EvTG&bR#9!@{om{lkL>&`WRcPl;>efi`yX-A!x$iHIKVYq317cP1KbS-xPm8w4 z!^Bhdg|zkpD0Y;B;_jZc1~+#5&j!KszP1e9#WJ^f?`OXed+R2$l`zp^=ACd&uIhI@ z&2vTe4-a2gK4>@ueEjJEkBWvC(bO-0$au=V{`y58*4~ zOJ<@69hzfvMw<`)L@OGu>jpe(6azspE7Zxj$k4@Wm)jwm)-yJcJb<>&uU%I-K-e=;H3HH*3rX%wJ#} z`~1_B_vyNG@s|SzO|ItGnH!<0tWrsuEI7oS!V-9I{3mHF^kmNOW$d+~Fi2I_NHWGd zr_~0otV-SM^h@Ww`qK{Ei0jHggVnIv-$7jmw&(i~AK5ZmWqKK(HEO1?>JZV{JyTDr zPzNUKr(0k9ZMwdhwmUoo8|S>0La*RMhkSw;xf>l$UajySzJ9zMD+_$G{Z8>OsP1&z z-?v9zCm5)nUB#~M3yh6C+VqLBA~EV~XzoSE3As_zpp`E{(n{e9AoUPnX;FwAkLEwD z`wN|8i#}tu)lVK@xo7Ga4rK9L2u>oZ=i>j`AhQS>hYn(gmGWWosxdH&R-W|fkxFsM zg}^|wsmx%LzEj}HVTN;X{b`=9#fr)Fx2qMN!&m-KuDekVPPp z+hR{EdLgU}d11`{cs?6LDTUr6Wn#$jDKoRfCs#Xlt&e{>{9l^1_?PGR-abL#Q;tB^ z(AR81os*<5wM9fSH0MN%xGh@$)r!9zG7#Rs;bLWseGwi)oRk6w9Voni(NK4t#=UU_ z*NNg+U$2B@u>6*o#G)O3zVjb#{|}L?SXF1M3NPyOetxvNjt6a2vFljrB(X1_LG9#2 z(1?^8B%O8qPdKq#SY`H%sxZTs+3FKpp~U;ZKLALq*nf!mPyYt|lA#gBHfwAgTJ(J`Y`I>-}X*-E885!yIL@k?Q zh=JCQGMmFzn}xq%<#>|$2f%*mKTrYt?Jw(}vlZJ|R;sn3{1e7@Rs^R&+BzG!=t(;E z+jL&kDr=~F=_l2nWm*PKRLjv&+BPWucQFqC0i|}Lr=Y9yxvcHrw0%pnV%Pg4z3p_CG#qs4Sf+2rB0qc1g4&&6*r59FK$^dM3iUM6tp|g@ zTg*`hw}+%yDKIJsf(3uxOpE^yuxZqO4Gmgi*yTR;ngaKvW8?BO6!`ycSqY4uw$Nv~C`F8w#+ z`)l0z|AUlU{I^TZ?^XSBvgD7Xbz3R9z$I8}-`~E8YqQ=;O?f4G5tl`I!rBwm*Q&r< zR$$PxH$GFJ9Ajf*CUY@;@;&m6uunGM2zP-N2E#x)hWZ^}O>=6d%`;^Ibgo@sNeGg< zAQCX0o>?%fABEZ>(dlQ3(?)JZJX_#=w2?iRH-b=EwsmXL}{a?r8K(t4z<^ zW&%TowImz%fc@I~Oa2r=Tl_Op4KpOH3<_|%@F|`IdA|stpDTnb))dEr9EH3eAVD{V z9yzeCLU}2MrX^NAv1{+GO7W4rj8QyNOBfGv8z9yI3pnw)8}ejrvs$8?A9hbAR=WfT9oF$4#Fe&;q8W$~?NY9yD=Wlr|311aoKc0>Y@F=( z7(sX_KJ>mYif^{xvlE4n8bOyi4HrOMv4Lm|qqHp?CQ>>TFM*!94lM?m!nuPtX?32y zfyH%wOV>X^-kDE&M?h6nuZ&Ag zxL~5%7GFHP_*m+=ylLOp!xBW_>Dl%?C-bvp78l5W^ zDb@n;zx~0O;adA2s_KL{@ijv(i0{2Mn&kk+bQ)iXe`0$yCn?prVznlvMxjAZb@|L< zqoF^29N3kJujF;ieG{--iYIXUrhedHX8&?4WuCh$qJhuX&xHgbqiL#%q%g_tU*6S??h)-QU@Oe-w8kEISf^(FztfjjC4}e3%)l3LS$4{x( zQ;e1Z`YSalt%h@=Rly-R&=GX31aebXUrV_&8`lX)=yJ&0=GwjL5NfB@o>RXQEjF0? zvcYX$;X{@dkDj9!p*7dLs?F#(MTjFKc6QJalvG+wK5|a27-B)bZTg>>{TBfL>mRYN zPu2te+p2vmKtg!{GMEXlV~Wd`USV)^^pfg+2sztsVlH9Y#}e%sY2~XaG&Th_l`XYY zIj)Eu@9MJVI^$*-yFGUy&%|`6ID^SUT5h(V{Zz1zV*4|Ap-sHpWVr=R4qCBg+rDdc z0gob*idwPEE{K}udD4xBRo~i%j+2F-kEcJ)x5)au$N^Zd>s3Fb7^Hpau4$EH%8x1|n)ztvdxZWs+g1L*MltWvBr6QoK1^Jr%)6rDIJ$RdQuSEvWnP&uB z^cim@-&LOW1R4KM2oJJ6Sr8I+o;uNQoVX5tw>ew2PetA@uDSU?MACl^$IBI`y3Nq{k0OP0V=}OG8aE(Ykf%H@I z^0NZrHb;vQcS<~gub*!Oka6 zp-gYN>o%XBH0dtvFk9@boum=U^qeji+wk~Sm^{%lsj?>AV`%Uc;bHg7{(|*T~ePySNq7ryYb)g}RNvT-qiG5#A8{dXE zfyaZ0_74Jc$tHr8OB-CA1@|`8E1IlxWwcgX+&3HRJd@tOj}0m<)n$k_*zi1>p4QXV zS0Zkln^72gVJ0Ao@;p!=##Y}ia?QKtIl=orYQ8b9Gqw%tH+TA7x-l$-WoeT$1&2Yn z^>MYwSo}9#yOZkrO&v|bROWYMn0fu0!7%0-fQQbi_CS}dO?lWYpNPIt{R8QhKUh?u zl0~hV{FnH)0}EcHmT0iN!?WW6sdnN58q_?+FRfmzzRNlfuk-FD0yR(kz88b47t|f? z_&!)MaV5+N{5M4Qch08OrM5VD8p#Gi{ws3+&g#o6*!j}hgOZpm@ z96ZnMQOllT87+Qt%uAz>*tCUn$mscRnp+TF-V)W zcTu}Zqn~N7^_JN{s5LB}Aeh~t<56`*w>BL$-0crQa}%xQokZQs*DW74HD&KRiK9lp zX(x3=8-w;M4fXVYXa-8LqPRE8=BMl;Nl4@afF3pr%7($IGUFb;ssz_9oE7krq_*Xx zwTIrcSW~`aK}t(7!~9-g$|=#sg=bThfpP{B++Z^T)Km7h!F$8=i@)ivko|?9uaiGN z0@e0lcz$$qveerdU*PzAw_~pHz*h%aoy9jP$Q3EXrErKjo;pQS-soIF5Hrq8GAv2B z^kmljbF1Y)fKn~9mt@G5P}+!7T$o?XrTu1^4XJ^SNN{29!gJT;={dym@@+E8%CnQ7 zb4{;PmR^(G7qiZpo}|G8>*}0f+&LBIPa*sDiTQf}B5fDLnwW_9?$!H_BC%{Zd$tLlP655o`A@c9SWfxTFcmtc|FS)2fMNvl`4~Ver>W-p#gYn}D4cx^wX0!r zKlU7)^Apwf+cTS=P#t~JU_#`yd?Cq-YH7oi&0k49Q|aTH%=y{JEBh#7&hS#>i4@51 z;!XdzjP8FQgAO(UTv^hO`NxT7pl-970F4*)$u~r2A9zY8^BeyFv`ruc<_pdQCFPZK zKi;~<22uA?f1kTwwdKC2rP*ZHaRmL`;8*4AJe)+RH(e0I6`HLS)t7-YLM8#yQ0(C% z8256W;mSRxGwJ*coUwy~LNnsM3(Ew_>$MZIN`6=G@H`AqNiBU}k-Xywa$0&$wdCCT)&x_Svp4hdXK(}2- zGAGa;NFA6R^#{=Fyzi!K8*_j&GltLvrM`E48H=Hjrp#n)o|T*D@^ zDVUuaCZUfFCq_)mhbBkt@DRv5Ra}Fm_r7mDK*jmiMoCiASirc&rtJ5m+fE=TvfKNb zkIMX!Yddd}^Fi|=o=^K7e^)y$0O^lctV=%mxdhQoUj z@m`6?;X9G48qV!<3v2|B8qF(g@00Uw`CZ32>3yS~n2$2J*Fm{Z z%b^POoVXD)fwHf7_On|A7=l~lN0;!Ck28E+-`y4}GQ=faTZmLqo@OcuO8+}(|K~}~ zUn6Bdpof45uWC$OdsX$mt-w|PBMdah8=F+;ID(rl+g>_8=^YhMP6RVLL{{zKq{>;3 zWZG>;jsz1SiL*Q!#_uJ~JC-bIr%aWBtbIJ%6!`B<91{rCKFO}&{p57Ks>|$axDxf7 zA(SrdHjz(dP;!M>dZ-k$wd~eiS#&lK2eNXjT>pzCTlWz3H-8cgzNM* z?MBL_fQ7Tu*xy4$<`IFC>kh>)&J@$6!g(Br$-^uQsZkrch7;X(-!T6{;&aw~j$6Jm|%nEIYHDgpKK5uBew_^{)#rp_Lfwf=G zF;H@C-3h2A{?=7Ivz8Y+ptE$9t*@5R1maMWwtBP4BWmK~OXGMbMoMocY116pvVeUUK!f3ECJ{l)O)O#r697J4b4*Y6F*vnmUHZd zXb_=ETLV8&!PP|#K9sp*LuE4~leq;l2LT$MMzto&^WrhX$nqJRJ>osEg~=_177&cF z9%|eFiGQbSx@kJqsm218_=uGcO7P478G4xH`i0-pYz4alh*FbY8j~-XS}CBPMv9ae zsLFe5G|y9@YZ#Pc)S}Q#77+EtF?lO5tIFf5L$*Gqa3rL%Va=2kl|VT^%AT5RA4mJe z!YjDd>vusf5ja=sw~w>~w?aWVrJ1^XSyxYa!3$VU@`pJ!O}PxogY<@-df{tAQc_Kh zFG-?df$Xl~y^DQ|WCX93@V(vU?s!{Oa=#5T-=v7LX}n09Bp*OX6#8|iY9`;q>hU4= zvIU-%^U7VFr`~(}j8V$)!9sg{sM?gW>&e(=W#$L1*I?rLC^$j(UeSf*jMcz*ztqX07V|V>J%Sr(2 zO7Ljm%~(O_owdSWW7Yqw;ll2fxF`J~lwHvz0|gmX*{j);&mMN)m|Q`#BIOxpEnJ4p z1J=<`5Pey6y4T=zn-oGwgd(_H*Q44ytGx)RC)(lmMtDGma4Zh?SOo@V{v~R@(mAxix1C7ELzNPLZ4J(T!+TmN(KUN0#_ZUGW}Ud(7x>21|(V&Xi3co34j+Z zbfmV|7nL8?Rjj7VVqVzl&%U?Ea3rh7lo<;<+VJscxYx?&OAT9J!+c~0_^1|tF#G1H zpv!H>q!Ji6DRi`&W5m9Np(XNrYh*3nA=h)Kfqw$oVluhEViDH)b){OlCxU$A(B~;q zQF9G)T%^NYwh-rm04C|T-FnayfpWLl&SCb+gB=eNPaTp91+GRN2E<0n7~-tykh($D zf?C_Lx_h|~xUVw$d3El}c}3g4!pCz=!?CuF9>;;o=9~LxHh#b^ObN4K(YTp+o?xUt zYnB~5$Ne0+CqH-YQ(8X7AQEZnam{2BP&W8g)3 zf_QyGs4_^@lac>uV()# zK{_lyg+m&=k!;*};U!v3*kT}8_}=HWm*eW}FCSxw{oYn}4c)=&A&rQ?y)y0|Yhnfg zv+tgV7F;V-z$xN+8*3F!Yv~~i^f@cV&bO(Ph4Pxv*1W!^D=@*SOYA0pPYETO>^vT! zWC}S$UiBW?&V4*7hxw~%6o|d@6w!da@wMtceq?<$>+fJfn=RqnOu|uCD@p}xS`HQo z=nDoR_q!LfkOi;QWRUG5<&fC>o{d)mleDG&`wpqx2_rY>K3=b4}@dk$WVS5^3 z0@HWea2J+1>Ub_iET?GGnDSvuyI`1~iq?qjTa--p2l4?k6kGjIR!r2z1RjqZUB5i% zHtWQm<x`E$Ff>|b`xh)h@YHvD8x3>NDy^&ulEPQYQ)HZ&_@g@w`e2GAt~{uC$6MKIg1t* zlAYy2#KD=lA*%V}JdoRVda;OfpckZeauZ>kHtgO);I^(`34&&X2z|WI(^{$Lw-co} zZ;n3p_Y7kz|AbL-l$%>;HLYJ`wUSdkQvI0qkyh|Ptdlw~xx1LnB1aBkAaD#d%y~gE zc`DOcvciNWWN6dMQCTRlX3sTx%Lc1*csdV!d6z1@cea0>(Dz0?312AWIcLyzOQ~>! zCtks&Zr0FeZGVg!BP|vn2}>s-I8a|-RmH$f!=KGGzQ!BDU{W<-NB&{OXrKO2weWw*UKv*E&KFA_^8Pgy)GN~#tZ3=lG;;H0t!Mri{*P+`WxiK z#j>b4JILk=XMgaa!|r2vOWvMPAe#6v$G35p)zVi>jWP$aLwMZ~6)Ai*U>es#TBc=$0x{E@Xt$Z{pA-^RR?)H3ek>>f!Cl zKLD!=R=twEtp0CzawFq<(!3>k5_c8UKby%7Y?%qG=tORwSSjGss@C6TGLm^ z%kr06*0RFA4of}}q7kuTsO0C|fmil7PO&=5~^rJnKN~V$BN5iL19Y4s) zMV@-k-ukG58KoPk)Rc7C%cYQ8yI=M@p{W5&yoA%bdBB;K zE2RKYibeo*o?q}Z(>w=JU_s*+xw)zLj8pc)a_acD#!vGFJ=p|(Q{vl3W%MS&@;n4| zC7wM=%c)CTr#6k&{MMWAoo2pT3K5$N)`-F`3Z$UOx7_{73mCaQN+J)e_ zXyNi8l|(`Tn)#lopVfj7(m_rbqGNUv07zS4E47T3AQ)wFn7;|TylXKzvBUQ<`c^7c zsVT-qUY;Vcc)o*QQhRV!)Wm-|zl7ViVYV&w58y%`rtFqo6K}*Jn5h3^)W9IJzJh3L znK)|aqA58-F3Oe3OpsC%X!R{YBYyMM=G9}r2mGDTJEI?rGSWZ%vS+_`iJ6+(0DCz+-z5j;hZ;B-p`p{^{!+jt(57*JRc#d&(%oLsNMPH6HL)IlcrMd%hZ?~RFpem z`O4t897jDTZa7Y13EM6c8D-Sh>Cd-wq_6&Q@5) zmc-TuGnWX8Jj-LFeoLeRzTP z2pPzZ&5Qg1)xm-WD*0EkM1$ibG>SYM&Kll@}FC^g_yP8Qux~0jvdho<5S!EYuO7`*I`=z=EXOVl9q9 z|K0q?Mv;nxr`^Se6EwR4$}d_;Wsweet2T-77P{nPEov|-t#n9IUK3TX_Rb4fzpE3{ z0yS#D-*nPTCo;;0$creN)!i1~l#)680pO`2G-MbFgXlCXWEofpCms&tMAxIVdbZ@l z7PGr0x%e2g-DmEr2p)TsZT|qKo#BmlUUxBjMLS%GoqdjzQ$CojJjTP<5RJ6^>5h0$ z56Ks_Rr2{IU-0qxaBwuOx`;N`ZMJg6=nw&Z{xzG8_AHP zYvxC%$Y1>KGcDlR$Y+ppEcui&r5$RkZ#l_&ma61hIlvtKzHM4raj&V+Gw@^cU~9uI z%#60T2WFqwU4c1?AhoKwKK5g7tu#3K)b<8pqkUGwOfRj5wK6HY21wK)i-!41jD6gh z2qxR$W7)G;aE;bTcO@w*GOf*mOQM2^+2}0=I1KgFwM&!r;38++D)>v%P~RfxPG7(J zdI{{@vMF6i);j4+LE?e0RH@|F!mtO>{dJM1yek80vTpWo&#&RMebjyZ&?FnD<;62M z)t6Kao{bqgjiJ&bhbuQJw#Z3FO}R!4c}z@+B2AW8%Gt4uLe{L+!y1xpJ0b}-_EnDj z*lS51XK=r`O6Qk-^g=6evE*U@(UM}ptz92Y`6Yon8Sm4Hh{)S5YmH;VL1fyeb9fH{KPXr; z@rMeKfht&uPI7iCD{0JmCv%mB_AiQgEc-lzH_TH$pr@{!z><%YF58@7(&?i4@X|&&!srk&8nIN&642y&)Tr#4e#M#$V zJNF|K)7 zx=y|mb+$B;!F2_hcy9vRe8ph?eb0s*tN4JojC0DWU0+o+zxQ>~){a>c(f>x{MRj?R zy^@Ty6M zYD8pXq6;mkoltgpa%wismFx5~hX09YmK|jhZ_ek~nyrh*b^lF*h%mU7?V+mhWo_`B z&g zHL~eA-Sc)e^8HQn=!>M(zEW|n}|DS)VvR)F{e zOjKSgbL-OI?L#0LO>vD$uWmOKblJN=grXjG)wa;$bcQx;r+_F8O~Moz8{m4_i#jGj z@hKt_f<$2vDG~{UmsfvzoAd+<@M}M=n3l-|&)xMF=*|y&!)8=BII4+@0yP&dx8*5%erpz@ zv~!D^T-1Xu@A-yxKrv{n_1<06tP4c)2On$R9V4PpP+X-LbyE2TQK10Y!w0l-lLenQ zkF+Yrt`9$bGppM^P^uE$ray~lFOyyF3oBDdOUgJvwiC=k2oEE9?K?6MS=sA!r<7A9 z3d6^bD%(@D6!tYfLI-5Oz5gET_^=+Q>OIGw63&6G0VQD8;%{2HoK%^yWz4dKI3snP zNl9mf@FRqV6;1yD)Yn~gG-E={LX=?Mq@PkH3E2RYgB!=@Tyyp%MlAxZpy&x03%0^c z-79>;(5%9d@({r1fU!PXQD#3qW9Dd8&fk(&9*Z@V_G6}37wh#LSf?b8hP?SjIcz%c zzQ{hkuEO5dHWp@iK430@No8lE4RjPNcb_@juFtlZ*R1Fa3=0M4!$nY(&4sV}pQG_B za2h4EezjtyhysZS-SFD@1m$>6R^*+Wqf zc%9ajI7i-LmENk1liJnYaBlPt#y5-8XWeF;#u8u%5go{)2vfAG?`W0msB$B6+Et-Z zN=9S|#0AV)ZyWwywpZ)$tN{&1#DabHdGq4fu?vaG+j(p1w}?s1O|ec6#AJYan0K#U z5ippBX;LtK8fu&2M+S+jxuL6h_L?X9?5bJuGdjyp{!MS^@vfq6Yr!^$L@RA8bIqo5 zvCg=Pg}xXzd@xr}z-eK7tY~cj>mJ`l6W9$w>0qY$RTCcDzcTNw;rK?X^@}&IsHoEo zU?+Jak-=wSLl;hNtu=!$<;}*^+3b^;l+&G*+xAL!NXH{i zxG~q@Wm+n?D8*4m;8we#ICitYF!?rzwWupbyx!UOu;Z$6X4j=PG6X+SI#!Wp*+47< z%P7Cw0=3sSy6V^F7M=o5S5ASywlLtc*i6FiL&nguyuylnNs4EAv9ls)m&fh}YR~C# zzOPf1Om39riu0MvC#HsIxN zK?d1O5?Y|oRWcTq{oO-Tls~1K`^+(;<$Csg8KV!0!tL=udfs_S)$ec(`LaM+?%5(7a* zYX!HA%a>yP8z1B(8A8ACqvRPo>$j@Biy*;%SNV~6lI6l(C+8Qzod9-h0*5y@m@0Mu zfVqkv>5I?hWbv+7`G&^*Yq@T%G;437ALDChTw9%S^7`fxqRdqb7sN=h&A$VGjP*7U z!*gr}yFkeqWA7u3>`)jbh2cuK;33%0F%*&}OPvvci>~Cc!FE;!l2P9~>RqF{W4iOZ zXNZ|u>h?Lp7+cj-b|Y4XttHuq>_f-*50VzjEjpLswt%JAlJp>@=Pj885+9W7aFNjm ze+5!f-ZnpPZuJqt_1|SzQgzZ=v<0LV(ESAC`hPNfGU&UW{)^;B_GWG0hpDQl$Hsp-z?I$D>5Yd1XhWcHR`VgYp@Xn zk{;{I)Uw@oHewdpP0Q~&^)%E35SoY5Q{-ew6?`_INe{58|Op zoC84Sxix{P~_(F8pRv^3sp7G(?iQ`e_CR6L_70ITKFHS!g; zN1xp-vB3IRM$cG8G3vfseYTK;8$76ln>nSvd{d1@+XJ4eWjDl;#3&E(=@4u2^vQ~8 zaS68rsh5Xi<6^<-gI*!zh zcyr~ur?iyLmpfi#@ro)jDa&Vij1Zh!zwt=qZkO_%O3?%jBDW&0NT7#)JuZ^C~E#55Va?tG?0!btQ{hBh& z@VZ15rdDQXDLub4_;>ENjO@-R(I+11;Hn#%ZF|fzp18L3noYEW2x`Y8!;FL=%q&zZ zO1Tl{3qv3@pJRf$348~Jn7TEjlBv%Nr<;W?OoFa%hUoWpT5Nf07tQ+(>ocKVA4Eoy zEa_El@-GCXJ|+p|+Gt_2aTN!7#v5D#ei70+?XC}hj_ds9jsjKCOya5PeRoiX=7Gh6 zQX%Ef*f#%2FBOG~glcIOeX{dIa=>XOxy*{IXY5_K!*X#-Zs^k2@1$L4%+SDNP`l1c zULq`E=1Dvx1!W((!cRW)#`{`71?Pf_v1#$X>{MV zt(FFy(erHG1}3EDWjHJ%w&fUwmuz%bdM{&~N3$dI3q zL9?cUBnUZ`Kzkr49mh2dG@VkD^ktfc0~3_J92279%)!CPB*%d}R^P|2<<7KLaSBjG6Q z({wvi2dp|T>Cw11!HMY>ByRg9mhs8z`ls*E>NGR>?wRTG-keLSF!OWE^YEd=>C^}m z)JA8X78)=a_2lHV1an$g^dn;%lYd2DQi?<}>LuL*GCyCm@6E^$;$%x}c9#gcxci1$ zySoW-AMWPXa1*0q*eGbD7Z!caoXi#cVYd(b)cV_Bpy|kCYhuqF*T-|%n)KBN(k;1f z>2HyzvP-=LAUJ1mcNiY3_rv|xAwy_Kv?^>UMPtwZiyViM6L;hMa5_4>rhJO8_toI~ z>=2Fn_EI37AGxMV*Kl!6;+yW|1`kBGd93}|6~pUPWf?T@^c?JR3LjQrfQ>+?s9B~0 zt0;Z#*3=fQIjynUF{ftge1W)c-}E3grt0#6`1JgI+|4hz?ATE4_FAyyodT=+EOQyDPta7RWFg zc#(y!vuj)u-aJxcD78lp?(33*U}_r0*0UYhJCs~3NiX$OX-3^XNgzXaa@A|MPXdj? z-c)2YAd5Ls2}}tlno=pgG=LWsYmPHveUklMjDdYghfs33f*8g4hlszBa20*V?k=hm6gUD! zp#p&b25trKOwG?fwzt_{mCIBc-ZG4@t7jK}sYJ9JljI}3)S4~9KytnU>Lv{O9i6cv z9EExor=LDzS1x{)r;}~H5EZTcAA2~$q94M`7}eRHxD#f};OO)Gjy4$Nj!giVicTg% zcO%NZRv9dE6?qIbs#8$kW@@s;fDMRQJJz#&+%V4;xbzTcblZM4t^a+^xIdX9ms8eI z`OC%#jaabaD_}9vDz<^0v;*N}fSUW9Ao< zB)UI&0DQ`FfVK^qviPlPNlG;oz^a|x?sKWWC*%h8{CqG!zSf*y8%OW^-A_U{nv2{h z*MH(2HrPjKfq;!R~7>PSgON?Y{z?558u&k}yn(a!TW*;6ZI`P5v31kGC8|EG zKZFTgb9L`9ZByRk0dB)J3J_wf7axrC<>20rwy>voq9@m0Icav!ICm>Os+C zUQw0M=EuWqfvq&Ny-)JeS|X|vJW72$#6xtWM=ARm%pNWy0aGO(h;&EeX$%;6I)Z|7 z?#MM&U+x#q;p@@LX@o248oI&#u^km)QnRg@;1m?9nyWO6RI8TU3!$ZhMzp z-o0MI#$6LYWX~xGjfeuDq?ED{wbzjp0#~Dh!|R}Q{9fb_K(X7cj;$;3bdPYq9$vzU zu9d95tj{|^r%LXih^%7h`?iimKe1l{M%^A!`&_u;)D3tPK}RxVwnx70;8upi%D063 z3$EO1Gk&JvZMg=CKeE9@A4nP5kGuD3%Yvu$B9fKWd5Kcw2e6m7|Dl&XjL2hlXbT!&IMAUc(3kWg!(#LX#BdS0Orc< z(tEiS!v_Jxw}Chy zOrrfFaV;i1TxxJzkQxGl;KS@+z{F)+!~E=B5FDD$CZ*(|xmm(5q3*2V8Xj(V@OpX~ z_aEo_niA(o!@hVz5~~yG1xJ?k^QOZyCHC?|{Oy>`KiW-a#exfZ6WE!m^8gnlQaF^> zRcgLy@T@PE`|G*q6J4iIb2*{@<*b6t_dIWDC-(vEpFxQ5&nI_WYjJK=x`YusU6)M#<$bQMb*k&JWAZ)vns1KppLT36-g2 z7y|>yW|>4uUlWZfX)u} z0t5F<^7}$#2goA0432;%rxy%XSDt_pV#crd)L>wgioG}KE+I(azLw4mUam^FrAJKN zVVaaE1cDhf0n$#kp%m`+Z9L(Rha1(<-+s-x&PWa6ZiPQ;5zx^tk){=@Jh3Fr zMR{7{3puA^`btIySK?gbhT)o&DD}N7&(?+Ju+RE3-s8uaS06`W7oa-eX4%+!h)|7t z0pC!yJ}}qikcCm!G08HBK%*Zvt-z_T&a|>=xObA9z+V7$+9a`lUe-%oLPIxxESwvk z#Yo7Sw&B~J4m!nV;YAVhvb3o~pe~`P4_^9ZBz@Z`{Pc@N#pG@NE4WZp)L6iU`UR&{ z$mZx91Wk7vn2vg&R1iR3qxEsC3=%>?V07n4Gqt1-q31z_P?1*#9;e!kR0c|fc#YB4o;w4en`$&IgMS4&|xO=ikM-&-*C_x7m0>l` zuBqH9?X}n&m}8Q+j}>=Yd^28X%O}tD^hAx0(+^+YWUElH)C2d4GNQ&@;|F#d{h2$Z z0>sx-@S=BeO?}OYd2@wYXfAFOiH4r;hWPCOmPg`6N0V3vZ|wHydB;J5Na8hKUz$b4_<8Xt&}l`6MIIU0xwcgM zl9i4w+;)U8P}o3{ujpYhLI)SJ5m|@UDV;MFiZv|H0JoM%L;k!>XA4Y{W)dTU*z5OS zODQ!4@FDT^9gUYU##yLkoQ345(;W}Vc%24JeUkWO5n#nb7xxD+WnHru!2l7AtrWgF?^JR#nP+y|0(J!gW`yq?cx&L-F1=R?!JpJ?gV#&yF+kych}&s zND?f;gFAtsi(8NYd3@h{Rqw}C-I}Sox4UQR*4%TtPhS%{SGETPOgzkBrC8nWFRc-$ z=gOslroL$J#Y}d6*~OSY_dpQ7hP}^XugCSI<;$B2Qk@{))Vu7A`T@j8mcs4YfmSg?=$Ze5qG#h9oG-ZMA{y|m+)C{bE62T2^DP1O;4*6=W)2(G`wXXIAhLUL zzr@mf5w7~ZQENE5_w_p7iP3bA`;mJC-1oQdST0(V=K8k3(L7A}Hh@Cl9DusNwg(PW z<2%})yh;{=>q5|Au1QrJ3w@)^mBu6Ni)Gua7(u+feZo)1s#C#xfP_bnepYjxm7ZTl zIS-wFNpk)t6g^E+ zczy1KomEsMQn-BtAf9qkv%xg5Wm^T_8;PoqM@7j!b9UGC*itX2=@mA zzGd$(C@CQij*HuJRt4H!VLg8`QBR{&qgw%Wr&Vgw_zPZdU8bZu!MNz`hg_+u@i>p9 z1$huyt2w4yz)d9^;FOHyJPQF=yK}ytU5<|&G{&M!wP@m3K-y` zsp&iBlzvA32LU^@8yr+gzugtrb@}>~G*w=F{Cm-(wmk#6B7NLPP;Qcu8DAa!xZ}b- zeBft*^xp$xc_1k$&V76C{QcSz#w;74sNr}${TJ^QTrt*RdyT(HKxvW^vZ4 zLeaP{*n?Dpu{4H;Wq8`^>+N~{H#bvP9vi#TVb9(P%?Xh^Q6i3YsKb?pCaolf+k?sh zU3I-6iDs@zEoA&Ni=h%(2dSukY(jo;6bm}(ey>r=HzU>AhA=V>X09;h3{AzbG0j zwC~ouW`s1=7R$z|;xw?U>0g2802P_^?5}ts$E7rdLtGn`{#x8Pfj@U>~mt!)5QYr1yB#^|W=@G`V7mdVJm@4jFr!+)O&q_tEF& zt~40^O^abaDG#*-@tB~Y$L^eKklcAvxq|p;-Bitq(jRULM`S)KdnA)R1Al}*T&q4UjV9Kps)9d8oVq>S6H>8#fsX@QjZBYJ-1 z=+$Xf(W*hoSIDiOhk3g|wugf#lR`MBx2p z(lT{;y+^&$$;{`3iRS~`$y5kiO5rAAzkL(~;*D-l7eivbMkxE6zzM5ODN#oko z0Ozf;BUv0t5>4s{L0Exn&gSRE=Z;!>rTw`L%-s@sY{|w2^h|tHX&AqRHyDW7#H zW@>B1hzP7>c+3P>p=ukvEUkx!21=nr4zfu59XsKqdJNy^v+ze{U!|97R>|1=<4)Gp zqmb&ta$h=Kga)-!I5YQBKUQ_%hRi87i5FNDLRa~mGDI25A<=` z|9ms@OVlKiZa2MIg-KpT?)I)fhM#?++eF};)=wAO`zFofYsvMHkG!%}$tVkM4-Hia zxRXGvoh7oD9FqEo^s#u0NynI`434XwqbHhG7 z*#susT>6@z_xe^9TQ?uqj|%>{Vv}-Ag@nWYQ*CssSDhik(%mR$t2SD~)|sOcy9yOK zo%R)pM(1p(!4n+T@RKIshJG^IS?-Rm*@lBI&wZUdHR;$no2mQ6cHjF!XA*@!mp#Bg zmFXYAz`UCD0K1_6zMo{P$<>JbkUp}Ae$FJ49oHb0G@X*G_Q2~;h5uvF`*h*l-`FTV zN@YY-3x9oIuuB=5WfGnW9d(-kYU zP4kJ6`nz+sP0{KCZb{(Tu^dLUuA-03^>j^icqJo9NtFVg$)`N1`-KzB7XSP@T&ZbT zfuFyo-B5cgfmcp5)=820)SY?cQ{*xR1uc{~LNEMS^y5$%Q;}PFEkISKvt@RTnaQ$q zW7)#YsR!2Gi_!*7O$c+p48yv8bW_!Yg@)6kCR)PeL2DFcut>SE0xWS8HTW{DK;f;} zaGxjBGn!+_lUptlK$;=-CvJ;t`A&bWjwV-*3Tm~2QU2SH3J;IFfs$tKo#*1gyuc`8 z6uREdo&Qdp%$J;m`()iQJDMqCf?Jnyk(~BR(b$=IPjaGDa%~4I_hWBo`=5mSMK8?~ zLB~Y{a;`ywd@|F+lN?RA8<(3)Nu1_1-GI?#rB)u zM;D?w%ylo;LIsbazUaAYsKwl;v~;>)vZQH^J1cn>w^+zNx&>P+wU(*a&St0uYVrOP zrBN9t+_3wPWAAMqPc6^8BKheAp`HWX8BppR;P7J!0s-kTELrw#FD>dyXsi#;RUmW) z56~g|1^A)-?5*3MJE?=!^CxGA9dFz_n@2z!V_DCNE}ZbZV9oP`S@%rRODjz^G(&lW^#I4k{OWh8vK-kX_|kRKjm#?C7qzW$38ToJn&dET;U0A zvq;u4ks%$it~y#aK|sFVD9Oy-Xd%52Ng|0Vx_q-7H&U0MRV||yG&7?_jaGrgL36PR z@@{A5OnAs6bD|jTEbAzT&`39`JgMy6NX|VpMc}D_3h})lcli*D07A)@O`7U|hqxpt z7hqIF!?u%~5Gsz{>%c><^hdc<)oVJ$GFYfHR)d`9?$d-ntjT^s?ob+E+@LGIn?^%t z7qlVhA0*uY{B{e%&18b!d#H$gZ@PCqwL&c963^VVux-puva5~zShbltdxk+o!-%+Q zn)zO#ELVvJUlzHP$m8k{NRg_xD#vP*;zE? z*Yhw?V1DNX1l>@wG8@ZT>1e9OW@TJ1e{5RQsDuM09wSE%yKtv#_~@KEXB2SM`;Sjn z5@;*Ir!+>`5?g;D(!{81bC}CWOYuxgFS1BfjW{m|@%^v_)&1IzM9>N${7eymC)Cm9 z9h@Q6-JtG!40W@SPc^caT(F&5yu9n!ofYFrp+OvmgAWh8)6{7=k@SAqt4_C_)_c>; zV3QvvFeXnFQKXiw7m*`9nVEf=!H*dFja&8vKYC7YM^|22&K**%=Y(&SU;Ak8NPYi@ zIG+{UD{r5$_;fkoW1ZL|g3MvIXs%8ER#udeF=Co8#bdARMFK0|{?efd=D5!MIpc}* z3%i{`1jvVd#pIZK&hGq+XzhgKdHi@~zX|TGh-sh3^wpy1gMmzn%tUK~N1qdsZKx663hpf7xML12!gRD8UCIwZZ& z0LJFYR!6JS4|3GTSJcXGUBQ91Dz-D@jV)SyMaw|FiIv@jDD5j4hjFh?4S`GSYSH7G ztbHMHLjW2NQSTJ$mQnDxwU?9=rl(bdOr~MuE7ow6`WJ+1ecJB!-w2}Y{s=n!rb}B}i2QN<ot_Fmi5HSU5dq?r$oI2b-Jr4Gb$h}O^CUdY}PnZt!hgpT&>wc(}yDI)ggmI=>d%l3ULi8h%rMobh5)%z}Vbq(uMBdnD=-RXEP0ZPXI%dK9cTW~_dT zAK>XQ-k>R*FK`tzvMEp&0g{@qDt&6?k zty-ZzI}m&kcPVS1OgT5!P0_^7G8QgmB&Zp&|7p^Uv(nQB>=I@VTK5(=QY(NxqWxaj zCvq|73S&UyCurtPSv$$Iu2(=0!N50-#w%;%g*p-YU}TyiuPTAm90B;e((}tbC+|K8 zipjG?G4kiG`H~WlZ>m-@UBnjQ@T;JWV+CRL&fu_Yg)lZfIji!8H^HY@+q-RjSyjEj z+Nw+era88`%jWgM=+5H=l;*RomYQ~DEG#2`YssmdhmF~Gu( z9eG>PUpCkG9!HuKm;d-AAdNm7F8Tm zk^%8wOHtURf|_@jtvV~$iDZbR^kQk~tY)8O9pl>bn*Ne3MLdJHClV=d?S(_J&1zJQ zYdg`s=s`4o2lRfaU6seaj7PlXxb}|wN^{rCVVcb#Pvck`H7hMS~?>7ECZ3b$*S%N0~|xmCItq}+x>-e znI_x@hA+QG|B!fcl(O>cJxeT`vojnuNzKSB+*=VdDh{m~p|T?Sp|I!MX98P2Nh+)- zH?=-$v0oDuo7Hx`11^|Xp@}Y@I;ApqR%HFG3uSA|TDJD}qGd=;O?WZK?%$nU(_F4n z6qA?tg#!CB;d3YFeX;O!wOrhD$TRJE8G#LZ&90nSaz+RY`=X0_b4fsQ-rbv>DXW~B zapQQ2;uo)ULm3xfEv~_hKjjE~M$TroMuoHJi%VgRbc&5HbEbpw!7nN;H3Et0Y$MiA zR}{*9h!>MuLED}_i{h8UTLkYVyS&F_eX1>XFnkSvQL~kB*@{_W?+>qL%^NHe#U&VL zT^fPal1O0=L0z4oyk9SmmrMCnJ&oG6nlXKufpnu@xLqdeV)tn#wPNkDQsJ<3kyux^ z*7Na^H-m~&M9b4dKef8xlFe#(qWoL>YYh?_eV{{C&5w<-SL1E!%ubDP9$*?86B`e5 z8WSjjGX)JK)_tChcCBtkJvg^RS}g=la+k0Dx-9XXfVH>X;4-ZvQsZbrDa}!)M%ZPj?Ue~g)G+e-lBqb} zKWQeqad*yn|K91a!*~dfa&yzS&#h1bGgiVYm}5)s;^aY*G&p_pWj*Tr#N0qJCzPN~ zhppjqh2=0tw6MgE+KcZ5f-Qy_UE@ZxIC(07jE@Z}hy6AzNtl+u1`e0Fj{10$Qxxwq z5fXH>e9n4kk`+AYzu3@kpx6Kh8d-o0;YwaB4g_849q$f*rZZyc%Grk9jO$k9&=+oZ zmgQ(;1F61fd$1P0UG?;n+4(O3HQ_sF^0H^pTU@b43 zXfNnDEfy>>E&H!kYwL;ki98w6{iq$*Ae zxIAa*{p8Hn7&LKvT2544@J>lDGA*x$%BZ!$*?YFlXSakGfNnh)4ACMG7C~#Tl#wgK z6DK_OAlPat7-9PSV1)~%7o`?2?<<`1#otg!r~({@rmhw;uflP8XuH%TeKY0QU^KN> zx&CDwd0yC7E#BdqzN?nmYEbd_JCXFb-+jp9si(GQp1S=^jo_PR?gjUQy{2mJlaDA{ zXH2S=Z1ydX#)b{0x%Bjl&w2&}7yRJ<{`8*=!aaEb<-D^(dJPoyELD|xvFI6atV7{(xj8NgDt(=d7TUwI+h=K z^?3(BjZsee@ojew%=9eX^K=E6N5eTRSU*_ge$`x`2x#aN!BZu$G)R@?5Ppbr@eJX0 zS1|ub$lc=c(FcMC_R3vAjykL~Y}I`tNj2X1(8LsE*PDhDAw2oTZH0kw!_?~07a+=Z z^l3W3OTyai!$3v-F1A3%m)kOzwB&D1Ht#j&?IrJ=r1mjC3w5j)N8Y_$3>F!uSHOGF z$xbyRDArpG94bvMnkC|e;A(P{1{3V35EhA2GW@35^LveEK1{Lm@(wU^aCT1<4LL%UaQTiUlCn+D;kPe2XjJFKV442RD;u1gE3GYQ{2LWt%X}_x zFT9Q_3|i5YkhMz(IyS}pKfs*di^bL+9w48gwxA++`S^+`5`=8of-K#OJidNwd6250Y(g2V4tI6>(6s&HbUL51 zPa;3fg-y-{5n)|C*#>iK?U#Inp!`oeMrH#zx!6!9i$2EdJLsFzN4#SX08_4rKh7oE z1A6S{gPoU>J6)YE`nFkc?2#ZT84x?;SEF5ubuiClC3|>o}iA2F74ox?_#V#2e}u1gP_e zubt4Q z{AzsUP#PCY0ZAsSstO{ENi3`CJI01bAZSPd`Ni#Fr}Y}JW!A@-RO4!EMQP9TrZq1s z)sDA{UqWls(hM)5dLH!WQE1k=j1)trHDS#5(Z<<*P~}>WA2=CELpc_hB>C>++X1d> zhaUo>V-ZPOV$>nJ{U&2!rG10(n733A)%b(pV{La-o{VG0JV(+>loy{;*bQs! z6PO5Y8qb(E7dqoSKU$sGU5{s- zu^OVLVPHqAu*?Yz=e1qGsh${)ABES#q*TVJV^CcMO5u1=X$m_y-(who@r&cy>O8`& zo_g{ZY@Rwv(rf88Nz;&D#ysN$t=Pbh2G?~3+J2wX{y^CbJMeHYXyN#gTbMHb^g7DV zmK7Q^xTX{A3gqMFrN9#nu8e6@k{JiP=stRy0j?o{^b4vf|?tbz+%_C{vPXbAyKqm zOG$<4>A-?tFkNPZWn>!Mlhh%{**M+L+L7lQ{?)w8oNNwDeo}ZtGGZ znO{?VK$&BeqJ)%%;syh;8|4E8D(Lb)myHw>41Ve)Z@|hC%WpwYI#{Fdz@K&2v@9T= z6**H+qhhJdGS$ls7@B4QqUZ7V@IXp9NH<_YC8fMNu}aI!A)S+gg(K_Sil>tU4e2W` z0az=PO>?N+BL3|^V%J6cb0v=SHE(VX`wrkfvgcdbD6Gl*=1{WBtCQSpct7B<(kVUD zsLx|zbEf0cxTF)p#5v8q;zBJtBG%2cmf}NmloDFu1@z)^UDHyRyJN+VBWfC_Hq|!? zQ-f)rg5UCm&$;qK`MVn*mbl{GE)iCqGIFEIZP#oLiYCT55 z0=*}Fm9w)Sy{KYCK^3;bm<>jq=bkpBId3uAE&F?tT0@1E4+! zqM?}o?U{+mQ;Gq9h)(t<|6W4g3ti*q-;60Ueju%n3 zrpq}Cg|#13a@4ksE0FM-G9B?ulupXupOgkT8ChsjYAUU?6^xQe7>7ABs-Av!DO@K$ zX{ReCv;|QOd@73uPbiCJiC2a|JlLIxP5?Cr?r+@XE8(4DhGG_6l?~78U=5Tyt2>b_ zE48ji=F=nVhHNXx2C*lhqf%5VX>6WzwoZH9&HSRO?8gK5F(kwnM@G^N1*#()BK#q- zd8%#J)t)th$$x;JRN-MM_$rY{6!a@Y?GNPP(zz)-dA8(cYD@s#G&#~8WP$ml!!d_V zp$_xU;u{PtIB8mRr82$|@RpurIGDVi=LXkTtV(F`Qd1D6BoAoL02^K_a=g1pEkdfE-r0vgaACT?f*O}LZ-?tAByspr-t$>=G&js}R{Jkj$ko*C^2?Oy@Kq$osO=I` z$VSB&`evd(^Eu!3);v@!#(@)kgxYIFX-13(_QU(Zp@$4`SfL{RtIdwz>0rjlDZj7a z?RGNnP@*uOFU&_Eqa_ae?H*HDCH7_k}X0xC6&MPTJ^;v|DsnCcQP4{ zs32B4k%@#CQVQ%5iS*2_tLPU>*Q!;tT%)Ud!(hm<>Z^7&Ji}DwK+M*VyZE73(=~;D zxx6`^V^Jz0-8DrgA8-Z`}v9;(32_)Dii*}p6I@zC!=5Lh;w;V51 z-cq7!VOZ~u+i@H?kVF}Mni)Vkjtk`d+dI|tTGFHGM_AEbG?^Q{3Q~;p_ILZ!r#^|_ zkW<{H5B1uXe%s!C<8JsJv*?54Qc*_B5T-Io(NCUVMT$BKuZ=E0NN0W#cMjV!VuUJp zXVfStR)4PLVkP1_1}Cc26t$fOIPnZi_CU`5>KpX}AaNGZz4MJ7`M=1=tjv(~3X*5J z^HPelmiin09AvMJYE6lf1(JM=#U3;zUL^`PXdt}t2lnnZ3|6{1^tf0apvS3K(}w() z+PZMX=w2lPCK}t31u3ZSgXN?R*vonbi;+VHHb3|)c022=3EaQ_UF)PBgG?>%U?AqXXjTSM zyhk1=wFSQ|n%H!d_y!*Y6^yl$3Lu`f`zrWL65lvUo81$92j&3p$2v014?s zHg09`6uc^^EuD0}q@aD*s)YTzY-F^0R*c3mb><=2?Wl@<23MSAQIztBN=@(*rg;nG zd%q`-Q|M+v)3pKa5BYYTrSB@_Y+z)PV2Os5qwq8;w__O;wd7!P?>4Z^YU(`3hT6Iz zfD+I1Mow7NDv|J7@-RpHsAXIOjot*4CUQ%N=bexU&XuZ-i#XZ{oi;O)gqJY9v=X4w z%7!XHLK--gv8~_wgVv0mWJvZoC&wq4IR-q933zFEmG-PT6HZi(`GpHR_|@7OHI;lKw?CSmzly`{ztfb`5v*e>W6nllI^AmvIG9==q8m@E0X9%(G$i4# zQG6N8+4GE;6}*iG2_7EFO={0@Vzx*7cHnk^HACqFr7x(PgA5}?gY+KPWGpk11+ zOVP+VI84^+@Uw%&{hm}hpy=vutYqs=B`7j!+7DYamP}LRD47H+9n$Ff2!=*R)}nOP zXn0~~`dv1z8%KtbqiwIgC9pK(GBWk3Qzq5()En&QwuZ`dWkhuQRY;v;M&t1x!4p_c z8Q)*ubtOv0P1*T|3VQc6@0`)i#i+^n?rX_dK-Zci%k6zt1(Ohh$(F)^Hw=GrV`EEI zoU^?MmMS&ya&dT*o%)QxGUn(A+1&1FDL(LZR##mN9JWH)e=``x&!T#L@<-hPzIw<8 zf3lCW-Hg(wpC_R3Y9f;wi5AR0pPJ-4jwU5usCQrYGvjS0u#t1yM4OQ

    hI{9ug1STecr)d>z@#|5=h|-~sY+}0MgTQUxD^C!9Jpa*{UKPBNHQrOHrG@2;q(>rz-(ZVv%m}j=?+Q zhL@S{nI&387fW?u@kB3$*xBAt?KVZxGV_;L+Kr0ORIUm1QUc%Ew%uPjM1FH*`Qn`{ zZd5lQ4KXRXEU6JZWgx4H%BU%;q9hs?26|s_kWzvX4j)NEbJv|Jh!}zF_%l7-tXP^f z!r+NIM~WL!Mv}rcsSbDYx`GkO0R^IiSCCP>FFf=uJPjk2LO?7k(dbuC)k(xn43kO4 zJ822?G<>%8Se>v*utuRuR0Gb+JX z7%ZdXl+nT-YC`FHcFtYVa36@_P+4hNxhnM-8A`AUHWw;EG7xhPE(Se_g0?txBhNFX z7MM`&CY-w5CtR|bNMnmWNla`r)Mg>D`@Q7UtLkvvui#i8h1EOILB)GuzQq+m3!Scm z^BZBURcgm{eK-|~)`C(^ki>^-<}Y;vVFvzakT5$nm;FqK8{B)LxmvZbK|!H*Er>A) zKB+Q4`Z5khWlNJOX=(_V{lPb~YLz9tLB2z5C5w z%qO{XYSa=@%3ar!er%s~)xT>IGf882SG0;Lds!ZY@B!+uQuUELxsmkRxU}*3}9T!?Ucfz@=7uz9;4#(pT z)CRDDer-BeGSYnEu_7bRakjFd&3tPUwT`^+#iBjKabELvec65Yi}83pb1d^)6QT*9 zKsWaf1QnxdWfzrtOX-)96+#`u6_nl=PBl zK5lpu5~?>`H@2(2KPD)Cg1%4zV{O$&!ay8BMXP$;IN!x91a+CeQ>Li}OZ}8m@Hfv6 zjHEGA{eq^4%dG2Rnr(A%1AF~3?U8Ig766}bvF7!D+9$Oww8|9$y%_#(I8#%OuB?QF zTG_i#=EATvGM3UB1b2o=2posECe$3SstOf2qztn3HgT;+ zaUakG;O&oAvR-n84Q5i-3uf@ov-mk$XXBjRzP$VctbD*Hib(;f)e8S$JH`5hHxnAg zz}^gL#F0vla`(A7bpeW`8iFnI&76!bPuCm1C^xyCS$ixgx-V>jc!`Fq8>J$|7J}NM z>0lYUW$6#z-zPl!?dIzc2ukvXe%N6CxjcRz?R^AQt<6XyisnSV>(ox-<2o@>=s<#( z+Ub(CECj@eTYf&V0U@>Ld*3p((5O8SJejaNw|UB!vvuh(93XlLj(k*C5&1KVsDj#QEse_En0z@Wi7!o}Xfnif{A{~o%4vL@ z$0)nBCgEnhz6D8)Fa09{e?mL!joBcmGY| z1-#b=tKw?eoBHXUXWC}KPwMMd#eDrg{$vVfBcw4a3wTCuUn7C-T-(fA=sLS!7dN8 zszT!9k;+G0&&3E^D+OqwfyyB^<jCo_h* zF4X}w?nis%72rw4f@ z`boF-twziX9j7kghP)u#DFqH7w_tUiQK4;v>^2dB@5ZKH`AsROorT`Z@k=+M6QdOi zm_VAG@RT3RGc?p96csl^=$wT;P=>|6t#&9Dukd1L1L+MA$>o#C+S#SG-$oP}ru(Im zQl8~^Lkx8OLWdeR;y8VGd5|a+Z=Ozimd^RX>2!bg(J34hj)V?MJr3wDgJY%nlCyim zi#{%;`jhwFM5yl7KY(6^*vaMj7tk+XD&&H?DTQBYLeoP(yZcueAw)T%N$SA|MIj8? zd=WA`gvKbg->O`HicV_MCFf~Z!$}0ut`U> z>6#Anb_zkB!$X$s@;m1DatZAqq}Udls@UDUJIB|v`AL>pbZ85mue&mMrn$-UXB>$E z>jX@tlr#md8%97|YE?snT?N}dZ`%@ZdOs=S^Le^?-_bBxsFyB}Su9%YbUCmVEPM9R zU0*P;`rXM^>g-#@IPw^Cl-lV+y^BJ=Z8#gxf)WdAc3V@c)OVl!JO_~07a-`;Iq;4{ ztj91n-!|$6<=7#rOQ{?34~TEF(@*3MP8jU#i_L+<-a7q^^6*TFbzPjxS48U3a0QA8 zDBxTB2CCJG*)7N*`U=5yC`yS$K9O_9F<9%&x$D4V;v(n!sqaRP@UBkPT;~@|QEMv7 zZ}qpt3=Aqi@&fVfDEA(+wRBiw*Mit}BD}m6cjOihwe-Wjj^~1p8Pym&5NpYN6gCRI zim|mxG+BRnt9r`|h)#f=?Rj|>R*H9xo9;6wmCmBuVusCB`@}Q1G+OK9UK!&*u!2+4 z%Ifc(t9QFo0cp0()KBjNgI)G-B2pkS^n{@da%kM~31M3sqBSGWcI42`hh53-p{y{~ zlS;tDa>d1!-dhKR-j1e)wW)DN9lFM#scrOL;z!{MJ;Jcxz1$i%@v3`fs`WXJMR#p<*kO)eL*MlIO`;N73Us??|){yAAy?fn*0OchTMxM3MJRqZzwy&5Pa7wp0Xs6 z3$c%#dFI6KYHBFMwIoy-+;@|Jsx|x9f_khN!7A~scx46IFN~kbvzcGk2X{G(uVlr= zMGUEq&+l<{j7KQzv$aGTc4z(pc6Wno$2oyCt0Vsaf4bA(S`T{ut(6D$*MfDW93Yug z$~_K@Ef3jz6%SnwD39^U&$-8Q_)j}8{i9AYO^e^q@AOB$^8Ev34&Mc3qxuUbo#@w8 z{{uu0zi5G~&Q|{cyw5Lgh~Hk8|3wt5AbVsj2+^bv$XYCUo0@5$rCPOgIW~TkCyJPI~boZu^87(gP<@3_K1misQ2=tZi4GVPbUnpyyB;)|&&}ml& zzw1+b@G}t?*m0dXJ4dg@q<u z#n%-CKS1}=OvSO?(NFzU{|9PQU|Vyf8&rt&w>#ifBY~6Uc<0lAwIkcO>P=MP% zTvL5~R}w=__jkw8HrP7GiPdLP1WuH}jTv-@hJaYpaW|O3b(!*$?f{ ziLIy}qw6FXS_%(YXa{ikZ}$7ZekaF_=_Bp5rW7wO=zT_K%*<43i<|jttiC8nu%$ZNS_p}#lvaWeWJ>Xa=dk1` z@_0vjLni+>J`P6c@ejjuSZ_Tqenq2Hn=~$a))&X)e}mCfNOSpD z4pom>iifk6<&a;WG>J5yf3{VZ@VPjD*aj~~E54Te1B|HBWFOguz>lX%^dqL-@1#7T zk;0ARnaK%<`Qs+C&rz-1&Y^x1-uYF!6RG}Q{kr(it~6~Sxqv1g)22Qn|>VBlDJT%WozN>Ie6^0|Fjs7yDm*JFIV zW>;mAH~m9q1wf@ab7`wlCz>qr3YEmk775gj&>9o4`zCj?1ksV@!uq|mBj{k|Mhev> zLS^ol9|@~r)P5nxQC}m78pU8k-aX>gOB8~jo4o-$ZcB}QQ_=#$G#6LFB#5U@6c?-jo~ z86|VPe^~D->9Z06C_2AR9tzgnAlLJ*1?x=7^-}Tx6}qV5x9+czX$~qV$OM57{5#jnS?YQThI$&e{pPQTXZmld z^Xd<+_Am5bXj#co-b;DOc#*i*(@jX?*XL?7r6I%>(X1Tyq<9U%FTirVIE{eXO5Qlv z!F1m#I-dDPml-m0y*j8yX&rQ3rQ^pJS{`4WFuy1HWCjLtT3%H*UUXvzEyDld!v6<& zewJE){X_Y9ZG*eO*ep8m)Upy^2iB+^GDc)~08vShbMwRad`5xGe<fuC+M4L|Cz!l

    ML=hxeEGXSPrK{{Ux~X1}lpo zZ>F!-v85?ug&jr;uiSLIYGf-!>V(oK4OD9Q>ha;M>a1dBhlwt=e^^j>Rz*Sp!t2u_ zfv0eR7w}H4l7^qy?-}hiMAQEMznfqB0PEeAdMO^)A1bP$)S7jYKUsW3vq6E_7~ECH z?{Lmg!iVD;0*r>{XgTfyB^5X!@K9l8^Nmf>?vXb^<1U5PJYFjJ6wXlb8V5PWS=qYk zdIhr_h_!%rI3bNR-~PxN5K(AvzzvpNLVyR)pvf#UT@*Au(R^k2re48FJ4}ks!mTDh vUPhE7=@I7Fa~8;{GDiUCY+k7~yNUj79>-zL1#%`Jdi-O$eQQ?o-|GJWb@b;V literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000006_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000006_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..56291b08dcd11a6e4d46e2b56b0c517fa290c0f8 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@be#1xNQZp)!FVsfmMneF92#n?* zNci9i2mGNoDmxki0~G>OfHi0jYi@pNaVpkomUIB~WOK6nD4c?ogaAp68YI ze&60}?>}qpV^5AFlY8dQ3B-(# zgN>VwlY;~BX9@5gfQpX(3jG!4-w6jB=YJ+VJUl`|!k2d(96US%0zyI}BBGaqm+Z}( zHxv|PWVEz&EG*oAdH@{&06YTRO9K2mAs`{b!y%)f!U6tU#S0t+L?mP!02~550s<;B zDgqJ`3Oowj3rbvS1Uy7e300H)czmQlnltAFaNV~FuJx(cYUjuVlIpaklP-yb+(|)n z?+faGG)NWlxPEsF-Z0~xhMLpk0^s2Ri0~-L$nbw*c>w~43(xt|qUsrbokTujpb0pB zf`-8P+xj^dlA5Vx5N$y)_j{K=-2hxvI3M@`1UMQ1JT3qM7w~5hfPnz_a`Ez63~-mg zT7*L25k@FPb@lJmenYVpp?(bD?m2MEw#p*cvmBB)xs~*Zz#ly+72rw-~OZJ zhZJwx)Y0uZ+XMWIEat#)WF}~K{7};)aC==?_uO_wV`ipIdox5hYTQ^*+YCqH^yBhw1) zABjU#=O4figt~oXIsVZ9K%qxzv&{Wp3WyN|Km66_(S+UV8o8#rfd3DxuUBLpdzZg@ z@dugsmoz5hpvkvu2B-;ogc(z4y?|tyWbdT^CRv2Ao%z(ndv1NM&quE>7gcrkL)vGX#C1%}( zdboHeBAN0@d1DH-|K$Cibv+3-%Wtn=UsOD*Aq_k+R(5A>=NRviNJP48v}~_+_gykT z+mA>ND5D2a^5Llypt#{+CDXQ>1bChNZK{U9n`58!Gcribc&(Az;{2fpG|g!uCm{gF zc9Wkz`KTj(@;>UiehM#H@7xqAFb0+u{<6$%2ro-Rl)ftZE- zo(S-K{!bQH41dm#816eU<61GM2sks$(sYP}ra1y-DWA3CWDnU$TiA8$97lStJ~S6$ zScvC?0MH&gIl_PF^xr=ITb9v^#Y|*;lc_VI!3(Hvro>A^Oh!_3tqGphJ)UHz*2GM7 zW`GXOh(#p+K;I{sYz>L2Z)+Sfc+o84jja1e{aU>I77g3yk!j~x0aV_UU zzE0BXR7=~_aw=|e1}S=PVh@_D<8%xA{~aWBCiSmp_K7@%qncm`GWrJJgP^6$Kzfio zjSLQzzz*OaKK>8ku-Y4A{3nUCeg8kNuB?%3Ts{q!h*U8%68K~WIJ|n}*bcLtmmFtX z$4NH*?rm)`>>E4w^jmAP?b`lYk^_vq)UuF1tes)yD@FeR zSoRI>i`PCw3JVL|y?Ey>uUD!%7hwECF!P%1$BDENZH)>|H8Wx$xXx4)znbE})Y~Y^ z@AAmf|2C?o$m1Z#ulG9O4`A5h-HESSG$m@I=8B+$;_i~Dy~pf6+2Yh#K6diq(5%BN z57x+5DSGV)1?Q7;FESY?eh(oJs6J`|m`Bd9=NWVe_Xi*be8EyOamVr(F5tfYeX-W} zW5<~S<(AyQ5nUVb=m5P{_^-65oB5pDXWGAjm(2QNBQ+ZJ~)$E$m@-o*e^{U!e&!TH^;i3F{==Rvd(0#Kn zLwjGe*R#o#&4b*9y}!%@%B)z=wg^j?=2)$P**|e%ru;J8^4X5vTQhF&zl|z+Auav) zt4jZ{!B1L#Wto(K<=CvcC(dJmAK3L z`;y2(ugwAdduAw-X>zv(FE5VA`;&p#Z#-I)7W zev}!}T$BYoB{gC^hC1S4%|&LLWV8|xx=2=E9T2PO1;{WnLPi9{oL@l7KAQoR4`9~{ zFH9KqSiCTy-m51%dVP+juS^!xGN#w&Y!|9>#U$Gb2Mlnrgm9M+dqfrZLTCOdN>Q#HL|?!}Ym zErU3@S9@?fXjZTGB(L{y=n>aWzX8Ez*zvvy?EOFJS@;9^?)w5x#s4p|)+#KQKH@id z7WVuBr0t5S-RC4-K@pQp6xBY{aIB9zy1j022I-a;F_p;x63R2+$j0V({DD>%72jd$ zpTd!~P+mrYnFs!?hvhD;i|4AN4!+I=+vt*muXI=KcBHgp+QX}ZD@67c@QGB+c%?m5 z6XMaN$L^nPf7au=2qFPBVPyiKXQy-qtEeC85Bx7OY$7Oy>M5 zj#gby-+ub=BI}n1wqxb}U;Kdg51{H{+-=cslIo&uc~qey-w>x3J^`FeZ)l@JyZZZU z{iwdd4D&g|fgtao8WKe*u?g)%?ap6axaj|vPibGJUfbU*`XBmTy>z89%;Gx6cB6(> z%R(!Gb>#$VDi?r+ugE9is`crNof!s{-Z!WY3teGpj&Fn#Y|#AyfTKkJ4QZ^wYpkPg zpZ#F*uLgbBaP}KJtLy2>t+J#?;|xySY?{=e(0oB8x?&6`8c!JH(=MmIWl|APtLBvi zzff&VREEh99Eq?n$n*a_#DBF&v8!Xn#(a?Mndk)sJ(QKp1bBGYJ#j2}RJpbsm(MKC z$8}JfVb*Gd{47-h7B{e|{a@J|gU&s-tej(IepozYvT7gE$(<_rX#llIKx)&Z-@sj^ zgHwg)=hZ*-x$eyeb?VvnrGfA`?+4*H3U~kCniFoHEvf>kCNm{7?b96=ZeEN1%hC=1 z69>b;Og^Wxo%W>sS9JT^5`PJHh$7*+Dp$Vd$qBYW&X?LW>t!jX_S@;*O^OKplb-0m zxtX;41JFCU{i0ZO?4F|&JN@)8Y5dy`3s)I`0J9HYsi$nb4Xjek{x?I}T^RmnQ2tLk zg<&so`(4&c+-EOxeevY)_A$2p1JO|{(M$H1k)+u->z_{X zw-Enn<%XfU``Cu`v5uj?;$E&5_dlb&Xrj^mnn#bI1G`=2#gSj((T?jW*YitkwfaAc zhV#DW-)O#F=YV!KLfj(jo7hayz))=MglInJO6!>p(WcjjV4^ry1HN?~*Db#QGvX)L zfJ6H8s5fQ4sh@e_Xq`ni+~;>~z`8xx(bK8RN)W_U$!xnjHzUIQ>)=tw+T#2Y+H~RL zL!!4C5l&dJYg=kb0y=@HKCsa2;SZqDC zo@=>_%XRLX4#>{w5aBh3()|?5PpEvy9fWB zmRD%gCgLhkoSUpt?$4p7EDs7oBhEvY#U5O4oWm{^stv^@|jI zUd@}FSxfARHzU5q3b5>_^snUl4HQT|DDHSbxJ=Kf@A3mDPL#N~i`x1M`jM)sA-#&^ zYOb>V;GX?hUzNl(*gr%_b-v91K$&Qv{ZvIxUuaM^Ch{fM+tCHXjA9FcxUZYIj|b4l z8GvC%{2v*1rJ4ADE&1KxNDEb->wRGf6CJ(vA!QA<9vs9@|6yNu7?fZxo?G z-Cur{X2j27o5-9|05jt5e`Y9ieH>O6hxk88LO6*k8IGk1yEfM693aC?khYu9A{Lw~ zibhzqcA*g6nCS8RaP`1ycHyrXcQjBwbIO-F^g~{Tchjkzx!@4Jli(I7%UwdcjC3C} zjUt;M6w7ng?S>CS%*jgAr+x*aCPaxqQ6I`E`xbZ2yykvW8<}S14WL3RNs!R3L3T*#-{uboR zPR;g=g^Dd7d2Gv%D?7s18?Fa1u@+U0D6=apar@>bT@|nbZb|fry_tNenbMoXu93=D z5-OnBV+5yksDMJhwow<&N4n<;B?EmkN)chvWD2||*Pzv+Tx-rKUxpXc{WqKc!@{-5 zoY4TzD0{f5|D^){HvLQeUkZSWa(8`K@$LgN%v~G&Za6!0(LswpZq{lY4HE1l=aPr?=`pTFMf)ul6+ZDUt?&-S^uTSeK|f+|itsXH5+((ZCX@MQAK zLffNOMEj>3+odU&Twx9o5(|xZGNK@up%P0b1lG4mW31LK=!;n9V<+s#|Q za4>;@D%`QgjylIH=6JY0UVYkeQIsw7M3EaghBL}djx)-bTpmaj@jksSvsYSG@1|8# z_*B~E&@xsd{M>{eKUDJl=Z19J_A35I{j4BL%*GX+@v7D@!xrtdf+c<@NYj2P<`Na3 zhb>0x)iX-w?;Ecf87w~iF32SAA2)r0^P<4>c94(~T+6vU@ zKxePE5HUEcsT0FLvm0J)Fr7zJC{s_=%9tt2(iqSq0ZVg6dA|v!knRD3+DyWUzfoVf2Qq_srl(XW@O2wHDmf67YI_K50Yo-TnxnRiadj} zQ|&8CC;&wiLo0TwZ?-&hx9o}>nyPnld_P}L+CMZ=AoVayswaNg0R+kB98>lZWeyuF zHu&uTsA%zOkDm%M@Jsy?FD{}xa<$ipa3V{k;q$0PCRNN2)UDxGdi!!O>@*KxBs$G2 z4FgIbZp!!MC6kUuLl$pqIMcNy@e;)8;9KY8sY;Ik4(!_o9(RETb|$Z7G-oPIOl-I` zT*Lzn_kz$eyeYGp4SvL|YxLSN(}T7x@LRD-t@izt!_V(CMv5-xf;_Vcr!iq^a~iH*gmEpK^v`(vF&t;p(L#AR=cZ|(ZpQxL1K8{S^Mh9?ZP?}sn<^xs z^L=!iaX~$8wCI=VCZKzI2#=gzM`ot+iIbcCcEBmqz%u#nn0ewr1`wGu=GBXoI62&% zv|p9xL(*34XW^-|9OUvboyG}5QK|d&hj1_EzC;@txU2Qk@`FWU4!DLiJJ;;$T}q=H zI;UpJ8SQAams}RB(u9wdrM6P?$TRt)8xSXE%}5k2(Z210^YUr(tG9coJYucJ*TGGU zlJ5uT4(?_{gk4xyl`7CgCS+4MuL@p-PNnc~ZU1My{zurJe;Jhm{#}2CAp@Yx25Q3f zAAZVdCi(;T=^OsB`M!nM^|uM-+{)Hn8)fOh_8-6pJ2);-M7W<*5h28#cBGKf-|#VhtgaVxFQJ zzeTsps(cua)v{-s9*0brS}v$*e`;)1{a~-O+LnEH>tjH_rN{nOeE4#%}5t9VN>N|Q{!1H_nr_AQdf{4y_Z=?==dG8k6 z#?gcfP-VX87?3%54s$kxgmVUF#KY+M%Z^)F=d`$)=#Q^U_HET`nmSLCeM}1ue zwC{?h1*VKmi$>a3Y*2cy$RPw1o$W<$i%B}hemg`Q>{z?I>k7}O*_(zMb_;af^YH{l zK(;X|$=Sp)FhIa&@!pu^&ee&~LT;NBJJw!!DU}KW6pk%{)yfQTF2U9?rO{Psal_Lt zFUn#Wn)D{IbAvHyjOE)qON=O-!jq4GjfT4-3c*jqX`jzuIU}c28@wS=Zs-v*Bf!jR z+N3|A@|+91<*hzYRnw{B6W&qiSHR%LKR&)q>0!JfqMR~QdEK{g??c4QpG*0nRM0Jz zV$4M(`VU}aVR=+#Ng8S~YwehqF1C!awWOoEf~g}m(sq=NHVn0l@5tf@l$^kKox*!| zkve`7@hK}b7$q}OJ11!nS`$qpoKOpTRIV?%bdc9zH$ns}nl__!nbO;-$tz-KBSFQJHZp1Fl1kLyomqua|?<66^CxG5t#Ba1}# zRV?z0sKU4M!QImiSVNDSslz*5k^QM1Fk6bU80W^1T$JrUpPf5mPF%*@dH7i;d@Ok6 zkG@m-T6lEg(8M`M|LSg0xx;!E=pj8W&{F4-R=1`lJ zi;2l6-hnFQKY-wg#Z-MHOfG|H&EsFVba%~Pt-=dlT+i&~kpLS05+b5r#&wLBh@g^6 z20a~v7{zxI4@utSEGfzM zbBf1Rpd~m5Q|&$xw3lX)G0aH+8sEhK7Y`gSPa&i)W8nB}u3ya8c~U`*LBi8di#U;{ zI03oT6#c&(Sde;{GzkcY^_xbUzjh@9#dpQ9WOr|~$rbfjTOm8{HacqfGh>NG9{i?j zBhFOE@Fi*Tey_x4_lAmOxznrh@|}p$^%e2L0RzP+?EMv&hK6;fM^l(;Mk1>kGv|8B z?`2sH)N_%-Yd%;#*p5{i9NIA<1}Hdi#3O{)&|cn%aV!`zjabBg=9KP9%SKF=x)Y1r zy?@}&>4{U!RQ*yYpH2#j!BOauMufDoHU{y1-1ccXU(|Ja?i6`$IY#ZQYoxR$cTVg? zxwh6XrGqb|J5$7P4x2xPPgcT1MP`yIRC&fF-N?@LaPU>^U82edNuVQtX2zoO0J-BG zD&J{#L0+b>k+I?IAaCXNSIyG}0nlVjFh+cRfk#IPr^N@p-XU9UJTL(of*0Ut+2S)* zsD2AI3z$=7onFw37U(o=kto8#3~ajE#;F{O*pwsj~Y7N}dtn$0}E=iN+&D8STBKJKL1Qrqph;jcS@iD6zRZ z=YUg)Ec{fHGDXQhd5`vLZSc|rfemY~CZ<2q7V7*+N+6FMWaW>3UYuF9Ss03AGm89F ztbiP_F=xm#e-zF;)z4U0t_BP%F56t9jHchoK1VipTvEgH@M=aV#cqrdM)aY?gcKJb zCxW~^Ma_so&C}#Cng(6Wg5<&;+aS~*KeEu4NFi+qT+WQqEcIUELR1LUX^~7erADN)AQuYlqF3`tiQeKifrEWL2QRS+J1!kHZy8DQ7ubAu*F{Ae{ zE89&Ku*ZrsfvR<*`-j|~2j7kf0+smAHN|Om$!EdbVSe?OjXvBuUil-Qi{EW;yLKYA zH?Es#8g;#wH&y399MnXeEml=t-SP+D7{8yKDSuh!DErzak%t+W&aN!ZGx+BVF zLZs>{$q1R7oVH5TS4*<&RN3K-k->1E^F<|n%zo7tS{>Xj;A7Y{1-asOpfQNSTiIS( zlBIRx2%VCgDLvqsV&#nT3j8kV?oOB?>4sz|00YTs2hsPf3=z!mk~Vi*mtXkD>8g4W zseV*57^xW>kY7Gwf0#qQTrzzGi?;O6_ie{#gzz z&g&?IzY6}I68+cAGzkw)h%N*F$QU8=P+?>^1@v_2=}e17CaJcntEL@&I(+?Q$Ig== zD$kJiGg}-$(fqqQ8W&J|=5A=kJ*z84Q}=|hNIuA{6R4uCs)f3OTX!e;@)%@2A857#Hx;`v|gbn-B|lI?y=oY@uany+x8(#3OLx7{R>*g)Ho)x`A87qs`uSp+ zG!zvkYq7pKn1z%6*e72Mluy)I8n|j7;K(28KAX*N$i=(np;C7YK${|X6wYPDz!5J? z%5_nP99H}ku%o7> zQJEiew}>;8wEV1HR>TObR7uAr?e*g4*+}Zk(UqAh?ts+8BSWVP_-yhP@C_gr??<_5 z4UC7=sslW!xOV~t1L)EN$&|=>c<}40)MaYT2V!ozM`0N6KML>=p;H;fN}TfxHN>+wE6>GGm(bWXp$0 zY8ulZ*R?bT)z}@x`rtbf;PGwXev^v$-2wGTl8UPpM}_}5+MwGX{|Wr@RRA<8ixP;3 z+_OHOqltPHAXlftlbU#MpjQ?5afy67 z9T%VKSiV;d3jHl?tAbbLQyfWx{Rg0sG3Cq0@hY6#F$1exH-&-j;n7?FX^3yaesLaE zy{CMqk+kFb*Cw~-JvTBQ*bmG}ysCj?)2ClS&O9_71EdjIL-9nF%Q+W%lohH835z9* zxvsF>sYfKHL*P8BC?3q=(9kuW%yAv4X}TyhcZ&GJeeagUh!KiI(@q>Q1v*P}bF>&~ zM^bdsDsNT2PQpkchrzBSOAK zX!6V4rdA3ymyDc!ijBaO*}I-01fc}~=RtdHq|UlX7d~&(HO1`MPR9&P1qn7CI0=us z@v*1G5)Fsze+RbCRNmI zuW>P~Kgak%RQa8qkhy*nryz3#g}X<0hTYaPXUf>lTVnYIoV39{9lTu; z*U>?HrGVv}e-z7jXe2ERnpM-<4osX}NEGbj97iCYg$2SZ$GSRG4eQq@A65SWkhH>5 zG#rcb=QV9)LC$O$DMhvx*ZUQfzWpM3KZyI=(q0x!US@DFYe@eOS+oZm%_RB>E0f#| zNJWltRX42%`Kwsw?jHc>j)wm;X=@16i8c&5WSj}16}UR|Yy-0Y0}x!C^Y{cWDL3tR zApYy|`ti#b5BHTX_U6xh+2pZq&+d1LqLGJBOw!#BZC|!LupnbQ`k^z_ z=)JaTtIScTRPyq4q2%B3)v&wWPb_E6Dx;TP^@WCLwg-lc&zbFpMvj$?*69#dX12> za-`JlOqtfev}|3RfIuS@5AtZFW@CFN!DUm?WcQU$58SMxjOlx##THpcmlT4!XrMr@ z0WA_ntaa_Lbf!sI_ z8R647Ei9>G1v4P#py-@`H&XmghyeR>PSvQX|HwfAll6+<40Gq!_-^_x>oaF{N#Fd( zxs=yMnmJa!9=|yTkD0?Me0Hrx&nDFqg@os(9)Pct{w z{gFgtvy*$-7={?)vzuQY6wZpizv=`AlHsf$;ifH{p@v(cSH?hQpXo}d+P4FfdsY% zvxpNQ+i@$KPdj6cgtCZ>R}L%XSfy`!I>RT6lh6&?k)-n=?%9$AMK@t{hzp*PPp;Ck zii45y;tE&w)dRPrf}bX2dKn2N<*5qqZAv~uF_q%Td6&UwLGTJ`bI43su)~ec4<>{{qJUI--V!#fYCnyYLsSCC<%#;9MrBWzEaBx zGrBvrFs_g#+6Ga6%G~WV&bfz=CIy%lIbP*ZAuKdgaNc8Os5*fV>#xi)a{1eCVbqHt#%5D)IR-%64NmktJ9@146&S;=D*a(b9H`4UCD#U&tiE z`H?=#VV`77&rULi?Y7TK5+~%)NbI`0-M~7KG-lz;-EZNN)y#ODj`y#6SsDyN_;tHn z79z+!QPb!9QvkaScq>_}R&26fs-?(YSYMTw+&y+e8PT0QK5vi5(mxu78f?pDq33f* z#SEMiOw7)OM2n`nO?Q%&G%6)Vh)Ha_-r3J1FZRS=WU%aC;%yo2vb@V&aAi`()qYt8 zyq5((Th@aetQ>)_;sZemVX=CK9!J)mop-wUn+vPX2R05Pl+r%DUXl|ZUrs~@jt8?l zjVI=yvkV4#m%KV~DI-bWy4OqgTZBBUPB^oScV>1#E)T2-ne+#xRSmJi5lJTQUst3| zxvDOnw0F4D&jn&T098ZD*gjk}ZzS%FyL|RD!8(Td5=Tkh9bH(QeqWXRFdo)sp*bRJ z_u&dXtRlQ4^8J?j*)35KFTd%Be2pkoG{gh|Nuv0>`5SHlC0vGh@UuhqV{b<1q+UbN zAYDKsepL8;87^n7GSO+nAaGzuqfxuv{1laaedj=}G25rFf8C$4s-k3KGP-JtbIaX& z;vN0*^_=#?v##r_V6et374A;sG{2j2!Q(gQ6D!n2eIu#JSLdraRu?plW#W4~f(0A6 zvu`ZQ_kU!qSTy>gHN9hSqeX zD`u-%c~w9O1uUdkKV+|AhbeZ#BFciKZ|ibY_N*?$4j$V&8p}+Y z(+(Iio+L5Nf-cW! zKQ-`8iKHCsZf>OTjcyRw(i$Z;|F->EOsmKIE9imce2BZaUH((7{6ZDvcILD{cq56e zyxKS$?R--!CS&IfLVs-j-I8LSbG}w|;{1FTJ#Tn6tm)ECdP>l9Vm<;}jN)A;$xm=) zwS17Qh*GpCp@1C^R4e_Sg;~-eR8!6gB>!yQ(_HB`r=<1HBga?|3?UdjKbkmZQspIB z+S#erS9-@_+U`kKU2vi=Bu-F*wKDQEXYk?fm~}x%@?)%6%=kPgxlB|@4RMWl z5P;2|(mk)#_3ycc7qmC)wfC7YpP!z3>-Dz#Obs0~Zd|=45c+}2-ZexSDN|mN6(yB^ z=Tmk{=tYIkd@*-s4`ftM?Hdzim11-*?owKvMM>?LtW7GV(xia|$=v$+JNPuY>}g4X zB+94+hJW&%F#Fs+rx4rjv;`IPWIvw@kaQ$%VA?W+p`D2YpJ>xmrG2mlXRelyZoZn( zZ({1bp&se)3r%xYj;CoPJ*>|VrWFT@il$cCjh%VbhptA+gbIlm19wk6C74w4OJjkU zhXGU(go>yu;!` zIGShdr>(cI0``bIqK9)ND-}xcFl4op=WoL?&?-vPeW$6L8n z!>I3rk_eGuk(oo0*%S%V*bFz=;VPxbZu4i{EJ`@Z#9P~5i^VNa<4<@Oxen?1xf=?KpV)!@LfP!*CVxNsXu4ubYdolhB(jj7*t zFVb01kB8$19{2oux$)^z0PvsCQ3TpvE$6`ej55bt-zW^lk(7} zPiv?=*BSdGYF+Mr?oT-=n{8OTDIA|;FssZdVFOiEqdkx9<;%$AYA9A0ymIrj8q7B0 z`@Fh{hKv=BFFbecOPr`dNDQ+5^GA!P00oB~uU51)QlD5ejSdO%kLQJwrzejxEJkou z@2!i->K)(_Pw4CwrVKLg$ScB{q2rhnq}|FvkRbgNZpatPbZq+@^XYZ3Q;5tOjuE;v ztgaz5p>EKlg}L2c=_~Naq?*~~W`}C%fFp$6+_WCyqGxS?yrLp$lTZz zA{U3z*HtGKalH5(o-(#HM#V}bJwm6bwv(MTwjqvJx+F_)y8T?EqrFr~{D}SS=GN(r z6W03UVaWRLckVodwI9doZM)^hyYQ1AdkEo7efIpYT!FLA4O-FppCv>s5r=*09~S+D ze%SIFjPNJ%`~kcVTI8>Z->=#Mxu1F)j7{rB>x)oQ>P=NOa4%2LRnE_o^w}8CQvX;T z*}fQOH)yf1*hW{u?;rA-XyUJ8)mC#x=j0$ji@uuQbxe7qj5f^h;#745^D;s=sy2*s zOA9S^kHPQmA_cNu_MoeSt;EM!xoPWzWzeKtx6cSlE=9jI5ST&9KQOtn z1i5)<^}BmT)dXeFXlv&fhafZM1R9Ji85CkGGPbeekWiN}QoRyd{RA~=DNE66lq z3}J-qaRhEzVPLN_i)368)hx!hC0=YpZ*RuM3tAY@`OeKg+vG@R1de7Igd(xj^$(zE zlm*&`s!J3`NUR%Vu714iP3EuTp1u5ZUAZAi7!nC;)ePwl4#7c8V3g1zos{Zpkk8Jg z)bnX%(dG#z@xSu|j;nuz>4d2=%u!#)sfswh;x!syO#qlH{}Y{`}v%5bp%1>18R>fy#owU52*zNW|a z1mml}`92Br9|L`y0RuUY*t;L6Y3NE!gsft_z97@9DpB~&l_-|!8aDa2GyWW1v>oAb zm&Lv5Fcm1!09{;m!jYUs#P7-u_BZyw@hs(8{2bAjGusX3NTT7-Zs^cj-ir*#z$P<@ zk{VYT<2gMeWqzTp*F}f5IiAYI zq4wgxZ%S>D*K!wGBc+}TCl$t8<2T&NyQ3ssE=WE~k{`FwxP0W9UvxX8O&rM^AMMB* z`@C_uvebX(_C7^K<6^D2a4aRNF)uwMdNdv8B0GMS&@ggzd0~rO&w%gC1z*v&wC@lI zvmj$aQQJVSiAc!b^f|VkZ9m2{szl?U*Yy)Ta(vPypo)NVu1?5 z)1W7!zE$S9B{8-lufoxSWjw+PRg|W`tRu+~D97|Ael||K*xXrVO;8}l)otH`N1>HT z6JCHWCKIdv7LD`vdY}jo3-An|pU=Y)zw@MY`@AE2@fGp2j&*Y~VOkq7un4)<_~0Rp zT3$ypZJRHVyj-=sbM%ciRI?tUT(?hz8ezR@6RBwVB_CZUB->+O^M?s!8zd!7SmGhK zn>yr`OznaqVjhj1=300+&ZdqKZs;yx*RHSA2(+t5MGB_Yz=uM=MS$HaBooR?a|;{a zEj4K)nvkDIO*6;@?5Qhd>a+IJ_24mxni*Nu3Mwh$J2iIaFEYJsMT6o3?7{+QlGXcG z32F4@dPEFSdc6&LEUOC`YDK>4de!}`%tg?8E9=RWbFyD$r5*A!T=2;G(S1V{>*he! znZeSICV5eWoE(GRGI;)=p3tdz_0ktcN!1gO-^(G}!CeTJOIF4vaALsS9T=V`ll~sB zVSnL6^cvgkH!q8kPI#d1<uG#b04qp*F zr{g#|<2&Uqfs*$n{IyN#=T}1EL(5>dFlIz8)w+^AusGtwH7S+n{Eq57QS!a-o<~kx z68MTeJ_gp6%56)E$+^k$B3}o+T<^O3fo5YJ)%d#Oo-3?_%?>w{odt!#1{~NI0V*}urU)5E zg98Z{66CRyC9!i;#VZ5%9iu|Gpcc9t{hgUvkn}JdDu@a#!I?yv>Rj&S|5)i7#TOc{ zB$E0D))8I9Jf+X&m4~1{q={)LDk!*GyLxKi-yRL#C6skVJmpDZ;jp&Q71f^lrbK?-4{D$C-h;{{W1jfbh)v+CU7&)UVQ>n3z{t%}oEv)<$`(%~Jl34QO z8`Z-CA|=FtxD2@|*+^sYVD6sdU!;jEalq}Xwgl?SJAzabIXQX2oQ3Zg&16Nx+A$Bku2G&*^Ql_j$O@F0%0b&AqckZ7A zp)Md;FG0D`g=yTU>>9JE!a33LhgvL8`3+Y)GVl1#9i>9abKmw~6DJ}#GYGVqa}FpX zp#@=kj45lz7fF9WBq+xi^OUF3q%sMF2Sx;F)&z19eJ(VX#Y|wABqMFTMY4`l1H`Z5 z@N0d;qfi{?NtdILQYu{ItN?tkuEqo|ZT}ea{;ugrH6pirIWfOi2Q62$@~A})NY+zC z3|&ODWx>HEb=0Ihp}xXcEem2=&BjE-?1ve2?QpBoLPwHd{BWj~;gKGZx#s zFtoat3KL`z5PgBNA*)Judpsz!tTR%go?0_ z=eBRJ;?c~#Qt?oc)F!eyx*c-X7ww8!No+w-0wa2W(2lToj&thar}0AQ6CWx_Tz!Ks zajZV?;MXS}%;5fXYpUvK#(p#FiF5gt&Gd3POcE_x3k$LLo`75m9(V8&T_d-rUoJ}a zwG?{3SAQCuQosNDdmgE+ zOsL2vC;DurSGwOKzJp4ZEj`~Bz*IK&oPsK#6{G z{21KZhpf=)SDs0XYJ^jv*76yF#dGg*PzKwhD|J#6-{H1SE&6=;NdqNbf7$A9EViuM z?CbAlMj0ttH&_qZf=RV48s%c%^MXM*BCn3`~4Zb_SWo`%=Tq zNPcR9(~t8K^Yo(|{8q%t4wlv58`Bve1A!Gp?y?`3NE=Bh3R_vqE|z}u8kh#l8J={TvWn@?!mm4jFTN=3$|>bQA>fM*Sih&#a&cpLIoQK&zd|9$^BR*D(?$vJP+z4r{f5AcT*Y^C@qKK6< z>|1tSUd^wwiip-CJ=LUHm~Y&%8_jlCCK|k54?;c6?$r7uuZ(28ugM-He7mFF91exkbr|K`&MNS>JRl0PoUWZ3u5YtWzz(b6nY5-*xsDKOk*hvB(pLcw#rNo=BHM*@9>!WLin)}RxHtr6gBR+FbGu1z-G za+b#Md#c0Eh>?%n8cDM5ntIo7p!xFXNX&CS&Nv^g)2plNBPG{dap&>kh1cFfgxxI- z=SQgM5TyHP7!{3!=)C#56f;kjK!ol{mS12O3!tb;mv1G5fNb<2& zYDhZUBvK`z=_S!)=r6}hKVFW-Sjy#B`yq)dp(6fP`u)8ebJd4sQ{ETZsx_W~%retm zkix1Jw?itsUD-rXfD13@2{k&Av}MY0F3=ucIwUas>wx_CyX2=9f!$z@AjWiP-f2AHNn3`Jo@ z^<*6VTjHictgU8~5a06z%w#SYgSCv6a8~s}>djngf2OW+I1TJ-W#LD+ees)J9+GLh zQ%@wJ4fnY(j$@JI>51>#`G{{X?;h!|;DyUdH_xW5IK1}p6d8N48Gk?&GQ21#-lZeJ zr}He^2=$!IjaG#>6PP)|%sgJlBtSfmE?sndntB4czqEJni|`)FY3?rNv^JDAfb_Z^kW-5j>S79c|Fdl^8VASeK;8kbP)E$X$mLJ zaM*cWzNO8k%r`pWZdJi1S_^M+!uLt9rwG?t1?6QB#J&1LBO@gt}r4cBokbnu8-?Y7;l)c9b=17*KF&|-3oN<)i`nk+= zMuyXuU`)ZaNKn}7IZNWkSYZB3*BSWUu?c_ptRNB(8iryV5Wg85+EX68(skz3R|E}$ zG?*j*?Ab@-X0Vcv#hWcfJSI4oxi2FOCSCDW_9D9w3Ehj{=1d4>+&SO6N>IUI(dtg> z39zX?HPf9=>XCNeU6A22sZp)ym8(JhbxC!Qw&m3meCT=Xoxe0z81K{Z_RB&mYx&2y z*cefz18#ZsbgJ~y>H_}Ok^48XVxbfqr^r6bjp?i`{w*DGUsW9L#}QG(G~l!x=Wwus zeKCE!VDKVe5hpG?MO*kre$*|hXsFecIRMZ^MafPh6xQ>9dEOp-N{(0yh-^?`ElNr2 zAXm)1`7V2^h2=bdGilIV*yzJ|JT3}Ga$Dogh6od*fKr;nXDeq#_p+avhnZDz(2i!d zX-UFR#O1{JQx@NuBhtm8h9e28NZ{Df(4`gWw;<|Hq|muHnj%&QVaOpmB6qm2o)kgk z<}v7ch1Ky7`7KJ!vdtLczc`E5`ym~DyfMV8GoVx!ygU1nR< z2Q81q+ANNT5QgWbpk*gNid5g%_vMu*lnK$1IF};c2|CaeY2k6$HO$`yewb81n}RH? zqZo{CFM0B5n#DGrIMx#2=OF^kZ0@RtkK-hP^##1yvpx%+n5#VYLp@kjk6k(cdx^7&kH`X1B5!vjjZN@%6((fRYf9K zbqA|FWbgn=KehoEICuWQ=JtJ$|W$OiNAnqvJ1Rlfs_ zgD6mg7eTRPg*4kJS&HOjYW0Mc)7hFTu)Z^H%AFHgR}1*E26 zYe)?x+C^rwnK9MY=R21$pWm~>quJ3_q8Ek5m6Hb0z!S3=BmrkcKbH57@BaH^UzMR z%moqK_vQ-8oJCM$5vlUx?TosWHP;c`bppQK9n{&hktVW6wHn7hnzY{qGlb!xfSm@- zr)JG4;$?+w?DB#+AGk+9v)e{2MVo|;2RCE4ld!DzW-X~G#GF#P4(eoyteKsUS7On* zYT&3*O1r>{QBYGL_8PalIPHm!&~3-aV zY5>x+$*ZSlE>I;&)}5j2k&>nzpK_|Mt8{qt(TPzi5v=mhCu6sA)k70#Aw!3afteKP zpX=}>Q)t1GIlh%OUMLpS6L1ws6xUtgKAV^lt)?f$xTOM8PU@b@Q*719?%^fz_#-_T zS9{80vtJ#LWe2<`%R${Tev)}uijPV*b`m$vloW4LRfbdz0j?4giaRxv!KpdK8)>eU zf+YBu9J?58Bfa}(Adp~5A}V!(w^qSa*AW0$nu^znuVbxrm`!Gp&A@q`YVCc&9^b06 z)^FMhPzl0!BqxBKIM%zIKUw63tl3h6f>bC86^cy>@^^ZxI*S%?tr&rk6rE;Z(jdNd zh0pT^+rWszsH_EHJ*S!sJKUoabrB>jv{?p;5;&iw*f$jr724&xD``#`PJ~PWS_<~S z?RiF8+GLv2eOTe{;Oe@*A8*fB_m=+v6pY=hxq>A?9V1HVKF3@WZcG^QtMmsbj8YXE z(_T}?elC^NMI*#MFsqq6(B_aPsjwK50+1Yd+cXW-VI^)G1N3OkPKS2PP3{OeAdN20ehDK5ihbR=!$q z^Y=T}6fV${vZK#{(~lOM`2(vJeHeS&v`f-L6h*S`FRBZ+#y3j=( z72BDCP^2Ri3T@JVa<$T@ogMB|iML5okW!QaiBu<UoaVn_)n3 zZcE&m!mjCw6GJYTotTVpr#ofbVNr-4dORl-kmy-4fok$?Nw}gw7>^3mvEUNE?13KV zA!gA{%RyAf3m_9Xb?;DXr<yuGC@Tfs0y*UxE=F8J!~D!Yw9A_0-}e(vpf zK{VX#bR=et#OPEzx(pz0k1aR05>#Lsp7Q5t({E>E=XPc;_9AWN0+ESE0&SB33`FRZ z&e@6WcM+5p=cJF8sfjzOuxuM;+gg*3+R|%O+fN+{sTJ9}9dFbN2%wdcdRRIgBWT%H zZfrN%_L#Yqvy3bDq@gN`6%uQv{SNB7H*JK45Ol;8XS$gEsU|ThHfrpg&?z7a@q(EZ zD&HeyG~QA!f&z>Ii39qeYdiHjhK`i>f_dgf%1wTd`-jydmuCmqB9W(!-sTfSL$2-* zV{bFCseF<(_IT6HgGv=giqnhf^E~`FYTS=MLzw(_wPVt=<7^%Dd?it!xAL3` z(zEooIeY}$nH4_YgKTuG2nvZYtUa!FMLr(pq^}Kd!Ne)#XKS2ApR?3n{WaQ6$q=oE zCZQu|%m-vVtr!wVba4ZF;m%A24b!Y^?IJ{t<*8SCS%b850OA~xUr}jVkW>U$T{dv5 zvNMGvygNH)x`$ZX8sbYrlCAEcGMcA9nR}&0)op-lH9t=mWZpsqiP?znfa5JnNs%3s z?4z-C(L1ZUHxDW8a?tkfSeGtdLeQu+nT|aCPdkn#r(Q;?jhvqvehql!he-ugWr*#k z&tu9|altgx#*WRg-8DKLk@&n6*Tj!wWkcnR2>}H6e88_c9`=MNuex;C+rf!hQN!6j zO{ug1#Rh6BY3yh`pFUtvEuugbuCnup-X2`~@p;G63>&N9Jtp1+;*xni>=%{0%L%-4 zz-<-PxMEpK(g>Znbw2vKB}u!7a5axTkmV7W0)kA>kd%*jmoG2559Q5CMR#DS1S)n< z9$@ZSxoXw2krTFqW-7;#j>lIrg{^2<(Mgds5v=@P3W8*B9yT?JSy@Z5+}HDZHKL++ z4ljWO%?CI>_j1TUvQko)Fd8tpG?B&uenbP?`p{6z|?cN z&AWMW5>lCOk&I*Nnx)-Ay&wT@SeE=#4e)meONX$kpzyv=E#VPCD!-S)6!uo@U0U14foX?FG@AsDf7GC%`nv&~rYRWw*0e?vcZTsv6SjW_vh@^9#GMR1S8d81cK;tC8LQOnDxoAY%v zDOb9VR}LQT!(BTr+)P`-quv9yzV^xJ%qIZy*WUOn1d#zpw0s@Z<*^{w3~by|Ydf~b zljpR0o3@rS&zRd%FshnXY7~M1H8l#>Mw6qkZXlg&GtQXUHtnN}y=>ZuCL{yQ2|iM2 zdD>8L@tGBQ!3$4z2cx)ymjOwwM{sIqaiku0oWi;bh+VPLm8%I#kB|``OKxRoDFpoF z(9Yi}iF9rxWo{z%)q*C78tLa0_PpwsU!S2BORvmA&=Y-1%`94)r74w8yQU*bx*aI2 zP&+yA=P#u=$uIzo2ba?6lp!M!J&~*0rnRDI6tJog)Tkz%-bUEP6HwI%H$7 zb%5PcSsAv144V0E?S6r2MbIuRoGJi9h{7YtPqozjBJ6}Lj+)sMwFV7w8qa2~TFskw z?WtizlR-iU3Rj1l#?~;knZlx=OvnO4hL!Vc0Fp{1l_sLK671ej@>69L1f&zRfm6@# zdB9PL2q_CF!7~^V1q^xfyrh9j0ZL2>HtG+Z3$JqxnOFxx2-peR8{SfThSDT03Oyy- zga9Y^3V}_qHF#ED>XvOzBLQEyh@J87byC($i==G$@ZjN@$vLj6;jfti6<9REV?`q>Am> z1!j9^ds~w+-ne%amOG;n_d1woTnEQvmgv8gj;C5jc_wy>li<3Stg713iIT7lc1{l)yJgw z^zC5t_J>;9_EL;8MzyYh zo*HchqUi`+VTl{~ZQ1zM=Q+WQ?rkV6+)yYWM5uGqwl>3|3gJ&J??Vh|IcckWAe|Ff zh?Pdoqz_|w8di-gdH0pDs*+P*>UPn03L|!;N%uAo6f_;vrW{1+FeWCuI52Od8Qo6G zP{kT$TTga25p`nVB0;h(M}zV8>(OIabzhp1fQdc-7P4_IO(`MKq!bb z)JZe%8trt%gE)Do-A2j(0FvkIUCW-0MM}fFo|DY(8|i{K@zch~9u!8R2Ke|E#F3x_ zZ9t9Q-Mn6LERKygGw%IDM35pk&_wol)za?;X+B&={x5}4s(?A?lY7}Xz=1wg_`M*% zn~LYmXex+}pNDR*peXF+OIi;CdCP4{AIsN4-kxNSb6oS7zHe&0`HBfpAGfkR__tJp zB_qc(;!KH(4R}uax1eI0CuW0>jabUV{g=D*D%C)Uig5AT_!45cc{h2PZ@HIJKbSy_ zcu&nvG}&G*$;*_RU?~bJBpuQ<-y?LjByL{P=geMRNU{krSyPZ9?g$W(u z8fiXGGT;bWf*Ch!g-SFv6Ikb`oLtDaNA9lbb|>cB#@6( zLX!aKsG)H8SJGWw-{sXrz!+2L$&ORLua>Q?S|FI2ks4|$YvPY0+M!Ab6a?-kg<4dq zv$tzWGQaj^2O*XK5>}9Qf<<;nH1Q925`q)9kaxtAM8@B4*d^(stI=5H zy;6A*$thOe#_EHM8meo>(dcdpfO$RT@T%%a`V01Jsi$S0-zYJ4d|4rNM&e+J`eCBg zg1LkdhP5VxPj4i=+ftS{7ibHIRL&_(qP0#BU$SThPIXDjhx`;89tqb`M@-NdTaAdkVrp@1rApH(4E<5>T+S3qCEk$N$xfK-lyeUCORA2;?%;JuD7(P-5 zXT|gNflvrm&EFS}#$rg~AY(9{!)xur6to2V$|RGz00*(q{aO^R@tW>^{cV{sJGA&S zYZMMb?byec`Ku};Xf@QCb}^KxYJwE9QjN3&hLfwVF54p%YBj=rtQRq=wE;Y`!iK3m z3T;BLg&3x>tOK%`jPG<5lC<1{6K2Z6fS&s1gacm54VQ0`Y$OQpN%GsbUJk16rFRT> zEg#q?T32@~v^DBdRmmp;NKG)}*S0AyNJ7Z~qj4gEL{7kA^+@k11(*UxElGFPaV1%D zl~n)*k_^}8I(7%Ed_NOL>SX03r3%qFBTY`j@zLcOD`cTCX68L9hYs>*U6gm#de$OI z5khBgZ#&|y+e*UHQUvsh)X9ht0P>fX=0v|i>EBL1qlQxAO)2eq?}aY!O5Cm zytoOJlp~oD{{T@sG^pENX$vYr0VK{bIF8L{k{e2rqA@5zn$EyYIYTA)3VMQ;mV{_; zB+vAYP41bpZlx(83rJFvj8$qUPVYB6yM(P3gIP(Wgy=xp@Vjt2G}Q{MZaY=JT24OT zt@@Dv0HnC301XtQS_mUrd3ijHt->(@Cl&xFQyPicP7ZTw0U<;MN2>}{LE}Igu8?XD zsL;@i21%Xjb-kCpmF!thcmg}t)D%IW??{2J&C$5IIh#odI!-ObMJ6~pmdF^FFp_Yj zZ3)+xdC}}E3RM!IRDNpe=F-TsY;wDpB-9P_tJ$^Hg>uHCZX&5~BNgeKZh5=}{*Unv;9Xxk219U_vs0V@kC;#0bK^EQBhd&3<8 zUy?K&XiK?+RbrBI=3)~tm22w?4K?^`@|#KM?&6|Ql1MNW(umzy?bl&C1B$Df!8F;pacX(zoC)0>hnc{C+6FxuM0f#GPRWCd8^@K;t0fMgNW9hgf^n|njJLSf9%Bbx$|nu;VQ&2CkQ+{ zr2Dwh$FC-nivTc@UiO;+h_K zRFFuH>Vmi$gXN%X?v#+BDON07``dNgT<;-kHIWjP5=RrP4W98c!MIvcART0+Vl)P) zao!-#yIhij6ebT1lN8f*P=mRv7kpx&!Vi%m0i%1$>ExoTaY&uCBA))%n~4DmF)#<7 z%I&7RUEIR^px{*0pDx2rE|FNc3SvU3yOM+9x!TO$tvbm_T=R-<`d!;2q$6nA7to5@6T%&X8c~8c5aMMVtu20i-Q6u=k{IX}$(FxTNLLIBQg56Ie2Pus6q>seaf( zhyh*F0o^9FuJ5JGP$+{@zi2qINS-m|>t<~(;t(q%5hr<7Yt7oywq&hId6b2`LI&xn z<@Y%R*+2=Menky6N{%Sjac|h_1Kp)~Yslw=sfE1 zRGej7B8^Ot?K6;-B^OE(6}40VgWU#FB9Tga9gApA6sovXepOZ~F#Kj-PkBP&i6Vz+ z^7pQl${BvpQ&Zv_e${Nhb{fuQSyFCPhFVQ80AYu9;IP zdTP6cCN)qSCOO&)6A83TVj!8U5<4i&Koyl_o0#an`foHY+TEA_yh~>&CU?*xZ$O6A%do4zs7=^o6N5%7{sjAZVdV zgH48E%(Feru8g8^s1STXrwE~XR#75UJGBx*L4goxK>&6dwn?v$Tt2V7dkPa0B6q=Dc>Lh4t%Lyr*Cl(aJCQczI+2z6PtVB%s+|-g2h|EdZ z$Ga2tSO2vOtUHLXQSfG2yiiAIo8H@fQXf|1U^*o9*>)&pls34AuY zvTpe2d`+-GIDl&*IssCT$_sUyH}9rUnhFw1N46Zrn`qCaNkS200ZgnUD4l8heD!qg z17_;2g{Xum?I0?8xRKXW|1(IJ(`nf_57)D4u{>|3-`c-k6anWVt_U8^MlD!336 z0!&3hdjnor!b*?KbyqJo$Ijj+TwgLHG@N;>NE3xHrJa#Fa|ZmZbc(pD6$3L^B}SCt z4+yraw~u>OVkde+U?t-)K)RbLq^SxeCk%?pdud)q)_*8OvNqCAcQ<-*1e{gi3=Kip z#pXH!thxmRi7-zZ%yK(9vo{cn3Kk4k1u;q67-k`#rniz_JF%Ej!TE=kYmTn?cQ3bb zrrD2N7ubeX)3A3>J28_nv{CNjg~cp^gcTD4C?m&DZh%ca?#>CKPlQyx16|5%| z36&AYaE)GW*cn;~(UiEW0_^J$H-%7Fe3sm{LiNa0rvZ^OxclMHKn5g18kgIuO{_4as2#>Y_PN5CQp*1TdO!qs zpo!Q7?Qsanr2-dqAWzBry|U>!=Mze#%3T`nA$NPBDaA5KG9!guBrBx4lnP3cJh!>r zre1HUQlg4rr4%5{_PfaoMrldFIyxDT>C<++8{CufkvL8Lh#b|cnR?2KeQAz&WZAoK zE}O>TAfzkH5D+!1nal*-w6w;e9|w5>2V!Fm$PPE)M`27`McZLkr(Nd14ri-Yl%>M_ z#~qxL%H(WYLQ?>8J_!UVLMTKJIperFZxJ42Hn5AzNR-CTB4#-pmRiS2l<6f{$T8j^ zO+ci#-axr}Iy9LRi0+-y#k~p%T5W3;+yrd)Hr%3~9pp}4&BKh{B!m;R8mG^N8`2YY z;*yfQAtp_JU|@D?-R-d&RVBO$#H%nO*F}h60~F!!OpH3t;@y(F z0XTqBpPO&R=VJ$4r-mzplD5R$STWs?`h?Zj7Y;k=flOoJE!e?Y-~^L~PWT0uPzu&# zY@RfD5GydXrvZry6|c*n_q^UW6087fCP5M?M$y5@3^h?3J1EuBrAsRy7#M0KMy;k2 zN82khoF6@8k+*BK5Tc~i-}(j^q6%7r2@?$Fk*h3iQml9|G4qXq`RM9J54q|+*@ zSyQEfKixj+``dzw(529cjy`UpE%b;c5^$L&l0cdruVGSfw&Z7Ls_q2@^=!Fv>U0xg)>U-uFcNvjE(Htvl&|`=@dv<0)abaM?s*e z)%_c)zb{$ICT~UMn#6xCCxnji;;v@Z9sT&SEuU`=qsR4EQD(EGT(}Ni*Q`f%UohH>Zm|FC}Yk2Djh57h53nnYeunwJ*UF18r%(KMYSzkdbMygN-{zhIE}BVVAliydWMryH=2x z#3k5L%2Nr9QA!dxO-E}4Q&pU6QbB?fI%_`%9ctp1U=nsJHdkwuw34IJlceH~a~-vK zD7R@!x{Xe$*k%qVNtB57Turi=UY(rM@T|!NRntcMb_(uvge2Rpz;Kz;46^1?B}xeI zM|gv~aHR&vrbgQ;pIE-0G*-S|#-(NkD$H-D7uDp@EepNkBw_<#CMim8F5t^s7VdX} zQfCb)N0UBLZKvQyi~apgW)A*$ZdHdC6A>r6`DZ4DTLgk@O5r1&=N;=9ZK*>D5+ySu zbvQHS8>Mheg-Bh^qqbibg`jFDwbw$FpC@Sak=>5h(sUV<`;I4tD4@vfYn&f7OEGP1 zW0bYwt|o$|EB2V)-_lJBQk4vVFq}&6#wTv=i?#xBq#zRlpz+Wb%zz8bB_lv_E3@fx z-D72U%1Bs9LP^^-^5G9VTjopNV(*+6`ndwmRMx60SGapQTrJxs+R_pfR7pC?a+xf^ zzA%K45RfY#LxXQa8IyRC06+t0io7tjj8lfjfULe=4GanL)Vy3FND1}ek0awo)&Br7 zCVZ>8(*|=GrhpoA4gAhOQabT1psLZ>C{JK`%p%;2mBwIj({m4gQ;1Frp}TY9Z*KZ))l^R)~B$h_Ztqq^P-Gs!l8T1jwxh z$CcZ*xRj+)SS9sqD^yO%*|@x3aA5F83bASoeBM)9w_lZ~7*-HAZPnKc0Vj#^$k`H+ zJ7Izrh1^o@5)KiHNtj%XCG)IpqXJX}ElL7*{G>;**jMdLz)6_wBi_ryrj8QDiAlpu zt(6P6AxGs(bTl*gI|QDPsG2|?##-pqZWA;kkuQf0zzt~A&63=)_Gb~3G-$rLT7@f0 zA8Tp|wrRE0UAa`5?|hquvK5MFxVQ86i#C;N>?U-zAx>l8KR?4|MmhmJ-FsTcl?wNSIMXl;{yJr1PE(}!g zHQBRl!8-JdjtAuPBeDA~tNg!|>i+V8y>k{kN)$qesGX4%SIxQ0ujW(qYo*8C@ix8Z zv$70jGWK%FDYSrrGDy`_i6>HEYF*g@)=eIZ+=W>!I@y6RU8#WMuir#KuNKOIO|3yd ziZ+cJBH0mn&aBI4M}0U|>!ymGlj{#UzX_e{jQ&l1AtK@imlIeY)FM;LIKQaq**&j2 zX{Vk7J}5w3sLnE-pAR;?xwxcZM0ZFv-S~&}oc{n(<+BpIU&y$2kCgeR-Z@`{%w#he z1)h;AgCHz?hOe4ZAvgyQw+H-0ZS5N-ol_DjGW%qFp< zae2mK9EGeRphL!I(4DgbE&9d{Y_yo@qc zwZ0|p@A5jiO|_%u0%AL+HSHr;yLc1^lkJ|yqC$x$bZM}@Ag=%$Qgzj$3#m%fnF)jH zsFBYJFNm}*LBueT@~JXGCx5xJ!pYR?)R{(UUGC!0DYFpU+qJGefU-O~NJhupV@mQ4rkZWBcr7}$<_nlNdU-_{YcAO+&p$Rix&7b(xE+5$Aqk%tO&U>K`+?! zR`)%0X!S>rHgw*E^n^!Op%%iFaHQ#2o$ag2C#C6%$ulvr$n; zbM7TvQHRvoDG@$7+bvt@$*l4=0=`oXH~K}9fmnMb*yy{&;_=Sj^qG z0v3DH+0tQ4!9nr(z9|5Wd~+}kmC6Yn*QA^_o>H1&FQ$U50|7>A+@+!f`MR?zfb)7& z0I~*HxUS)AMB#Z_yP!8jTLzvBFatQTs5?p_Jgz)q#-7h>b<$L!4uEo3y@Y{KnD(@k zMxEvocQqM-oo)xQ07oBzYgPM5saL<=*ZPY}t$CRY3%3eFQ#+c{xuai8Wl>p<2kC7K zv_#X#WVwJl72&z+o1QTs=%M4a?wkB`wjhAV-x~fehsLbT1uBF!sbrC>Qj~Ft-rJcS zN)w4iDM-|l0C|_>v;P2@^4WPtTz;W?O+~TWs=EYn zV9!zs8asCHjTHP2L-Q?*{mjr7SFu&-k3CV_Ba8Mkh)5uf^1y`2z-8B3iWcJ$aY)hH zH}YHYER2OG&U)`8GFgCaCk?xx5Gw&dK_0`} z{PK(XPhVxfRgx-4&uE2@EfvUTvHZ^hqOZ@@uG{^{c$$n!2eFoJK~OEW!H)GgzAv~F zGXdfu0#v*$l2Tx{gdiD;`F$^oKv|RRh73@Gs_4j6(?_C##k5duU<(sEc;|KmXw@&X z#mI+1cok5ck4ZFWxhX6NIWqPjYwRyG#QE3U9<-_FI&!uKXnzlj=sqC6be7Yqf#E+- zxvhDdLqa)=0Ce!>A4>rTsjfEs}5P;gHEw#eE{ZcoM@Ms*5QVLXli5v$duiBaM0?RC+Q zY3PP!GqiaA^Vz(Uq4#`TNFYZ6FNj>FJJfj|dqo83S{60(?|f>>;yX3umm4uW9+YPQ z?Q3Zz@~A$qj(5ChKB(eM4|9mRv%S$xYQ;gpK5w8<+(`q~uPij2-La-4N` z?iqKmq=}_xX+h)48Wo#fLx z8Jf4+1jwChE}oft8845%3S?~$V=$wL&YRr?-342rEh6M3)}*-F8-D_A7ge@9N5QwG zFm$7!79dw+;_l$mvMP5tjLF(xP3lPl$D63U+NOazarY1!mv987(#2I5TYUp-EH5!| z;gu1JS_+D*F1pv=N!%Loz>asKsP>Ig>l<5Me+EQsD=K;|Ylm~=8mEi_R7=f*wQ?iy`w5~yCC9dy<|20E~MY>-Uj`MF;ac&`;rK9@}w!j^Rn zDY=DLIFs*c1gVK8i;5|kkloT4nmEfBV%Y@kZ5I-Dj4oPaG|_QN009HMMEC{eU{E>> zyk`wCjStfGa_@C5z-@l=_*k}_I?s*GK%i_@c>Mv@yLK>`Zn$uileNqW8Fv~H39Wd2 zEn@;zB#|R@x;d4QQb&y*uG|9&1(5yY+TnVa?cuZ7wNp4#QaT0v6OlmV$xfIP{5FFpCAFJzQuQ`4)~ zGMi(|_bdL9xWAv~`olGzRx9^bxY0BS9!X6-{9PuDZDLb7)_J_~o7V|W#0nsG3Li5i z#=M3|4vfqwL30@nw$1f<0Sm=%ZM9V=%W8WZKCEGA`sY{m-cvjRXQH})<*RS^RTbns zAE4qNz@{qwMnm%kXDbN1{QG)AqS81jv*dXigGFQO30V%xR11K~=knR)VzRJfRGKM1 zW59Al&7gHwFDNBQDI!9F2etgN{#|x1+_q!7NTgJ3I&BZ>DY#R``7XXnQFRxRDR4oJ z>GR$+@_hRBn$H6p)(vR_8LIMyFX z2Vg(l{1^s6JDCI;_~m#6%n9~;TRX>pjx}Hf590C9*x7|=TK2WzBU2pT2Ef%f^SqcZ zMw&-Iz&!*Kz&zS+T~YbhzveHhBx_c*C;;oO;VPr(gGE-$s5@GesV@O^18Yf1K?+v? z05@Ao+RK^=EojxDckvA&@d|1MweU5VRFD~u0wJYISu+oX3U-mQu1nBmrBsskZV7cJ zDDzg~6S8Et(gCo#>Kk=0V5hb377*MTR;6TJ?+r$_3k~uE`rF_yb*G7?P zSC(N|dHkNbVxXCi9jz-~6E$eD<1Rw57*RTSqhUqVe9^NGpNqi6ZUqHI4dz=HaMGm1 zOYTzp8$t!k(xZgN_+`bAv?mN!C5TWaRBH8cX6Z^ufGIoct6T%Hb@(=*2s5jyu!JE5 zfn7{+cRC)9q~Zf(sdNdbku)2c$T;LwQ5q*g`ic2X1Cr=+q)KQ&olm{y-K5?@l|^EZ z+>+}&1YkNbegp+#l^2bteWaNSb9Y4WL;TIIr(k%vOX z5JIX**G{03M#SExIC13TRW`PSDJR?H$TH~UYQLOu3UGZz*gc`>DHX#UX!&^-!U0aR z$myjeI&P{fW^;G*7*TfOmYZND2`9P$5FsK2#@Lg!)=R>On(r-PZ>?aCekGZFD-OOQ zUmF3~iDOuMU>$6D2_@(vXjzCcv!%Dtt8-X3|&+}4H#~#KzW|Y-t3Fp0MXEj zeG-uvorK#85=%|erKm)}Y$pmuWq^)cLJ_8|K+EAJP}HEI8yf~$6w%&74e3!XKG>_W zg_Po}mI$)24iI#J8`W;mQZPU&JN9`v`-|?4w02SnQdO}b!z~ZFU?E}&Grh9Z4w7|( z1FpM-s8?g!ln{v;mWIgNh#-kR_q4`GPxlt!$ zSX?eeUhZ2$uz|j@vz_5c7KjNmSg+voG$ryG9L{p;t>vv7%fPFu9VMG#C`yo-D!q2v zer2Qp2{m#@s5<2;pV9vS{j7oK{{V__{Jw*3_Ik5DU6>#G`EvYfZ(hQa(_W|t{Y#$~ z)NNUDGIqA;QG&%DwOvQo3QiPc;=jdYfBsQ$m$G@>t;1sGR`6OvR7J2e0u?bT%NwZ% zhJ=FSjr*}ZJ?%d7=k}kc{MxnK>&w9kQU(zrDpExaK2kK;+I5GP9QMf@?I}3G^J~iI z6e*R>&foZZ-`IHlCU;NL=Jfs^{{Z$Hx@)ukt?%(}e`)$Ir&^Yr#wD$Bf-xydj^*E2 zm~~x^q_=s%+hifi=L%!=nZNdX`uYz;E1U?A(&qmF+3)>Zri=H=NAs@eKjvKi)89>d z@2`J|zwJ4if0uCP-h_wgFJb=x+h-;}hrhf30FA6bp>ur|oUNes^ zoBsf3%zLf7-A=VEUdE-X<%~*Fzh&Q8o*d83BK6{t{Oi?;{$k_)(;A{l{MpX{S995y z+?P3Py(EmbawA{UU;4MTTg;#QzHe-Q!)7bP{WlVB-kM##{w2@t2k3g`YfZ}mDpF)! zCkl!|i-Xl8i%C_c{%oUwZfh*FDum{+N9nVZ{{Y+H{=?#}Unw8(*}4A!?XUe?jp=V~ zyL*T87yYL@)g?#+p~25ftSte{5Agob+-&I-oihdc+4O57q6t-j_rS)cl=AA+H2(h050L~x1l7f zS%QD|xxGJzzx8dYw|qq9vva@n*^jDrI^}9b%ONUKK^Lf%DKiWHpEu{t%cD!TzrDlU)6(;#ol%g>F z%2K1bb4*0PG@IA8H*ZV|^ot+;L5)=}_Rs7a+~Hq$=6V8~r5FZZHg^-1%y$0(%5+ko zn~)GJlv*t3toy4yI* zjzo$5I?|#Nk+|WC?`X@5S8)TfHp8Q8;_3^tX3)1(lU?sKUeVWV+O(WVK6PrrD3)Aa zp{imxZEbc7i3fA%SNDs`TQ1h(qiHNcOYJxwFYa;f8-fGta5E`OumBBh!$!t&g^x{) zQA@$2u*e0W6_$A{Tq7c82q`bLjx{&5&8cDmM-P_TE<`4I)(w2TV`c|KiA z=O(&&x!@#=sr8z{QcUe~my8K?`c(0GBWF>aD&tnP-cc|><#p7q@rS`WhSEk$O=sFj zo2J;)$1AS2bmUSDRir6WF;qL<5(QG~v=Xk&b1E4|u-u!eiu>NmLg6Q!;_h1laboO` zrRN1bHEh9XHddr=QGDr;l<)Dy`YipVDIFml=+*fZBt2S&QgV(C`uxkdEueApE;+Rt zcC<7#YCtiwgIvX06zVNTHEm!q%2w@OQ#)TRzW(qyfb4;|h-*@a)yNkebLTEW;9JFn z3N)!e$s|Zq(8hp?pa7_wHC2(bv{I-aCeimaB}%?^iky1p4G=PFL>-ch0vtNvR;#I zWK_96wEqBE1NuGX(JCqxuPAh2%1f6ydCfuj~&Tsz! zZFl>HsJoaG5Uf%c|rAOGL&y$Ev9M)_90BbvizEs;!u_42wLiJy?qW17|{gLv*PqF&%&FB9B` z{{S#mkQpF_D9j>WdpV*m}yvPE1pQh?LBQOMh-G8}t`JeNvzr$p| zz#kUG_5NuShR#(pt{YfpgjWiL>UDQbII?qq%v-jjY#-?lG7Ind-S6SDKj06G=lRv| z;j%yA4~veC@d-UYT_$f>or3=W{vh*yeLx~3#Jn%Bb2lKa&Q)YgVzgBfWX2^J?0NqH zJH7liNBjZtb$vGNh%Apw)Z!DE{{VM?sdd6xNa)gIV?gWd%062;ME?MG))}D{!l9(R z9=g0JGWj9@0N@XcZ%yq=aF83e0owfsT$sCH>D|0P^#k)?=Xbw{$$x-8EvoD6)yT(d zD!po}ShY3PO)WT9Q;Y$kN{jS8PJ6kJ_kZy2zc2oGd-!~p_ygkFI{yGPkCxU|Gp-w0 zW`tJ?gz8yMII?mL$fTa?dp(Vl*w#Ak@c#hIWPjPy@qGUPJH7lqOZ)-xbJ0E{lhSqF zF4^+E{{V;_0W^G9UuLdGJ6To4#w%F0GJa(ko(+HJSHFkJe}FzN@21_c5JI6w2N-@|0Tz#kWO&GlyOT)l8767Q05h#w+!)bh0Hvah!TMCQxv zHt+KIx2DR}kff1O#6GA3ku`fkAOIeHhO?Poa9v7PYeyl`T2SNW<8vSC0_`V8nARjx z(h^8~4|jfbdAtYQa6C;sm#4s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu;sdWS%RJ$4&M#UbH>91f%G z(J&cYX*7Tk0B_TKlLd)|-N z-k;mvtTt9vtD0-9KKhW=YL2=7uKwKxph-)BBmfYQ006|t3-I?HfGTU?YQ)9D#LmRZ z!UFia0r(1lM@0OD_zC%60}U1JzXt{e1~xYK#}N$;0|N^S8~gL;&mUiWyy4;D5fc*< zQczIR)3g5_1hfJGkWdgG1Mt5GG!zT~5&{+u9s=+`fB1j`2?Y%Uiw1yzf`o*HgNB8N zhK7cPfcZd)P6ma+>hGW^VgP*=pHDuAS=Xn;29vP&he8w!M|mE0A;8cPJ5hzbo^t~W z4t0Bdf7p~^3`Hpt9NGW&0?^?hydeFcAjkob=m02mz~6NM(#P45=pUcI0G<*Ui{P-_ z!XV`g=^^ork4mFFmiNm($-&PRBY;9o%S*fLs9J>Mk1WUXGYx_?JTzVD*3s& z{|!p_hRO5vw-EEGKn=ZBa$^!11sOG60bc6Qch3p!I@k4CNUz>9$N(jABj@R3O`jrD zQ*%Wte79&P$>h}26ysakrfB}*wraSXf9qlTVCU{1cJwSZ?IznUD=Hs5{(YfuMEV5( zk2xB!uM;`X?K6?xCNti;W^MMy0lV&0gwa3t(8gjn;xr;>1YpPQ9Tmv z5H!HFBMtEWZzu2k+rw{qlgu0Syato&b#Cb*O4UUGDj*0BrYR|m^(!HTC+cyYkLj8; zM)=`5ZIB;;lF&~djre~m;O6U4?^ZQD?7%rg6$C+$gVAya3sgTN+a;(@>#DzA=#s(^ zBDTT|gxZUNXq*6_{>@_d&$dgZ^u@Vn)Fhoc8^W#F3=+KZQK%b#r z_d`q@2%Q95sEOW|@7P{qw79t^(biBy?*9(%|2XyT^@DXRAIGHvFWtlQ;)5~}EtxRx z4-xO4>l}%lW5(FY%&Z#bBW#R#8);yi@Bo?LN7nw2`1U#d3m`Mg=s40SDWT#5#W+Vg+^JW6qsiae2OquFA*?^U#+NNk{zFAXSMG^?+K8vmbI?n4us#@CKJ zD@{JP@pC6&!&rP$@7MRlw|gx~YMRh#6+fWstGA_6XGjLpD-0qs?qAPzQ2Du&{7*Q2 zdK9%djmedtQqR^8lSu=TY4DT8kaRceD_VpUM+ue{ZO`Rd52|;vZTH8Ge&UK1M(>6| z{QRFuoH;|2(&8?4I5a8(RBRQYQ&24R1EL82UrFTupGyjTtW3d;racZ= z^*Dm}G;o1h^cV2ezPkwJ6vNDw;>(q-`=QCf0o{Z&u;Y8jON+Jgl#?5c_fGRj-Jy@C zTWT|7^=e0uFi&kZ2NO-!F_Tk^Lp1;kVH2*YKXZls`he)M>@NVdXT$m zN4tnHZETD(^JkBklO!`8nbH%Y#;LPFfAmPFgxZ=6503ZXuIO`ULT0tnPND; zU3w94owB#|7(&Ti?yA>~dK<#C**#u0nK|P!cSidI=IzWGXN=($g^Z2R4AEY{l*l`>mOl}K-TN;mFzx(>`Mhlo7YS_8eQ5!K&4D$YzT4aq@=(>fwiX^K_fizn}>+uN)ftbl0jA| zxG*2j8#+j@y4%>Z&Kwhc`4m!X#kUkoov}taoujA7l=vhJh~y=!W7lN zfQ*~XsV40|bb>1Y;82pOc8mH!=efXm@?U_>2bU9^AO1)= ze9G*s>T{j1rG

    GZe$8MNjmuNpn(ziHnS5NG9-J%%U>HvP|UD@ukbmT49MK8cd$W zD8L*>XI^}dzVWTBTkhHGs;@7}sQ#pMws9@nKtZUB_PR3JR_#E>eF5X=0N-Lmt-q|4 zlqflHLu_6q%}rPFC6xxz%4+M;2<)XZ@@t3pdHY+>V$WXt7dPI?;BCDs-vN(TCf}#K zAKqs@X*Z`mE^5m96&LeA8jAtJ=vsq!Y34h3J&Mn~Q!lSQ^Ll4OJ~2+Ym51*Me*qyw zCrVCL5`@AI_Y3G<^}Pl3+yZw7Dr z+ySdSr+3MveP&jr0ptq3UH^<4|0 zZ3_Z&jbk7b0nL!j`2F98CeQ43Jo8TbG^9SOe+iihx+6JySqbx8^JV*eMb6|kk=1Hj zrVipxz-Xr~1La{|_`k8NxgBkuzgV`oUQjYnG8i-?21rebkj{Lj`ENa!{{gJ?P9TK2 z=$m=^2i5ZfrVoJdVRDB3$yn=CV~i%LcCz$=7;PMkIWd#hPHt-SwpNxru)zgBpkCrCa{cl$@FO>AM$Uei_j zU%)X_4cf!?yUEI~d+7g*u8Kjyvz~O9M`DAOlfM9owtsk$c_;W!2vPk7Am<{Sewo(& zFD_Af`ki2ANv1; z8>P-<#e) z+5I0j?Ea&-;s4XDrjOM7$NiF*hyR4k(|X6e817@T4&6FwFEp#6ILoh14?7#xSvTA- zgjv&b(gM-5r1-s8w?6vuNk6){JOq+@S#$b+cZ=9)kV-wT{-#5`ls!xzYh(*BcG%oP z#Dq0sPaIWvtXL>so^|jcxu8s=aR@_ZVYgAuD^}vROD8}!Nn^1fk`07;zwV&_l8aA%$(YzJjO^Rybk}+7BNx^dvLs&*BaytlL1Lj^?8q8VD8u z<{u2w@&Y7fagxu`kot&}2_m?~z~++1Q}b*JkwLC#i7p9RtD>{nw>Ix~vWR+Rzo>h0 zE3B9v(fiES7Rgw-`f)@!m5tG(lU6e{&H}>(>>$EjtI1=g>M107xuuG#(AvRo@!v8C zuA+Vo;T#f5${TnHlX0-A+XZ`YVhX_gw2r3I#`Q7CjY( z><~+DPqnI9x{FinX6_b!5ftr$(c4-=lyP}3o&@mFUm2EcTTYJL4X>nZdQ8-kB=+JB zQ_Ciof3nJ<%ByGECp#B3pEI2VMJE?*U3@zaV2!%wZ-0vkLy8lQ$u{{TY``aj{0X=? zHk731%442lxHQSZ>%pxO%%ikZGcdum_o? zm#JG@eK}~cVC8lg5NDcAcAlD>o=}eI(6c(O$=6CE@gzTE^E#~P1e2K=LdSrwkXCu& z6HhNa;sR2@%1^y=(E^hG0HZJ7*kv5?+QW3TU_b`c7Kzb(?t%=JxpV9}DEyIP>~jC- zq9Ea0W^-(3^*&>tR)U|D6s14q0rVwf?9W`jr=G{64TV@nE`LASEa`AIAkiEa!J04G zQ`ZFHY5b&))|>}qN)t9(O+Scl_wK@b;#~rjTip8#_sIj>qq{{*_+6^5%X0}(*JT;v zL*T31e*^vz>K#Up%idsO>cGl2Z9Q7Ws-d7OIeE>3xC2|m!&w?qNyQLXnnP2Ms4Y3V zXrkJQVCp^_ub$+VM^>VW8dKdEDL^~`qCCpWu5|~iX$EI!bBneTgM@i_^U!Xq$GHf$ zLNQTx(e;I-VtBD>V1dqYq6=s@nLf-ey#|O)sd?Y;-KG08b$wsCfh!JzGy`k$YG?Ih z@+855QBX6wGv`9d)OV6@Xec*a*6OA>v6}PXbX^|irqy*_{Qg_K^@&_ow9x-CZA90l zE_p>)ANxZ{G8cm>4`J#MLK%IN71=mmD9)Lfc9qPPBtoy;%<30(5LL2oDags4CqS2k ziW*z%mvWcbQ?mhR;j#`jIC71_XcQ6=nCBLaChR4B?IEVw^C;Rk*acI5Z7)se%AC^X zn7>JH*RK+!s*v^73@-LmK9ws41;mSNgSRiHuICZ*t7KQq^M4Y4^5Sztq7trX6UTr^ zNPIA)HPvFpixTcD-Kw26(K=%YPHdw?d?m%|Pk{OU%P_4HlG|))B~hjrfEGq-n01VW z4S|P?CtqM3P}>Ca6O$T-w2w!GhL$?%Co_C7DTFfBcUge&v}u*U#&Tx3meEz7$yFQ# zc>^1DgWTlF!Uaq(Ynetk*I;sn2(K^{WBRbFp*}??mVO*%TKZmYB(KJp%3#SpOHJq& zWgMk_4O~3ESidSS8hsC;n-njml-Vyb3$WNb5mfQA6xP<>g0>0>hCP6;+=zuO5E8Ui zW2M=tPz1@HwsgmF=;Tvs{EpCuMjv?dI3j~si>a15gfQQQ*z-(j8$P?q!FQ2 zO(-q&XHV*U?4YzV!E+&JPkEr~PzlzE4dQ3N*^3N{yYwp}6CsaiatB3DcgE%j(CMm% zmbKdTz91qbRSAeFXvThU`UEIPHTpun z$fMsdDI&d-Aff9n0KugwN$y=kRCQD~2H(B8*g&k3==%M|s2gSDW17V)*@~K;%xo2| z548yeTHm=gn$RQFji{ugpj-(vxaS|kFV--metBz|xWx?KA2K80UaE|yL475^ek~Ps zx-zfn6RlcD6PrGFHhR~ZOBM-Cg>l1| z-Eoh*Gs~K2d#&Ob!D2s(L0?fdxhIXm8W-nsoVSQ-wf`&5biRNzLkUKT)U`vo0P$R? z=DJ3}k?RX9`7c>QRRs+DNxK0IWY?u}<9F@}uZfJql8Fvi71IhT@&+DN(nIC1SSfNd zSzi5D!8ddO&orBCDgbc->MypxfP?G3lplNP7E}>#x*a|!pHIxFx2nySw|se;UZ$qB z5pEbaH0FcRK2oLmhr^ z*_S=-OsE9ZOy-H~gwgx%MM)UhyL{n)a;(bC`3_Ksq8}2#3<1|Mm+XxF1?cp=&;4PY zW0G%6l&{^5y(Aes4|-^eJn#p$*a0&z>f;x66y7e{GfTz~&Z##S@M8o>ZP1mAdAc>9 zU&Hvi1^B`FL3lp@Qbd_CTb{bG&o&b4fW-6ZdPHmDTm5-O0q z4=BzHhsH;5#OWrRd2SN-R1Fg^ES`tNZ1J?^B23+sC*rbj#;7P0PS9srx%p%SR4a$A z*57ZeBF-P89$GR+E!bhEL)J;pDt?4zKKcw6rfIydP4zf^rz$-$E=l?luFH?h+7a%p z5TpAB#BE`s`KXh1FfJ<^AM#Nv_5NKa|EZP#Eh}1(?Y}Gk5vZS~?R;P7tBQ9kiEFHo zXffQ>8M24{Bt07NVs&zRZN1s8qJ^>xb>CH85t9KWSny{^6nG?+mh(rv6Ma&Bq$_LS zgneZA9WZ-EyLZvA%A*7gveHM_5qa?PK>`huu@E@`%18egZ85{2pW7m8gVHSlgei@0 z^NWz36^>jX=g@v^2oY~3Y>Uy565kAi%XVp4dpWPp-*O}bKq?j}_rmWgfEtDvcF|WT zGM*V@`v}++Er&*{KAV#PVGWYn?HbA819JYbYX!E)fBT|gR$i`(7>;gR0@*67a%QCe z4cm*Ye0ERjz2IvWuP`gAK8OQ1hMY_qUh4~(XRf#^FWh8e)hap^I}!_9w9F|XUku0Utxt=i2)46!d9Gq-ftg}u*; z-l<*<1Z~wNfifkUir6RBCAFD()xXOkD@(!ISHDHcC?H`#hoxxn_*1xJ0gEhxz^7B+ zKQkcz1Y|q#J9}FfsSrvS+v$%+B#L^o7r9LjAdj^wr{`R}-rP^o0+GNG(RM}w)rgS7 z{M5DI#X9NGK#(4m#+8BA#2MQR05i3PkN(b_hETRCrXAWKFn9UqlBF$1)%%D504=_T zxhi_IT`D$oC;8bHvjb5Qx)X>I>n~>>er}XzD&%WxNo26(#S*6m;1pO;OkrU zsI*5bUe)WVrt0uVH`@{eQ|RXC75k6|z;cP_Y4D^p0l$DAq;ivg?5Xjvw=gGR({W#C zUg?#06pam^=^CGxe|_G`f(kbj5CJ}scPwY&7YU+$INrpLy95Yu7)Km8K_Sy0!3xuh zuCaA`=R%J`?b%WAy{6I5vKsH_0(V|vzYkfC=XKsWubLL|q39wkL~7YeUFyWQqXfxh zk7tVzbO*znOXrvixv|3^q@$#m$v*1>TlH~}vp?eOBP>Y|TRJi=T>_2Y7;mVp->+Ub zQg0B|qv%)(dfWq%yae}3oNn)Xysi7Vd9@RvP%zToe}E|9*Tk)v4~zH1702t!T@ERX znAYfns>9@D0r1O4?yNZvvkf=;&8e49)S8N3sQD!iYiJIRAr1T0h z+Qr*m^OZ`NxiB#dyEWO-IS6*9)C2MTI5x@4t2gT{$Fl|abzi@^O)Zf?dY3v1HJCk& zB(+(8TDHyG<%!Ub@;UWffVNW+-1xW;T%bP82L{mwKS*V7U|H!<4m{zjDqyp?n&lm= zR#Nt6=iL@;lU(Di`7(5N;<4S%rs8-P++@{hDIS$z1Qq}2;CryMYjOFk91V?i$onNU z-?BMRB(TLdkoHDgA$cDmos8}x_b@CyV&XLRwQ^yHk@MjXi%-0L;>DYM6CJE6KZ)Lu z_(n@L4aOxCU7R|l0oDMG8#3yGV^3#`O_@nOtVX?y&O4Y_v1#KoB|Pk{d)51t<&Hm#r{#_cV?c?n&Yy9<$#Z`t~U9#V&k7Kcu~L2-Uq?~e@`xW3s#Sju#kfH8r*Q# z&MX8h&!JVst{&XxIM=`h5(}~v{hUbk(SDr5)LlA)xmLv|yn!E!omEZbq#0m2OI#{y z`t9rk(D~|Y7C7BX?Xq;MP2?}Ai22`?3-Y8S*IL) zd=gzjxrfZ;D0{i<=(a~|lL7GesR;&7asp}g92);(wUIzny}Ns$l&Y9I6Sa}GGBu4en-j#!^AfiLiT!U$I0MZ_>-Bf|^ zx$ndkR_-Z#-}%>e6czLZu6Uk&Ue(%V2kR1fSX4WqEbY&t@#IqJCuoE=w+3p@p;SDu z$oyT=1F3R@<6+tJ{Z!p=0-j-TlVWWIoyFK_0UYsb#LMK273kU1ql%cU^V-Vxe!>EH z5sfT|^HDGxqz~FZiqQJpS(SBfk}af8GPSCKbkdgMNLe#W6@gm3F8DdwVA<;vpkx;~ zn{KZ#Op!04Ac!PrfNm=xJ^BfyeZDSLn-K?17?D~yW&qAr^S%*Cd$?@e%XgyVzjyZ; z$EqSgf1k=;!m^#DAZRBPYi)xFF>Mh4?v1 z@vzv)I(^#JQ%igaD@zO_uK4xw%)Q)}!4~VZpkiPPe|=R;)&L#>Hc=P9yZB&}R%=L_ zGKw8@o=ebZFy9grEuK=HtcRKdH|yYbb=8ok3`Uw2`p?ANHDZ!+q0~k()Oa-^4;g<& z=R)yul(DkycFr)$qrNNj?1kraQMq4)^Vuzvh&r!xP%m|yFMMJpYJD`Ssy*BQXF(rj z{L;L3Sbb7pxTJa5U4hs>E02ezb;ZeRLESbC@9Elt=U_6EI*r}8Q_qj0)yE>Z%Wvel zpEoN4<+D9J>N1C!%ltGoEus*v`a{w|6RDUBgSzQ`<(4_stMAK40}bL6D8TPWm4vi% z7STrTNFCB%=9m-xA#WykqAX(RIx?zp&)50q&dGFfUTzfN)eCzj_d;1!>Y{hma^&ap z8=UzLOGId-s1c|T?VY9K!bA;XewB}vfjaEwtp@wv`Q`1*a!~`0{lhJ5HdP?JET?We z8Za}8UU4!tHDbcedO;5|K{egW1A&GVoVU(Uc@b@=Efb&3MYf{rO6~_LNNAeaEAP(h zm5y-qxWRHu%}Ofrmku{OY?ZJ5?Vq83nwp7CDXYYy$aZ_qKS630zk`S{E8`s#f2 zB5eoEkKd5B!h`~h$-!&GrD0zjZmtAYQQm!b8L${?s97UsBoHY-`HEN@MzAfFw4Y$S z5?MqFa63Pd%#sO~aLfOGXr30G zZ2Ggw8cD&3vHf|KeC=djn+5qD2^*{%P>rD?Z9<+tw;M+2ac!%)Nl^k9Z(WK3)*%2# zNCyWeiZGYe8yb3gYpU5`r)`mfYKBs}z+%vAQ0_##Y~v_p>BQr0j+ifN_{s=aI~B2V zlO0ygEP>m6#e6aj#E$xV7_?l+gO||W`Yg3i5{)Mn(S-XX8}D6#w)nwR67+b^bM?5N z9T)=dP$fHI53Z^o?+TdaS}Zqd{hXpHbfPAOVmf%U4D2p?_>mzV+v^ z2+PJN;f|tKE=PmHQ-+iGT6P@QaUCpox<5HyG9naxC(hKuQjvFQpI1>bE!JSmC;G5M6gLA@4HC7m-RM;)H8VP9hQMV9FWB$t8Ln)M>R%BSCq+@B9l8PMiLE zUj0iYdTO%_I!@ve?(0HD3`M8E-?aW@x2`A#R(=rO2-X!@{%tt-Yn3U7Y=6e$Gsq;}!g_{DNS_IaAWCAsX2wo=gfsb8P= zn>0L!S5S-~k5Cnm4uUR`DImoXKI^EUH@U?W6 z54SbSrx%cgRltE{T5toz5vkP1sUDUxWI7HEffSuq6s}56`(XB;xXEr>NQv_|zFZ0B z{vWWt`m*+x^84z@vN&&O&f1MW&|@GqQd3$fXs{v7r)a78PSw+V{u|FgJeDK}AXw-+ z59}bALsk1^pu0v6Y^k802$NtxG#Y4SLTud2KXu1hRn_~mk7-N^q`w%2LzFW}DtW;B9&uu@s^T=|@8fyN73O*KmRFVY5j)d3CmaE&1JYaRW|V zYZsTog@8z*FF***B3p{57&-W^y=;UZmc^ zbK_`uT##F!n-}gMY)LT2D>C)`9=X(cS?s%ISHD&e%FPVM#kgpHe*2pBlW&Gu@X#EI zroM^jxN5R$_m_E^(!9ThL#l0NsJ>Bm=dxDJ`I@bo!`;eC861(fi=V^FiU!=k3i>vw z)pI0LF+1ksH>*gDFm*gTr8;VJ&c~i$$=Rox=F5jwLg=TQ;oc_42UhY7xcX?sMenZ` z!AR|v&V)Q}bqG|_Ap;Gj&sRTu3(+ z2-=P57tb4`z9_7%EIxwdgo%8W<1M<025N&Q{{k-Vx%O}9fJ8iDjKLYOEqQegyM)rB2NCt;uuoE1#{_Dg$JZu#vYS6JkfEKk@f z$B+9kie0SVh=|?&^E`uoWMh`~Gj4bBA}#doSm~Ca$ZNF$V77`Q90)^P#PGM za3Gt{@v%2_Pd0;DUBA&!wqyde+WDTlNlb}yZR}K?M@*_eZHN8$|L5b28YUwSU^6(^R_}(8k`)sQt()G>o z{Jq&CpXDW)-+Q@}|j7-?kTb6zz`ij0ri!*C2;LPG+Kw21vSK}zfM{`kKS z)z!Z4H5QosoizZ1#GJdfrs9d)5#vX8=H0;+e2Wt6)h{ACnRP}QSSm&V7`i#;Mx|7a z0Hmy#iwv^;tw~sYg}_ea7a~Q(3D;9D(hX@<+QbeQT+<2H%M1}e@}w6n?L=${+-SGb z5xFX95pisA#sR+q$2N?qk)3CW5usX4(vXL1lN2rW-3+hp zW-v`C-d0((>r|BB5;nao3ZC=oBTccc!gD^pw{r?5BM}!|iwW+$El4&B1PaHXgmjGW zMzZ!5Kk#+0bPR8@5D2ij9)+PnxUn6^ga!jtRAIyh!waqi14CCImOZRI507S;pWBb* zTkxFUP)j!6P7au{%Xf9tq7Z-yRT?6Y++*8^iZ|aR_R)HTR!7hh@?$!bc|w&R2Mtu5 z&unA^$wOHr^zV8kzFccoV8dmg=oov6~ZI6ZSU`fWN zWa!Ww96Uyn0!>~!Obp$W!mq)`1(fUQ5X8G`#K}EMJ^ElXv=Z1|Yb*kFZKCCnsPTcM%|0h@^9OKNICk$v{tUqIkmZ0S+P1%N{F zB)@y~LdE9fu8l#Uj;(MnKd~__Fq+b~p~qr21cl(qrJh>*ZW3}miBbx*lDkID4}Vd~7`=*w~m4De)&>*vOGb%yFc zxdTa8ywUZrt63S#Z4)I{Q8LdC_(7R@hr(_nb`|?=E^7jG?64GSo}BSb)muYFEMzn> zKLo4G#7)bUnr_cb@R~2}rZR`&q`W|MtP9rNWDp>Z{R;+I&n+>u{yWzPLdAalHUd}C zU^C!tytE45hqpXiqTd;Z&wF8vmWsn|u6%1%4XiuntGh-~Esu_jttYip_m z$m70Rs9Fr8gc36$KqED})WiG^@&hVXgf7N?>7F=PPDeN>PXu=VKVkzUXiyeows9zo%i`l$(VVo?KWPq4X~bQz%SK z4?Bkr>`(SMObrI6KhBHHdu!I?WDj$Ly=W95T^eBJS`C)PjT$ogR4&GIx3Voq83GGxKEeF~5&i14 zD6bqXpYhS742dPOW&bjl_ap0hhY85z zU~ODzw6ikLrFJo;NHGcuav5YTM&<1A9s-`)uSU?ty`RIbJ zEJ0X%D{UT}3(wgJWA#3BWVd#7o0kC)n}L8dJ`I1;qZ54_*J=ACN@b_SDQ~e~+HYrf zx%x4x-NI5=3Vf0E>9>K2t=u5>8W>PSGN{Gp9QU$NY)w;w5puZR8Uz|QD?_;7zXz}wkXa6?2fs%&k2fk?~hXMurL+#_bLKEgw{Da z#-f?Hw>5Gl3#3BF@u0ZHqSv24i)V#%o=DEiHA#c;7L`NJveSvSiD+N)hTntpg(RyF zd)(dgI&e}lfz8N;jK+-WEMbVMNT9MunTOXNPOA(EwJ~z)wmY-scaG|B`-yRK60?r0 zsT9|-X}+O5h;5PM)ui0=`viI=CtY59RCwbnc2J@G3qutl^Z`QM&wEd(nb&^-tXgv> z+#;e57>$LH2wR9Wb&~_nQebvlh z>#s>KD5YIGr=Bu^Pd6?Ebn8IGDZCHbtd0BJwG z&|l~wKuA&2jAOwv&EjA7exR)%l}Qgu!iqF%KGjsMS%d*k{#!+cnUrQfFfnWN?Z=Be zhrfrN8Rr=8$r{4xOb|j=zTpuCB^&m@1lB#6c$mAvZ?yRM)F{$?t*Bd(+xcZXZLG2Y zj%l{UHbanZ&R9H3;43ar24JAfz{P=h@8knBVB2k8V05GrUb=h5VPMn3-^c-8*QgLVHm2B0g0sNGnEJ1P~|@tTCs>x#sq~*NZm!)P?j)d^I7p7)QylfC>Tt+ z`zEY7HJWH9rXr(j!eda2XtwY@WY)w7FYXVE7U-wki9}f;Jr~zd>p~jHo$udjTo=Wr zI9F^8ij+hy0xT^AKP6No@)b?LIa-{5zg|R|Jx=kivIrkXmFr7s^^=c8tTo-z;D3JpU<1vHC9nj$*=3d=z@Y1zgTCawpgftvUpi21?Hns;s~*Oei0m zS>G&#Qbt@O?{MkXLJ&)%@IOh^{v^-y{3o(a zY<*nr76VdDj4(tMO^~11Q9?7m5_d>TiGP%pzeP?D!^Ull(TVwKZ|N6=eeo zl;f-e6M4Gekzz*{@$30Qz_V zq_}w}OhnYCr_VmSbd(>v*|Y)0Bfx{}hdj~ve3HB#Y6;0LW!K4JC=IBC6ZZ{xcuVOnT{_|a}#+S6N70*iLg+jrk ztljK2;9ODkARO3#Q(sba*`|uz9vyEq*Scx!MBWIl)Gh`2`(^x@Cw#1Jb#u7d`QUC}h5GT7_y7{KW zWWDiKn}7Mbp zE%<2OV0lfCUs549W7LN!W+foq_k2S?mqc*wS3 zSG+Hq;xdn~olhq18pO?Z?pO|fqE)4mP!x5C^w7pSv8>-IDVUqzp zYM75vC01CDRfTEY4|gc*xBJ61os=nB_66=cP3ea7w)hoj?2vTue>U|Jd1RPz_XaN) z%irTvOT`gpOq;#^(1{MfZ@mFjf~xUlKhq9plgdDG)UbZkZYqW&-j^@RLvwD(pd(_Nyq!a84mj*nK5HVxd{s4FaO?Hfq^ zEDAkbpHtAonym4_SR4WyNW8jyC`p&a=JaSJPh`zxxRQLe2+7XxWt5vk%1 z2u3rk8nPshPO<)ZDba7ynwCTE~l zI!X;>6P_{?2wGvjfO66C9+*(W0+rrH9fP&^BGh=Mnf(M1A}sJ6c1P&>Q5)*&WC+6V zZpUKfPnjlc1enLCFun^FK6KMbjopZFOxKQTee9p0@sEXdhw)u{`U{|HZ4gCi;~%j- z_tIK%J#|!vDminJ21(5c zrd@z3`PgzG*>ob%&m?V05esf^ zLDo^g{$18G47#-xpL8a;z4)can<+z=RIEJUR|UHgqZJ7Vn0LYZf_v@`-JhZSSbJMxAZomp$qZX6Jm9QJ@)^_Fusl&Z>r^HbM|H6|i+Mx9lKeuh4d{qMLxyGF^k zinK1NL4kvp8?u{-eTesACsnDY`=R?aL>DH+@|7yT&6ZghagzPN&!=DPeLSv1+?bM? z3ZXzNCA9_FZl&h3^iTtDHwB`{|iVugI6MYQ?R~>3Jz{4{#v*6c>%MboX>cb zsZv;a6nm_?1LXo$i*IHbKkwqQ&J6vF;+^x^PQkh+Lcgo?&!U(zdXAmkKm!%b2|z`H zWPdsq?-Y;qsfkm*kiw>^MAo{yNfBVb@UW-4Q#;jhX`*@25Ccm#S`1m9`n`bq0q)QY zckAnd>!TxgxU^`3STNFY*lynua($9Mi}?3+0`L*;AU%6ga)nR6TD7k6e)k_3M2$pB zW6G;F2H{AZ*2ug>cHZc(G63AsaM;Yv@k9eq@TT-4CMJc@d{%RlmT2Q2@;>RBV4BV> z;X#$SIJvq;T4oeK`D6=kH}RVbh}qQ2oL)}SS!CSeP1(_WAEOA#FV^q zcHkE0zEX+d_YrA!=R6HbPgx3;)Mki1D>to`fFY>ghWz+zzY{G^SANP~l2||fsa{ku z5kSx0(YGi4342X8qnGvog{Ve2{NwC0Eeyf6b^e7o}qM9<2fV=Sdt?9_nDC*tl%v3dwWAxN}I$5?*90o;k^aVqP8Du zt!~+RtPU3J*hRQ|n7N^O#+uS<<&z@Q$pickgwc2r4aPBhctlSw_m?4BVp@8mm#JO@ z8BdlGm9(Ozh2_%7;`WCeQJPE74KE8*+B~TfTC`X|>BY}{lXnLEr>_7-#Iuwo-PW3+ z-uc+?$Hyjr0h*0%m;$9AI}`P~4^_?ii;KlBD$@uW3V0mHrOhUJ!hU>D_=07zyYXc5 z;|?P|aC^{@^oTx3OPiVcS_K-$0E1=NtaOx+M~IsNA*<%5H}#(Mcyo&@6dylt=A}p7hqfr0%&-tTCg-NAyl!OhN z9%?ZnTym|g5CfqonCZ~d(XtyP-(xp+j2;J zTd9_oXr3@MSbuT$z#@W6D3)GK(V}u>>keP5+>})Um9csYt!BpBjRM7)_1~oJ!*K?uHoL!^jF_ny1p7drI5Y z_oR7I`brv<6deNEX&NiRun7XHUQ%YLU?cqnB$*rLfbY_AM%jcX1nPfRSSs3=QG9kz zjKU21wL>J=^{^hwxIJ70C}WWd59Cb`&%nxy(qP-&#mS-{@8bF*Zv%vV-=&36$%@L> zbYVRU(hRPK@5M}c!yGngqT5w5(?J!2i~+GrO4}B`(g^P!CR#f#L=A1K$PaDtPr7+( z@dlNlhwa64vgYKO-zqgYAN~cXzW9Yt1~!Dsoz!{_D=GvwTbSaK zwSxkZH-=~&BRmIW|EwW7{xC@|3Drar|M`f`<8auz*chpniNNaJ{aFB`|OMY zB5X;eRrj5O(1(b_J349SaB|Q7dfzqv-19szjG1=E5d;3q4ZA{}%v&K!3l|C{7sM zB+VLEKH_PSi1cbol&``I(QaO+^08!sO#$gwDars&Q%t9QMrNE|QETqPuq8);sjO}c zi)l&;?v(J8(sM3bEnVO=ASNh!^6K_9rbptK(0fNyBM=~pZphbeWX$nmUAPhoi8&8l znKWBVuuc1<1Ji#4*26;9O7Fc=_ZyC%VCrm`~A+U-%3l5Y@|*g-W(oue+-k@s0j zxJN-(e8IT-xeFH(r0xcwDuxKCRBs`}Y>70mf4LSHj-3PTu$xLal~~+0*FkSwV?+(w z1!N5Zm98n0!@6Zu^v$)oSgY0eCC@o+asV|xv&XN2FSk||Xn}-n%VDpwnMrY0=%8SY z5$tkSLJdq9HPk_xS1NV6X++JzE$oWCWpoprdP{YxW;b#6xh`Vb6p~V!M1r0sK>pA( z;sD;_Zo*IRdQqCn#6Ve}US;bM)!?oOYG>Yk0 z`kUpjCaWR_K$?3VuX%XENd|*g+v}SJQeiadvW~_OMKbobJRRRqU;=5cpp&)qgx7Sj zMm>CGvlji`p4n{H(1eBbBgL|__her3d%Nio(oJ}eSG*NpmojG=AO z6LxL#@S3GY2_&8Ik=5*%A9C74%t!vy$FJJ;X629$Jk2*A)01y9q^N?7IqP|?nJN8L z`o!Q&eO@mMI3V%hd>)dnodeuX)7h_gvE7@};X(u$o}F!&M-$ohx>Eq~1n)gKF;KN} z>7j!A&}IQWnV+fLAm{Dhv+b)^B6}l$sCx%RbyKsogV;^U1Yrj)!=QNtoX2O@%b|>; zt=uUahi0xVp;HGLVC~_i8jUgAPYv1st=^fOeN=4(_q+ zFFpO8qjs0GP)Bribe0PK#`~vdr?ur^V#}-A*VV?qXH#HQRtHX|g4baxE?Ecw6yyZu zI`r+~S08#3vQkEjhmAPSx1Qu~$>IhYXHBwtxXJp(5~I3Bd>)O&g&)G8 zC{E3laU+^aaEMfj69TzT$HUDGYE~RlA`a}Uz4ItDl#{uE7{wG!Nr}wGcntVBE^sP_ zHc2L>QB`fMb!=p(;KZt!(_SrLQ)}qSN)opy?aue^D&2DHv*J6aP)Wc301)-r&FpJ zM=&uf%d{XVF&P26DNLRzV$tQ)s)Yl@j7d?A8aTcynX-aNRAdbec0`S=TbFrzu&{g6 zyeRiZU<`#Q+Fd(GWR;t#^N8OQ)_fs~EvwJZ;yC9}h&(;57cZe{2?|<3tcn^k1>tkO z0)ixt_gv)6KLZ-Lm0_V;)2HppwO}sUwoV)}WN+crSV~?vB$0%JI<~fjj`3IG5YZ^g z7lC7#T~d3w)_MmOcNA4K$4^4!k`R=6WD=alDwd0>Cf}BjrGP=6hF*!&HdC%&pbdqT zTqQ(AWfS4kb2xKINB~S%Ra|MXv0NvrlJaem2XSNhOy#`8(WtyxO(%?zE{9ywN;Ix#8CLl`+*LFclzAfpV&HI~d+e)rU9o$JtnEYDAlf}lAgWHPTNe1%vp$RG! zQpPHrB|K;-6syLSeIC+q`#Hfw&JDn#* zsWZ>6)!Kn;g(kZ6pwDQQMe#GGIZhjil)CWzkm>lT|JPm5~J0rwGanjn6jc z3KfR|MZltCx(kSV>pHI`yY5yYqogM=8wL!eWAEVVPPN&Npt|Stmkp)!WKu@wBt|4{;f)2_Z(Vl&r zT(`r(mzl!1^6?~+k%u6eO?TFw*0IVZR0vX(U#Mbf{X5N9{{Wt#@FLiS5#8Tk z7Z$Y=s4#sTjoX6Vu@$Ga#0Xh6GIv7}PfFkyY{p6gc6hfWsNt2;cRiz%xS)~_XSck2 zBCOk7ii{GX4CFv0XEc$NH7P`kfxjpf+Ef5*2XQ9ZHQOiE)Fi0HL@M24wNVFFL6D%d zr6@ay>hDc;K%S!Xgu-CfO*fj5@v|L7Rr^O7DdW~$a?T}zNXvJ%`a6gt5{SfbOlDu6 zx{@xeY*Y?0N30pNWqe65m!}$*lM#lyHxFxd>nTl=os$*PMte(FCm~?uDh|xI_CqrA zyu@7&*CFy zogwPz-uqWiF#iB35}xf8kZ{r~8+EjZCrorZS!%ek<)r(DNRPsyt8mi7FpH#am^!x8 zUQSacFLM_XtR*|?9dBu}O{Di>Sg@M$)yJ{P8#8c7s1$%BO*|_1@PM7?0*?bTryyMG zp-N5F5jT=;tYn!K*;#4_ceJ@HsKipLnSf0xKss??-sRVf0ZC3N+m_vKqRNc6#C4R5 zs>7V(fwl)t4*3ybSxQNTPY?h)o_RE_7f$H=H+0(M)hh}{QyqAecr_Z?mX&t zQS?T^Jv6>Tq|;o(n6-YuLP3cWo{$EpH4dAhxmPFC?nDZ9*LStKD~5I}0V|vma^N+Q ztc{K|rWSI~N2_?183fVoHiC#dX{C?Ls#FqZy-vORoaLKlLZK-NN~lNm2Qy=wk|VsC zUIGT(kB*IPcdYqj;erHK4QWbM&6|IGg+|%Se3|P#jBA#tumL#KB0r*seAU8{pSg9@ zro=i`!kvEB(bKbZm&GI`kOyAvbvZE8=1;<-p-)qNLB=DeiKu+0gQ%Zl5c!kvuc=Q! z-1k+M!}Vxu%v#Vz1t4XrXG5e4oaw`MpwcbhAAwm+kSh2}$HWLUkbd0wHX769TZ6qK zwBhZkc_V!ekl$|+BM3d4*-7QD&|uC?_jq*T-GhCrr0(RY9tV2AXK|QnZX!ZQ2A@zbz;L%nZa#3g-;ZF9m|N zs7y?I8v>Dd4=|9afdZ$ny;!iIuu3C3(sl+~uF<%xFs=Uphp;+QQD`iY3TPHpX(sVd zf&s|>D08?z@T#yaXqjl|N=GI}bU*`0qm5l=YnL(_$W`NYeb`2M&1NI1nRAz2qy%t({GbmH60sNb?|TB_3&8Hk~pw|n%!8v3>(O35;w@!7fV?WNd- zETg?SOpdS|m5df`c{hng!y+J%O0w@H5nXI6zb#o$b_%#ap3v&om*ljg$8X>EA{TIz zUA>I{U!1jR0V=nMo%G&Y614JWE+fn)o4c`|-ht_Sr{_NT5o{}TWfa}2UA;VZT*l)R z`wMrKQf-@?dgfY-_(ShRMPPh011;{N75 z0Phqhn73@jREZHvo~N*co?u7?IM#qRABuXCUhaUarsf}pHPPWqSvrA0D##_`njtBL!|I7oz%CMzI#wR{Ct<&NZa z)nh;+*}@Fs?urdE!tyqza9d z8_xDRE#T%5f|^QBS=Tu0a{StHAW(d0^{aBl7bxkJS280W7(3(DlgrriQ;j4vulEzMs$=>?(mwW@gzxUPl=+v}LH_Q1F?>2% zyv3Xdlmy!|n5XZr77( zBvQ+9{{Ye!t4mb|9Vd%bo>^f)tOFrJhj~c@Ov5t@N%*i(Z8O`_S(a?3LKHy71Z|DV zN&6_5WGA=C&K~6n{#XHHJ z{Y~7rAcTY{k`$T}l~PP?XP#ct%5aL?ULb3nf_*CCw1nQ|I9;+&!(7ERmmab_^nqC; z(m3kL&+{Cevu2Qx6s1u7N-2TA-i+LOc|<1g6OxR{n&c8R)!Zb98Br!~{7^A(8Um<3 ziy364L35B#TqibiEukPJ&v1y_0z+q(lOTXlzV&vGv#sLZ=44p7kqM_>eIb-!feFeI z1R4-hYo)msZ7N0ra>I-c=m1b3U2Kdsf-~4CHPMZQo`080!tRA7LUREoYcm{Sxd#t; zgAtj_;@j5dr@cs~W+q2txKgnRf|Kd&Z%{TttIJ)2iMPKvGmrNp1Ip3NCsdP!Jr%hj(i0h#k)zi6NZIA&*F)m(G#Vbuj=MI-tQ9uROz!ERt=IXR&vxuUxhDkC{FzagN;@ZO8oQk0BHF)(5) zxcIom+ViiFXbA%sx=vKy&9iMDToe$mgvs9`BArKJcF|7^LO(HXV-inxDVIt$uI-f& zLXrrq@6NML``cB^*i@6aL5ffXMxEzFNaZ)z zkZJ$`WHz2VcAB|#^7au>p6SJrRM!+pI_PL*Ju1z=yfB)@1PD)dLTN()E3wj*b1p7` z`9<2bZfCr#=P?BQGnDUTo@o>DaO~jXu3HO<1S)D1@S|Xg=R-LL+g!b~j@suM21hbx zrbcOW;<2)t<6$Q4vQn&aQ`BarbNZVR%v-h~DqxZ^S9v`(K8{+-?pz@Nr7{59DISJ; z**5H2NS^7-5hjOc4@WJdH}^Tqs@2N~xo~4W<0i3Fm12GGVEd`le7nySS|yVQtk5>7N~+1+MwhK!9kuO(7y9` zq#zj2VQF&4#-1x1Bko)mxx|B)nQzz}=m8hfl+YV%BykEr9vOqzkJ_fs}Ta z=KlaLweu>iL^~XD(Ae5gxx&#kooV&b>D_UzA%)MqtV9 zVhgzJCiGI|fHRE7!1>k&HZ^FKkvqp@lu?A3&BR-`hyqV%*2uJ~C0WeTwg6~d%Su`n zJ{1@=1w{Mi(@qyMmb~#h$dyq#1&cBd7|n8^1GmRzmH8{a%d= zC1*7zDoSZQ4+b^Mv`UA?-W$W*sM=L3K&wbDsBOvas6^L24SeOrShSTO2`U1)WKRu@ z%K-PJeNJ6GwY;#7*3n|v{34FM#mIj0 z?INtB?5T6$CVM*8k9*5XlU_N0CyAyV1Kq>(D zj{TGrdKSgb;%Hkep;!IIW$4LWtPuB|w8h@cSd+lsW3lb(I7ELg;!c(Q;f5>S+ry(I9jYvQgPDvtNV zJYpfFD5R}B%s!QN=b_NeC}N>FhDWo(jcb!iE|rBCRLWogXk$n<%QoR!^*3f6TSKvh zUDYZIMCCj(e8xgkgmWkYv^T2d1P)TF06 z<;2wMFOYUb=K5wE0m`MNoXf!~O|WE`8|CJE{0mywI6yS-XOln`jNp{!B$9K_wX|FX z0Y*W9cd(MIWl^!!_rMuIjHR$DR|FZ7TLCFf6rS*|&FodG+6sHo5yDY!Su`!^U+(uyM4S{G$QS%W|8M20I>O$?LD3waW>6lbS zX!KAAP7Qd05se2+8ttVllAE9%nzZ{HUvUK~5+YdmR248{4_TKSyBx}eAxHa%{v)}l za{SQz0a<<|?$Y#8?L}$6B%xBMNFoE`Ri5F9C}`Z{{ElR0Z$yt_ck-9$L{_935hgV4 zZo5$HwGu-LiiscsJKHMURc!wN+)r5RaX*yiAz4t7yv%G7zjpSOa`(wh_ezXG?Bi3D zG_GiRdr99i+*M;J&pmFN$RGCr>~S{vPGaPOFq*|jS>GNF{{Z3FRc{tLg%CzU0nBFS zOnIQw+Jn<}+<&+?XGn+NGTa{eE2+k-kh>P}uMt)$6iyn`T(It2Rk)QBW)5OH*rJ~l zH4DL_Y_8NsG-V$U-23KQ#8ryil;J(>#J@k~t$zuwKpGWJaWs4UzcNszl+h*)M1z-2 z?dV+4XnfH=i>V5t&}vN-39Dv5!$K=T9k0#~TbQ;y!XY4(&k>l3t9pFY)+A1Gm)n~h zIW)uDd?=n!NFiI~b<4*!=HEaFJH+JHtALU?y#)~QU1n+1I#Q(BKskt()D)y_>DKT} zhLJF+$U322jJTnJQBZSZ+xxOscZx>GQlWr`>w*F`Fl8O(gQ@45ux_r=3d4qJw=OL< z%v^4`saiBOgB}5QFyJ_VP}pk4N>Y`nNC6TB0D-==Z8Aei7(;6T1zK3RnGi(lW7_4- zlX*p79n52NL703&!XP?=N);!6gIHYvRQZEgqZZ+KI8G+XIj|e;-YOEyJHh_1T zDBV!;Xh|$nNjg~^x$hlLxBt>@ovrUO>iGW8S$7q-6PpN$)uj#UTXFd^dVF z1w;do$@1B$dB3|+XJEs5Vlb@jS@tk_l`G9SY9KZs;SX_&t#J_>ZrbhK^C$tCjLAD( zm_p!E25d|R8JUW?66R^5RNCQOw7CuQnysL48Wq4G0VjS7U0UwaIx#*Ldn^2uM`LWNl=rWl+Wdw{8G- z(`(XUKo*OrDmiXsrZA^hO0W(<8{?*j%`*2$=wBmePS|J)?@p5XMA9|Y=Dj}2U>uRt z>V1w5*H(kz?RVv*{Y(R{!?_xIcsxNnO90r-$h^-Q!na5Gx#7^yvDNW^C7eWpRj9GwsZGM9P z0GH<92}o8hSRH0n0qd3?Re+2j;!ZG$i8LmsVSwjVM1@InJled=YQUty72Us6?5NE# z6B-<)@%elf&^z~2E4Beq#A}{iPCnhW`Gp`99g$jVdZwQPWf-vqYLZ06Vc(n3ydgVT zXh4FXl-wB<*APDZ0OSYC?WW~qf`Vk3s44MsiW;%12C|g~KzqEpA@NC2eU5 zIMTdleQ@E~lt|7PqQE%t3Y}7n0vUG3qkK=yNL0E2&VXe3~u7hxKePNm{b7~0!TSca-^|Vm0VcD72WR* z(+sJHRzI!N}Az!BwaY!-&|)3vfUl}A^tdtXYRP)`$2LRbRI22eXss$Vbx zxH>T??COrv^#^CS(i;IvG1kOds^laHm^ufw;|;Zb5$b(mwL=+dJhZT^r~zH9(M~)n zJ*{N|By{X*!o+5kosU+kt9reS*6jj<4RKc^K99I?(o&e)@kW3dVPN+Vs5Dno#iT3~ zNYKm%ch5}^wA&gy11u~FNx6?L?8~yGLUWMV3S;f?KCZw5MKsT>HDVQIDo8k=E}rm* zz0Z=sGLitPj-T3F(xk-=I&5LDH_)a~1RK`+2BjJuMThvbvkpbN0Tzrb>Mgev5_AT% z0p8W^1=R`b_LkVoixpX@_(O{(3zD?BLUlC6ElP=bLhGT5kVyk;C03M{3~nmRT8gJ- zbNb!^!?@=3S39SV*w}}qL0N)zL$u&Jhl94tAk>4hp+3iwMo>TZdmMpgBvNbAUnXf3 z*Pm11_i+0CuXu27>C_knrs6eCdwq|Iorg>0rDr)m+6SR}IrqYjjJ}az7cJD^qqnu@ zv9NKg)hCA#doYmsB6CsIK)^dq)$4na6?u;J4D{3<)Cd$-iD(4W)^gh;(q5C^$frq@ zJSZPi7KLlrbVC;E#^KU>n*qy!WS>@*T{W)V-)j|SB=q=$Qr9Vgv&Y#k(z4WB@1d6! zzkK=^7ky<}bS$hSpq<&BKU}dRyILgHdmivdOmgTJqKOhtd>v6GI7p~752z(7KZ>oN zh^+5=4@wk;oM7;87bNdvi+2<#CFomJ@oXFuHf~N?Hu;2s8gqD0^(RB!yI7W}qgx=* zofDj;wA{i8Awfw|(NAe>T8SSpwl@?M=`u%SP%1)@N9%2;=_oU9(&z{&1fRV~3B-0{ z-Z(yYiH!Dj_UaTYh1eXrP6e(mNd~kN_C0f70@sfM6au~JxG920MuK|TiUCCz(y#WD zgATYGSg6NqmboY%Wwo1uCaE~-xYxQ)m!3U3b`gC#c^-GY@#ij1%2jujSQNsftbG#V zv8h)SV+@jD2nn7CW0(1QicQ%dig4QcC3;QgZX}m62~rYF84z+w8xFmVg!zfym06LI zu0+QSs>ZmFKMd0kct-=NxeCTrFP^qJM4U^ve&U7|U{98!GQtf~bKkka5pw8N<#&Pz zlA}{E(Omn)MAagkYnIB!Y+{59O9egaks#&WH0*a7cp64@601o!60k7++?5RI4LWRO ztxCcaWW?c(;O&|<*^y`xWY&7}(AjlI%R*6_Ph@4A$cD;`cx=EjKs7Zx_m(#^a^@>2 zLGIj4s6^!*Tp4QnwL)kUp8Ufcs};d83UdBx9>yhIyhL*NLFMkL0v4i1D5=D;)+1|* zJeQD?zF|rz%L;P@@s~AtM=@-Qp($1&5(ztX7%nE_uJb(Fxpc1z2n8c_;S)}`q-53~ zf6&r0@nlE?8iHtqxtiEQ|i$elRh19ARPr^i2DtI`>K{7Jqo9Ir~ z$4q5miSDd-juJ#aRDpn>?I1rQFw=RZOZ5sRGf|xO2cBmGEbbmK?#b!V(9=8pL~2&{c=<`ZpwnaM$)Q3@KI5*VtZ@ts}SET~SZI3DSM7dFH(p z`kDi{YfNX;(d!u-3(&*7t4w=p>V_lTQSB{(fK+O%BTV)O(xI|=!5xww!W9*+w1etT zqv-`%rCo_0jR2&ZV>t(m_`N_PK~Gp8_?^I#YDS?4we|qvuk2tM1Xh}4cq`SnzZ(Xz zms=dAGrvN9w&b8@oKiQcL}&!C444%MO(W4yn5R1EZBmSUMQ5f`mKuWQOzHZV1JqL! z2hr&PBv5tN{{RzONZE+-A6ujlWYe**G*Cx*Wk?(oPM}!Exlf}g8QD$bPgH{VtD1Cp zh7259#Zstm3kjt)m)H>`qfrVykm>JKp(BU@HX$BOxy@6;8%?AjR3D^~*!GZ<&^(mK z;yr<&F@-5irx2mf?Ju5PDh_9>)xAjW_fsc)ne{Cl`LAcx*c-uZR4CKXK4HVWY9N=Q zwHg^|f_Uj*4vhuiSC}Vex|@} zjna6T_Oy#?nkPL?0KhPBL7ma{7oPTJM_6yS+|HUj9uZ3+S?AoQzZ5}ZBk_%u5hR!If* zH&6-OUTxfg1|+>I2YQw4X`z$+t_38Mgu>FHJBX+u+nAMnEjx%S;yf636*Cq1OaZ#A z`$`u|;&SyIUprf~mVe#!;B27byMZZ5QIklT?~L`mT}rSdpy4o*yo})-ax92Z3d+4I z@orQIIT0^DCPW7@%G_1Ma8n&eR{;0QrtPAvSZV+qbIr<_G}$?j%UzWDX&`Kz_H24< z+Kbleh~!Foq4gkwNM@xW18nH+F98YZPh*WlSUmznXT7ixO+#rK2EPe(!QuxWusN|D znGp7hPS01`Ms?BbYlzA?@A{en*o{@3#OPz&gNa9ZFJuVSucu?lN=Ov;x&uK)R@9xY z+qMz{zY@&(6!h)Q=$gqBu0yR& z4&O30T$Q68$?N)yBMFG7TQ5DGH9mDWt-M0p6fLOtsXSZkk1ph$ggs^Ug4ZesDs2v1 z5zb)!_90OYr6DRRs2W>>WLB@K7)+6^%GeHE3P=SNkSO}KVx6QuUs>*y(gBgWJs8(! z5+f9VphjbaHn;%AaHB*?te%?N%0}s)n_mT$t9N4n+AySlwvooF?5o@*RV05ZOHj8I zlGLQAekP~~sAwEuds=|lnd2aO68MS=9?uZ=PTtbk8-%1a6;HyF^@5T}8ed13wCDw} zJB?%QcN&i2rmbu8w*%MW_?0AOMu2oLJTue>tJx|8@F%y>IufOdEvZ&WsZ{jhz!&B$ z=AocBAq!EGR5S;`yg49g`kMv7ekBy}NCUNiadh|3P#uj~T zF!h(SBjT>7iX4*I8U-KZzmaDCLAc;RT=?t3e{E3hE6Z5%7RCm(-{egoNo( zeY%@tcp6JUYY_vy1%Ifq@~ns7E=3l4n^SJsNyep`$qLmvaUq3LV5*bzo&K93Nc_lXz*knk6h;Rih~y^*J?i*Dc|>X@3<+%RMz~<@tP8QmW-K;^h@SU7#pqUgyOpzQ-fQ zn6?E1yX9FzmQ6qcbp5U322`mKk=`E@=a!}Li)ow#TBuHnYkt<>FJWj^g{H8#`V=6`0jP z{*%xC>BcLZijMQ-EdKz&v*@n3^UL;c^2;c?g3c~J){_Am|@SB6g| zB#g4+XVt#iLX}z=ISBSWTuut;28{rO%+nC*spUp0pgL=1TS%JJ>3O=0AZv|{fTV@5 zc9JLB*0d;Y(;W*Y+y(o*E3wjXUBga+fG6Tk*N-y>VV&fO-WZFwh>%YTFbpJ{o#78C zD+?f<)T=X=wh)pHYSnclUj$dV?JVpA0XULDn#k1hZJA6O0UTMDju{7emY!i=T9Xt9 zHt%`?V&X@_RDZM=!fvZPT1(JL18~yTWSGV@c(4WGQQX{=VHpB-$FcMxKNI0jz)rSU z8iJuxfx#k6Vr{4h155^_rCXqCWd&RUq)yg=Nw;uSMmt_WNZeKE^1G`I9(hX@fJ&x6 z+a9fH1RR6veCvbZUst0^Srh=z7eErLk&;pe8npmTW?oT)@uc;$Ql!Us2n}EWhSJYS zeL{S*CKbSmY+A-yQ#a2Fg|NJ@8wPZ~6v zN=-_uz`ps|XxJH8NHYo#30<-AVr=}x<{HSwx)=jZ)cye;tz}1go-lyilVs+=8EnYU zv5acEsd@x^Szcm5qy{>dJJ<%+5r-vp`mqmNbOXKiiOQ@3p#)SVl!Pkg!5uQqi;YKn z4(-*0gcxqD2`k@rDyh|<)WVq!N;H>2?j0;hbYypldbdKSizFGY#=B~XdL@Y36KO>z z=M&=XyP-CW=RU?RI4X_h;P*stdQsBheYZhGs_qz#r$gH91>qwysL;l?qK@~yK+6&e zRmHwW(Nt>aRkfi$>67aEqm3A&dFqDssW%D|cdNYBeQ8XAL4}CXi3-OOegjUY!j9;< znVIzsHik7&=ibojaW38g@nn5iO^rgnA+|dh$|Mpuq|;NvlxldNV`eD{q*TvMOeM6D z&yGzUDTlof8^{q>tmBimVJHH%?1ymZ>TduT$+RSrB1qKafp}zO0o4z9VkT#6er74v z^|lLoHHvk)um(b8?bdu=@Z!#$pZH5>mZ^zOsp;y-BDoLcpGa!J>IoLp8$hVy&b*%J zG6o&w!wjUP$?riMVD+=EfV*+H=JJ*(kUM+kzSirQ?k5UTRB{0#N$PUEhsbaI!9wCz zEZNAc)RHs8o%+l_qFMQOE6AljQYBGUs;WA7Xn^;IOPfywGst|ixb@CZ`$ z?jE~g`_5mMv&$z1n`uDb8}*#Y*7BMbPAA(OxWfR%V{NJ|G^U~KP`wJkkz#W^hb{99 zSUHPfdlmST{iiXZmMR+(7}9u&qvZbpH2u})Ieb?+Ja^DP2r+aGKtlfjmb@Q`C_-}t z5g>|4JVXNj07;pd zO;G*6%)Ud-{J8|jooo;H5HkMc$tmk2i}4l+Q6f72$H}F}?Fen2U@1^R1`btaTp?){ z355*KY(SJHbfMlS>h`v%r|}V{J+2-g6-#6q)n0p679tW0R~v=RuD`Bz_f%6 zQly+mr(i_Vucl{V9@Y# zkcQj2ibw=T*glU+RFNaHkCs4GjE;0}RkH>wLf+OfPZtyBVUaj;i1xfBuLpWT>u5LB zlLP|Y0ZId)1G($SKZ!s#Bq1AA-GP`^rtp9StgAq3HFHLL+_<4xYOD;E6=g?)BY#aV7?fWPvQ1q9 zA!^R@u^l(Qs`a#dJ?@)Wh_rP#iz(N|uP`v9k3UA7AUCZS21In$(k&Hcz&aEiSx(^k zvWs~!O3Ju40|K8^TBX(eF3L2Mn;zv>=vYW3mLjg>RWf2hOfu@Qqp6G*RcPui+!%%# zW6462nWTqmHjSeTjhaH&WExdjXxi|F{6w>jv2=oR9j(0T1{-Ne0t>fE(a@o(TjB_9 zi)jaKSfqt4GP#76d0=n)nMTFFUdoPUDts7eM-yNaV^wKEvs+43BD<>h=FOz+h60jR zhh(%cn~@lYKp6N<>;%wCM`LtAQg>l*?x_h#+>sjLu~mtIZDfqFl<3W--7zR{F!D?- zDLY{3&9A(P1EDREg^b*(;5w@J*~QU_4~BG3=b)7kB!D=%IVUYlVIynUN~7VPKV#}f z;ztOmee%`2smYkcqM*ptx#*`6qGM5bNKW!W)knG|P?FFM2*M9~yB?C9Lo=w$#ucSR z)aqmd5L6SpgOEFK5~x6@01MnCDto!o2|bbZIZHA@uDIX0_V~aNwqf+?8O(&(Z%!Q<4r8GGNm*;tPcQs2fH%bi{ zCkrPcUBnYY$FlwI-7BWlbOLyZdN#-5$8Tx%yI_}>uHl9Top>SDmx8dp1Wmak5~5}pK{-r`E}^3 ztq(On%b)()(*7g)pD_vrBQI<7b29z6u?#OB|5qYY)wEWW3h&^6?413JRV>? z0U|cJibY%|6(r0#1YaXC(*uc7Nl+VW(6-ySn9geWxKIj6&xfmxLdpt<^%i0xVpBr& z2sI8UT?%dsX`lrc_b4>LxU+@HtCmXOgA)5@*OE-+Mr6>tv;QxGq9H zE5IlloKiW3M|w*>*WXnH9-VJJB!vTuCeR|Y4yM3PXV}+u2{{w$9&8P+@SljCPl>{; zn$m=hFH;#X0&tD+$5s)M>8;?ai%nZd3oBIs&>1M@kK$I3t&9QAt%l(P;yrEH1s^RG z9W=X$s)2>(1!P&a`CE(3z}uUUHZ2Ux=_pQ3>$hnMs*bsj>jm@}x)1F!Fan^~pl0D{ayhsF}#Sq%&@CMxj$(iHm_O*GS zg?=L%o^Ke-ixCPjCpl)ru%iP48CO!6!|v6_+{BzJ%1d&On?&!YP7U%9tjG5sYwwsw z{(-4Tz+!$1`&!nOcYmrnn-E4MsVi3ex^=z`LMS27h>-S)${e$sGFrAf-r4H)tU4(| zNgP`N0x0E}nbg_|h&ehyR#MuVW;YigXALBJy$%8$2g@)-7D&LiEJgu4TLvML1a-cI zTc}cq)mK82pN6s2TEq^g7rTpqWThmD07mhV8XC~BzrYZ8)_%s&6N+F?*CH2@(%o@t zN)?faq1M%LII1W>&|8<@&wmFzgnPf;i=*-v6tf@m3jYA{+kT0B&N?JO+I%01`523L zM&P07{VH}pPb>N(rYTi~6N#m(nC7{DQ8@41O{DEniPSSEgYlf_?_HG0|s-(Pha z6Y&wR3)rnm9_K9R;CauU{J)cw`JQse;J042d?Swj59S=-&3WE-eA}lF>eJmvP|r-b zu);Oj^qyDu%LU>t@}tQy@TzVnjX+Ajx4W(VACl+!Uzl6-IeUq?1Wx57{{T`=Mv(si zI0jEI%--d03$`vU(#Az3gR5a1L6BRZQa?!%6=WlrQbZb#Ilq{Ev-iO(!*8EX6PKQW z-9O%uRO?KR0sRFR5qSw+wWVlPQlyYU(=JP-$uP93>Bl!F>}JC`WxfHj6q1z;K!OO` z3-~w8IWx&~JjIW7L??=KNz#%Z-I7AGe>?r_83DHy=kg?!R&n5@wne34bKcN zM=wo-)rbx=jyKjKjmqvL4-c)4ZVF8+G<-<`B*r@3O7+zQr|Rkxn3?s3qrmVFV2S0qMVF8oNut2 zCy8n*3^ZLNcbt`*0^ke86hEiM=Yc0}d1ro%o|SzgD*7`%?$F2%(&O%VYCY89>9Z#m z0Lmp*J&%D8XS&BXGXjpr9+VT1WTarOBH@y?AjmM>tE*V4X{H0NttlqBx@}=8Wlx3S z-B1RmNDN+KcnH+ZMp_(aJMkw`Z7t0pu0re_S0wCa-B<}$ryXW-9ZmNSaL&h7x+^h# zb@m5JAQBHdv|a9i~XlRP5%Jvy#}}0)=1v9%&G!;UTz2f0MlRo z8_FzmAxWNBn~nbf^4HJ*0PAu;IVw%dAu3Wq7iS8JL5rKlaSfZ9*=c z^>`Ab@z|BVi}wXNUTpsW^w<9YhS6frHETXwoBsgQ-~Jnt`AU?be-f0eps7Vs_je{S z6C0cw7j&T~&AlW8{^IM~R@#C|{afEI9z*vf*|1H^9MZ8zEzQ(_roa3*e2!`olgjgS z0Dth;_ka9sHQl%j3%9?_U+p>Hoh@m(VJT9Q2)a%cl+0Yu9y-*MP zi_SH@iRJ=0tEgIRnZb1YT)#9p{#Tg(8`w?MfBM|f zuljCh_)mW+DpGG1DM}(E5{jd_cPSzFa(mPXo~s zvJd>PHV^(<{D1h`?I{W?3POQF!cwHE3Q2%qg#dCP!V8ywTX+21pS0%EN9K9Y{K5YK zfespri)!}gLTZB3a3{Jgf;W52n|<(Agn|r19ts4(X4yE>PZ!E$2&&bY zPv5RYjnt;*)ciwOYzbap=A5_6?OgMUij+naq)L?j8VYlc6|oW_SvbCM_mio}Cv%PF zzBKCbZDMPe{e)Xq5=q$jtA$#&HMSrEYW1V9O1mgffV-iR=`Fat*O&sv%txSGH z<)8XHJ3qCit}D;-&S^TE`yVbN4xdNsKYPFb06Ba6eXrA}nzbYTL*<|PI~#ws_RknN}Q#r+~GeRp06PeBF7*o3`k4>R~W@sHCNY$Z>{yBgBKPCP^ z@pt@k{{Z@aOZfwIN|>Qg5&bP<5$1a@_*zH7R3JWb4d7Ya;co~+9{e5 zSWwzr#uV=QP+8QD;&l%|^#t%}{{R}h{+p5i03dk2e~sS%08Pn%kUUxKzF}vm`P6xq zbzc7f@&^F$ub*>QBOR_)Q7~0&5fW<>qSgL4d;K>h{y_0=P`7pmsrg9mU;bbCF26jA zp*n}7T7G}cBjL5mr*0cqW`tH0HkQBfyWi=#FY*VAZVyUV&#YD$4I_)6ubs~S0Ol3{ z00JMB{{S1m{-2Wn03dj|kC$^dIh5bLxC&j;Oi)e~?IpyarZV2{gxTp#Z*Q{@K-$n= ztx)TS=QEsL#X(R5B3MXL{Lv&x8YY!G{c{<*U4}0mHzf#vQuxI2Kql(2i}c)m)Llh;GK%FY0>oWop~QH zMV@z-xQmI8EpeDRrxJ4?i!rhJOGy^^oxf9vcFh~b6xa)Ix5?`5|w9S~$5M>Sp zrB>8?$*-}wkgSWzIwTvGQ5c61-*s4!bT#yy?b^rhX literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000008_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000008_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..18925d17fe9a850e613d5a65405fc51a20f3d887 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(@|4HF-wafbk|uo@M|OiRdN zGRnpbALLYq96qDs$RRKqZpfiAnob5XJxl@CpgpX)`K86FSgT6F0)apc)E hXKX>NQF>s*0TLUuBZ2X|7aGrWr$I@wG!aPX0RVo+bs7Kw literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000009.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000009.jpg new file mode 100644 index 0000000000000000000000000000000000000000..91e440b28394d2bc60a73efcd3f4ef10f43cfb57 GIT binary patch literal 32197 zcmc$_bx@q$wm#Un1a}DzjRk^3a0oPX2X}XOcXxM!ySqEVA!u-S5AKj)8NQs~x%b?f znz{c>)$Fd_^1kn0>silQYuC2^v-D>RfB^x4K>#Rd008Rk1^Dv{K$p^YG2~)lW@lz) zVFCPE1$+h|A|t;;euw(EV4!3CcfrEK!o|gXd&j`Q!otD9#l^?Rf4lJZK}bkQO8Wji z6%{ojBm18LKpOx64FmO70RJwqFmM29D0l=!D8Szv|9$5T4h$?DJOTy)3I-Ym1_=%Z z84em29t!RaCME?87A&iXyuLj)TzuXoWu3o5?=)M2=r+!5fWwRdo+1_e6?5OCbW zaQP^m)~hRH91a_6N}qMeX6oBC)h)rISR4e&(8>dyuC6L8>bm}`>wEmKtN9OB(_VE5q zoAXM0&~piLu7OAdJ*|<;nJ==ttIU1JN}{~Iy~RTNkt`*5v(A|MimSTn_~s|aRnPJRYp@rV>(e#AXa`k-`LfQKys7GT~jXXjlWs~M?C3A^;28h&Z?0uc}oo! zcTevl8(HW`joF6Dn-qa2Z}AO>!{(aOruHY9?acj2Yh8KQ%(kg%g13f5OnLg!=!ox$UA8Ne8~e@4XGA;8)TaC1D!ikp`61(GG>5yUt%HcHN}F zAt#~J|LV52rOvF8xM7ea^!H@$h=AgW|Dn?tha=8UpSm;cNs~C3+MT9+mosz&C<4Q& zC7|S>V;8cZll-SOFA*O2v}T*iPbJ8X??-`W9daze1~djZr5uvsA!N@s5+8!KIy)9; zI(zHPmgs*Q*qfCJ)e|73|H}oPe|hr>Iru_j3d|^f7CqN$^O(!I-%oVkZv{m>0JwLmfDQJNN1Ku zC~o!SAj<~n=Y<6V-`$u^IO+X-YHmxM#ASq1fFuc8QJ2!8Ba)LaNCfo*Zr1+8g&${p zWvfRf-Z6}=wifNsSP;|}l4sH*^idzX@_fku-8op4LHW$^b(p1~R*tZeV5JB1e^8<- zxzS`~3^JIATu!2gv?o#1(1S_eS<>SM!bbOTjiHxd3E`R*A%Q-8E5a$_QDBvtVnHPR zPdkp-D;hlWt+ZMU*8L~T$f+TjFG5H`G|4 z^#<(o06h@Mfc?G_X7E45x?B5=_Xr_;i|H=r{j=h88hRRdgCt2lXOC~w>WsdTjY1eU z`wq|B3K8N>bj9Y|soR2$dZa>sGl2PDf}Sdti3uy#&h|w=>|K9Hqh1+M4MWlh!Jymy z1K7Dnd^YZP**{TXT5ioYcbUtJD#f7}#z~DfhpvPf{U1n-99$fl+1GxsMJ_XPCnSS7 zcIET@d&~a-#EqGZtJf*7arKtDu_~ICir`A&4BjGRIieV>NBZS|V1c_O392$@RFfXN zw>C~D55S3QqWk3~myXJC_9JYVctPIgPR1}zyUSyKke>4@N{BvQ?la)(`9Ff)bN((I>_LyI>~mX+HpjPkzgcrUG~psw zFs(@RsuY(3u5E1~MY0>V4FWGIk=Cs@uM#FVuI#R?)o|q0W^wD&>r}o4wl~eW!+nJn z`*LfBwXLrfw^HOj9uKLAS2=Uu2N&SWT)P2}KY;$($Vp3EQ)ABN zBt`d=$Dp+b?dHvFiGtDF{S&U>rd52qFi0Iv>ProBiOYBNjEVIh>xcNzuryR+A*93s zsn!A9mG|#4SAK__8U&FDPQQS5NlL*Z)P=te9GTDaXlY7dJPJ000UTbilEW9o3wh?q}IpuZatj z&AD>lr6GlC@?Hq~A=#V>PYC{$W0j(l{w3|;lZ5Y2UX?nwq!)=h;z829;M@1VrX#jF zM17ik3i2$&gI-MUhzHsuQ38n14s2Fh99D3V+5!oR+#BdCB?DV);l@HwOH5aVO79~6 zA%JYo66=`3sQvK^Pt%%0^+;=lg?e8OX45s<*Z5XpJ{@M$8&%POYDwh{@6X7V4pg|< z9VVIYjn-z7eF`39sVm8n$#nk!sBrROcX(9=Z1x8uHy9(3m5PwTa=9#ufGYt1@{!bC zlrjF~4Q{krLNerA4VDxsMc+!^Kp&BSW zfLB1HmJbFteaKj6%o<-+u4t*FakEE3B&eMugoqZWPq?&>e{0%o&oEkP2^-@4;Fqln zlif1qYKxWOUYXUGCCW@MKNs4`b=dKu^5I1JhCOK)LBaV7@COh-@k)Eee{?8`#L$i07QC>t>2BHL)<(4Sk{sDBPP3{=KR^%kQJ~)1Qmg=5&ytdUH*nG6@ zKaj0zdhnDLFl2XSbkeS`M3y9HfG)rY6o=-pZJHshpd1z-KENr`xGdae{M}`gmd%c4 z`Cl?@Ue8U1!c!ireWY{3yKgr`#|5lwI#$F?+P`pT3QIrvn$x%9_YDfuf9$`l_do2Q^TZcteU{IepzTB=?8}`%_ZWf>S6bhp@UxhNW0oB zc%891zOfKzhx&CUivNF?S?H8)kN>IHb#S71ULheN=qFA>Dr;x>&RE0jr$|GNB6`R- zHw~n=?I1^^Su}zG6tT~Y{~K9iT1OxD2elPyuxx4Qb%Qx2r_j+ugiMH;Z>wEinlw5* zvG9XF=r<$Rrr{zDax55+{I4j59oK=CliSlq%rk5w&}$V~SBO_wfKq?r^lGOrkDaL^ zovJ27DHk#`~d(b{}pSb&6{hT_oUhuj2cM= z4Dt-}JI>Qrk2(MG;KlidcYMh*pWKoCTLZVROGD0J-QCA)t7n<3|4IXQSKZS8($MkLl=;dO``0kn*y3kF z{S#~NjoGpG7m5E;u=!u$g|Iul{~B+3exI$qcr5g91van$2EY6(Yxk%dJ=J4X*R}Eb zyj$^K3d#@vZPr_k{|nByH+oi|QnLHUcwc1yrR;A{@R!8DdGn9)cmJlE;UB914|m)( z|M=!FpF-Ha|5E4cz+X@^|9W=#ZxUDki^Ttve=QGGqr5M=|G4rmaepJ>uMppVD14}S z1MTZ@vm^|SEky~gGRqTqxll_;*XOfK5I(-B^LKazHFCzm$uh^n|B}aiA38Yq9603| z&V+y^>Skv2<=79okZ2g%`3`4g2q~g4%k8M=n0}0@Oc_9zcmq2!Wj}ZAVYV}-pulQR z{;?l$tWbd6lIXs)01APYSq4P&JrfotV*4!!r_Y*h+=xrfQAd!lTt9qO=TSxJyrPB<5CQ%U01>Zc6c`wb_itl1dOH*pxO#r7u zN+qiu%gqD_3{@+AxG+{j07#*tqOCm4@4*$@oh}(jP#~1Qj~-mm@f2peuhj}hO3N*J1{Bh%dsmJ}1|EOk2d`)q*Xf4(sPwwiUnUk0 z5sLTt<+{vC^q#4R#wEX0^0p0s{Ho)HgreA94SbYTD{k1V>(r+1Gjmb2{K1zhscjhY zGWCffO!aWjUZVGXO)~?|WhxEZZNo(6fa>gY$ChgNYHb|j=KLXmZm^PXq6`8CiU{7% z#8|(3?>YZfzzqd*T1roj$!93@q?YRJT@S+MYvIpJ1;_lqZ+_@_ahXllk zM%Jl$mgaPC{RWH~6g@|(bUbE$h1?r5+G*D(4dcS+i6T{E3TC9zvydVi6h87s72%~j zEri%0YnRN{3vj25N1CS~w)%G_m&h#k>14F-9?j*@HdAd$B0T_|9D+0JgLcb+6~lm$W7I zM+L^SjNza%nzMoHv1MDdT?tlQGW}--Fs(~X-XFkxENaUiz}aimqLZz9(V5^}GL3

    _qh@w4M;P2D1DqvLzE@7(`qRe7vx=MX z5WgpHs;rHsO{+7a^P;Z#4^^!$#eyH^1iyRIzmIXAph*j$8(B0}lzK)Uq`0pH7C#f# zuVze>DleX4Pp65c(G89kd8va(_os_83~xkk+yr0HkYy*$OLoV?w{p9LtYzUfEOn~X%LeEY@KQM7+<%lY1d2(MH>Nm>-p&vUc;;O)yPC5P{*@hx(<73O7g+et~!-@Ier!LPmNGxKD=|8R;Ec&JFepAMPx(G&=lB+TzlC0aq@E9 zG57I|=<0$ZqGPK9om54*G3hGRA5(t+BZH_Cc(q?wzG=o_Xfw07n!>!VH16{U#^1`G z%2?cJDo8*B&+i!?Dd{`~3=YGR%8e107Z)T^oJ+7Uc^5V7$D?9-^feCq0SBMxp(t1j zyM{6NUd|wL7aK&$PM#|q+`&A(*X|wGj5zv{I7Huc)KvZC&qw-j&^>Bh zdn9GKMP1N8f_mT&BS+yx8F5ZM!5GB21VUWVeq$#+5^CLGohinU1#4EzFSy@Kt(o?rY>kN1SK;9+~w-Rh$HsWg>xnjNS3~HhUKu z3bypfdB{X1{!31!Q_*7lj9dj+{DPCJod{!#!KXlulKW732br@123aK!sqQ;?i_z}% z#tmsu@_;1=wBImv09ozGZL=n~KTn@6gUP%xYiVjsYs$4agGyZZteords?F6qeCVVo zRyZ7;Mo!+xQ=gDY-^~@@w#UfAChcPO^XcuB{ym?>pU!d2g{S0qe*k(f0LAC0zA7Sd*7OC2HBhX-$58jKTzH!dsp%#MZn~7^5wKjNI5xACUSq( z7QS4A6oUehIJZbZB`=P@klGh1AusW%%@r2%lB~t;frCRa5o5Gk7W+aKs3nE<1!t3# z$_K(IoVQgu^%1yxcJ*jS_ zY(IJX(SBn1KYE~=>PCd2PGxv0sN4YfS3&=MYA_P{DU8iSp}!V2NoIC|tS2NWJ(=7e z;7<-3brG7JEL47FE15pp!43yU_!$6cw0~V0`+@OB*vfm!k?%HlPt{OggnKBd?W!k$v?>i&)09C-fyl5PIgv9qNa=0{89&x*oWKgQ zBaJyZgSoT9X4|Y1&R4Kp175X07Gh9G9hM^!hhi>w6PhA2Y~|Fbl!@Is{EZ&cz(T~r zdxMh11}_HnqscH9Aacmd{4!{u5-Tk+NREIrR%BH7(>HTi#s07z_LBQ!cRF68lTpBC zgLHbWD&0v=qc;PP6_gxZD{{l6Cb7JzNMtjU-BEgt6rflgD(p@=IIx%iPnSEZv%=y^ zNTPzhy#W+z0;4lDqYQP>ni^eLBzsK{5M0p2B})={esdO7)~jZFA$x+RXSgW&IQTuT zkfAC2W&Z~Tx15|Si`h_{Ny#^!lC2|NisH7+LoaeWGSq0~aU^n5Z4G1_8aZjchY;oQ zdK1MOsI^!K7v2(EvK<4mp_rlNx9*&w;2|AXRI|5P`uW_WA0nVi4P`1Ri6F?uvX=4D21ENB$`7*MmVIcHy0kJJ>i;_Z1jIm3yv>oG0ho z;b4m+1@+*;Pe&Xg<-L};2?`V>efkiyeQkE>1y3=b7G@LCkVkEOVJ{u6(c+HqDU)q_7OY-Jsx+h~c{`wSfHc z(s6*>wWT2Md{I`CZ#6{22~w6vZFLGahw`*WXB$hAL3AJ9Gf3g&tiXp>eUl^*L1Mw~ z@-**{&tnB3PvYG$bRK&|#ooc2G8g7Qwyio8F*z&GlJ~%UL4v_i)bq)oyn~0FJ|D5M z1n`5LS8-N#Rc`?BJzus6#EnSO1<$%(x?6}8y8i$wf4^@ypZey)BNNUHt zzkh}s|M%*N#K1xTPvWp}eFQVOBXH~8Hop2qv3-1~Y`jgWZKW~S&=E4!()7>h=5K4~ zt0p{^Z!iLcM$s0YyxBJPT9CfUNk@wglqn0UMO)1o)g5R(BB*vXIDW}h_3YjX=26Ta zNbH~MU2HDUw>1Pq-+iqX+{%ab>Y$S_cV?)3-ZT~V7+P$VlcY3ta*%aAQd+bFxXQBo`*iY&Tnm8Niq?R`y_WZuHa3O)Vq? z8}}Z*6-k8qce*T8ax2P;WKJa*bj(PDE0uopZ&BGWkj&gB(-@(nJ)&~OFija{K@mUq zat7YJvRKm|vP$BO6BfGpuU=X!2c`Q5U2+T(yD1uiV6&E5ibFE_Kq~^c3`-^9>@9Z7 zGTopUbL$EFn{h21>{{oTZ>uFL0~JQ3#4#Cnw-q}v0yr*~B}RDDMsA(*SmX-@_Ck2- z@);TdaX?eGi?&uRgl=B}wFkLIbac|2N@@N~=j1g;z)Zethk80aZ60d`?q)Y5q&jFM z^=$>>BSGpe@i(*>0fOp}&^-(2s)mLRh|2qf{kncPR%2CPz*+vQcHm11IJ|*x$YqsAH7<{=9Wb zu=i2uDfGgIK0W)TI#|27R9B2&|KZ#n`Qo7wUn?RPQXwx-%QUtp}EqA;luZn>Pj)x~|GM7^+wRFi~}kj8;Xl z%Dr?FGC9|;ymCdNcQ?*I?$^Xm8zjik?`VoT^O0hC@d{vx$`Ww^gF#fZiEGOi*HyUN zh&=w{iahp&aMVM%h!~%DaJN*ca-kDRPDmeF(tmQzpOPQTylfPABbnpPPb4A-2oK_9 z8MrnBr)Bcde%ipPu^7mSuqmJ-ElJKuXnqC`X$7O6yvrcExdGH4zuc!|+R(K)N z^R(Kmy(T98Q2RlJTfhO93q~BE_UdsFiZs)U=6XOgE#L(md|?R&AB;elQN6HB!%LUv zUSv)-Y3pc^Ok2KJ`XJz4xGJi2x3u&;-c+~0mR>{tYoKzD<$}Kytp=~{eD^l%SyY32 zoAta^4d`zHc||0mI1PYLd@9hKW$qAj3x!BMwT~nbXRenIvdd>^`iqmtT}VWw&Xm3= zT`=A7MJv7cLFbOJ1}hR&vfEYXf`uJ1C%aogTVj4+mXe`ab1b1ndL z@YyQV&X#3nWY`-MAF>Eqtb9LEAOWwHp9cYAnl;6T*950`yRE+pU*WO5sNqyh(SC@I zHOpIEBL+a*TTx?^52#S|Ya=!0C(*4_7J&m&i;r$ro{VY+O#IxZTZrjL6-St1&yvd5 z_eEIYyf|m2-F13Maa1_@%mMB^ug)s7oHIF1M>I|NqQ#%kiTK~E2Nlqi=Pe?FO9>Q2 z8VW_iQ>#0p2{Vj9RAIxH#o5@S&cu0%VTPi?G9?pe1Di?Hm?4 z0LKk=VjCijYQW_sk=Pqg{;Nzd@w;}F=%EA8Mf9Wwx4FIY1eRo*wH^kQw@D+bMvu{` zNxA{hIY8!B*E0~4^D>Sz!JS^f*;=}fLv1hnNyi505V=ihil{9G&6yQ+446T%{DDac zcv5h}nRVv4oA!(P!J;5eI#*!+8pnsHvQ0#{$Om3>B>!#wJ5rqrJ?6?^a0Kp{M7%lu)oCBgn9Y zWh5A!dz@QUi&TnXke#2==weP!Q`)2>YRP(a>w(|bTRC0ju8$kdJ&;u=E3m6VDcOve z?qFk5?DqUyAAv2dH1W!_!wrh_c&nd2hhXml()ae9#&-~@u1ISgWrCG8ahL=p+Z-D^KehAG>s zD_~?($rL-s)YJd2EgrB$W-O-yqPY z8CW2fv~8OxVoL_6&zYl`OLaj$3tyBa?r?ro@S{Xyx{|$f=`Ry$FF-&+d%W;g5O2a{ zA29QX^e)u6iI0&eT;)ar^ro5h2@7nW!Zc}|TFL1628b~X27|Dg5C>$ky`z-Z4w-j_ z>1jni_XUCF^oAeq8zGV_Qx%2-S)HT+jYfFhjYVh6{1kpd#4e@)XwC(x z%h|*U|CsLSTOYsiweh{~)JjNmgOwEpJ;cCJNA>4p&Jf%0=Y=nXb#|Kg3^mAz#W^gj z*mY;xOf$%U_v_ok5Kl$bTVmbY0@T(rkPTRQ|2dQWyjsDUr||hwaOm-s!7;5OS^2rI zO{M-tMfNd!r+Rz5c!f!%94o(U`F?o6;OE;=uBeGyn#WxAkc#lZ{5!;S3jUDQWU4uv zPd9nwZF5STE|~chTG^jo1D_r8OJox_`rX}^uAt%|?6Hq!=T&RDb15Ij?RfqG47sbf zFgOMF2LrY1OB6P>&$_YpXeasr2Dy5cp=?* z+&i)Lzgb2>3-RQaMMXp@?;YRqh0;El0i8dJH_c@lh3+Ks(zWD?Ya;I3Bhm2US6bf3 zYL+y%z9RPY^rQ0LJPs%SC zXaY`j>B|%%Xy7{|uv$@WUnc)dE%oQ~m>^=Rewpg{jw!8BK&}G7!Egzs z_WII@XS50cb0mUqV-}UaL^SN+MVKgdf6aTgHNEtB_xl&-E4w*jvvemrwbX(Xb2}?W zKq5PZ(#}2v*JU#hSs@H^4hN=8B)8t$MQzGW)=VzK$x0@UVzfE546n2{dwvEvW&&l>SQwoBk~;cTeZtgo)AU@B?~8ptd)3`(;CG}UwM{F zxkNU-03(M7bhmI=qG%K|`W8p&Mjf^GHlzdL!O}t!dX0!qxC#PTx?l{W$;kRj;4U-D ziZU=rN{J*IvN({y6HIox(ntd`Dv|?3)0F1<8cpyK(}Vv2KEc#{hP`&(y2hwC=i{=Z z^>gF-{*gBGp@MQrUc=~nY4p=Nylnu9o~j}nUoYwyXJi0qI1R^NK;c)tC@4r#+P5p6 zOH;x0kU!L0KntdtH4oBcF79R^Z8rztL4B@DhL~^jN<8>26~Y!NT@`{!hZjMV)yRZ- zOYUAP_-AiR2~=ElSe%$6pN}QKjF5cU1?~u3? zL_yH8c(-%@W5lj)v!A(UUYkEBl}R8c6w21vELLzufD4}v((wgV;u2#J*^11i%dk(D z7VYV!^!*EchU1K8wRsu{US#k(=#S zY)UJ1oV?>z0`zM?@&}Qno82ajf_Wcy4}PgiF^68=>(;Ks%$-Ox_7)>IL7kux@by@O zNh=v@#&C5mpR`{d(_p&oE^0Dh&~x_QQ)j*W*0sZa__|E;lwyw_GK3C&qf6mN>pHl` zpPg_Q<8>fpilRUvW{I=>(Bg&Mas3A{ zC8E3^5do(3les6rlK5D`V`76Sj1<-n-dnDYDpGI6s-!juTa-*c8J?f!)+oQZf{x!z zOwP4@*);gFBsoTTKp{d$0( z@KU}>hwk))&0|Y_qP!QLd$!Dd|7vrC*e(-(fF(3LmJ1t zcuBYfai7$|EoK(A-a10)j;?0zTo7jk3H1k#^`nn=CvmQP=9Cenu|i>>o0VM>`y+`A zQrh7-6@B6)&+9szS|^A^;nx!FQgP(T$eUl*<(#SH(~TR4*b_y%OrG=AI~Gck8gH|o zJN?KnFjx@KFJ;3pU!-8tljWbk62t&3D+CJdr81kUneF0{KU-ChmCz4TSNh@4sFNl{ zqTR-hmFgrjfrTojF42nEA^UFdR9NibX0!raDTT^lC^Zetvx2%f zShAJ080)^X=t*EuZqW0DIn@`qiN+~)P*`6|2EQKN~D5h*D(JzjoaM&S~e&6MG4vu})h;3gUGaE%X@lLp0qcfXsiFpKQD)}=&p>&q{g^<Xv=rFw`IrJr_X7CKaMtq_5x62XT!;3Vp~tNszSldVH` z+L*`KIt!eHd~h3f(u&PLyK%!?da)o z`1KN1x0~1ImHTFicaG;WE0OZF`)&9n`h$=o+lDF*BrZMcscEkQS<1q+>tI0Y2USU1 z><-}{!2Wx=zS3*V6&>YWtzRXdJ1(3pS_^1sBe%8?v^7OCPPUu~s7WW!*MdgMXrJuN zeuu?n85mElwAd_>13=W#4WG3iADC$p=Huy$>1rYNHZ$9F;L3M?s@DhTpQ=o6l^Rv0g5hu9G9nNMu2x z#t=|C^X=Qq_?IzeW})F{s+jefBKqTtYM>z1<~iDEzE{c4Dk)75B?8)y>g^(<;b_Vx zOaa@(frowX!Leh(4D8tIl}6~dH!gPxa0#(g-Ya)Z*mU9T-{ zl%_CK);e%ccp(ys1ok>SC7O?**i$Mf<6skCA^G)PzX< z3g1KD3aP{59)vnB>~7qX`H-!a)ohLB`~+k9XtTdz+*vE2*K~5hpMBME#IcC6m9~od zydHl)shUjic5a0#>L|Q8)7i`0dFs741|xQ#{-PSp;1ox*--Td}v4E~8$=zW9&C;v4 z7jMp|r|d<&UmWf068U(BpH{P{y@tBYwT-tI6n^^cB{sI-o^m0Oc!%o0ExVNH=L%95 z78%GXWxdUYJ?bt+a|z*1@v?I{-sM<`P0b)AdMDF}D6nE=@x^&{zq!hmM(h$pr!cC=Wbt4bK&I<~^u`<0>lAK@4W_=QzX z4c|Jre1uq}Xv9sNH9k@dru?0APhrIVlFF_ftT@*MSuGxqrxH0nn=){Qp6!C4E%|2w3(A#w}rckK2AEOCZ5n_NhJ(uysLl-b(R?!&g$cDzivbb6c)SXMo; zxTBUWzLtoT5W5jT4Cd0FT}Ef0?2k+Zk1jOoTXyT3Yhp}~37_6Wh%#D2@x=Q^8d4d;Eg5H9ly$9n6HFAa~B_yfK=fTNwC(1B! z#WilkvnVK8(5oBbR!Dn3TO1-%Uo`ZG`CufWut}^y)8eEO|N4w|u{1zkYT%!5YV0cL zco$e1j2e=$-sVi8+-vIA3ZUZPHUYvWOND2YYp`4bzoK zm^a-+TqI{2&3frVh{Jt;R25z%b@PGHS{6%I+au!!QN9COhMs~x5k_z72t^4CF~J92 z`paq9t$&!*NlVI_#`$+E>m1;ICQn!mUQ*$ve4G??jaaxq!T`56L18x1I{W#2?P(N$ z{3Y|3OLL^8@2Cc4Qa1AvDCy#AYL*zOC)w+j9ayffl=n*hElz?kC6R0EK_f8eAIu)* zTX5BsefNgemG~}75OU)~MZJCg0PH=HSitPk8>CN5OD7w?jV#__)D)D`U>NlTu7^9l z()iQ#N47diK3ZdsUz}UAbtgNoI2e<`fw7E%}hEd8COvpl1gyA>KSv(mb)7) z6~4x}W+Dgmu2((<0%(A(%6pHv@N|6F$0x|7aN`+-Nm{JbbR)~8HlN&$H9?exiS6~A zvx{E*D@tT2<|1P-#?Fq{0TiH+68Exz>(_LQKLCBK%%b{CMYSTX??j6Y@v$kA%9D8E z*!5^h9)u1}T4fmdfF*aq4|^*u#?*_0kO{Bm*^%Za39lbX>X#!(&~?pj4}}~is0B8u z<_+d|)4QAmGtTin4vOa$?+~eCeP|lnk_x_K#v|xm%>ABUy>R?GYfM3z2rUhCH%u|& zi3}X71p?zs42vjs_IEy5!;Bdw)|t~m89S3ti`mYqkQ8lK6Xwmwb(n)N8?%avFDSTe zYVPXB6jGe&{jUT0Gdnm65D!vv>Gxu#S2N4cO6j>3B>XV)I!Q-)I{TpUpiK(!Cc4r& z!lU_vrTgCY-^>g;9?bR~x#E*WCBf)0ez_t|W(7`LqI9^MWy9r52QpZaHTs)}Xq29Vo|+L1@10p* z@Wl>q@N6l*bi1p>$1o*-$PZ^>8CNu=x-o~-M1SCqb$;*d-FrbY(z`~&3K7gNZ#ez^ z;hLSHpK1UJCUG%IN-UH~(ieB^5Zf#uXP=JdGr{5j`RRILgZ1r!@jHT~ic;|JaHV*5 zQgc$0cXuwnN1Q}!k+xOYUL74<-tPBg**`bZ$4oyMwmEEQi>R?w?}*3|(ZOLzjXMOq z5Z~WF7`EZ;ESP@C-MIm2dpMsMrFn(6x)9Z}ovcgop;Fn9Hza!`G& zc{_b@(`OwtGJrym%Q-!=43j1gOQQ&pTJ#weU?gwE42txVMh|+pbLqjG5-nxL44RrO zq52gC`9f*6R<9O&-AMd3!(kD9m5JRB`L|a-XiTd(WO0!R&1Y@rh zl>IDv>eaX=_%T8rsJ3Xm&4L@7!0kl_tt9StHa+dznmqD@JRDx8skj^7g_RrqvbeLp zZ)7EMOP>RF;%#~5nNX!_rzs=ssGo#C?+qA+5-(Pv4!wyiq~=fJrBV@tjDaX7(&2X71IzZ?Dl_9SmA4&CLWGVP7J$dPN4 zYBFI*I0uVlfrphau-jK7*;2GVO~yLn^35e%mx|m>{;k*};p|Hq-{3P_1#|DAKze6V zWHK#;X~-(Z`tc&`^J`D&9%s|WayuOB4M0PIiz9+JCAaRq7bQH zKjzN2K^d;%0eaYZHAt8zVPg){J6)Ba5XmiNk_>kuW(^MVff7-Kj79%y*~r5w5B9<- zYGl|!r?Aa~nRa^R)%kBR2_m?6U*`iFL}^REDu^>^u=*>WMx2s7X4F|C=peWR=(YX< zU@7mAnl3uB(1^()QsL+NmoR>q#cZNe4d?fKH0$Hq%OTu@xiHRSP#xtU2W6V)x%R6E zvgQ&$WR0V`)mpK()W&QqOhL_Ju6E0)JSt&hi0j`6x|P@yH3#IVAmdgCNvIl<(}~uR zJX6Ne^J$rLlB1$>af9JGPh#YpOmbI_rAt0pGsMw>C^>@}YWZX0~n(OLfWlwQm8(^Qb+cdRaO@TC! z6;Fnt6QEoM;imuH$_d$jC`nGvm{L(9IPNkO4&a@e&wY@AsP?MU;=XvTa zA5Z4fwDcXzE??ng%+(G@X#`XHsnq4=%w(ny+}~CAGKosyf-F~m<<;B#;C|EN`XJYj z26I~5-eD&Xv5y)!ze!aiOW#SS)m+bP^QL5n5P|piV@9aUrwT0(`Xey35Z9@O2F;GeXcs=6L6%(psE;$q7iaR{pg^>$K%sJINM< zMd}pFu%K>kLc5U+WkII&qore^{c*|*T$+sGWclX2(EbkB(6?CxPjEze!x4YG7nxl^ z*(R|D15%|3Uq^~LQ6G*oIC}l_CR2EQQsjODdQNDNGcymCjz$kEDRX=G7Ur<5c;4E=xD8ke_5eN zHYZ`qH*?u<@J73E714EOMDD+a7xBt;p^|<1sguz$!<}rf-&k0@z_0f_fD_2qltoq@ zda0>ey3J06LydbvU*e(D;$24T|DeQ760+n%XqadBgc^6sHpD=t-%Jw9 zOXKte`g@3m4Jj1#Vqe4~Vp*XRfb%h~^nXEavP))8*sMXZz_7a3W11)3`8 z3o9P;8s51a5f-o*oyL=*{vjAy4i`FOxfw}030qW{X;ioQ_jKh$MsShJQArLoF;5XA zA~AA@K|y@`erYX(H2mlcPv;>zM{OsRDAl6x#@-0hd!oSvHix2SQNN=6aGV~--?P;# z_0}2*?bc!Z+yYfJEV+VeavOSH>Uc_sz^+vcU6KNo@#&TF)xyy1O zzK>ePnY@9i0t?IYzL|g3TZ-6+8oHfmdFE{CR&Z;_(Oo22Lk>qZO41+0t23v`p6~>s z(H8;2NX@2^FPOo0D{GKEhuqZ@GkT_f<8F$UR@PxJ_rt9x-@Cc>?T;lwYv1vRE>~+y z%hY8b^NtvlE7T;f2rw1PLv`_T$n5GTQ|)L;QG>wymJQ)}Ny^n!7-+Gn_&-MOk#gig zTEa1?h*6CFUjc>T>V>v&y=#38A`#z7Ct{Z{H_hxxXsj)pPkmxNZ=Q30FU4UiQ?c6y)B^qkN5lXoL(;jx(V2)hi&L7h!0Y3>go0-Lv| zFJFRU(J_LCI!ev|V)T%$bH%!JTaHe_Mt_gM9<7D#wb;R{6BVhsqhc)%^s@Tid!|rN zN{pJcZMi#nY(qBS66Sz{uQ8N38%`A`S2Fb}QzRaqTC2qS+g<34v=g!C?2l;RvbdB; z*rRCK&(A+uI2U-t1Y|P78G&JLiJNg4dsZbxXuoue32HuuweD3)h93wp~iZwJSbMamyQr-F!wmYdF=4u?$d~BV zq{7yD&dQ%!$TAzc3Tcs-=Mx&KuXLm&AnKT79dhH#N z0(Nl|V!Odk!Dsc&#&0`0(k{Qf$2lErOA{A`@6pesdC*1-ho^a!6Nec%5Ol~k4HJh; zpi`)G(9W-WYZW7K%r`h3+Vq2sFwAg2KPs(uOzaB+HEU8(#rC|oKNd>_MWAwB5^p+k z4c7+>Q5vKRhbVWr;X5mp-dT1bj;*Es_WHSt^0E=shMlFR?>zufdhyuwROIk})=7 zR*|*4Yq5Aka@FD!*Gy$838c@U)3X9t`an8udOqapjK7;+OG~ z(WGzN1>P&)Z>}?MMD}~`v$X{i$+OuDy|QYUGbWW+jhziwXM~Ov$N)}nnt+v4>CvX+ zvdYBnw=ZzPmRYH=cFWyAV(rpJF1|lCIXpaVaywjE!xbFs&ZI>R>6!dKd9c;zG9;S& zXS8Er0!+7fITe~*wmP>2<~uwno0W&6O;_ozIudp?haU`fxnzSNnB(>fi8r+xuU-)l(%fyM_6S{V{CXyilbOMk`oxrAby4 zLP<%2Q*ijxi{(3uOf zK&+aY^1b2%LN+-vE~XCu0C(K@qk~TUZDyE2H3zPRs^!8L96i$566uH$quW|OMDho> znI~Lz+rZv7iGcuTq#;1uSt2l6p)*npRt~x_(!6<&a44!MBuA?} z;Z3FC!~&DzR^Gn{{Rpebs!4zY-46hmY@gO1o?gMQkk%*(q)U7 zJ|eA(uc3Ob2GMHJrIm;wQ|Ye_x<_m8I<*RqPbzS0DqRjC$|z~Gl!ks=#8i@gQA+RP z^m!dKgi^ZG&r$a70IXV(D%qhpcQ?;Cg1wcayuuUfJ6?n$%G0n2>v%lQIY~EY?p~>w ziTE|zTqI7AKhsy#FAcE+VR^I(1P`(R{{XbKNQ=uOWS!it%atHa5(cBcD}UkBd?k3I zkxJ-G)%!{2qj$lU79~L;Nm%|F8>F9yy^nbq?TciQi^hL8u}!7(SRJjaJ`+kvuF=@q z9gzzP`Jy6fsU^x1+^?J5O((U``oheDN0f_Qo>05 z!?~$4Cc;ADaaBsxCcCCOhieZu3akYtqY07sL=0zX6Yos08)YL25$!Z z5ris8+7BIipP|a*=P)2ikQ89TmE!C8xC=@lK&1j^oV*{f=q6R8)#o|ASF^T?9@$S zc8&Dn<}RQUZfgYcv@YH$KmxdqEx___YZ3*yr^JIJ676mz@Yz$Y%~|TWsFSQl`7yhT zN)v+VLWGPe0+pDa+I6?VUOsSE40;CMp}8euvoVHJBDhTKP(pc5)5XUKibTQeau<={ zF%`H~UMWp?ZG1*rQk+l>fT*khQ#&^pjr<)cWD0bMni&4UpJKu;n#^IOYcV^x8Rcor zSWSYMP9SG(NgBY=Mw0l8tF#Ug6R15~d3G10&qCgD$p+oaqSds?IH4q&r+x7)YRanJ zxdjJV5wR+aTcb|5t5@#dAq6G`i3t>uvPh}g8R|h-fW(A|p@S+T4I+FgkT$9wO|)Zm zlOS!CDv+aB=}@JGET_a`-)J@8vB9M&vvMGbkdk6SJ7a#dFkhgy3RNjOfyYLtLz_(A z9&jZI2Bu7jh>q5zWfX1%=#_k=(4H`6V`@PLuWV5|Jw#+EE7~;EUni0zf;efTJMf7} zClUEl0h(h}c1*EWoGKVi2&SN@dsNv*tqU4_Xh;#UUc4Vto^bBv;N%mP#3axWvN!Cn zmb@FzT{cyV6x7WrhiIACLA|U(bB_r>Xb84a6iTNEnr}P(SnC)U6dU`UkI-P#Knjz` zDO!S(xWy^)Mx`Gi^#kJNx*GOZDS1ZnP1P+(Qc^JVe zX;}cNBju4IdcE|cA$P6i`K8!Z)F}{3&&)1;sKP!-(^?wTy|$%-QI{0>=dMC$*~-w6 zN{9eSB+>xXnc2nkDJq2m4Cr<$bc42EhRt+|%qPP(IovkKi8m8?T}`}v*)lvR5UeA@ zO=?*Au9dUV*??_dRrFcQCfXBnN&?WZ6lf9R=D`@iJ|nC@b@#ZgmFI@0cu%3a{(66^ z?B7+}Jk9$?72^t$l>tb@929F&G-l*FUom?1gcYXZRGSD~j3r9Am9LykB{57R3G@!X zD{AWOSXu3~aO&v;XEbv6EW(Yo-9nPAK}bn3B%v^~6-_{;OAT4&-Mf~zV(p>tN>hai<1Z(TT#jrKQyhKFEDGJ&=>W3y z^?_TvA8;#i>Mek1eSA>}5~GA^CWE!`Eeg>|JcJjRjrzEp#pR2O0Y;p7IWBP?(}!M8 z_0_3q2~t#KR1a6A662S|sKq`;aotEO7=mCODca-z01sWk(K>Op{GrMoY~Usrks8%8AR`k@j}&WZ@gT8mz^;R?>A!5|hFNHsVhA zftI4~0GnzsC<*Fti_I@X6;Mvyu4ekuEZiTmuO}P-cG%aZ=!}V zbB{OIR?9sZZvuJCUMPd(SGhoPv+{XCl?w@<6dIY*UbB*>T&0sqh4Vvu6r>bfuiEla z4nfD_lC(rk&;BA_3Fc&&n?NZy2C$_8T_`BkWkV4l(LpD?^1Os|Cg3qDxTOe_uPfq0 z5@ZEz+76ET^B-KviwOxp39hyEmz=^%k_76naci)w7ScC-Tk(YvvOW>-d58s8=}4j# ztkn2p-ucby2{BDIZM`a74I(G=g!wn~XM1)41xi67Hp=2p%mfN=*hTf=JIxso5cztN(2C7Of!keW^Dw(5kAP< zSZ=ps-Mh+Mc!5%+Z3|H`uA@fF$xXrt`GoN>=&~QN7!Ct|slp&LF6#=7HxuZ#yIw@@-p6l(8diKWc;0Tq^Q7l(UZp z5duY3uGtQ2ID?r-H*P3vvF|lw9s#&0n(8S&=cHY6>Qo!+)bfJb?5QN-AP{vYO6_O< zB|MW}x-yrLLA`-w#sr@XeAuAsoz5pQVK&ed3cO`O7><}s98fDKNcDSKk+c2WL;3XY z+{anQq~VcB98V_~9yF=*pC)~~Nh&02l&Ghc4`|destg!&Cew0K3960~MQN+*d|)U? zh<6auQ;Y=sqJn=%7p{&%-Gm!uKQMqsl%~phzIMz=g;=<5D(FovLSWG1QipoOAy%#- zU>&e`;ji4}Zo+^91Sl)1^D$d$nZ&OGBYLPSvaP8$$EzSs>u12a?CPh%xL>qYh#gF- zhf)BgIBnLhH#wJ1Ug$_9$}tj6W=`F;62M5W%@xz%tLVXK*VBDTeA1D50|}KR(_q%V zZZIKyI{3b%2sQJLSG5Glf)mC`_xig@tc?^J;)3#+ZAn=ms1%*F@SWRW&2~8xeFP{O zgD_)WK5i1)K~h4MsRDM1euJ4p5UX+W;uBItq^6R0)#W~BGfh0$8bW#ORq;(o*+|!f zr0)(yE(qeN>llx{?MQ$C*nlTwY_8a9(hzGBq2%(sMtY4@DHw@5!O@aRQ{msr zZ?6Zga}?{lyfQ@`yW{l37^V zq>L*vjR;nwXC52c)qdKQJ|aTOg*2e2V|9cTC`PG;M-X1r83KML*a`rVA~c=AcD1Q?mO-kO;lejY=+JP3Qr@hkuL2STlj2b%C?{C~O4qNE z^;vry(qTHQ2|g)Lgx8jLy11^R$sHHa0f0ssCW`QilQiC5x#YnxVKMehOzo!qjFV{z z5>f~9$`*RH1JUO!*ok6+#bRuIEiXBe7E5-C?kt4;!YZmHI zw42KCClR(1nLaI&BtYLB8f@7}Hq@mYH%=Wl5EKYMDV-D!k4z}h!z;2tX3`A~-tiQ9 z{6v|oNSbkme&9=pVKIj{#xn1SIM!X}Q#hhXtO{PEFdCN!!RFah@ zfYZd}FeH#w@|mWihnH(0l+e)9II|N14J%GEtO}I^aDYjj3H3}%KF!Patf18~g(fiq zkeC>>p9)P2VDT+0N3$ODPe#P;Q3goc)f_jshzT2AC2be3W*2Oy5*KWv<_g3LO35Gu zh=OE64=BxaeX9$#a?R{hjaTiI1%gM37<@R=2&Oe1b~ww{ao{td(|95v?|V{Gr0FAe zuTNRliR*cF)JD*dqguYI&>kSAa#B2}wf5WztJ3ZyiV*MtBAYa1C0LHuB90JDr>r`K zfUHE0vGHXLeb_NAF!hTaMSEXxt8)+-d?o=qgtp*T=fzr5l1A{|fEBbWFg!<1{w=($ zq{2eePYH?NK(Bm0K+D?_c}Si7lK3G~ce3x`JoEuJ)tQVDZw|6Goz7<3LM@>?l9ALH z%z<(m2Hh@*cYPmPIne348lpku9kZxVB$t%s*Kn2M3B`+NruI}DVgSX- zM#fvCQ}LA?qf%?S^QF?n%V z!yii$?5g|}?0R1?u!6`R23kkN1D)xd$^8nu%s?M+u6%#QhZrW3^-f0FrCXzuq zj9R-+nQ77F(yHkaHn}XYlQU455l@@Kdkd90OA1jcwj|WHCdHI!fcAT;v^DirJceZ> z8utKo{M|cSu?4M5Mk>cAE2a{yH#((+C0I_v`P;U0w`}FRgPp9Hr15R8b z+Ei^F#+HYW3Ngj>t6@wS-QKU)!bZUPg(qO@Z`I}F{)+W^c%E9yznqXj0O16B@P3g? z$Ke#?z8>%=a(bSr+VgV6M5ST@t+=ESJUmaCg%w=@Zu55FsEq|!_=h?aMDS{i!t)9# z!k=gk$>7FHv=P0zDi<$X*&zB+;b{P>D+U|nFpF4!3azwXlLZQZ-)R2;DM{sHC=^3o zL~im%3`bOuMJI0n}Z zrRNbK2#p^C%4!C{8lVeWg~Z}b3M7H(?W>0fSq8M7jB8O@cNmD-wW<)}pj3QOr-yf^ zYgtgofJ;#W5K>I?zQM(aHUTIl3QVh0q;PAyH!p4*PWOv}Mk8b1I()5h0jdQEtl=Ae z=G?N%ka#G8$i}=p1>(wzQv!VpH9&IdG8px4ynI_UX?BoVU44S=9mUZ^zFbA2>*mO5 zvLf1h?QeX{%;Qzlg#@;=-V}|~LVMBB!j^=kB(!Sx zcIG4z8nC(LZel<{dS8_*&cDyZQf&^9d7Rdk|Hun9^&>hPu zO1dg^K1IziEL%zz2-c#k_Yx6USJj4DtUKDXh|ZgmlTDG>(Qj`rxu}x`q@$c(8G#*c zr~~1o={xO@^%ue6P!pm@oW88UPZa!H0GnPFn|rqJm}W7P9O}6Txwxq#bC+TS=P)Hg zf~xZ@6kd&43Qffo96L(~n_a({M7l^}!sL4S z-=)r0wGfb~+UNBFtNv<71GT2thY&m-LZS4D(SS366ng`as0DKJJx>1sg8<4x|_nsCzhyj}1GAvR=U_X>0`mNfRT{ zw@t6UTub5MrvMCkw7}$zuKLU9gS5Vc@=qt+&=~Z0sB*h6yEO7_46jbN9c)U0XDh`? z(pc%XZOooM=sx>8G7CkGcsH9-{Jb5tv6SR5czKXt7*cu0A@sje1UTA zfIc5RT0awNK5^gC>+u#vF$XlnIq{5cRFCNmr3B)LY^@Cm$Ra;%<9o=~;`CJY}pbDU+es0&m<3`P~_t)d78p z_%j3QIwzkY!KXEU#pBb0eh&%F-XNS^gW$|8DU;Cag8W_cTz!M!@8a&^`MVpy!im>q z!JOsLC`9r;em3llI-2y@sGXS*T8YD@7+;BAgag(+?_Ba!Di5LinS>~jRZGM^y5vQ5 z*U1~h0F_6`o<_!@brBebk38}>yoE#BZfNn(r;Fx3c7Q`*46+CImY{ci?6qa_#Gd_b zr!181C%*>39FqoEZdUEIrrM2Y4%Z7M(i$5sw7V}snn}WCm@diNJ&Xu@LR4C0lD1Xj zGIkbjA-%;}K}w{ZSx+W_D;QxY<0b9oNyA-nFb-j1UQtS*QJIg2R2Ln1%KPqihG!a% zrIw&h13xf<>HXL{LzF2KY<`*YV*@uo#aE}3%X!v!ybx1NH1wXhU=k^eA;#xp!TH|i z<|Xh~5GPrIE>i$BRE{Khxj(}#N9_0XVOm0yaU{?J9M?FO*b(bdw3>;ADa-|ZnDu7e z!EiyIIv*m!yp*Jf5!df+Y0L_b70*|@!wQEe^Len5kJ4FcN`BC8z!<3FR^bn@NlxdY zoFRId(@67pK&-R|){I199JF{+6g4c83hPeE^A9mfPtvb^wP0d2S7`ptf`Ff&*W)Lt z)PP2iczwhNOr3!8$nszqkgc7KkTmnB!K^124hDn%=h@giJcN6T0NDXL&tu?LAs*+U z#FN_AL7Br&0iZNyC%>G$B-Ufi@c5`_kwr@PHb~J)9+0NM;ZU7Ve*2$5Cc5??C~L$C zf#Do!9z2^zpv7itwgw&vNebUEX zYSp8KR+NMK3H@MuS-7H#lh;0Q_@Il)!mK8C6oPuwCDP}Z0G#ECn*Be2z3o}61b^}d z7H1++R8(#v73m_bF6o4b-X6*kmlD#lf1b;cU-bR{=a9U%_~Qz2VSmePLZuSk4~X1o zm?pudGDi3#%szzW%1A0oxWo~|dx=VmrFcblwS-k+cPwSpJU7Bf-U*f^z@!aAqiFl; z_>u~pxS>%Y(nX@|qh!ycjVmKU+LbAK81KQ}8y*0KEh-|qciip8!$tzFAaQ(1#WD{DADX<21m?xY;LszcXYth*bdI##tts0X&Qh3o`=d0R`)wq{N-iyupdY z3(9Rsl#t#;;V`U5wd8A(s2W3TDJBU7x`2?XiD2-Cq^fZrv$J^M*o&b`Ek_tAG=t>c zvJ`+403hqUIb!tDEs0yp-c zHSwU_e8yg4JDC3f34ESqz`0B6CfW4!aZgy3irH*RfQf5m?9Dau^#tK3Y$ME#i{tR$BN+Kl{?M+ydIK(l1~LktAH1E zN=}SPCs}loXY`AqunDW=?9y2f`AMkvM{AH46NaZ3e@DDSMw3JT0L?ELgth@%kBb)d zx`IqMC3Jnpzz4Pvnu(a&-6ZX6S=#W0!362wb94jL6gGRFAny;P4{mr6UU6_fkktWF zl1zC|o7qBBt|@#7S%Zaf$lZk}hvy%AKrAT4X~rKAB$&i|J}*ia2g+6TeeFs&DUQAQ zSOf5`VMIqQdtON(!f|f-lks_rf3_YD6rTtqjaWJww8RxF-`Pjs6i}%{>G!=aRo_zoRS>+puk}pJiKkev9&I zJ7xN%6p(yiPRaY`+OHR9mbt6AZIYm*gTHASjz$t~UbvtFRa$BBqq|KTQfS(4R;B2v zOQF+QhKm3LPo;&2ta^eO|y7}riv#)L&(ZrUg|-d zy8fDVfb?`btbl5*1GtntFxHinm^(|-)lCU(EG;UKJhwBG&5sbIQN`I&`KrjZY^WGk z-5fHJ>#!$qeZrl=>DRz{#dPRhd{qHp8Mt9A1p;B0&JD2Er6EKYXkjQXOhI-wHYVXM z2z#wKi{Vjd4Bz<}7*5G%sYUcF2%~lYqSDBmC6$#J7l`q6-nyz{JyY&)x2B|?AP;$E z*ah>U^PJ$&_~Z?(N>ye#8dPIP)!hi+EqIFbNnji*FlxYZ7)y^*N=u{$cC)hDE0u!I zwc!-n?SNdotw1E6R*8Hg5RKru0A4YvFH-s`Xz)?@H3L^%0G8O&y<3Ki9Kwu;05F;{ z_OyYZQXhzsmb!N(FY7WEMiS{(-`J$%wb*W+#jv$x{okVY>@i^?E~+{#5I zWZHB0V?p1Q@aL!IKtG)hCef6M-uO*ffpiuUQ}Kh?{{Xx0Z5hrEGYE%4b4WzD;!rik z*!D9P_jl_qB41Lbu%VLw0En^nr~dD7ZAHwg$M~T<3RmY22Ijtxs1$1l#8giz^^r_S zQ}U`#1+42N`LhYt$J|P)Mv`8zRByJmFEmzV^rYF=E4zc*6+rcO`G9bGYHLb_lql44 zyg_8lBajv*Gr6D{iwJ!}>#I!5U`H16Ck>4%Ude}4ozH=M-8F7XptCH87|=LUlH3xL z>jn1jjM0^sw5(RnwNGJlp?FffiC&Gg7inSvg^hQRJ`9_A8`+DAw_@2{!*-!MAvlyI z>`HX0qxZcT{W*D7`P{;P43k_C1;u`l`#!bc{gz)XXDXaFP?T?hRUPMPJWGgexi;QP z`%XN0-+>8`{+r8Ni8lc*ntoRgLcQ1{I8)xUtD0KsS~Ky)XT%# zT@36^2$1|=NqCL8Sqa$ob{8;vkvg?y6P}hh*Du&{cL2?;;#X#+RyN#^45hwmU2yT@) zFQUI-ZQ@yd6aE#$xp=GkEqx}GUsbY3ljhmc$rgfu+Z5z({{Z>qH!s;n`bOEgb1I-h zNKrrOKIa*K`HL3j7mGD><@uAUW0wgU(j4w&V_lm3w>s#+vttoKI;PGUJHqRx`EsRR zejuNL0h3&3EYi5sK2PX7Cwqi8N=PIF2U<%tH^I?kt_GO&(aWW}06S zVsgzqC#H4RUop;ByM_{58ZPPT!Gb#LuVIzF8ZvxOI_)KEQAwQjxZtVEdkN z#DNEtH{4;Jt)LmtFMN7Ep+>A~wA)X)7tHhn!sQyByI`h~9Q~Lvj~J5{4I~+=o;S40`SiN70PClTI&qG0M3b zm9ML4(JDhzY>_RgNk*(OaHPbyD&gMiLXN0OklAXUA}OT7B6^yVuM3sPX6)Lwt9J}a ziI|XR2F^{bIZ&>Ff09Ot>Z=~k5~w;uW@SKKUKqnP!=!pqr53Q63Z5LkthGg24(9DE zP?j_dQL(b(S8L}T1OY#2KVAJ5aE_If{Hn2WCY(}5cgPLHIU*ZQE|0jcQKa0^(#sa){PWNRo_Aevcnmrm5qbGtOuTUvKLdUn5Ma~CvUEv0dG0h#$uDu3529HfaP#$qO5ckHQY8zSuuO`}*Xdan%(Mc+%;Km>VoI3f3j8a55 z-9@pYluxI@q$Y&&d)pbgtwgybUQ|XDr;)@hB$_c}3Dvp+Qu6kxa&`gz+-g)*r_En2 z(M`|@ytr5#j^8i`t4?G5UZ;%0q>op!S{vcc9MEEqgmN&hTq1)UX4kS~5#jePK zS3&0}DEsnIuveLzgignq4;MT}L&r3-0SR;mCLbm=d%&R{6VzL%(5V?mj)txH0yJno z8j3oVDC$|jIFIJV&9TY-nl2{+^lu7>g$zZa&0Eoqesbh2*u;sK8+*XACuVA4j(aGg5DOZti5eZttNS#3-9>%40W-RVT&9j(VZr~uG zj}hA`(suGy_dAqEUf|~D2qq?CyEeHD??)!WdDN(oNDzC6jFqvQwytt~Bmoma8#NC+ zU*~R#wf(Koc23A%PT07RM~hx6FV9{xzmfkS4nI2N9Rc9U0Pt?sl7ygf}T)c~7y71M4Zk1_A9!}OkCLgG7qe$U-es@3p zLy*Zzar4(bi%_WLRN)V==m3J zO2_`%JWKTggY$@f2Rr>$o6Jfw=^14EK;pR{q(mpsNZ$NcE7N9fc6#oO{-@~-iNkeA zHEY?pP9gd$GJOnU-H|qX9(Vx|~>|wW^NxqNb06Ediv3-Y$e@)6~a@p%*<%_AfHpZzt zl$}B44H{)4Z7T)r7adUAo1{>*XDD>c7pC&R@*N#^&=Rzca?$;?ZjGuEpI7kNE3ii8 z`=97!sU`exrZQiG=?ln@1=G)4-Gx&l(Ljz!e4Fb>?=W}7^Y}2EP-nsq#gtN##aGUX zL=6RDxLo9XY(hcgJdQ;d1`$UQqah_w$sjYEJ|IEZ)n!DiayHx?<#di$n+PddO$cv* z24NezsLM)$0#ajSQbUgE?b4)uAYA9h^i|R=&P}rCZ18%!7k{AbE zh|oB^PF2K&%ImJU5G~b&mVv*Ij-(T8xsb*L=D zKF6a%4~xF`;(Qas+`~ekme5c>VE+I#o<=&Ai1{{LmE0%3kD(Yuy;0j0qvQ;wPLEYc z<=M#fZ8BV%Y=oq(RWM_R-rbEvURKz;u$4_@f7)1CkZs+<4vpzj>=aztQaJ1(r!Y#c zV>tBi${^kB-u8aUtBKo#Y*b1ZWEy@CgC{^bZq9g{qHQqhqK5xZmNRK~}=xr^5S07*V>Q=zMN@92*< ztkP1P>4^E#cWVcGS8|kj9QUf^dQO(6Q}dAEvbRzv#f0@gN1MZT!ATP&QTfE>?TpQ& zYMYp;`YD}W8*lct@n&(C?P&D5d&HnkTJxy?0LtB0?8|CUEr67W19pMyGLxwAbaj^%F7UfEZ$pC{#VX$LyL&+~qY#Pc?`90~GaWwO@o=9aY*N|e2HsnO`2 zV=n6vlWYvaZb|T~O_Zbb`8&zytt;j6re}23#ay%Go2c; zyuHGZOsNL8*UFteL>3w;QThlHH@`XkMP%kOsJs*7u&?_SMRgcF1i1c;{;0|4VHe{o zdruO-l%v6BBavNY`sJiuwo+=9rXzSi5b~P1vZR!e@bxPaff`gXpp82mh3h9Rm|cZx zi;7Pu*nLGsl;AA;e<*Qs0djV#gz`9KoVby^jzK+jYqq{lzjGjfdfFC{D$ou} z#}L${1rk~@G7zy00zM1w9xaZ&3Na5Qv8`IL47%&)Aiy?_M=O_b*37JWIZRF@NY}z) zre0it9nYOyp{?m(G39xB6jvo+cpPlQk+M_GrKTW=0oRhGhn#bua3D3G5#=pl`tmaQ zq*UwL^i&g|bTC{h-5js5`c&wvU{E+PS9GlI8=Q%N99(T~AUY`N)Hk4bShp4h!RYFS zm4xWxZ!DFOvRsPQGs?u~u_&nNJ#S68*{e((%+*4GN zJsLl!I$=NhKxh8|y2!;p{{U&DHJ8p=x~ms5*TT}2l>{VaD5(NfmQ~b^4G9IqT)lWf z=dTa{0IPHLpQrwUW{v*ITdfxlELi59HoFL`mT#*hn}3GQ z;|62!*Y}_Q032vu?&a>>%3HVie9knHu2(Tl`dTh49oZ@8Z-1J<)_#kk)TJWsB`Hyd z=2=(RyQG$*hEA2+&oXnhrnU@hC}jnnV>Ggz58-nf%?3WCM+`?w%-&?EwX7>~{c!e+A ziN2HVRu;asH-MR^Rzk_m4a^(DfOxyOtJ|*v*sSf%sg`#SQj=aDRo( zf3?m30MlRmn;m(DR3`1|-_9@focByENxhc|Qj!R}M5v~Gf3il5ii%CE%UiBJgN?+8 zR#!Ft*EjzFPk-`le}v8d0JP2j0MlNl{{Y7KcIv?Y02c7w{KfvWm*{n=V=*gI{#eAN zDnFNZ+9#>ha5j>^0k-wXhWs9D{jP8Rn*RXg-#-nW8fCL!ZT|qv-u#R4H~#=ld);lNc`Vx1Cn=k+{{U}$@@{jc zmZV&=5~ULp4}mj|=T{Qm$sc0b0w{$l?CS=5|h^S;7uv`h4Li`inbytZrt2mYHd ze66(Rve}j9P1I%cL>wpo01cR=0lwDH}u$2k`+06m^S z@r$dXQ={jFd!ItfQPlN5hwU1pPhsJ)Ah2b6d%(y@4S;}xSwSWPt*FtU68PLxEo4Ip zFzc@+(V?g4`ZuTZy&VY)=dAw#31&h^>?>680-MA0PfP>b^qzky(u=j2v56Nj#p`Mi zq?G8ObWk%BxC#aa6n&%F1^K&b2v}ENBlR!M--?W{sg$(>R$M5`ZX?z09ObbJ6xpHj zZ|2jv@4_fNCE0KpiEY!Uxw-Km%chhcTK#y*L4X}itU9|DPodoJ9oN_HP3-JZFZ3)k#L_U>AEZsoACq~ z)c(cSHN|s4XmnXTsE-H2(A?g)avdVbknkjs4L_(dznOcg z6wL^(6%FlQ^H;yyWWUHC7QHutE7HbwBN2&yqo%%lCvF?j!}*WPf6ZS10Bw=~03du@ zRnys z)A^)M8(CD&xNTvYMR2H2r&dF4sV8~>v`IpC&^9#=n!G>s*&p%~@qPaQo4x+qCH_G8 zxqEM@S?Kz5Iokt;@AZcPY*p2C_G;u~wUt~aLw$C01#d`}}F8|w;y8!ezJ*gIGXFrwvbpXH~K*8~FN0g2Od zB?vbGRM$?You%BNI0q}1Bg0>|Q-10|qHZj8D844$YI>RZx@Ip&wyxT-Q>*qPqWB|2 zVIXK9OY5c3(T$CXYuMP97HzQ_wYDeb3=sETw!FF+vu<=_C^Oy_RJ1w(b@b|A7$Z{V zve%V^Z75s|l*oUGM+4>_8_WLyns?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@beuL2v9zq<(htcTjM#W(v0HH_2 zXEYp;Qv@;T0a?u`do=xwrXS2SFu3x?6krY7!`+l6YPP3q^tE+bH+WXqquIlc^uZ3S5075k-RV4rtG5~;t_yPR-2_Vq0 z^05)+=M~}=;O7VYS_UWrFtM?5uyOGI9E1df|8o!%6O)mVA^s8)5)+e>l95qRP#`K0 zCn_o`dU{$~W@Z*HE}>t205|}EjDmzX0RKHukWrD)&@nK7*Zw~Kdjp{k3Mv{pApi*l z83hFc69W|s84Vc&(E^!>5rr64K*vMAAUVv6gb7Xe?Rj_Ib(XoGlCqSbZMA>KV9`W&zgt%I`yz=gLk4O-U&*bu(rB&htQ|imv;4rLME~E zyp6K8Aor#oQ{JrZIsOZ1K-T&6L_oD|>fDc}HP5{80_c^gVeX5F^@XoL{zy}Hf(-lS z7_Brc(th;iy*Dymf4Av{t0AB)e4b#zwW*%YF$#56Er-LKK;YGP7XsCVSHQ99*3TbR zZ~p33@>{1D+Mjn^c3hg`94D;330f|93g|4EU99erHu5>0QvEIdoiYz5`hVvBXB>3}q})G+ziR7F zggX2$-5LHr>>O~00<%7SA-&vP`wK9=1d2j!T;;Rm$(8D3kgoaTLc^PMG@Km#%QK6$ zHPx0E1jl9*LdZAfx3nl8LX8~}`QkCYrUG-xfvhS08J(Hlx9{g}08OK43t6~%XC_mG z^iqv2J8ff+Pk8;omoYay=VrM@!Skm%JR?5b+07?%J#*c}2>3qOObn;m!W67v{ za*wHGrAZC;(8BPOD+{OkBZX}`pW(dS_E1~X{>9eDx7zf2|5oNZ&*rSbR{n`T`4lw&X8U%2zu;x_=#$vo(3X<1N20`Lv>0B5_^HgTR!Dno>UpN^ zTM2*Es!_cOzR_|t@v&GPlqKSG?~f=5-*l9jWY?>6uTMO_B+qk+JwG&M=uJeDv#+&@ zL8AMYca1)LrJ1XWD<3hb3ltobm4KJSpD z*=#)|x*_4`+Sc_xtj!=g=O(}X+(I$IQEKD_L@q6JaM?SDMscUThG zQ*ZbSVB{E*zxsLSUSW?s7VP0N-6k+ABP{3l^WR=wwfM9!YH{KwN#($vChj3R$Mhhl zzQP~&Z2YHDlEV=%4U7+12Ob@`dEUH-Yu8mb259U9sgTL}Tl`zfB5iI?b=<8@T>XvO z=x)wzx_Wp~ik^;ckzh=ha z`2TZfsWA>Qsf`Ki8Or$}4o0NP3A+d9lT!323oXbfb@B%VsW$s-kgwcseZK&auW#HW zzGc3Y_yxcVU>$Nkbe(S6ybx{~3xnZlh{3qd^xYdwDl^+M*5OF`G$apFeQw|EPUmK* zin2~}r(LWmEspKRUYFC&4RU^6;(LDOCJtJ%yIVh>yaGvlOOr;3i+}X(&hc5b z3@yj*(aTudnccLa?>p8PYcc`-wS!Bp3xxC5JvKG14#^%3nQfWN9YqIU8XeL5SHFwp zJZN-MojrN~LpYdbEQIYv-d-;-@gbzx;_-pQ!2FnUrjMWM5GzHe)6PW4^U12ct%XZy zkZ>^dSO`y|7~=xeB&s?i~)&7Cgc5ed$Rwu8UNAC!=-KD*}c#i#RoTFr=Q{b!ip=9Xd? z*y(gjo^}do8|RHUeDY_U9E3~$E3M3SR~mM5A?eCzzXp zHA@c9q+2PLVAN$)${}nb}51}``xuX+J zuWY~AxA@O2$G?F>;`d}p_z;u&U(`rB*>wZLe5UClVCpzsC>^20&QGJHFfC$#LrU=H zpRxbjZ~kKa_s~DoFT1uo`x+912PKcQ6PovOc+O~EFy1-QprZ6E>_yo_F)+-Mjyl>R z$>+W(af&(MNv4DLcYn+C9!HoXbG&c(4qxMs0q0uud?Ny;63jUx?mFy6ig1*yJJP6nmE$~?Y2 z_>IJTrR%~no{f?$N0%tA@8=s?xd-(hN_q$`l)J~+l06j6^Ii`xrRZJ%IUMcr#0!DQ zmRVkwlsSE)N#5XyuQV=!ojUKt<$Yj&FL$8qlOtz;Nq&l2GUaET%Ww{J)8tMPf<6)K zzi|04N)lPFzoa+4uSxk!D8okoSEN*C_y0m{BLCauXz{WQ9-wa#wC(B5j28jgz zCgD~D`@OHAH?Y2z{Xfy;%>PiVbUj4{0@adcwE-{AXCr^4e5D9+`S|c05f+2`VEd5~cz`fiV2PrBK3fhTj*~GYkSa+7SYn3WR@ZoKXmy zFmVP5EDhH&*G6EGf z2N7FZlyE&4{SOgAB}1||xgm)*yW)H;rx$l^CT|}|R^pY`SNqK@6njo%eWN@85@l~c zUaz%mQGs5*Ct6tgF&qgEFRF+na<%=IYgL{D*7*39$NeYC@e4_dgtIu%;_`vVH9^4T zn4oG@)@6+^SJ`6@YXdL;>*C_sCoJ0GeEz%t%x-3P0e&`2vtR!u?VL&y0GH%{$W>Z@?cHA7xP9P{$t~SdeX}UG6l4aVad7iQFUK?nmBw zWk?@w{sIhM>Uivz@O`-G`0?T@@Gh4#jivrjMv%N9xS`9MyZ|Xmfei2$3(kZQnAU_5 zYJY19kqN{9-ca~&Ne(N{X8j@pyYhzbrxBz|vx!D2ZA!KzgeqBzjEO0BB0~2%^gC?# zrbb!>QbET`kHLtkaBb2@7|!_@VE@4C58D1-SbVO7^74`p(N+qkk=dy&Y6xcm)B-(k zpJT2+n+M>bf`RShCb@>0V_?0{zG7=tFdnCN_Ls$6kImMK^DdpSC;`-%z;Iq45L46e zu~hi9dT+6vXJC%oay|U6+-kK}hVJh5Xgl8LD zSfWidrN^{?j<4!(p2jEhhT=~|?B?v$m!DAP_bj}c#4ft1mgR`>vC?bFMA@k@+t_)j zndQuiu@u~)paz0U?3(G$*O}Y93*Z>?{0-v&MEmdfMTE08!VLfMi=tc#2~LIE<8IW< zFJ>j+(Mz!WTM8bPp<^-fZXAR1B17zT%gX@&w1zWg+DQ>QYbQ=#cVZgZY!WH2;( zuj@3BJV!L*Nf@ys^$Ni?EH8Y>>08-}k#}~%);>h}rg5%Cc$1o#3g0nu$iukEj?kCU z*@@U+jpQ-m=e@G{lQzuDmeX(z|Na6r`KmV<;Ruo z&1~khObt3UNEQ)#Lg_qS2}GqGf*MBh14NqELzAO$b>{u|4h<4HbHMGI&cu<@_Io`V1#_Eb_7y>k4LBbPp~r!qg}sD zf3r(Uzx50cVIN}Kn}ksBH`2%p0E}i}<6Ot=?@C|U3uDvO@!4t{E_Ze}d=@HeA42b# z8Wb5$v_WyN@SN^<)OIfYYqf}RCm<}irt&|I&X*G>%A9k3DPHP$gj*A%zBj_FsObdn zGc;pu_}n}7z66%yFVMc*H9kuxlMpR0loWppc;%k4Q=?;%*mXHtmn~b-5dpW=NP3Z< zMR}U8KTzA=X8vT2z6MCQ9_(}Me#5)uxxZW1EBIVfP29)(y=Rlp>}JgJ($}V9q9M*+ zb@_6P3O~Mj(LnXZ6oG5kgpiM1>oOUV^G6cF;K|}_{UCGACou*-UQ{8g7{doMITk&f zPuIbRc7kCb;@eGWcVRx2E)Qu=2SR|)p6&K}e;%E0p0ggVzhyzS^WX!rO$ z6`@GBEIAY4p(j!Fh)<%h>Imusz24q9yUbJTJzq0LF2;RXk)XewN~CD;SughU&w(jhUb|)Dd8AG)Z@DPL zM@}bmAN$RnX=Iz3WGAD55feiQXrVJ=>i2%-60P*kzuZ?aIDt3fyVkCke9Sw(@6 zsA4yBQB&OYCOz+?qKnAlY#70=3=SXZH)gfuc@xdboMF55Q%Urm;O|`WH`P&nlDJUH zfU_$%im|JbB!UnfC1UQv2^z8_#=cX>leWJ8kR9Zeb;)ym%P)TG%0Ro z-B4O8vb3;W!8P&CU|9lGy63qj`XqmFFS&A$Ii;)(ZxVaf~5lg;lV=MX|&u<3YF zm3H%2k7|g<8!S{uJAsfA;VFmFBElvF?^X#fJ&a22*krn0z%RM2$88>dP#3lm{>F~3 zLg*)R9IehTpNIP<44GwZ1H|B=flDBye;pdY!H^s&17|O&-qZ^68&^*2GO2&L)I=8$pNz zWVIj|P8^p0I3p#XM~s9e zyZ{7!X%qXTlCd!GS&vA`)2=nye+rZz&{eW4D2y;@@of~+Zik!I_Y@{HCx`3|L}o?5 zYQd`NjqNCW39@x^dp`k<7I!hUKQ3)@s$f@@bTMwDMCDQVo!E^LXJY|p@^U?rlO?`G z@JuQ^VS23olJ#5%6)fe~1B70@nmTxs?px>5 zgVU6Z=4sW1pmht01|7Pf2fq)x)3ulL(*>?HuU_Uec_+N`tCpn9%_`M?-MiYU*;)n2 z<%fG-^i0mRCY8eViRep~8Wuk;yANxvPT%4gQTA%k%LGX_ZAAw@_ZpOyA$6Yotle>U z?iEbKwY_N8Mg=~H4>Y1j0m1-OEL04wa?C?gSQagrv0^;+M_@6OLGOW-@s=Q&`JCOf zw_K|El4E0R70rmJkD6V}P`nF!AV+r-bs)vu#NIU{t)QAg?N{~GwR+!%=v!LNC4%A# zzRYiK34${?eIs$o_)YBE9)>>7lT|(6_2`%5K?@tZ?HDMHL<}_z!w@bDvAT0<$m_@{ z6=0M_Llg>`)=47a^h8p4aW|7g_~QZ`IdUEiC(FAmtsP3@2oPG~<8Ly~=);3pGzuB% zcb|03ja_dW)XuKny=8RaN}?)W(%V(-TP(9M(nS-_C6k(5 zJ+ykOR;4I^nzK(@J^b%Z1zvJ9h7b#ia@go>_vn;jxAzu8IJf3?PIIk%xa|D z4;N#qp?!>)+6v@V3y9@yfl7fYHZqrC z0n)h+k*@a)4=FmBu7-YH#Sy)~Hz_8bND zxd1MLpwsQnGLw~eax_n`jVgiFI?)Elb)j(8B+)#YCXGI)#(DXpLpAr)3E1NCEjJBY z+t6(}iM$_%xLoq6MMAKlNc9BW>{X$SiU+iAc%l)d+?~VfXRI>IzN>xHT>PBIF?wNu zfu0+F(aZDPH#L>o)uyA_va)_efaBK(BPkv?r$|PWyVYjiLTt}F9|xG-6!I^t<_=@MG;p|!>$!((G(@g@r_2%;K8`}R7QB|$)Ea-?JYwciay zD5D`YxDAY-98+*Uq46-`D5`d!)A^){I&M}>oqO7?(&N%$SiCX;B#W|-sT;2F{l@Q8 zfeRDyXC)$Rb+QIF9<)&ea2aTpQZCWk@|h2F%ehyzQ!!0ykQa_wn_Rn(r9UY0-yGo^ zq?M9&3fR3_>o0IWFi>TWAswKsRF#@l$7DcIOI*gEcSu^wPFQXRr&?!OCFP|u&Qy}*?u8E z(<;+*N_J4w#1Of~e`a>J5J0h2^#iAWhWR50ItgA-o_^1itXLT^4F$Nhe-(dV`yCz_ z8;N)`dVTGp#xm~1Y`?j$SKIz5%}*~v-)y?VmaeV2!sZv?%VN;^I;USn4V|yYp{f%; zdQo{#x28`&20Xv%+rS#yI4W4$A+L>Wq?GVM3Dpsk#$H*eU#Aq8$oOhL`8mc3frAk{ z&>KBl(ujI$GF%gJ>ntrSjvo$`>qMjV2 z_8fy?^WH~@B>>`Su@pyE+uISrALYC*13a8KT4QKN7#5w+1JbD2YQ^<=%HD`H(Y!RW z&=00zzNzHBLCU7f9eRjk4R?Py^|?MpGwM2#7CfSNrao8A!;_|1d=TsR{K?6C!?baH zzC{2Cnnq7>k0HBl;dKVa4|r6MVK!!j zOEF>yD@z;&Tb>BHMuaNrs6<#v&BUYF5w)3!mVZy`$OSHJ3JdC_oJ3d)I@rbq+RY^H zMcCG)oaCiFj2dze*GliR2(am$2Wxp)-Xir%PIs6ro*MMto-75(dRj;%;TjEXwkLJc zDeqUo@msq$>C3Nq@Lua2&U-;PGa}Ee>u!@d;+a3#=PW-t(J!!3d^#eD-q7nXdZ;6` zm_oK*RfXTLMq&;c>VBG|8AB-AP1_sONM(kE*dhC!-EvS8-)5(wbHb&0m&O{_iZHlt}RWRD+vre>Ns9o{1)AnnPoWmXzl4ic{_uK2(Pch7B zm}I%N7UCIscgmVBp^sx+OItlFd75pTdaSwHuf98%vd|{KFLif0P+hGiV5$RmLr}t^ zHZ*4TTk&Bl9aY;QZTvkTo-pvT>K*?lk*ix?UWeS#n#Dm;3)*LXBo(uR4l;hu9pj7H zbAn?d%{y}_9wQ)I#N#M8+J~_teKgfAJ&?A8k-kb;F*GKk7OXXysf~C z=@~uK&hBXSVq=q=1vJ7Idb~i+d+sLiHm~{d(ox--4_ryiFdq|-O>Prs!smiQ|aUu^?Ay zh@jLXdR&8JC564hVI6Z0!wzCbpoEDC>zhu5DGyfoZy9=og^f(VRY`kX^Lsr0QYLN0 zRn;btic36RJ8yKU=ab>NIJSx^;tPWWJRf9vyddiy>qtCY5UnBUFF-et?2+dR*B-2$ zxxW+0V>-Ul`mD}tC2-vhwWU|XZKnCL_=vA-?d99t3A}+fU2km32iT`LFQlHKs+A<)<~xJDU^8}2c2;hdsVicu_w(c?GHC~ z5H_U-c0V-aGwU?fTKlKIu?6!t5_hS1tkX&w8Fb9U>rud>Rt+f7Y^DBcIPJESyQ@gd zT4PT7sVM{vD{b4~7pp|_ zs>DdUy`|c58D9p-E$$(b%tL99i_-2PT}eK|;lWi04fM#Y1Y%BCYt`H5C$3c8syLN? z91^}^V!ROE?jasojfF%LK6dJUwsC9T@GjKpM2)X&1-Rvr@S$8Xt|H$(cHTGN-7mm4|7kORaM4`#?qq3?eO}M*>s5!ujN26&NFBF3x}yhbgDD51Vf z{KjdIc<-RjXzW}3t6iC+~lWdAW20GI!3hITos3^SA zcm>5-=Z3YP@Ae|U_12HWkK(+$UrmId245zFE9rICY$GYmAVZrD?K%%Dk-Z4L&3$`%@%{6SejRY4CM#pG^N0yNZ zfHad0b{=j0;F+qfQee#^h;nmn)Xx;t^*((; zc?S;hgudP!JxyeBXY||{F5w$g7ah?-QhmHK6K(1+HB*)p+OY411>(!U z2e$2u;{p4zV)VH9dt1FYS$;Yh9YO=#y`|%b*fX#xgWED1+)eOgq?vc0y=CDQV_(d1 z>gXlE+DiJ%Uw~}hN8=a4ve+v)Ba2_nQXdLNfn)ZR1HBezgDr9Xy{o*P&Mel*9(Ygr z7_;s@JZI-|o7((i6{OB=PM-Kh;WMG{R6qz5<{QUI4&mgM7R2kziTL8rhJArrR6PO! ztxkS$Z0^=0*OmDJh1vRU@!TLuvzHOoflOQ`Uf0|o9aOpc-uqY4DNnmIBj5FR!U*8I z0(o@NgZ1|qB9|D)Ns2-L#pN$>_bYAj> zP09{@eC`@XX_V7>lfvcjcxA$7ZGN+B!jvT!XbHz?)M6BV)tdKci~PZeb$MqqK|Gt_ z%*Pin`AywcpAuF3u=Q#ZoB}i2a6mCt|3(^$R1);EUs`Vsl@pN^xXHNyMK}XzL zl)mJu>V4doAhl&Ji%Z(Ascpx!n`&aDz6lLyuEHI9JFWR^P<&AK{!tYiTuUjKGeOHj zhVT*br~}PbX#J1cxhzxHwNvt$n<|34wiW9DikDp5qC-x+y{ba;G`8nQCW(U3yscrC zn+yZ){;$@y?(gg#^q8|ss`}rwnc=|)pNQYeXYmPaE#5SZk@TjVnu~~h5Y1#R3sl_= zavEjEOw=?Y{smZ)?(ZCJO|?+_1&FHa+hn=+eZw6vYi1lN=8VUoaf zUQ%Eq(CVg0;z4NsmB?Y$WvS3fqz$R$Xi+npJ@e~C`s(4X?iSV}%v?~UO{tndiz8+#<4JF>jD_ZoO$X`CDawzm#X&aZ+{xeFT4!c4{Heum~(us z4C?l#*|zS-yMcw(tak_6j8^YqIq6*EGIH#sEm_=(R8zO~0VFl}$WrfhQh9M({1bDT z9NrN)?*^CL%cr)alMBudg^0#BcR}omiKWKC)RJs;{a!&Mq^4Tm<9Nipf~iN3D7rV_+!90PKF`KSSm)cNMKOb;; zva)_@q7a>V%M-M;Y-M|Lg)f~0?X0h~UUI-KnlDG@iEDN)C4#1<)G1uo9K}6M{rqxs zr)!LWSKEnXw-TFnQ3`tw($86w3B=BpKv}(z`V*d5D1aD+(hQ3?3OvN^uf_~+pI&-Q%v%f~CjP19oZrV+JMAhIgCBd&!VNM_I_ z&c&^%cfo>Pn;bf)D4A6y)-Qkw`x<`}93IZ-C5S_6wp&-+(w*G19F6boaB?DzkK}fn$L4%aqbP(@)o!FP``=zsK)g8Fo;IBM)0bZlKb! zB`NnNn=KXiVuBM$+1&+Y6(vv)83taNGWEH5FUZDt)W>D&z1NkZx+)sO3nwc)jol1~ zAv;oAV0ahZp0<2Hd&17-d7Dq0?#Ag)40ZR1`prSlhs*To#@i;A=6cx0{7r6}fVjtz zO2&m~la~BrBPOe)m~Yy4oR{;?%kIwAw*SK$-U;jF~y?Wr`9~pmd^H~T1Ay|ZN(IX8+eYA z5$t)yFF+{nF=94OGC;2>vH|XG+!bN#O=;)Exdh}w@so|#<_i-m?>><_YU;371Jg_z z7n!+)%N#M0?r9V?UGh4`*COdAIQ~hyt+;1B9LS;3 z>(<~Q!kh4Q5hc@lq(1EXTwQ}PJFYuMHRCuhBuk!U;>H?D+z;aL;;I)wT$sC$FFBBn&H%Sg+=WiQRDF)S0Q5Kakvbgm|c zFP@{BchxFi`gOpudG&VZRN!VAt$#}+3e{4QJXd(RMKt4et})@;dsmkfnTu0u5A=2? zsrh)d+;8ua2+Dcc<5kO!k4e}|K)K<>=;8SF{x_-Invrwr(WFr~)gnAh9H`#q=;-iJ z8}s4nx53Spb|d4~G?1nsGj616n%ZqP{*ggs&I%L;ve7oDgupW0jUV~Fay&A48%<@> zf~~D=)6?H_2UbTZ>ZW>zpGDdOU7LFHGc)JZN}o)^dU%?b1%FhIv879FhK1jj1H`lO zU4}Dt=s;P60#%XNs)rSVPBcccLTRwf5O!K)^NT5AR|5V$bH?h&haJySVW;?GHUlqp zO}&Cu#RQsO6Tt_&WrfivK5Yb~bqWrDqescU_Q{YZ%l297Ef7xo1kHjH9Ms;}5Xo$_ z#JmM=`JEV2Zq1X6%Q*q4WDcO8A*H27Gy1D%XQVM&IlI=^>2Zb&@W(J_jHE>7l$7ClL5Z~SYI_tb|n%zT`W-Ar}4y1@V1fJpIwglEk|PZ zG3R&`emKHC!GZ7lf4$CQvfyd;q-H3mAaWDyxuwO1-bcrm2<{M-)pkPvX=@oMg1?VhYPpe;L|gRdCVy#Y%$~7dN~Y zzukChgmv^ysHO%AG{NZwU;EjN8I^;&X$16&n$7Gpuw7nADKh3*_`q~}w+wyFh~4$< zWu5X={krpmtdRfQ!g9XIg2k;>??ArDc#@U>S(8bd1-t!dd*!l6^XTBmH@i{D`0ulI z`u*;Gj3Ib^1IwbTt1)k2VFLYLL*%ROR>=O2qCC!!td!S@)T?#wr2Yj-!E2)hUDlR< z0%~E;%btaEOfTFrrdVN5b+?4Us$+^nct5K{05F;L) z;5ZAkH%W71Ni$1sL$#V$gBhwQxv-N<$fVOXr+*-B<6hEJ@z!c04W(EdT~$}mb+T*hn9Znp z@W^Pd$IMw|sQc^6Sf%c&{g(Llk){H&WBYO>~3VjA=(?6XKdQAs>*o9MN zru2q!&Q?|D54>ra@S0*Z0n6cfs^IZ35g@BneT@)%!yTh(b7C;s-|!&@L&7@}CkuB( zzI3c)32e{=d8DXVkmpr1v2;>rSe+5$R@mb>?6{Y6vL_rxV@oU07L(6TnjtwdP}!6* zY}#xZhIo5c)?xb7c=$DiI*l7oQiD5789CR#8OetpU!^ zm%Ml$R`81zhCEuWqL`hyJg(gZh_$!4gPi}220WR@7s7PB1P7Y4q-CU-`+pujhX1_t zfZtNGw2?#~C3Ef^zxKRYbvN^mq7_dPcT;V$-bC8UH>?$2Q`rB~e9Gi>EV%dP#qc5p z?L!M%wFk(}SS&mCB3q^m6cu9iDi=FV&eb@NPer&1gy&?6=>AOlLyCSUey zifm2WpNwg&ASCOjXyAM4-i{wpL)ei4$J^bx)N*vp>9yQ&DpEu|BR258uo1{RK{xFQ z;_rI(G-?1d?WU!Q<%pEuD3&}b!`oN>ImOuxck`l^6FZHD#B|pTPp{6)hLH6UlW5*Y zd3&{^-@pz%^%5!HX<>Hmf(s!I^&MFtsBmNJ@Cl`*)0wC!%4Bu`Pw#<6(7DiStpQtqZeQTLqXY+elwAG)HzU5 z3lv3VD&DzZ$*y(Zc=_{syfjre=ja!ps%^9!ih1&$`s)jdAy{v=>n)$Phxc5jm`isy z*b1q%nATI6G?OdLx9j0KkL#_Qahq#=v_CR{1!>SJV&5}_&$s6|oBR>D7b&BQc1x>c zZ)g?z$>4HL^o)oZ7HCel!ov$vq1PrLVf)zMq9yo3Z`O159q7y@!_4RL^Y_c}eEFgq!3_db;k zvN@C}4;)$stED;^s@A8&r54&D9NQ<3H08*t8UxdE=}QySsYZ1X(B9hTs_ZkNylpsz zp+^%{z_%=TN|kh0H9pMMq*7*#8w593M-?UvT@S21jS=Rz)ECs7Ye}`8G#*4%&s$ib z@I8#K8ES8@4>7>wKtYE1Qom5;4%K;Pe0u6r;PR`DGx`s_A!d@>aaaY1S04yl=P0Bt zJgBYqb`K5ZNK_Jm<={;hsf#YDsfZOD6(poWz7=;qcjPwGAxc`=t-8^u*Cwc$8+B)W zmehS8bohMzu&2ld+dE2nGC-dUq+P%Fc(WAL@3v~SP6NFe9@#BKN@jQ*(sG>ukK%aY zCzvP%f{H4*+>f%n)w%?R<6=@xVe&o$^|5c|rx?((%sa=VzS3Y4 zS+0@%a4RvGxQ>%|^!#K+HH@As zS)P?lNA{z50x!L7s#XHOS`%sv<{O*76i-Td6p>HiC>q|@u}^3Dw&EjjivZbRG9Lrz z&N++x7397zR$_YAy~JLswl>+Xk2V6(kYoP-%rgAEvk^`Uu!S`UvcxgHEa9XWpOw8) zgMoSZ#b@T!BCQvW!AkJw^d)Y>x+cCialEfs@}PuSu1y9uns88a#KbK#h8RE$rM8iL z1wbTdYVKU(`=bYG-a_CRg;{JZ!_4QXvt7EbBnf1sc<~y{jprH>)|a($rlPLG(9DK6!>DU^l7Spzz`)P_E zzwm8!clef55*o((?PG8*I{Ca+6PoiXu zpNT~~0BpGEy_yV*#0-5j7`?x5ZdMI5fm_^cQCcuN`96v2@q(ayADIuCwY^Y&X4;v! z^N!m*5bS%ukdG;3A$c&?T75inD_NXk!DgwfRN-?haEj-C+$}2#_ZxkB0Zpj_I(Vnk zKz@pQZO0O&cITRYdi}yzR8$)j%8G9^=6{qM-5!I;ryH>!8fX7HG@aoc2j!r8Ms#+ z@%^E%GSk2fFXXETw1!AvtkNDV-cpbr$SowVn%f3vJLEkyjq#trK~*_Jx%)KPlcJ@6 zJVf0=reEMaD0PF8#^fFUJX?llX{NDZWqsMy%?K3Lvm9* z+s`)at~HaP9^XgtM9SDXQ&&bKf0zylnarn=9gBxKMQ&|bfDW&VVZF!V`gB1xE#Pm> zG^(^AbKR|UIxdu9Rd`1&v=Tc#quz>trzLF>to=^IjF)zV1jg||22sCla!ePYukT%nRTKK~8NSreDgu0lV1Gt%b57h{JgW5dm|BIMS)3 zlPy4M!a7B<&Tv$snt`f3i=maN-M5r~tCg+_nbpDG(jX16`HUmdLbzzWyb_Gem2YO_ z5r4AWG_#9cop0Bc|FPjiQ^1$ac&zPyz4{4iby$U&52K4azL6hyQyE%!mDCKm$O3l~ zde~ayqY4kU=ev^zZ4@}r*X2jeyC>jWq(!PNAr4A#zBKndQZ7bErNUBT21jTSxp@8QZf4TGI?v zuZrnWiWG&FLSe!SJx|6EuxkUK%z*}<{Sl! zl#?9kLdRMYPo*jyOD*)HJTP3}dIsyG!xi+Ku5|Ham5JOqg*QfDpVBZ_h%GmxNz8o3 zXC#!s|6$(HJqxnE&>)*Qp+)@f2iPhJyWMkO2wRLYQJ1?$aC7^J-t-IWY2U``Yg+Zp zu4%(O=3Q;qm4^3mb=}hy#2hp|4S@{9;hBErBuu5^fn?eK;1M}y!=r&V8PH2LbNipE z^`G_ZHwvcwJm(r^NS}j_J;*l<>x?nN&#!RIQwH{0iihQ6M9bdbhZnrFbQ;t{Jr$W_ z)_g7q+%h%z@qWL4t-Du=2*v8X;)(wIca5AYdMbdJD^H#ubfqLUMt<`aU^-y=TJw=h zU(Y@%Vug4g;2d2!l>CHPie*yo1luZ@%zKD7)-?!^ly+>|0;`H{vc;xJo67FhWOBni zoC_bWZou)XbRJ8eT8;)o+-!^e4b!1DHnAxFivSek$#5_Zmoa_Lf!`|-sK_J0?V5pK zyt#=kLZO zeGhNF$}`>QiOcU~2_91(;EWqiELi)h|Me|*>6-D)(E1tvNAIBX)@+aId^@EFRt1rCQSyAa+_*HGTpF{ z4`K5qFZ9o^KZH5GQ$Im=%%9cgu9^jMq+GwgG5-ZnJH!r^{67NTAR*s^<6?`0_D{4f z$TSD3MhE4DH18@`zuV~vc~ds}N;5RI(ib=VjFpR}Rdui{RKC`#S3Av(tD-bovGR1O z-e`Mpoh2sGEo^F>JIXQ(oFMb58-zAg+|h*YE0$d6P?{nKUfBA(op7gPjb@6qr*}&> z=#nL*?Nf*@-Z*aBF~Q@ai5K+_rF#06qx0DPmwOtOovT`ig%%My>|F~EXmPKz-nn*~ zUg7|nE{iQX+{hw$l0E9QQq|WC1WNPuH!gnB_65_qajg~hDi(_$ar*rG&V#HfwgSO$ zYe3ME5lW|96bGqA&1<5B#}HyU5r_zAH5(>`e z01b(gI{J^?mm`4#dq=rRxHNDloO;y+_(q$FB&Ex=AoPMl*BB9r&Kn|wnMKXR`CJV( z5IN8d+&Q%zp<|+uM_qmBZ9$h|>JEuny{30Ztz;424KyW5!UWAX&#e$#H3V#k_7u4y zgP7r-RiHJ8RDkMK?oDVXtDD#qZUS^}oJa3KInHEM?o{tMoHN#@TZjNnXN7&LE)(6^ z&B%azp4OQ4D&Ug-l%&d{^akt4bw{wMYv?G0 zB*hOWw1m(=p#ym&o>h=U=|=;%s@pW zr(#AqG5-Kh?+R1`VtKyw18um8`F8cnOLF*{L&P=3&nK+#q0vS`;zvaL6pMi4JA_ZE z0GC*~TVcd91{#9{sQlbG>nb^GSM1yld!@P_KbNjJtDT|Q1ZYSH*&dt{-16fyCrG4o zD~m0QGgz{!Ut^vmDPm2)R697@t(iI_K&n3MIE41R9ZYl|QbX9n>66GkbI7s;00H?Z(8Q*XTPbzt0O~tTT zeXtr&Qs)O%b+3Q6?aP$bIGQ!w#Y78u%p_u}b{A|_yC>}BS0bG&v}jexGGG#)b{x@P zPL#$7o<5yxD!`frG#{3Rz0*K?WGP`I zR>%BIdqOp&jl}u|AD8l&oy$Oe2AwIrzgt(}-J#7KVDidfYMPH+!kbvIVun zXY^$`#(wX-sP7saaXi!NO@COf-`%`$Ai(peLjh>{kYW zbR@dt_=C)V(Su><`;g(VJE!-l3_@Yn_Yw!T6G?nlH?*h(h~wPzy+Ez{vp`ASTs^Rs z$qt=&7%8NKs68?5gaF|KOg(tM^<26R-f`Z2dzi6anM2LS=rWHTm9mw6kOlP-jfH~6GMO%#^b~cAbU@-EXM-G z=HefjyocC;Uupaf=|BEd*2DNg+l>(&JJp5_=1(G^$41ukEoqVS1qToWcSzi@Jn)1AOpZMCPGgBblbKg*T=QH<_CQW~ zb3_tb)2aejF&!YDsa46i-|XjgQHu_h=RkALRK7pOGm9&sdYs+k8w`K-3Uzj zky^Oy4bzj8YeP8Dim3PdDBkd_LQevYibKmRQh{U zAFKZWVV>72fm91%7YUAKChDuT;6#^o!m`(}=qi9>SY#a-M^C*>xFj6I2LtKSq9EdM z&?=YAv?d4R5|evCvj}h6U6Y4n#x^w zN{@$83ipX^#*yXR^($e;%p51ADi`AuW6HVeR+{Lx+3WU{bW(00aR+dPjeT@b!uNp< zcHI!$Zxs!msgG)z5Nn-?L?pG?MaFG_cxA#W zJvtH^DkLZ2ZmoCpk%CRM;ZV>{@Cu$1b#@c7QlY$B2TE=>pPG6h05a%pB6|G-lYu0d z5z{KOt=ni0Yuw|ok6Iat*pjdlc3Db(63xihk6%riXyy^jtYePu} zI#OFMrZLW<34I#bI}-*r8Fiz)ww&k*n@@MlaRH>69@MjBH|98JNRp)IF~hph!@!Sn z%WS!E;eZ>3#*Jtt#Ptp;<_)0H?K|Wvd3=^*IQ$lD_J@fM?zi@d)7Qaz-wyjRF#O#F zf3(wI`4Q|K;d{##ZP;LCrZ(G029Hj4=!T1Vyo!87H{to5mBP!`U2N(mp!J6e6oF66 z%D+M0%h}D@?C)D`#+ajo^v>R}zJ20bjmQsfO|nq#%JZ_4jiF7v~L!-lzP^HlE*h(BXv|m!+QDx*0T_j z_QwAJQjY@IREOJmS@$DfWYd2;Nj3c6NGlss!O~)nGM|^D5#t6)0%n zP!5Y@fjvIu19OBo(|U<%c8!jeF39Xq85@F$!^JdNo;=E3lN-e>3UBbY)~z<%$d?F> zuoPJ4H{~W!_T$^mN)S!kpbRw|oKLfdN~i^XUUhToo2#dCTihHWw#e%=Kw~iLj1H)j z*WDyTTt`3yks++x1C?0<+NBWDy+0eh0>?)RENzz)ClY|4dM0^NNtGuITn{=%94Ax- zrdzyf+I7?{F1QRm@_G`xLlKDcm8@$)t3l~NPZM~p7+;`6!`-7qdU{b{dz-K*2pc_c zronBeuWqO>CxoR-*`s-awg+CgV^i3iPEfmA5n<%E%gHpG;5QaF&q?K6Pe!{xJ@Y!om zr%J4DTsn_ZmIbgwYM}8YG~yvHp!~Dx)Tu}`K~|ar>Gvp$!$&b$4z_d=$fIDaE%tXd zgHTkQ1}Rc=15PkW<^9PHbd$X{kWOK+ZIC>p>{eRP>qBwVdV}5Gx4GV=);JcP8m8il z@3csBx#*j0Tsn~ocI`dwm@tuixsb+^9mn`a)qcb_ z^__XuvueYu+7Nv$4kSzXN*?fjuMC0R7-Pf)vxIsX6*$@rDLH|>Hz?z~dtcMY(ppupB@NpdkaV%$s{6a$;UDhcEATC=^yz$dk`rP;HM zRzQ(=4JuTg>#;ZrdAi$T8rFqX*PiY^rF;O6(&My~$fO$c^Folsp37y%vVDr!6w6r5 zNat2CT5rz$Nx_V74s@4`XYN_BEZt*%Mvk9JOPAa_fz+TwL8(?)!Kgf_2=*lPB{`Tl zs}698mntL~L<>wVA1Z%mb@e7aI+rn}z|vL5KJVr<^w_Bgn=hYF-n7|Z6>@eVwN4sE zVL(v~W-8)6A9}YfwkQ-!?YHF$0OR8b!3&oh{xP{hp@Zi1=2u0gky`S55}%hEhZS-u zEKT4-OPn}Lju6ulIuqM$6$Oz7L=ypH!x^bQ)Wc-(Ay*7(Fy{|y0T|l)VV;jt(`G9U z8@<|cE8kUUNc+;edEw1fY!SBWt)Rkph$p0blCD1cjFxn?&$bG*_T6ZZ#~xo|Q*JwA zb>s=#c!+!Ju#6{^1tQ&rEz<8vQEiygTey+zq5+Mz@>QVFz^j|jxA}n^96e$b+nbs? z-j3!T*Bh*RFLb0 z>E*9K5hcsA%k6zME6&)X`WHeW2;!*kimlSCjLgIB9b|Q`qc1Yxp%b%;9AyTHvg=(< z7F}p?GX`}Apzl;&^Fo-qbTtTM@%|<0JT2%O7dh?mE-P z7ciEPc7ne|0t)!Q+HAM!d`{zlhTa13rxQnX;so@s9=4iUtb;uN0A@I(k!=2GJ!-|G zA0fsMB5}*&tq-SC@tHYA#fFvZ210Lc^GY560K6v6yKmLZKAY8pcG=-={8a@<7ta>; zijHNGa%w^3Rvvtlhp6b67d)9z8-N}#j?9f5x1tmYc+k%D1KQ)Gp!KOb-YSZ2(NOPf zjmeF3+o3+2(zpSWa^BJGS=u?&EotOF)aKoHQByBGSRAeO0WG$sSkKI>_Lm8%j!$A+ zoJUfW1{-Y}Y1X6N2?gC{U~v^x7G#|Q(#wT1+uH7>YKmAneVeu>w%#2~E6#I3s^`2k zniq7X*enay+o+Uww=U6*jo$LBVG2RmG@{l(yxgmeUisbcNWE#3-Ffz^{{X0Q)zY=D zFQ^7Mtkd2Ii($H{=%v}lrDL4I=_-RTf|}N`AUqQa>oVolBtm7p5XRmCR&#{w(63x& zhPE7Q(>#lp4W1~d54G8zHBEcjq)JOb;H3)JSQm^0r(7pRCo#Mh3uVR!Dzt#xcAfKL z!@8=`U^BCAp0oRmNVn01<^&QvQy6XUGX{}GnG}P(PyGJD4A{qA=$Ofs_t~S_$66h3*+^mab&6))J*IHRh zp0T2&F1@^Y)hQ=6%~kOoiO#)?QOuwm(yA!oW!D>uRC_y&Ol!yh0-7$9!DliG?XEXa zIg{C#99p*!E1v1Zcqf25E`65`1jfXU+2dGMF_5O)!fFzL<9ifHh4y=yz6|GI7A-ID z07zd~UcO)byg!q!n!7AXcB%z)^Ijv9$g*$VYflCa^Umgq`j?2#{j7WCxl3J`{7aDM zbTm+Omu_Cev!s0j_3K%7HN68}o$Ol|xbUyF;lNt)0fgu; zF}>5*aidM_r|^aHwI!k{AElXg)%FM4du}n`E1G4eq$!CS75VSO{8KsMkHwd{gddrX zs~x?%bcj7`>V6mYpDCWhZvO!NwdzsF>fC>Y+&^X8fhseiX`b9(KbpOKRgmCn7d-%x zqJmm?)O1&i`&VPN%GtjXZ9KK#vi`oO0UOt>ZQZm<&sZ6}rTaSLVFP}x)|#wMVv9$W zrjLie;#nh*O6zb?;@+gkiP}$vIVtyv< zLte(3IypV5;0Ol^?>0R;!eQz$$`cLZKPYhO^eR2Iv#Vi32^Psy?3&-1EZuS7tCvAk zq};|&bkQ-kb_rF)l>24u<4uK7TNZ~C3O742BNQH`S?{?0PcleqlnIDIa9SFU&tlLN zOsF-jF7G&j&X`zut?bgl(YI$T!HSOSHRMcwVW2}q@Aoc$Qq_tCA1|!N?{ZjVD{W>j zDssjND^|fkxC9H2(WH)`r*Z6djr;kNZMYFe$EMXlp=sKcCa|3XX(|}wN)!c_*&IrR z##`J{4fklZqZbvL0^vlu7eR~xD&b+^3b$aSOPxt4sW4^Eu1;xdbOB|{#wpH#h7#Rq zuqyrghLYmC*0QEM?)#B~q$m(oyOz5;$URyWP`K_Ag$a#vdh9U?cy3u+tmP{ ztbe_FV;HEbR^9iTO3!}$pNtmA=%DQxoPqxU{d8OlwzC;~&a-zdd9P?|oEXu5j+mEa zU3s9>LP1;}*7m)%@ADu2CF#D@xUObDIhfR9@ zz6pUH!qY$eeBbq4$HG!&vs2mTf$PdVNBlrixW?gj}?%StvZ2`(746JP`#(xe6e$E-)PXAZarGJ{-xjp z_b-{L8k0PdSX~wtEN9kGM5cCE1km&>85TsS#nD(NqwY~9%~Q9Sb-onh0@W0TJ)ut* zQtI}$ZgA%Ss`som;G1v`%0XDg;3((fJPfQRbsnOUNoj&0jYfqvw>TA|76p+10G0g^ zx3Ic61JmzN4vK{C+57!Elj;NXBr48p0pPCQT(>RHPYj-7hpZ)`DEy^Wm4I0{b|zZH z?j0%YvcNQ*>gvk{P|d1G3qsc_>HXA_PfG?3_-j*a~4=1!h|f zQ*jC|B`oTo2Rj#1Q5A_%B}Hf!00?u2^pT$7rc{J$_M z{{XF*Uq5R}V}E2Xd*SzvUhCf0vBc$W0v_B=Am7VOjysJN(VDeC{0|KNr)T^t(3U7K z9`kW@eA`9pe%1c~{pI=M(H!g0vpC5g_C6cK{{X?*{{RZ{9zEjM$B4e^wU@a2%xj!q zXgZ5cLH(WnM(aD@D;JOk#0`ALp0<0Mq2Z$R735dUq|QJzs^!9LHWC?p@=m z^K2vr{>b(vx5x8L`E2D|`!=x62(A|m68`|5-r<+Y4b}&}N2c&4dn^|5!tD6x`<6I& zZ)yJksJ!3v%l`oB@?Yc+dSkqMGjr#$lzdELwTdLkiE)mNi5)hg)$O7Z8yBPSpO-QJ z03YsMUR(Ly{{Zy)FY*VyG9Sws z4|+>YxFvfm7Quz-E!@MH&X4}(-`ryH{{YVKf2PTQkUi_M;rWN=n|kZx6Yi2=K>1s? zRwo<~mUpm|*m`E%1Z+)^c&cMSwRmXYRKDUnNZ417E-T&KV|zDlcd>XC@6g_Li7>T6 zs%z9}m{JM}?E+N>0IaZ43GCv8p=%0blFFeNgea*{*c5wtutKX`@^8>sk0Wg;EA$Zw>*}Y=ug6?%h+iwUxVNn%RYmB83veAUlp0J;Y kC7}R0t$xYtRf`71^eq7`5ypw3q{JaU*bAg1iP2I2*-}#;>i_@% literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000010_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000010_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..7aa149f2b910d0736ba977c961d3e2b00b58bb4d GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|J%SBl!T2y5CJW=k=#eUp?r#_$ zMx%$%ND?2(euu>~h#n0GNN9|P!)Q1_LSQ(A!xUf*+QXWgUs{}swHjsuR>M-AP8m!P u&z?PV#umgH>NLbRv?GDxvlkjpG@)tda&X+z#LciUArvm4Bw3mWB=i8|Ib*i~ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000011.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000011.jpg new file mode 100644 index 0000000000000000000000000000000000000000..441881f89b116128c6c8426db1ee1346f7ab06de GIT binary patch literal 29045 zcmc$_byOVB)-F6qumsoOGPqlC4FP6?ySuwP1b26LcXtUM+}$C#JAshgNq*-&?>XzP zb^rR-`ieE(MR(P%XWO&suHpCc?`;4YSR5n{fPw}9pddeh-@gE;Qu;22A6S^!nOIp^ z0KeA&9{~tRNN{a^)ANKcfJP)mxWmugJDT zE*hWz=Gx$+L!bZeuK;ueC@*MV7$|Z8G&%qV9q@Y<@D3sX8XeLX0{n<)EP}^!4aMdo zciO0~9tJrcs8)JclFZh(X{uX-L@_wC|~|wvTBxIKLG}Jch0V%7G2K*~g#17}u+?t4Ay`6*A|7 z?TSPd>C$@_S#G;M((lZVWGaWyaa#nMf3&IQDdXQ3u;Yd`J=gdQ#~j=I6-)U?>=%bA zBZnXB?dRH}KA1srtfYfj3H3Y=Jwbvx2@As&;l;G+CTS5v8y$??tW^cP?0B104}F^y z|3dbU2)WUHt9`2?1j%UwC1nYBAEBE1=E}|HMnHBfJ%)HZMb{qxZV(7O0iXjbdT!YMBA=N#THgT=jvt|6xsL1&M_$@s&`8nvd@-mJe%I2N|u zRxC2wu$Y1z4^=C*{#jl#^8J5Lhx`9-a*y`B%bE7rVS*bK?xQD29*~$(WcG%=#4VUu z6#|BSk=f(3e-Icx@kO_#C0<6g;u#zG?rT?y_8=fr@|am)5{W!(e%#)yD}H2mTz9lbtV|A1P#!gCozK50pD683Y5)(wKxB1W#*%nV1-My zjXHJyFL_ac9uAQp&^kr721D@PtO!w@p{oe1 zh+BbGYMN#H?muFu2b86qv7uXVD_82n%_sxenGoFSGvP|B~SBNnyz8S=3qjDKlGbXCOl@{2Gd0 zuI~+r&}$88Ot%)7jZTxnMn>3CiZvK-)K4@K3KY5+OV>a8Us8-5TpXI&H&ECjR_M9n zlf*gpQ>XU(@M4hu_x}p7T3z{N@M3&Z z;CheAqPhYC8R)G-jB6KS?rgQzNu*zwG0vsZ0WbF2ag~-rg3eR_RT?%d%k}GjODhM%ahsmaOYPMKr;)l*w))8@eDfb**#02yF=@~gIIq2*aq z&PX)g2OD{aC*t-uz+GtwcIrImPT-~QfCczveE6Q&qBtzyEH$jyF4a)LSvC8XQ|Fd{ zOJwr3Y}}-hZlQv9Z0`-73VD&p5~?$+AoGyWr<50=jB)iAH>R&|CFNk(c!*7(&jC;9L=^Kf?m#l}bT_}uwt!ris=midpoeL|m_ zyGh4C@TSHb{{r3nr7Fa@Wm)w-d-+=XAt#;JnZID^q(|z|CxQ7?>t~CQ)S+O?pI-Ar z`zGi-z`NuiI!l)B`gv;Z{Z`S~^VrX3q2h)dP|AI^kEW-#`>%vMu2mnihtr9> zDoOb(d*-uNPnF}%`oDW?tCqib{*7kuUbomW=ks3-TXX7jzo>K<+kRM|bN&VpKJ<3a z-Ac%mB-%gM%<)V?AVWap97jI1o{9fdrT8@g0koIB{4dZpgPjwnDi^DwwZ+FGY5>>T z_tKJa@b;)PzcMDh8Dk(0I6Uq=oL#<8`C;=SEB|!Z%rE`dYTOm447&^V;0~(Ysxt5S z$EccslUXhVcuRsT{q$BWYzNWmyAwP`%{aW$tTm49mz?6;oX>`fx)9UY^4v8G82{4D z{-gHX0K_!iHVV8;0@kiL%&yfmvwg0nr{>(J zit0|8Z`L2ue*>a@vZ-EVCy02n9fnLERE|1~)v(f_Ijws@Gm1c778Q$NvYg(p#^egj!4LCbyx8J$8&_YGpR7Qu7vxh39tn(N<+(F3-`1 zX__5rYp)(u!A@Pv|67!;`H2wW!*E!4!Nl#Adw*JI1>#uJ7eqRGus(9E`P*u7Z5A;uFj0_Q8I{506OnF85dxWcJ8D2aol-QhftggYO&YOKzW124drWv%3 zBA3|@iILo4LgV=#ng4C%|1i9H&C}uWuI*m~898U$OtthxRtZFbio+VwkwJ3*a^ zMc(rZ6k^erE<(u!Huu#ouT2^qo*1}+4EoJTwQ1OhgB**-_h0@erre(`1@4mnM8Bu* ze`FJlY5RnFPDlPX?Ly=DH*G4EyXc!hD0yfuPOVd~*TFn+m!`!~s&r%I7gi{if-&Zb zXG;h_`Yim?oSIYLD*QuBB?8qDqA?uUoH>Z87%Tf^K}Q7+1I`Ug#LWAoPZBX1zZ(Bi zoO$@ms}P3bYF>&thNO#U$?(C=q>$oYfq$rrjl{&!UbD>>(rtX!c7yQSHCMZrk1fsK zJ@wh*A1?h9*}V@Cs{1>rrD^F#cINL8hP`_TH}*id5yBQe8I!L}x{!qDfe>;IGxp~3 z^^eYmnE9Lr_o=^t|0Y@rBP2W4R!@Hj>QFt(#!T#MK^S}SPeJ*!N|TT=Z{I-*+a`p* zZLO;a%u>&|(jiz#_a3$iASF#Y`40fdB|3R#Z2`43TqB-uajJg8P;=bw2j!56O$zNSo(_I4y^nN>haO7qf6 z!YSN884DWzQzQPV4F6jrTDPzf7iIb^~m9`o?U-j zr?(}4aJ>Atj6j^!_NVwks@fkZf1*YE3-E86#>fmOdi+`FUpJWNc>a{7KZU0Gum88? zhafeLREEeLju00sUXcwb$%#A)i@^!|Su{RL1S<~pKtNHJIW6zQQm0F!KAX!@gn{MZ z>g~3IlmyjMJ8KROb;QL3n)PyHkXZ35h9L+MFbr>8zA@$+o3tjUArlUYHAr7*`nAZ~ zvIgI`+nSLq8ALt&sGK5V?JWBG8xWDd?fYYbkV?ODV~lQ0i%{FZpA+N_Jdj71udjNF z%_9vN6HuDuXyO>AwI^=T)=(-(AWlv>3hOt_rdf{=Kir+A(G|g+g+(Wf;glNsrWQ?C zUfXykh^*}G=#I|Df918-!uwGeW{_R!JSg>&S~W(~IuLVH$@8a^w~5sNZjgk2@Cz4O zIM!2lY(E+~k!M*w6i_u_C!4O%K-+q=FN4cc7G8WCEbFY+dE0beFuX|9!jsH2{GQcY zjg^BP^J~Y^IO&Y`*2l_-wXzqtm?!8EgXzsp$pBC}DLrXNm-$yvn!58 z^7D45({~Q+XA|mL*$LXZ;s49A@a@XlAfA+r38IK@QnH?=bFMh9F)+#XqM=|0W}1Pj=!NsXg`F%w?wG& zPmB$r4-@J`B+x@X6x%}>dSU1lbA`p-!D}guN#J(@h0~!?<}DhZmC!(N;&mN$zmT6L zQ!@=8eN#zitAv=dRc^{i!#i~d+2CSa)!_=hY`+l6niiiD)=S3XhC>Bxybq>HK`6d{sk-NQ;XuDqycH#t6iYjN|+bu!fs#~hK2HHJ&JJo4KHv)=$_mP5B%pG4eQnUw!-E6PkNxu%!2*x#sFs; z#C!L~FCuC!#kd%ZqjnZY++e?5U^hEhM%Q{@mGECeb3mLVi^)@oo zvQ(#k3D1aoGcRKxS4 zpwLLgZf)EG2C21^2)_yq2z^|@Xu!Ps%Agc)Vy0#gKKCds|zPg<5 z>WAKFV+cTU;Rz+J=#75xMT!ZKo-bEKpfcaU0If)v;v2c&wd6ln>3q6F4_}?GrGqWe z_os1Lk}z0`I0GL~ph`SwJUE#mXZZgHe4L>}t8`=R^NU4>n~_ZWBIm^4HP19xyCEW9UCvwGV;kq32s7yf;J z>{p0Eangy3~@{t;W)L}t}KHO8|t$!FfL_E~q2BtJD;%{#Cvee#A=FPp+= z*osh#)OM~GSBI+Uiaav8B5YYPAo?VX*!!In0>NftcpArICy)1P(;L!X&ts#81Xgqi z4j{fvfc?^hW|AEFig

    ?k;s)-wX1{riv!J&DC!Jvbhcs6(=!y%*Ew)slOM@i>=C5 zOWMea0#*`QV39o>4hAa>K6>j}$ir{I+DWs8+p6K-m79Q)E<<(e8Y2d`Bp8I?rh23o%_dS2GIK)7~ z@~(DAriq?-5Ie;|%tn?3IaK@ECq!!*#*PvOYr+u*%YA3n^P(r%ToAZx`K6MVQGB&ed4{z^Kj(AbYy!7S=jKxD1j)1M4oUcfLNhT|<@Rt{Gja;Y zpN@BUSI_y%FU{s-4UX#EptZxSR4F3Vz#l0%VeBDRVh&IBI&OqXrv&B)aGR#ebcifo zPyO=>urWCFvYzS&uczFFSbffmmWP|S=(8aFq2bUqS=YP+{M&~c?ulfyx`?J%__Jbt z+OD+PqMvTK)|B#efZeZIg`sp51{S}(;)AH*NUK~Vva~y+_Kh7?n53Ls(aK75IpN3y zEl9kHq&MuDoM*DrLbk}hbA62c4UkQ{K@5rA3Oc|US4(wAe$wh&i9kpBXvOnpeK0(j zWh-c$>3~6uKg94Yr1Kea#|M|C*f&Oo{A+Z;Rv@DR{zqy~<*JNPkn4I*Nr0vqL{4K7Pt8iNuKa_gT(AQxP4W6#HMQOVi~Lox@$L;jo{ zY1Op){>(5z>N~($XSQqBOLZX7PsU}Z3=JXcc$=y{-b_W@?}AI#fJ06geB_Cu%^<~t zJpG+>5wg?<%Ez?$KTD!~)V6{^I5J5zVB{={=*o~$=&NSJNYqD*l}f@}{uy-HSWfpF zaIQ*#loXNyb=TwIh{G64FC2YQAm5PmLo@lt{p^0NMFvO#tq@zz+@=vAFxU3;T;rtY zs+{(C7`Hs#;ER&BC56w0&8p2b%)N5;c|bCW8kM$Jxr4gXPP1>=B5>+G@JTRU)mX{< zXugSieEfFkHBy=@D-M-?QP=E7K(0G<;&e+L(a#s1Qy~{QmNS5PBBw=P97PWHH(<}d z^!riQFXj#BtjcmW^^Va7vp9ntN|01Mnw~*e6~ITg%xC=Gb-hZr!)!zQ#Y`O+fmUn; zk7WQFA&v6sbZg?vQ0s31-pZ1zg@f^sV9m$a$(2(A#2yp7ajjKkJTnwL%%7ECPG6Q!tN>WLtpFj`4_pwvk`rB3^rclK3cX6QWF?9#) zF3kFbZ$N}Ti-CVe^yE_J@(P6_VKuLb1!}lN-EKD#XGsp(Gxq-Ad_e~+vLfif2~$t2 zT>&IndG3o!WSX{IPRXvjdq~C|VzMddWLM;NSA2t7%;AFEVDc`?DL`$lgqrv;iF`fY^tMWFc^g-<{7`$h_M$lO9#^jqxxoCmJXKhe5v)-xT)Hu?m-PW%6$j8mT$)yv)C&1 z-Gf&99^bZ&m#1p=o$EQmn6j1YJHBo4JAijgQ)X!kTKjqcH)*U0nPCf)N;Iji3Q4R= z6g{g0Wi5z&H^Q$g@uR6aJ1$o|eur%Pe6u@|+=0|L_L zYu;Iq)42%>sd!h^WlvDkb+(@|8Wq0trgpO;B?C%$m`a}AE}WPZmli+Womg-b+kd}H zE6VtAjj)38^YCR!!p#uuK~_x??s_`@+~sVi*<#J7B+K$schsYw-A_#cI=U$}2BETt zxpb*KtgjkXJULU!;CWNG(!o6mcC{U=AeRjEt6n%8wJfkFaQ~J*uo5#=Qb{tJG%D$f z1EG1ZO@FT4G=AttiOgnLdAhyPBz54ugqy9^m5AN`()^7cKeJ62B z1|ZBg(4F8ljqrdZK*dEU-aGe{x$6KduSBXZErp++t-3aEImn^UJvrpmc&i+KKh>m) ziHM^Z-o$43i!|J=n{um>NZ+oK?j|1f8w(1FDmo#*R1Zf%Dd!q7cIB}|kz8yUEgv9( zuQjT}lcc{!SEc4saSLBbf<~h)tJ6khCp`Nu{q{tAOufPn7{p*N4YZr3@5#DFGNzR$ z*yqT0{GNiP;3SnLO~uFaamRg6pgN6gNmkuN{w;1b0xx!9Xx5O4zo&NGLjZT`FQyPX zK`a0<81Qp;G!{oGfdg>`-quMm9Y3~-3OH-pQJt}^nzeqf_zKxjqP;d{WLD335y_q~ zu2H9JRCMN+bwt@Jcg;B{I5+NI%%GJnGFCvhnFfQg%OnAol2-*+Ldp_DO2WL?k1WlV z!1rOr&u;$DL*0d6%uk08GTKcOjzZ2|jMYRP7DX;jiWHLKq@~4ox|W5^7o4az#=f`= zUTd}dlqAFuHEg{4!b`~=_*jUjRB9KDbUD=Dn)7kqBgb+O1BSTLJD9-mmzfrxBk^jey=WWq4dr*?^Epem_p_FseWS z#>CoKewHZP6LWfL;N_>hPh6crsE-#LVZCH|ZpO5u!t&)I&xZP$LA})0Sq4*#B(;Pt z8g`h2?3!}-u0;2{t@_(8pM?Y#5f!W_Oj*P)A0_6^m8Y1URt_p;$~`x{HZ6uEm>pT& z-O&6LTVjz9g`d=Q8(-Bkdu%3(wCP)O69cTmdoo_jPT8|2%Qq?x9yF1V6s?Wc=%DLK zp+#QvS{Hr*86~-6L^6}PH=U7Tk4U5?G`Jz%b+i5s|N%!sOmQ=!HhjUtv zPJ*i`UNL%s)S$8Oy^uPre!@_REuRjymX5)S%1Gix9`SG%(HM;ae1DJ{H-j0d^2Y89 z{_9WtR!Bm^*%eZqJs?o56dY|rnFXF*qEaMH%gb{KDrZ>6tfI5Gf9#AGlLTI-&{3P6 z%fPDS@vjlVF0lepu9PM8=dZ27IJ~U60Dqw*I=>c2N=lFdRw`+ksIzZgb-UyP51zJx zeRd1q2aF7rLSNd=X%)`GXjMmGUVuSvrDSE4+37~TvM^8Ezo5H z%CuPpGD~yMuSFf3X@efC(2ltDpD4kE;qq~7$nIY;&BYkj?bsIe=3{&l~QuASvti0Mn3Bqh9bLR@dKvE=xnYb z8XOoKL#~?Q+8h3=yYiG^L!q95Cc`4q;FAvnP==JVEWF3UXF**@F1N^hoc66GX)&vw zl2&5&r3WH9xfAoTM8(XSYYs!#OU(9)OxLOB`vIpPMH`=*9%rSBKAKGvajf;eDl$Gq zP^4?b#}@|1#!3Un-^1mj(wjeL(f!zeNwd1E88r1-cGi(sWh?@ZGolFv^M!Mywq4Cx zUaXu(7M+$g##e$_fWqHXu4LLkOg49e4b0MAR>+Aj%YUtz zu)oi6X-Kx(ZY^8`R|Xa4f#vX$!YmVgrTkRyqhf9rmA&Egal2D8K>;4am?^y12=z3J}w*Z8Wgvi(oXD#6tQrphlZ(h+*^ z#P=HsiWH2zRMCSeOlY0idaON0a20H+7EV1C1IZ-ZkRxSA| zlIq*Mt?zY!Q0kDBnvgWhOle{vK|eEwEm z8ljO*rMkJuN^;>J?Df=s=r5P?QP(P&)u z+9(}44~bB>!n4HkAaIP?Y46G4*P3&AaucaKSoUisW`L-L597k-^GZyME2_(o8Zgn< zMoZa#2g8oa_l=ZmWYknh$K2(aR{XP#qfp|>i?Mq|9vbz#(a9s3Vgqw*26<5*)5~@gGt(Bdw4R~l;%OeYkvhJC4Kq4 zD(%eQ05xNe=B3Z6YLd&G3Zcu1C$bdh*%>-d_L{sXLNU%EYmp0`v%mIj`fgs=i5NRt z$BTgD+A`s(voSr-hZP=TP!><333T!REW19$Y!;Vpox{%)h78?-N2a&U?LG94>`siJ zmS9l;-6aiBoE)yi^T&nbc^`|``_CA)p22t9a)a1lr6k08t!>8xCD^ija`A%-(6!d!!K<%2b? zWV5Wds@0lCe0{zt^gcPX+t{^R#U?EBu;1!ZeLAW)0gJRr!Q>-dxn;%zJAK{a2w_se z+`){)2@09TMo{mihDcxOX)flA$cx{+eLC@Q(&9tq><#OGzMR% z;WuU8&y`7wpT`|!YVkD1V%QBsl|-mklvS{%FWdE8h|S0nx0JUmuB>fW@YmJ_%aw{M z2PYyn11cx~A{Alt1M4dRTn9y}pJoa!xvUxVR4@WQiIcmHORy_22bIy1XB?&&fn2Co@ZV`D z;&cL!r8{(7I5DveQTAgXfAqRCS#7jp+W@fvI8x{AKxthDZG|EQatqU|yM^*!dMomh z<&A8)d9ZeHHEAx-^QjjOhyi2BoYI=6L7|N(C;N)P?A~#mwf!vG;eqRgIlP#8*-}j- zWaxyQrw@beCzljE&}F-p@G-?f?W?upg&<5mKSEua>YiN}WbKPAT5;mg(m@$InRX?a z?2xx9KHn=45(Zp@+@d<3+K+(vhIgJBeRF}4c+R_^#}Ard(gf;()8b+4DgQ9UVI(fT zAtV!-t0 zfovDi6$Se@GS?BEA5;{GX~~3e(0>DPldH+-;KB(Ci?z>j&W%Mwkw_(H>)nnE8XE)O z0SyVDi}_iGIyG?9L=%3p_+X0C?kN1of1S~Qrzm7X$qybbfUHX2I1@lEoQFaRPPMVZ%dt(Sa+&0 zK_0l!P_wW%_ICVQ7rUVg9P5TYRbgt)QG3sw!cG^=7s(WkJHJ*%}`!yLHQDbKpzERCbS*F!J)}2tJ5)WwPZIfD@#@ zhCiUVyip{%e)#O?oN(_4KX}qN{K?{c@GciKh_$x2inM*{i~`k}HcwkrcvZ^G4RM+B zBoQo*xg`a}7b)Z&<0)eNFyC01RXCI>W?fL{b}+ByZk&lT0{v+-Oi@v3*8PK(^{3p- z?{m!{aj9G$2NL)^wj-Kj<+@L(L}1UA z#7@}=7z%~C?UJBn@pz}xuNTU|$513wqQWu$u0-{8YY;Mn;zQa_B|)DHBAb+|S{Zlx zEPX|(l8gYNDmGtr^7$f^Qr{8wVLlNrXcy>jVh$EAv`bl3bwbBa6ZCpp|0!`QS~ z5OU!WUUGOP>Auxodem5+{!T$pcC0<>htpeE^828yessJN5n(+dYR_}EMvP2uKkj1K za9LvF5U~}FL_K;W^bJzU?0z;E+}5fJ?UDAtJ$cEmX}(!@S6LnD@kzyqa0T|~tIu87 zF9ynnh0{A1(d$&kYX~pZZ#NYsn!weKNnk7^YfEtO;n^A<+^h z$u^Z-aNS+R-75JVxp&%IIINWfL<+OAyoaI+CXu*swJ+4$mR;EFRp;9gwH~;Ze%qlo zwso)4PFs8Jch1PVeMwlfYY3P-z&gA#edKGEeVzLa&^*O=zA3Kmn;9a6Ja}Z1rpB9d z%vP%+_&%pORgWgxmB#zW1Vg@amEfG=jozL_jn-x?3GWd3p8SKDRF_Rf&Z+9t$WOP~ z-S8+EMcv*y8W{|G!jB?d>oW4D@*#Hd-iR*UyPX(`s={zQ)t5U+m+`ml^F2p3KaH7X zadCG!OPU6O9>(i_JUR_6=yGiiu_cAoAnv0II;D5ap7XgNs}kHx=3Q*EOB9f6{PAUO>&LEl@`B4CTBYX>i{qIiQ!G6_C{CBAv9&E%TE_f;~;Mg>BGe zJrdMs&W3$7r)pb3C#4mi(8sgUhIJ54;>sf|9?%dG0*0z#%`r4@j4g-hux)Fkm1Kj8 z>;1l=!J4UzB)N9AWLkGt8p`H$RXoX9(KCy*lc^{I+Z6RVn*S^e9PcV}y&<`8^)^Xm zH=)NhH`yi0msSoLb%1&}pi)HJ^}?0DQLJyjvbNKh9{f1TJmzx#eY$Q_1e7(?;41Dq zS)HwMTt@-WkV=n!DxM-nVJP2Ctdr8bC#=ENzj7(O*PyzV#1oMZgZ~vPfU;_}*23K} zJTOe!OL_~(W0^<$RXCP!vCYF$*>+=%XcjzU=Il78?uz11#x z5lf$w(~msI_t0zY!{~gGLn#!r(Rl*vg*U*DbYgClk}7?MKN0{|QiupdhxHGlZ+DRk zUEk-_pc2H0b{mVIqay@jV+F{}K)Rrl}I%Rx=YY5sH54HMZDJpDLUiCI0@d zM@l6$W(PU_$HBriGjR$_KFhqHG;K@M>F4(Y>M!S|v|r8Z)xEFz(5nQQd#{ud%F_7q z-v%<+!Nlx7+w4p^UA$h?lE4z{_U=qtZpb{2^pjwbkg!b9N?VRJevssnN!e_?$P3vD z&kx%;s16qk*dVN5%!QhV>O{1P?I@AG3>GY5uX5y2rLaX(Y${&JEy5~6-vFrk6y1r- z(}9_ME1~3a^VERQV(^S;Y2+W;OzwM0h+kb?TG^3Ufe&NH1wg-a?V=z(XjaKIIrN4N z)ZEOVB6=i6TA6TI5T9u4leMD&YlwzU=#S24rWy=Gx6qy47rL)S(f9Um23>4-*qJd6 zmSp@fX_Ra06NV#CXjh;8`<$Wm0Mst$-fTaXlpH#+r5u2yh^d7tE)_9wpAaQUG~azC zpe5!_9`mD-Ll1Z6|KQkaZAkG3Q)Xkq>*Am!lzd&k#7QNtNSaSG5;&Br8Xa)$f)&_{ zwC>L5b!C)|e!zXTx<6>8Kr8vwMG^qr2`XC^U|JDnwbr4@F-9=3js#kV!3?X#tREe| zt5-0=&Rwg|PIIZEG>6C8bPf)Thzh8-F%b0LmZLm-J!L?afKz;Roa zVrKjjmW26{6#r??rgwaN?G0%xg?-d>Js?4b*p0`I=uKfUbaGEMC=F>9Zc>IIRcBO_ z4nW45C1w0}8)=H>uBPs;R6Vkq16>%~f&8;3u8$6aEHF zWL1k!=Y8@`XcAH*CWWhx=lXm^K+6yJ8zA@&(}+?@Qm%4*>cgbOJ?;J7C+>rC&)u!p z&)KTnnq1h@?c*Qo735!cq^zJe#JpN%c1nv(O$|%DHdx}!kKb)!vC1~1QUkC{ z(zoAEwMTW@otk=VC&=HCZ17VDo=U5R7gJH^a7pnjFKiv-evlO>RT2F$s(`X&Pe=P! zSYGuOs|c7DFD+%A%gkUrm<#1!P=mMfohuQlZFzB~1aU6yGQ8@_#CFUu#~YUZ$@ueb zBb3@YQb?MU$J<8~O4Y_s0+-WNC>f9q^#``ELCBmNH6dN)OS4(&&dv)i`Q-%}!44-Z z^wPzjJe`a>xNsUtQDRd=)3RWpbVQo&=6z#@_lPGYjywwY8P<@-Dc^Q*$PbVs^b;NB zxfhpPWF^PER$W)hk|{iWpXX9VgL zOilVw?4iCW{%#4m^W8U9t8_sIS$EOeG4htEORx?~DC+y}<>sLgQd6own@lNQJ6NOw z3%xZ;p|j<=3u5_9E86+g!kU}R>@o_Yr&@Gz$M~d z@||@I#Pg6Pa64VjAv@8Fy@h4!^8RWI>8_vb{+W%cBnIiX9XLM9J9MjcVR> zUYJck&Jx_yk>kVP04{5xc3Vq(VHTroXoTRTU33EX&v$Tr&Axn}Q3BNQ-B*}w?!Iuy z{s!c1xTdUOuD$SI^9>v~JVG|)5+t)?rU~1GkcEHZSFh-(z0nc@9G3`qvuH;V5$Jl7 zSQ@T)HekjaXK$^5aH?VDCh6pey{9UYkIq)c-}>%p*yUrg0>Y9T0fqiR2h8(Z6zU!p zDKk!iVlE-zM6or$yZS267D~m4+1Juz43V60a`7FacK#1mLD1NeGh%4&hRDRQqKI+f z=CzjJCXf{3V2d+F9y>p)xgXZew#+{!2WmH4ZL9PQHfK6dKaLQDJ~j~`N%|%h8qCnc zdbpz+@#8YDffd6j5my2gUF)X0Hk;GH)|JO_v$&#MXqM7CA8y&tUQ?|~V>`LHMMUoE zFV++H)PHFZnK~F*oAbPk%Drt4EdKT>`uM{#_d{Z{l3KA>t0#52@bH;$vP{2)I9~ji73!_|4Y)Mja>6Yu&~TX%NZ4*G+aoI1b#Fxg z7Jt#3;GQxpGS|K<-N8=dwe|kA&mM=z827WKGGUpiedOCgoV<<9#c@gH{2VLhPm>!R zzD)Meb2#ViYXpTzD%HlX)M+7~T=>fP%?ziX)3mw`U%pqqK`p};OvmjQu%l{KRJE5Y zV=bMIfir&FKTEah%jZOlI+Izmd6rZ?ExJ8a&&4X8q;>KuGF5H*G`|FFRj`+1YV_35 z^`kYJL2Dda4|e;h0+W5MfZpA%s9ZuKuy+6vaIvJ_`@r@=IVqutczWQJ1BPb#Pe z=jt7*TMlm6{9xLA3MRAOMswqq$Qx28BVPn~ZCbwKhNcb4wx6CznWXw~bRt-Mg&$iR zVM}{~afNgJru5JaRwMbH*Iow|cgy>ny2X7A~DTVOoXgq+BNz=@HI&@>*KH6-v2E_%3MkfnSWg(9Tlx=Y?;MfSbwsvvIBB*st`oexP{xy(145`$8)= zSp+>nvcOr*5ik~9!zv*_Qr_1c9+T6}{tYGT@La#OmR5Rbrq*~NeP=!yzrMdQ2z!&h z5Q_g$g_aHFHIu^94zV>|lF*==HP?~_6p9#n%dd^EWKV;?<2UqqB^UmO=}LP>CbT%k z#|T|i$8W>r6e5B-{A##`$DL<-rb(Il{bht4C@A0Kw0Lj0*~q9k3++hftroTi#lqsv z%P3^QQ80wMYT$|@=jZN6M;fcwlj*)rZ_4f))j$~91xh%}>i9<~lhlY?tZUF!38y!M ze+6ct!{@;}vFJ!f-kHxxLN9WuhZg;VBek_m#OIu8$Zmy-4L(z!wcammONrT4sNpw7 z`E56Y<{L!q@ir&@5tlz$Eo$sxS|L_XaatjjZVflgYDDgYt?8`(KsH5({D)8Rc{GxH zK=g4yEw8!uCVG!81IXRSzUq^}3_cz&$}&YnH5QzIYRjDC-5joGX?|L0hb{J8!c^X4 z6-^O^HLMV)-Q%A2G{`4;E&4tXn$sbH0P#D$an8h>InUe9 zF(OimgssEXfr`yp3{5(E%y;bFVked$tisz^m^J@x8Q}b!BAVOPT`RVRQdUW#XTaC) zhXyp%98OofPn6!rNWe@}5k#O5j8n%4TkSs9@R5~z+4ksx*f9CZvpWaz8YDlZaQ(7_ z*K?nP)L1_)))8@7l|=l979#ip&t5@ZN6v%Y646F~(c>!1W{$@O1m)a%QJGyh%m%gH zb4qTo2q*fn1FHeNdERzHiD^;8`^3n2QN@U1Pm;iZnK-q13x4;F1z@o&2JOImN7y-& zIea{F^X{xD$1-{Pw{XEBD3P=9eF85ApNzL_3*)&0I<$8wto2v=%SagY#?!CY&qgQO z*pbR&3MG4TT`Qa&jIp(IELxQY;I(LpQt9Hn^u z=fyz(W_Rwi`L(s!ouM73xDjP3@TdJt7YQF zTaXH83l7zE#HX#1)qYLb3=I(M;}0IdFr%KLZZW)*KjAcs5WDS>g-uocfXI?5&=cS$ z>U;T#v&u-^B}_{riWuR6|U(#XKy{zS-2^Cc$~Lp6SI|`I?UN z<0PO{w=8dLa{d$;53;<3AfZ8GGi^$Qo#ubiIgzOwU&8R>7l`3W`>}HFz~%Wmj!ZO& zsIO4uFylYsfhCNEJrht(2n7DDmxV7=J9=sO-gJOX2a>W*Nj^0eJAQ|965`;L7B_Nk zso{YYP$&0wq&ILNsv`tvJ2x-Lu2PWTLn|2|5@AMT3ddZOF>z{u$AE{(s@R|l7tic6?H@ZBUp>oP73G5@Y;gvqm4L!){Q)qr zFr*IW_O3`N1Y}>wEc=@9I8oCaHpXz~BSm+V#qrZ-Hk7}@A}PJRR;7zJpYdfuOX4Ub zNmj4gF5mNw6yg!kZ(pm~AX4kRW2FMjp(L&@qLiZpPn^~SgDu(+Nrhu6`n9V#Nu|ZU zZQn(Lv*uwO{d(0@L*$%PuAA{~@F}62FlzYm!X!Z(9Q$suajeai@K`iajWIYP3f|26 zM^2cdUAJx93DVUFciel>Ks?|y75;1+ED^Tkehd!Yho#5*NOCu*Rz_R#`_R~ND^aqC&t^MUNv$qRgZyPF*s#mj_$-o}R)tL&}f z`63m}X_fZv(1nwd9)L(=B(B@uS~oeRY1G(IHo216#^-kC$;lahM95&EoN$TX_I-dCM64(mnNh{S9M7IJoZa3 zQHen>Qs&b?Q+bX@IiM?_o8oTXFv-qlO=E7hXTb6$czer5$#dpA-beJ(?1>V-O;WIJ zW0iMBzHE{t5Z9z!@RHH`R_-uY;ySm6|i)dNGVhadg~Zpfb3WW@;}2og&^xyMJS`DuuQP zK_y_r0joIYgJEM~Kmazt+7Jf~nFt6vBEv=Jeeo@f(nvPEUanP`wdf@|HqHnpoJAX;m9OZ7z)6^|Bp=n?5>0mhE{482q zbvaW3`;O~6w&CuVOnWj?yU}5;55`-eBta$Q#`osiTcnL{P1A2U7H@dd%5(=tqpw$} zzh(kzwYSp6Tkof|y~pB8xidT*=h!P;(ia&9mY4x6jA%Aew_Sk1(K zCJB^yg`V1v=JIMlHXf}RonC<#pN~pHxW%HgIr+l0F{NvHP9!;iYQ6TPA6!-#Vgp+g z7u#Gbq9epm4riCWEZmL@!|3GE?Anrxx&z1^!Y!V;4bZq9E9TA4;wA*AAFxdcIKBY% ztwuqNXj$z;TEG$e)v|o=Ttu5?xWZzU@UvI9xJrPnS#wm&%NRg0biklM+jLte%ir-6C#E1qmfApaT=GTH9p+u_w2y=-BHh9?NykJl0TrV)?K>jq@X|#sL)d z=VSYsthC>M8MjEvqVoi4S>@^{;&j`B_z?u~^p^(fy&uG8E#8?BePd0Y&2=J*zXtl& zri2g?Aj_SxD1A=`Cb3Mt`@b8uWO;X=Wv&uU6TSY@rlhA~zsAgnq6-&VHNqa$I(IHe z0*x4E%|mBf2nMDSYf7>q`O(HLM^L=a%}wk$q`X%-BUNrWAFS zdsCO`2+H6<4amp9TG-}1K0i$bm!uYf?L2QM#4JKyj@Sr3A3%5+4|U{75ug^P{R-S< z8FbUGvmYoHB$}pejqJzd9G#gnfTU>JbqBvvH|p|U1e#zlhd-hQS*czJo%Jd(UaPk- zDMHWnE*LLt>Rqy=XQx*{XZv9MCy_B*c6h?qhkE+9WqxlD5KeUffxdc#lFP|B{Y@lwj z2n8!y`Q_-ARBIHLpB~QixIW{V+dgyfIlY}CU)h~KrxnVZ^#Ac-O{-jG>Q+Wlygc;- zM_O&iUm-hEZ-_C9W@^!A&Bz6rk2||ERSLg+ zlgm?<9vZv+Yk})iK=g`QJK9p1AVT=}{=TQ$7|WN$MFaCd8`pm0aD5BLrL0ylHX25q zhPskw=usZiy(n*1N#NVM6&+?o3toFk)(WCZL-^Fzy^(QRhrP~nZYe0~Eda5Ed1l@7 z{X#_#mwPo8jM?gv7l}{4utwW_&&?;du^sVo@RMdk(nz~3me?S;3vgWrKLDz+hgh7R z+%@3Iqyh~X$<_|6dK0}TFGJs|?HxKnx+?CP))SlDVJxHjpmLn?xcp)oo!?cl2 zlG}CUi^lWYz3RkY)q^75x(K>RcJ`#T`fr(3aDJtdnhELM=XP54z%M6pu0m2bsuK}& z8GsW(zJz2j8bv(M+wYHgUgQnOSOx$%+Hf&hy_T!>4{?lE(&k+{u5cEq*>qhKxwrC}X$5QPsj3D$5R!Cz+eu`ud5} z)pQ|7AHy(DvzrpFh3hu-JrgnbeJtuhjQdAV>u3`mii80%@`HZvqKS?U`2dew|<$f^JTS6 zW^~9PQph(P%a2Y@+dTH)n#JE{Oq|ujdU zCs%Sx=`9+}Tt7H*K|F4g{%eHM?KEv&mv|cU)U8<;;<3NA)bZH!Ra8CmOv*&3HSIKm zSbyLK7O`_C1eG+Xx5;<2$dM}pS7h@)?dcnPPf<6pM{&W-e(qmHHxiId9DxsQ>BhD- z&fDypH9mSoP#>du3Cm902eQlMY7i2y8c6_q87f8)-vXt3CL@1|5b^1NdSM2aV628K zq!+HKcp}I0Q) ze~7lmtg-f7kl~b=RZ?`Z5w%)NB}icNd{bcvE7U*0oLmy8c!{ifLvmccJlNB>oHO%u zb4!<8axl<3f`la5wAM^YXheMe4Fz?ZjPh&oubcl+nE2`ngNtL2!Q7#Biy}{-e{O5PXG#Gcno=d>y}^gv5#aC2 z*Obji;<*$!TWM>j(qy<$AT!@@k>lV;@r1y#g#q8xadbXl%YjXtT|MHuUq8&D?DK<1 z3V6YJrZP$!JoJ$Kb>?MQIU$l)h%mk=hh|(?t~#wSnTWHfv%W@3r>4*|TY5={oC*O1 z4Zx}J3c~&>`3bkcthe0-M7&XYRtg$B$U8EsDwo>@hm`3Xm8Eby6&0F}TxJ0O0dUhn zR?Ia5%4-~8`01q5R?X%_ldY|}2u6aJ8kZ7WyyhfCP@Z>cQebaZ(c^#u zJvRMm^I(@J{Di>e?nJXJ5#?2F z8lrtc)q0@WK*2wcA13bU z#6VAS)dEhzq_oqFt3rktj~7Z_#T@>yn)4pOiQF9^!;wvLFSvIB#Tj;Gc3D1 zI2l9_6&(vWeYrn`Un%8gzmXurHA=Lhfi6*g*Ic7ceJP@|`AzKW{9HhNX7slSK-EQCs*;f7>sq*xwZcfy=xvs(czXh2n*~U5gmNaW66& zwCkUs$Ax2wR_A?wr!fML<%+FA7Is zG@A*hawT!9lf})+`l3V&R>~j@60hlwH3NT=GqgHiU37kU@e>n9T_!669qIaQ*`gLl zL6tktcU(w|L8?`^f7!mCU0*u)`j+1?QXH3J#i|IEeVFIZ8l?=oEsO-_RkY8~dYCMEE=dbcrQYrw6|f#Fkj5e24a0?IhyRRili z0Db{e3ov@v?{IrDwjP9ZL5XuYGAA0*kJa*efKh#0ttBvhTIm zRnei70EpzaSPt*3;<*8EGC!%xxgSc7GF?F8KkBmZB5t#(0iTr(e>9a9- zxy>ZPDmc4=1?0;*;6k3Mt@(GZ+S<&9A4JGr${knTSG1W9Wi(9EOy{|7kb$RdONwu^ zxezg=9-6k=2&GE)LkLT*3Kq5h?1kI;U`N%jFJ(b$Vepb zYdwarCXM{wC|WQBJhXQ^A@#l7{wmy$kW*DuNjr^;Ess59iINDH285G5`O16oeeY;% zXr#2~Ilsti1B-u7tPyW==|Bvj9V@?{XF9XN*1AG^TEU?kA74V`HoF1L`0ZdwuFQIp z0unAPvc%?=(F?7h+R{WjnF#YqiOIzyXlV4u*7ZEESzh`Slhc!sK`6{wE0u~MPMSyz0)7aY%W-)vC_ZLE3E(A?3w{( z8+Ry&QZFV&PuV64lErS3iF+bwm%`M~7LpdrB`*z}7uU8w!?+yml99E!t*!ipETICNZdX`dZg9cu&OS!pw zomh>g7&IR(O&6J8W1+UrWl9){Yq4g&_7dlO zyQjA!7Jd}oFc7mm41RztKYbw?3cv^H43Q#(+?lMb-b4JvYa~Mkocp|YY;<5Xj?-sg zjG4Ad_QtV8yD=yhO1phkvdon;jg%K$ZIc*{xS{>hxiwHB^xA@{lf=^ol1I*fuBm4* zNf=^kDBS?0bu>dl5X4AoP#T9kJ3HsIe0&Va-qsQZ*d1f z9;>Q`>4FeNXH)ltW^2J3F0qg1*PS&IaVW$hAfobaiGD{LM%h+-JWl_4%6LFpGJ zJQidhc9aef>c#rK=h+4=GQL~p!BshQAwg8OthuzL21d`WII}>70Oa)qc?l$`JW)UF zmEiq2O(AI!)cksp%!-=OtFa!YkS)2u|N zbQXk_5=4N$YyebA#|dZEXeW+ckm(xyl*}`lK&wDZI0?lF0{I@+k~TbPni??yVqvVJ zdhguNOlsIZ{@Ves1zl38D5^+ts}$#%9~Z=*s?&G8b4D*(32cfKP)CR{!>1{Rf1U+9 z3M@xDAO)*z@FOS;DnCW@ZHO6~ZOINj<;wb8PU=M6>>?EU?FeM~Y2+7$0Zer~y!2D2 za@!MoSqYkP_0=YH5hxi1C;(H;UFsKW%l)*Lk5Pkh4ETVWS$q=(*7 z&Othmgjtb&PM>#`IJoH51Gg@ZndW#sJQ*O)7+({Mmcx7Cag1*l(VVuCnj9sTC(DFB&J} z%=Ye?EK+E3+Dcx3- z>`i7pjN}XJNE}%LbIL}qEdB3}KE%s~@;2@K3FRXeSJWBwN6^F5RGQM9G5YiB**gcl!2*%_9O_xNMT=SEg+AI~(3%;x!{$ zG*TP?<8#eauL`doQNCp~F}bftd1?d(XSHt_KKnwjgMj6;w4sXg$YLa}iMe@|lh$_H z#Pr(|*sPyaU~jcnXV57n$C0&FGCFh(dm(IC0{N+C&SeJ6OfKQGs?szffMM6#-tv6K zKlXn`-lxq}P9k2Ty!MksPgCIc-4813Dmof%lV?D^M47j=&-)P;+#72}$a7j(SfXGL zPVJ9v%=5+S#U@qIPXUD(U?wUGM6Te6$L{$xjJ*dq6NxV5(c3TQUYOcWXN_8@#I{k{ zPA$H-Y63{?m{k4#h^cr;l`pfd*}#Bvtc@Q@DVxMLs@XncROB_&Krb_ovkHZSROrBM zLMc^_y@jp_q_>!jdv>8m{e}56Axs!EjH7{b>W3^>Dy=6UIjmQ0+t)BuKaTJ^BVw!w zI4{o8J47LW^hMkauZD*ONkAH>CERW=QRvkAACx5N)H^qS1B#RfOH{&sanjXMKv=37 zflQ_frV}`bF_K6!iRTFHg3^1IhX&VDLci?m4I(1f2h04vaXa@Bg=05pDWflg{MtIG z&+BHd-v;X>&FQX`hlM}38GH(y?`%^MC{r)7CWJ>%JdSkAk^UV!pWy)w6sv%NMV2%B z8b3r(f4XqD{8O>#;8HRZrTC*w%sd7ErD#0VClhJcHi_>bK)_3^-TO55yU`J$I?-8P z!?keMi1xVD2czim)3S6~DV@Jh8aYll$VvOFb}Zf_7xHuF^&9hEV*?_%OtWN2|CX)h zm5;BF)k%6xYtPqkZ>MQF@c<#tN{X;WoXoR%lQ4-_(dR#lA*=S;Zy9h0al?kTW&c`i zD;CWVh0BLJuafvC#2dSnzleTYthy{rGq|gnet=hrsgZfc*=4PlJ8Wy>)I%}D%D%2? zP!1TG(=)ft(7U2C(~_i`S#3xg>V5l8wjFVb=s4_3Ua#ICNm+o}RD+9Y;T5U{D$ zn4^@@&SP(M(Ni zpXr*Fn>p-|mFv>v#M1eclBb55yaI72Y_Hn)~%5HTwwgIDi ztkSv3Y7iDBel6nJ`ZeaY2kJyh=xXS^E`8W6V4wspZJ0>v@{HSp_BjlXEVM0=tZ?bb<}YwXOJ&hL^+}KwJ2t&!lM5=kg;|xOjxiM3W{~6L9`U zuoW8XYUl*NVM13aN-1Q%DLCDxT6_m5Y`zj{!GxA=_u4~7GYQi3zA_X~C1s==B^6JX zZu^`%BM1r^PcPd6K8zycwW~;bNbYq9lK?EA)XR{HgS&q+O~xxnRtJ41@*)Y{^ap86MbazF^S7|0m5$F3AOlrW~+v<6wu7GDgnY~~oH`{fWlH`2||`AJr)EgEn}{Ykp|j7y}} zab}6wGN1?@rUih(3$W5p@_*bdxV$gP{s*`fyiTUHtLpuI_{GHGy!6F}=@6EBD+SkY zplx9OdP=!kB)g*7<+dtLeL*wyenw2>NUAaZle)U#uBd@XEwF<^!YEED_i_y{`>^jN z&POHF%CutkiLf|0X2a;_oVeP*8|_O1jEo-A@-ba+%Q-@h9!=Sw`@i04Kn>2XE3T%M z;WhJwRyA&GQwx?xChJq?y^EBdmtnVxdQ};V`MEfg-TZRw;9V@jaHNpnh6LN*B52lF z7q^Oze2M7|4#e+9-FKCVYf@H#ToSuYTRRjz z&|ImAzI;MHZL@8^tCsHJgCv=!$cws$4zjCBO3~paTk-XNEQzD@<##uB@)1fYw9eY< zMbWtN6F`y_DO!&i*qd-;qYL9S=PK=9P$I~|c@9da=fsM3+0$`enReQ#?;t!xqb369Cdw;oqnM1 z+YZ%%KC73}`2CE8%@D>kwO+SUP4xGvbTV*cE>)`ej8lHxd&biXXR0h0?S665UE6oc zf)MVCsyaxtvoRuh-`adm5pVi}T@PnK*-<5eB&Y3rZsJ=*87J)XmaNS`y$<5;il(Y$ zQry-I5-~Dj0vf_qN~t9bvfLor%)K-hv)L9Pa_uUy+)g*cPu9r^L7uK0t!v4@rU zg6cTkESgILj!1ul#oRpA0DxSJeXrG@VMl2=9bBtM%I+HSk(Q>%jyV4ScPOji>&De? z!gn9*G4GaQ*&ZGn-<9o`=7^|F*h((1t8Om;GJbYA`Vjk!66Jm_xH_@iQ6(v|H0K4? zjMGcWqN<2}Y-ph|r=<8M7}H+&H1MzLKS0G_%Vrot`slA7GBolDE;<)%`&EV|&if$t zoeoPkgIVR}LJ;kqsBQGdtB2?nN!ptDFPCPc{{VlU$jq+)ecllq*ZoWW$_VPBy>FU+ z(u+WsADM5O=7=4zT zY4Abdtl5+8jk^?!C6WDqfG@Z5gc_)Y*|giCdqdIv`>-QS?JxgyS+@On0^K#OfPb>% z($wHM{Hm|>_73se-d8^3FnAl5f1O*IAnh=hb8A7U5x^l#c_jL}+h^j=;C?oL(mI)c zdmlFImuXyDAP8L-MS?C-e|bD%nXkHrq1@j5Pac*B&c!+$Jb%8uVI{XqysB#Ipp_7P z^$(Cfk$L=JLLM`+%xIvFFpoiEdS>F*337pH-eBj^o}HPj@>sHG3WgWt$zT1 znQuL#EZ952@dsT`?(PF;7h~3JC2uoD|LA`H)*3w6e#_DHE8n+r6!Bskk&YuAtBLd# zycuW~(?_m~ejC~(`ySp<9(;Q4_}{i4dXFU?i3YFSTXx(udYJbC+r9tSINMpXzfZeo zAKmKM-|KF>r$;GO`TmmK1N3=Eh@tHIKTr+t2>tYd(z``ez^Mb2keCUoN|u>B zfqPNSU71T!oag&5x_A&dUem7d|E8$rwfGk$ofzQ9>r*%(7tF z3M&{a&av{qH4qd#xc%YTwotcN32kZ#sK^JBsvGM{mT%;xrfsh)ENnYez2VJz-gr^v zZ}!8H^A%=%-LTnHOcpGrw-c;=(B0f3O8r8LAaYNT@+f`0ndeDXS}PFB`u z!4wB%DGSFFJ!nS!-y!ggq666qP#>8LTF=FY1dI-=NK{{A1sE>7(k~#3us@YSKD==- zYWy^Hw>2eLBgOIh!E0$ZSA7n@F2w@`7!D?fwMr0~b)Y5%NjfE_>alAIK%Yz>n-h&W zDzB8a#t+y>bVwjI#ih-5nxc-ns<^l(0B~@z@q`3%C8vI&9LC6WoLC`I92j)9o= zD$}hT3BX>%W;?>VC1dEaX@=j(N=lS=ysS<3l>e^?%e3cO3U1!6K0Vk7F5qcS0#Da( zQJ`WWt>X5R@%K1$*+XEfdOGOrzpRi6GIJz4Fn@JRqdGKe!T__Gp4l%JwY9s*9IRlx zN}G3sAW%N7=*jMNL*7$xKnw1|AcT?C2-=*KHkDXUF=kYdMi6aTIW1r@`ST(6Pom{< z<8d!LCrPl~6Z?hiNR!D8Jx%(@&kG{s;hbgT2izZ;80G6HO|!0}js5{_Kip-ma$KlC z$7tt^iauJnt**29lAj!4S^S{=<&zuMnw5Wm{)X;L%c!55EnU)s#)hr}UcSkZytUJY zN*Pd7ou$RwPtd2QhMOYjNT-Jjr_uBU%-uo|_&X z6qx;q0QIseHs0voPwx#E!-@a@j}XK+*ZEw%;-Gk=`yb%pf!Q1Xha~=ZdOvSw<(Zx# z?S*0vo7)IIK;d?Y;8?>)V=g@bLD&IKY-Is-Sy6Y-gvY} z)ST0HEz%t~rLQk!=n96Xo$|Kp;zd7jseRw??d#(6R zz26mSU6R+%)5|3(Ljx%>kJcHdcY-2yHF(oqiIeq#IV1gOlSZ|!# z)TYfV zx~AiY+mK*b(%?uP?!&n=IeYX^u+8{7$&+VdeR2Nt>FVBecP3 z_)_c0zs8n3ZL+%>@=O)E8()==c+&rfL&9zuUtP70R|)?RszAM9XSY(+Ly&D(drXr* z1S;s?bZWFS5KdBKrF!<@DYvvxdh$?2DOB)aXexpS4^q-(%ZhDMHi-~yp%;Gwd+YcN zK80^kpGmV_2TJKpA}E=|d^_I^vpe4|?A*TUTwJTlz4BmsBy^d1WSEn#SvSlWp=zl$TH?AFC1WD$wQdvA7nv$?bpp0+ zpMPeybXF^ZetH8_sPdgVlJo9>v-O}n<+fP7uxi&al?RLD@O%dJN&9WG3 z){?WWd*l%@k1=AwqXH697evfrj9BofK%v{J68Jt4F^4f?!J`5aQ5QtaVvP9TcrnlF zY{gpMaK1V~_JeYL6vQFf_zi2>H++z`sU{Eeu~@u|Ro*Cf%g_Jy^XI3`?{v=eKM^}^ LLh&SHCv%N2PXkiN literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000012.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000012.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8f7f4045b26e9190fdc8b41ba8c4e2bbeba6b8e6 GIT binary patch literal 29384 zcmd421yEewwl3PZgy8ND4Z+=Ax`VqD+}&M*ySvl4y9NlF;7;%W!5xB2F8{yx*=O&% zx9+KW_3G7IUCXA7ImTSy_~x8rt^V`l&rbk4SQ;b^fPw}9px%CfKd%5ZIU{!yZdPUv zW;Rw*1lXqGMQodzDkX zw@VEvDoX8at0mMq65N4w>{d;FIJEb&zK+)4uWsq$QECR{B`{)2!Im;^iolmW=+c;Q zWqffy5`Tm(Lsp|zXdu;x+V?=0lY}xw~Y2gtg484ugrJ3=*K)X}MS39d)tERr40YaJe}_C7YWAHrqsq13M#QkWp$F>%jcKW7W12y0@&X{T1lLVq|6WZ z0&U|p-+w9F5W4$*>9Mj`q>R>@Ys&w3DXS*8d0Uf7`)PH&O@ABA)XGZyp9P@)zsqi0 zav|x$bAI@>ffV{;Fsmj?Lwl$LC3m<@5j6TcU9iM!=p5vzR3UnmuX}p>D>-%VH53HH z%3`+UBuaDG?OMVHMxPLe83HzE^ zVWy|*P62z5A6tIay}$^05IhJ}vP{H6CA+tm`4Fnt-L*8^{k6gJ2k>NYdqyQ7gq5%M z-+GIsup{Za_V?1Gp8BqIW`h<>6kw&AI5?@0%CywYxk+y!xgkyepgY-2BXp4AG?fXY z4fxg;U94_JUC*cR?{a1wP)atOx{P;InYf)Rfr_7@eSyLlVPTXg87$e-HrWU~>n+oH z85LT{Nos{F{ovrhi(sf(h^HJY-`T&*fjswg(Q$sLVzJ*&k4P;y?oSAIgpeU7SFBAp zMju?`rrAbcRO*uV$9)wwuQ#!Qiu?D=%lC^~YA_*T5=Zlp*>A_oGvvtVWI3HcP)x>!Z7m;w9M8*2;2o18n_12YoyL zA>rs&df58zqnA=NpJ2m4BzFVOHnc`z2r}m%z(sekaEIf1hxbCXyFFczeha+`101pv z94(cOXcbKAe}IcBxhb_F@4zX=;JbErN+nwvVJr&j`>N-Kl$NwE*XVERpH0`mwaY3g zxCJn)7==ZK|Cf0}B z(J6q0)I1$sIS$u$uW9oH_-d){YCQP&xY4tE53GD@0m{pa?R8n1byFhc$GR%fpFu;F zweZCIK#OFryjn%9&dFEHar0`1g(_e}40eT9RBFg^cz;ITD_V?`Q*9o0{9XRM{b$ER zJ7JGMfav{}wubtwRWSF>&}Q_!f^K%)uT`6ZBj@f1?}d5hCQBRsuzr{_&tc*b^i)s^a- z821quBn|9`J)76zDbhw|ik-OZXsX5p;lvl|ilpxwQ~C$fRX|=Rkji$pNZK9Rind9W zCqepgTnJBH`7>XR$gJ7(?#n8|ZuQ#Yp5dh7davTB!y}(}oWK1BL7e{1iQTK_IE&t- zI@!mP-?~gEQ@bdqA#i>s^c`$l>H~iOkjr28W!h$D*W~xbktRn0?=6H?Msi)wmhT<- zt}+Ao0^IMevR`*@uqIdU632K>_Z)QE+-Gq@dBDhhFhhZ(){+qd5q@}E-TR||DYlk}}I_(|*z6Wejm&MljUxpv&yKe$}4>qg1y;cqGe#!g+U`)JN z918&tWsw;A^Cs~l2ExTU`g9hYtaU0&=qsctK@|mMs)v67|99OKyob%tG!DAVw6GII z)!?Kl;GQ_mgM0?M0FQK`+le z`^tV>9KR$rjg-EkxaG+E0izqTe>MBx)gR>l0Hj~^cP3ZvdMRl&1v0*;x-$0TL4kna zq*|qd!b5wOoF)%EH*hzoGcMLS<*Q5*JQ?C6>%#xkHIi=Ean{>)OqynzNlfb(5M$Xu zI4pT?F&j;<3l@ehpwyv$ZA9ogV{2kF1bdtM4*)ZU|MkC~zwMjnJjnIF;e2(M=Y{CT zBVNE}<2h1xUvQh*HTXQbSiZ7^h6ZUGNW*I4g*7zRG{qNf!dU_YfAQ2o>evc#HJwAn z4@QxcNWFLY%T4(A*PH%@35S%;-wq2Kf~R#pSJZm@A9jcIRq8MuXn=;HT(Z+>Xkj8@ zMCo2j0{$yC`d=d7_2aUx!fDAxg=}Zpu+RTbX3O%OxoB7%9W>%u5ts(qt4DfQ!r!d_+D?MusvTeP%$4(?OZvClSMQJh_}jtC*bX+|x<RrV)W${CGtG`gQGxp?s;xCudEF^DL$TRUXiBiig}C6RV8 zO*VZ7l4Y{TE#-BP$r((O*{LITSbcI)V6U(QxK`$ykdOz&jU&LH6qadAgKL1+ncx<} zDNO{`rlK^<*fnifdXgACX@)t&j~37cBi;T>pKz`d1U5pKsnHdCYnC`wphY*2&64^;y>`g0QW7~9ny8*JnmRYHX^u=D*U z!7h00?bun2*m3)WKgl7^`7r`|IkIx{LcJa5@Ku3M@|-@39GjZ_zaAYEZ9`LlbmA&- zVrv!>rFSe~VUep+_o3(|JLW69?bf7}3Xm)VXj*ad(vn+vao_gp$?)ib>A}IS>_b4s z#og`VT9$;I3E8w@-!R(*7{!EyoGs=PiV1$fFk9e1t^$YIqD)8|BHs4@JTI~Z^0hMQ z(Tep8^)~E=tO^zDPpBa)Kiu&!g88Kg4@yFi(}vdimBm!gtHAn0dR#0y+^Z+t>zMX? z$OrVo%tZJvR@6>*^(Od-P%$O|6Z|1~wiu`w@4gXpwg?mx;?F(%& zn7eOqBmr%Dj8lhx!UjMYzZ%hPER)6NZJYb@wGKb78po*SQ=rH2l;1=Q7!aIGGSOFy zZq*o%Ej^@me1<<9{rOe%H=8gOQCS38!)BvR$dkVKqwsz&A`tcx$(NHb0~$MO3fKH@FDheud_2Mp)ktYcl=x-=84VN zvGAp6eN$B9h&ugW@0RY`Gbg|*)`OoJ)V>TB^?v@+q{`N^F$gqCJc75RhA|r{f&ennPV38Z3sM84-+{JzRI_ zBj6_CpZ%B=b!eyjYo8+9eo6nvN+Vo^*qL_C~K-$mMz9#afbnv8{#`Dm|r_1RE}lviBdc!3xa}ojz!V^{FX@Fs(kn z-SO3N)&@I&3WBcN29wuT3It?E6jy#b);5f6x{QOkmO2AmFqf>1!Xy(7=sc8MYF1U8 zWT&yPW8FmHClc5gM3wq0kx!PwU9^xTaZ-aKDOlaJWco>I7^w0L=5qVGtuM0VPGUtY zIvi-J9C~V8OBlq2H6!)jlLsTru)6gVLeD}NDh66h6&pU3nC}PYKs1TRLGaF)2!mWH&TFr)dqN6c_(wtXsj&hCyChI!18aKUzV#Pi84ymEHqfeHD;Iv{$ zZ`V~^?LPML;mjWZbQk`}Ip2Sw=05>v{I6u=&wnBEf5Vx8^m+_(!!bVo&0iJ&g4*9G z^u7s7_~t!~@R?-$b9bJgZhHuzK|agQDL56n-Y>S#y|!g;_74DYYOZi6kluEaP+-() zkUloKeAlkjR|#5dgTMUbv$nD2&>)U?u+N1@Vw=rsZd7dX>{>HY=5#QrA-w80 z{=S!5C!lNPEK@?gku&hZ_RUcNZ-e6)HljG6`U&ct3Yye6zuG?lRW)xi)_wN0tBU%r z+v(==d$ndrj*_ID9qZ$ZWG{lIRe;Rv(xMBVup_J{yi)BKBElVo($H9yfNL(!ygiH-kXJ!3fd&bh* z^%k`<-yJd%cT=85xGC&y>%yb|&e?OH2eKPw4$qjo>fTA2tN=B*@gTcJB6Oi+@<#FP zf)pyD<&fB@Ptb~TOlz6AmVJp-6?a|g{s1hJ z-}br~g;nnXm*VcDT8j@Wfy`nIwb)|(cm4_AH!cl|@g9a}d&Bh`h5p;^zue&s?2CWp!z4t%GlajX zN>dVG_1>uCR5bNS9~yOlyX3BLB2~~wtc>ZbeW_;+63pU-v(|vLu3c@g+l4b14St_S zmC+Co9!73hjzI~NSY&-0-`~3LeE+e4j=8u-=rPw`2h6GIaptN% zRHbrok#axQ*BaGuOr}aVrBx-+8g{87FIH_AuB0R-sp;aS#T|bCQA8rYAr*NL7ARw$ zS9&5wlDvLeJV?Qy(ZvT}Ckj*sDf*w?p+fig91%O%o> z4)>!7W-^vH?V8w9hg8afaU)11@!{QV-4=Z&A)0p9YrsJe685~MUu`la6$EZUfRav} zYJ;d8j)67T^OOT_*R7K|@z!A$2HV4XQ9cy zp|fbjJzpH%5;3Skj;ei+fW+62jqOQ2OnAXg+*?Bd0?2O2L0fZ2&TQ=o&e{*d%c8Vv zg9gQ|w&zcS_)*AJRknV94hg20Q^{_&3ATyoly&UNCRErB4h(U?v$ zI3otjx-&KTxZfv&R8%Ekt&WOgN*I3OfCO9a!p+ubRV-HlS4V zb$=Yr3(xhA*-IiP#{mYMSw-@yG#|(EddNgDxW1A_tIiR?hw16It6&Wq3@%h1Q4-o! z<#gYgx!e6vA0o0mDV2wUINKBQmf-&^(=MHSnUuC*?rvGbiVA!knc-2$2aVAbE$i*B z_4=M}089b!>Y2uyqPM*@h%B#<^}`ZJM` z0t?$80Q>n2*5ZqA?XTOrpBow-CAmwt&2?K9lh!-0StO=%d?rker4}8TKAx!&2f{Q> zlbNfT6lRve$Pp?EGbEqRSdLwYUgdO;MkHdWENCWw%w*CFgRMAiGFeVqd!qLr&Tco}FlIeU5;hDnsZOPH~B5# zjc=Ou`mvlW0W|z@=v-K=k568LEuBRZ&54D5na4}pFN*HIzgB+av@WhztPF1=J5<~d zIO$rMYx+r+vc(_gIhv46Lva;21;_@a1^d_s-^fVMQDGmwuO`m;mhBLnoCkJ z-|I%eAy3n}XWe<{GGl@p5V!d&zU~%m=bI1?MRJI|j~&dUf{rIxJ?|JCG$Yq&A!{Tm z6@@@-h%Dxh#4z16HL&_@#yXA!q}1a`^L0PbL)@{E=Fjtl!(GcGD~N^(^GHsmBj0_6 zM=D=5{XBMHb*I7e6waV7>iPo}Vry4HDM~-rQbQ+?KM1)PmAOc4T0YZHN^UUOc|RR( z>NAs&NKO{;S%)o4P^MakkyAP4K7|*ieCKOLP7iDHR)iK^g5ofK6t?Ok=c+P-`FakF z2YQ3+oPdGTSbz1%={uZ@|B|lof!3m7ga3X_D?!Dlf+(efr@K$18swHt;a8o)mo~M zK*klYvVkH-923KRXa=2mm%AdxvOX!X6**+e+x0lECSE<}Wcgw#af>QWBd;@tu?esU zn2Tqu8Pl50|fU z;aUvF5{w=9GiP3Mq*bmHq8_nqjI2{*SYfG)(bGrxjy_|`Qmsi*raVi;=>4$dmQ@F zpIQUthv0+Gux*|P88bLv<9DLx9}-o^&}#OKNCX2AUZxBtlOMjM+?(rMybw%z9HfW; zI+Pls6PIbKxYoPN*6GAnI;z+y9hrID%-R_2ipC)aD)DHVJP9wXE3GJhJofy}>9ic% z_Cu8?t0S{3Rt(6>2`^0DbDdq4Tpti93?7kfjHxbocR;BoK!^GBbIm81ueX_;%o_$P z%C!;n48p>c1GNyJ|Gp#CHx6iEwHV;!Wv07>9Gk|VXtjOSXjb(T1AHApQ#|a z{kD4fo=*#OG37H)T&I3SC2JSs?iXjKbutVukrcfPo#FRD>@L`xL8`u6W$S3i7N??x z`<;FqJ0j@B;l+w=S$H^aBptCKGhJu&4nd}Vq@_^KWE$zm9TH*iVuX6UQNkcX1%3rN zTO4-s%!-VaRylMu8B36}D`d+6)=vIiJgBcMM^#vueMECQMp88sV z57W*NVP@JT;-GQ(^4hieeCGo8Aiccf*-rb>Nh2kNA%W1vGL;0z1x!WnEQ9PR;=Io@ zF+UC%-P#mYn`4919gG$87A8B%L6GGlRxIq!)WB5Nz9XXa6iiG0sqTSeCW8iJ=-oPO zC#nXc@c3mMpXg14a2afJ3)w=8Fl#8&7>K(>g$d0 z9^%G*b!9;JZru5z>iC%tXUxPr%A>w z4rlhI`)#{(vnSV+47>uQtWRM8VBst1KDQp;6FL%`qhQBZnx)RXT1EBEftn*ZbWv2b#7@Y4zBJGdMHa? zoVIPG2m$IKgkV%9boa-l{Id1v1y2kPQO0eQH(ec>Gzo}lNA=U58Ij_KQ}lf0XxmfX{s zgW~)Ya*Ff^+6*y-LB)o(=;CHU-73S+m=K}}Y8Z6ytUj<~8ILIE07tho@2=dWw|7M0 zN0Q%>xq6wLZDN!*PsGP*=)L%*oT*SH0Zz8X%*fy}bnZiP!gP*euSFWBq|`v^=G>@- zjYsi$S+zScX7N+X+0!F6@mQP2X1h&jv8Y_{!EyHa6>H@~tluG(Qe5ssVOezIZXCPt zgE`VX2Od0={F2U(q@%GZC&zTNzsN^xCXo`i7L%*fRzSh!r|2j_CkvgKV#^riHxl0q zbG|PMO?8J8AXCM|o4>cx*T)Xk2^GVtxZAkt8>0@j_T@?}@w@MU2CFh~PRk5z=%Jfu zC|t=3zbJ@O2dPrJ1E_io#0a;$O{$`u*B8j|onli8@&wjG2&Dny9gMv{hgOiZ6kfPC zi`#y<^t4A8GpDQ!qD6|*U(%@-*GZ-#D#4@NSZwFuGe{bTZO01O?rcuNM>(aiFHGbib zs3XHbHP_m1Z%HzRK;N&1XQXu$6}CE)8;{4LcLz7QfPpmD243CY4c8fT*|F> zf8M1Zam;LZhXmyNO;Z|?W&IRw$%9fYDOy>J5q#*L89sMs+#@lP*~nvjehO^Mt+hVH zSfMCExw`Voehd0i+wyBjcWsU7bV?B=K2J5r{&-eiXT291x{4z9UE3hQH2hYUDLnN{ zlpD8%J{V4+sW|WSaw)q>yd?;YOcA5^=s)Kj7nX7m?4yZhKEo z%Z}fO&>`=FdzM)w05KhWO{; z827in5+Y!W*H1EW+Wn*)H#l_$6%P!qGiW%#e{+53Q56Ezi*GhSnff>HPf$-dMboQS zk2@!Um!MdV!;J}(Xrk!H4wL8>av#+oK%M^Ph!ZB!j91@ROtO{mlm7PYxQGU&o+PVA zWPvR{WO7LS_t4h+kzg;d+& zDm(Y5sEHw?dPw_25`Z|D?|&VfT8}z@!YFLB?J3A-vtv%jgHx60>dLt}G-ky?nRIAvnqwl>;~rc~k;vmUmH@g| zo^gdvE-oDnzJV#ERO} z1Y=IWX1qJcoYG|WE})7tmy#ysXJc*;m(^?GE#Dmm1JOmkgbg#B7P#0*$h7^1_ERQn zV%fy3P1Pi~sf2oCPAkprZ2A1*=)3snzRClhi?X5LytA4=<+&ekEXCj)1u3nX7GGFB z9)wu4tqz29Iv(5P6s@#G&m(EQ^wuV-z>P(L*kS-Fg<@&Fwy=rrR;JG3&Rn|Loy{kr z@~9YS(G`Ww&yj7Ht`-~t`A`zsDxw`!O7TxwN$6OPi85HUmDTh{87nBq5CH>i@e*1Q zmt97?9J4|9v$E(GNCt&yC$b7vgQ;Fg7TE|%t}1{6B&s!1Y!Al>8i$$>crQ1 zw$Ty^paDDF*;{f-A5QJ+tD~NOpfAmdUt;8|`wDL*qu6O22uWd|Pk!w%cQ(eXH zOIfeVeJE4s$*w9z`vI%VzrJay@Q;hKBibaDab4?dbdzE8jRJ&BBr4=E(i(o8R=bLO8mcEs0Yq1$R!zP9 z)v-QlG$pLAu3oZj?PE=Ku8Y8xCKhV`f%+JJ8=<*FRNf7~AhzzvLJ^ra#l)SEp<^d| zx+vKnks4!WZTWR1(q5cQZHF{1{Z|D~uyGSb5o11Ix0Mt4m-q|oq8$@KG^jnQL|pRd znq<9WLwi_#e(o9BCdktZbqO~bG}uA|;~hb|sH4JMz=VLQt|T2^^Lt`IObm_U3rg5H zy*0u6A3msBh2^F^g^6dD%b~)>d6U;!&d~>t?8SiZtc+`ZsPG?iTnKV@AwV&D-Vz{o z5w>TQ`x07SMUIG=@CaUZKJN8I9EL1Dk3-&@i+f%RV<4%aE=}t=j4+c~qlD*?vTnKTHYEQ0nEdFH z=S3r2Of2th@wEDKovJN47<~q-;>(eh8IM<<@Gmc@d5R9@zsJ(;bY9KIEFZy0x9L!V zhe`VLZ^Izr;wJzKXv3H+IZ0iW)_WzKC%|rW>Cwipl*@Oix%5|3j7m?}8P6`&lF=FN ziBS06HL10rjL>t5eP6}#NAUQH{6Lf}1u$avvy1MyE*_=|2tPzMD2wy>Hmb>eSLT#D zWkQgiH2vPQ(-7-$q+6wh5TBJPX#S^mXQB7&&yt99YeuAlC4SMlcHU*vA~o_w?z5Nk z-0g8@sx%DTrm=_2ck@Ar{0;{gUr@){*Ya z;KkOVD@cmk)>#ln?DX!;Kg@OF2b3t3zKHPcTzVtfM9S~!6|fj;t+aMD0Q&xrSl!%q4c|hcbCK<{u5e8NOyK84k8tY zEy-{WN;u_}T}TpSJE9CPMB0oY2%pktjAen^v5a|ScRN0Bb8cE=s2fx_StIO#Dp*U? zHL7DxjU@rcnWfahjSbNjQ~r^8zP2Gv8p`Vq`^uQj8kkp*J03PI&G;vrbW70u9?(@yO}!rw zBji7#jQEo5Xw)xmGiZ1FG1hui0a{!xvBD&W0pNheD?Vf z>*Xas%Q%@f*m+BOvF$x{a)CnUxuv^HM^2m`Hsx%m*_!yT%0eVD3^7;}uG(%g<%iF3 zTdDn!{5?ytD<0vG>DM>GqawVYG*`Y2sh5 z88XYcPz!Xk%i}V#HCaA{C7rs8Go&Nex^plVML)WwqFu~i(DO6tHZ8ZJ>=+sTJY_+<~FDIdyY8XG8fpVzvD-Q+Z>j`oiUbB|+gq+t8nm)#8VmJh2 zMcmZpUo{{IgGQsbTY$ zKg%R0$8zBxK)lEAvHRCKuR^q2i()HIG5?uQsKHCqiVgRzb*qg5R;l%c*EPCSek$*F zPQLrEW=#yl$}0ZZToMA-kH@uQ(#?)|1t^a583Jjoiy#tEnFN3S~2_=RWn0a^)?(JS3>;iH*dMlj1>O>K92nXynTNe)#iR@`Y`ch??p|W8DCQKC{&Kcv4dAT^r;EINEp}1 z&S3IaEF~zu=fY&{FLs7Sa?V+TLO-faduT?TbN&Ec zYKZ)YL`qvz6RFLwF6(527L96OO#p^X&A%-SkbTcIeoC8c?g z$nLXNwZ%l1ZY=>12rrWUQL6}S9U^!F6h0JA9$1s_aH~Uhknw{X$|PgRlYtBcIN{gs z+q<02n%-?6VamgPnf2z|yj4rEM&9qSYHVb3kpbtjv9MMSuQ7;k2lyPY8)kK+=nT2g zvqujqymOy(rn09l#5wbCKj}Z848BOe7HhpX?E5Xq;@HC%kn>PNDLP&~*JK&628oZ> zl-WbIHZUwBsQM)K#08|hLBg29ijq@F#hG~xayad#!wQp`_rl}Zd>l$Df(^yEAXH?u zAr)NIHfZkS6Y>FcJHubvrS_`Tf*YuETlc(J^gxa44Fe+3fVe89r|qiP$Q0 zIh1?4e$)JVV~{F%#QiQ0C4~5EDr4%G6=gPznMat$Q}o7SC07jARnbFcSC^-O*?|Id z7w(32QJ6=|#7;ld=bxXc--lp`NR9V86m_U}NW^#ZXr`&p&u>v{IzWWXEB&OC7l=Or zIoRc8Ib0pO>vrah*9^T&``RZ> z(c_obnr&wUi=Tvs`A!w_kcr_Bz%&Rw%kr8|nz)DT`y3{-I{*9XNU>ZSjqLQERI{++ zK|UV|-=leN9jl6a;~Np8!E3Ja^}ZZ>xj}%?(Q!PoqfJ-M>EPS9|8;}jL_gnuouk54 zz%-Rek!1jP_%k!i!KU1G=^|Hm-zGR_b}5oKk8K%hwKX9#Gp1$uF)1M`{{c9Z&96b> zX)2Rb$94snRLm5SgiMquT%WjVK?8p5tqqv3^pSBUFg|(kryy+*HzKyh(4#S6jWM{K zZYkKX(bC7yycZA@XZZnRoa8-q;>?DXF;RXZF6T^hW2hK@6_+DOwu@=2hku6AhSpg7 zKEK6aUFhJKDFrCkh~`Ko8@d!e*rWCRP1s`wqDdN9*t=rvCR$?-d+Sr#-dg2 zga@B{=@Di%|H4SHKfKx3d-SU-yH!Er3!0`R5H}NJptS?LcShn5f0NeefVc5CFJ1N( zK0jqI+jlFG2Rt9D-QhY76aCfp>uS@sUi<`}vh#-6r+ba;61ouGn>N09p0uWk^x3Y( zX|=8)3Tq^~lLYSGD&Aqey*-QK`0&{{n?t^qYvVIh`~7YTdL>D*R07%DO@W`=B3btBrMck9oOoVWHf5k|=^B>z zp0iGM>6p@~3#pUs;324J->+_`6fFiwIc?)}$RMwV`=-~mDvm@pjZ!U9*jY-DW3^*) zu@XZlD=|FuCSe>JeM!9Ed?RERoK<*l%OJLGA5;V>BXMwAK?(#M$(jJ01mZ`;v9x?1 zm^fBZ!Esg-mkB(Q4Y>U+L`U-BRLp5Oc(|C6NFy#^m4jr=rKat z>meBffvK6Le1f3%k!ht3gbbo; z8`HvWlz}Hhk*V%$hb;CX6Xj znD$Kc$TX~M_-QR^Q#$*?4m_uGB&j)=*buNh4k6yI;0oG(m`eV0w8$mQ#iSt%y_XKt zNX_wGB&5mM`!#^==0nX|pA0Ct-2!Dakn3|nZ8lkqIo?h7odfLFAv41-xnU|cR=SEO z1VPPg7u72Z1I?Xz-!FwcS?@yi-IvQN8xq2gw299bJ}@5R*B1lRyR0bqSLL&fJmsN; zDB>~l_V?mD#?h-=D*phUy{>fR<0x28;Z zDtLAv9l8D_Cl~}HOtR?fM#t+od(Oi&?>8!K<_a-XeMPaWmgOBd=4;AdNN%zpJ%!Vi zrVn)buB_2sf8-&|))N{3@j=%oY1yl|ENG!hi0D#1>JPx9qqRNhLCSpCA%c)?d%k@SL;epVN)Om z0Le2V2Z1_V_U8N5d4|*<40MvJJd68^5LF@zWkvA)O`{jjJXK7G6GVfCDJG>;eI7Cg z+|C$UtD3IlI!QEnUtK#)IGh(XhPuAVD!hwgm@mLYwpvyukNh!%^RSaM@A#Wf*BQ=n zjn*T+8F)s1*Ta7=#ISFj;{BbU_hZqlht1++kA+)rWfv~2Y;IO}_nz^d@tyrqq%tOe zt)uJ%1v25++|ru)wGmlLB%G~pYq<7nfWRV@%znj^2fOvQxr*YVrb&Y82hBwmhFiqo z^2BKK9O6_gc&;C_jkxJ1kP)r{td04GGM}9f4y1FkyEy)j1vMX)u;IXi9;fV&t5Gy8 zi4-5P#cxT_@aE$&rSE<6n$6y~^Yy*2y_m1(^_t(RZ?VC+R3ai(o&OP-jD^+q^X472 z*zd<|tN9E84=LS@6w!~z(yD_RVN4qOyu8B@SG4W7LtBVK^ z$(k}(>!cAGgQk2W`Qxt`SX^jaGm`!|u4CZi z5vFKfoF#AG`&^#sy1{{@&Jz}g4_IcW0|(UaqVd722lBmoy^5O`ERD%lRr@FG$t?toEtPl2=T*a)f@*3XC`Sa<8PWW z5kZMF)YQ&CV@`}0<(Q7~*!H=KeQw+}Sgw^O<|$_9&@8r;YtPuXH{L1!D+x--NGb{; zfoMt$WVoqWz0>8QIa+0=lFl6+R4?mtx`kQZsNPV zG}9qb9i}n?U$Pa*fS_Nz$k_2@rp4o0Q&6wN2{fMaE*KgMJ*upUGAzO4yDdXOoS}(m zKPOJcme#J%YI3maaY1`jN1or*KKt0%9*b*)%4{Q81rG>-iO>`!6tC0og8HZutQum> zj>+?W;Bz5Baim;`l4{_>kGPHqTfppNgBu=}crfmJDHRK!%G!#LJZ!|OQKs9OiymD` z&_*2fns;e^gt=3MQOuf@IE{?o$aE(RGp}>zQstvHs%wN8QoKbikOsCCaA+BjlA{gK zQre6(le$@1X*6*JLZJPb+i@GVutIPck^weC4%K=EXUt)$3o{SDfkuqqU9GCCAcu^N z8BDWb@?M1SCf8*GhH5(M)y-HR-TR}$#-;CMfzQ02J-8uO9qUl!R?20lbsrBNa+et< zE=ulc^`$=gUQTK(Me>dJn|Y2(ErgVRD+-oP9FU*1ce!Q}*h_1!VuzFju>U}mMV_IA zrzaYXDpq-Dp#iCVaEn?>?QH%A$+D#~+0XkZH)Sw8s}-j|t~Xvn!Zft5Vnl_ZQeaFA z-vwgIC_SAUTI}VP^4lY7vo~JyodajZEBRg$lJM--dJr#U3sFYdu954rQDViR0Mx~; zLR)jrSD>t<7zgw|&KYpDsax3m%eEAcH3(wVFZz}(_kxs|zTD*DPr<3HW{JEdCurXG*8;_YzNzng_+ zu7Ttz>NW}5`CObF!XsJJuXE(3DW>K7I%#$isO_>@%XtgJ{nX<^La`xh?ha;8`!%nk`R0CIaLr|*K{swBKw6SF0zme( z49Qya$&;v2J7ulQ=k6+b#tzLp?fhB|-Xtw*#mu_Yp?Xrj+A#LFNyKzDY67kDe*yY4+`+-OI_hgr^+{-vf^a((i5bkzjLqw2qqM zj7TJV8wpA@q!T2Qqo12fI(5hugRk(oKq3c+QBALLdMrY1b^ibiydV$(1c%V&AlOFT zXU;E~l!br<84`QnO_csnPkl=gPy$>lLBR6 zd<|Qv0|-0dwr?OUAaS3A=9lbP=_JE1eyI>a1BT}$oJL$EBqIp|qI=t_SpdXnU1Vq! zq@`fQ$Pw1%^k$k>rl%F!t59R9t!PJkk3uo5K|UXP_qgWRy?S9JThJZqNIwW{uHoqx zBX}IHU4TYDR={96j3f>3n_8?U{ncI+Q6vp?JUuRT)xy`}J)`YrYS>0yDMCqA)Xv@; z^m~qvr@-7@c$Q8>E1E(s2>^)zK=@%TQ7GuGr76)M#QIwqQZ3lDk_?0qH-Gfq6Y9ZxErvD^~GPNekSP9qcGvxEHbysk@Al zEhj*BRo?JQZK%Me*lrbglgdZHy4_Gsr4lLYdDg)R0cx{ym7sB~&Nw7_!`9^G^AiYE zl6KDr4%$j?VvX-e3Ix#A#zKXpkao_U&C%kci+%!t8@<1|kRG-s;8ybF=P7L*Rwt;q zR+O~iYF5upRY8q6bu-a|0&XNGA}G`>3Su0MA#>7@pDd4yR!LQ~fN;%a=12**))Sgn z6s`N5_TKt6DVAttJxqx zBS^2m4O}3hBbBTYCxO?jkA3fNKq0RwX-m3FqDZYE6##TJ*N-EjvLg@kJV5w-COV>e z+P$!hcO+uOxD*=z*Im4hdQGVZ36GrwLF7134QEYi=60EE0f)Sck??&2vdUVeBGf7&F>ObZ0VIiaUR2shN{ygQm}O64{Lj7^@Fa)`;Kn<{YN;QncIhEksVl~ zT%W0o1jq_I0#o{ZB#x$}xgsMLppal6f$Nsu95887q=I|s{d-H6-Rn=Ovz)Vk#T2`g zzni#)Cfbsc5}6|qkpmVXsEvW9UCAv9Aw(3N1Q{ZED0yMdQn72kj}eU~K-X+b?=DR@ zj+9LNSqjcjVK|flFjK5?PR_dM;as$=P!eXaIX%72c+6C!@Y~cIQCWd6oVh)JS!>b% z00qzYfQW4>-W*`cv<_v9OIkWW4jf@R@@(&Fh)7&Wkvr?%EmZwV&`3qwqOzd7eUdAv zHJvo0@b7GHZO2&OYW{;V;6b`R%IDBmv7uzpudLcsQx?sfv7_qN0#c%oBmzfMPs^$Z z#cJ5bd@-tLZ6B5i$}-cW)u-4>&Vq9HBXIZ@*Egy%w&aQNPn>tTcYs~BB*Yokfx)np zqSy&e5n5My@`*Lo^8U{eLY5@&XvWgN#D(9QPyU=at|~!RYf>U9TF=4iJbD7JQ8zRu z#m+{bk^ov(-x)su)UyP1YDuJnzh@wE@&edOuo(o+X1!s={{TnQi~YC`!*<@S_+=kE zAd#FV_E-QaaF*Ui(vxV@Yj*gH1tVk)+d$f3?>D0WyI#~EZs*(kUrBVKKR6PVTqHt_ zLF#CDc}Dlq=WT;@stNFlcy@3hqujQ+5p8^Z;r5Hs9Tp$ecaWgs7293UsmgyW)8~2v zV=swJFhBLN_6>#w+Y zI~1iyvD(-P2c(lUr2fX|*_p#5Nd!}V!e63-h2YcU$GDx%w7RY+_7UrA?GKx9{a}g^ zP(ZDA8udP(3U@vmk6!@Dp++NfvPLaj;Dil<#=U?LB`M^Gg^3Db4J}Jiq?abu)Tb7* zp$;roHoehobr|+bRJA9y@%^qXp@!SIxC_f5#Qy+!;1qP=)Qa~|H0w3YPR|#y%f4|M zZuPJd4OC}&YsT1GQe^3}Jxv$Zd_2=N2&MvQ7?ZE~-wW4*#sObyR*0;ZOM{t)YFP!AI5 zRrYZ^19H+rl4pC24WSC@g3g$w&c1P`leM%|_HhE>5X|(InF~XkeJWDqgAf30sXuH? z*Q7_XT7r(0hqu6^c+wJp3TL^nMjMv)%7pc{Fp#WL?LH)ZAnHXFb7m~}hLdb=9|t!u zqtWZR*D8`yd0a#z6xD;mk{p$y3FUE>M?otpvXrZp$IOzegz#=<%md2u=#_Oh5Hg<2 zn}UKx&uG>f`LEsNq4oiSJ+nLJ1kyxbu?zhvfsZ?%0$IL3)aj5?Q61}gdUu9fD zb8@&ARi#T)DlzU#8MF0;-ewn7ogu}ivm@`3!Xhh6! z<R<}) zpZi^`C?@oj-5nY6VOAVNb}I0iQe6~#J0iaVJuanW(jtFlb=B%E$B*AZ1v8jj5RArT z3c@;V)vO>6P9e0bM*WhznVdLJV(QA5HG%}0?r{2RS4P{WQ$GGNC3LI$u}iMCd(N-2npDa1!Cl1)y7fq!Xp z(To;k@5A^pk)?KoxhAgYCtA76wutZzc7E~H^%jBw+DUJp+NSB+!F8C*$RL^9)bdXb zElU8%EdKQ7MzP%R4S7MY#cY#7NfJd#1IJGA^fea z>Fp^f*0`I*b25W+%Frnw)x)xbZ2>1yt!8U2NYy8i$LPtc=ARDJTnajm30k)%M{|1u zw|gCuiYj)3A;3UY(|H9(of=eC-alh^B&$%g+geuFyi3CoGcuIlxEE1CHhP>> zmPy<1)aEiaKDNgb%j#HCQ$g(g&3&b2l5xn=2ej^e_F?8utw&5=0k(x! zI*kV-v)~RB>i3k9143p&;la)~f4d8Q86FcuQOZEx)o%lCiHMRS4|&&bdrat3HW5Ed z(qGkwFS$6Tqo+z29yVt-fSkg0ID&xTN%jq|tbFcT+EaN7tclx7f;T>i__y}ww=|@x z)&0{NaBcLeBVur`oJ4XrLyvb}=gA_Bq(X-)btA@I5+zPhOiVpkFEE(X;PTpJ7a2t2 z)y%Wm?{K3?Ft;T{kX%)GY8$Cp!r=z3B%yM5J1u@Egp}%Lcsk)KGBhrM>}DPi<4F$q z*b*&FRo48{9tDynUA5Z4U}DtIp}K`t(&P_+*xLpe64!SK(XuX;lSazTD(Ps%yFha8 z&PMIN%V}%T$+jlk^07!tp(l%-W3o>lQ-g5FTajUY1Ik@wHA)q$5UwqR3~K9cEjSD~ z2W{{V=Z{{Yi+kYv>IadAlsF{j!J{{Yi-(lG#(sU`#&1aNWx00xNH1Y0G-1eFfw zHu9#~?)JF&022BjZPiL@c85PF2HL)tpfttqbMkA$Z9{DO1J42to4D>^BiuHw0f>RQ zwvu{99wqL#Q_3EJWP*; zOC=_|8cqT$>T*KHAy#^Ur8Wkks;B@7;L^6!sgXq+bXiWaO$X6Se5e3P6*Q%F)v#d; zxwr!z!rXFd^*C@4XZXG_<3uE`LUBzCt;Jzj189@UqtJ0$2am1ozzTxDht~2#5<;z> z0(+y@3(IK=Qs-J(PwQxuylisfeN?ZO=`Y%wDl4dt*+@u($wA0wwwz(i3V7b-=mN{$7s&!3yeNKkL z$9wB#FJyRi+I@~?plW7)5aAEA#0&}z{)Z;Q(xPK?!wj0nG2qFL9rYvfUcXBKBoR^ED(kpJzO44Oi zJ5TItE)djmhc)AKa(j(AHzt*8K{PfNJWoo?WU8!@M^K*-a`2Z9$(xnjT7gnyR~Qqn zP?sjj`!n9yx(bPnuTT^b5O%$ADL_i5Wx5ry?p_H~s)!0YJc8=1a@F4$NWvQ!sNGF> z^611UEziz9QSEL7Cu)y%U9kTE6g%J}LVAE;P=u$mrzAR+LwH8h-TPZNVrfPLOI(L* z04$ktu1?9?-mxg0Qs^z30Vzyxajorh)`NU_0Wq$gw%RR=bWj|ar`Xq_NNXlqm9Gu4 zJG+$~px6)kI3<9yquc5S(cFK7G4)L%i15y5={{V*Np+>A7CrLmE=@+U2J&mNFLvy1b z)nPn59-{QhL%3x=``o3b5KjkTm$qpNJ^taBM}baQ6Llw*%d|=oV|$E&D(i9$1U)b? z46K)+_`%{GM``N2L%D$E<0>4CN${~&t}3A>9dM8$7!9jeBpn=LDqGUb4(u)e71e;CRVghx+51HDk86>S*x0mAG0Huz0(BI-r(u9OVGgSW0F9J9DBQG4jnv^0 zi6(oK>T(Ij3Jh=KyU_6afZe5I7zTsIJ^ui;v6Nc=uC*P7TGK>>?Q$@br^DaM?5!$% zCs;oQ{)HO2q0oWD4|xobyLfhDCVR_z>On}2W?e!e%qXUYcW7s8fQ^e3aZv||06SZ) z$^H>uTt1NKP)Sbd_P2$woxb|IWO{ZTyLUY>b){pJ@YvdpS8cxk017w(>`NWwP>se* zfR1fRVIV+*#kf~VgTs!^2es6tGUzT!6J0%R{AnS^6A|Q^Z*3J2SPzXsb0lnVz$L$J z<$>1{@cTsa7cH>A=Gy-Ncd?K(Us7=&opK7da(rAYsqr2{@k$^o--B~5f|Z#YJx`;0 zYE~$^FS1?T6{F^eQ3h?!vJZP=(3Qv=4D!0Y1cypUxj-C4h~cTA*NJjh*-LD3(nA6r z;imX*h{R6H(~;$iWfwZ#-~}czj|NEhHa1+fBoNv*_dWI$bsdr1?%PI@ zH5!KF$qK;SI+=w#otWSmC=g&)DAERQ@Y)Y39)_uyk!+A+2-1b3;SZ#HN`(a4op6p) z-xlnr$tRQdw)!-eXbI3B_W=2}p{JoP83$^okdpO5N)?hDW5Q`Y-nIjx+O>ubkv}|l zddO~~RjOAUq@RPsZ3kld!1b2Cus4k=?JY-xe+`OuM<}0Xt+Uaj9bwel+5lGPL8`ZH zR7!#O+zEOU0-&Dw{jP%A{D|bQvFQpx)4N6lk%VoI`S!=L(&QiMU~ozQ0P$=kcSfEb zuTy?hx)?m7eXW2P3fyJ7MZPEC^++0(asVO6N>-x@_vK(o7)NCz-<9NVO*tFF)&W&E z2)Is)aHWL?gmTBQHo`}gJK#O&m$VL2S&I!Qz`Bh;m`?ypXI-gJdzI1MSIkE$kEzEn zHN*bVMj@YOHC{ddD12Lp2|bWg@5imhCQyU z2`9p(5!CC1rj>HR4Rdbv5r@SwGxH(H2@0$pw%(WfBeFd$^p_N@b4a`{@qOP z9?F2?ZItdq4B_n^h_O{51 zXye@QlybR57{q$C>)^nkdhh}_y9(*;U=c4co#b!aTh_rR7LZ+UBpS_Y?QjB;Ew6Zi zM$(8TpSibjM->jh-m;XE3bv&6;^5e72HxS8*(E9eofJhVaf#m$FpTkWN!t`d+`NuU6e+y<^95$DzKM*Wnn zM1-YAlK@Ns@B#Jw+4)-jV5a75n+W+a{L{;^9rAcTMakrH*&BTwi)pqsF;b(%>Pz)Q zVapu~c!+g9lgRyJoIfvRGjh8?W$qoR9tCoun(rK@U!CWap#>>I ze51%1E~m&nZPcVIdH3peRkIa@`|uAILK2mWX&eKsv2uJMa+=3o8_*_+ck%E6+XI1D zQannZXHa$N2a-?xuP8Q$LbUIf0g52#fIsH;yQbQ;C(_u)qoqDBz^`hh&`iXkOh=GY z*xFsxjewUMq}?x4l`BLRpb6>xW}8%ZwsgH)PTLU?U#!8@D26zG|FqOvJ=FMt?QoTywl!GpGwgi1uC?VdS<`*I_YP%hOB=gtDU;vobU3F%c_re>x)ZYt0_eC+wNe{o$PURAQ6>ni8c%zKqDuTU zosz(LNt(F{DYS_oDCr*Z>tzz35SsB0HQZRrKq`vGOLhY;8Bsb*t;2{UOu*W7>v?fr zGfmEz35}Dz^fXouy+zN;(-`FRwH|Bc%+ER8=H8#jv{l7b zYVX)MCIfxS1_Sm(X+xx=uFa&VlHYxe78*P@(Qp*4A!$&kP?Vlfb#@O}97j2RRJC^% z%$XLIuSB|_{S#82`5b}y=4U5+CbMq!wOdyaL&!x4;vdirjK)(hoVIP|EvDjl?BWQh zJVjrgdoh>Or{KZ!nrVEF9)~4n0QcDYo^t%5dfmle$g8GIe<;%6`YNDbA%n_a4VgmG zi5C*2C>n(V3VA^f#Vxr;tPR>SulA+9uF2$a?dqSS8d1nsWb!_25ZJ~&njN>=a$H0h zS6i6!%#m*lR4qMuejD~ny&9)eK`0OQ#VRKP(AwH3NcilPlZA=ErjadP{aaFTrI1)rGN$xM3Sq*D5TSt^MC6NX9hMyBjSw{3Yp;dEB=+U;Q zV2Zf_(Zo8r14d*bUF|pRDiY(|?H_%N3V`|@n*C7i)iwf8Lbsn6wU}_%UyEwpO+tBf z4Zx(NkZ_a1R<{NA|p}B z@Nv*EiqL}IIMp6KHdUAv$bTn@=PT6z z0OKrw_btC#B)@*o&{8-K?xnE1$xQ`cV@VSbYQ^RdowM)M(v^8?+S}O2Orouv(NQuf z!&ab`rCJxy+SVMPN=*v~9R=X*d@VZOXlS}|bL1&BsAGdpBY=V^S|2%cR!^J(7p>Q2 z#KLY4MQ!9VI5|N|L8E)63Ib{DaisAG@-A@vxthqR-M9VX(5!*tNd6>n@`ovh&MRy8 z@&b}Uh$0T>;ko>=%4V-wKQTlme4SEz(*^wf?X?Y1k`Y+OK0vZH6>~pepm7tZa`sPC z%Q8aEPlWL zBWx(06D?Y8OoR#F1JdR+nFcXo7=N|7|H%CTvOt%HQLysx-GMk&?14h!64%aMO z#wx5n*HqcFJ)(I}seue&&t z0Rc30P$fPAJZDD9HPz=xy_#|`H|py;tJ+Atmk?xJK3i(v8luI=4InceSuaBO%-hN6mmNEN+i$=G8FzT*bez zxm^-{Zv~Ahv@8;8`!lv%8!M}QC&J_>Dw+~W_&8T6 zq6UDxP?P~dN;Ip?3UcjsLv3pSx@>C$!%8!xZIdpRtleG0POiw&%T1*yFr+ckgoujh z?l-r7t2(QwdP{FZ4QQzWlI2)d5_ubH(TStWD6e7`l5mp;(BxZIDU30@M++=Y5)w~y zP(UEWRC-^Qc(cX1^|GZz2#9Z@+LN+5Lu%+!zG!+e;!RXQ<44TqSzyq3XX4#5fhvaO zHphEuV5(;ia{}YUCL31@lM@x(sBaNn8&=dL2--cZO_&qX z1dwB^(%4!_p_eN8k9zgEVO6OwND%G-gQ;pO$%NGro(Mu&_6Fkgli&`dag89g#eCKOp7< z{rrEXx_3WGn!lm5^qrus6BThv)V5wJXta{0sVPXQDj)&MOXTz4nsOO=O{`gkk0OZk zkDwRW>1iEnPtEh5Pa;0v@)C9K)&k0 z(fBL0kzJ#?(<(>P3U{}4cKau|>D2sRn?E&X@|bPrXEkeiKWd&u6XN7ooFAxUIm;=| zR`KQT1$js`T0W9>9J(Ea#&yy63*zWetq;D|<4$%;$YYz^>i7yFL_a(H*=G&w8KjCW zCffdzYM!u9Q}5Go7L(68U#;_f9JgY7>G&^s# zUt^fzTkQ{hj&F0x0o^`O*y!Zh(S6>R07`a|*4Q^J&bYgYaQY%@R2_w|;R!u5b|=u< zD^b|@2;Af*67B$Cbdx~b=+L3Iv3Pp1XlYU3wLreWl!4ja1?TSb%$pgd}OBs5>5kbjhoB zn;xV9?3$_Ng1sq7(S>&uHp7;iW65>Aa3|pNdVwb}y|pp*qAGF5uzwDy*f z0L@&5epu(AqJE~D&;I}`B7K>n=l)qy`o?}X^L6llugktcH}Aqpy?&q*_P0H5=#sOH zr0%rRu{u=#UgwY_>VseNSs(uZve7eK+n=$RHm+rE<48)BErj2IsfjstT}U)EBo`Rp zzpqVu`rDtZ`oGP~Rla+^5Tzht6iSqx4Lc-g$)|Yf_tq{}mYUxKm`^w6k83ZRAO5xT z{{W`srz3Gc{9M`P%U}Dq0A9WF*T1c~`p={0Wvyt;Vp`W2drDHjqr2r2>{@QM^%b_d zgoWj^SiCt*+<)b-5BnUHS3v>Mm(^3Z~v`W>+`=0Jp#Tjvv)?b^hy`;r-hG0Nu5Ids@+Qz)IGc7i1+$O!(>9 zA@ic`<3DS9Kh$6AJxeTIJMboE$)hQn2m3B>{{U}%{{ZQ`bUCl`nZNzL{{Yl#yv(&M zU&OVmvjs|0qrrE|CG%*#cz}9q)d2qhF=PJb(y#W``i5uNE?G`x0M1)Bn*RWnzv#B- zS~o^4R&$19{{V}b`=`w&?dtul-|H9p&!OjqsTVBaT9OF4*-=dR{{V&`Cmp$5zilh4 zkAcon%=IO#sZ{85V!!2cA9UMMxFZFe+2C@SpSpbbUBZ8xclyQp+V?hGJ8jcL*l!+JJwU zzt(r@E_|)u>n?t?nRfpG{=G(WBd|r-1MG8j{{a3>*WEf^xTPi&YdBB-Q#1E22RkUH z<&c#rG`%4zQf4lH4=Y7C7C)uE{bK(BS)~X?)6D(@B04aoQfy}FyDOOVT621FAIZ)C z0PQnhbke!4N>IF1r6`OsiA7Q1yD^BCf{V9sf^OaffnJe+tn1s`r2hbIdN_Cti0Q6e zxTPPl&EeVo*E9D{A4$>`(Lki(QdET@K~M}7tOKD3O|2KCyYiQRtY59K+elUH?!Veq zQKw-K4lq$|X0-U%@Ff~Vbs#COqUVHbplRmM7WgSm9~Y_2Z#cQZ{_jIgM~dO(bFbT7 z(2{{MG#p$zNl;yQy(|ia&TP!t$Yy$L%EEw!B$y=0aiU z@j(9oYxG+;?PgFeTDY5tIw>R(`}jYb+J;Y@Gko839$x82@v0TUNj?+RBV!#@sQMd5 z@^2j#Q20J<{MD7sfp0PfTIB#I`AR=aRPrjEUuGbHWEt8tFD9B(?L3#9^50a?U$iKY zqEp2R5}u(>=7o2*AEbU^%K|N51AVm`a;MV#{{S|SoD_|H59ZBfn@5qXu7=TjVb^|> z)ZS_BLGvKp&cSemmp12MxI<9%ialqxmZ#CQ%RkIdD_MDXnHV;>w=LUuWZ`(Sk(P`m z3#A0Pm%#Wmx9FbUSv^RH3gctC0U$tBO@6xqvoKuRx_saXueI ztGYv^9O(;}mm&(u8tu`(@o%2ow$iOt~Nc$x%#) zGTLQ4t~hE4?2bUp086vh!jxsgk2q^JFzysi*BM&9A{$Ou%VsU5GrDQ^?T%VLwDZ%b2g5 z8LYy>P^ro&5V46Sa~XX^*G&x{Kr=RGKg0R6{{UxWchPegql}(eb?A>SGWy@n73bUE zLzipm(V4R&`-e7v?5uG97sT`b06Sj)01;KyTFcyB)FDbMv=vaLibvnr~@gh2oXQFc)aPRUizEuMX6UZCmpYuqZHgbvIhSnLO z6~dvU(Y5|Fcl@?Qf6yMbdv79FfE_`Q`c8SR+Z#MLzv4b${A&04e3$hD*4;e)nYkG3 z(D4_n8S`oYxm#B{cK zFMgP4r$-T7{hGPkwUt~l z6UZ9_pYurU+RCSX8(3z9R|9tQ+F$%u3b`Oob42LbI5mFXWhe}zxhl#0jl91^cSC3vX3}YDS(8thoZ$9UFw1lO7C84dQ)PJ<l#{t6G<0(v)PK zPo?;|h^vb^&k`$88U~)#V03BZ9>?Tn1{I;?a9s1D_uAqVkBg{*Z2h8nN42fAG1^Zk d_J+1}c$p1t+!B%jBBMs*snGH}wH)jt|JlR!j$i-) literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000012_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000012_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..a3444cb9d267f79ae8aea355fa680e7f8405ac81 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTgb#YfqY(7_fC$YLN>Fg}ci zg)58?qesPIApoIa;R2&!vM@f39&X|=|G;Q;e~ya7LI6UKhRaeE( zYtSCn-2BqwRIJr76R;YV>U7FrLUtGV1Q(RG5{!O008Rs5Agd1fUaQdZpz2O#?8jb z!2$Ta29N|GA|s<9qoDmcFwimndthN<;o{=Heqvx?Vd3E5;^O1uzjnM{2nh+v$Vf=2 zscD#*xqlADa6wcv{alyef`()@ylr?Ki8&W)+~CF(Zb|jRt63A8rc@qR z(;_L}!O-ggw|R|bGg?dlG&BGP1_c)8-x{HyF|jCNIK{D@X9|Ln8jQiJIIwC|T$kGt z{cy=HCRek+`v91TP`=QCFi=zgXiNYMCgArf02K!6_2#wx4sb)G3wpzFg~8-5b=0M= z7jHggZOD1u`3^L`XK!9Zizqj3qK>4A^>BP#!_(LsvUVj({pgYmR#gRU=uTNcde8;- z_3jpK@LU~yY#w9u&#GIx_*I$#AB`Y1(MSrKy?$Sm&s#wwww&I3^I_LmIgBc)P62&+_;F*9hLYvi;=( z!9TiNM_~R||L^62|KBZktUVQWXWP5^+>MR$^KEf?AV_hx89H-rND?OP)tIy!{O*b1 zFc3oZqoJcCO-`NVwR$2q^qsy34zx3MrVp~1D$Zr8X^hF8Jbesj09Vz?ySQEGG#XqJ zIWAujrNqAP9sh9OGtM2G(oApFy%Ns8AVYTRnb-hsWLhLRe};;V7BG9dZ3>)>0&!3=a|A`oT@pEq% zJr7G2yTeX;WLmk&Kw_{{F*#yN#rkw(%-%jL%L@LsdaHOCX>i~dqme}zl79ffPt$6% z#mrpBfm4=oPNQPp|h#LYMH^4U$u@*%h-3$&b!ZVnS2Invmlv)uWiu_Xm z-FtUOcg2_wr@Hlm$8tzm)r`O&slFiL3Awgp{mIU{o5mv~WC2_^x}(7k%r=WVShhc| z^ojGIJ*rV8`VBB$Z?Go}abaZ$Mez#L=C4^5s6?E`Re{ziIl;%OzK&{*)uvfTU`J~6 z#6H3(&d4ZNcwaBj`_FiE@vdm&__Hk(`RY%sDXxRSAk<-oMEBz}pT+^nNC2^yVM1()8O!B=sMjVL@*vCkqQNjTufx@+w=M-wy0B@(i%SQerM=OZt`MOH&x;#A|N5w zd}d~}WpzKoPppFf+@7Z2Si-<6#*D?fvzJk6_5p$RA8Cib%wB&=YYWsFa4>ALWvPJ0 zC=RQT9oz|>wub1P#s>A6<6Wrl3o^HTZE*%E`kJB_nyF*-eg7vv>>nh3X!Dbsh-p_* zPa_VasQ}W*4Zy&>MKTfcZMff|<|4#BXQ>ef5{BOLsF8}|#c=eT@ z_(hDSP38%fYDqKtmZr_SY|gTcQN+4JB{X7?E(+48t}oOELP|Kvf3z-4%H=Q!9X2(- zTN%E|n#(FvI4i5F$zKa`?rsx4qqn%R{ZdD?B^Fw2mH6%SeEQQ$+hc3~*g$dB4A_I< z9xu+`w!JVh6q@-c?+1Ts_GjvWNGeOh<2#b%IcFzpffhvvZblnH!PfQcHr4~%+~>KB zFIpN^5ViCXWc6xkP)IE~vWd?(f)%mT1CxYvi}zGJ+ACP&|S3Xc+vI9&UfM$pP$ZGKh}iIfYQx5!?9;gf$!Fj zU-c8ppIUOBbji7X0~GThO?mRB`-Vqt&5z`YceO58%|A!kM(6*taO&T>)>eHJN}mKr ziRC9%Q@z8;vvS$;$W+mf%-U-XcXJ18~1jtSIslL*Rl z*Lb!TduuI4Q`m3xUVWzb$L@}&SFgWhA0Cj{v-5jq#4k+gb~u~dNACouF-jz>Goa%1 ziHN;v^-+6kN+^BvJZuV~DFN5n4y7PO+I-UF&I6`D*7|3Q{zvRTjg#oep45|2=?Vi% zq&aiJQ?w8=4Pya$D_k7iYdyH_A+?)BY1=~_6efcz=8VW$w!Etg{}uFqv@hYOEdN8^ z(2X)rU~)C%52qm+5+Ss&(2G?u-~|mvX%Hutq9U9Sx$>>6;E&9Jtb5 zb={9|vai_i>iX(EVsQ>R>@wHFNfc9ulc9u}pc zEQppCCJ$!KUu^SAr>ZiiiYI<9y;JP@SL{mmFR-}vOR~RG(EaLb4l}`YSXG#}2`xVJ z^pWLLFCp%w)!t>;RC#_$HX4$GnFLkGyHXaxp6 z)PIsU;6FRa`;XMuocbH^UpA@hnq4(Ho5&D7iC{9X40EZ}_Zym8q$z^kT)S{c*d>_-s>l`>e6E zsU=Le+g5Z3;g}P28BRkOCR8LtzgE98^!KDIV^t;n2Cz<$yWWgG+qD@m=aqhcSe;pe z?|RKn#oqvjo9vEN?TP5l5(An|5cm_aO;SD?Ich*Fak`JDu>Wd}eoqw203OF0oR&g# z=uVa`*AwOaJCT6a3E$TU9?bu|pWEwwjp|L`!nJ6;%az;DiDJnq_pGLO5z>iWauYNt zF4giw)I)O6g|H6xdS|}3!v&xoU7L{%5Ekb%2#TXi4=IxZSCGIzt4@;~sZsO%rAd4=o$ z#GAW*{4-w#Rt^3~YW{yI!v9YJ_~QC+@_u#n=G?22)s9hO<;hRp@!AO$e{<^p;tC(- zq#xgGUdiYk*RNwih-lfrIiBCK>d(H}f2^(A{`zC9znM#{pW#CH-}S}rZ^!;&-~TYH zEW_x+Uj+T9I86PYC1^c2eN+u{>3;#g(&{+58~1^3hw&ip)SOoUv9^7Pe$Mxp(%*na zfl|@6<8Ys&#&3VAh&T`Tv<+Yqn*TKC8dSCC2e)JWRFqz+v7^}xg*z*o+aOg#VO2oy ztgcyds9qd`9ou(;%hpzjRg4TkicS)(e^DfRWKp$pS&wOnRKPN6e; zyC~c>SdxQ+%v3mX#@F{CNRVg_guyQndZy}qxkr-}s9L4@Mo=J@M27V}KXVW;uN0@W z72HhPu+Tr)L^4K&dOlZi#vL5o>GNr3f*N|t>t5hHhk6bZwl=J-WJg74EHjk|&+*du zELLzN=BQ3QELu03wuQ}g)mx7y?mJgPd9LQLzE+7?hIJ%t!2H*OinSh_%WMyfdQ=i7 ztl|t#E~8_i!RIW)TM~6s27UX0@<*LK zQbr-m-iC*kk85Wr9x3lNYxQCUk`-u;>?jaxGm5Zpl_t(`?$Y92?6aL3gORcV(NK{! zodPr^_pN_M0D11|yRVfMhcrwHevML?67&&_aKLlMiv6br6>ADGB^Z9~`@1!Skf1fp znV>bq`Dfs-`3U@MT`A!-4jXqOO-8k)Axj07w3JkUh=cj870CPOO}*}j;-IP^4<0z| zxn2MFgmN3j!0swzQPH>>)WHT?8DP5RX(&fd@hQQGX!ze+{$F&d9gg3wa4wAgN+xMHTtaTT!{*3}q%dWWUlcNQLtgoFWr zaZk`19b4vE|5hy2eI_Hbn3-8K4iu0{D9w8A%#5-V;qSvd%u$o9{j|SU>0k`gCssEu zckbB|8iWjM3>~oXqKq6Ic;!uTRMzG5)A>o5Mn{S03k_L4tRPK^R{YtTE$=-7y0KC} zS3#E{cNPwwQz>08%(}wK{I<$$ia_{$BE4r+o=4BnK&=f<4u_&UUXgSV*jk%|8j;Lp zCqW?!P+b08b;p}i+w?#I#-PKZ|nX4rJnJk+vZ zon5Xqu}uZjJDawpcUjfTp-jbF?q=t`U1JMZOTCDOGS~Ne?-AP`Y`H1*`du;^1WI=k!>Q1&rWitjc~lgaNVA z`yaHw0ct?nyU*X~z4pDaddlzVMBgTJ^Cem-Aa#MAK=Ab5yFQv47~90jjdzNFf0^Ai8Do0pxcN(Y`Z;AUuQn3 zB4Vlw7rVmOvMeq;wj2U6S#_UMi>#i#1CY=OHScswrkI@b1kwbFEkn#*q%kvNXN~pMyCU4NVr(RS{K#qRlYcp(BK)pBpvhO1M^Ut>&&OWNpp0 zQfs`4q8Me=-E08(Uk>cH^~zSrw#h6GJKhs3Baa<=cZJ`V)XQW|RH#uWr-G9)2U7JL zI&zXT-wS%t-1mS5+gS&tnRLvoc*}|Jz&^S5x*eX0p*umEtVkLSxVN|K`{s2oWIL%U zxRKHfKN_hbBNah%k-R6~oM4dXV-q2L%&eQ!0(<_75#Q@(`^E?fjO<$*f)|nxfL<{Q&z`D+-b>8+BFHGh>^886Jh1bcM$x(0!9b(ZJ zP*3S*?tAgGaIh|QNObFIa+o^^h@`6#JEdx>1%B@~Cii0Foi*9302+mf<#f1FboHoK zhHcwFqmGoA$(J`hTiKK)<4bevuvfHx5MuFo{sw6I-uwZ{BF8J^_Kp0TgZ~%U{cq6x zH?+L}F!_J^6hW9Za56|J*nbHw0nJ3Co>Jc{+j@HE_%?!2yCfH$+#mueA!|1Jia#pI z^ERO&gskCTl;20R!w`B4AASx>tDwEfafW{1_3)K4n=wzOPsNpz zwIj(;ZPI@O)<@XDh^(w~dRa=PaXN>v@%MN*c~YsF^KYqBKLm`Oo6^{spKhdqF2{sv zx%}Lm>;klaSc^@BP3{ZL;u6W0c!S%*0`2L@HOxpzvn5pF+214?EvhZ{67#8&40YPW zJgxL{WVMr@9lLsRKTM(AbK@mn(%*ldunpAGPA5l-h;p~#-&+XKW zT%*4M7)!qaxQ>NxZT5JH+3vGgsexGVfe0Q?zXZ1CLfz-Vr?P`yt*EBt5-^Qm4KdpN z8iOy@(4Bc&zI!YTvk)4b)MLu;1}Z?@@HnU9s==bfkF5P8;Ta>MBjW$5)PGR)))eMn zHT5q{z25yd(0}@ui_26bp{q?7;W_IHOTb2?9sn8kv$c1SJ@}o?3tP z2Hgxd65xK1i&Y^{H8vK5w$ywu{XyU!Um@NJ54E^8ig(WwS(K{4&P2x705R^FQ1DuJ zc9Yi4I1-mdag|vOE-_I!XC0=tZxmZSv6tv!eKzI?cVXx*Im#f_zN%R37$KccoDUCNDb&062S=COot{wv?R&nAZfu@=<4S6d&m zxL-4glQ^c(!Q_q|ai-7R^VFoUe8Sz7;P#YY)J<&9!k0&-E$j+&fG#>mceY97*u zwOKRxYgVR%w8q`S<7ozE$>h%Q=;+#op;J3S`AJq2o@3`Me(kiy?*t=nGynIB<=r=0 zZ5rr>r{OBgDri*0<#`g zT8Qq`|1QIWANsq*37xYfQDL-Hx$*WKD`*@9m4qecG!Poap_8S=EEUB|b|y-@BQ(+NLE8>inMm++4SD&sy3N7P~r^*uaLlfM83z& z1R1Qnn8t0gI1jv7Lc^JbtK*rr{bY7%9w@#achOVe^;<3XVZFen#X}|G@p3?$G0LK3 z4~{50{0>0}bkqPGD(S)n8^fC1!Z^-itx(hkE?aW>Ftp!P<0;#Mewfb0x6ULhIjpSU zA_^1SsMQ+&UJ);X5>HFDbHC;m%OYNe&dW=|DJ5?0i(g~+`MYbaVbPT+`#$iiGb^pm z4C5tYK9o`!H}@@PI=RGqJ^6j?xI{(-S(9#0kx|uO{HnVDUANRA$^WNkb;FeFPCve8-p4McH(Vld2hHPbe+hNt1 zM^$!`Oos*mJcQOzV20`pT5qn#6?+Wgm+>jGLWJ*D;JVGaQ)y6dW!6)clN}N?5ol*wzG%RfzL2+?}h85PED$h zh*NrDSm5k(%qQpjL^WN6OR<;#A$7I9t{+xd>}KZ%O3g(TX0(A6r$fo;l;ui<1?Mjvq*dYxIJ{riAh;o zCv|}5Oejb+C98jB$HUii>#r-Hg4D+f&(Z2vt?j7Zqt$8zxA8Fls>QFo_*WDCBY--l zn^bMveicHc7Zng8`ohOyLos>gu*QfKnKj`q?vC*uYEm23aXQY0G3;}j8wNHuR7)7Q zc+0mBa8fPVJ5=Kmew@&JK3VbFM#c0dQO|zE{Tj_y6(TZ=W$6-L;#9^o{UT<0)DxCk z!eki(y!o7S>{i0@wao>q44#u+V5k*?`;Z2*qfpF_8$cAj|9(%7s$$d9fmI}sMog8? zZR<(a_#nF!K=14G^K7?l@W_MzjlMr-=tJlvCQoKL?&f*6P2pwaT#&HO_PPvq3@;-m z_xw4fpQBT@+1OzB`}(LWn1;B$_JS-lM7Bz9QuB61gDzsT5YqE-E73OsA-+9}etvn`64THf4c210(OPv{(eE0HegkAY zzY~^kuOwSeE<79y)|9n1oQyuf9DD%wnRiw#hp8m%vbZ9{b3hw((Mk&uxS{n7=vKCu z*dR!@@j7D(52mv#W(rG~=z?{6>LrXKFVc5Yoi2pwP5gE&=RRVR zn?^6PG0>}5=+8StH$H7~cg^=se=v-Sk{t>jrQ4fsfCBK5n>S7GkN0?w7QPnMzssoB zZBVTDrebpvt&!8(^o^TpxS*=JFmlhA5}aIEpG{|ji_Polxut4vO1CYt57Y~9CN!E3 z3x(Gdyh@(>6UV%74{^Q!dWrbg^Wj_1VfeSEA6_5%UQh3{kXOuVy%wHDhR1FaAyWOY z!Je)Mx4fy7*<;qIf!_ce{}3dz^>-g?7c>-6HYG-j$a)-QD13Di^yAXC)vfZifz@rC z6*7{YQH+re%PjyBT{o{{4rcmM4h)m`7q<&z&3G6|nENXiw0$q-Q_udqHr6`zPltea z_(O|XHp;?oF(`iQg2g@IUUY1Sh0rxn@zy{E5iS!{ zu^ow?sTZt{VoPAEZ0JtpI*siKn^t=s)KD_53=%wj$nF`LuXdDIaT!~EZe_)tSgy?W zqMp<`_S~o}cQeO0l*#+kBs$Z<+Kx`3SDu=?GYZ;TGJP7_QH~8R&JGWZY`5aA z6eAky6~JcIZ3N0%?Zp>m+jDl|Al*;=@bxm9*QU4k2i@Q2yf-PqZ>yL&oiZ4td})8M z;5_9brHu7KjS!FZzM6i0(2D%)VT%!;B?wOR$fG|U5-$9+7KNOicyq#R?gQ|L`YOa= z$6ekCi?53~<}N{Aed$g)b)ioD$3}aciV*eI24Uv1th(1aR|~1SvX`439*fJ)6)EuK znV94BTuz8ic<+dbdeVUqUF*(l%uEt#yVP!J?{JJ3VF#_u95))QmrM?}tb>k-(7_oGd|UnF(_S zY5AaQ?S&AWj@ps)QJS}>if7&)*p9&eVWI9vf^wF9i#Sp(CcTB5_Gbyc`erxOg?XqmVr;HP4Ov%?!b&;v3*g|`P` z6M5eb?td*$7wDNe@a&|^CT8Fk5nVhR+-8FqpIY)~)$I65+ECb1SAi!gf9({k4Y>zD zn|Zx{nNT5V_VrSC+eXOl$W4ZEKp9hqKb-b+0oChMjsT|gj*1*l3Mc8CcJGoE7m(Ym z=;!y0B>Ym-g6ev#c5Kk~kwiXEYY&{HA>k`_G9Z(|*URR!wJ5tn3f-ph$pm2-rJ| z$^%jhmSa?@Iw?IWJ6sa~{KC6Xe;;W&A2~5GvPk&uTU!{XF$c~*{x{R7BXGuv=c6n*;lhKFO{ z44*S9$Q_djGJzq-Bu4p_=VT8>@0s8uzpzjQ2zUKBnd+{W>=Nphfd^E zFV}#cR1HS|>ztmQ)b${7S?c6!qz(k+QsypO4GC6@8-6S+^=jbKK$2RsUC@Oi7Vd}M z;`|*5Iv^N(r&?^I>LlSO|B$Umeqy!_tnNT#N2LkU+Rsha-8Gs-tV&xUI->rP=gf>8 z0RYc}B<%5#FLJS*^e57ZL>r~EOGMwR6-dbsBwBiv6PK@mmgRGapc_7@{G^UCCE8%n$Hh7JPq$=C%Td1pF9vGA0f=6! zlRWW#I_&NSy2!!Nm|;fb(h^oRqcwHL0Tr|9E9ftp&et@(M=lMKX8e~;C?~PQsFIUx zGI&OIW`&}7#*W(-_|=k#BlsD5vFscjk;c@o9rP%*ctaq9fl1b%ECc6M(<6V+m2U|-;Qku zM`qw2B><%rpA#?gK^yH==swjpK}7xLYj?&7W`-Gmeb2x-*EZo_8fi0hF&Prj|43U7 zA|u;&-1Z5Z6iYxi?g=}`CwW!r6E-$*M~qP@!i|y?mZ3Lf6M1<@@|QVDuj`CIg{`4v zBo>O|C+==LaciG`0KxtIQY97WRAI!1MngXvO*ABMplj_COIusRXV12sj71ZqcIC_T z?86fitpz0qC(XdPtVk0*&HHo8(Osh2K<=wHEo2^0<4ecq=x^%(gr^QRBSka1|CD{* z@@*|R;PE#=*&#e_lXjKW;Cb+4cQ<6LH3t=#{O)o%Th_JBASO0EyJCGrw8M)G^i z?b?Gc%~{HQ+q8=GhoNu`@KpH=U%FPRK*-T^GHD`30A>6o`Yem&o{)Xw7y(I@rj?(& zbO&6V5DtZasL}%rdsfKpfLPLyQUc>}r<8W%U3#nW3O^pMH0I`xR$2|;kCFV$UqxuH z4{vupgQv?^R8A{|yl{7qPo^U#+tW;jEO;{?+R&cEOJqVcp*%P;;)vpP*{TjlZ~z$z z0l9mXlM~V@86BaqU}E=+b%K=bD_e0*a!#AQw2=@?GM;ooe2Nyf8Rce3lNa9w+<+8x zW?V#-V`evd;c_=vjaIDK5wqF$tvybKnlLFwMdq9KwBe%s2JdMkY&$9uImqMT_r=qa zvSf_3$aW;dNR`ASqVp3g29+v%d5lyX6=hJhJ)Q%(Wfj_QpG^P91H|TTEsK)Z! zT483cS`&%0je#&sQ2=@=EQx9t_*d=5gvVLwUJD$p1h}98eLqY;P3D2VOS}$Y?oN4A zYP&UTRlhLDO2ZV7g@&{#^p;oT!fH>LZ@DY0 zo0TR5J4)&!k{lCfD^y$^vxf1|M@b}Vn94p0P^wdC6q9OTA`?&;usQb|5O=2{UGl;; zY4AftLr5hKY(v8UP5iduA7!>-)`T zk4G2TY4=UOX~oVY=(uNDpF?Mxt&w>g_WyAjM;(*=#3(AsBl9{gPdfBjl7KL%$bHno z+=rBLKTJ>O7qK-iV+NhZR_Tt-jKiE518vQjx5lfW#E2B)MJ%{!!@N+z4OSB#wn zKKR@jyhHDWTm1DMQ=3b|>jDa#@|3oHLPaSJy@;t-c%MaRZE1Otv6MtQv>iAzDRbY;*-=2poHP*$R<@E?w4{!ez zI-B6*094Dh;z01~uoM`Lj2V9pM*70_K?7l2I}rz%crF4&1K1(*_Acfxxg3;McfSGl z$T?@56=br99m>648pTw5*+CNHX;C04R>&L;FuL{)=`n@9ztAUw=y~|*9aIt7U7OKo zLuJna_l>&yJnp>CkPlrMpD7uGQ#Ytdu#r)X|PhET&c-2qXq7 zn7r$d3#ushs}$CybcveO(29L+CZ{+a^ z6VJwluCPid3E#^d(vq>nN;8{ovN%eI%{E=yK(>1{Eas8IeDK{^hl9GWzKvM?kuyGi zB|KCqhNWcp525RQUs=g6)62c?CA_e(&9+~KMRIG6oo`AP*6z+Jk(LwrY?a`Ocffu~ znYGD3^txq#18fa+Kh61Se5zNM4mDQlM5~pqZ%#piU}V6f+OKZCdSNk5R}Pp%0V6?X zv94KhlvB?v@AZ{s9TZ5ABt|2by!Avz7>G(Y(o1 zWFDs9e#xE-aAp1gd%zN$0%cuIP~suFS5&xy40}kn)|=C>!}^&K-VL+K=iuF^chZ?r z{QUmG=oWi5-|*3faJ@Of{9hv~%J*@9iY{x^AQwTEm-FU*5c(_gK%|NZ&DfkLJsCWkM{F14|gtb6Pc#^8p^bsdwfl zp)9Es7>HQP{iO^*Y6Y5LYt2KjF5ZRWsAqfkhn?tVH7i^xv2VN1bK(14B-t?okIS;wndazY)VII|YIQlX1b;k0PMZg2XIqT?3yMxvou*R7GDfWXp40+r8l zR2Bu_QmdhTxm&~i&I^o`&dz4@!6HgT#WEFF3euz!^RUqCs<2)k!@`2Xkl_6$Es=S! zdX(MI*(^Y}leIQqegqep9OEIbLKnwhQM8|cl>*;T*zE15qHQ6N)k(E%Pfq67zdVsG z`d{y!nX&GluXQFj#3bf0bz(U%EvuWz4O08aK7U@#6ge;sgVh&#S|h_82m7ZZr-cTZ zK}9P-KMdqIs0u+fQk*El&f{gfFN1Ndnf3~_5_Cd=DuY7KFg_<2<_zsEEoMLcl<{yf zC+%*p-$|V+_bmD_Gutw{(lY=BYIXD&zRrdr(gUyJ%Z8a*f_`->$GB9SvRj9j}WJc3jZ&>st04VL^#4Tx>m4-2K6}^sgiOsLSMYlPa zuUZMSs7sn;*d?w4;UvGlQ8kOe%I`rmL@6U85T;=Z_Z^nI6IEgam=DKt3yN^{dYmeE zKGR55@{tKh())nkrJSLTC~Z8u*H$3)2F3)@1UqHo!eaPQcw+Yq;%iRq;bH~m=VuB? zlq10aDUdihVA!UM)x6d&?UJZ~Wofsqi;k3%p{4xDb7mFhPOJs$&Wh8-}EBC!g_SD_x9wo zmtASG>0FuMofR|6LJeXSM-F4_({=@1eDwkjI=M}mQC|jgl?$A^JA)XtodP# zv~(g`p$^fXP|N2pPHuNL(l8}N_MVEa=?RQ<{S*Y&-nKRL-YuUAM@?+X(p?{B3{~0E zWFT=e2GufPJ-1ya4i9Onp1Y`epXJxb#7Jlm<2Y)$ zR(+GdFT~jsj3lJLmzj?+vxYwEZYyr^`ma*`Y1{d$3tn9xe2teTE!Bm)h|Q04llypG z_;uzg)tuH$D2S941Oif1#a2wl6?cx`L>851XEK|+SK|h}V!8rLIFf?lO02}Wg$<#R zR5Nd}))>LSWREO0dve@sUiRx>yUR97%Xb%!@oRYRoa~sE_@5Pw!nSHz7`Ql&NG}rO zioRZgVpdOPiVg$?MU)+lZ6a`rfI}X+@}`)%xY!-NxPU56X;dBTg|gfsYt=6C8}FGg z5GLl#=2@67O1?Nz(#&1=N+OpZHApaU3mBX7r9a{ z%<0KJr$N;54d408seR2`v-UlyGtG|iHv+PSVETkUp~!v_0~4fzl!|GN2Ug|UMNWts zT=RJF?)im>V)JJa(IqmqaQB%GiqeQh0WVT>(`dSglJ@KpF4@S>WSr4LrTQRf73$EF z&4az`@kvT)Cb>SEFg5KtwI(B5o<{5|(lUz#HPyy?Q(K(Wu+$`4o}>~+(3uHVZZ@xb zA{jW|1gA%eE|>wmMqjb{&{HUk7P_e}z-QBhBI~Ft=HTP^59X8%6zDo_f>9YsZNA>m z*-|%5^zZ$APZn8D89H1t(oC43m6ogXJbL5reqn!m@h|DLtKr5hwyTr7=7Y`0 zXffteO1noF?ou@JPn$?S7cyS)$X6V))`A<^CuuyVrBNBSEuprS!=hthoNw9Zdko7T>w=m;E2 zX90W^?un@E0;4!k1(+-jP5#c2)s>#oMmd@qNtoV~c+==*h4s9AGJw##vsmGbc-dtf zXGM7rep8v!Vy}2Zi+PBS2U;l3XPsqso2WO0$z-@7+8prliuGX_a%Xc)YR2-_X+@`o zf=0rKXsj&rI=yDmEHhadpZT3z&R##x<eMb7EG9}X7Hp#-LJGaz5#z7~pzy4ZmRWlL;Pe^!*e*Ma z*W?JW5F}(@?r+eeXBwioZm^cnuy{@y%xj{Zp;k@s;Ow1|)lZy_1g>gXmx8nM;Aq&Y zPa~Lq&I(sy_@*2lf$I~a0QgFQE=?4!A-```pmY!=68pK*QZ}6L+@5WtMTtY%`Nmq< z6f*0e=V+pKqt89jEyu?}t}t|{(ja6iqTO^f1?ISR=DsoM4c5==qx-iI21wolX5?~0BxpzsvXJS|@XBL3Q^KteDh zQtI0`mBCh@yjWS>Y_7FtRQ!@rVSo;t`(kD#Cde|Ahiv^DkZI?up($VBFcI{zB>S8(rg5 zEsE_$z~hVD73<8I;18`~p`b>mFWw86F7Vnw*zJzN0URN(98b|GFSi0pnZ1-4G1R0^ zaYbSPMlLq-0N8TtnLVNOTBpsw>X9?L4JrNl?hyh&Vjldi0+!00<+LrmM_kiyF&-$7 zvn3T0jbz1}Ox0CCLHIj{}r1g55;y|}L?#nY;hZd!gd#MZDYThiZa{-rtU?-`g*k!F?geQbAhTxg>~fSPtR~wUM-D zq?H2o_~hvaFuIj^HXgS{GQlH8%`(10*k+kz+QOu2E}I}a^JVA%IIw#m8j?$m$VQfaS{4#{ zhrqN5)`Wd%P4pGL+IPXV(!t@bE&T-8E-C%Seo<;70$Vk}Bz)&g+nK0QA$lR4Zkrxi zT&C8Ic-euXBR&X=T_Rbggr6XmfemAI-wC@NRjsMatVZ}AlSYY^sg8q~l%XJuK`x{j zo0t#(jRU$fI<#k8m+x!6rbj)|*Biif#TZ0VO0DzlEZkBq40X!Uxw?`PG9WhW*5t zeuyV0VWL+Qo z^q7i5fj7boV=Ekfl8)EULg*g4`zFi30bycdk&8i{SU51j2?*r;_F7>eXT*kNZ6-i5Aq*^F*k3rPH53$B;ef@xj%v|-k!bLQFS@sC^jka{6!AMUH5CqYb4pW@e{r){jK(OO1CI zWF{?0Wo~)G*&AVPTxeaE&?*Ya6d!xrh+eezFp_b(?5hm@Fr7cXYNVrl1atT^LujYvmv>;d5Eh` zYl~I#IoG}1j3L5Y3=+9R&Csf+cd2KDP_||v$4`Qsn(y^)y*WDHu5N73CT{DNZ-(hH zA4s;w^;TfY#`bhSXNZhJj;-g=9Fb+F4EOMjVHVg$WEAX_lr)5~hzI(JvGz@r`n1)m z@AC{@ztWjvQ=?$l^Gi2JnCrDhS10hd3q*g_f|>o8U-u&-XN9y8g6|oDaMfBr6|;9; zaq#^tVcc)<>wLeEE^<$ZI?F9=-fsXs`c>1TP`9D!V75L(UO?Ifl9kgzVy;SoRt@PJ zH*BeZm{aEkKTrXP9kmR)T9J*EonKzf- z5}%eNvkkWJy$4#%I8(>h4yXRnC`>vaotn;XKxbzIU!%D>h^bHmny+t=D&#zDC7Q8k9pkHZL=j7mgJF~8hn9)X~CSq1|P zKV_(|I~t3JLvf0;`MLC_w}?CC!|`#^n~Fr#xSd+iwS_Y)1WSzb`Pu9vc^5bV5|F6w%`nj62JJA7$DzLfOD3K#t1FT?Oy1BJwEXU!^ zw3x~=AYotl9vo#rkA4P*qZ?X2OR)&SSR{6 zI^Netg8NU;P%@K#;#3+ zx3%f%B{$hS^7G1gY%st>Q7adCjzn6u?vw*I6Sa2c_U<9e8-W+PnDqtEc$!b(eF_sT z(bT8g)}3u8fiO=m7!_5zj9j=DQsz5R$DKpgnp#LlbPMfa7sfzz{ewbE1tsS;9Bh9|@5s-%-6W0BR2xf}>QlXaYCLK$ zA#6=)wS8$VcJa!#;>Wc%66XTvBN9s#Qpa{ns6Ed(VIslem_ne%7)Q4TVM#5o)5pDu zP@a2VzL$Ld4bWJ99C8Aw<768#tN5ylymv`=-&^q9dax-(ToYF$N(-`w%+_bp`LX8M zz`glg)p%N9q1-8wFtk@%qLu5t{p~lPQ1{1?3Ff@V#ky&XF$AYjv*0Z`wo>|x-X5$h zi`&xnv7YhgCFEYmmg`U77VUS^Z!cCUDu?Hx3qxR*d|PMJo;)=|`(WeIyu7LrpjF>B zi{TzUbTETv198JlgUx&a%lELmCv0tPiMw;UDj_IvV1{?Rrb|7$NLV7ByhC+}u@vtS z?*%+qsE!mWfP-C*c|ErQ#2Nvz3wh2lnomP%4|3FAt}GVFf>GEgav5xxlDECKbs=Bc}o(Ek$%DfiZs*so^(dNA_q69R1% z$v8~YUm)Yxt z>iheSx^jvUzjv^tPW64DHdK_FYv1ISw0kcqYxuQ_>TSotYIvT`v;{K=JQjtg`KeQ! z9ZxL#ANXlbKDYb-0MRzu({iaGfMP?G(lRMkn_(-~NikNZkAwO0S>$qiaw2{iUx*iE zMSL;5Z}+*R$J*ek`t+9K7NsV;-KYK&J>hvr4|s6Emn!U#8)#39W7gt{_VRqSrDP@& zXMe=CGO(%!>irVo`Yv8AqVH)o&*tuTxB`S4)?%)idV+Yi?p(sMB6QosspokM0+i5F zHdVZ$Fqnr9Yd!72sWA;ey#;h;9%h77e6Xhzs(dmhU1;bx4)9N|@w{i(k zluBo1KbtZf9(~e-<5q`jh=NYj()6PJMHzn5?I43#+7BOdj91_t&PwYkpIK?-Boa41 z=^f)6?IvRZA?aM{uEz*UsNM!O5_h|VV9gfBQfkFp#o{WauI8zSm$0R2NYJOGwHfFuOAZ(xp~_6aiD!;e00E&fgQysVjZ- zN>BnggYGP_r1+{fHm)zkCc69Br1GG9UT(MNIGCRi(py(WC`x*Rl%-*gT_;~s+DcTD z`@$gUghBY1N>NEjzMG_)BM-&uEF}U30Iz$6cH;R4aRp=R@Xa)G0`cZ<1kDncq@VAB zN6D1j{hd3THc*77gVG+lyCzE4IwVjqtdqNc7m@%-*aOa0yNZ>S0lgpvO-zyi0BS>N8H)=5g&<0>gsEsk zwtiKp9lb3dm#^i-V};5GEW2ryCIJ-N6EW&u47lG;{{W1-qW!zdQc$F*q*Mu~xyXaO zSG$kB;(s?<_ea9H%Yk|oLRDz^N?V-l&$J2spM(b028MnP^GA<&sPk$)h3grM*DAFE z#|?S_bXAgcRGx=1LWI|5-(II3Y6ugt=07JdTx}l}N!dK{3&$zV_%;pBDPX z40`ZzQf#ef(kFLcQ|Xqr`M07fqOq+#mZWdj=XcoHyaqo6C)&$O(xt^?VWj%)az72sQOAn_C;&*pNHSzaW*~LDb4G$T9zL-3 zDg)PS?doRP8*IzSN(2O&ZW?q^z^FC%wISjSpTnZ2O?Q{?_}?#wl-q09Y007+*l()R z5DJQ*4O=O;R+7AU69D4FikO0CCybrJ{WbEmWUWg3S7n(|Y^@nasE@)X&_&EQFDsao zs}Zz<0aRQDi4u@7$TCQwXhSn;ScP$ZQjiptK-x-^yLXY_bEf@&dd={q;7Ww=F5>Ev zD5LA%?3+1*X*Ug(fTSgI5J-gy*GACvXaytB<#Kb5nzjizzw(52JjMS2mF_Nr)c#^L>Eqok-dfX4Me>ix(vzh7K^z!& z#9`Z)8ky=1?+sxp;VK?rALb6ie@m5ZBn{ne_=uuF_`T*Y)G-Fs6n`nBz!aL{^EYvm z`HY>x+YQ5{_pwS6o;}d=ErIBh*Zx7=p+i*x{Gs-|l_N^`)6BrGlubAQ>xODlPfL6o z$MTN1q^x>MPfwCqg8=~SF5wZi1FX-&UGeb;f22oE1E5hmyOKQe<%2OJN}lJV9^?+c z{^sW>56Z6hNQ1C7g9^75X}ixaoVj}GnOgcw=t_^WI)2y8=4A_5{{VE4i|YzyO3DJF zYDw>5Y}+KBrscyAy&J+jjU0S_IFHR9_Tm*s4q-&`YBdy2S%mI)w0tfCmQ*3RZFo}+ zMR9aYCGM^lFp}@$R*q3Hvg=xeq&-UDYVah)65CoZ5H*)hjs?_JGU%RMlCiJF-Ch)8 zBAxCbQC5zluEJW%6NdtJx7Tn)Lxzk;i@5Z_R_!~Uif%A9bht(!cQ@1!M!^m_{o=I6 zN<5itxV3vQoJdT8Q&H!A7iGY)bQm+_Y<6-KEZz#F&~!UxQgXuqs8JrJ@|>>{4|Gbhr}S(18qr|A_J<{RdFEhFN1iFBWajTC&CY{9MczpoFI=s zLym$IJxLztTXL*L<&cZcHs`>#Ry-nOzhjY_&0#_?fTKEV6yV>r`>Ec0+*`JU;s645 zFay$}iI+2%BhrrVQr^kkQCiH*gvoWkleCnD3xplYtj_ErTb-Pc5p#g(f?sT~vw2|@ ziiq?2!@0r$B-GKRD;?AbQJFpe04N^@@k)iOVOsg$8ZaqfJ3W5KQ-}{=Sag%`_fM{_ zhY=e3>!W~-oSGGO@mMDuY=Pp}WekUWy|H09P7?&-nvgeX*V5$4-$G>0&I765hP)=A zdi{@XZK^INN5AMPK{$+15hGEvW}1q=ZFy^Umuz^#kZzlbJ5-1;gQQZETE43u9W*AR zciQ0J%AC+CQtU2-@69ARZ4Ac%DXE-b)EWW+_QK-Bsaj1riFiKV#CYCFIFu>7)T5 z&v}}-_oY_Vw_}h>lsHWEI>910nPR6xZL2F(uY7RwyP(`cyhNyL;giE)=#KWGqJ=m6 z$E-e!vpXL(pEFB#3cWQC%OZ&%n8$E^Ez4p$W4);7Y-7x5C}wA4=h>)z)`#5E{ZZzeIEuX30m-evJ%H zAv?rtK6in1CwAXmD*dmtdlMzva#oBNc3ngT_v}#X?k~$bQ?PUWmXmQv=Gh8Dh6pA7AMC8FcvVzc!TjdXS6xy4QUd)vb535-M3J}pqH9g=kRtMT>V-XjoDQgPve zeNSG-)P%tYNd?&p?~|_N*Vnzm5!IGSNB{(=4@gowv?pL3b!q&yOah}6C(Rqi*lTi5 zC=?_K+Z%qy+JQ+?&|%$OAp4>35!y4YsWBj??h{ktHKx(!c$>F|sV9wm%~C;5-yKf_ zu(WHWB}W2COu(tra<=#JeHA^&MS^>%lAHG_P7_MKH`5>~0)#~<2sR~vlTE?zYr(%!{VceZM95kaW za=f9-H$M;noFeLhiUS%INY%6eCjbDL(tSRmzo4~5AvV%XLWx8HUlp(30&Ag&-ulK= z^COCn_AQnxPF~VVjD#gIJA^}g)wf{^x)^nsrk}CwvlfBCE5@saSS8#^iHa(siOlh^`r92~0Pk)u> zG!M(~{{SZW35`&m`7W3(fvo$UL5Wr`MFHHQL`i|>bT=^R0pU`YpepJG&{nB5AU6P= zSI`M`N>)E4M)vu+bpCH%=L$AOv>y;u>sm|EaWVo@I?vkjq|e&x;M#k=YQl^u8&&Jj z=oXM*Bp&wUp;`gnTV-~W{iUzu*Wm7cAbkKVNrd$WstHbtBdx{!VAv;&f0K)IKB4(X z!{8pUuuo9gjYqnVFrUqHpi4(N+p0PyI0NT+Zmj-qfcQNqu|nLWL?LSHfFC)Z0A_R$ zQVdMjO6=I~R->sWytr4Dqw57s=>?e5l5|HM9;U*`CWEi+WwlN&=OkPro8MN?N<=ky z4jo~*Z2%>OIEgW}>eYx49V%2bFoxOHyKc@SQx0Iu+ejv*fdD33vQt1CmXwZUIoQNd zHHsjKJ$`p%GgpOPF7oY-Gvf8`b7@kc+v?_90yd9>gGhh?jjijqJR&!^#X!?WKyuj$ z(7Gs<867V$D_)-GhNg6OfcgZ`@ac1G+eiWhI{dB%G7{DDfUN^m*KgYooU%D;HA022 z3Qoy(fFaQU0|qq^d_Gq(SPEzpJR$kyd07=|GA2WOYdlAtw|OSwff5^*(NYa`Jr5)s zRwAPtPnb9D7z!J_vF3JCYKR*{s?t)K-Uqy^%<>5Q!mS_xxT+UlBfz_(pV) zk)h?=Ok-NLgY2e!MmmQV>KSg%%XWRBn<<0&mTI+ipD=*?!$r$(z;iv9_jw2?UUT<~ z`Z4GVes625PPg5p1c(3t1OO-pLx_V``CIo!DmxKL0lJ-AC^8+fr799YdNo9tcTSF@ zM&ekF98YR}(a7hed3efRHI$FtD}s;+jnbX^7v=fB!pwGU!ko>bkYW}kR_=bvv=OvW z#-aTYP|KF>-?poF5Sxflg(X80R5*h8%S#u4#NJkX(rtvN#7bbvR>&RPYWCTgqLV;t z!0p>zl&keF*88G`4rWB*fJY%-j~pu|okn`Y^GwBy*-NUmc?!0m6Gc)0_d(xYx8jr~ z7oYHC6nqskr3SES?4*Sgv<-Kj4k2HSF`xjq%82nDPoqAuSIC1naOp{kI`Y#pMv)_U z(hmA-z2DdDVXL+aU|NP00<}{LK$w98eu2ro+4+lLb29hspomhHX<;yvI;k<@nzuVN zi#fC^Vu!2IFVrmdozNg%AQ+(-k~*&h#+GHHfp zBfq{jHlo7P@Y`os8-^@PDphLxF2~ZpMC=m-e(!UrRO{B`uM&1L*;14w5i{Qpf#)SV zIy=jZ868ogzD|Is77)lRxex`5I&whK+>*NnN3sR@5i< zQU}GwD_PW@mmE90NES22(p^-|9W4@OjxEng?=d6uHdL@n)fSlIb#J(VQd`)FY7}8L zaW5utx|uGdov&*nv}59GX0f%ofm?noLc$xo$qt=MObFdZ!h}`StHkK5gL|bn84bX& zQsWOBa2TV69ZkF{mvt!4uAvK1;Muu0)B{i?rqYdQ+qr5LUCsVv2%5Y%00wGUvLj{y zgn_-#SW$uj?||R4<0fe5ph>Mmj#P1!nq)NCxUI1*kBM6XY2GZjUNl}k@1U@cSb8gG>R4Y1q`z_l^@pbhN zg9XVjVav3D6g&P8BnyZ*e2%t29dWuGjKp9vsIIOJo3@IuAw^~@q-r+@0V?qx9kthW zdf~Bn00f;+uf^_S;Yu|0QnT`=Ty&Ft0!=|a{{Zr2fj+{32gN#STpaFXS~RIC-y2J5 zcaxQbs3^^8(ZR603(x-Y(Mmw@_@o&X&`$SKr*B|HoV~kuZOKAOld`UfMbzE6n{$Lg z0+1=kd8>NOAnNNFe;PD9RbwY|m7D*tVJo}s> z>0fjI0PXWL?WBK&_Lm4Ri2JAi0B?V1jUUW?s>lMb>1h54+RSbq5))EfW#1bd6)dY# z`khos%FL*UO*pvWUNP6HZHAMb?QEZC58m7uwdJS9aQ6*5)Iei-BdtQzk0YOI`c*?u zU5}OQ{?^HMnkL1-R4Q$%6LCFhpLmhQs}>C=j)})z!i8@QHr8oqfP6#V<}TuHgp`&z zIMFmMHw6ny&E#^~`)1x^(pEN4q?5fQR30JN7%ky)b#bKjknxV)>{Rwu)L>GsTDld4 zpGi1Kl&FJUL>Ld9NUdPMWc$mO8IH|T#b2>(KC}G>xLK6g%dV)mN%sC4_?kRX+Imd*2ciA}Z{g*KMB))H~pGJzS`m20NQR zy1JW1Lzix@`3ksXfrMx@B>CPYAR$UUQOB=iwjT*qVYw9)kWzYC$-S&u7UH}<%%cZ> zy-t@E$wEi5q25=sxyrz9+t>#5nJxX;!NV1f8&TJ@2Li()2}uuyl9DwW4z`B=~`L#-jv6fUFG%OJFg= zWIc|GqcKs(xTo5PS;)7idr!n%%YKhbtT@pjv~_oEl}6W!Gge*hCdM%L6V?*bBm*^h za)~?P#8t%u0o5d^)Vh(vnI$rzvVQ(dg>pnLuHj#TTo!fJr-BdqWcnC4_+E zuBwLnihU(lYlhg>!a;XHSsPtd667UxbQCC55?gld;?%hqr&cLG%rFcnPLSEaad*2x zMKzY$Q_?a`$3PranN+$X#$nZd6=7sj$&{JZ%|J;Kp;r|IB)7`(RmL|G&=a5oxI>$4 zQZRwV#L57?d?U{0geXhlBh5E*W|T(2dOWSbl^vw@A0wzI(IPm&Ey$$->!ap!A4owF zFnBeR08WEXGeAf&GgiK{2q%F)=gwpih}5MguCmKU;%*E@TX*hYzNw;IgsDf;8f|nA zFUFD$XM2ZZE{j=(Ed;?41kTqanPlLp;Nk^yHIeOON$NCr%3ByuTWv&1n3+EJBEr&; zL56z(f!f3(`VcfW=e($oGoRUND9fm8`^9_`*zGF|zkMo5B_h}bKu8`VNp;I4*tdqF zvmMX2Pfx_{O91|2*}IrgolpdDmZ+?vY@gkUJN_@6wk|_6 z*%jc1RhUh<5iw2f%OcXKkgQ&B^Km(R%a>{fn$F($OSg2_Yt4TrGRMSVi0Ukg)QRC= zBh;iNUg?4%>Xo$2jR@K9^Roj-!UrDxJDw6^o?hna(X~K{(dv^4FaV&~#V9qcI)2vR zP>e)+$aIoZ2AhlzJha|T)DNVaHw})Y2Und)^b@Ai2f^1Qqk7_rRw4w+ z>uIR*8}dCHYBT(>=5}xH*Kq0}RrP68I$_E93q=QDZIRaH0u1`>bhrsH3>|fc#O8Vr z-MhlhM(83>JyF$y+)oC&Zm!|4L*;u8nKQTdhxw^$d9KjP^mR%sdD)5>HMH|_+Ol%Q2=>PWA7fyMdvX#MNV<|P?Ga>&yWt<&G4JEZY^ zYSy)qrKqV!q?06_QjWS4P4AdaUK6l?45dP!NuA~bX) z9YnAmN|uzX7K5u(O#|F&O|d_gv~6M_@NKPSNl4j?Mxaebn1R-YH)~gyP?cLSF~{YS z2C);b7Q$3&l+(k9?RP2y+EI(VHu1bcw354~MQgWq-OW(j!+}9^7bgw`C`j6sKm)&B z?ufB%H?gEagYd4c)r@&`r4TTP+GBTf>nU9xiE3WL>Of7~`^B(R9}l;SWj4**3sw-N zm?8y8+V6Q-a12#AWQ`9EHn-tE71SFb%62OkUhR(-^6Kd4ClREx^y{v3jwEnxD9LSW zF3|mybWur|n|$BilXT)-e7Uy?rsDZLsm#lu)6Pe!j$OR z<^+fV;8|9s=qnv>f&g*Dv=aHVskca2jn0PecHH6d!y zGss#S!O9X0mrOv{Yn%9Gt80*qm>P+B#EiM5CZ#1@TZYBMNRGcFT?s|80Va3P*y;s* zMf1dSdV@Jz2EbGq#S`!HwIZkyg*EAI$&4vc3;`f`hrRM|)Z*QMm1sNmxyiJWnIcPy zoG@UJ=G&q~Xz<7~d16McYL--jot1EeVm>Ht0ttB_z_ui7?sM(i5d?`mYUo6U+B^qu zTLJZ2!3KCZx3#xTt;^RHg*E)a(Ovv69;=t%_?pf;NMEg4U2+Rxyp71tXR9vHv$Px5++AP6D>&uff^C9-omh316^Dz zsyyWpJ1H@+ZguC>8?wgWLezXc-gYb{LWGwIQdKT$6W|*3w!#}Yw#UQX>=Pa@>o>qK zJ?*?mB%}f-sK>dw0=u};fmIpMLy?GJ*JFx7CTZ1*t;jnmE))(UQU`Tj1mdrR1nMM` zTlVI%*em6NK$_GavBC|l7MCE6d?f9S(?hZ81nIqqRv=ZSUa3JT&>jxvTP0fb2G@Vg z+pV5J0-wcOQpfeD%l4Pt9Zhs+y`xU#jaYf-6EjHvaJ zQj?7X;y*t~Dn|oW#Wtd(&I=vf0_hxqH zce5Z{S2EDn%Ic5g?MdpU@L#I-XK!TtOavudo4-V|0Y9*GQ0lJmABHd-CPc{5$~K)J zNyF#6cFDVx3llb#Kgm@Oz8?YEe|j@nNJe6FHnvaoss2~y92L_2F3xi&+SzNIi=kmj zDNy{#ByCEx*IhRk&T8V(w^B!stwjrnb1kC~og-$(#oG$TL7_rAU3imN)%>P- zis2HVJ$>%h(3eQPS_Nbm@b`N4ln%y?dMo8-zUoHZhXuRmJ@S-bkZjYVu{+TUxXX4As4a zJ>)D@`t&;rma?P*HV1f7&=)4i5amyE&6b40mRgNqdhmK|i$hwMI~Wbr<=2^r@Tb~7 zVCE;t9U&+QF{_gx2EL4~thy9NxKD`dr@8XyT9*^V%qd$QSBp)LQk*T3aVQn&V7>{U zdp1gFt3D$e+KE{q!%c-K%!INs*v~Agt?7{tKte{=;7UVT$;yaawJT$hZp?9V!e)Rs z(`n_*#H2;#BvysV14%sr*(t4Dj7d&_6UFmwvz?UYVXQ;4R!+1XPSRGY*L56}j#^}- z67yuTn))eGbO5Z@j^^T)V@{`IZ9q5+sUaeQUQQ7zOq0;uNkT-bNpvbjb?Vi?Ny=Ei zoL?s`m**@YS4ierwi1M?2`VbTcG`U$^4Zk>v$y`2O9 z0ORu$)y~VlB^PfLq?^~O0YAOD{-@O_;Z3V<%9ZtcotN-_)t~z#{{T~v{{Z)H9n{{j zeO9jJZ>HiBg&YYNzzUd^WmS0~(9uK}%^SCM=U)Dn=lY*Z_o-?}ZeL0YkT8h}Qj?;n zJSSt)P?Bz*&)w#$*0-C?SI}nv0O@ldr07oARYA(e##8`7ZAz$mzmPzU6#_04!Wk{-Zaq_gu&6IT)3S(d1_QDFFWf!sd0S z{!`{f+sZ)w?d=Eli~j&qwmZK|cW3si{{U0Y7=QciLjlp^U4P;8VpLE4u422tvTZ2c zQU0Sh{{Tyv{U=V>TGG29DMaTTDpHjm3%=24DYu&Y)7yhSQG~n~a&@i1R zHspU%n>&Z$b2R?|(s6DZi~j)I=7;|PO_{I$litixP`pZ%q9QRVN{?ac#SV z2hgKnH~C@WTdsoW5M7ELQ>%T@$i|Nxhy}K!!+{Mvm#&VQHNR%{%L@y(wIN9|6q6pj zTMuSf9Zjs-^HX}{V^X|O00jF`DP2r$ZzUlF@$)o1i6u(5Oig+k)yRPURno-3IKoy7!F0GLP4KP!~T9&oopzC z#?uFdcR7JraU||0jxv%WOh^+Tfj}e5+EUU^=%|z|l=530VVllUTch!hr z5#D2t&aYDQO_S7q#u6t;JGcA1?1}bAM4smWL5VPv#Cg8PQ&KuU*$+et)b}t!GrMF* z*udj2ST?S$8$yWtTgl5LfJrZ`?HsMEgBczmH4)KJ`S^cIc{AMyYp`uLZKv{v{*E%1 zsnynK@NBi^0G&W-*uZix`Wl@>gRFY(bs}AgRupiaw(PiePld=QS1O8Z!{~C$O}x5P zq7opIO*JIGX39mypdZXYFyG9`xT|t-+oDG5d|x?6=^Uo9Iwxj(Kr9qn^i2wmwcHvv z8LPdtg=*&|2@pVaB#p)@DM-4n$$m^Jfb-N@)UjyJI_89;ga9WV#YnqrWJF!M$T>>UX>Zho1myO|({L zsnvG|g};|blSh|8QIoOfc?w&@LonJ>)rzkAy$A-bf>7nD6dI23+EQ(#5d@R9x_|(q zbwEVCSqc4)lp^+WroEq-2YCJVS@21!Q%XR6*Joc=TQhY@N)pCY#oN;|e-mm^Setm)QeF`&nF)cJ^xA;kA`a>xR}Dp%ucRI=w?qq@9#;9)Ri%;zT%K$1nc?cF6w#K%Xl=jo$w6 zlKy~sUU*N`9Y<{c09#|K_x#3#4TbjpX=D{y%BYEqR*Irbn8c$M-1GS5{{Zg!FX#uA z>UwuAq&l9-ASV9+@^||eA36cTKV&w?Z|0Hc+RCT)ZDE=bTq+yeU&inM0C&lMKs>HJ zHsF=)uu9dhcFgl^{>|_CkD2~CfBU{m`T^y3S8wKSdUkS)iGr&|TAKPwG2OY?>C|Me zZ30O3mc7|Njq$Ie{L8PC{yBg9zDxQ6Nf+0{BHO6Y?t&2^1S{xd;7ji`T^y6;XhMHQ`-agwtBCBJTzIPXyPlkvsWV> ztg7N;6|7MvOkz=O{{S1k{of`10P?-hO}T9%I^N128`J*Z=3RW^eG$|fBe%0CoHlZ) zop9PIni2U_G?NaW#;*-B*)Qk^m7bl!E7@R`uuXLM4b694&U60&)RX=MK3n+R@9z08 z=m(XpWN)uCZvFM)#Jj|p)6ho~@pzmt!RP7TNo*BVtWXpIiH$>%mPtraIv9>RxQ<{X zTSRgAzZ=+cIj+%2{9VuF{{S>NsB*Kv_Y3a+XmL~Ke^MWWc@ed+a5}p>zJ*35zSv8B zfnl-A(ZyW>XQz@;-m^~0vk!F qdx)e03xymMpph{r?T;%fKGdCrA2-3@2v3M=?w$M}SVI#^!T;G1{x7!x literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000013_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000013_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..02729e700dd1f5979597f49bb95bb8485cbe370e GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;0&1A6$u_%Irl z?qGZvJyOJB{)Ex!{vQ>Gg#d&e4Ij*KKn{~pHa)^;3a|$4Va?4iEl$N+4Ko3&VX021 m3?_(U&Yn4A3u4iM-V3#7l!k;iN;rVRX1D@Sk}ORG5_$j>AZZ@} literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000014.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000014.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4f812c2e29fafa1cd0c2711a16a75604d325e79b GIT binary patch literal 29096 zcmd42byQr>wl3PZyK8U)q#Jh$?i$?PgEnphB!@w!E~QGynz`0DyUU0e=4iU@4k@eY=GZTfCK;;6%`E?4dd^DjfMT+2QDry0Rh3wCpI=NE0s{*N|02N#z`()6!673e zAs`?i!@jf!z~WHD;j+VvshQ>{e7r(%p}~8X$WcGnw~J4xE{?b`?`n2U%Sn*b5L7^? zA@SMVErhEdEZMj>uo%p3F^B_zg$2OFpd!G-!o8gH#SoYmeeAe!VpsX5ALkO(-r>Qk zz~*p# zuLuG`8rRJnyODJ8yk}JaGupA9A+Vaeq&|iOUHKzuPfM_)(%}tj+WB41+#^Je>2+DL zWA%OtVJB9Wcd9V2&k&`Y%IhkE% z-3K+Z$KTsM*V~658PW30Or!WqEnLryQFVP)Ji6AC)LdNbmc0*DV0l}NCUn+4b+m2Y zavYUh>J&~wjyslFPDl-b*}#;)ovW(o_r7UGGZ}pNLBAMBYOM10iCkXerwld~R zRUNM#oIe&rgi(Rk$}?{d4Q@A2@&EA;h}LvL^1uCo43lH|$7h*z@PcR6yV=1155+zg z1FdEkxdza_y5E4xb^8eTrcG88U{YVm~mk602yxNztZ~GMWdmv z=)xwa_UDDr_Bi?^kIgb-iF({?OgB<4$^O!3X#N8G3aiNXyB^i|dOT?UX<51gDpw&e zhn;^D822|DwGv>dg2qCJ6@SxvkU^K-EU~^;TUaW0zx3Gq!)m(@78!HJ$2sqkiE6Y@ z?og0##y6M*C!&;-c{x!^^}m^XeFOS~il4yUqK}5F&|*;pHx9;Sla3`GpMk^ zZfax7r-v0HH+f+Iq%?lB3>9HW9kcBuTtF)4eG}|=CSOJAg37gyW*cE4~FtPqZMeVO1f@AL5M&E|_ z2CZhBACDk;Tncr~rRy%&e3vW$Gw;t? zUT}FQJVCft#!IshVbWUWp5(Jx7?tll-Bf zDWz$}F~K{w?tNy=3ogLH8*ud;TdZ$}^y%`j=>Ib%zDRczVO{5}UJes?4z}4;LFBXd zd=GxA>*y8H*$H{PRSlab*)oZb0q&KtVEcVy|0J5c5Saa^{{U%a`=M?1HkfPgf{=)N zc-IyLmcwQH2m_9X`^VdVpj?~};QUWS>S6!?T%6QpmINNgKz!5p>2X&M8DJEw1l4}8 zj}Atdq=88j=3)swFx+%n5iFfOt*6vHdC)O`m_9DzbRAc5RQOincCO-UdJ^p7Po8eA zT?wg##9UEH=yBL6pZ%KcS!;cQs#=(W4bR#f*+(@A^*Z-N(Q&$_vwsCS&l`If8CPAd zDvdb0%+9;mRn%E2gI&Q6CpRv_TCF$Ag_@?z)w&L!Oq!>MVpf&C{D6_iJDp~Y-}Sfe zj5=C>?FHZ751DPhXi#fzY;MT%SWeC*-1buT&5YsKznZueTsPY4vYKsq6u-d1@OZ;m z+HalU((E(Q4(&ES#E0O6;7%+k+GLjR~pe4aWEEpSHtfs*ZAW+YI0lE9|7i zAf;={W=zZF!{whxcla@x3&gs{dvEKRzsn4+D0U)K%R)Qhcde^Fm}+8qmmtnu=B}!| zyQWOt(Gj@g(tp%2DS)oqAqJyju%l0q)DttsXuMrw1T=AVbqdRzoJ^zH1kLfUsjg>lo6wsg0>asHPS0)vO4&^tP= z7E%AO+wC~-@6f|>u;}_~U2mj&KiflTL?1j&_z+WT^2im_53`;RZsOL43$BrM-xeM~ zvv;K>AnUL)AhJI@Y`o<9)jYExvo>YDW&3QOTN3wq`@t!*N!X#5cjJ(=*?sxKz^iMo zZRbcq=P(r&4xtVU8#X3qaYbiX*Y7t#ZFMuU+O1>$nyYV*Nc(?Kw`;I;0%EW7??0@c zZd-%H_WhJ<7D<<}SjpR=tzX`9s_kTEf3-(U%x+r7u^g_%nQ?BY(wH&Rx$tx4t-rYwz_2nJK?VvReBra?Un7IhC!jB}3 z>*pAn2^Jh*yfMMmUV6@{k46iMGbEwwkNkg9Q4PMG`*$V^4|so(9iC(9+fUu|lh({H zof4A;?D$I5Beyz~*m0Wcc~l7(R)WAW+jvz!F!}WLjp7<7atT)d2USAXi<|CYtDs*= z21d)hg|Taq2O*&+BLDWmKWo|lFn1a9)8>cLZ$Q|O4vFRU8~yC@hhA9%~oQku+iTbzLaX3_{5jYU$Bm2`D;aK+*s0Yu0&RpEQGe$V& zj~B4|%yibDzI{Q3(;t57{s#Ovo&NCC+q13f0{jUw2pNf-p^sjC&itFa566XPTYrf7 zzZnoqqO$*xST*PRnlHbYe;_>a zZz83Ne}DCF!P#=-<$Ir*7p|4I{kvQF_VvZ9zk)x!{-^FA0ROSY=Y>H|lMr`GU9`pL zjAj4xxR+k8@V{#_y%=+8_(Itqsi4B~e4pG#y_v2*F<<4#Y!Q|pv1+v${Q zXiI;hd>;2V87~R?TY2tV=5GMTjYoKO>!HkN@$VmA!uO9Sf1>@@;9u$+e!N>Rbov|O z|9hRkPIdj&V$dGP9C{M)*M^sVVcGwm)IxsR{^?~B?(@Fq9~wM|GkZkgQ3bBKQ6$J? zCe=%75KpFKQ-P(=gG%kv0eb-1bQ0A>s@6mKF*vVPu@bem3}!$L8(i}}K=NiRsRL22|JMDbYb zKEjh`J5JR#qaCwz`TIs9lg*Q+kcfo}>uf3e7c5B3 z%@u-M6}l4L8MU_Mr@II7V^a)FTDuf(YhG<_Zb4;4xJ72q3$!?#25%w>MIm~A6Vy!R zlj^qP9L#~}v?rpRC{}Zq9EZ|IFDlkI4k^>FICxAWktt)CxO}XO0^-scy}tpCt-n-z z-rk`#eK4+fQZEts=-xkuu#cuR0-y+oAAP>+k>HMLnO%;+0Kl}Nr`DboKIc#(Wp@O> z2`@3%q_?{@P>iLx+iAZZ&K{yRUdB3s>tse-2yUy*A9>SS|<8`z9;oUs6<1k z0m$OmY~t8~@YKlI)X1W+*dc)bNAaQpfE_{wBZn;er-d1I$nZ-mK&aJbcsB&IgWyib zoGaNpu z9#T{d8u_wQu@9{mQV%d3u;$&HU(IT z`@OT8IhwuB#m|7`$Vx$2w%EY1cnfFdY-oOT&MUWe%K-~&D}DuZzVLnFqpgmVbyB8g z*UkdO*H5Y)F;0Q_Lay+~Xk4`d`=`}?gL8a&ZZ+do_YxT^Jb}#%14ODN{89FBKf9q zDLCtW6E;_MlvSG-Ch#JM4e$zAx7wkB%#nnwYu){iyd>_8xFblB8w&D17g3sF(x5Z! z5N5hRc>FI|{=Y6v{=i)?1SU#WT$~+Q-24UefWOeU0}Szuau5RLKDxB0!ob6)8vEXj zxQgT37$N9Jh!Z6G*%nXYF3@gXY6DM*@8WG#c%}EsO{%yOIw`WAcq?YigQmZpF)H4J5vd9W{eI-uFcJbO38jt1O3tg)ZwVO@a z_gy27)Odn*Bf_<<28@v0&ZN@h=)GL{t+G`Hd?ILFM+C^Nz@pBw2ZgHv2IuU>-R}+D zOY|W0GcKR#coO$^R@c-rI@aH2I7T0|3n9)1fe1T>(In^kiX@6yVRuEPXJwf6<+uQ8 z1f|TkJkQH#9$kFhJh`t*PsG=weIRj7$G*M_W%CA021Ax9+kvBl79S}`85%pxcAl%P zmjOJ;_51LxL(VI`yYzW*ib~vME7P5)IqKDDwcj=@8gbA;I0l?}Vn(RAS~vHPm-3on zUe=E!U7?Zp=OP7L6$>DJXX+{Y7TG}t3S5|K@%Tj@UcY>gH3<9eu7hK}v$uWcOvxLw+9F(T8nWb}@SGIl(Szn1GeMbA z^+_0A_a`PHYlg=wr;}Wt<&@j4VdwCqeaCH_AY)~!``Y4K73k&2hE($a*Q7Pl3q#r! z|7!&JOQ4sY|DR#w|5E7v@1sXEz-*D4iro$mYI^a+##d}Yk5;lNNTRLXL)TGTUkA4- zm9e3;ldh-iLwMAO-Ap&gL=CW@Ae;{$*jC%O@(X&s=5H(Flg;X@y0uRH zdYZFBxQGnDC?_azU&OP}Yo8;bCrTG2x+@R9G|nkE$sD-w2M){0{Q@4uGE$x*<8EW; zSnA~s%HLwtV>zesu}3_Ks`9d5@8!8I*LXWp_wEIgA&E_UWK=GHs%C}4Wk6%uHW)>c zU5W?ZUv5+QEXl3ECwM>miKf9$?66d12x-(~zf){HUPd)qaV|ht1Kn0@ z@qzRC5aHgQ+992W;iQ!54xdm37?+ILNbPDQ_Crrf2)NiZ_<2GUF;n)*b(XS#&Bt48wCC<8mLfpA&1dvb1+j;{Lc#@|Q z8i)KP?TAO@OzAE)l}ZU`wQ>%?D6HP2+B=5uGcLk%m3jM3OSK6R1_8GH) z;Ju=YqDhC)Ji#EUtX??|H;{4tFc^&m|1<2~{LTU#C%!g_m-6Y@aH7yOCEVKB#*2|e zFrv@~No}&4Q-E(-|By&7rXU)mw@a?lt{F{uLw#M(J&l9;vR)&>Bt>w`yItJlq&f;R zuLm6W`}R`RHs2X_dc-ETUYXFPv?)G>xeT~IG;*b0cJeBY7`Uv?<*Z*6)z*1nM%{&r z0*!6#hepy6t(34-Zj45L&Zs%Qr8OdVoxJJlS2#^o+Y(=7E#OlWU!;O1WFf3hP1A}^ z0z(KLZQ19h#1ryEqs9I;_xd}#a$75Ld_~~%4g_tPD)I1Vd8KBEC8vZezAE~E& z&IW7m0Qmu*mjk^gOE^+v*_8ldyk7TC)~^3_XDsYl<-XFx@wtTAz(NRJ%RzVK-R_L0 z0kxr9qDQ)I%&1$*_+E4@32yU>(bHrV<$@EZCNt2GC)UwLNT9#S+-wj?OF_qLMUAEv zlJ;kc*o-XtimCpfxwr-piSxEy#1)dtEs!nt4G4J7%i_e~R16n6aDARsfyToKj#(vd zy$|wF$7>BNQMhT|TV|!7_syue8rq9>@9^p5IO5`m)Q=3qc??-<{oId$g-79+l27Rr zIlpl)8~~{E#dLsoD%e&c1iMY`4LUJsF$|QC19*Nif`BB%r}>04f{_2F!u%*Qp-R8m{JrEfxwg*JN^uh0i`}(?pJvIJrYcxbP0sSVo%aaG^H( z$0|LC?I6e3$7QQtXUtQdou0lGV0f>oBUTv6tv4$pjDDAz=0CTBo`y zI*o_(Qvj?t6O#y=IyjKnelqpBG$RcyTwEH~zM01aXK6@IKPMRMx_nmz8}1>AL|vm6 zTyv|doLCkx?k-2?y-bg2pHc?rG5L3}CB5wShut@iY~dd+7Q&`DnBh@lEjWJGPbak-ay&EZLrN0QmZ-#rxg zr~~?y*IZP=Cihk~tazN80cy54+{znJ(c8TT)&f?f9|{1+3UYQlr@n_k8+UV|@%!kL zt|$fQK<#fpl2DxUrH7%hi8yu_TpD(0K`6k3>#VoeBctpD zf1v79&%IbCkKIC~mJlsU*IPj?vN^-7On`-%7+)}g5h~_JQGPX7SFTu0xMialv_wU3 z+*lfi_oIc8iS8o{GooP>lAD^?RKTo{wmVwpl4l~1n$R~Q=^MwkxdJ}PGU&pT;bhBq z0{zFe%;?v7u-dA}x?N*cP0$YntnOH1BS7a#{9%3YHwu;p6Z#)1L+l8YRc+K?rFdDmqYQ8v)5v?2UWPp`B;x*y7mDRDWilsIsCs@BR#i|M2t=c}PQm zI4zOv3GXmKqw?(&9z1Q7H>lHsit7S|Sgs9`KJvhY&Z|pBfp>3-80Xa~%E@4BPVX^S zFCL0=PQPLe23PZUK;{$FGYG%DBH=~KAZx^8_JSNXnN_;Vzp=j|*m|v@>oA4Q%(>xn z!#WdZ-5*T@by3;PYLM%2NDCuO_^h)4nV3w|i~ktVG++sbp<-C6*Dj+;o{B zMbv3w+X7kHVSrqy`VU84Z7KV2(9%n-WPA;tYfQFD02b*Bgdq_oN_yVLewpD8GwmhS zrhZDr9s|eXt#q-+tQjRqTxiTCw;l+#l}9`FV+gjND-pq!yW}n6aKkprJ8&{*SFq=K zwZ?X8M#M9bV<}5nO8#^TI+%Eh>awgt1QgnN-lA{p3z3p)Qq3w3f}c_)Cq+wdCfz4N z;Iu~e$sMP+Oz*nCHhulY>T0Xc^C60SAR^QYx!_xi zlM$Jkm`q+4-MKd86DMaHVf)Keolslnlr$V?-e(Si53_Y zB|`+(#iW^L?Btw1px|bn^H~zAs3>M$Vi2-mS@|ilJ+NLpegob%J@T6pSb2#H>0Rjl z#%D?(Vw+XV^I=yEJEEx!8CQQ<-zUd+gVNI)S#DzvhHCVJgcNi|*=Z>_u&#oi54Jwh zh)vRI!COgEf7!LkuU6D{bl1meh;}_5gDuF*ON(Uv@w&Zn?n0e<%3_LuV`84m1Y@Nlaw{H48Qyr|nrdC%W60mm*G zIcT`|Nyf@zcu6!awq;;IJ1x9TGg1@MNCqORnn7+lP%| zED^Y+*KdC%ie>cq8@+;t=Gs{MV@Yf&f}z`XiIrEx;y7rkRighj1ip;YFlBn7wCt!#82o7}BC;ez-2-B1Wxq@x4zt^C-5wo#>ewRvM&f0d0cmcAtAwuMYtihGc||fWwvR79 z?19hp+~-MA7&B8Ly8Z_5#QMLt&fN}>zHsTC`z1(~wF3YYe`DftQ@U14vc7^&6}L!kCY-Y3@@ zbxj;WA?Y`#?)_Glpm)opwpNu!Fq6&3vXPNSJ8+{8*Uu#P16V#CM4K!gc*XdoENT!0 z7b5i`@q2dU`cKD+Mwiq~@bH;VrkW_={X59AcB*sH`^%kt_nw7>j*LpB>SG7N7$Z7I zF*y<4SC+FLIhjqXN2?${b3B82^0hQ1HXjtUif*wV=IZX*1ru-B9j`v5trGM(95mBN z;x%wqJi`8rv+n_(XC@Ar{PZ^Q%B)|Yo^02RC%BZFC^>>J0=4U=uP(3A`6?aDJNKJ& zy^*hcLP%?twaPgCM3XY&_f+;#H9|1H3hf>c`2|H?ms{K^={Zcytl>1q`HoZo7f(@| z20t*_A;b3WlWYFio8F>kLjm>ru@72an=`lcHLMZ0$r>5``(jKZDsJy7d^Ccccuf`F z?HcYHtKFTYEKLjdo9feXY1BC7x6_;ADak^ggw6K$|2Jwdy}}>q=EbYCCLr)shf=DKs@y z1ZiqHWRi$?A>t-xOO+ZQuVG5vhs|tpy>VsI-#-k=$U77`aIn*pCGMTA46J=8CJEVW zU!@SYf_Xo77lWM4K;rzta9kj)epS}ps%H$gXf#thb3Y-3;s@sbH#uIKW6vTUmX8nX zx_S{il^Y(2-1Y4B1glm%kT6{@o+*z?5etojWX~FYzOMKxfDxlr-;hFNFm$jdqN@sF zJ4&WwqnhVb0ijjZvd<+rYn{}jT#NK6xIp2M@3jkczij3?@2SV|m;)GE?_SfUpT`$@ zWRPOOAv!Zj<;lXSs%(Csk#UIMpwR2yn&!f9Ic4l)Ir0j@KLyrz&>y?9u9q`eg(jUZ zO_Ui$HwD<$8i6%IJJ}`x4W4w5hdPg-v&NA-GCJDV%wN(1C_x5~_aM`S%%XTbN={J5 zh^HR#5=0OhP+Y30nJLygWXCz-gbHee;_sAw{gy+U?gq*Tb${ro5ZKT7Nv;rx_Qg|$ zLt>PdT;Pe|@h-gOZN|Y4{owV!#sG+Xs9|=Pdr9m43Hv*&Zw9AD@nqY=CtK-x8c#fQ zvq_$Tr`15u^iE3AvNek=nMlFYjELXi143|geYdG?kN?OABNOWw{E**rFF467GB;VI{O zKLsn?bMR}{Dbrk@>rP*0YL;e~x@c@mQOsDvnEt7I^;|*t{_W4K?zZmCXTPi#mtBVT za>}>uQL%f)eIg!ag!w+!BoQyGjQ{y&=$ab+h80pZya#4xmbYc!(;*GVh&g!pEU0?( z8Nm?6CgS`PZ;eyz0lL#|9KV`g&S%0GMeHPW7DWa}RvM})aO`NNsFCb;gn1Z)=LeGM z)4m6GOT=FIQmCpeFR(90ep%I2P>sh45l_U1H$_zqIkTdcd(Q-m{f7g9K_>B+Wf)Y| z2&s=(ut%26EEM96xDUarD{qOm+%nDWjdp2iRZ~taYZ7!g>}o5*Ik&Xz6bZkqko}C| zB_poe^>{$qsNf5O)J!WTbm_=DV@}F9JveiJ@6^`g*{>#t=Y?^pRdF3zC)^-k!4u?V z_`$Q^0U|$^3LPwJ9Wo<`o%`i&MryLd1>c5Pl=$o<5Ys+{B1`RKQdBCeQb!cx6K%4P z!Kg!RH^F;s7!H?2#<}$S;teY}AQvM1kZACwdNKAL5nK_al+T4ao$43)1D<1G_FBI) zWVgnI^21@NTuNB|B*Y=ja7#uPdZ`6M;izah+}L@V=@WigK^N6$o}X*TXj|c*UJ}G; z1v!*r_F%VayG*-jI~sF;de4^gYR1l+R-r`&2HY{d+uB*a<xlCQrMIAIY}#CW_G+Np7|?l6a7?Uh`L&dGcX05Yei7z`x49vd18 zccOdx9CS|I_sr(r1tiPiJ`joy6YAR3`e>!!1~*mm_~#@qkWL;Y@HUC-i@rev5t-W{ zhLGs&hXC+wRT<84z#jOK{mi{qCesG$$SzNEjv!CT8g{; zyO%pv18fnDj%Po4m?g7o#=E)NC$tq5I(1Aw5D$#8oT2R!I+-{#v>B{8js|CIZ@8uY zYA$W`K@*yA2*7KQR)_VytNrlocP=mN2Z`g|l)?WA0mp@iXAo6eBR4Tn3&J=@D2n?8 zhS1>8mh=YUQDf)iM~i3C;MUP1hw|wqM`%!U;GnFciCfU%m=lJKO49V|;hADb!XQZyp6Pq~ve$}#N?KLbGJTZzR#d%k@%;A;;?zJf zN$Qt!RpCV-$*?O)tNQh&L0us#H~Q1YQKTMnc2wa^M7jPQ^l|FGHnwnOPNr02+ENb8 z&gUR>$~QdviQ9;uV)&63qRq`b;h(TkuGmpb%W;~=mfic9tf0*<=<2@mL1}Y5HY4uq zj}RFKo5F_{qc&*;&L}z~TH*_B3}OKhIcH97xoDoWlk>|He7`L5FlzRk83AC{T%^yJ zw&eWj{81~U3HlnsFG#R@Ot&R<8s)reu9a5N-ZcpCmLq@(R6#sUd~gxioH%5KO!wu4 z50;!5&X?tvAq+OHg+pqRNH7GBv#2ht_I?tM-hC0lW5P^w$b})+x``eLwADfOZPIrZJ3mx8BM=*VLqweKnBl(`@^h^1D~lJ2_lk`er0E zFFXD8@lpwvch-C<(;MMkTzmk1G9`~{Uz$SCDTz0YRR_04=-{i{(0h^TR~Muzk6pRH z>bl8S!ZBG+jZ->gPBWb7S0$Rx?cqsNB;m+uP%eym>TTseNd3&BJR%;vT~!Nxi^^7O zt=1>*T!-_V2;XGvB=D_tN%w_sQOhJ-&Cdl)9XciT2!`5KDAKgd^P=- zZ^UY$q~YRhFD>y+rStM%ehQ*U`+XFmR|S_pYkhA6X8_=F zE&MFa)kd0B9Mt=6vAUeS9b-{rN)eM7US}P72Zk+XM?qTd4(CJa3V8M(xXyK1N(^V) zn*a`DE_oM%JPbmkfnjy{dt0B~8rYe%FUTwq&kr;+%(h9ZY*}p?D%_Hg^?Q({DxHAv zA?&|kHL>qCoK$SC(R?XZzU1W23cs1q`PhW@Dk>=p;o5lMT4U3LW_2|>6)D3IX-zuI zcc`Evn@Gnpe{G3<$-Xow-|nkD0Lo(QrW*P2+_PAD%-M5$(l8$V(;>a00Xt$7mj;>h zloGp&^!M5iB19k3G2o&{PoSN|c1$Z~&YcSh0}Tw(DUX5HdLwVYsg6)(=LO8q$O$M9 zIC>d*e2R1ZNHufCg}fGFonsfCMq@1uaZqp{V05rf6dsiiCzFalU!A; zbZv)8I;d@}TX3?m1k0A`AV|bIfeBh0aJOP^$LmH+Oum{dI&$W}Hlz@waK=1q_bT48 zkk525D80h!3{;BQQ^5w#aK+*MEzm=vfYR{Co5S$7-r)HZw*TR?uhv$ay6ezs4Pgyi0w?TFFdNELThWmA z^4V$A$~jrf1uUGxW?d7?$f4wf6Eq-J-e1)eD=tCx!v%{(TyTpDMJ21_+mX^8@}4%%VaGMsY9&_0oM??u6O zw{7x13dKBc8A1>IvUl)mJZZc`b&*|eH-OO|b`mXeoiy9Y>MOTEDw#lPef1lM4%TM) z4ir%4N_1enEeUI=N2KSd}wZwGrS4f#~*g* zkDD=t5btU5gpuW1%8%H7e*>1enuzK70OtPC9chFdhpGT09`jpvbp3BL%oU_Li?cQt zJe$u-l`0-e;Jj~^^D*{AsL>Bvy9N%7P98>(X@_imiZDhzV<)HQLS13B>CLC({KoPw zvxCez+k4$ZUbKvF{W4*g$}~}ww}b+ztoa%KeE7 zQIOQ7diI-6R~BicuZpNTL=!4U1)D0r)PKS~^i-bx>|UX{L4TTC-+R3DH^*iWYULs` z!R8<%nAke6C0besGHzbc!ajIf23Ve}uWKP}%K3M!tT~+fncBZ{&mVOBDC)u5Uft5V z{LS4)nM(qejfsEK1mqhmI++Ndpx1;{&#mEs)k|A=yH82^<)Oy%1cYp~u;279r#4gb zeRU|n>YSj3owivkspFrOs6bbmfNEEfp~K4|qC z8QmY&3^iaq&U#}Ek~ySO8u(*7?<74b!};7~Tuv_=mwL_Vw?P~9D7@GRCk*#ZY}-}{ z+h3n(sFtO+LBnsJB^0iNDJySyEo@oQRJvjouk|U-kzfzg5}KGP3V*bU6}7C`I-jC& z+f(Y0BF^uPU|L=a4jmfPlqg~pt-MM62*mCjx#Xo-VqNHgT~uklvp%ArteW}uVL!nwY);6e@9CHsnw%C8>Tdv)SueO}`dS;Hpu+adY z&Nhs>EcS4#BQR52yY(K^NwM+z97Xc3Nw56mdxir-y{DRcONL>XVS)?%N3k}7n~Ck5 zw2319S%H0a?3K(1w6KBjF&DgcZH@bUtaIjrS*cel;y2H!&~?hieyH}_>I3gopef|} ztp%Ssm~_PyTp;Zo`c-q-C8O!`EA7kGcNGk3@^e6(qVx9R@mbms5vV?lBMltNlnXyW zTFq00XZD=gtD5OadWE^Cr^vL}sqICG9jc;TGyhd*#g3I@!NA0foq+?6ORsA7Yg_1z z#E%*<-LrCBY@hoEHm^fx z#cXkDX$S0XitE6PHTKb-8~n@B(iH^hUd{7IR*jKi7!R`y^aT4QIq& zrI$PNI0B^g`)g9ctHLN+uh*voKEL8@Bk;JMD3)HfhG;J3lF1-=H*iRfVcT(Rm6e-# z7F<|B9f0ZZo)Ov!{0Xhp=M2f~?eIan5c<0JFhzIxo+j#II#TnE6Og zmB68Lk#yHuq`M!hpa8coNiLzQ4iWIgoXI{ou@<{?2};8haQS+@8}NVbiam}y%y&Zr z>AQgoS9%U)l0nB$#^N48z68s<9hPwHhy&FQ8@2?IG2ms?<5FetBTM=a+wE(C*>;`z zGfvjDFEc@+^USS-n3KPl+9<-T!{7FCPP{xsgMP@LM4>GroMg05LPx4;p^k^pcs63} z10rIJ<+{=MQxbL0Z>vny~EalXO2Dk z)vwBi*qSRT&khL<3q6yBm|ovGiRk^0a_5efEB1qPbVD1JuL#%c3>{WLg2-+rTvSIL zZ&DQ$2TGb&B+c4L_|rhU8Fi;(`=j{Hjwt@@76E>^OVOAoXF`n>p+iG?k$JJgV^wWz zZK8=XQ7kXFA=_?X7O>Df_#ic0oM#bCK4|nAnbo-#*yz@3LpDdY$u?!iNNX)tNkU>X zhZ+jCxR!i5nDpYhtxH@iEC>~%{V?)B($QQTps%{^VGnsQGV z=B;y<45>`*O&h5q?}s!fD(JY28*!SZ_>RzKyLVfdt-?dSwO@0kM%Vbaq?HFb4}KtY z7g{SN3X?2>h|p8}2dX3|OyZ>rPn0}1zHc(@r<^wereniv!19o`cost@1A8B(YUtY2 z6~_vX?8=6JsCb*N(*VkV&is=Q+e$6Z@QB-yR2+|{gH$0RXMPCbD2~l4p~Mew7f}yOu9j>8MuS9u!t6?z6UOy=1zZ(IxT{)7g0`8|5=@_onx>P9Dz9 zWZJXR*)mo~hmMOMtc@2=40%C`b6|v)sa78HNgzOb9@dU1j9v4qY&Y@joZB7)7|AdQTWx6b;2i@aH|FSTKJwV4gy8o+<*)2_&y$j#u1$dDvVE zQ+BYS4ieCp*JM+GfuB;o%pl#vGTaDZ)LJSuz%Ep=1!g$<_wP4s7u!`xP+OU%VeA)GZ zG9Jc*A5LmP>-hB9N)djcD$*-E#>r-DWz+w3I_Tc)jjab_?yjv%totYk6R#sG(}I6Z zWj2qh1@&#?pd7}+NsaAA!+9xx5w;&)WH6gal&sPH`G}p=8=JcTw4R!7ui=Du)R^FU zSkjAP4NpcSH3vAEIl~`EO(8JWa#83Hpuw>SN{_F{$ujCcI+~l`6rQi)znwR=FV8kG z&<OV@>Oce$SC@e+)rlEaxWc-|x`y&HnIoVBmE zU9Dgr=?^;deTsCHj0veY_X{8(i{q18t8Ld=D)hT0Q9WnqG9SSk%z<3;Zhuoj622Tg-xl|flpNtr zyEHOaj}L4~UpC0}jjk_{DRM={vDQRpyIgd525FAOnlOt#9= zhKktWM=UVs)1s>z1CjO;Sx~zMLRVWRZ{83Y=vZ!&_$KSeVc7tdU_kkY-E9l>ESX6Mach9e>h6^ zshs)^@NESXV{cG**hh7iWfmJTrA-^$@jMzlDd9cWrXd&o24E4i=V1kAvC`Ono&I^Q z#U5dMXJOf516dF2KJDZWTFB``OBdu*>itj8G!H#oWi}$9&Fr zO|n`e5Ksjd`l@?e z588>{na}TQE;SNmfL`_-QEiAWbSfxm9TylH>>pN|^GxF{t9zW0HRyeGde;xCpK?=B z#KXfwq_QLkRX&VXF+o?(zZzp{W9umN*)^EwzA&e7C5xCH8}+=58I7*_SX+(mP|e-m zToOEA^4v8M4W97thjYaB;GX(IYgW0z?sccp+6);q28%WOA+M8TpYwl=h z$*8lYyxEaTtOJEdT-^7->k*meODri?`W%pYj|qI zvCV)+$QW16Ji&tyi;!Tg; zZafR8jMTKiYq-N&eMdyG+%5hYRz0RA*3vg)i{m`oCP*Boo9^2YUG=T)xuRoYQ*&nn zN-wvBe}JlaJ}yU9uhdu5HJP3u-qt+%4t-G1{BFnu17+rn_Z}O=n%4(+645l{M_Dm` z@Vf_90_*S6&S+ltbSuvZZ<1?vc(^TqaeDnrCgkkPu|*kBO0IBM44iV$h~~XB*Y$OJ zO0LE=#Dp3LUq35_73>OLL0x_W7Q6J{=)YY55+g83Yzi-yJINqSw38Z#wyf{C3U6Wy zjIRrJvUFb`u~y%ZFRU$Rpn#|s5=`-b0|>oQtK8@tT!^>T##kyy!p?qJIscSE#v8`p zVCxnNZ=Jl3BMRd)%RsM?|Ek~NXF4;`smVNbPkzt65>t1w<4D-c2?)5`DRroAwR)}v z`AxE)2VkFnxzxI9Kj@tw(T^LFKWLJ?b=K0i&<97Prph$y#0vW+12?nf_XU0fel{4@ zviDgf_a@%uz`;l6(Cd<#io(hLV#uORd+fj5v`YV!f;M!m$HwC&O1Dx%K*JQ}Owk~| z-c^+`?DhOsQsrRwS194}Q^y@hAV1c;6T|t=kQ<>-Mg!yV!foIKGHs5J=9vrSn|mQw zh8U@dcgx-T*5Jv<6$Ag0#e&;$mEQn`UvwvlQ{;jfEUOZ0nC}-S#3WkbT^l$-CXo;?fHMp1BJ`ADU5Bu#jgLs>pkC7LosS+>)eoQb)0e9=-n{FJ|~Fj zK!eOg^ac{gHZNrszP3HtZoIHYTkEY>CH>~-g*vMxbHCb=VcwA+PQO?79v_`)Qiq>s z&F(@|3*Q-yw`K_;cOCKEr0}jOs{@3R=r^GFXRymouI}Lu_oGU<(>L|Nz4jHZDhQZD z+7$*VbBnLqDrv$QtnkJ8;Z4)n8_3F6R2jAh3v4M~{FUZjT)iC{YqZwkxzDv?72b=a zTrZcg8_kovnPIPWT*ym`*M`4`?W>KSriwykKKnA*l)$tgyEVlUke{}nlK~uDBdf?Y zax+Bo9QnT_;BG{uL1mdXhmYrsV?Hqdq;z|f#+?@a=#x-PIP!UeSb4xbFE>MTYjSD# zhNRTRrmBUU%H^TLSY{c8cYj!4d=;9+Q0q?Oe5(LqZ(RGwmKsKRn?{DSD!< zz0?D@mJVk8!?{SvMG&#{EbAHer(dB!7bY~9{P<8UJY`0i+ej(bBE$;TWNO8AJ8U6c z!Ck$6%ihbeDz+CK5T-QsD{~D;K{RsZ-o$gIl3yT+D~-#Ob23MQvSR9Y(N~tk9L>Qq zz0vM`kuaTL`3|?RGe zdJ~~ptdjN8V}oRpC03wQZMQs2MG_d-yf{K@H28os5*1slb*6_My>}~(xJz0%4{|<$ zRW{n&r)+QTQkuJlN_UGxaZ+Vh1cujYbR~(FEWPbJr)nEwaX7mpkndJ)Jch>=mk4PQ zKx%W(uEYVBLkk|~Y;lf+t2ZH=P8pt+o^IuqgK;>VwxWfYHP9M&!?}~*>SE|UdzEL^ zR^SJoYmzLNB!Y0$c#d-;-^h|$ zNP*WSa4-+0G!C^!qZhg0a~Y>TV0Lgp$#=-IUFOG(P#zg+i(-@OmL0r zrLrFP8rK&_IQ@{G>3eChR!&2< zP&AfY8vq`i*)71M4g|XY044y;xQ?|*cAm+S@?AC0E&zWubB0JDZc51(#(*GF2O!*o z-zq*6zu>1KyDPKlNC|>s2RR(IP5Q&UC&NJiVT`v&@{+5M654uG9exM_?6?n`=3Dc; z#=yo;gNt`%)3rwCpy7!jt=LTmJf!~ssb^04RcznZ-N*8HiZ`2_7Vb5UbswxW?PBTup?`&sm>4&^-nt)GYpvpIt;qZ&xEj-7!=Ce!NYmVcK*&upftg;O-@MwAS2Veq zlZ>wvR>0in7X$PxF1Hbbl&C%2Qdeqrh#83QQUX%5{z^e6CQLM!MPq?h*hQ0xk9$bk zJ|YUHJJynS_2x>t5R*IBdqYywBH+8i;12%)Jeck)Z}y{KIP`W2Bia~--PoXtSA`d! z#70OiJ4YhIe%oq)+AFBhKA!&o+!J}p2q_*(bV)@W2b8;Wu8=WSypFxW){|No{@q01&EuY`M|m=VAW<&=*1Wu07bDUF5ssQhu*L^4rmc zW%5$-k9+JNs+LMi{{Te?T>#;%8p)58B|Kt+AaCSdAFDTQx#(DQ?E~2(ogXK;(cju> z>#mcZg{jSPXnJy z-}EPGs2r_1?Hf^INaS6!h54(D(%NeQ!zT>q!AaJ<*J6-8>c5)eLpBO@ARYp>0AK(Q z#qw74&J3~v-e{CvM6+-zug9#L{JM#1TrLKIvQjNPl19de2y^8@t=-a!zy?0k?qm#4 zLWAGpV<7BZFE9ZRxzH-(m`DbLgu;I-N!`-0&6} z55KJw@a_R-0oVB}Ed=KZ8!>Lh2JZI+#x@VL$?;PSK8w2aCbSN-@<@;kJ4Y6rMJn&nyy-s2$aEEQe zmFD60ozhRlrMC6EnRuhF)y~zL4V6}q(a`$CON_m-U8Cfp+m=Bm7&EvMxJ~vCN`oKa zGOtN`Pe9Asd9B0hl*ekVJ0f|Pp0?n56G4P%6*IYN@ODK?<-o!uh0m~Jd)1quT+d{} zjW~$0V*4)_wm%H+$nfu73uYKvM*+rHp6ves9s7WK4*vi%S9YzG^)q2fw4g(AK-AG7F5kXTT{uwWQXCdZuq2K{+V$V%e%P%O4vBDHPk9* z17|lANcs}kNNgA#+tRET7h*DCk;s_MEP3ufLYLE3hTz9aRPygw*T~DCf&c|7ftX2m?SU!UkuSiz}B+RE{l&WFC|Ol>!fTYC)6NuQCI(+4K&j1cOc7lhd|3 z$^ffyy}2|RZ9h+Pr1xq;fY2u@<yo7FflKIl%btArnos&i z4Wqnsg?B*L2ROJE>z|I~NU`fz6Ki>6%t|71Oucm#o;-HI{5m-~n_NoPhrTmga1zh^(EZ za$>!D=k7*S`g{4IG2@>ToNSYxdv`4QyN*=nj&*pg&PeOmXFByZ{ruxfO4mpEGI2SW zOAHkk4YIGBVcE*FYei!4t^0P4=i}$4qB&)EN8fBWi)-qi=uU=mpg7e*#xh-fPJ`-f z^Bs24woVZCJ+BgE#T_Dm=S4}{LeLMHXyU^SFJODIWepE5HQYPbRCi$AsYbr2oie>{!cNI=oifM_qxC=P96VR{%a}b+6RJsuWnt%FrfBx1J~o=UQ&u+BqdLvT2uoN-qw`j%xdp#4N#fc6`0b!NFE- z6U&{lQDd@Y(4yY{MG#KW$tp|*#-I-W019oSQdVgZTuMGNt&KD8(XO_5W0S~_s$6g3 zJp_L>9;7_WysZ-HJZiYWNEA6PI`kI(hJNVxFQ1X?BLlDqBB@SxC6^onjGf-(G-DD9 zk&d=#n<}yJ&?&bnp^nRf;AjaqEd~cVYLs3Rp8QROhrpm*PHP%No*OkZ?o4>d0G7*d zWBkW9jU>S^PUWEie0fd{21ycVtXOkfh?5yYgQDrgK#(9LhLV>8xR&KrdrOWM?FtUK z;@|}VAz5hggE@4m$cH4eW)N^Shw!L@-~ex&libmt(4-qr40L@c)m>&1+`NP*Hh6M; z5#_&h^81yBv#AOjF}oLlp5-_hU>x5Sun$x23czcCtvU*Z$tvO8tuoW2FM1l%58_gG zDX9a1lmRN8kfNTW}CpLQAFnPL;No60mu|Qslz0;0@XCr$SxV98?<>l%qr% z;>jvv;W?$8Zln~}+r^ZW!%dH4?l)~Z6Wyu2*3>kDt3}4u!Un~z6cVAN8drhT?6O7E z#gWPvY`Y1A0y^(p?#{wqAOJ%|(>jIGzR>%Qoz158%|Gm6yZwtzj6FU1m_Z;kw>UX( zg-xy&ur|4Y$_~}H9XO*hRz@HTn(V2XW6PXbN*4n)?otA5Nzdtph49Q2n2h$sCIoIa z9t7+v4RCLVDK%2JXYuXn_$Xb#>aYM+0T!@gckvPPO@JUD4DH#JB#iu1*QHIOX;PB{ zXT`P-W}9Rff;cK-)$cK|;#pbVXnv1+zlI}YiT8J=h>^SqO6UsM9azpi;Q^;^{hf9u z3|7fC6#$M|Q`&IxGt=@w3c>P9!fKdZ?HQQPJq*d>r!69rwlMq30IplUivu?+0R1z( z`}VAw1y;s@EBM=zQH+22Crwu9ca)v4Y`cMbH(Da|gD4*~S=_o(=%WhR@t`_Kg77$+$M zuQFhcVH|oKoTnu0j%%e){77YT}{HqG~o{+$HH5PJNj=B zcE-{&Qqf!N0dorWvYYV6u;PA2fk`aO=<)oIvR=eFqCiGi*nKM@WhdL->CoAq|CLtSRu zjuwJo42z5p437(dsoBls^l!WMlVDx*7y+w*6CM68Ro#LsNv!bLvzyT`#kKTgP2yj7 z_j~P=<=(XDE)e#)vRr6GguonUe5zLkw$2jy#Bv@SF-SBd$vCMSr6dwLI|gaXNd zl82OY1PGobPF&oIh{W#=0G@u8kIwH_aqdid26?ybem(h$a*w4X>h(m?$89jTm%^OW zFzGvDnJG1xb3W1NsEQ1gFL@gLp%*Uiv0swOw0PgDx6(X`_>NOSMJz*lN|VzoJK8qtv<2M{(C0holhDk5C|m2Ty- z0F(;LI#(Ap&T*45+?jdf0RpoMrMfl-`A_o8v!}g7s~eJv#g&1}y;#C`7*-o>2!Y8} zZMNT-!XmXv$x}Jr>!x!XZw*T)uN3)j;ShP$-81QP$0$;VZYf!db}tLPFLGm8PV?NX zjXTc3u(Y>NUMrCEFP-zquHJS-eDj4?;KPgeY0p)(5!JO5pK}4DW0goT z^sN$mmbUNlEzL_20uq73?fX;>$SMW+x2gl6FA@zBth~#*6;_NpQ}LaFA=vAeF$hb> zKM_&4g~q3qG4|Shn)IXxS3z)NaTK6k=6LQ0FEORehVFv9_79W>-MwhJcNQlm=k_c= z%Px8RK#o$XKg96SL$|#@C*m7Ss?Bj1T<_v5Ga}WpK@D-IQmfuEdX<>xD{&I^VLo7V zNdz6=B!IPT<)DHq+>m22Bf1K$uhl)6aI|HdpOFbOe2?k$^(odK0A6sAHu1t~%kf$x zi7~S!!t?K9b4#KO8V`DwE5(N7oGmy>CY9?|rh(DuJkjCAU~d4dCmAkjg5MF{m%uMQ zcI!xJli_Th{MT1V*IC-q~|vftp!LFf^0fH=BFJ--Q3Ho+FgmQ?9=^L7h91R8wH1S#X;cig~zQ9 z{JK{oO=clCt(%N3%Z})(p{CM4$u4sm(#Y`xCB}}+aT_&5orbEWKNF$B924UsKuRltqLFQdU_qNip7u zt#>AB`(Huk7L7d89P3kJ4%$PQ4 zs5_MLiEUyx=1*OVU2VVuT1Nsz@_dqfw&9xZ`7BzeiIB^80j~;wB`H1;ks_P2Ra)j! zxl*a1&J&zdtx}*LISKFJqECBzHdQa#>Fbi&+4K5P1{oyLVkHOwoaVd~e9Vcd9v-z) zqKpm51Ig`80YGLIwcdIne8MOd8;&HN^{eKwM$=`@fV&@2xp)k~>%DqLTDNc6 zV_Dn)Qc(Bhl2mfaZdn{gc3~fmJBQM(vq5NkMb7Dg$hs(4xl6=`+dRXX+hlIXQU6!(QDTMns)jw!xdS}5XslCT%#fnHNm zpm-3ZS_Mi2m^Ko6@jwy8_<&EjI^8Prc}kFo$zgQl6cS42pb9rXa#g<_Ne$$cg`r0X z2C0E)i*3sZ7m{x% z_`;+b=G+cD)ti7kw2kOsG#q0o99KV2YW0_#ZUhhpWY&0v+O=vMwR7(^*|hsF;a?SV zLgI#V^}1$Tw|4tU24wE3*-k1p34Vaa+v0!OmDuejgQU2Bcl^O~EZJ$W)(6)Yoxj-t z^HLSX{y6xT!>p;P(&`_2^Oz(DR0VdxEwu@> z{{S9Y1f~3N;r1qc21f3+OLHaR(@s>XzTlb3HG*1ah*_)%Cpn((^b<86B5M@>W0)Wj zZ~51u2Me;1w?7vs35bBE5_V2&O%h2=3#zn5hAfKp95vfmB)nNmpsAv67+D zu(T)#vBm)fF#{&%hX^7;p$4G$_pcY)Kb184pXzByu<`gaw0pL*pb^1)g`rD|mSi8E z^|~EKhUb^4u#6DxKB3XH;Rpm=H09L zRm>&&6N#cvlJnyK07J4p&W`fAC9>m|BctW|z~|U_ZE&F#9-h zH((-MMQSLl-?w(tu32#M-UFECxDf9UjG}ThT$^2I7FHO9qZ8jM$vS*+{+tV+stU<&RNd0y<;qvhmEAoyz=9)sk#0z8%m0l^Kh2&9?x2=@m*qW#D=}OxD)Q) zw>;BvhO$qzc@#6Oqy``%9eULuHWn>g&)1~@c)NFH;01h|E^eOF&C%gBpK|UuO&}K@ zW!ol^+pST{!d^fCmF=`bH0&$Xeu2~vxPCLU#Bukyq-zcmNlyKux&ZOamh-YwmE)cj=U1A?&KHP==n zoB63+x8;hF-M8F}86O7GJ@r}8myUKTE#DtPM34+8VNGqf6^?thC1bt6+>j1)8o?z} zE)F>>HE^wa(q3kQiGg*xc43#iH+YdnBWl)cad$`~k#kvZ5nV436c9U->jEXAwQ@_Y zTNQ<86_^MmL~h+|`nCc40;_emxrB|`g&fyLh?V&ky1{{W}nsyevtbPkeM z$sHSU$jf2@cTYU3cg%;MNc@WFT=0gC?D&<;)SAZXbFER-@JBXbYdl1)z+6?2EbNYj zt_BWBuIdG#x-wQgnauJhw!n?4YC_ahWDs#?hd`r#3j1s`h)?nvxE@tHkoBCH#F>T7 z*m;*>wus!(!X;V_2RI={vL4~g(fN&Yi^G8CmD_WiTL2{CoU7B4v4-QD9u8UPA!8?O z#k(aE8MXvR&ZzNcUg7j4GT9`EKJ@mV5CR8&N(5bfhG4`5I0@@2pP7Oj>dh%1CDVBg zvnI4$qReOTM}A5~bnM(?n?+hlx>_>C?v)!t8%1G*`k>S2kw`*yztqy*U=nB}d^j#~ zX&0@4)BgaM2mPhfynVRe^GdhRcT5PU54h zviAnTz(y+fW{m?p+at_#o?2rUQh8HP`45Jn)*9l;&b&7*e6`(vAvoP`nBaopoTOd3 ziNL`JcUcK^l13vln32>NK^36))d1l#M@YyfgH5McJRMI7cbBHR*$V&%zsgp%kc&G z%5Pns+lbJFWR&}%vlUdIaJ_aLsFz)RtjfK2s_(w{?WW+WN}g4iL3Kl|lPjXjKxNXj$e{6&%tO3fxlV4cy~AAmh4V4>hCG-Y|}Ms>f>oAxh%Q{b3hWT z3aQ;~t702d-^#0uNrh@BwU&?X^p|sPl>)xB=IpQM@rbpOg8k*I~BW-Z9MB zT<>U;#b|}N@CLEH+&+~s*ir;!q(qf1oM1zD)jgxboohsSL83vDumTE@vO%ugQ=@o} z>XkNQn!=c2#fUg5u?xZ8JM&MrHVUn9cjA?d4q?RFd^%Lv3~mr%%qNjDV<`$&#yXct z+mdke8#qWE>I7wy(>t^F4YHL_oRB!^$ ztav&cITtgt-UXp144NGCA$2YP08|j|ik6|iQ}e|&*geUohLPium)(ls@nkvk;H+7< z=CqlJF_7G-fr}&1y4qIZXG(7b2HPH2f!k|J`+`Nsw7v(UvME)%k>VBE^C?(eom;*Q zQb0!wqMbRdgP(`H+2#lUhT=IL=ipR?9~2jXxzU9J9j6X$>t}VwP}68%VY#S%%;+}{ zn)vm%pElJgjlT8u8~TenW@Ei^w%h4Af$z__F28QGH`=UhzS@Nsyf9*OgFwuZ031mx z!0UfE*tRg}?fJx0%(*RK?r*g2oGZZU`%hci4x9I!@v*+k8{E+Y=DSOBtg$|yuKdrU z(vsUp>n|G^Kw$p>SrO{uUJGB_{h_bK9RC0++5Z5SYFa?u)d*41No{ZCo(#^qim3UEnVI{Ll-U zK?H&_Aj0!ly6rZ;-tu=0FR2D)5h@bBwYvAM9TV_F$A}l7sGzumH zysL!D1(G&}KRbe0y5z9|qRj19*wDcNR1b2};|T#~A*> zYaMmN%(G}UjdNQi*Y4YIwDyPwYP!E?J79nD2S%$vfByifTl~8}Sl12xo2dT)?$fUv z+td9Q0);w`jl zAUb2krgheXOo<1A<$2e>{_+0+`)+>H^*@^2=62=q$2o%tkk>i66CTMKB(zHUIdq1T zl-oCd7N1yu@Y;{lOu=}2I(=Mk`nIF=mmpoa_qV^qf9)>+0A)STZTWMZ?8YZK--7P) zQ5fyX($&m81OslLYR;_x0O55%q_!^4{$E)i_iab%E(VHWuwcPgHvKr?$McKmQ z+41dy7Y`l6KefCE{^S1uX*ok~E}pKc9)x-jUI+ePSN{M@sr@A(C)%;A{{W@b{*qT^ zxVe|HIqvpgxz2QWF8W1qn|9_A{Y$by{{Wcp+DXH0B>w5Ums9#n%;UK~+TH&E5&rcRMJT3)r0JMjvRo z(cru36PhCpxjI_0>0N=L3^Z{{Rc9{Uzp`a_In@mt=rv z%zy1I$8tZlyZ$5p0BI(+zwOP~JqYy8bD9VAwVmHo)YGTRNxtik!y0{2kN*JG)P9m* zYrV|5WHrugi?SN$Hz)r93F5e(WzqitGVl18KWRB_!Y-b^Zotrv?WVpY+0}^4{nt`| z{Ym!Ua84^azxrKI%F?gdZgYj=T;~ynF*v$B7jnfbf-c>{51VvQSImCV`@s$yT5s~k z{5u7q9-8IHJ^|C}-5Y+ZsUQBSO0{rH{jr4MEpuGQ0|3>p0of38UU$vA_m^Y;0CE2S zw7d4m`p&XVKCtbR_!y#5hL+Z>D?0rUr5Z(+!=@NU(_fN_Xmg>U#@P|>Qf*|N<_MI* za}>i_IZ65!1PgV~Do#bBLE<~)73Ht$bsD>d<@=Ag$EaEYMIunH-I7pPHL$bOo`_Ar!y)X&rMJQ zG5Oh&BAsZgxG|6#+5Z4AOBij9oA&(E?O?ds25M4#z*VeLXb2;lcXHcB&UnNqeg5?3 z#txkKgqk$Y(CCTTy)Zh*HH%%E#BN7S+7?C}l}sX#09Oj#;LAc0Nn_8s1 zH0WiQU!8^$+T!B?bL>^spm2&Rssg~GTkeKtRbuy5AfA+f9OpQ#YYELvCuFD$ge_Qr zG85R%r7hkT$l$8F{mt-tft&6hl|1)kx_Uz(wH z-etqU*VZ2(uSpHJ)BZ(EWzG&CG||$<2z3vnm}Bs&Vu#lEtE|hVGmmlNdN0fI=WC?r z+4u7Q0N>O1uN1-IZtNr6y^mtvI$cfowXAgME&?7gG=qOlGAIvmqVxk(W@G!mvp?wU z{>pizWSt)}uI1|g05<;s{&Mu~Z_Ro(>a9rs0C4Qj`YRu@yhm^Lwf_L`7hQF&HOJm# z9OC;y)ed;1noiYmhHA^rqm?Vrc07@&5I+z3F0U>9_j~>yMg9QzrVifKGZq_KR?c{B zVVV(KE)$tm4Z4WBXeZ zy7&J8U|b*-XLjzUDMSUR(R}{{Z|xi~IrcQ$4+@yBO^0ADm*fiX_SQoKlY6MoRObEkf^hRU3c( zKjgZ+pZDGW0Jzcr0DwLzgSYieoHlf=obcK$nh{(s6PZLdnv!;eHI-=Y3t?0pw|sx` z^k3jN;*amk{{V5L{{RGhRC-Uu4^`VA+S%^C{&2WPycK75_NwG#wWWMaVzr7S$&5}Z zf8Uq?0O9ms;17z@dv`3r^}UzuYCqHdORLYaFgAAuTz3Acp3SW*e|FXxp%ud6l$cBY z`@R1F52F770DMw=ZNV$b##|zc(c1v(HQ)D#XZws^L;LRc{635P0r6e_%)b3qtJmKc z-+jcz2e8gF%A#=531fK?JdV(A7LW|ml55aV0l|dDHkg$b5?tPu8=znh+PI8_-hrpR zCO@Y5rr0aRFI}i{@-B*d4*viu@(a7$NOaq0V+TQ_NIVXx4l>mqQvH= zAqy3URUrU~p$a83&6FsqgaPPu-MOi?r+$|A`^A@xIA85Eacj63Fl17YoMI+g-`)9- zg7XW|+OD04Y!7nd_bd9oqdDR}BRnJ>t$L=(Dtl$X80M|$5e#ECOayAeYiaY1y!;Cs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@beuL2v9zq;h%_tidI;i1;5FHhV zg#|10XmtORCq4yO vgZ8lI=9d6r&9(K%=l4yFoeL_GiUaK+%X(!P?9W71QL1x?_NEC literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000015.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000015.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ce69b713b581a4f0ccd6ed2d9ccbf8708eddba96 GIT binary patch literal 32846 zcmdSAg;yLww=X(4gy8N3Cj^4KyF-8=g9RTL1{<6R?iSo#gWJGhf#9ye26r1YK#(Ml z^PT(Nz32S{Z@ucZy1Qzxs`_oIUERHR|6BUE2_RHgR#OI`paK9W&j;Y&UjTuomA8#B zFOL8ZA1^Q99|E8Nz<%)p_XRHAe+nT1;r}kg#KdG|WY1?pLSkZ4QZh0M3X123=L;1T z6+JyIEi*F<7ni`legGT*Kt)4&E`a~LprfIpU;r?&uu%a2A^dO3GaNK@3`{IS016r^ z8X7hxCMFIVDh9@LXH=qBXz0Xza__A8FTMu3GhvX-=oKb4^kOoT%6n|h%?4ju3pA#% zkdY@V80h~fQnYD;dKL=`ZTE+;hAI&OP*DNssMr|jX#X|mGY}M1B0jWN#P8%Tte!_B zVL~tD*K3&B>b)eD*B7uZ@?ggJ*8?EJM)`;uh=#)SJUal52=H$O@bbAgD$(=34B$SQ zyCp)3L0t{&F6b%ni&DKIpK;70 zPqWN0*rVl#DNk1K0{^3-U*^TwuwM-*bx{(0W+xe?Ahipt5$`_f?i;#qmGTJ~@X3TO z`g9ttHZ0Lfi{L*R8Lz929#Umk$2%_5uT^bnymdTV?9g-K;o$}j(Q+Otduxnlk*HObuO40|2fWbeY9!MC&K{)_$M&>~*YT zjOUuh#_lxu14w~`xp`m4;U^vIT%BWi9bMaLaPQnc$^YDT{HL80oQU@SZGF*~u7kqu z_x?Iw#Gkx5Gg;QMqW~1=qxDw+Kg<8yolI!>D#>~6 zonC_t*`doHVb8ivdngEs3T3!+!FoAl*7>-&@+8*)TH@aA-=CTN63R#*v~PB^c22f# zH)XzlX%W7Ui{XHYsC`s8GlTp8EZ~RLxa0AFL*tT`f09LCRMW)@oGa~T^EC7DEjq)` zz{8kIo$IDmYODZ1o&RXF_P0YaN{s3J6Hx%>##&sl}&^i?g=}A0on0KdMm#Ug_C6MTUuv69V3T)&pCbAzm)u25v2UAlo zG{^6?rbYJ#D{s-*VeP~*I5h(Yn}q4-Vt}hPRAnGSOoIQmYQOaVGI4)R@ag(RHU;-m z>j%Fx5FYVqjpLr^U5-!cS6B4M3?IwudL$mc7@eo^(HQ(|tNkwk#ecfLKloDp0M=Lf zzIpe?)ShuQM*CIp`l|)46YqGZU_p#|Kx(D`z+eaMHl#@yrxm5I53-Ja6$#bUG!T~1 z#fteK>%;t6?zqy6PBYy+TS(ZMgIH;}bqYwrORsConClC;S$f)K%k8ffG)Ada6y~^= zu_fgxOE2?3OylP%UhMmG;p@UQH(#4{0f_zEarCY}4pNhUfHhytPmNn09ye9wr>m|h zk1lSLX1KJQxSGn%K?W#$AODXP#kEXJm%|!x-GE z+Bsw_tTotKs1_(8RLG(~pg;Gg zkt;PayNE^UvP(NR*p^I3T-TUEvn_s)jqtz5{(tckHza`P@|YPx;ZYB-6~R@5W~qKn z!NwH>2>hR1r@{Y!S(jMn6p`4Rwx6e5gpsB*;#!2@?HD=N^DPXlb|#CdA`6Xqnmz?< z7%N}gF+31=ASA#0ZB#U-r?cvndG6`RAiAbq-sUm1dB3HAQb5N)ViQsef)ssyA~!A& ze<2n8{P5b{HW$8AF3IJN%o^8}ZKT8`!!Hg$pDe3!kfhnxrM2Fb9R-#)x4srgv2#)C z>`ugc$|3GSKXSm>tIE(cViMQvbYL>1*5bt4e=$Zw8pLb7{G@V2@yYfDe;JnV-=f$G zTU)lq*=Uz_o{^jbb`Qw!ykgv)^y(Gz4QKPwSGD6no8KK}pJEuOVmjhF_NCFW!OC&i zIy@(3XfIVZm>fJCNj+8(iB3%B4^e|+ye?`i8v?nu%G{&8YS*9W!@|H}9VH~GS^h&e zq_hZR{LZJHhZt9WNiEL1WV2KeDZ{uwJKF2pd8X}qR)3lvqFfUQSJd<@7skm2KhOM8 zdRxt`;cz>3%>2o&bv8<9bV^EwUppO3BfuX&9u&h>~Mt**+Evrwo`}gTA~Pjoip7;$Lp(yQe<0bN`nm#f?S6x=~Iq&AeJyeY@7%HV-r)@X{VxZ@txm zO9036rKVTxUi#Fh<+aQj&yIz2!QO2`%z;gon9q;j-(j1($uNSo|6V2>GJBZ)$VXG( ze@o3i`vgV6ieQQNXAPcqYbo_|M`Hb$eHsS9`8ln_%ICTLe8E_FW2t?YuE3*`cp$#$RnGZ^D`ie#&F&; zq6IH4SYrlxh5z*2M%WX%AT;Nl2Ah!vA63ZB4rsa#Os;4>V6ZL=omLC3*DmDy+#ll3 zWhx5uc6)4ao|g&920io2Z>$`x17jE3j^nKDTNsTBT@wj1R6LSG)qQmI3)R|hlFsao z%;n5yWn6C4SORi*BuGLgcE5A@a1bsE&2)+qX8i*Y53YJGiXX9^NVfWz4&SJFhg|#^ z3Rv$W?0Zz%@82A&4I%%t^ANe(7Cz!8Xt^y~72kPho^AnZ^=Y5UECP^11eY-&=9fKB zT&FGRrMI0GpJpGOiS(vTGm@XQS>E#PuH8VMg-GUS^vN(_=!W&l z^MEnPx0ce<5ZV2--nd;IQoDSregtK?*reCV_B3Sj@a5G=nOOrmWn|bsAHipO);Ix~ zXV*Z2tYbL(2Uz{`PKZTe`d6!@29^7w*-3x)1lV#70u)5+SWDOn9qZ7}_JC zf&`zIDJej;gfcB`kS=^T)-M;u{wN(k5%hjk8A97|OhW=~L%hz#G!8ugR9PR!`|)lJ zx{RGPK%G8O9VEBjCIrpn0%`U77xD(_b1y^U|X6^AuQ(P-CnXXwmV1b#<<&^DH% z<{nvR3n=Oi`-^!=a7U@*e{h(}Q1F*g=8CubW@DyuSsJBXC}H*fs`S+L5806MX9XX8 zPbS*mGJgaUF4hJ9YRYu}1I*McwoE0pq#gbKtajbFc+ZnNlxEhY^V^l`AApTv{mbD` zhT@FgnpM#`BN_9JF_jysp@)sWD|&CEpLde&d&e?;zEk&Zf3N=ml+&Syf7vElS3jMK zf~n1RL|&rQV8C@A;5Sz?5xbJ7bya76EyJd*ee?=w7Qzw&5e?%h`7Qashv@JBC=IT< z)E?F>fOr&t+mqi#OYd-JYhLxRz$a^LY{>rseuJOka3}cv0~DK!eclk^bBQ(8srjtt zx9mPMcIM|bHvc8OA^&RWF8@JjW2@TX2_|*$H_qt)4fac)@zlL@z@m(D$CFU(KS0`0 z7xWkCZ+Y84fGN^4@w)ME>0H~@SbF6@0G^G^Uc_{B)(kLxDAx8-~S2Ry*_;N}#PU^^ zB*(v3(!cV!)p_8GgK-7m%|7d{k)zy(ZBc>4;zXmT9z<~>)VTLp5H@`cTr@N@ zpm2x@UCfY_Mzt56H6mxtgmErDi;NTQFI%fn)-V+u38seP#-n+$(65OnWmxd2qN{DK zYY>MEn!XhaEZChFeKXW-kOe{Mz&ZGH$YSxU-_4j#GFXwH{ls8W~(P9|dc&3Kg zWG=pW#4)Q&If?=e znK{lfD^2zuQdoaLMMfkKDKTm^yq%|^GOD-+R%_$?+{go zA#Rs0!pkjnScsT6z=P@Y9RD@-Ph$TENYL*w&83batF{wbGda}5b0(gXJehL(U{irT z#q>$?n-{As>}X6u+_hwQHB;20H%R4-oDf1f;eNS9bdt<$n?yzjFMDbYE@GBfcxtCu;a7QaEx(My4 z2_~R>iawIZA)?=1WK)mL^0VI8{WUPNGLwu{N?h+Vf3W}&Fiy4_x>#CAJ2MO*{@yE2!+g|B`L`Iw4)-<-pAxcnNZi{o?^6rK=K3`8P)dugac}!X zM%{E0vS&8+QAs_7mGh znN_VtYjl%mv#9X82kl++y8`lbz|P&(9)fG^!{_YmJEUp9aSKT7Kx%_GobRi;SV^*Y zW?Zkr#Ar;-n8iOpz0;J>_PJdZJK>pDS=;xIg7#&9D5jQw75C+^WY>uG0cV%p2_^P% zKoIYC$D+Om)19;@Wn?y`>A(V{8NOGF1h4NDl45b7zNCL=DU$$bnNkhJ~>h+GFx{e*2?_1V&ac3AG(BIQ;G&HAl6 zvlBl3;JJtAZEFFk6|+upC6p52cS%#F44UpI=kylp@#1;uy`2h#eF?!ECgF{NmTRGAoXoW!6@RA)`GpiHap#h z3yN#QK{(gs+FwonDZp_LcBU=#m~vI?xR5jcR;YL-J-=3+WranSbUmDV#4RqQ(wzki zi`&4xl<1~(6-Q8ByBGMB1nJqM5O*hUetrub`1rJda2>sZvm|LWsR__jDqVi(RgO87 zQ7Gx4{CsNqwLe{Ds?Tc;IZ6;B3!a&_l>2>iNItt?VI5@@0Y0LYS#xQDBE4^mt&e%D|xu)#`r5+sax3o;T!1BgLHwse% z$LfBut`4JvgSkqTXxoj%JK5=Dx^=orSfi%Y1T6tqB@QHL($mLBb1_4C83bMB2^Wo% z;qgerj6=?sue(EfBzjP1?HfvRpLA#TMbf$Fo%u_nGT6t)k~^5SvhpSlz3tq_s8pv9 z34`X&}2@H#JVOJEf0E;Fzx5s>Z%7m>J05 zH6_{CTumdlaKpC(S?uoXgyxSCHxpO?H z35oz~JaP$h7nGcbA(v=sQ6VAqO%Q4>&@iNW!mKSZ%VzQs@c0uySZql2-Kd~rNZVhJ zte(pa|X(yuh524n!*4< zz;dJidlaJ7fu^QnHDpoAEZg23{+!mr7&xP!PUY;@vm}#|u77e?Tn*7yAwmer`*nS! zJkim>OQ0m88=q+~>+4NDByx;F?HV^G(ib-+Sd(JZ05w?X{);=A`W|OIg z5I+m)^PEz#9$%m9Y1H1SqrAl-o3q|z6NSoj*Vkn{wtE>M>2O&Tt`#I~ z+;9UfR^|OHJCNlgOpCI3D2APbL&>PD0u&DK?;8?FEaj z@P!@)k+QKS4M~83JEz|SgGuREPoX}+;zm4J8kNkGz-Gj{v2>?rSY<#jABC&HDJ%+R z!sEvWsXsv5;N<|YI5jQxflg53eN2^eTKH!s#(%q?j9E``0ZRjyHvM@?R=zCVS zm~F6zh&D0ZLqRv9GISSH#aq86ge|>lsN(j!5xL%5s5G+ZcsO~dgs3D(ShLF>EqT1;k`R7XUeEPTNmRPEWTL5DLw8TEboh8u%6rz)G&&WDu^Nk#RxZEr z;E1=JL8Yhh{BW;_9pO&-^YyAkGfUx0~aC2*l2z3!k#O;P@1t;rQ_FFk^`m4--P zEY`?QS1^I8m(Z{-V=XhMErAai{^ziuPE=crMs}qHRQcj9Bk>2y8J4spjWEaexJhO{ zgVFr^3=Z(v8%snnq+KB1nJ3AQW=~o|nw+GFvYnP3vTU?{{^k7+-wCd!r-swW^;H<+ z7ZEIJg^bJj`+_y2NQj1C!OWAm1k#4XMBW>YGyNteL4;01*iD^eZP-QdBgO&?!S7n^csldzyX|Ah*&DemnI!+_f_AemdPv(Jl$h{v2f{8<- z{)5@mn)N3O3M>2%MX4LpU6`yBk75E$=v+UVKjk(@q5n$C%H_FQu&w%DB>4}pK)gUn z$Rsy?sp(IH-SCcqfSD?D-7c7me;I22QS_rF2 zE7q2@=)CBHwUoOcfzISQ>UWOYE_)p+QLCCLT2Hzg4#S&wn27>977s@Lj;URG+oNmF zVkP1_QnS^7h3-uAZWc#Gd5avf#eIStQPxZIFqfxXWUCt?9+EGQN7DU%euT=^SA>(P zR*-a#XLLdD`c94FH8K#J_7&9hO}KPNE(@VAxTxUe-Qb4_4WK>q!a`m@sc``H5*4lH z*av->X)o9bYVd)DaIQZX%fuh()z2IFslQzNzBn zkc+}f3SfpN_CENCPEXFk+Apga2nOy};p25gDzNQ8wy$Dwztk318=+p1cXTKfm1 zTe8H=WUI?2ST#DrE2ufnHnFf>Y>$<~T;k=zwuYuSv%kX%^&iX-mt=F09-nZy5s|8U zB8}M~NJP+)xsa(BZ!V1UKCEVC+d$|*PRlR;_w?x+-!gV35%XD8_4Pd?&`H6jKI*<| zccFUg{`xhp9=NW&NA##K79@imxn2C-;-eh$CyV*0)hd^{xG%eKOYVcB>so83Tw_uq zS2gN=?daT!w5M_qTg!5}z-du!d&pzk$&iwpy@cTDPRD-OFeeM`dpa^SgI4jXwk^Q- zfHCy`>O&DZXk|yvPZ821IwxV(;dA*RzUImwhnk)JGUcwS|Y|}x(adxS-l5`%kW`1&0 zP6B796!Q@-F}Zgowln+gTBCTuVOQ ztN>4V8s}tjv+=DlcLtw15t5CO_B*92@qy@zUis-P%Sy}@a6~{bcO2eK=ie8yc1zX! zYV|AapGyjI>ys`7M9iOV14u9-Yg}a=Vt<{~KkVhs^$$sh(aU<`1YK$S0$xt7NlyoYq3CL&QoHJ|-YI0B?vhL37eq=TBJ?23Dw9qo; zE)j}fkTh+NvOH^eoW*W1Rr81DAKHmJ}zqY^;yfejM*mjgrDB07NF74q#zq=c)En# z(6$au(#|BTmfpbz#^IWK+e0CJn3T3n`51`Edj$DDI{Ct@LB$Clwc}+ z2+LPlEe9X@IjDd3kU+L0kK7!KUx_e*54X5Xn7p#)+QaZHjY-tmuX6<{B(r-E#$zkn zK6F#{BTgxM$*<^d;`$Anv|qD|ijwDFQwZ>(qdj8H87CXS+BI0`X!r%2@#8#eNN@O1 zLQ->799o?)IZPe*1bv*$CJ@4Q^IjGA{=H1-gSjaz1WBHO^w6w*+Lyz`pCUdFu)hyB zXK`XGalyKh)%!4oE~x)u&RqD&B#ekutQpg5MRqPQDXjT0<< zdB`E)_GSB!FkSE~;4~flQ}Ai}t~POo`lx)+`(mkCsSOq1k)*E)qCF8QvTU=2UZHS1*sa z5e<1p2N+)EZW`D)$EAtFmv=KH@5rz;H~`<~x$)3%Z}D@M*uU?G_j zP=*L`r*i+;gslYYWQNq#14X*A;H|Ouaz#{OXrgn=K<*E#xgT%i$i1?)TRb7Z`?(pL zzB=;s6=+@`%*WZ$OwEQW0e8b-7H@l|MD{G0jdH}e3LX}Ngl(4rtGDJVh+AJ zrC`zHyQQn=KY)@hrs-0gZly`h*3#`JUu^fqQ>MjYc%T}(?#q6BL;d-#oBR?Bh5(MY zP#ovP%@4{WD*+UvE?F<^Hw$=Veug2Ce-`ZJD@uO0R!(GprCm0kJC;7Tf8--y_E=8q zROMA7K*JDo_-*6i`n+zy(_xKf<0z(Nm;B^UsI_wcC@TI+`WeShSM*}_w8y;#ptKbaJEXfGIKVc zN(79DI)ldWPD7uM7Y?%R6VcDOUM+IvC8%&6(h^Rpm0id{&j>}u8y(aLw}r1?7mrlMcCuwM(Swz=%l)Chq_<=?8#4)I(9jAM(c z03jS_2VX4LLjJQLHUtA?tADm-O-T(TEq713v@9|6#wVRA)D;`6O~%O!HYOjgN=mcH z^Z`CV6Tm~j71|q4C~|u68bMBPlEJ{M;hzk(2S+G*mjRnklKJ@Fh`8*Z3-&B{CS|K%QcUrHTS0S_mS0j$Yq(XCvcbmG{M@!R&M;wjacrD&%XUsunmg((KY|@kUv)0zF5U>Yw4oeiL%p?oTObn`qtm zsfnJazYB9cO5SbE(kbT)8EDvyH5b+>XC;PD9-p8#5MAaag?)FQW%iL$ccCSA@`@9` zn+x_*6kZ3|n_aWLR*|-HLL|ISk;#{*x=zw~6sep$_Lw+k`z{;vLOCb+H*$0>{{;B}i?yPT z+)`OG#N%itRcZ5**DVfRdj&a>5PNHcj5YezeQkdII6P?7UK6F%(V{&M#oE%vGz9c? ze~tSt2Y-U#Ouc)?y#-VGA0XIqCU?Z9g1*49f%uJJOBt6r^OqLXmz0e0xYi?S0?Ts? zmDx(1c=3IRt9JzJ`arHJkJ0|;z;>T?=V9L38`{d5+Fn*-j#u_p>%YPbf-J}wH~js0 z*__By*7Twyy6U&_A+v{XPM{cDp?D`CdpsfE3|5;-09H&4tdqKntE5r)y|L_>KMs0O2|G=#M*z_>Hl zp7Q?jvwN1bVV9L}tZ@+PohPjQqM0oGXlHgw+1;TJ@pk%hV%odQT;+*7-!EUTxx?swIlq)E6PAYbqL+HL7j^0*On6AmkL?RIy z+4_8MdKF3_=GNG^bRr1otC(R79r;0M^~dEIrgLvmd94@Weqq1#!NvaO@$VhAj*V8) zB5X_NG2qocpL!@)WO+l|u5SBq#I&rXL9z2G0xNxl{HdT>e9kH?m(LoS`-}Q?6;lEd zs@#>YI%#zX#a|~bNR@>W4TiZvbF~$`XmsrSF5ed}Vqsjh)V^ZvEn6C)4xZyjX5Szr z^e)j6!2&1(?^IZr=h6oa(ye#o#I2cFamkoQi<*)u!lTVV@9SZ$%OuUT39 znUYeGojoB^GpOjzgL&(z*=d_(wN9>)2%O9qbdVfofbbD-6*1|YAX>aWriqqU?mqxS z@Db});}8DcQf3FLg8^*Ib;q>CPcu{T>o4TDGTB(10mHLV?@9~*oc8Ni>)w(qT5$M| z@h`;{Cy;1)6=i;YY9PFB*7Kw`klju>WLD4z3_pLa-_J1qVJnwwEyxJ)R2A{UbX0dd zH!4)#>64YN#&QafAUusAmSbWz;mj1{$x8RX)n%@74lr4{&GJ#(()Vs2WWTrA`?u%LqBIo*LhG$ zu|WBpe&)Wgb;c`f*qzE8!8*j;JWqrr_n2-K=hzm>AXO!Inmz&%N=y2bPwZRS&aKpK z%R2$3grct5E?{7GU~hCuaH&P%*VsoGCj=XdVF(d})sM_Kop+Sp^4pjA$R?}1F|fqB zDLl2p*radoP-v=HhhU}3S%jR7(U8nKylBmiY)QI<`qPu#N+XezdSzq;Ecdmx@4iGC zUor1V+^^G>>)eNGP=R7D5mh^B?sx+{b>ju%l}n`fn*=B_x? z5Ge%tBvaF6FXNv49!MJx{T}3-v$>QiZ7c%P=$@n3*Xvy*k$^f8zrKls&e~WKb9)Mp z?qVZ?rv}8NVYUT5N8>3KOyb=_VDeDD_?xZQo)*?EA2B<7q{93HLTS#}BYMHxb$eEk z`-W1m`*|Ozzma zm!3`?UGt%bnx+kKO;NKd1lC>x$`kN z2jBl!C7YD0M(gWwycgC%E-pB+Ma%pzs^`Cb3oakUCQN1>@(g(8MUBqIV(+$fWq^SR zIcy-~adE)V^wwoHaXZJKZzTjysay9zhERWc?+DJ-=|t0faZ0(HGF+!6B$vJ*u!-7| zQ!3mtjwQ#Nf`7p~kPvDDC%d{_!jXX^QFKD&Bd{gk*@`^A?s z{wK}Cu-gl7?RV(N?QZ&C(y1luB_FHyUy>l7=8U&zFvExuGtG=Vu~X3U_n02R;7&EE z`yGySi?29mFYz)+seh2P`90{YO%vK9;Lahls{^0u|U^&P=%IcAv=i!0Ms>zxt|_HK+Wi=V76?Ubec63#$XcPUZjD*wTFZ=>D zdvk`mu656QCiOfP0hUem2x!hMU23K;UAv3gCzx>H-Ju+otbJ}X(A77*jZ%0!dsn#< za|RTi&vWUXEU%$Dn@wQ$N-dpArUEC8!NYS||8zDIoM!F}ZxI6|SDZV_M}NvrD`Kb( z)q&-uR#}gHDlo@qv-j>@sr$UC zNLOF|3C9x+9s-`nJUZbdF2Zvu5o?D4^i)bur~L!G_fFoQuWBA-a#7HvW0IDEYoAdN z_VR!Of9t!xVETSVcY_ZF9y(``iZ65zG(^JOQECFN#+n_rJc9$+t;@v=(yTkhz4I*b zF|F9VYDeM)Rix#zJ`aP(|aM-WlyG!J-m{hYW7mcC!Wb zVrJfwu-5po3oARHuq!kz8Pqh~anJO(9nLb6z_6oLocL<+-ndq8E`}hQW(z~0HQm^W z?3_YDjUA*bG)Wq=H1VT7cg=hJEVMI(LYcSdgDq>luV$zC!)39oc2K)w)Z&O-=EnEZ zNodx1NjuB@5*_$U^-ikFQ}%qR%89R(fGm!7vU9X2FaPFCm10tQS zXY9C%Px=?6Yi4{YTSvU+*jGxS&)F;45@Z{Vn^))dleyA36l&ha%M2KhWzs|q?wB~S zs1`tq-m0hXvM8`3Fe9&hgk9N8X1{C+p3# zPqzAk4(U{l?AcOiqB{i>w`g7z{pJM)&P8_iyX)b5?~WK|QNq3F2hgelbVfv5EY_`E zGMcW`TZ$ZC2qP4Ak))gGd03mtM$O(uk;{{ntSLkduM5aM0*}Z-E_BO1ij}hy)+mbcJbO%RwsrPnWi(dFY8Mw6WaVd??bfr^BWfNpz%o zRj|uQQp&15@j%8r}1}YkQhXsY^VvhF*!~ zE-V&$rAcrW{!~{8#3NHS@w4%`)DxEt9)lc1Lrx8qu9a~dr{5|6sb}UoFnIJ)kgY_= zzGcjsF~EyydyJwFl}@(s*7@rpS7} zx1~sEht~;DE=%6BxBli!GYUFX+7Db1HJEEo9UK@_7>5Qi1($s)wBxJu8pJp9gKNa6 zB2zo!VLSG?Bk3q;?NZV|V@!eOWdiXF>4aQ9HBr|uCj%ESmw?{2ce1d)m9xWNj}&R6 zC#L>4bY7`PNW$sKo|K;?tmYrQm!actC-A&&)=Ki&r!)3NcoTd3f3C4=m0l>14{+ZT zyeDN+$u{ZuDdQ(r4f(69!h)QRk4` z5)*qPc)2Nco`fPN@3AboD;5HyN>p=+ma43g)nUsDkrgCEyLl+i&kht4bZ2*A4`f7Z zl=IH=D&5M>0LE3jSPRa%%!}kGbe1Yjb8O|qe29gPf9g1P9{dIGqccFXQ?jL>sQhK$bmOUCBFD=GPQFUc+w-rlW<}hS zKC7+>Q7J8%#(E%9T$mi2I%QZ%Qn$UPXFKmo?rZ#rmC*6)#w!ztl>5Fz{7)7+tHiei zf6DYuH7_a#(FI0f!J@p8!gEomsno;wWj=%90XExG7`&o?_P@^2w75ye4*uEw)}!-_9gVT76jJ9_Hqv`kK{H9q?}PEIgPTO6`OUQgnN*B`mZf zan$NtC#!^TQO@1WTkq;pVnci*+g@jWa~w^T0#qV`Fo~d~4ZMB_am-BV z$oI&eK~w594A=7>CSRx&_B8jtRSBdKFgY>oO)VAHIThTSFN_Jl8T{6WF~x3wsacEE z)n?|1`v+i(Vcajv3v5SL!&a^^RF4x43S7$=>KQhUb!>18gvIr)6^`Jt9^kHbGHyf5 zj6uyhqdA8~=78oCNuR~w3~&tEU^Hvkz~t}OF@My~smIuqsq5|cEV1h?Ki8`HPg=mF zmS00javoW`#-nC6z(gjvng<}dg>^axl`Cb-lW9n0kT70|v)dYbjby>7m&HHI^3`s^ znNa-1_h5&NwelsslPtq1YK7Swv{oCjD5m71FJJfm933ZQfqUDS)hB;dyeUk+3YOB&Zn>frh^{oMpPZ5} zu-nJd3WA07z-KA}t_~xrtE3YKGMVfgDj$aO$P^p*%4Ody(-2mRc@@2{GRd==QJafS zi|~(Ve?)mhigk9%L;y01;wI}EryHSe}VurqI z`*ry$A({)zX%4CV?py!Mt8shAGrnunCFj4o=R%#>zX*#3Fdee~NNuj0a$D0RX1v8} zq-y}LN`JsxblHoA(oTZFu^E*NMAtffYn^6L1RrVSAG;j50PWO!Xq-73f9uwZQ!-CTZSGDv$!U=5I}-=G@`0gr`l*2 z@jkdJA_O2^f$pl<%lP=-RwyO5{1SatO2)1BV8`YnLeXl4i{4sWH5RJr_!Mp+daF2h zhK&cpG6zN9a5AEGH}OfeeNiY!g5=`Q7@qT~lt(z+VT>^nO@Zd_%&_wI5R~*2YK+4( zuDU|0`Gall1Tv$$gas-JD>wD3ihq+N@FJ*V^J_3P=K?31T2E&}eCFmwUT3BL%f_RR zJs5la6<^o7P2`VGp0kch!UiO`)9KOQoNdGr=M@mZ9v2|Q%3#^y@&IS-vVQVcfx^pR zt4Z`WPfw>vyV0~P@<6W6can`ik4ZKbAu1qXxdol_*8mGzH=d;#3sHyS6(r8nH`bKey=)n8V3N_c*U>!^3X zjeEDT1{;7seIn~eLz}!+-QLfU^^QUu9y(V(};bSOv{o3fnjRj$oC4j10GMC@0 zH0OS0>!kxDsqapi$d(z)ElBD0v=7#el{PO@!^Ej`eltCLm;60C>F&^9v*+;roD*H_ z8H`gW3B=+=nv#9+fDaR*CG;&z6Y6R@0PhM^ zts&*$+^>pv^~<&yS@>|W!_`D;6UWI7E$A7{ius4`BCZwkIK3xqHp zf?#?AAeS$q5dit@_An*cAqzPuJvzDN4tHpQw zm(T`Hj2_W!b1hJq`eMwL?^hCEOnZChvKPAip%K_l)6O4f_#;y7ohG4ZflGQy{wjS* zfLx0c6KIN63Q04p;+DQsM+1or{ll^dcKV9}COukKPw3eui!+>Wr7q_vr&GRD`FFy+1 zD|_U(vg2l>LF7un^w_&5(f7nFJvzK7GZ>DuzeBA@CbSlPEewjkv0|F*uN2Y-Iag$u zyH9gOtLxqJ+C`{#S8Y9XOu*yY z3FX+!<`uCuhbt}gI8r=Q1DI5wd@%{Tb1SiV7vJs?nzh2n+vHx-+x7^~`FE&9d!`th zEd`OY`4~*ID?*c?xZNi^9p>E3enNBOVE9>$oJ$G^mFB1CL@M5!@O*iJSMKL&8M9a= zskzSN-F}D7$eg(r#ZZ+In$fT&fGRX-OFG4jZWufZSbopf56l#CMS!4e9G)28DU~)G z&D@HixL(m`DN_5vi(1c8sh3;#zEDV~(6fe&O!U(S zCp`X9)qG#HZwlhODK74W??m&3+%SvplUmb0g<|<7@Z|R7t>dJ9Sa)n&t#j^iwqwEr z>ClG}9xM-t!H2%^Re9Dz{-|j-_)NJj42UTI2_^t!_yg& zKyz*=9fU{y9-U~TcELROxF5w6#dsPCY;!qT9X%4zlPWiO{yilMe9^t5WV>@FJY&TZ zp`4|sSYsErjV264Ls?&WQ+Q2;;h{Qd&mHtNE4+ch_m*kdx2vAJEs}9onoA0{LT6-8 zV;vgOMkiV`CLEV-)g<9>IGN6D(g)&-S?6@TV3Zpp+*sMpqrG@36o*%t-b06)7^&v# zX~!aH&fZl`^dnoex^{d3K1F`5RlwY8->0-|c3|%1xFjQV%>SvZ6mFs`JQgNMX2Shr zg=%Q$Obk?6CD4XuEv%7={(gP3<2npx+}`_1MblGAj)Br!>oaJ&nH zyA()7qP4bq?-lvP%+F}ETxy~vA1_w+MOSmUe83WCS@X$wPL2={>0~!<-Zv-PxY)rE zF#uDU3T<;XS??%0rFlXMl#1pMEd+60yQcVKQk$=Rkb01*H-}A6>eoD5J>aaJ%&uuU zJ}leI)Ml@sF&Pbh*X`WMXv{z*>5smw;@1g3?-U8O3-Tm?5NZj{-@23C){0$ESrlvk zC>R)7XG@{V(8SC=A6rUw+2m&{(cijf;syrvD}zWT^@U^RIxOcy#>EOdy6uGG&}Ug? zG4FG6GJJdzfTdA7fc?l9|5pIGJx9Vzg0;|$N+@?W8ck6WkoLLFBxs^_%&1Df0NKKZ zP)Sm@!X%pW>|9t&rhTiP^*07>1$Gr?q2NkkIl*e|us7=}tX0#aDVkN$!%sqE&dJb& z&Jte$2;Q4VH1Er*xHh#il3hQOO9$dupAR{#KP<*(T8_WsyH7td3sdw~RV=H@eH!s5 z;;`n;+rZsv?&o+fb4b}WqV{^=2%P?4Ez&{wN{yy+%d7%T#`!(MCL5-^5;1{N(X3#ccvl)Ji+FT=|Z; zVf_neXgTa&Ys$oOi(E#t*pE2Z5e#N%I#jIX9Z;p0D@zkzi#g-|^r_3lBlA(`?|J6iv~NM4g|Fe(@;JQD zCn4=58668Q?chdT)Rnb_aPRPRdji^5w}Gsr_O#MgYJaKJhsC-DqyGS^J`PULF|w%` zm&j{H4q@yBu4Q&$v}Rr*t~iDr1x$^56Q1x$fHc=?P3l}^ZdGsVTymX@qgq`?n$a^; zByB|5ag$XaCEPjnD<4zJu5BShBs(!dVjV+M-f3^{SLd^Hw$(w*b|;aue!_OCOHrhg zGi=&pt-P5fq)>M4UXI3F%xM+-vL|I0O}q`uZE)pgiL7Z}rLBXxZq?gX9h=N`#s;Vv%y$!9Zr&uQ8RCX?5(EILD)O!aK&8&tax z75hckSpM*kaxw{qw23kX-Bj)|*>f}!B20TzoJqJKoS7T|A9gbBHPtbfdyNhoUfRWB z!KCijUF%KFQL@b29RxA)m9O1AC`jtkDxBiN+7^{}T*0XB$9dgkG;8ZZ$z~yz-QH{+ z+qGM)$0dM9O_~R4@-GJ4i%d|Bn|5#g>(mTx9(gY3*0L|*&HLf@`=QZ0K7R!%@m;2w zk8;4y+o9VVE3UB-v-d6^^wZ@%NSsC)_;T5E9hyxiWPHzBs}>!1LL03$=v2{$*1_y@ zhi2`ZXtrB%!LS_>UUIW7+sNbI(C+-va$4cVIyHoA5>06Hs%s%G*~=%2lnT6d^dv@B z{pzN_29COaCaZ1&nCG>ENdPRS*Ls^zpDb~JbupsXdNOD&*I41*bcGg@U`eRbsC$;# zW-YMDHLml-K-ir4V76Fu9KaZjoB<_gDQ+q{a2p14olmt`VKm(aIqsC=8RiRx=el5M z-oR<3YOdZ)@5`_q}Tx*V2=L1$aih|jZ zS&8ajpP1}3H51;MiKf9+b{`(hEctJ_-j?OiK@qf8B*wb0N zUlV7EVg?+sq!u`XIu?Q4{g%O~3sU#e{B65HHqDarGV>%x$ZJT7ZziaZy~}rLr`z43 zI~lu-`e)(TJVop|vx;@gPkp*J*lV2A6h3Uk?aMq8FyBqESMOT1aQE$-w^+dE7G!8~ z0J-kXAU0&vMe%!%H(uwuq>@Zd+UjJHJItlz6St*vzj>K`MeIlF(BMEIf)3H5wei0M z`c2|DvA8@f$J+9Xg|plMsH7<6^8z3=do)@;IS=gieD_%pFo)`aR&_g>zDdL1L2m~ z{{S$!PbcQ5h&|4_Ann||+49S{!mY1qmTu=%dSPnVu27i`io)~Fy@@1Cp9I&OFGbj} zcN2<9$DTeXlez(Wskc=708no$(0La9P*y=__SOI?ou3f4Q&rRQbVDU{nnBPU@)B5mZL5P zJ%!#Y6rDuVc(cJY+>JKXX}B*pd%YQW@K+Y3opl<+HBXQY%h30CXRUAH%-Lb! zu^Es*z5%J4N;5aBL9=30wv6-Ei^SMO+vN4HJDJT*%R1ANHE~6saV$wd*T3+BZXGx4zM-k zu8hXEomsWBz%?TCb3L0QG4@BzPZ6tjciW;>@4C5`rwTGLMa{Yynmf<$SJbwP!M7`% z;kZ+Bqu$q!wbWG~glLy7Wc@U1=A3GV!bR*eua@O-Zd10hwDN(mMZ_|(s>I6mJe{#~ z5iK3Hf!El*76^S@mb6ap8|ar}&~qDh0DFgce3Y8=nd3~R7Ls3(YgL}xH}1R! zHm*6qkTwIf@e6*#yxe{wr!STSRw^5&z_poT(@R_!Gc~zqhULGRS7zIi$DhFQm~8hh zFb!w0fGbHW-2f1$4`r6fYXBffjr=z3SmOQWIt}4)kWN;*a?tBa5v(bGzMxCZ`Dq%s z=N8$Kyn9v&bA-C4MFzi~*;*H$Ou78Y(^200y$Ttu+b-C_sMPy^imAa;H-=Wk_iKm< zymPcc(cko`8}l3ff4-{jPG-D_25DU245UarG~SY2DnQ>=r6wL+Xsful0BJikSPlrgtn{nmOCr^kM-*l+yj_u%u4n_g#*Xe^ zY{ua!)*8p#4lU9(8}okdwWjW7$@*^0!)D+n2Cj(hIx2m)IBAqL!KWDbuSQ|_8Xz~4 z=I||S3EfgU5^*iI>sijp%H6VcGq$jj%(jH5dKe+34M%Jt+uAx4&T!)boocclRcc%5 z(u6y-bSJgJrKfaz6kOMlr1Rp6$(A#>DS&dCQYKPi)24+R-en+bvSi4J;+g}k389b( z=dh@;w#~sl;X&buqn6^CJ|re{(k^{{WL5V6#`u z({P`V_N9G;BZv=?>qK;Z^kbWCw_HIbga{K&cph{}x+`onxCn<-^zKbb@haaGW!`6& z_cWiE$rEb^Uv6_iGZfhx>_xv0$TgR5C-Of8PD>4>ng9g2@!2C?zQo*w=hkJ}!UyKg z;v$so)(Fuy?gtkFzCBd8L*)j3hr8mSAY(=bN ztuh;zu4}h~Hr6cPbT~Idu$`Nwb=*CJY~+dA-q`~|)5Q;iZ+wy*(;QpygV3p{y+Q4E z>*5$N;lbA(YRgS~ra&>Y!+;$KCOx%pk#44?%_b|fnK8i<+jkv!k22t7O&)8!0(SoN zQ*AN<^DetFs3-RwfEiPVN1-|{k3u%Qs{xJ3ENDgsL#o7v9>IN8fbQA_EjekasH%vknkKg6!C?&Dgke2?pV*f@%*!wp>E;d=2~fk zq*rMgo|O`{&qJ3r$vYHnGTZG`nR`b%EsU)}z$LCCa_PYkE*V%iXDp?-mzE`Y`(S;z zfv1}9VJp$@UP}#h{kJW7Pp?yxh7#~EE$J@<*Kl4t#9js}=j{IgL@!O4uOEw-;%-jE z@>P_n>O7q#eN57_)|+i~4d}AxJQkB6i0VYxU>0qN`;GV}E_T-ev1W z?8-C7tv?0max`p6!~iW5q(mJEGBi6nmPU7IEoZ&=G?_CsGuF2-7VSBX%*GfaR)Y(9dShnpTNfG^;q8mA{2ws46dP@g zm}^H608?C()=ivQlHDfo7j)&e^SQ40FDlp0upxSj} zq4J^rb7w-hKg`ZIh-~Ypr3Q(#=nvsk=nC@1hGvhfJgz&wOIej;XK+~lX>l%%R=vJU zV{lV0LYE<=GhXzfl99rcfI~xwleyK1+xaMfTSG>WGjQzKsjj4r2`;9?x2mE7+mYWY*!%$TFLmpU zb3yJfwm%7(>p?m4XScR!wOm^3t=TPb=fziWMK}D_Cr1*@>M(BSbkOtKk{feOhoG-U zZZ*3CgRgG(eoEBe0q>K+7+Ed|?z^EIXEaf#+w)y|>xkMq(Ee-N5M8%!{S&s!-7lo{ z8Hmhwm@%&1lnP>%16@`a#W?-v^pxv(hmc2lg0nlu`m9`AFmY{~SAla~m9>XO4$ylw zQp4tDw)bPZ5mBV?qQG^_rRKd3{M0=6jZ34oJ&uZ}N2Vl~=QFTv(udg!-rzx|nsYR& z^TQKxiWl^hRlAoiH(E}llBZ)ts7sJjtP;;Ly|Jn6r^zPeiyC?@lNDv=1w@F4zTM^A;#GaEtdmM^JD->)NMs#@tymXq>H##yOPoj)}+RhsjP5MI3~pBEg%iLb#C3d zHCEbaz1Pil1bH}c1Y4Ix5@3zHGLMdd1B1ge?G86{HMYoe0cK`ghXa-5y}FIS%3Zq! z*Zj8=00&4jLL_kMT28{+4S8m1tapsjyi9Aq(xU54&tCrkxeeBqorYtdV;=tiFyZam zeo!XZ%bGMK2yY4jK?ZeLZNQUP5)ER8G$2XsjUt}Hn}xdDV7}$RWN7^a#7ym{N`=dp zY}1qmfy9?Y(8%6}e^rX#Ek=Ec-Mvm>SQ8pgUg5`7x8Oz0p^_pq6FNZRcKdiCjym&y z({KIJ@lLaidxcsD;Xav7sSWv@7+3dgq5h+_=UGg*fVLVzc6ptVNz_$G?QNjUI{SL~ zDj2I?u;D|rb}03-Ze0-^Mir5nfP*v*0wY<09hCgmDP__AmCtqFgmuhIGf=n@8fihq zbJbCHzw;I@tb04A`$uKxhwmP@&h)!gWEZJSSZ!I_T(i(Ounwz}=YG8d9~qXoQ9 zv2fR-v0gGlZ@9gG@@X43;*1jjOlm>={gARo9^? zc1>i$GyA^nNW?pC$pf!2_AKp3{EVNcoJEW+ys$)=GcG2&QoW>aS+ zD7SI^mOb5#C*Tp5UiRA^;XJ}>-U$X&>#(_>IP0N?w*j8bhX4H z>iWahTbj)7IYy-H_N>d;oB7K_p96Ik;1mhdiB2v!gkgz!Ak;>M9Okq%m>MB)@H2IM z(N`g?aoi8VWz5T&iHLh$0F5VKl5ja~S^Pud;<7Oh0Dy_iODycu`Xz4Gi0GI6qvQDu z#=j+pNu-a8#%uEmRX-LXcoW?O7;HeHsq_xT#pk6h z?|=_Fe)XoN>oQMFKwojqxX@ynnr+Em`gW4wXiD&yL6AuP0oE=Lt60c z`1Y+%h{h>kO^uCeV!e?wx{B@P5eKqGE3>en<+EAbPUExKnQUhzt^1&nwn(0RNj(ac zD4JJrw*Jog0_QQLN8NEXg&Wo_41vrL%6{A3-OU?56im+^NHe1*ML~n8rikl}8QLOq z5a$yFNED}4Aa<;GGlz3AZ(wq^>&Tb)NNWux$0^;myCX!=f@=VRb3T>)Ac4Pr-HTAS z%`ojKcT_rdb#x}YFffRnd5y=4Ti7hdj~j|;w{`ogfj*XK>t{&SQ;dJT@0Z90K(f6&k@Dtm>nCvSq*|@<07}q{)E1lzK zESiO8*L-yFDPL{@tu6*=X}%Vz0ce0qQh#l>9;E*ibdr1-in(U^#4ue@$wk)lw z;Lk}W0R7(u2QWIO4`|s3+b3Pi1IomKn7F_?&1hjv;;(_IF~EMRHa>`eQ0(f37oBL*pv-?v_SAzrAsxkr!%$B77e0PN~3 zOyV%F`;!n$x+yD}u(gNC!y?Cnw)Bxq`D z*V>YDjloJs1XYqZIg)!Lvxe>JS%E&8JO|{3E^n+m6#^(jfi#NlMGlO26Kv%MR1L!^ zS8lYu(3R#1fh)306VXpY)ZonF9)BNQ7n3a+&u)v=;ZODnkGlT=YlvBGw%L#b@7Ln6 zi>=-#HuV`x#7~F!Zaj}rk7!%L$9fL{!2Ct(2y0k6YPGIg5HgC-$!9lju*!0_51M+_ zWs0cWYpk)?94OH+vu)$e{K2h&BW#+_Vw!bp0xL@^D?%U>2`#A|t6m1>t7F%g8yxD@ zJ&q;7NvQ}+mj3{>G1CA6P5P%fe0{@d2IDt&=l7x0y*2fbgW!o)?Dt^f}*3?vo72@pW1#X+XTAM zJ--sV6n48h67sdgJHy$Lw>Xira@9asIa+&BS;rNWowj8+Y?Jg!Oq|qRl2X8dBvD~o zepWTRm!io+vNDO}j(v*a&0#IJXTLL@K|P(T*Wkoq?N6!}7Fg1qwfHyYw{5-{-@?>Q z=o6EWV!*}QA82{ZM27n(iq-QBEX+&ExJ+!?v4(~~4~F?+qPfLrPK6(apFNUiU8Ki! z??AlwnH29GKMCHnaE+UQ%|y~QqSjb<6fNECNgLjsXIJ*AF`wEe_?&(#tqxZc#E)uM z_C$|Jd{ELrF~{1ThrHs_G}CbTgt>tZ^R^du81$ywqiTblzn8%|gvO;)7-|C%${n7R zMq+@zPH6rS9VHP!h%L+~@>$pME6kg)?5Ah7ZQ+SFn^@(4NTODpu3o?67yE*pa1a$*RqPYvBEh-*YUm8sW(*oo0w>Iuhg(RTrgdlH3{ zOu^D}@9NpxD@Moj7L*POH`gIXdwCc*n-Ot>%Sj|ZHiOlT$jdl?K1+uu>fjxqNi)wy zpe>Izui#tJUztTUz%ji^IkE3``|~`5Y`NB4`J-J1l{2+z;w&=`<{L!G4hL@JRvOM* zT5_E!#?ZJ(Bn+O}P0IWP=HGxT8s@d$WK2`B7iHnqJ3R`W^|9Rz*Pe>5#kh*mW@bgM z4|9y2&^LA~4*}t>nks99Uqe&cmuxsW#ns|~41aylXf%!nsaGvE?E@?Z?$WxaGc3;m z(xm4O{@EWSfB~3WPc@2j+_>Sbw2&Gp*q>`YV;UV3+}xFQN#dQbun#S}73D+w+@fni$r-TVqX;wKo=W+Hu|?4}>vkv8W^ZJQj-vTySjBnP}Fn zaE{Ov^72ECbkkDlcfwGQ61HjXpSWmE;ffVO$=0MMu~>xnF1nDUcCDjS1v4#?#wH=b8bL(Xn{3{ z9Ykxej?*xfEnNd?6obA@d{?1?E;Cm*FJn&aY6)EJ7$!vf0S+F}H|+}4`ikGwdF+=h zkQ_^Qh^XmS&p5b4j2+(rE(D#?)V1*pFO&B`%y+t-##geigu(*fGc$66K^668K0FiL zsH=@&j(v<(1Dqh3f=%px(N%|FJw*DPVl$$ zaaEa3svMn=y~+wL9qCl#B{J75sboPqs7&2x+{b)EEo0a(eks|BqfhjcT>6HYu(@*O zn2$Jc*Cd}WQ@K5(C;-uDdBp4pXAI6RGgYnF$6C~oHRpQb6o0X z>jUJ8Qoy63S*F%qJ*!DpQ7NZD02t>Qk?}{31S^0xB<3h?z;vwZ{{YN)J&2m1?oM)q zfEE0vcAw~#r-mh<(|u4GGu&X}ehW&=ayY`dU<24daWh)Lgqtqz3E4eraZq@aI~^RY{c+^!%CHoTrdl44IggP$B?w-`x$>k{*ZM483&uo0cF)8~3n+y#5m^)f` zx`@}VQI=th4I4UCD=J-@D&5vr=G6}_3p8L3V&ivg>G`7(Atc(sQJam?p8P8q3xyyO zc+<5ljcwE1Cye}-n1e&MkamTGYFg}~MqwR?sD*aDG27iL~V#$AOjC+E);!(uSb@-JI;xA%-V zMIhIRI|!o6V~Z#uewoH#FxdNkRwCPO4TOkuPWHG}r?Gsm#Quffv6=V>8|)wHxczf` zDGwb7flgU&D@{tjN#R>?n$v0`-+0GwwG8_=EV!}lEpXHwLN&Ia2NO?iS39CvAkqvM zY4Y5J`xaY{a4mvEgMcN?c5Y2*TUDeQHTb4E*{QkJ{3Jp6sNB$90>2M$HNZdBV|W2y znA(&rS{48;ameiAYG9h_tvDl`*8xF06br~;Iu=0fACh~STbYN8vfFJ1#JTP{n;RLh zxC)LPi!5++yCZK*uWajVy3GDt9^%s(fUjD}w}1^JI5}N12Er%?zqKNutz(#|(lb@6 z>C|As2UD~e{OF}GX`vhS{MV;twzkl2Q6a5i{6ufs2|=7LQ0tc1!s}-tZcrO=(nym{ zW2rspt^yN>^nln8w<> zU^7qyTeM6l+_-p|TzD8Q#9xpc$LZ>Q-l$V{Ez&!)B1E**Y=!8bmCfXW>c?FTdlDPv zn_ygTA%IL*x;A`aEz5wda4~+{ki(h7cgZoh%V$>Z+xxUoeircs?|=m6hZ5ihy3e$s zABO z047&FmZzHWJ1Lr(`6Uq=C_4}Z8YJ{62j&v$DWOptyn)(~z}GsWoht8LXr#&*Tz}mQ zNQ^bs?XYG6t>XY8{LkX{rDVA3nCmO3-@ZD6un7;MBHO@ppMmrs0Cdol;St`G*F`M= zdB1FGQY}4?yy*2?vx!K|)1ybtU5WE9QYh4`Ml(@S(4@<^sFGmSXGDWS<#|9HqL|TW zed+943|`K~S%&k~s*|EjOmWpkF@;%qG6csI`3NR$O<7SR$pdPKn!&Xq=Omztq=3`& z?eF-g%FaiP6KDw_U&<)pfqh9$0V_YvZ(&8Sj`sOK6djO!(b(irrfOfGtI)yYQB%*zejE1rtTn%k2Ui3 zYkN=L{*P+X8BTg2Jk|rlZ{u3iTXyO*oxg-__;FuD!Qkv*t>v`Hn3o#4-KXWJ1$}}M z4Rken5X+8Q>*7#AQV9%oQJ8*C8hio#vlW4em_SBU)@ z!DCCVS|4V-b#nmn-hYjh6FQc`Gg)6Q@lOlIczeLS!Ulrld(3rqI0|+U4+X_6sYAnv z0lKUwX&)tznD?jhS{aN)q&cm4p&eE0SfC5A`Dh!0bK5lFv#qtvccB#}@Cr;o?-^fB z%ZGRt*D2HE@kDt?F@jRA7c^}^6?R_G2FQcdqDR&c+GVH%u_?C;r+8d0L=!_Qs0wq5 zbam5IP&tbT8wqzp2s#}19cSX0$GExBfsPkE%YcIvf;JlM9>sn`S)3B?^qACjQjtJA z5-hQ;E+o|H9^aB{ujMrxz�g0HJJ_xW!)j@odAUw3ejCgm(i@QA#)ytO5p;Fb85O zlE#NkCL@_ra1pS^oKa)8d8NRK0(CI5i{5sXDwMTR?Y`Kl%QWsbD(v@{AeA=iV*mX3tA84RTA;nb`& zCkHp~wQ(iMfe=7akfz&AI?9=Cupvka%r?b4SLLI~=1nh{p`!4u|?fs|Xkz~!^*gp`60)4*~ECyWU!UTzq#6`|o8=92ugMttX zOLAb*CgG~Fov~=)xd*r87FJ@kEIYZ@lfXgCHxbK4=0ROD32I^c7j&LE_)_wDwd=#)jrLx%(i zBI(Y|r+ODG2&$7EnC(q0lQR^BAUh?tvowi~;bvYUCa%f?VW>$x8 z0Z#c1douH?UB}1@o*efy?ukH;L*PUEWv#P#mG$RnSIImY+<2xPv-h0thPcPEeO%~Q zZ@2khkL*$gg=z)Cx}fe|CV_ODD2Dms-xIkA`_F#VI95m%Iv~=KtA}|E5LTVL;PSsE z1iawW({?uj=KaMRF*^IICK^Yge{6xk|hLdY6{@cDXB#1g`5 zIfB$}jG|_C?@cUWsOR!mV={wGK1Q($?T1@66z9Dd#!LIv9QP$$8D8oV2F|Uh0aKoG zjTHHGaHvhgn^5QxB{VlHgSV3>r}rcRT5{?)7~Y?h64{3LL<#BnA@8`g!L8g4H`O-X z%;wlJNe7`S1IIT6^U81?h^ts;TY@S^^`$mjyz}Y`K6k3XIq+RBO`172BkKv|p0z(M z=TaJ`UiWIDQ@u@+O}1R^DxdamS;4&cq%^svgrb8dSt?rr4zX|Ux_9%1CfFNM89-4l z8^J&k^1#t0#yBVD1ufm=z)YQr4P&l41azik-mX~P5CfZoj#Ii6?XojPd&jnl0_%=~ zs;xEeZA=2^ANz`*5e}6c$)e0{NPM=CfIB+uUDrjiQlw@o3(+M{p0 z@HhGuVjGdo193EDhaS{j$H5(Ga1p!U zw*c{6DRF`IwLknPucX|?9T+2y)$-2=Kie&D@BaWn%jiqH+x(Bm_BYFl#3NM$ywkOL zY=6>h4iZ~r{if#TAFCa&sP|=j?;YaU&k;Xcav8GQJGRGGt~^{xT_x^5fsOi2gu_d0 ze`$XTTOF?3`!Wvo;4!`}jK`m@xN=YO*)3q|p7&{zvDA=erKPvH(Fro zCZ81xKh0?3J*!Z$*5rzW*ashy%>yk#bnV(bg*;r?5T->;^XvI2NC}7c$@HpoiMg9* z$+p>VKD8^C9hI3#JN=1nJ<|UGyDZ*&%&VLKoL z%!vEnG!p}!T@($j~Z&OG1PO?o!I9=38$bd{lhE zb|47`t8<*uLK2JksL3f%5H54Jt7s|Uv}{q%b_h%f30$fkRplx~MZ$Nc1~e$b=|Bz! zG+kljFm?P_2L!u6%4cFg7v>%b7aT~b+rdde`gNc}6|1Syc2af~V*)0tr8L=m=vDa- z+=~FrHCTTU_yl4W!1CD#c|>LlCII-K<&M>qwC`nj%1+&tJqZH}^Ik~fQ_wvtUoPrBcWlrFp+v9U|U zK+f8UEOWFQ2f+w__h3hdn2rjV#Q9E%fzlApvhrq~KayjO(nfB9qAt!A=B_zMx1VZP zKm+6-SSI;dL@k4d1CnF40RHJCRIbDi5vrzroS&=@?FCKU*6%F#an40Nt-lqDC*fTW+2m=Jf0APM93~Y-pVlyPy~q-zf&^d6|^^ z41MED4W$@j0xE7>Cs}nT_)2@);%DMj-ossM?U<1yQonTi6+txJzhW;8FaZ|rxsMK^ zgv+h-oHl!YVLk{D3Z}7*9pVqmu}y%HW|>TmimSH6YGYFxnfNL9UNm!~rAsqN0zOKB z7VNuXrpJ$ktfG8M1))3l4hzmNLnGugPLSm!g7L9OG_AmXa1?V&OsRwvHJqeM3~G!_M<}T( zK+_Jv(z7na^E3^mO&{{W=p+G@|*k^D>K3z0>q zN9kDv0th{o9>u`9x-W)!Z;xYqH@erGwzk@NkUt64yds7#hf<>^wOBaYHck`kkT7x?K5HA^AQ^W%R z&s~R}*i#L2ZMYN1PR;3dK`e2Yu8EG3yz}e!sR4FhJ=3r&eH!sIJpM^~x*M-P#Fmf) zf!?SBnD(`e>``;!{R$2p!9Yw zz_=Sz#B2#UNrd*LU|QnIOrQ!S=?DbKD}W^uKylGbTPlDPDFF5rY%y)SJEP!?b+8!i zv?^l9n88h&q}a^z{WV8mvV`B4F^Z106g6q331`4PKtrJ>Nw>aoBG~`AkqYez)5q? znkPsN20GhC2c=uCX3=Je*on5`J@AHrZAx3gXQgPg0w`fmEh;!h{-qbFj8anpAsqZT zHEa$DQDxFWRcnCD!r8>)@7%Rw$-MDz%y18}Y&DH=Zer{AZ8wzmhy|b;~bN*R@}Cmr>ALqbyx#tyZ+x7{Z|wEBvUaL+fO4F;r{^2 z;(tWE$C-2fTfgSN^_Jg-z0PJMGv4^i8Jy?G)!p@pL~`LLUafpT0kk#Po3E#livxGr zaTC+!A<1|A7{BVcpV2Qk+n0+$H!oBJ{KxM6Nty1S^=|(Fn*RVwTXNI?055>{o4x6u zBNzQw6Z$17dH(>%#s2_R#QupV#h&)uvKr?$McOsaY{TivA!Fyc{{SV}ulkRt{mP{b zxVE0|>ii38Hwn4_00uAmt|#;y*;M!(Lw7*_`)sV7bn8cP{$GuR1PW z$`6}*pcD0v{bZaWN&30){10jAgPpd-65|YcycKup-ehG0y9MUwgi~g&L{SuVF{tRFBTux$@dEBDwu5qo5U-ev1=#XA6_cG;>*EymuNNb$gi-XneSgw81{;l8h zANt6iUR+&0-K+2|r=%`F@Z$dfs$zdckmGRw0HKOM{{WX0`X$~N=R95|&T$!IGo0hd zyQXPgeV165&9)ed+)jDE9I_-2Q|!a zFbo#FH0VLmU^?Z^w{HIcnRE4)!)`s8tY$+W+7I*?qEL?3)!M9Un57y)`HqEiKomnc z?|Y=D`LHyVftj1)ZfLPJ%p3sSU);Q|Vppm0TdaN^rL+9O{TDANy)2LUml29e!%Apk z+_7oFtBQ8S=x1aIk|-f165Rp`VqsDyB`M6@fzXvwtwg3#YV^@TT%Zp|d}kGi!+n<$ zeBd6}GUJcdFpcN$h@E@Y)e)Kjk+82XF;%$BgtQT~E-!GfW^y5%jeV+M!W0#v>r|2L zT#rI3h6k#2JaR{F)HaF3MIt3{xgx4=f)X6;u$adJckoA85$>|FUuu>{j`YOt6YPmo zoqMg0!SbR|arH!R9*IZ}IW(NzKcBFweYQ9Q5O<1u4HH@z?J9Ry=T%+lAUF;my)*!K zC9f^qR2kZ~adwzSXsXAwCH-roF;i$^0g0iApwJ+zG+fppCdos#**XGLuU>mylo$(3 z3Q5DY+fldynTK)Kh^FD#IFv)qQ7bLabInSfScTkXS47HOCKUp-8FUHO+|ppt3RfMS zh{O#Wp`zRW012;Br{XQNK<`sx$G-PHuFOfI;8I8^D&_wGF~V9Xo9EhS*1H-xT?0jg z9_x-XLBgKhQEUJh0b7ay!+G~PJ<;zw&^@VnW~N+snw4OtZcqeUXgh2pTWO8rgOqDY zAs`vhgSlWDwuqrsPM{?S49Fvb=qrnhja0TP7}i`YM)~Y5+b9O?O@*5gPrmnQ-5a)~ zwZ*1Eg-%ggP#azn@a#@Flm7tx`{JMcY&!5w?`aPinn#<4 z)NP%!NLvlX+MoWz!#}Cn{Z-Kwa9#fZa@4Dl^J2YE(z1WQFY#y1dKO~s$o|3MpVX{= zs_`B@;@AHGyd8Pgu-6}TjB|_a2UI!Yl4(187LbNI@-Le2Go^YD3I{FmO_fc-0|{xTSo4ARGr3x#J(rv-odyePZ&z-r1CO*>2=S}K{{R~W{zH8#U*C7X+u*;*9+fXG@dMO&#Cf)RFMmvX z7BC1mc=jsfXSIcVOlGy3B*~1q?xSNGlT_V_RI2c;*L+)}(OK?6nT*wS$r-9P(-{RS^5 z{rP|Wz6<<;>0XP%@4pjn{pY}Q@3@#0r3=0t!{?_mhVMFXQbQBSx6K|qTOa|LrV9L;L5#>Jt4BwqcMhI0P^ z^L<210Q;5(tezuxV$p7cu}mD++~Nd)b|K4*FEZ@N74e*@wb4Ws-dXKVVUQKj!pUhB zP^(CfYFc}0(v<-@ipc7pf{24t(iJhSbjTFkq#G_GNY_IJNQ)2*s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTgb#YfqY(7_cB$YF=9W+aP` zWWSH50nGeDYI+zAL;PWg9G;_WdWXOiU=7;Cnwwu*oQky?W&&2jQk_m2Ob~~iJ#%I+ Xh&4(N?{J`ZGDNlylq5?NfrK6aG4Op? literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000016.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000016.jpg new file mode 100644 index 0000000000000000000000000000000000000000..34f88f3e073df6f6a49ece1d6c01ff0bee793cb4 GIT binary patch literal 28131 zcmdS011`pg68k*oJGh2JSv+}F3_}d`#Rb+C`$J69 ze}~r-vaELGiq6Bj(d(yQ&bE-vFNd&dTYj364L>%TeUF&5LHI z|6QOs15f&8BB0G`4I`*&rZS+I^zvIHQjnd~DXY@$9jbz=;lK>1P zV5^$97{5#K7gF_CT`<*Ocitz#A;+V0PCfUv6S8&a?lO?;!yUgrrH3jZH9BE{lE6ZX zAGLGT{r=@B;Wv3*FRcR7|dOpuTDIpck>BD=_Hq$Nz>f7;^cLP*s|dYf63w{JzLekFl{ z29_QMM~1cOp$Nw2B5`0#Zj^;=l#zyVDPEb}r|<*d;^jZ_Q1aqB=qYdbJRAm|J_@>ChKa>Hk#b zm&&~AxAh6-Z<3CedKksv808!2g9+RDjnuRb!=XBi-k#2E2_`IGRjhoH_1Kn)kEzG_ zCiwL~3~Id{7ue(~(-Ri$3(m9kRmsj%0)R=~0sjd-TZ8|PUs!u}Bgd5s7%_f7f^)XL zet?Kr;89tkl|M7|4kChD9E>NLT-J2*M1Ie6Sytl2->nClZ)XY&R=gM5TT9y>fop4f zZ)XK7=grOX5O70V!dfb9$wgtvc@e&<-vpdibnGqrQR2j17F)9`x^#T~o6A4^rZ=3m zh)HMrN-Y?xT$B3f1a0hhR@a=pHp^`u@-Rbd%hW1p`?JLpic1tBbluVXv%dxozseg3 z$D!X=dVLo8yVXB`KfNF?;LgI(=DPa$*(%gS)|q73`TSH;!@8qQf1$BW|1<`0$iT*t zO5lp$u03ir!LfsgfyE)SB2?vYy~la*&hy`b*8IVG>Nh~xE-oHo)(L6nEI8+n+uWw3 z0d~6nvKMS4D}Lq=e~=YfYPq-kyJejf((B3ZKJRKHU~{S?G>~L)LsrC|1q^yr7h%r& zr}fuUF0RX8?JrLM7haZ6@>B;SYi&K_%a2J>cDoDD(g*D(QyPySpPK&EUCnLM{r}HH zd-%^H2cv&I)aaGo?@1F?-!)>gWk)8jbF)SYSzl>)k764*_ziefTI|4*?rz;*Vi&x8 z7QqgFu)nK3ziNEh-Cl7G6j69^Ah=ExepSL>T{k#kypQJ37qk-81dYeXHk@f#xgJ zFqgE(ij~FyGr4ZxYCuV&5v@!Ng2VbhO#cAR-y(OpouU7!;(qbg5&Q-%aOF^Qnfa%w z)HDfG5oW3&_uWx1+6_h3A>_9Gm;y>`D8k|x;@XYdKfv-o$Zhktno}bi2R^sh>7Q6* z6o9OvRS-t8z+iMbA|Z>&n)2NH7yX7ZMNonC5vu6-otyuSThG5l$4_E+KK%^Bvn{3o zHK#y}&lr}enA6mJt*~oOGjFm#&99Qr$z6zzBag*Gx$6-5Tf%>Ye$`~!`M*UKd5mPe z5DB?bNELJfnFw^OE*&{QevFjb4Aq;r*A}u(LI<$fJ}E z=|opU`yhc;iKen8g1B!IS+*)m)PcN8yOrW!vI&r&DX8xBj=la5yb|W{%fi|lwBlST zS9=Pmz@j+b(FhL9ZSy`Fs!Jf2;gfp_mILj2U-Ee042hUC#>hY2`cE12JUSqc^tw+8~{1bsA|DKU#vr|?#Eflft(g)Hp?;r1Rm zH5{ql&;P}T|AEyQgQPh)-mo}U9t=$Jk;eO9qs%|v_J13%{)3bBAF)O6<&SuEb-l)U z!1f#PXE0iSBzuK^R%6%YYeY`=@XF9V7}XS`-6IAdQJ2`s72Ugv@Y>Jz7U!EP-gU=A z=Q-vYghlSnG+Ge$7*TMyT(;$+*a@4yPGOLr25!k5SEehOV{|JaUtB5{p|o_OP=9M0A9Y2xU>U}H_xaJ}7Jr{KCr}%C;SAq> z|7YjobCAgV)Z*7>)FN^z$R=IZ*Qu?ww=#ir5r5 z(=F;)0L;S5%6d|EOW21yIwkirvS##C#~-hdEj+G7gyre;dh827>LqX8r`jfo7Ye|o zwN;#7r2Yn+d%k6*@1^=WTD72`uFJ_s-Ga+r^UGqn;`ufB7caGDBtFJhEx6UU1V4Vj zAGnqFMlrw#^t1n)phuN|AvVBYfWhwvu!n#7KM<(-`F<|gh1VIg)h)X!PXtnGyIO90ID^D1gh*O4+%!A+{JFB-m>Ac+TWAZY1@PltjU*?vU zBWQv??&(@_fyKgi2!8_*a9!jRQ=0Qr(pNi8k^#$9Tbl|c9?{)av}=nNs{z+-akhmi z=gy{ha0Dr(5d>mP$+#pF^20cx38X!w>DrLfpNz<%ShT!vHz+<~X7G)=5l>VU=>yp} zs$Czf9QLlZ-~<}47ROc+GPme?oH0oBT4o6nmv;J{P%|@5)55NY7D?Zh9%PVI5I-cl zWP3~%`%l$8K{_dD4D6kE(0&6%Z^S{3`0LmQEE_%w=pLup+#a>DZg}BF$GG*r!JJ|Q z(~zf@##?0|b|cRP8TQhB^pH+#|5EQ0u`-NTM#|(x0c5vf}Y*b)$?uV6)2v(LEEC^{@wIeKTgw_AW*9*4^bpM{!>WX+P8g^eYzKo{fMsV)&r zG`@zG9UsS0GA>KXHHbEPE1a@sD7LJzV3CC_`dCIOhkhou*!&%p=k9cV%~?uOLGb9HX1oq|qKb3GiE%@vs?`-7bBKx~AWknQ2PQY-6lPif?3-w^mnxAW zf_-dPxi+>Ux-iDgN%H+bvIB*m+DKQcg3?$>sM4&3z;OUFZ?wWy29|@=duz(#BuNue z7G$LAxbRQ0z#E~U0gVmlJ-O{ z9v`etnXrUtTOt?iK;q`q)7GBC1LU%VlMeik2zapv=H2>+ksi{*!A@x@$}ZRWz9$jV zMy@B3QVx4R+PfIcwot+Y_e=E{lyH~l>dzP8++cZ`^F*iQP$UM=h_zxX%x3d#ujR^+BW*P$b2@DOE`_`D@B(5fk&YWo|_+d}Ve=N%H{~+l!5p3eeaWybLleaFQ zb$KGvY<%!Mjdem-5I<3INNF;q6P{OX3syvk{3OR4nmPZ@yncDllfS*qoZQaW{y?yW zPd1-%EQ*;H5PuA7azQ$;6HQA)Wo5CNq0P{#$h1hzdkhIH;UaVu#obpYG?B|YE_Yt5 zZ)2RcqoE7Sd44e0Bx6!RF(AVDpN2Q6e&J4@|1uo@*8%MBG4lU(ansIA6Q{}%dtvXq zb)>wy*^y<~r?r5(Kt_(7UOGeqBh^8WxXGES0f7N1t?icPueAjHwA0#C&SDTBfKz}Z z`X0c2Au(^+FSh$>mfASI|4x(J8|p#~Q=lDRs-rZ&o4W0X*vR*xufnHS_ zd3skOzZ{KHiV80uoGf8DWy9?YT_1JH&$$UuJA$@HJK>OBOrnKq1c)Sw2J>oNu#@bF zrAvm?W>iw3Ay(?5d-J_3NUgDGU*`SvS10THrG?YmGE;NeZw@s95{lg=LL9Jn{J|Gu z0{6!madBJzqh`Jq(J+O4R!jH&@R#I}$fwAawKt{^Ui$@xCkOr>Xq=qQP%CdSgr_d7 zy##Vb8KNL$yR*Or^4){g=zdOVncO$=df>v@d|$pk}C9KBuGWRG1)*BzbO zWxCy%Kj_fPPBBwqnk9(%R?CrNp#|2N|;PM>~-9%kt#cQ(MB`V=O= z=f02?^H;ju$WZj#lBv>|&FSN4fvm=(WyUsNCV5Cai`2zs>DOVCmZwMcWd9iK7_#qA z^ABlt{f(=7_enYy%5xhzgbsUJ0n~-!7BNIwf*=OY%M<~jF8FrS8R_?D2uq82PP7v0 zUN+rA^4b_odw#OdIw^byo78U3d?`xh)6-$G)bCPqZWsb}3{SG@SSzxMzRxtk(#xtu z%dZ2+BWHn2Xj*Ub^Hg#8<#?lCK4|B;io{m6%NycRSB zZb`bn(xu~AmA+Zk6zEWRznfi=kw&tMD?!D`(&WjQ+fIGDoO%A`#3~xc6f#42aW7>j z%D;0lZKS-VD3g9A@RM|$_-AfO^R&`tQpMGrqqh7#k1T~DBCS2VoOIFJlFT&74kw76 zn9R5XJ(o=;yl@9&OvU5=YlNqeETzs+G~Kj`L6u4pXI4>BUu9lrIrN**y{FAjymW@U zW2XrLGPuM{%Xh7DdNLUdGF32{qvP^esm8fk)J&$#{1N1j16!Qv()++E97ZQAL1K&; z4xr9ZsVL|9y5dQ69EPfH3M2cv{5;Y7t^5X)1KT1~W{e^i<-2qR3ihITMdkDsPPY`7 z%8EKSm8{*WK&=8WvZz6anRjRDYX0Ro#c(jyvo}_D9Lqy{t(cipNe069 z0|zxiN~z_ZzGg3AFNB8s1`4&=0s|n?dR0>gYG2$h2R@@F`a{WZgBtJuDM5q@_w=xY z3ikvgW`2;_CY^s&{S9E!ou80e*zedT#6qOxZI`ykzSmBTry)`5XO9g0C;5Dxeh~go zP7vY!XRtqiSjKfgH6zuWOkhyzu!GMg`xJzFvT$tG(pE6IwnDtCtspdye=J_P+B5nY zI0eo^PDnJ*6gnHS0IM-T$L}7ExKAp9=b5NW>$t&T95X(=RIbygt0<#1jnnLk!FAxFkMpNL-Qr~>jc^0-E*7y4weiKLe#p5 zGyD{UXL3mq{>6k&FrA4sbH4jwYH%xVbZc|xK5LYL&c?!{WJ@19#<5I4FQ=qTj6_b+ zpD+rQhgSadU^6WyMl*QSGD?HGzCvHz8~KGU9CK;e+nfQrR{oxh$Fb5TX2+TGdR37- z`NpT)Z)OP@h$Yk+H?9dw2*p{;sW=fYjli|S5X_GpakC+HJW@`hdGLgl&dObUF+#pA z;BbS0;S`c`_UkJG%0A_T(x4M42fF6&)Kb_2t22?dmNE@%_GDfgIn6x8|H<2yTk1Fqg|%NQ<>c3t1j? z#(}aZzB7%&;few?rzE2@^wm^hx5_4cf~>u+HkACN79qV{#aC1d3<@&@yCwd&BZP7l zBrw2C5$xHd3Q)yw0K1^D^;pLQ@sDSUy)C!SaY|WJ-FBXsX4!#ppoedAd4c zJCFAozT-)#T^#u>KS}l(+GjzD#x?8d`S$$V_$0HdJzjPV)lF+NRk3^3>T6O|-cub{4G_=4qiKpCp=T z^T#{|vWDzDK5rFf=qU{`E0`DUeqVx&0(0u#U7U zX_zslZJX1p>=RLuuzPTD9$1x7EN@V4vb2yDV)@2TQK;^mbZRUAD1JW)vE*UV}~Hc>&M0nEUL?VL_#Xp zGLiKn%OH2J<^-qfwOIBWz=9Er;W9-7XpN zk_Cmz+6pO&cta@4fXVk>as||tgPsNaCJrR4>)(JMiQ$5TmGGjTCM7nzzDzh33dm@8 zT$*w8K0q8emDBSxw$JogsklvQYrcFf!%|5_pWdYIQluSI!!a^@n|l<1CPI(iyQKP5XzvS5LH}!n*=S*A8^Nr2H|Ob^(p-~X;07n@ zf>VFqWpsQEdjO*;!^NpXny8pxJ;^`>MJbZZAZ@}r3Svr#67DR{x)SKpF0I!9_2mu^ z_dZ%Q)U4=&V#r<=%)!%?N^I1kcMDrk3RvJ0PyvvE1q-gOeL6fjj z3|VkEK!~qVePa|Ou7*aSC4CW!u2;n;`F!XmDl8VP5CShN25vz4h1&5w2&|z>=IWW$ zFH$Y+(#)`iGvxp`)<58AMk9Y+vPUBqUvdSaQHj9NN-a@tq4{3QWEAe<=|a#AT`Wo! z*4=xfiN{#CA<$X2>gg1?)3{owuLKr$hs=)rId3r`l~1~68mw)1^T zz1)}hW%MXO^kP4idPhu@<|5+2V1WNH==a~&?$(HK9Z*_;MRUNxE`z|n&%K3Ui<`K5 zHybb1MeD5-dIs41prNjwij_I=(VDJ}(}32z{PxdqS3cotc}lD*^zOs8*vw{%bN){`15Y&&IB_ERYtOy6UX=1JipNDbO&6mPbs^EOL$ zGIDPztNwlgwxgU?mE|mS^K!cFh&1wLpe^W|Mo)&KO38gf-L#jDr1Q`pr=2y8jriNtiu)?rs`noh~uWM$A$Ew?J zfMFBle6ODKHA}wwk1W~q3$X_^HIR)s=U%?6**DD;nFd_ah&~&Y99~lEoHW*JQxVZ5 zn5M~$d}4K9o0G1ty?Fw%jN*$}r(0TnQFIO3uF}U1Yc6a0Qk?Q$%m+HYQfzwM&tG0A zMHu<1#eEVY61&CMu}1Sy|0%!zOlYhk*vj4gs~Jg#fnRi_S2|B~zLZnR>VeSdXR(`3 zstQIL>?D)73~-cU5kZ#r&Xil=EbId3&|cWP7@`OgM(#?(^fV{!tpd8cw#(DSqn$@f z21Zqv>+8ugXIHoH$eDx0dAv5h3w}irl+pI9OMCbwzX9U~hNLH3?VOxMG$|-sytmIQ z>D=an&MA43zNoTMFWg(?nR_^3kEHg*3KbdHc}CB1j689D9;8&8k%A?t@L_|x=IO8) zI!B`$SgW{!t?!#14lhEzSmYpsFF7Tr0uBp1U$8L{$B|NxV|lFXMlmU&mc(v$k8WCK zOxWa4KJ!@KSXs_uX87qv+fUGA3vFpdaLT*?G=@yc{QLq?=Fb_7b(gghAU1{to=;#V zE}^z<<}ASWtHLm1M5A&}MB3C9LW1LmGGc&Sqywljj3MK~HH)>`!rhb5TcX}7z+|E7 zZtzO@$Eh!bR%h%n>kA}|y{Mq@4I@s0)l!B=Ne&GaLSv!Lwb^r*LT@I9dmS|XcI)AJ z+cEh}BAL^ohP(GrJ>bF&(jwv=NjIKNOvT>|LoDd6}oI^u4F$yI4))xdijMJdikj z&n}`2sL?p1eIyvHFq(2{B*yn)s@!q!-50 zJDbWU=7{~smQg3Sd|1IBpX=DMYp>VT`sI5jB73+g z67(r3^(?!7HiUhrq!#B2I%88F%SVz*LHV>OYmeVGQUh6^Y-UW3jQ#Gj214+M@*g3s z@G(67->)2~EH2Mh?;}v82*)XF3Un?_-S0YABYp!|r?!($i+J|d-pv)o89;8acKy!K zUEGukpXrByh7Nr7cHss;qCI68$)Tu^A)57NuYoyMx8nOoOc0iNN0Bp2(nLplW0Lx0 zInT=Ik+!0$xmpDiKU7`T%Ej4crQYlt%;sa(wL@D?OeY#9DA8lqu^>EdrlbQMU>*1*_ylMj-J(F6PM%eP8W8lF$G+78gCL}2XkX#$V0sosHp zZsQMdcyATZK&yN~EsVZm7cw#)2oM=n+l=1RP3qOJ+(KDVi3lM;`%qX_Wf7RUXG1OS zUMf_+(&tnS8wL$A=XXZft7LYTFTl#{>x)`gS`3^E6RuJYRkr6Fm=lPDXOH;1!V-m7 z-rDEFO!3xtrrJCYiNv=fXTrf=K+z!)Xl~B#;6(EA{rGP{%$F!EC&U@Lp3(4vr`rHY zanUtNduXjQoi?w%vLxi$-brz@o?O|xX3s1JC(l&( zacI!hxF>%jmz#(4rNtode1G*Kt7W9WeAM!>kPLI$zDwce*+)^{n9mJsn^7?zwaxJI zV~B*MFJDyn6H{L>{}V*I=q1```2g7tG6ukcS`1=6-Md)ste2DMa=D9RkiumXYMMZ`JejBB5O zhX#kha<{7<`H`nC@lf`p{IpQKDq5_G_1}!BKE~5+u)XHL|MY+SD`w94BXTmx(1!fG{WSTsl)6%FCC)-H7&4>;S zR=l!Uvd*5T%ehPRafOryCSO_5c|Kyt?-K1FNF3CM)eGBxvEIwY$hZ%Qvem|B^qe5!@LGDZPxb#N@*+Q`b zL;4}%;*E!hFQRJk{>LN!R%cc_j`aO>P!u$b@8Ru|cPKfwdcAYS#I3$Xh#R+mKed;I zPS8@BnHDWRL)bdjiB5=*FHW5o{@xQd7II3nd0HDl4xEyC^^MRg0G11aW+)9q0110> zNR-OA8y?meI||VRMyg(fK%pOiyGXPI3AnJJ$3>F?ceCUxeMxo9DrN?RJ_76&m=cK< zRqV|b3A91IBJ22dhflj7H~|{Fe{9G2PG>wSm(t37UFZ*SSlh8%sn3ywCi4&EQ?jeo{u;7$LPciQa6$1b z0Y-2mqoG~z6`NA#GEB{mAg)oUd~Fu*O|lys+Jy?+Pz|_l>e=}OG(yyet6Z%1I|e={ zH&4m4OlAlvFnjflz~k_f1XMozhFT*k!jht6y}&bBcdah3PCheNJPx~xp#R)UTs1Xf zzYb;-v(flod7`?hP;uQtUuGGrMt;-Ut{30j@xHTIg!N*>bgCmr+fOs%QTSBOB;@OL z1cU0;2c4v%_6OllN;mf?bQ+bDh%H<4&m{+F$I_a`gLRuP7Zc5vdrv{#*7_Cht|Vn{ zxL)utRCpF>v|D90l%=dA`fmLv_aG0%6;8pCQE+I85_Md|#odW*%<|YnR?xTo%&Wu{ zs(^?Q{Ey9kn{qsH(FdpQex;UShPYUDH?@FEr^F|oOPkf-0AMjllWL@99Ws1O|2m<( zxpWjL%rs4td#xycfBn9ebGQKg+*MoUdU3qgG(<(5gUFd1R}B5gJ5wKk9hjp!@9<~_gz`rc6WAtG>ZQD$sm`0^YDXjLb6}AUMS$l(sbbV8ukoqUj+Rw19XIt zV?}2eKFH(z>Pr3%_z-$!}y_8DGSOV=l)s788#QGC|4z)ull>9}PL;Yy#VeqId$;FLXW6JwmU-+IZ;4%?{fx$-( zrJJ(joaIkH^#o7|cOWPSZE?pi8R#ydB>SwFoz#pd;*pNU0!#=>*5#HcA;IIhOtOtb zQJ({|;eZ5j@q?R!&PlF~7Vr$qcZNR*RrEtg_PCN)iR#Q#2;?t6#eo|XL1l>_I;P({ zaOa$d-e$C|#0SvQNT8HdTaH*7#r;G=g3rcQC5Td{A4psul>#I~(b8Gxe8QAlo$v>6 zG7k182iOmuvz%kw0k0H%GbzL}Ra4#9@hZX^-Zp$DWBFpJT<vQUbI0v%{OSmjsab1)c+J+;#_LETnmk|SV~y^ORg(mBbM zl#T)v>x@T9!9h4W;3601pxCdb;}RxhEE!p`B>~DT_)5wl8D1zXra}(lyXM7Zr3A$4U*9li885?v?eqJx4o+fT8oETw;_F9wll`_EBRn|k{gYp>2 z3FgCTdw~g!i0S6*CYoTyEZFdpPHiRzE($KQv>g3F__b3#9~wnt_u8E`LdM2jmhSJn zg}6u){&&K1J@mMt5&q6RT>Y^D=xE)j(ynX|YSQ8HaY?}*j5jf$a4!?j)X=ZOJFvs! zt5hs^ZtpTAnKZ$b;L1(md8`&vr1V_91^;r1Qu~ajLi|;t@Wweeug9@B^pn8*Ca|pA zcC?$H!BrSFhD0HUucuApS>E7Z7#Ed|%5O!Fqz5U-iT7L3*2lTH(GV4P|> z-V~;I@86{4wsc8uYf*)CO1eEYrQMLnzIHf6S19EvvtlsN6r#y@WkZD#aK5**No@w| zPwwYW_*bzPw$7oRlPBi6}1DcsYW34f1a!r3=H8m96ioi(~!wOHZ8-hhl> z9Q?=7P|76@LTgitsUI9tdDDW zao8f?e+6;c8JfirHS#G0f*~Cu-v-9d+wiap1-m);VehV$?q6$AC*Q3~C3fG5 zc8iNqd&2>_P)xT4CuolukQ5SI;Hcw1h>8zTY%*IHY?!r@d$rLw0!+ zy?z58+|rU_OtErdq1}Uc_bxnbF)uM2a|?qY9!*^W!Tq{@C18-)0ur4~katNDUW#$! zO^Imo8UJ~f&g2s*-7~|f=JIvbTjw7CQun}vBTM~!8vO;8IU3ahZlk2Wtu@dod5I+G zQ!zL^Wxp#d8XQXvMMRtK4B%Rqa?f#@%c&1a?eH6K{gAcuOjFL>80osk{n{$7f7Pvsc9?WE!aTbkk|| zJ7RnW$i>t-?GEz?G#bk6R^u^&ZK*qnD9h)eYwqCSTVio#2M zKPHe;uDM3t!Q?+4;k!*0wm@3PGS0U3YwGe7J-`m%7$RdhGwgTd5To3f1!(?Y4>kLag{v_7{4W#%Cf1*16<%7gxeOqdUF;n3eqU0nG`;PObiMeYk*Em=T)H<}MN#r4p$HQ6 zZcSDE2aL&Ai%811G@24dEYU3M7{M}GX||!`QqCVTDaS6s{;)A*?Ch{MIj4p5=`}1L zu3Jhp<;sj*IRNBJ4TcsM44T1PU zipLqK+AS^f&9UC7?dBpX2?cBtgxO7ci#Ea0liLw?vLvi1eoEF5fOIgK1I!>=)d|rm zM13>XuXS<%+%0OEgWFaXO@=~)uqhcBgD=bCfeQG=8JQ5hl>l?LPGcc3MI^aZb@tKsiWrnqn%Sf!}ww!~vHR$QPe}aDHF~NArom1$9NJ0lQR&3~xfH zzOI!%e@u!!D!sY%{HL0VxfiZmsaeNi%@6sHDM>hk1r+2^HACSJTz{>i`&Jv^fB$od z@chUATJN!f_1Zx305;ntVT&uz)r75w$aa*Lnh*tHqxE2{tYPW;G0p=%X+u!p#TzmrzdYe z5&zK+zg&vJWK;{?4lDlKDCEp`+Dve0Q|Vx!7Fl-Mz$Us&#&u zp$Loo=O}E*{muJ$RkydWElFCf@eB;XL})`)l%CdrA`V76{_AUIjIg91c&M;aS7T$~ zczk~ALCf-)^wbaGEgt>;`$xqhgac%sci=FE!#+-RB zSwr8YiYtpDa8f#>ad;7v1eq=qwH2mZy^~4{K~SRJT~}yrPmhiOP3apJtIuQNR@(3g zeiZ%&sPZ(FKOjTd9L?^I&&&72`E_fF(aQ*3fraG+MW^2Y=2(|z`l;Ad?yaJpbqB$} z6qLiMOGhgY$UBHm(dSW}-fJ7L~FYWLC@uW?2m)d{KLvZbQ zWYzlqD@_wdwQq~8lvDEY?T<2K#lbR<-G%pWOqW%2vEr(Yfu~EPzQ^&|%ose=&OH{H zjO6<>gQ^V&7cjWF2&fgKjAISR0J@pAo_uCT1ojixhp)-X%>!#A?IEi>0v^oH*t;h= z7W%`nHRXet-RYzO5!pTrgXdI*VrDv`o%-gaZ*w(MbLhbU|M)(7ifzW>ck7BUG7Qut zv+HesR?y1RX0T$ReBOT3$W0JGiM#&a;I4Yblg2lhY8eCRj_Hc^`sJeu$ zmyzUZO4v111rVtxW!7NS>MHeHt?NBO*2{PRD)yMA{eVxBXwqBT4&AB1_2^sfh6H%k z9Kj!Xu5Yc2;Xsx-A@aCJs}++^&MoQOQVpds zs6&n>lv{A1oCkhzs;-Ad_?Z@5y1g6m3sm7w(UF=3-(9Jlkax7H(?H?* z`@}ZhwF}H81?pTAM-|RJC%nn!hH8a8pI)j(mOLa>pm%XsJ=M$J$?|Bt41romKT2uZ zG0GukDOnXhx5W#0$v_yF@RD8UD+=JDP9Yj}!zpwY^$9zR!0gHmGt@NalNd5q-uMCO z#?-N7aG@K@!sYq>tX+@$OwtIAY;8h@0sfD=@PC)@BgkKu$NntZm#9Uz_qJ&)gqJ0& z((RV1;C>beTQ}E7Exu4W4qumWW~D8U3our=|8)|QBVNo8T?Q2)$(&jrh-+n(|RvjI77el=x zXg49lc5la=|Bvn}RyKFHxvCYfQ($^^oB6$4XV{s$K)OUrJB2 z-Vi8N(oOXpJKe)KVKjV(W{bt&)sLe-U-{!E;8tnHQCbM9KyKTev>0^q$Bi>ynzmo# zUS3B>{MY-+iL~ENpXHWUxHW>4xyVrJ`J@~BEuL<9$(ei|_1bnQMt#?9>J*twqi5|PE_c~hJpYdUv{ zmFn+nazv&g3z-eD-Ta7H5g#81Hax~jaerGLHvFQ2$grkT(Z+A(6b6547=lBN56oG{ zJ#hH8dU^4tv3!a=IYL#=^~Y|9TXa(4%ZZFeJBv1BvkZeGG+^q>NzwaLHztKAkO~f^ zu*OD7mBCQU>u$-=U*Sk2$u^7!6U-S^SYx|mnDOOZBlGqb7<8bx2`Y8DkpQ)BLzzzq zNybm9N$Q)xjI&TLUMToV+feoQFH$b3V+GHl!VB>XS;bOkByy%HFoI4e|_+IrdNA9lH+Y z%+Ywhk?RTS^tn-^>`dEk=)p-KD$yAB&!o!dVIkL!zWVU&hRu5H!w9$IrjH$HmY#f$nzZ;5FU0EXu2=B!!lJ$9+YE!TGQ9z zHreXSA%M2^RNVsUn@V{4c1N~#T(j*I zI1>Wy6fAbzxt?Y=t~Cz!J{DK!acd~wC|Fbod)8riQ5>;n8tfmU(1#i`iSP9Dr~R|< zS@bncLqLMzfZ0$~Q?*#F+w(LyHgk1mCUh2aQk3)Pz2Lg$UN~wh6*R;;wx>rcSdtta z@S3+EYJFP>J|8Ayh&eKg{&3CGuq^)a<+Q0a*4_BcLEegFe%>z0Xm$gi%1ZQjh&r;G>Xf^(scj0Dt%gRzvvX^00GFe}v$ zBDc=W7)Kasm@*~M`NG_ri~6O^)my0L?tP^~Vopk^;(pAo-1gS5H6vT6JAzJEU;w~` zc1tb6TNy{Nc36^9vnze#7Or6;d?hTPIed1z+!H$P#HhDnfQH@ z=kfZbX+pFCeM%`dsGq@N49itGEG(o)#FuUBa;7t2Lj)(U*lO0fggSAbEV z?(CghOB*Fk&G{dk`Bz@-X411|2l9m;C1dAxSJ5iO&j~qw(GG+JKC9WjfNh@ju`Lf) zg_+xV-geY~mj9q6G+|{f*Px3Wy4)ZLQQB=kx>GS4dMVutAITb`e-odOj{$*Vj3kg~ z1UpfHR4v5#r)!DX5ufbDnOzdBpk;r__$bxSrSLIswU=hosGJwEs?_7})P9iB37Bfl zWADjYJGw$hFCp*|5aP6`JSP;|%tFavdZRf!#XwLW6YAhv^=0ZUjw*j3QMO%BrV*AQ z&_f)@-Wc;b2WH4FbGkZK2c3R*et#GdJ0C#yQHfp%O6_wiWcp^Ot4m<&0?Rq8%eX+Y%`NXeEYy+uF{ zGn1Q0kv1C=%-q>V8Bd{Uyl}y9oOM7;Feh`++`ZA_hdOdk-tjMmFl|jrrhS112(HD2 zsn5({WfhCT8GVoeHzPlh95!Qi9e`Y) z!N}Pb1i#w*#@nxHO2@SsgYnS@!&;dxU0D4@bHI{{x(|^w!c6>{k0TuBWrD|> z$>uCJv4t5P{}XhXVW$-FSW6~NjwPbtmU8HOz&=>WCYxt9ekC=jKK^6wNrZk%6I_BZ zqJBHffOHsb2AQlUYKCnQ!cwG*ULl$ee5E? zv!)4Y`Dk`Y-}FB9*mWBVQdD1)*nI36&FGz;RfkU2Z^0W=oD6e%)X%L z=T;7BbckL+Mjjl|&=p&v?q}cm($5|Y)f)|)*fH|J0k_YxHJ?@vSqW)TDrcc^L^faP zDV||;z22F0roH^pq|U2_>QS|3fP1KGsSD5Z)txS{rZD}rf=*GNK=TPh%gAGvD1?@M zf6MNct$5bX491mfcc5iL_FAfZ(n+f|Oh$Di6cHp5D+_q}`dZ3Xd&Br54mZx2ee7F# zMVu<6_fk|B^^J%DRH3CAS8kj^S+C~mQw`Q7-N&tu>)~3`?iy46^)_-Qf3~!lhyJ$?kJzHGBKY(fG#Ub)2U5D9ZqO~^VUOIp(8Y9}%I)H&v22)&Ybf6p|UHsIL zTYho75z`Q0x2S+k7S=hloP?NAcq~b(? zMq8;o`<+LC9FF*CE< z89oa)5@9I`occ*(bY^np{TVh{>(d=8;Bs4rVTScbAJP1BH>C*oijT#6CSEJ@wU?l zO=_b9(0%W?M1gRT0BIs5>PZ9&%XpiKigMd>zkwW8un-9m;)xNsv~dRBT*@D}X}xT3#M^;g#7pAFea0Xw_6adQ_gD8iNS z4yC9eOv`A8fixxs~Tm>!u6xQtmw;flFXI7+kb|&a;ARKdBa|*g%AuF z5156mAVFGtLmJN`3huNRy$F=m=i0hL_iS$9)#vILd_+V}akZGdGEx$&X^PgQ{7W== zSxBmri}#En9&So&UP-o~4$+&I0^sv}vo#*E_e*=MET1P@b4S3@ua(h^H0Z@u`Np$FBj55)4S?~~KQ#gBAjBUFp z3i*_wMD&)|mJ|k|F!M3i3Y3*3Q{_I#i&AMmMKMQlr1df_YPwYkijxB&Ur2jau3W=d zB}qBbqib6WZAiK`AlK5bacE7%6r6~MsmU6i;>tSy_8wCce&xfcx-j>MR1+ZO&FmZ{ zu_-c77nGy~5;rx?+tIsU}9w8COJ;1BPzwMUN;2qLH6Vvd!HzIXbW66lj_Q+tA~klYVAA zwD2yoa`nZD{$L4}=m$ zXgXfm6Gw@1!mO;oB_}7SyUCl2Qp#VwhMl|m(=i8gnejZtsG!Ui@Kn`~G){5C^3bzM za*gk(R>wTv36>#IO^t#Qp$4G5%a;wLYAGtLe-|5AfVepV))%Rz2}?DiKC29UWBnlV z2(iiWu^eOT7zY^p3}75%7{CY6NXWi1lz_ZnSmVq%dABBYdUDCuz+)$u710jar)gIe zQjpqt>BOC*v9O!g)x(?QlwaCeKGNcpt1#A99VaR~&BUZshLZ6l(LJrfOPBOHHO!%Q zN)BVUrPwgUkC=NJ#FLu>)SQWZ9oQi)KXGzl@mI*8gifD*SsQ{Km%aowfJeB&EF!6N?BAIj_}_?Pu4Ow4@j%b zO28&3i1e$`w4kKLMpWIi*9|el2mZ&!4p~RJ0RxTK22fwyk{*Ps+ebC1bDg=&;1KWs#*vEEwHq_2`ORbgamDI?of<$(x7Hj}b78ua`dgx0i zHooF)6k7aVy%VM_ft0|Z2frrqc#v{%d9U7s7WLRz3IoNRWb{?(DCvN6IXtWt;c+PQ z0a?|V;9iaOd6|j<`AVtQ-V*#lRl#h3?MM90Pzr98&;C^_qu|^#DgEK`bd(>&CU&VrdfHF^lDUxpTt|D7k`swm;+L6j=tN1$Uj;oZ{sN;}7B%(R7th7j?`%)vNn8^U|({!RpJ4(xj;s zuHRsKN$U$Blq*)G$TZB1;tSK4+nHKP+ALBjSOcEAZ4?mrgw zyUDkxY$_yj*!Y{N#jGJTB2S8)!a6c~PFDIGsV8lEPGD`V30AGJ`-&*L2m}%pRE*WjMf4q@$!R}0;y-rEb6HiJ!lI^byt5=NWYLY0->}7XrxlOd7of*pv|@9OQ6ux~&&Kd{21K)eYMaw8?sI)tlXS7*CmE zF3Lqce{-tg)(9#z3`atC?tmZJ<=qBwHI zW#H^y#uTQ~6t&C{Hbo4boXiKIh{N5&+*P&w$$(;GX87LzrGvm>uonEg7Se(BkJee< zrET~PkIHr3y*RM1c>wo#!YLVfSopc;~%XMw3r7!W5FbvPfEEaUUtO=iO>KD49$( z?prFVXs?UVRb46&=$S}cc?(!g#4pBtJ9d}0Q$R*VCDQ3SD+hJ`KRMoS$_Ei)S)zXnj47n|JKqt-F@)4e3%<@Gj^2Dw29T_8wI= z$R8Y``HXj56k_dQu!&9U>Q|0IDL~wGnh~0h)ejTgj^1yZNiKH!BT$2r%a*UFwVs)b z89urgILFpD2N?Rs0Tw>7fN_tkYz{*oSkMuZwU4f=ix$!elPXGJW=f0)mn5^$Gc05^ zm5Q_zt6E5ynG|uvb(R;a#NQJ+1c@jA0Q1UW&F?}fyh|#)*F6TJ*xSJCKDW1kK3<(` zfMP%bWLG}VTf*xP^)KlJ zPBpV?SFL5@cI|Na`t+}Pvx)0$RxBwiSwbjuC&HCM9Ax*C!RAbuG3q&(<=L{2ib9Z| z5sLhyPq3d;<+4c!bo^h>&TekyEKjxB>ZiB1sycRmMc-9QjWkpku0Z9p_`IzvStO^7 z?)$r*sWC@F{#S&_OM1#|S+^>g`Bj;axyw>Xj|r@Iy)@fEDODjp1nSO%G7jS}CHKu* zC=E7)lUb9*#}^*IS(*V}HG8(8+?r0%^KK`}CrMPEJM#|wJr+;*eXY*f znL9Pf(h1xp;a$9|rnw0n#yTFV)=CK83(4}HBaxIxRHe6M_&Z)nx)CQZ_3T?!%Slx$ zC$zdvWND&m@kBJQ>mFGY@p_v}Sg@@tFk@G*gQoJrfP(WI$Z0gLj;itV5yriMQMeX7 z)D!L(m266bbw4@|;L|c$btK-@G6)|Seroa2aYOsZ;{6)TZLBuYX424-P=i5IkV|ov z6H%3FjL~^itIJj7&WigRi}0;rWhOgZP8!;Qx#HCX@^pl$zS6rfxf|GZYO`QiqRO+IPM-XD8&2XKoZ1Cc);y{n5!RnV>v5z0^EJ-oM{#4aFSG^91`Bgw`lxtQmu zN5%B4+Y%b4;)=q~ay=;R*W7kWnmu~csVW=656;>Pw>pG&X8jN{0p1=yD8iH@i-y#& z$(&m*Ij8$VW+mv^J^=DH8K{Q!i=H5gL>P6FvCC;B+Gf4AP`tz_BJ!>CkBex;X4hIZ zB&UIKSy>pmwbZic3KZDf>gyNp+$gkyt><9puy?(#tN#E^<1v8?#GfL}3V-FgI7sjg zCc9#<=vMHyFlqAbBq1ZT^E-K$(+!s;o=7CLGWx%%F!p?R(wp}<*4lXknc{n)`iz>k z%o6P;Ut)@mg3XLW^4)!J!DV}jliOcmexy8JxB*x-SLISvJ7e1RGr=&VEij-6I|z_p zSRO{qOZ`?ADzsqxrz6PpyiwQJLm)S6DYo);5>x8;v4 zX(+ETws*Dd+Cq~#_w+;c*RX{(M4lUd_vBYNdxS@YRRHXK{Xg7A>i2%qH4Yr2(|~YfbO$-B0KK z4L)_X+BpqsQCf!b6;#$>JieyHbq&7`30Cb~xMNMtLgBB=s^v)B0X?tQNSKj1#@@&E zydMs>{eEJ7i$kA(b{2UR**)?6UsArm^@x16MEn_!fmAHK?)mznoiS{JI(D!dJVTJ zODL^U?d&3@EDkAB=Jc~!@r6kQ0#XUcOcTp{QtmFzt7^NeT2geK5TiS@_`a992&UYU z*O?l92H5id02mGA9eQn>N?A-whS^aP6HYnNwPz5U*3G~QM36Q3LG_w@T;s%T$+e41 z>B_VAW;`ruuB=70>w*-dCs9m{yXf;SW2VL8rD#$Q%zzY=oCK9;#z!aS#w4ifDG-Z6 zDG5w!gFZMJZ8G>b{{Yd3-W3X!=k9gk+HN~-p6wSBxza%+I3U5?VfRuLCP5qKkH;KE z!)-~jy*AL1aIZ>&uB?y=IcXAUB1}DNHp)_fvR4*~76mm0t2L%$ZrN{dkH+aiJgvN0 zUDt&_ZDcuZ*E4Vsa(#RP0D-2F8wUs32WWn(WN75Nyz6BJX;4re?Q-R}gO$LATUNi5 zNt%)2jupxI+%`7G7%VAa2$SX76;V0P)pwe9K5gzHZB8+6DNRc|(z&&!AC2E5Y8Eky zxo;VSQl(b(p9)m~k|~fR#beg-zZY?K)}2HWq}3(>2_GmQ67p4vT9tC)samPPRitjh zbk=sBdB2q3;`cXaiY>>}g}=L8+U1iwW3@_+QHn|mN=lS-DLEB_s57)imXY|D$tlCbJlL|W)a~uHs-z{P7C&d-czL4fkUDE-p zHEdG)r9f_dwjesbWpgdafys~ zy<1r17z!GFW7E0e7sTfB;)H5qZX1e}nF6PiXA5&)vIQyW>v>a0J9w2_qyya_1~}R@ z^NHv|nq=UlKqhhprDqhCE5!*Bw|}PBqG3odMMn1)shLYN%b?fBJUpMP!!07sus3db zM**rOeU0mJZ7ELR2V>minnOU`OSgYG|=1o{-<_di~o_hBP zeu<@X4QvG|Aj}=5sK~@OA2IAL!Q>-WU_tJqTNX7Gl-7`4I8{)Cv9zc!4OEFuRN0Mr z3J9IgHqFHgs5LNnoc`zOl$j9^453N5U}SfC2WT|YFSY7Te&%mM;_l$uvb}cFczW8` z#430DA3rmhv+;L<1xWnl+bu zPC{C0QC4NLsTvE#K!o=F&o2-S>vsZ)GLk(`z}CcKY+(>?U%Hz~-mK3dnvVbrM(G|H zD5j08YX^hOw~E7JRKFo%P@_zh8xE6L;_hLW6wM&uR8m^$yi>*4R27p9IMBwVs5CJc zOIW1B)5$|I-|WfRlbBZaF(!EIRt;EHfK{j{NHOZotE4|y);7G{IZ&xbh;rN0-2ADB z#kjDsxyz`@y0f(RI*h5rIaS9(Wo7Z$QtNCB5U8VNq}sM*Lj+CIwC!v#fJ4oIj!rs; z_OFOY zp~2ZGv{tZXLSl0GRJ3o}K5$yss7(>12ps3w^0y?LsXoEI#N5-(9*YTxLT(GnO0TRa z(MANo@f8rFqn^yhZ(j|HQmf-S50yL%sC@Qp6+#VGI7Ql8E2TemJCYo9GD5dhSR6C-TI!Qkdp zq*=LpPL&jh{_h_CUC#o(BE6zS`&*1(Qt;Zdx3Ma?K$2nyE{A=dw8i06S8n>~0%~=G z84nmxfw|@a6ikW}LA}thE3&i$XE}Wl^yzW-6?;pt^jA9ftS_l5H`am^QEcf?coZjz z4vZ$H+|{Unn$VHIapOI$94&yPm8h7SarDXPBm{~G8=BnRoVApnwUYTQDqOTKwQ(rA zqJW_^06=DMobqu3OXAC5 zL{zz~z;Q_l5?w2}38N?|k_4A>)dACORBK`uWCrTZMmEVQLwIz{pjGwmhb4uRS)-xC z;>bC;YkQ!=#?b682>C}BB^VDTN(-?8)bC6~0P3h3(=OY_63|KU2eh&%qMU5ed`;D$ z%2Y!gG9D_)8ut3Y{dCU}L)cqM#9LlQyNw^8eGd*O8aD3LQ^A(SRr{nov2jBp*|muY zokOwZp$>SGP)Nxj9>;N1TTgq3nuVr{9Jcl|G^6*6%Rt?NPHn3{D_LVnhjogA*B}(d zSt_^)h=<%gvubOcdYj91E$KX{wl#vQ;#?}riAYH>DZ_JADh_VbaVCXS=JJz7bLupj zrg#Th%u%oSn@AB_Zgj5tZW;m^PQarMqAn>WpL5X$uRU<2!E1zpvF@)ka(J^n3(Hgm zsApO2YicKH?_g<0hY4UbEnf%`T;CUWSP`g47L%|u35M<2_Z$u-2LduflbQ`nF{lD{=n-w&1V%QTs=^_-sZZ{wZ4TTvf$?c+Y0R$Tnrl zk9Uv%09EFU8bAGR5BX91U5~Z%7F&)9{HuLrFy=~`~Z779=A4830m9o3OH7L2r*Euher)wM{6 z$wFzCx)zwLxIQRId9xy~p-@~&jNc?Z0iXKO_&PTb9bjKnJ{?8AR5*}HJbu4Z(M z>CNL`^D#FH^=aL}Y^ZgNbJs$O4Z1D8i@At|sL&maEn#XSU~7#~pewxp02ft2hjkK) z4Sx8K>YSNs-nEGI-qaTW5cTk>(t@+M<@DmzvNGF`F}esq*u`!Ss1OQ^E!66ACPNUf zD&C!{^ez??<=fPD)vs}0b>0Ea^}|HHEhxSnBiA-vXlh*Ut7T_nrpk1nPTIMbH>np_ zkOf1UfJoF`X%by{rNRK-*rKsIGf65uNp^vgq&v{C4P7}M6StBa6b~_HQI_wR7eMPa z(11?nBU^B5F;K~B`HBv~b5$YItem3ksb*ANVx}Fcw7JRl+D@?{?y@Os53qeS0o8QT z+}66{a}G`GcVL|8y0YRa1Nvn;_%ZSZ@Z!i5zcMXO{9}jE=p)hVP zuz$Lqlid83>`nYq_1?Ivir#&j0N@=rzqD63P=17TUc+|TYnxcy-1#KEnJ?G5z>@0@ zOf*4RI$i3?($L7UjA5bdE;|u~Kg(RFYE@L8j$!*8Ct2a~FrV!1AE>Rrx_0-$;QR|4 z3!~}{Y+4P>I)Kc0hcVNW@;FYmzhV-v=Oc2Dh;mo3zgdUISj8r7s=HEXq0DbI@>-rg6$RT?<}7 zi2t6Jl#BJ@LR6mPl!3{{RgI_$}#4gOK6HmI=3LAW4m+$vjhcp^eIY6n&%$xf3ewr5pN4?MnuqW=GR1p5~4K|vv*^a^jlI2iaf;uOxw!U=51lM z&#!AS?OO^_U2F2m^|r5Ju4yV#l8^@(?1x?z7ef4FN>h+KU3kn^rD)V4Iq^SsPTAtf zqQ5iuBRJ)yIg_4FrCPcKn1>R&(b%OaVgZ50+~^Bd4aEUiIr4i-R;8xWDG}V)v*5H< z*xAE`peo{wPF|c`Clhm$bU+h(K_MwH%OsOwCh;_@D%Ev8DE|OW_00bO(kJ@X3xEFr*=k?GV=&j1X6_pAn3SbN zTM5Jf6iiC6=ITK)kuM|9zx4P20Hj~-KUw~*Qjynpf)u0;B8gE>MuLQ8v82;aT3-JE zhGg1yuCwTQtC&n$l|O~Xe^d)nFVFe6Yy8{M{{Yy!dfDiUto#h&8~i>i{4Ov0z3(v0C=B!UTx9Oz)S>HfQK10V1 zizEJ*7yVxU0Oh_X3yTB(mlyqB_v|#h>)%a#`b(d*{TE%_xQn=Ol`SBPvJ#@1i;-9$ zQJ$8Y&n2Fm#LSH1adwOPbR{Vp&1z5f8%c>Z~ead{5+==z+k zLlLQKQ~?^4OiC_fR7`g?uu4^(_RuxYBRj~&5BqFi^=tnCmfG^Ke}={X09M3)TfrBv z6oPMF+5Z5MeJz1@;r{^3clt}8wDBwrRZGAeY<*TS8jt)gFF)|t{{SuAi+F_3{5CWC zwf_LicXiU#idd*=H2>R-_l&${%5A5vUt1&gw?%f`qKjqi{JkM zO@H#*KZV7g`&?i3YySYS_d20USiDrFD2*|xMN#0pl#=|qFJ2%YI`qIZ=3Esc_k%qV zN&2b#ITek?ayLRT0Dscr=^ysj{{UhdH&$!mQc6yh1tm#PR09Pn0E#t5>!<$!CDZ=^ z>2syKiv;3vWK;R){zSaB3T<9ht#c-lPvR}Cy=1CDFAeJ2P$&-)Mz<13y-p9LhhDUe zq2h?7*^zx;%vW<%679tQ0BWe=@N|dhN9A7wPalUkff55DhMPI*y?j&8mYwG#wjVdWk5hy)4<26wfJA} zdcOC9lchmhi}d!^6p*Rd=HiW8DNY$FG8B8E!sY9zQ4%;eTG!F2NX$9P36Aahg$^R= z`p%-YpRBIe34C&**jGcfW8)b{z6GdOlHHkk&5KL=rOl!nk>+n)PnRg>)5y{7;*cPn zpge0}6^>66Rc9#rV$vk{^J{s9Q|f+o0<|&dwUdMBn+CiKr5@nl?^6=J4ZyKf4lm$7Y8*3}-LpOT7DPqzS zlU$Pxj!NW+YCdQk~Vr!2Kd`DI_O{{UD6{`q=+-g{rB7^_l0;5xHs z{gsXX0J{8FU-h;B0QVJLRjf6|UAC1duFzF=EO=zncDESny9`~m@jUNi^&LnIVg(H8 z-}HBPzcK#xcl@AkGQV7;^d0APM;+OECC+#s&b6=Y0mv{e#h z#HhOe0N$?u0ENK+06;ofC^p-%ERRdn)J7+cpZPAoE}7DtvQM$Hp1#DQblJkEew$im zgjN(CU+*`6!(eIu0HAcYfz(y$UsQQ26}*Xw?#slH4`+0*|3CD-M@-md=uhQWV8I$fi$@kse? zVN*Hjv{N*S!i45?c#JAZ-%%8xcn2Zy?RRSUI{yIP?|+BEa`FUxS^o8PIbpCv{{X;r zw)WptK1ZVK*z;_1z5f6^^Wve&U44qV8tq|MomFd5sU+s7SNql9@YpZt2TO8;a=Q`g zI-ljlf0zDCugfDO@*c->dixZK(`yQy`fX{N5m->(>;C|Hx-!FHf1n*LINV~dOAe&R z3i@z+tHWHsi^f8L9(fi0#QJ9Z`0Kml{@(@t0O@{{s&B6oX7%O3Qtzmcnh%i~RQz5h zoFOdk=u&wZn5;kH)R8)*Oz{qsD8t$7Ff`izgEfuQ4 zQi()E@8NE54U`yLY0b%$qcmX^?+;tO7!7~s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5Eu=C(GVC7fzc2c4FTdpU<$AX z?P1N$FD*{RS`9M+t6`~5rwk?rhO=kR*uo42viE}6qcrw#fQ8dgq-h literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000017.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000017.jpg new file mode 100644 index 0000000000000000000000000000000000000000..51cdbbe2d48994946cc40d1430c572466953e2b0 GIT binary patch literal 31577 zcmd42Wl$Vlw+1@6gy3#L1{(qdf;$AAzyN~<4+Mg{yL%wW0Ks)|cZc8>2<{Lx!QCa7 z0l)>z$jbl_5CH&$r$4~&M*xn3q00x} zSM0CZIbXd3{9XjS0ia`IKEr(W{O`cU!TsNXkB?78MD+BDi;Is>NJvCPLPGL1@$^MT zMn+9dNl8x+WMzH*y9dw!03aeEJPp8q1~L)~01*Kd4gJsTpYfj)Po^LuA)}z;0uYe? z2Mch=>4WL=0rar}3og&n|fQoU}+1D#ytF(Aey80z*2& zDtk$;wXdouIYcL~KNvau?gZeWBX}YDAtBHK5b*#=c!1vv04yYgr-LV13~&?221g^b z#s#@bZFLLJQ*Diz>sMX3iowmD>T74>f`G=HAmox{FIQKM1#N@agWYtMizay%=0dnl z*;a#^rmPt&E%-+;Gfr51H0<$7`k zcCa;YP`)}(V4C4oyJzb0;+$uhS;8#k-l;=IPj>sIjz+qH$*TXx-D1WHwwx)(9$0-dw{OogeTrgbhTsJe75*Zn(Q8NbeAYGaFWV1A1HrLO; z8~(5Br?{JS1a(*>@wrG7h*Z`MKA#-0mGG670hn4bdtzZ>3kR8wE#4+JN{EqKlQmlh z7w{%wOg2-05u?bZu=qD4AiisU-Wq3lRRlJ8e7bM%6;% zbi28N5&WP(p(f72xTB4ruvPm8Y3M3R1n$v$)!N9cmgO1?f{IT)TX0@d z#^{MYy(IL@VX01zwkE077S@iuO)${9J+i}8i%}1!8JypC2tkYfH?Myk_URRUmRoCE z{4g0T7`;W{vRp_iS%qJPsYVWz>c+Wic#|(#uCKSZqn|ZbYejfp#b2$KxrK$p^WWil zc>awzVl;|R{UUg&$>E|z+)xpkgRho=K>>fA%BsGmEktgz5;@4LZl)_)J`bHrh{1|< zYpK(Sdy6#Wnj})^(Ytc`v%cb8N6rfQ+je@F-eHio)s7l8{S(H#{5+|d5zS!0{Q{3Z zsj%ol`Q}wGxKk|-j+2T3!bCF0#a5>`?nWB>KYHYEytGA(kI!WyGXI3~0lj)4J&%JkO3 zBEAO5B2a9FZiK(ku7w8W5@$?AK+{-A^UX+cMg9vs921tBAFqPunrwPokUF5^atVfD zrIFm)+@8ir4~C`5;@<%80y#OA&XR0C2sL`tU1KHUKN&gXxqF-M%}lQ-4wjer0>{U`U#W1@h*+IDz$v9@))RA6%ZarHv!qy!IAkom688OD zlzRUdbER3!%cI${I25Ecz7tI;Ul&$^i(iLDqtgEy;QR~kVmRbrc&1dR#$IINpy&;m zGBebVg_CRo^_Avc%tyWkuX90N>7Pg}1J*&K>qRS3R zObO~>V}f2aoS99W6`yVWa73N!Ez;os$;kT-q{@Uc21|@4pKoPcdAO2s#Bi$gLPb<{ zm!Cmq&-a)cJPK5n%2&ND)?q0JpTt;VmER!s|Fi!;$f|K3nAB;{Gw)qu27#lx;?(lu zfQE=j(eGdXx2}$9Q zMqQp}CQ0WXbgr6xwSTU){EyTjb(^?8}U6 z+WED`Cj#xYaA8}A&5PdtV=u)O{=tK=(w zYyI-W`XjeSTv`?~pqRw@2d)2c@n1=ZI+vf1p?cE&yk-oi;nPwsSSh+7K49;nLxv~Q zv*4oKhv4UUl3T`*df{ zoo{yc@oyBL^jpd9=y{wr73}po9;e+|Z>7CRD!ejW5e}@FXFX2}V>;UzgC<=o(x11t zsr7%DX}X4Zx9>fQT26V!Y`gOrlxgl*dpy_t0QU_!J2%z2N)*(*wt1*Od&KGWh+htM zJGJT1JU9I}Hti2&Gv3>oGX`rymn{$d>uvIf2WP_Zdl$Ykb(!u37fIKUrxg!dpS660 zelFTxy@i%Eyd6L>^Jsa;;M3&o6 z&uCTGJ$iJHP?pONc#B~$7*;@>Dk^{M%SF-GMfE&jD6C{QI4d>vRrG#}VK=TsSlCBz zvY8hrf86czev$u#)IaP(k!jiS8$fZQpS!$xL`VEH+>#x%o8T7j&H}zvRr};OKmU0v zWVoHRtm7}Rg4dprS_hOeN=w=+zD{=dgV3i!d{T4t2ep4m>S*EUPbG4+G^|LuwDe3D z-8R1rUXV$q(Md4Ur=^hc1JTH$mSk{LG}f|cmRP@=#Hf$`yg*aJ)RsL zzqNd72lkI!kAEy&@?o8NLh8}r)=ZtQc496_C4pSSqv-qbgSM}NrPB4^zQOevW?6Go z+h6~4t?{;fcl_%OjtkY+wp)@xWz(&yj_tjk7k@Zy)qj_dgl`W6eLg|5pw4pA1p?RQyj_J=Ma$1!i0O)R%~MpBU_F^R(jg zj5q1b!xObXQDx@eod1D&`=U=r?ukD9DtqPrmRjZHslN%Xw*IYvvtOh2zTf_e?Tb&X zNwe^2k-q7Kp11QO>;{d>+u6V0z<`v}`C{hZ1HJe5vwH6}`)A)_)msPFV>p<#SZm=; zty6iy8@wP9Qrk9-C7Q2N5@b!&2OqEkPm>nfy|Ovvvy!pKXyD%-2U)a^<#_y1Xot@inrcY*8p+3N}{y=CFYvsy=CB zWC*guis9wye2z-x>HgmSD&_`^> zWg%RHB1~>~;Q6UC*(@xWu~L>V zKz@BxZ#_OmeR5~(1Dm;qT}mtqUv z?mgwZ`R~M2&Z{8*PW<>cG0jiOsRCc$?%_UN@&uE;Qq`yB8)#JAE>hj$C*GP<_hKrp zZt`-Q>hk-^ZZMc?{8~|AsUl4jWBq6YSzTBVD^`kpc&@Odmd;&MX?tH)Z0k++%mb) z{IJ{hPH-o4FO@6TIDSwWa6|*a%Y1`5Q@Av>T#tud4=O4hJMi(;DwXwtYvV$5!{8rn zWI1Ets1ny#x0fqZoCV=V_T^&2{8aSMrT6NdJ<^fbUe5)@)-qslR%3d9W0^`7S?(zy zm4murpGfG+tWjY-yJ}f1Zf+g`|ymmWI z=S3k1$Zk<)9%~OGO)O<1Vw5GKT9%ZLsG8mw)<{`~%>!r-C{1le=A=w}I`}+x6WzHD zn{es)OZeVz=UjYyzi;y!z?3`RF}TO>Q#kV@$BN#G0hoE4Xpk{&X|tQOqNmRq5&kMR zfewJsmkec%SwD`AGADqF`%>fD(r5sz+C$x zZ}wm1Vz&v-UF-Tu8{XwV70;7Upi7%H z`@|J5s8=bglbtUjOQv#V3-q=&{kYcUkX8)0l`yT3r;2%Z>@mFzf7zC37Bskv14{`2 zqi00BrovqoDm~Q}_u%#QowHo__VzebTi5J(j*2nMdtd1s4Gi_IZRtiDy99hRsSbvs z2bFT|oFQv%z>&O`2ndNe&$R8jLGW;NC0gS~R&^S?q)Ei0!{Y@cZ+q{$S@^APu@0IjwDJ{)h-ph!6`$;OsY%*8lP7QM-O~IjEK#zvU%~r;X|CZjj{oIPBs( z{{HMtsJrl>|IXqb!&`Hg%BAqfk(^SE8HM{HXe4Y&9-G#Op_qkPeYlUAcRE-&>x&ET zFaC@O_qqBRRVP;TCd_ADId-62*i37|vvE3`u0Y;+n zMCm?=?-3Yk;KI(l0P4i<83Uchk@IM0wQ+G@cEK6k8ZpX*giDc&%w@(`0QiC}27{}s z85c}1eP=XwFQSXQ43J9S$1=9;@FsnPFzA9|mZy2-N?fk$0pit6wBM>ew2(%$>#bNc z_Fmmq+jSzPiwcNhL!1YJQQy^DpJ~J7l)G36(SmI4-jWd6#kXvu(Q4y(iq60Vfh;Ke zFhYB>?5(8ao+7uQM2K;w(9{RT^!Gb$So*=Z$fGAM8HXH<(ijR3=88jcdf*f04$1F_ z+c)Gau}F*`lK~~0a{i5+7-f_haYfLk{x@+Rq>iR528Q>1Ut2AH0|KwIS&m{I8o{3! z%)bb&o6{ltartz2{|0E}U;Kyyvn|TM;Ig;+9;?`@6Qq3Po4vSqr`iQSZ&+wmN<9!I zf7Ci!oAaoz)t9uP8B#-xvj9>zH2O-8a)YQs=Db3!o;hLGzp0!-@zCD@8^u8g2P=l~6wmGhNlMlP# zGS(VkBD8x}Gn2UM!9Dv-LCLy*Gdgs@BxIR+az!E1x%vg|cf;AnKixXC(S?*WbigyzFwE!$ z_1fI~r+F7DR4R{9L;nF6>U5W{U!Feg{_A$n;2oK}=M!-m9?TFLy)fLFbLb@vH9VvW zwW=NIpubHywf;3}``Y0Ljf2u6wq~!eydbhw5Q|+6ior`@AwG)b+@tA+;F`~6TKSL0 zs{Fbr`A>t;jgNj^L7}I&Keax&{6tAr8*|l|Z|bwXE%2?G_eqK`XwBtZL5?_-TOMl3 z%${+Jj+aVsO;RQV{=h^rK8|*`^JN@&%P)|N%=y~AvbNBL8A2e5kO9>E)Yw~CBn?(8 zQU|7FB&{v(wa<*ul0N9J=z1v`-xe<|!xnFSa7;@dR0|vBCK-5R4>-(a3yhs{9lqT} zK|0h+!Jd01`uK9Vd>07wU&Aobwv{yWU0RZGX;6;V#Vn5xZEbBjimmVMj zGl&FpH`yd7x4Tn%3%1WPWFo%V|8a?pN0sMLfcBixDLJFi68tK@nIX~YWdZew)X&pV z)vr0E!%OnJ#6rWb8oR|__G7;wdF?P{vlF9PZDZI|j09^lXRE-B8V8z0VjvZBxzCVw zp@^dm$DcLn2vej+D~0Yio7cYT$-~4KL3;0}k=;6`vuld(k4Te92ea?8KvOrLW~VIW zfN;i(*lH36!PQog22U(&;Zx}8H}0!Eq=?KQh`oN=idzh{4qYxGUlwaHd)IhJ1GWG? z?SU6!BffVS;CyQR4s+0N3~E^gmf?Ih36byoWC&;w_>E{XgG3|vd`#8iBfV;n4LqIS z(xGyx>zhO~DmInrexEGbA|ne%IHTY_|-tqbZmB_xMH=5 zp~EUy0HC#FIQto@1NCK@vL1}V0Nuz+P~ zVxg}&cK0tX*!YMCu8^X|E6TfVi zj-KcG1r;J18YFF@y_WZtaV%``!p4iLb!X$)EfLwYV0%wd3C@kCDZCPh%q;feMhIGh z7SUjiIL`*Z=sz*iiUZD=$IC0LnMk}FnJuZiCz-jr8+@Fo+3{}mc)dgN8=ziV%!a+M z@Ed?D5sjDHdqmyju&`eUT~`aXri9VEU5jM`A60-KA&;Y4DwQnnK9K%g;O~(?6U8aD zb!~Q|xHKHNRSI-#i{gp>h>L9zR@^ud;wSIZwHmf_G>N-IZPDRZKu&~_tajg&5x!@# zFkMQD$h8_1#$}$#v_NI+@}l?cMfQ*mNOtvkG1%h%DEyi_sb`dRd0hh^iVpRo*esw| z9Y}*;v+9eKPs04YP56t#4V59gxpqMjhNNHy4VvIV(y3W!q4@mM~>uq33f3 zcJ*}1VcsG89Pf1S%&4hC?m1!(X(h8nyKPA9>S-nIT7v6e*foVIhdo_;O4Y-xfxpkM z#}{pOK^|FOg^bM1xG?4Wjq@EcdLYE8;`^Fg^uDRnWG)0-pqK#X)oSpZN6*Kg?tLD; zY4h4y>xCA3-dR^-vMhe7^a_|u?8|TnrCWRm6XAYrpF^}Q=`6S9vrT3+b#W!k#>GdT zwPpusndH-RY#4YXa|zk=;SkB-_Oo~%oVff0YVlkned2^{`svahg`MJh)TvF;rLYO=>kXM)P9KK{u596#w7d2&enwj*{LyeKu z4v*Rum65JO&E&O@8H;nw&F5T^lKZ~Wh#(o~_KiNPSj+7AE5%3G`+kw%02rsU;56O( zcfT)?vuNKs>g~YdsUavnZzr>`X=xI+&jusA*|kDU9i))L_(@3*e^d3Eb!#@8El|u6MU|5(Ism$XoBG=V5dhz*TB| zh33J|To^V;msuBAQk?gKenU2Gr8HQ(6ARJxRqfk8c#0t+s&xZ`8g^oqzeVCNzJ`;W zV+^@R(2%1_(XLe+OKsa`_qT{ReN6{7DC%XI9H)@nS6Ep7u8qB`)AlI!48n0>C>eL2J0NCR0tNlIo7j{7x zBzZQ!F+Po~tX<6@ac{+wXO%e#@_5~jiOk^G6=wNxdtSKxNR#K}a+lO$ZaCXJOZBk_ zYBNc8~ebO6_G_T=7{as8dgcDN=8AufUk!vFKid+SfAIPh4oON)>BmHve6C(uB|xS?$BJpbISX~yP!oXmf!Xrn zHU_ulwqz$PnC85MxEt5BY zh(SkK9__n$qGY1g3>y&|e5n*;3S}5Bmgd;bmIu0Ask^T!TJA8*I%%>*noRtWC!`Yi zr9p2u#T`lF9l+V~LqxE|Y1Ib-SvsXXw_O#qx?e_6BS3#FP?1lqN2x6&BG1{wUH`<8 zg+9z~??`A?Ir=;cjxLvPDtJ^hT$X8Z@(Q13GB?8?J%dYGbSr4N z`B^em6}x#xi|hv9c`EROB8q&0Il-k^)~|b8*&;9}M>Dho6vUWz8e#8?4dE(f+?Cla zNd6J_+> zUorLRq))M8$%+@Ef9^UNl%8Le8%~=?$KuL0tx$Bi8A@jrpWr{d+Do-zC5%g=+&ebX z;c6pMB3NB>eRk6j;Zvx-7|9^ddCWVk+C6b>h1r>msJGSO}JH^NZ5yJ*Nc<^QNNaNM;ZQrk0-q0TrO# zel@70H2&ww9rqeT3ZMVkvx6u|rbeKnlFRaDU{IV_frkZ2IzZp zi^T5EWx;#f#5BAqx>}XpQZD!jf{BNOYP6I!K(Q@b1#Lq+&cqk>EyIqaiE%`LAwsAb z#u|P6aYwE*0$&|5IulWeWpCsASF-wZ4l4*nRq%;b_35C<40@Ld1$@pzMg+eP^r`@R z>x?QE8PXD6*ORgey2E`Lu9UYgi5BY4M0`k4Hc|rN$9EBg>RqYwzX97IEXS`A0@bbh zn_3N4q7F=%e;22o0lf z()G$-)y%Kx8{L93ubG7`2~oLpo8SS4->W#I0&5us%%l@T0ZA2$bMx^F?N1xQ6BDhU zJ4!H-3*j_saqkgE4dQ2f^1tsO0Q8z19RPrjQV9fj(+vHV9U9j*1q#+$>Lae0s&zIr z_!y;M;mnR4qZZchZhM0ZfuBo?WW-r=dO$6gnNzEKQG$U=3TL7&Gd+4u`aVo4GnPE$ zd%JDk@1_(%b~N&|%*bElQZUb3^`>@hbud~y4tkijz%zAlPC_|gFYAEFO9s778K)gX zJ$)=Wf=H*YNQs_T9ZAuUf_i!!t#w#}_x+Ce`8vkD!sWI=VB45c zusD~H;5FIc2?mQ_3=0gq<|mE?*%9!5u~BwpXDhyjC-zLnUfZ1lgcY8zdRa~-^sSpP zEm&&Q`9r#T5Zn~^3nyZevN5&2OIj81P0oTX#Vmx#P=gEIFFQGVWrTrdj(MKIDg1Nt z8%*ePu2O78-f5$GT|7)lEicA7XOI@vOIllp}k0zZTw+6q-G>Q`f+_f@wtTr%L|1TWt0dr*GY^-5uJ)AV=g;eKH54`R*OyJ@rwRVxJ~bChx^G1ktfWE3 z>2@bB2Q_wvEdHI1dkuh$h6uwXd;vp-ad%8THQM#9DY9TR!azxF+#(e#-dEd-i&M0K zLD8#74;ac|61IA{9fg{HrF%q<{$UYFNPsV!aoyOq7`5vV{juZ_ZYg++S)u=5MDQHFbH>iS{yz*lD(HT)R-^asap z!{f5y)@1-5ZDWcd;!DK$&}t7Yas%wzJR=ooi%&nOOe8O_G@J)GRBY5*@-+s=IFtTb zQv7PVpLd#=%fZ|LUqHipfee3alVB!Fk-Np&78=p+&8K~dNSHCs`+pirGA1TRUUL2o6Ala3@?HU@W0IHCYhFp8nu zmR#NT6DBBa8v{-mTbW7&qjjZ=t)olEqJd&xy@=E@M~<|>AX^7VYkhy=Im6#ha_4@7 zdpj0ri5=1@cwmzj8TL%ER$IHgfD|(|7(EXOFSH0^pjSL&$^9Gf8-Ps>H zDWW3e#I4@ZQ{(VA0Ey(bpucvg&FEm0%`qrJOHwU>(e4KZ(RyB55f-ahajV4uXK}yi zaxHsXcK_1s)t7n*gRR;icgn_ft8R5mJ(SH)_O40^W5|aisZI1gsgK?Qvsxd}ylnE# zdgXK%KOupr1ko5oVk zO=w<@=vRfh){Tjo5hon2PVV4egTz(}iXE|f6YVrrzB&r;K1TRp+SXySTb8rri_?zL z3WbY<(E+6qM1~5sew5RzQ`AiBuGWi!EzNdmb|sd?@?-~vpb3omg5hNUclO1^Y%K3) z5?DRJ1QDLrNBbsb#F2odLu-A|9F{8UMB-4v%c#LQRSt=yk+UU^DB zu1qq?DQXeahfQw6*(z4$<{LggV^+Eg%hNvML?q0M$}7x^xKOa;_>m;uZllYa`01^7 zryNFoOc4$8+JMO$JcIn$d+1r(2BALl5j zO91L0vrm2lO0=cRc%QXBz=yVrP{UaW%N+&8vafJEw;}}DEFSQ_T8-gHi8%c58tbhy z)vK7Yz=X4EedVTGF#Gfkh!INtMGlxbq}0@3zAO(yKK^pnnq_UO56DP5&S2qt`5R!s zG_9&UiIaTLilm(`au@W4PK`(P{0}T#vX^c$SqC#mwBJ}=vF};NhTLdc^bHM#C zq$xu2kBzY$2x2G^3aD*KaGl_sAkwe|KaU+U4T+VNy}bCLTX!xBv0Oz zPm2MXq?R@P--RK*9J-@m`$)8D_N$r0pmikt9HubOsdu$`^R#@=x7MHoU%2Ajk3uqjmO1bD&-ZaPmTX6?Fx1FW%9ch~W_mX?UH-Zor8jQFZK%S+42;dSQu8 z2c(2ND^gw`3xftjU5!8}e1zD$3L}k0A$(qYTPlXnHYF6!_K_E=;IGbHQk%>uFy7BF z5#*YazDA3my6qd%_|SzG8|6r}7zV*lAV3U}6WSZRt*+tesTN*Nc5W~3nn^RG zVSDAMLZvnqt?r1&G{ae(C$)ThPPbde_)9wmB1{t#+sw#Rb^fb@ZsF-ZDqfsM0V&fm zagvthNG;2=cfUw7&~4s?@G1g>i7tn}+;ZY{+rBS;5Kq%dR*X6u57q=9mh~ApoW^dRhoMdVx`w>085vbA%}SMk7lV9fZRsSJY|M@_)Rh zvz$6jNR|4!5HVqaBWmMw7-i8#;K5S&1}bEBzlmjR*n^NK@11ja!FSm9jqBl^>}+|~ z{ck{`1H@Zp@aLYZ0;SguK(kQAFApQaIXN_XH@~RhHPOB|^LwxEgwp0*6yuYExfxA- zMJC1)qgs}{6XQ8$OwxR&R}2eMo&K|SlKP(GVI#&{q9r6T{%Xr=fyv$NW;B;=O3*spKnX*mT zRqI{&u0eiz3l2MOuE-DJc5bi3BW7tDW1s+)Ox0Ve72K3?T2;<>H*e7O$`~t(J&dDw z%!{)r47QF+AVm4eTw#Z!^lc*x-{+gGK^g*dW*+=C7O2#u8CFIT@h2a5tSU6BQATL6 zUsxWN$dbrO7K-G^|7@nX=Xt9*5!6FooEoBky7Ba*3H-MT21rN02rR}m-HOPR`Ag*b z?OAx@(Aw@O635;7iIQF}y!>UQ)0hARrQL|KI=+cZKwhW~3d*JfEcH#%aA@T>G%EJG zw{7e@ybbDc5ndGMBXYX$G0f1H{mSR!*rr2JEY3UOe=`HSRLHdZu$Q2gXMt!)(fo=9o-E>Jd*<6c2;(yGJx zFYo2n&mIGT6imyF?ig+zQUoefb6J)W;BBwNGX4qKS}NhDg2@Ludb8x-oqoPKVQ)cw zS)7H%vB`l;x8%`%@_5t?h-p=va89yPzAduNxk&Qy=Fntr+x&eu2Zx&QN9jUF6w{#C zJ{H-)PSQ}OqUshF$P{n_F$yO(IASTBK)AiHa?jr$z<+K!Ik>6YOL6DTe`eDq5<3L@ zA;OCD(NFxdeL(>`2cs0>*>K4NAK>dX4MRu*T3Fz5ZUUP<5dQ$WD0MGI_}ojUE2cQ)dF!yqPr;$=9~d!z{i(ZF+J0wtKhb z==!EYwRlPQ>6N^stnnKObkN9^+lQ6ho%mvAdJwL(P@|FoweY;g5e1OqZF3g0ioJw< zbTPog;OX&_q&2MofjR+wEe@E!Iq~jCtM6ck3u`!D0hAyNx3E_!E}(rHAsTBRz57e) zec$-`KKF;E<`!StK8H_~&BbTV9+3kqw^fSLsupMu%+#9c-@^N}*9=O0e-)c=cA(mr z_YJZrD84ehilP3VdOd)GR*CeGct@;&1qK0)jb+qDKL(mYiF3`u7!#1YqQqDV(7M+L zBYTV{^57tC#DI^f+*igz2G;cL%KG@Iq{xL56@Wx0Z)Zk!b{yr^$F9MK!fIG&)3i7s z`x450IupWK>A6Bp#6XV`+wQ>g5S4g7;Rl<3O$(Gb9aGnOHl?%DS`(=F_%Y9-_AX&k z%(6q>WRtV10Gu5xo@;!^l2%Q2w0`0UsRQ6!rcLq-j}ecRF4F&`ia+)|8pln@yKHjU z6q$cni>9A=BTxEE(J|h@&JZ+{W29Ckzra0$pdAsm=YXbard;ZI`fK}QRYy1NL+)_z z)WRe1$|Tjgef3_=U$;T_ix1brSc8$DoIkx5+F1Lj z*vI7irmKPh$c#cdY#ci|MIBOx8G9%S>F-^f`DCy^UE-Q3>P1Q8;ZFWM9n3^DwSQ7z ztJb;4Q8GifqYcH3Yjg3j&BnU%KUR{jcn&_=_Icbq&@`{ng zyzKmCBUp<`_!85@2@SSuoh>cSt|pBov#SjGaZy3Cmihi*O2B38uVctG@fw+(S=_;QP5{Y;iU5!6h8m@CnS z-md&-7Ogzw2h&my#b;Or3gSxT>Fx&6mzL{*uF(r!p)ZaEqBOlSH zRX6ktojz5&;N@ygbYu8)E;cs1oVH>{k+Q{1cuhWU8O2+*^`+R>!;!J3ujW?eGr15# zQ6pkjZlW_WO>+kA$fV~kcF{$HHw_HBkk>JRo{w_XxQ5J_J?4qOK6|x;oIKt}LgqoB z!jU2sjmTZa;W-O*psA$0Pv4e~yg0;L)O0w>B+VTQDp%)B+*ABR1(-RKeHb;$)t9<^ zi5Xsq2@Tnyyba~mEuA#Q77w_t&9oF4gmnuWc@aWlLl==dG!PY zOI(grUKMqJ$&N^oeKS#qc*zQ60R%I>*dmKiY6>mTsxd?tg9#XCKa<2QWknF`na%j% zb48u`%ivo0p$t_hNvJNE0Sz5=-^b)kN%FH%Y0{ya_LG`|--ZV$1hd+3QlOk@ne_+`PnrIk@{^I0iU)%VX01{0ez`ES8otxYU;L0$S6E_^;r}H!U=s;9gNe%r!9QDGCh_|^=WU-?%&6Cj z)yB)Y=XWLjq)}+ZdT1C{IU`i3azun&-inR zk6w?t@QcAOQ`1a{*2iECd!ZG{m?f-y%)E*o8PVxZh(?w@j#-YRXiTN_fvXD|juHD$ zZ^|(V*@WGtfRU^nq?Fg9?tVCg$iyr8OTTRj4IzT^jZlZa2Z>g^tj0U8*-)l4|cT>%EqFc}gU?rJM%vVy4}v z+`X#KE&-UQ>T{AQoE9NHjnRc>+xg}dEs%*^f(dyh;guuN`U%UAF@03Z%H4|t+0>fi zKvHY0-+-6y&lvc!4L@KKi>g0gt__jNV?Y*HG(Apu4{64;3^VT=V4)PEaIQ?7T~mAXmXADsRRk7T_(%{AiAea3jcJxrO>mN%mrqS zz4-cbDaclKj-y%bzW6|_s-lH@tmD8VSPe_F40ez*wONK2fFeeWr&FhOW*RpsjI33?eyUrjc{D-1#pR1qWUXzX68z z)r$9L?8HbmE3|ZJrebv+jrIsT z?wyAW+F_9Uy*9As>vcbIzx_7#=>F@hgH<)rp@!A;T*=RE3{!|-nv!{Xc{Qm1xkH~h zMdQ(d`{kUc!zQaX9P}fbn89be9d=)jOTX7MsQxuXfYOxNomX^BNJg zDpM0OjVBYRlrjdaHFZsAb+ zX4f_&XSLy|nK$jZAI^Y~$M~bcTJ2i@YA&W~6v3ZksbFVj8p=1XsWW8C9|cjJPTSS# zNoW#^q2gU!XH&{Y6$chIa26q!9`${Hzytu%ToYQydZt5_qg8LqLV_4q6o3Ol)o2X* zwlaY%*Cf5(WesBi$)VA38oab5(M~|_q#p_UPzI| z6++)1XY@yUjTn9Gg^80&dYpZNqU9@CuAwXZke5T}!BZ}a?VzP++y5k0#N|NgC=fk^ z+AN*?g_TQUdGJuMx^EFdana}t_Be15*>;mofR|nkR+z>}1MlsMOB4S@pI_?u$!Xo^ z6O(~;y!;f6=_iRFVx>!EVB)r9U;rS%E4cGnUdGGw6{w`7dPuU$R%drG0S_l4v*4Y6 zaO3LE#Ck~nt^|MerYQD<$2P#&Nz{hJKgGC8sxH;BDS)Se03YwW*O2$^X?_v4EJ)6R zu!QTo(^AU#I@;k+XN<>^L4DzouY)NU zlr3)B?6Hds^;09F`%AyJ$n_AKbjaEj zAy_~NR48fg$=TA-S*+^g{gOzD^RBL71fyuC>1`zJI@$pncms4k`pY+UbB`~MkWlzeHWxxT*9Igwnc6|e!@-eOA&0^Q%GS03dTld z3qt6%HbGm_$iD&1)D8xwVg;KAhh}?RVtPi?d0B{3XKQ!*Ud+B^7Gvnq@s}i-_|jX; z^YXMTSKG^L{&JF4R)rN?0-Fab86a9J`V^_3a zjA-uS?BU)IoY5%<&DTjt<);TAMUDr+pI%%L*)NG;cxB4FEGRC|nd(Q6>4qJ`GeRql zjn(kedH6RVj)^9WVlR1FaTK_S?)*bOUnxadXs8$ITX!(Pg)?aQ{N?8olwhETIpsS( zg_5N{d1i>cjGzsRY}uC7Zcm}%tCKL^GL1<|jOxS8Ug8cOBk38~Ae$JJOg&ilxT{W5*Gile04zUGMp%x#89*?&JCu~}&VU1uZ zdGUtBpy4o;PC1nTW{I)74*Y4T`~jQDf+ljrm(A-&tFN<-TeqKYcw8S1u2EnpMk%wC zOylnKpWgr%>fM#tJLETS5$S0YU+(N^a+LfA+=`*-OHD!5H2A6y$PKxCjJL*+>2`6dN^QQ}u*6PH~a{s~mkHt9|L@v{C z34V0~t@9o0sFCN8FS-ki;mODZIpH%5A7t=D51qL=Z*v&M+ z5fGm=J!(3MVsAk*`B@!-0nZ|O18utLyjH%qn$FH*3Hs?u-=n1AFTQv<*RRVD1gvZ} zku1k$u$bmqjYZD1_fh3qpWRpP+ zIO6fPy@kviQZn6_*zPGl1;;jd+qP9~j44z=;!92*r35mOAdW{G z$H$Wf+U?x{gsTdS0|+}%suPTmc5;c!aSvBD!$fUPe2+VdNaL19U!bvT8gcK8LzWZU zoaOLj3pS=;ah7HplM$3a<&)((+0I*-z62W<&d&g2eDXZ1MK==CjRmF2AR0u@&pP%MKPaR{Uo1@=)SBW)CrN1_C)qWhX;4RHVvkzACAHy!8RTm!fS$li zo=l7#;F&uxW0x{3>glvQzUfLn*rZQin~NsRlWX#1x*fy`5(QwL z?nqWvMK{;r$5N(n<-0tioVF_?m7T=|R*^N`8TptO$yi82N!kbA$q!&pXQvi23=p*9 zITKvxJ=q4{?7=-%Lb1`<`bJ&sQD5vQta_VTCG`gEXxlsY{-Gk5ZPJ{=mStLHUygjlfrBQm<*rqr>~< zsY+BQ=154~Nsg$>7*_2Y_eit?ND~K?JB@4bTTC56X_T0idT?kVLOD6@R*g0-qUI@C zfgP1#TGEA*tHoB*a_2o@xTKmV)0iK63$nX}1js+yY&KADb-E ziuCOByyH8G9H;g;K|vc+Bcyy@u%>2f>B3&|0A{n#_6Mr3C=u`JmH_6}WEpQx%46U> z<Y=`p4$=D_B=9bjP#xU>&AQpZn2qH0)%OgF&%N>) z>Of_h&ligpN7g&vN{I3|l!n_N0A8D&n4xL~wq5l1%S%my}eTE;VVwKzoWhZ7Y&C1~k`m2^SJ>0g#qu!`<0EURN7r zRF~NLWUAH*U^UtmX=5t#(5;h4Xtoj{(eJw-%;fvKSW~nZVoT(}6sn*8c$5 zZ7ve>^*$yMHtZo;pH7Bgwt@Kv1HYnRwCp2*#pRDkdJ=Lk=Wq3y?n_IGDnrVWY0seg zZ+gT>1r9Z@V-E?WDMO+UOK9iR{CDzDjuXFq2TMr@&-S&IWrUNTKCItl#8)tSx?W0T zOjOHYOwxnO^7T{E$AuUaq;B-;sdic&b`JhN1CDkSl%0U9OFEJM}bx<>H&V~a+y z#pVdBmkJfMNQ1n`Yu1-ay-`rNf>I=cV4k7r&CPvL1YAu(SWS2lIHQ0hvQcRWRxkh| z0XT&+9DuK#z0^!;WO@A+uysjIz&1(5l>=D(vBmWod*b0X(o!)KGl!1s@^hH&$tf6# zGqNgrAlE zUPl*aN8MPytH|0A5LE}&=fJ(i`DAJ@k1USp^vJceV;SHQo) zaww%e$}U>jC34L4g6$xq2V@_7JQzGXPul$Ve{_CtPS$AFT}a$xh?h3k zu^E|h175WlDM{+1VAZ82w*3C$)WIU0N-;Cml$WA|nN}VY!|-wc0A981;r{?{!A*(O zSzLmkDZdw-Ij=+la{TH&k~(GU7ww4u0Mc5!%V`w2lRnCU=?}GE%C&ZVj1H4If9d!< zlpNLNZ9oK8XVZv%Q9Qd-@DSh?9FJW+J?|=zJkj>OZF@YWB7D!AM>&pT(69w1mt;^p zoIE3~?Is<1fR+ljluIQ`5Liwi5R<{P5nwG?0Or|l;VnfDj7$|)@4KPMZeZM26$8EM zC#$Q+R>j_h3@ymf18fwSCa`j26W&}KrUUdMfJWf@H%Rv+dFz&!XjD^<=MHLR`psgZ z+;MT1bcnq(BAcC0LNps3_&ZR@f+`LYIPY`-q2-<2z7NiPa4XjP)D*l~wQ|^la}eM` zr);vwjn6KFua3UB4m$M(=te|WmksWNu@zM}b9F?eImq3(*K=CbL~U=1?uU@6IILge2_)xT!zy!b2?LDqo+r$}>^_u``m#WZW(6Gz5&-#0sLDA@ zfM1o+k-^!<_qsj#QGu`{49o&!e(^K!c!QNp(|$vIX8Kixpv@FdxW}F7{K6z>=vWBF zJ%Eq3_M}Ga`)Az9!VjirvE%bVpoRttW=~gV#o(c^D5&#S%h30T9yB%d95P@e2zm<` zr|f!>hE82Pgz|hicux4s(ij`NMd1Xd1cEzWNi&P6SEJ6?j0q&w(7Z$wQ4wBMpL3vX zq7;}Q)bH{>TPGPNxM;`6Do*Nv%8}iy?*tDg#h>H?8;kC)+`~jDz^IhSt8rZ0wxB*_ z$pdkYalP`{*-#jWJC5#`)gP2+92gA*r~#1-d+7@(NUCuTt^G~67qg6XQ{dNiMK+U* z+d{h}FcsZ|&?#w+(iA`Hm#^X`n9#1UN07r{s3cS#KAxOjl9mFy!Uob0ScQBKxu!(S z#{gwN+J;e0KWi~|z(F8OfD249F6Bq%Jb1EjV_89(L6$wMHiFc%5=Z5vZrxvdtJqN~ zxUq&caIcL*3p5yf6%@rDqwfq1E&WRiMWXa?`v%mbpxfjF00 zu3Z^25yz6FvK_g5>h!}syjaO4ND_0!?@2yq0otV~qGJLlh@AUt8f1R zOA}9NHxQzu0A%iH2i_y66<|1uk+O`B_Bj1}5S5~C%a<*w=PXgHhQF8~)yarkp zl_HrIAShUax;&3bAS3|5+9fHDzboJ!;UY34-qCSbDaDatU?oR=JgDQHus7Ya=Mq4hbL^(!lRC3aeChLk}C0Jipxq52NxZ4X()9DcLX;gY7 zJW<7=f#jIv@i6R5sS<({bza)KUsb8Rh=m@3=L|mZB*eTG0_Fr-BvY=vzX|O7TmHVW zyP;9>aY=nyhnx$LqJGx1y;cY&MP~xMz_U6BRUeVWVtzN-~1J`x~k6Oab3)SV(howd} zl5(ruQS>&J7_X4qV`w=vTq)+jl-p5cin`B1jZsqJKpZjqSz1Klw?5M*0#RMT#oE=W zCJjdI(bV7=r5pgwepo+d=hD1gY2U}Wj(yvWjL_L?c$Bt!ek1TunLrkt$oi)InG^KG8BLW?wLIW zikbPQlCL`hX+Tt;%QW8ll^TOznhiI!?4>3F85&k!0_sH!nvO9*Wq`_rn2C+92bA&h zyb~~(h#Bt6DFx*+2J<{}+J-sMG@RsQ?%z zYUnb7ku=YLnjV1wnVS3GL7sf;_B@oy%+ssl9R`AeN`iNWlrIV9NC(})>MGf!tehKX2s@{D zMJ4ri%4tK<=V0hH3tikMGTk0$C#YCJ6TW@VEmJbUJK~c8IeKoRJ!_Pih;gV|W@rdy z;*iY(qX^zB)H)h2$$~c&^wJ%=FyU7NQHrdToJ#C^2U1~oBPou`xdkjF z%uea%ThIW)`z3x<(>3W{Q+JXuxoUv^tXB~KE+$Q7C$wz3;n1rs{W|V5c>hTg`;#M;S zi$`%q9&uMUoXg$@F@%06j`;9u0E1ka8w74_CUEwNB5Lz6-UPg=%vOEOYYm)AhZlvC zTIIZC)Mo(wIpe&;kp@59VS{7=1nwp};gy=Ljsp?xe5Acja*m{eYGPrK5~2Le1WtU} zc%z%z_3W5B@KBO*{m!2hBv|u1C!LOa_(i3QrJuQL|V~?aU)MS$S zpH*ayeCz8h7Lb)Dqc#=6Zq3364u_JBI4BUpCM43W)YVuWJbkTGxj+KuQoTf!=S)^dThd ziu3fN-Ni~k-XqA?ASo)2UXu(ADs9Ljcm0isxsKA)5g?8k?|jrLN+S+i`yVZ!l^CRf z+f!KWbPKelTebW{8)YDq^KfmmwyNbDy4pw>lqZK|J|GaBSdYDc{__Hr$dNnY^&4xF zWEqby7PYG#qMiJA=J#X-TiT%x151)h2WD*|03&}Ng$r8LtV4feZ~zU{oUk$hkTN{v z`>6X9G^plBmE`UaXN^Sr-lGwX;)5VQEDVa&;;#VO6EFt-{*cyV<=@NFUeZcPlN`4& zGjdZS<|B0mV}lb>rW%Bsio^&;LNgoYth<{_#t9xru~bJQ?-W&;=R7{)@6$-3{>grd z>|Kf7>AArs*+MO@enBzDl?>f3sNK4xVFHsqqqBSIHF~ZFAe`;vMH_&31@VVJK?Pv(EW3n7TwvX`_h|{r7#f>u?w1&H zGZVq@9|(tTulQ@Pwa6waZFl$2Zq#{RsWMJ>q$V&L?p3@^HYh=aQ(r6N)yeLGkMB!3Qi-YtB`soY9U zsO$Ikv>=dHT;@Q@3fP^{(2ILUdGyRWiU6jTt=b|#dQxOdaf$4QC

    VP=Y{>_k;?E;PMfEE*B)1`0zb+rQv4)M5vjnEzBfhR3c2&>x z2Yp{LcTa71EvZ7dPp3m~)s=eEihnNdKWmfHqq);2d8qc+$n<>*B3lr$3}zZsu~K)R zD{Z(^hfC8l#GXMIDHP4XEm+p2dPe4akO&*ZFcyNBgl&@G3y38rF2g;qi%?oBCOTS? zj!fkl2CIaE0%sQS$?&7&1g6ml#EtVut9O!P4JM>C6)N0Rk?FauVs0f=NZvA@rl1^C zg$h)rwaYI}%y2d-Dxp!_*i=Cve55rJ6F_sL0T#r9W<8d|vvS!HQRZv5C{j%Yb97t+ z;R7dF2@1rNfkHFv>v+lE6nWY>r_-g?uL~L!kvqLOzLkT`(hW+ay?3bxI{J1oCR7I(yb*MRU5TD{^IES-yXSN#f;uHktz-^)wlJSfWmObwllW`>Mm_0+v*=-6+iBe23H;)&U z;7L1=TjH&)CQSkKtJ)&nLa|86fwLcIc?!eJ%jwwr^HME?*Pp%f4nLV$dVW?0p&}I~ zyxvx7@ic<9062cvU_{SpbRA8~7_Yy`=uslHX}4^Z8pl`cZML1lO9DF8l^9VXgz-6U zsAdI0$>fgcl&I28M$}2jjOgEMxeEUPFqf$sZo#gZEy*JT)x?FXIu6IOgd|eEFFF|& zYyzq~sG?8%f3z8PN=*VQ#*A%Di7^>}Xf^5q5=c;{_I=z-OaOCxwv_ux9j_%-*imo` zO`w$ROtF_HP@0ktQ090yDQa25la^}SI)?M)eNM>3J-l5g^jajDxrdv~!Hh;n5u3;* zB15RSrR=EZ%3aik!hp)X43sN;9;Sd+c0`s!aEZpE z)itA~6Penc+}{*VNtotYDN2?;aoW+cWM=dRm4JkXtBO&GcoyX1m!)JtY5?T`50zfiDqd8r*YvYp`m0@vMy|=2$V=pkH|bS|Ny+n2j7lmSw>pu9F_Y+yWT47gE}8)2t?Cqo z9qBunYm#@=NOFw_gp}10Q&MJHZk$UC&vSXUN@EwNOG0g{0K_RY0w>D&9~{lhL#u;k zGOKx1Hi7bkTE$P00^Qxx1kQ~?KX(>>g`Wuxfpqc3R zwLvFngTzavF=#0(EfJN4x>j#$17xerG_ntl|Z>$i!7SA4(1Hfgx7Y zDLzKXGwk#9hg|_{vW$P6M;BC?KP-<LuBqb zZWXV&^r1axfOf%xlmHA@vYoXpn58)<-b+_1^-I{cb1=XiNyUf`&X&ZaqDJl^&>>l8 zz6t1U1PPhyhe652Xq>nM?w~P($on7Y$#&L49n&KvFsz>uZNY}2?Wi=m2#bw^^ND7<;+tsK=*XH zK&eU-mQRaHML|*J4%RYx7Yx7>=pG?8DWg<^sZPtuyCxN~_1(=-py)WHl_+9VwQPl9 z6>UYhQN(M(=%pK#gs>K%q;7ccQ-(XZJZF2+Hx)u<+!2{{3qc#(X5C0N_q8b@Cpq&v zQzBd_uvW52m%}_BMm$?2hECFj1edabT~hEWfQT*$Mfyw=`Fjf;Ry69Ol0o91hF<5= zGNY^{k>=E;OCeU^Ot}-M!XO+&iDD@W8Hd*rXH_J@0-?PNrsgWr`T+sl%VU-?HH8ow zyTwTb{S(3e07mMy#W!_$wAN^FnqFAXWh88lYUp}j1eBC>JbsHus_`4Cv!=AGt2{DD z_LNN$Ig$r!_d1`@{v~Tno7S#d)La5?Sxlsl{k0Pd?ZS8m^}QE|={!2mbEUMKNEI>j zDDhIhWgcXgF=sL#OgzqBb7yLTrY=BNACN7|VHMufBnG?(hhZb~G zM~Mb^ypWO_#?iy(;PSg_0nCe;+)V3c3V47^j6hZ(47mV5Ht2_zZ3m_lw0d)Exo9D3 z8;wKAR@jbE%2+ph#frvHnb2B9AwHS3TUkoF+qCnuFElrmz&pj6tpl`{JKJ}ZdpgBW zls#X$-P=wRnGA%iTho+`$?~|%qZk~YIyz*6P0Vn4H(M$@AWV;Y86pHZEfQr)R{zS2w}U-FD7lMy9*4Dwt5!+vFQh-EoMhVEuYv~6zV_l%#K*++ zKUs3hE+h^aa&cNTB{xd=2<`NjJDcY@YIlmzjKK$v_+=$CnU7Ak9y|n?t`;Q}1P=2~W(J;pj4HKuT@` zCRsO>Fi`pYuPuQp36c_e>ytr918GC53HU>#C<&+?jl*bB273S;_q_unU`Z&GmoF>q z&lAscNw!jUtIlS79aN?U6%TU($s~NKo@QN8DUkBA*W41;Y6O%*ETlWp_~L;*&uR^l@*fhh`_m<{(X+X=#y1#>lqiI>2%gAhpM=rDlD zLPXR{-NJ)QXs;hnJHk-ATS_bobR-#w^yqv|!~sm+jpUjZf|Q(?48)^ZY?4t-Eons3 zTZ#~r#7~ugq^40Nex6o^6P6~)11Erq&CoX18+FXStd*HBprjE3TQw`Vw2Uc8xTc0{SU0b7M$k$j}eKLh4}2wxXOtCapy)WF>1)=8@9ytFmFxaR&%##8jqewhCG>DJSJ{ z9m8zzk4|lgwqm8>NyAsUAZ(teY@$gP)cA1|DKPscc7y7GP@`B8aWl1u;%I`dEyHIw z(v=#Z!`*;*So>PfFv%$@8%Js4ZJg-slmSRezbHJ}dQS<|>+aywpqR}^Ov>xj3h)UOSdRVYW#aC2AXPyJm_=>tz^N`zXlgxf+y1f)l4;&_u%{ZNd*NY%5+ z_#f$gABHJ6F8=_@)Bp%thRP4}Ca`~)?QneP7^!e=TDcSQt0Tq?C~pDrX`|N(DG)Zv#0+k7 z+9MXoJj;(r0a3V2Gh`(+eA(1mc1(877&#nDCh2J zT8xpNd>M*JLJ18CgH{A`6iSmMoz6sp8kB@2$Q9~qDy%(b2b^}uID$_}eVlp8FC-MF zaGpOgdB$QexDI!F)T(kX1yz6m9$wb}0LvZNo@7I12ml20-{=?!5}}1J8A-N+_MtIGnEKZs|GBIaNQHla!ig#E7;*0o;3XEQy4&=92ek{os0M%t)lmvIa4 zjv24)d|jfDB=$PA25g0I9i`!B4MVurDY@W3qObLi%m1G`b3;Vyn=Uhr43s%t=ip@)@_g7Q>W zGh&03C^RmcnF>)x`D$4=1K2*_`h8;OglQB#f}Yf@gYOPr@5~P;=f6!;!Suub0C)cY zGT)$H=k%kJ{Trh%g-J-MW3KnI!YV2?bm*cQbtDKyI@qL@875u>2UV}q=#z^5)T4mS zN2wx7>&cDP9;VvJ0D|N#Y9r>SH}l^P{ZZlCrFSb*tQpuhU?qR<1caa7j=6rudO0wq zBhi8lL6)0}D1zY;d3N&q8UFxSMZ`hbIr3wJgF(r-79)ES{fkJ17MoJl=x zFKJvcF;Vf))nBHpcwMor(Um6Ju3=rHP=cUgNhY{tK?ItbKb_3W{W;=#M0l2iQ>HQe zL=>iXEL95ZPzSO8W?iLQqJaiTP&vq?KRne-_Iozvi5n8EGnBG5H$GFqJlleh^{q(m z(;51C-xIlP8QW~TE4pWwL_FzH2uR~rR-s8s(K(qD>F#;rI2*ll_%XcEr%bq17axA% z?<0WsmcV#Dr9^s#YwpGbjyS#^c9DC$K;ylKs47yi9uV7XLSCh=reUUm2{2ixRE7h% z@B&{mrP1l=Q%s@6uJU96Ccc4~g%u{7sA1(+cgJsA@~TJ26K!Z&fFAFHyL@7;3AR&u znQuIfZd@csCb|R_wR~IAaDq%xj1hrV++n_Sd(5Q5NKgtJ>cqHqh~Cv~sVTx_ za1Q{B_aP=A8J+Jm2cbsd2SIAMf|Vn1GCDl$q@Y+5E97IXX5oVX4W#EjR{(_=JYI2; zl3P-hVv>W0f2Q_XU8D*Cdp#)iWC94^0G=h;u6WNZgK91*d4z|x5(EPSl^V)(5|*4P zN<{q1ReA$bkW(9U93IPAB;?>E5U>ylr&|Rh8qFk0%Y(@ZSIs4vBqTt?P0Rr<6viAd z7@UA|My5?(Rgqw20cb(mj=nDzR8q1@nUK|qO3m;XQ0sIO2u>x>3bd63UygO{d$zby z>S?+YIEjTg2?CH+JHyrxcQqk$qs_hGf^H=xJRDlqQg&Re-;rHT7|hY_6nZT#;q(E$)FL-B}R$(xs6hz%-E3R1)zr)w3?p8IA+u3W=F}AtXG=4uOih3{@US%VGX3c6lI0NrZhfq*7m0~Ul^Ll7!*WM4{&oWmI=90LN3 z)s%az9bGShDI%;wQ;L~54wV4YjhrkcX_@HB;Wk1^3ah#SNy54YLAQWR(X0epT|Z;h z#bZnc0X@yTiIlG)Q<(c0VvSaY656siFgUAfClEDy!blQiwE}pyg@AUlU`HuZqSzGy zlX~5PlCo=4VEXDlLgY0*Bc#v*tM^il3C(KiNUv^OU(t{CN2V!JDC2d0 zA*Ij^SMH@86Zy|lMSEg6zXPxRRj9ov(xO2eatGK(WDq57aD>9=b z^^){w5$v_!s&CztX;a*%+f{BJp;V{-uD9dvF6Gsg+)|dKnNgB?l3JdfF?MzOe+>OY zt?ae=xAicU%?_L|@nlBRoTUHCZ?A(C5do_g(@+-s#JDdE{ z2?P*ejE+Qhzi;86s`WZ(x2gwIZi65A$sfE(3jY9gf0@cU;y%Wkr99b5T2$aPdi^Je zATLjGE8->=pWaSm$agsh1A>-BllYJ7%@@^mx0WXj;opzmQH2b>sXd|ng%?V5sD4NF zhL7uh9GmE>wCluwad4GxobaBYUN<$1F}9xE*C(Ozdn$`Wc82?xJHVOTAoF=mxWqUL zI+T?K5&1-yh2qtx_|@9+ig%|wdEGQw6@kD=lJRXxLV<1o zk-(JH)y0IAUMdtH7b0&Y&`nc*HRYl8QZ0r-o<;?xmE zJa>WSfb^jx$!Stj6(X;JV3>Q3Wx#g^EZ)LEd6q59g(f#Um_+;vn}2L3wTskGXg2wIdf*fRsc6Rn}cmbnR1sFr2blQVJf7T5kw_CUP=Vf z^sZDM(YYygRh zvn?jt(*)P8k98_f%ovj{<^nGyTUHsD%Rpk5-s~+PlUU88R;3$vtG|->UxrGqH#i0e?;((SDKF zfY-SUQn$ce1Ipf7-rWa5t1XDi88Mc1@$KQ##C@$r?K;KFHx+GZ$w?$1cOBvV8^b?W zg+;x3V<^0UWBuvOb)tWD{K?ijEf3N5uUphxP0LpkaSF(!$UP3-wl7;6{{R(9;Pq`dXRVk>3+!NbwORC|zr>%(nYgSoIZv5NAa}S3m05Dm2wsAW`$qGp^Q&`Qm z$-^_gtx6#^IR*l*sY<5eDEL*M7?fJ^Zepuyjl{!2xDX|fBZz|J5osy_$ziK#;>vui zyGkK2L#(D0N>v-E%NHq|66H+3B=K+@xoE9&83?#+qX>mP1ao%)a0nrHOsn=fQw)M2 zF%+@6=nP1L&A>=wTNvDua(G}zbXfX}IT>;-H-wI}o*4nJ+r(w_O%i!Ks96JKwP{Y-<{wPaMrtjoO60{BPWLRW z;fn>t=>UG@4ex1HdqX1u;KU*nc`!G2_5rdhAhRv3;fH7vNVk6d&g#X5 zURDKMATGtQgr!JHC0Fj-Pt3G{B%V#l@c#f3w{b`63a<@Uhs`;SThE55o7%p9AE+Px zZ83j7Ma#u0{w?DmPw`j%XWlS|6xz1^O{HhMyW##XaQ^`PGaHB2bRYi!*5a>Q)Z4nd z7wld_&=R5&pR7?+5_G2eLD|Vf9#779mvc9-e@MUUKTZ0fMIJYNAgKciXoV>`6CWwu z_fToNdUWGu!YLNOskf^=T1`m*0O2+&^~BK@HGlTHk3E;?JIkBy-Tsk()^mOuwXHYt zDN2kniAq##UE*3W+mlP{?W0&1$+3G-O>JuXbh@7>bk}peQm47qpZ?cVPyYZD^S18Y zAv1RAr`|96&4%UU{w?3>75!%`ZFl^=4B$lUFE6X={{ZcEKZMpr&$!k90NUz53G2nw zwA`{1r6dt{aHytXf681SUBjK*-_kCh^*b%SxVm_}SD?U&^jy50^!l^Q>bj4@W_QH@ z0D-F?{jR6*m_8U(p?eaRua*jwRrxOaN75V$E?giU_Uw>9#8rRlXScV7KdW2i!?0jS zqPUQ(tZKpk0QGf0g!+{&8Cli;0NUz*3Hj_-e)jM5i~h5j@r5ZjEQF~^rP;!wncMzS z8EwI){;ey!=xYM`3pZ~o=CP}%{{ZFG{u5C*KId2e0BfoIC+AJer2hb$clt%;(&v0A zsEg%GQijZ8Qk5GQeWF=H5_GYv{a4^%L#2OltN#GC)czCDTF3Z0zx!QJ;XgbVa|k`{ z(ntsAU-g>1#r@^q=@$JH6n{w}0>Ei`yql%aU3N>LFQl%+<++_6jZ>AQ9TH*W(IJ>tgi-e(AC{;GG# zU`MLId3OIO<(%k;Dm`MKs zk*xm!_7C1cza{E}AifGJpyUc15>A!k2={ zcqx_?4)WMiMr}+}uT5{Vz&%>+LDPL*R}{NI%2J~t1_;?EsSr&D&nbA*9f{z^ZMJZV zaDHv*>Yrmf?aM~I2@F`I@nI-XhTBEue>329bjbezko7q`2OW4c-?0|abBzyfMdM1m zD#ORh^_nI`yW;8#j?`Tu;$wAvI~i(NQUM$N3&Fb-`!tQk<@JH+E+k0|LERz&iRp4T z(kgj4q##^TnSA#lhDXX_Hq+k|88vFt0WrFa+ji4wD#JajYySWte3oPZwrd<)l`#U3 zHqsl5)65D9Jj-|hX6sDVz!@RgV(?^Jc?U{yrW!?%H?xjSOn|hf4;EFUn?rr9++cgL zAoVrkEn#KkhZ2yPt6n@=HC`qkEoAf!gHk(NkYZTl{L}lscGnmea;PjIN=$ki_mp8SUf&Jg8%BHup@8{_|1NTGro)xKGjYZ`Ip)ei1a2;mni73DKUeyuzB#DJh^p$XX|68v5~UT|3aC=WB+_^BeIX4Rvw5&0D(8GENWKJqS|8lH{EzqT{{Zy* zFX#uwu^*^4Xq+~*shr~08KD)zp*fehZPb&pk4V{1PbxxMYCqfki?7Zkkd*ZXq2v0bP8&Kz{_U(YLMw$sDWhNa-Twge`Y-4Q#mBdB zO7gHA7cOtcf6cMGX7~O_=0CqL{{W}ae?UGjs`2ej&z{bef4yrINs|(%W4B65^PoPj z(RfeEd}s3i0OPv+pZDGG^!hL82gS5_{;3}=ttw|cHn7bIt`!N)v^MHXP(Ip;Na_{r zb%p);fBim-`UL!6f4;AOr_p~vJ}*Ag=2O)8&-u2xuYYVfHY&V-RKJ?ml~FN@)+m!E zF(|x$ect~7Pon;Sd|v0ba=Qns@T#K!0Q-Njb@}w2g!KA*iNkA3na>TZGf1u# z45Y(f_to$8`Y-4Q#pUg{1g|R~S#whwon(KgKiEU^{{Y`tztiZypdS}2qSP-`o7a~C zOS(zIDe@Wmn}ERafGQT|}`+{_>|V$VCR z0e4bQvIo4e00Eh*{{T0$&RCD%xBXE@J{Q-*`l6hBL&Mq~IvQkKsfIg!qqe|)PO=!! zVAM-O?aoYxmxEvg5c14WqZGJvXy7ktYkUq#Ttr?(q9__h(G$G9SUd>2($~O`FADJOYTyF*+rn6f~aay#2sZaAIDm`@`VDl4FQ~_T{E$&5IEVM`e+0yTcr~m)} literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000017_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000017_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..04d005e9071d9a902dda0eff899cb939099815cd GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuW*wM_Soe{#S!5$8WxbSz!nbJ#6YS? z#S!Uelr|jrAYDOP7B3dik8xk z{?2)x=e(c1f49oA1^Q9&oV$3fQ^g$1osL4KNsOs!vDF5iHXU`$R58535khGNy*44C@3By9)GB) zsOafwX_=XU931?A`T!k|4Pc<4JT~!P7a9ft9Tf!=3;Qt=?SDxDprWCmV_*^jP|#4( z(6F$vFfcJtu^zLa0*DyVh|&3^HB5ssND7jeoa;YL!{C#r&oyPpF_}r%Lz2&2%(SM+ z_#0#k1$t9}jcyyRa)M#z7NJE%08~@}8U_XyIx6a4B#(GdiO`4{HPEF=`05KxC!JxZ zpMpu5l4La3pPym4Fq?%m6!OE#WdHO4h_F%oP=nA=m;k6m05l@NpCteu8p>nf@%bg- z0%!;&U_T)Qd&uo|7#qdIcdBaN{aSriV(Efxn1hD_-|>ObE2G>tHmU@gx_`8|ttZhwq=vXFj9~ae3x8%pI=Ed`0z^*A&h*-7?drMUHlXaG{OO5S1cmwCO%;+QhJL>y6cv3!y#LEG>@X=#D0>So=M z0r7ge(v157*-Cpnrb@U>IWwcqCSCBMjnOxwAMPf0d;qZGkSYkKnrGA$C9UT_syFdb6pqqyVy47L*zb?09o{_9kfmS@_cEhtF&hf+|Lc z$4b2fysw^^QUdhksw&pOBNoP9=6)s#46vDm636QMLP1ccVzZ2V6NZ1$vHKrkx|2ma zgxgB{M?AO|=tSYDa+SES089Gw`12{Q%`{i;%HKl9jEst?HDRkFo=ifAYdzur62Eo+ zfID+$vi3q^gnTkxm}x3fswG`J*=MyVIzJ9Y8^H46b-goqf<5b9KU}59em12! zwZkpui*~5R3bbYsmO`FS7DayMW(Ce*2_AC$r+5EbouB*$O*sspb0jXmv-B;M7{Y;E zP>Hpltf9~r*Uaabmnnu_kKEODENym%s`&nRI&7gu*z@D=Kl-!L%K7uTlL{(lPh4Dh zky9}O1dO7T3Ytzw|4*#u_Cf#2MslHr|DPuhU&Y{$-w04PV+x){SS{fHq;l}-u=*XSHRu#W^v^zyy@6wvU}ZjHY{Gavmkpm{^t@hZ&K`K)Cg%REryKS>KbLymdFwM)VDHht!bDsp z&-${m{cT0j4!@^si_fdXe{kA<`&)_mQrqs@@62(3?l*s*27d>))cSNGW*{N9~-?VO6rezCR?V=AC`|VXb)dn(^fi`{SwZ;Q~ucW zQQ_#y_Qg*m)}wvwe`XCHb)WE!f4+*>Z<+a>HTIk3Z`~jNDz5&Y4C#LW$X}gSe`c-B#= zC?#PzKP~trn0;qRKlPjDA9I>7y(w}mUVHhE4k((JfIQiaF~njzNjB(&<_+Vm zHBA^=^VE)rRSgD)QDSA6=zG$+O$9KU$C!xVKkWENPycT?3(a24OE_%x$E+Si<7$=Q zLe%qlH34S;(ZwC*%UDZ_i46|wdKNIkXLU1OTvq#k4~@}iY@3^r-MF{70NR>?MraU= zFpC0a&a?@)&5LCqD|H>spr;#@gnXvu&r`xx1T&9p^8Q7(M-dO_T=(pJU;U4ak61+oL+qK_e~b3;-#UkM2K>Vc z9^0m!Dl11zhZv-GFa#Y#Mw>E>nw_XfqQ+@VpInAps-c9&cqW@AP8To3p27>8N=fAD8cF5}#Evws6~&uEc(&yJ0JJ9rk4rQJy56^|*Qlvt6dmotU3>c41m)9(i~A2H>054?TI`&*&PP~X)XXKu44 zmoGcw8{BRoC%#20A4*tQ(4>k}KE-Xm%6gmINIqZ<1#2JtCR7PurR77XjC|j5 z#s6+L14foo7$_r6V1p`vHUvuV5*?|HF+ zrE24@s3*PTyvrDR%F$sPnYPdG66*E;0RF1opXOWri+?|oxSR>`<=DUIyna#1f9t0I zq$##Rss-HFDI?Yw1ZGg!=Ey$DYi$DQR@&=L;~;bY0IH;v*>6VnyZbI8u42oYJtsx; z+VsyJ9(0#FcC*sB!WsvyKUG`W!cr`uEKsqb*&oMGR}OFf00RFiLrrpU=TR-ou9GiZ zoc&W}Gw90jmX8DeMshv69F^Y*6Tc-5AN7}e?(Cm-av_|7|NoI8D>>%yV_^EWtaYEz zufY1-M|nnk9~l*Be(?vOmip*%+2~by8W*=;BrZAQcYiw!2>fg63Au0o_}#6)ez#4e zd*bc?t_s%8jb9SE=8qrbX*~>TfQeIQHfq`XIZf_femK_fgNbQl!D9B|y%w_vJzT749qLnf#Qx9CwbM_xB z?`F92*>N*GncS@-$JqU5&)W)rH?WaaHYvNMg#wz+L+m3jz8@)`sGU5hyXh_^m74Rf%RI$b~mDvUEGfosRm; zp&PJ_D;(|sL|DyUN)bG-HP-ncsX!q)XZ(aD+PD@dLa6iQ%7N4fvS|y~!>F{rD@v`J z$h3#->eI-dN%RdYmw_z9a6S5Zmq*K|6oj?Gdb#4mu>xsv$RplIu)VVXca-bAEJZ7= z4j+eIs>T=c(m`$u+_y*u{VBDk#0wUHegmocuv26y`wfQL# z%gI71-?(UK-*SyL9`|TU?G~#AGW$dE|F)^uB@$EUvL%1ue{78ZV{U zoyl%RclM|ZQLf)q=?c__4?GVdWoQ)L5$1ZrO!|J=zCG$MtiKFfn03aP3Hma9vS@Iu z^w)Ik{rWiQFN*Pjb=ixetm4YQvZ!R6-_R-l;IMNP@7{6z%`y0GQ}NgWExq*v8&0eDQ|bRR{Z6kcxXDRB9$Sk#R9LB6c5xmiqjlE@J zuZ!7<$OY0cvwEG#5)AHYsjDf z!;TFhHmc-kyHgCh9pw2S;xfz7SVWQ>S0(T6Ssg+ zh0PuBOjy!ceF^f_;vlaYd%3I9B3s1>IwAJb?Nt$??eb~lwWOc2s7?$smptaObDF=< zsC1f)?pNdxSJUS#oxY$xR|>Vc1T33gOYi>of<{nx#zI7e0o?mlWgc7K=tkSnBZLWaa zfy5HC=i5dt9vh`W4mskrm|91=IX^#hdhKg+Sb#iG-E5wvN>yl_i^%I~uaqGc^Mqx{ zbz8W9dDW~3u@6L>UI}bkUY9qga0?c31|BMBEQm7*ptC@A-9@d%#0{n*DTCnV z0mStcn^Co)DAJ%H1{LuvFS^b*LAennVpGvE)>3f3@_Tm?PiQ+dowYDiDf2cnlgpI! z1Z;yf0pAoPz1LSl-0IAqBsP~=xK zAWQ6Aj%p?Vn*b_lXbEMsDXO{Zx^79aFYtMoc73{0slp=d+ z-M3%?ngm8O&vB|7!DD@gdAN0hePX1^8@gKNcD=phq0O^8DRHtZUo_D#55=C?h@yH^ z4Kb=9yteR~Keu|-*9=cK_|b#P`sS$t%l2(aG=tT3e51|opj)`ORAuCHj1Go*1`@OJ ztUp-4ZBnA<$vlCUKV;id#pX+R1*R;G!+0@9jHjj(Oq|wt(fNY8zAQwR2io`{A!$0P z|97p7gYL`|hY7k(9FGeORQ%Tu)&b=NG!9hEb%?rw%YJ_|(e%px7AFLUT=Ew~r9*xl z{8ktFn6mxB7`83#rl7?i&A1k9ZgBXM?B~1Gm7^OgP?&G2Vn00y^bQ=Fy||D&PrNbK z4~a-H51pkPENHK@@5r>-G%zlXKdlXGbVG*8hpfC9?u-e=R#w*iX-Q>q#PV6feYyvs zm%6#sJf&i;D(NY1d4w-_o|eWrihk`m*l+Y2?BgcPz%#O}S}7rW>8x^)TaOl+e5=u{ z`JH)uBj(jhty^k=yAFPJXJ}3tHWMou4^*1aHw+?hB(Hf&k&<-> z7yK~>TnZ*9tQba)qlOzBtDU;iMJXn~vD+Bex?+2#hc~+HFM;r{6&*BltMGh$P0qtl z-!&$I=g(RD`|TnEM^_fnlD@9B1~i6G^rYqjH}#;BHo219G>gRsxS#dV(--M=QMV@T zS#&9IsNwH!99(ve6TxdbyooSfx*zMJXan6(W4!v_<90%XZsdLc05Ixe%Zkm|E+WLO z8v7w64y)1$)o-dlepZ0%hV>D`0Sk~!5|#WO+IU`YJ0H7@v#pr>*@aknUS+g2tFouZ zz<$K}bYCuKcInX-^U9}QJ?BZ?hz#p$-~5ayLBLnXr&270=r|%#;!S1rNQSqBr9sJb z21t0m4)iCfJeuX-eWn_lB-K)=UHNMQ-Oh;b;q8v9*byXE@~ zK~V}rYn%>eF<+a_GX4XC!B?LxE-3W7lbLaz(fmHH7%lZ3is#H>smxs8^1ijT+#l)p z!^6vp(uu}^vjpi$Eeu=W;<&eWiH#8R7fF3381$qms{&zgD|gOE*0;_lGgZRG1zcWB zsHU4yx!b%w|6#=6s|Jsi_jZRd!^Lww=kxvm$YDQrrMt)W$Kbcyv!S zT+;mcIZ2bZ7xN;K+R^o;2rmJhKQ7-E4_={>V;N;lSTU+&( zEceOd^fJtOomxjLkd+fY{g^iTpzAf5dLceFg#dQG96B^vKc9<_sTEQDP(6LGr)q6N zNGe7-YiIZE^Z4E*%Cj2T@6KTj~}w>m!qQF6SW?y$udxa;4VT>U8gi7-bUCC z*kl^n!CACfA*@o1jXEYW%IW1w8|!Y{LHKJ5a?C_<54r$YfEbhk;bfw;#KU6>N~E${JLs_RQ1M@uyOmMT2^13SEb7Ctm-hl}Vdttl|y zVbttu-0L-NhZe>s+CTWPAyH>KM8mtOLbNZN3cCYQch^;onf?H<6fRGpzZOLGR1Jx} zs){G#$q1Cpb=PKDKO&T9;y$xI0G@GtyLNQ^1BmU62}q-P>*qHiH964Xi_;_aK>z}S zK*TC)$z#R>y`Fb&*4v@k?4gIThhWnnP?c4frKVnd&Yt$CqmO~kp5NCuc12r(e{?z~ z#*@Dj>1bbf*Xg8hC)9(r!wVO)eo%ix+Of0p20yQMreh{nZ4Xw6$Q;G^0*0f?sF)B? z7CC>lc~y8)D_#X*CS{UJOa!0`9ecExf{}$qqG4YapH*x;Xp#^OyV$q8Osi6qsSxbS zq^AX(r^$s_g@_zpi{PsC5tF9B6riR)eBWq~I>lx5xR+ET^Qu1pJnj+(^b40GlNWaH zRADYUG?4I#jA_IH!+J?S#5g%WbB&5#Ut9pcpx#Jy1>aBNs1)B1K})n&)m+s&f?u&N zu#?UV71h5C3~D`N=(`!EkX$m6MrGC|EV}zJVa*5yu{pWgGwB!7fl&;g!NAPiQYTVn zgp?q2|A>y30VLcrL>q@jb)04JQ{YSx)7#a`GU1Q6{OLFEe2WGrjDHP|A$3EGVdFo6 zNy}XwfnF{Y+F>9+7QU^BtEb+dR2x5^d)>GF;@r&@U22UP>I)$i@m_Q~^{im67Wg9Y zRcLj6^97}L%R2{3Qg6HjN;0qGWq^GL@@H>c2eOA|Kw&M&zN7P>r64tFK>_z(dvPY* z+y8uEMf@wtv_FGVhZ}m>2kWe6 zd7C_jx-kA7`uE~;5&1XbJ9zo4G%+@_m(O4C{FukRwP^{O+JFiJ<`U!xW`i^gc*~&> zbz%{S(*)UF5@+=(_OkM*YmF>ByfW=*iXDLg?nF87e#>{=-cu4|an>s1IgQJwI#F%R zI4N=Pk#%S#{TPozSx(p?o*yR-c-lQu*ht-e0+9bJiUF zj786;K#_vuYlO*Tp1&~GGu)=#6-ySM&V}1AsvqPKxMPJLoU5qL>1|kBX#)`mUDTP&LwbI+H9! zLO7I|h%m`0m56^@&bwX56U}~M8QLRG#2QufT-mQoFMMk-TODQgI!4Wq$6>ZZL8{LU zEc!HluIy{Y%V@k}%4&imCZ>Yk5tVx{B_$>F9M!p!RK2#hMWS#uy6^$rZP*ss^zrg8 z4SPWpG>C0m=X^h$)UayiR__SSky5=t>)grSNhu`2P7x0JzHiB+GXw$&#TA-U35v<# zKae>&=&Sczg6V#!eMn++cXPoFIR@@gJDC~R@%#+R0AQcd8FU)>o-svY*o-W{AR~k} z!`vtTF^3MZ?i788)^y%-L=+e`RB#Eq)hl5D7_##ZvN9ceol%{@mBk z(8mBPNS*5zWp58h%Izt zpfZyKD_|ZRbe@sC6f_VFU=y^h7^^&aCct9J#;*kzLmpSK666=46Saowl%?$8<$9?a zoI5BK7RdI4Q8fA@c0PLDCi6H7Cn;!y>Rf?K03|7P15+!G%LAJ1%4e?pK zL=fUI3PrE)ICpOu<*ds7KteyxhutuS7qVaWLy7XIf>VSE!BySf#LIvk{)!#`zPZ2?)9@7JqP z?NezYQc;@Xz1}E}wINjLH_`5g(X6iON?4a5`n@Q`st8`ukqZakov>|}=oUs2pA}Y{ zVmGgj%UZ9vB$bn7p8@sH@Gc{ZwC}sCrz<;j(U6ZrlfS=>aR?$n3;{UUSb-iEXO+9D zO@^v0exCFPDMDgVb8>!*BP0PDWmc|>RU0WC^E%Fu&uqAUBpv1# zMIWogu@$n*v>W4dcaKtc&-g4JHx~>qyPiV2t`oAViBqaQ-ERq^B(A`*RbM=Cy)PmfJXj61r64#>Bz>f;?hCK9 zU;3O7DMw>H8znJLA(xe_M>INoxR?FK*Z=-XW@=U_ssZPNRK{jMT|$AQ*TEma&p<){ z599i#Cc?kPbo+kB)=F%;l2=#gYFI)sHJzn-VN!zWp&xA3hrM^Q>V`&{vN$B9Z|Qzf zaOL2r_M*@`Vo|XhJgB?f%>Jys5}EyNz9xO_rR&MOI4JJw682!V>OriSG4OFBLjdh- zWvRezfUz4`%&aS6q{?!}A%Ka1vRboJtI=L4*_pCG(z3(T)92Db2om)1;?pB9oYrhvn1C3AY**g{PUd?0 zceOK~zLb&|Rw-El63SKWEH{jt8wf)6j896yTK(_oOJ8}1znrKD%p8}z(Y7d{Rwc~f zH#kXd70B{9T9WVQw&$EsjFu@E!lqXg5vb~dg0L%FoWbErx}z37`{q5iVA749TF#DC zgukU66I|PXkk;H@Z0E}z7r9lGFhf(mu_^<+6GY>0GRsv;0sSC2U6IdpA}<$KqLm5R zcUk&om2wdotBc;v0l^WyC3bNH>epeZvQVpe)hW*xQ zOKpk?Lg=8|c9pDRj@!%3eThC%%1mpSwY#U$K*Hx+_m9j}jY;I6n9yTA+17qsG+{KQ zWXxn68)hy!(1M;#3Dtjtkf&&Bk^@|H^OakB z>PErbi|3Pv9^Y~a_W3I2tQnp8;p}~MCJdIrky!1Ds4^&j}wp`&I#ew z*2IM2wY~<)Iu4}Uf|T?!@>!w5PP{`Wv8F#W86yOyByJ&lTG+INZYsfcaJ+J2b5{{3 zu&(;WQ`WEVbC*`l$Cij9IhIfy`_~bj9N7)#ufDuFCoB|QD|(sQYQobbdzNBspdClI z%0xh%=>a+NXVuKpwrz7qF*i#|K{!`ns+sS<{{sNM>^M5PLA>%@tk`ELF(5p_rvOb* ziH|iUIg^ix72AuoI{V{FsTW{HR$em3mQ0rIy0i)?zF!~inS43m-Mak|%nw)_;9AFS zyC5No!Nm9Kpa+`IPSRH8;eSpJ?j+D39hU1|c_Bh?;+s9pZ0h(VIe43V)RO45CHzb8 z1=`B@xbSTXwkKY%w@DQv5;xbq!;j6`RZ)ke1Me=!9_`f@%jc$L%cA$DJ(nwS& zAQ8d*WR&mA`B9xS`*XWykgoAL_YP>!fO4YKpBcNiqHi z%$b|6rD6rd3p(Y3KZhoNkLo3WDecyE_TId<`!!I4Fc?KA=1P7SRQzFoiang86O-%4 z?2T(mXZCEEu~Mu|p%qz^>VnwgM0SpzOY@do4b{s(kY#=T2_pDxV!ss~vgktoE~a-4Z)JGfnN=RDO!dy+=&ue9I*#A5^< zT)233JYmspt?p4geTz02@bJx*uFw3NY)||+Pv&?ue?Bsxp?LajTN0JIV^^?qRZ8YD z|5FVRjLlR{!(RiW&aloz!Z1+imX2C5FuXi`M|;O*q)(|S%d47lu(p{UK6~E^q0KTV z)U?NeYN}_%`A)O>rYBrOEI8aE?R#8CDX*_?`r4DH(eMrqF%hxb>b*N2g<2_GG^g6K_~rlgF{ z;lxqTsV>xeTs9L*Q*t1^M7N{2h!lCej2d($Hi{g4`o-v1^x1m@n!MDXx>gq#bD4>i zc=a>9OWGEH4GN4+bUXcidoWte@{?w+x&%E$`^l$zOzB)&YX7amW2;X(-22HY(kD2H z28nD$IMG<-bKLJY@KEcd)|H-LTAwt*-A|gJJDeP&p2Se6I7yB5`jYQ$e3+iEjOCjR zTe+&~m;^Z{UlLC_22UG-NHGP!N~YVCcXR(_Cf&!5Cn-W$pntk;N?I}!Wmg5?JYUjS zu!VNKYbH4@&OH+dCTjE*g_1<#PH{AUCW(f&BLk{hKPmBP$e1Hvg|cQatRPb~YrR(I z_POB}$&o$ll^>v_k{bMy{9Or@J;jHte(~b?jy-zA{rS@T#iGW$!^3c5?ZrkvKFg@j zSjK_6Zga*wl_GC{FF#3Z?RW>us70*2(a7Xosv56hHHH5>)^d(+b& zQF2vC@ck}haTkwMu!*-=f+Fd>{p!_x*@Ij5k6u>hx2E5|e|Sl=N?N5!cHF^U%{r#t zNBT8R^;-fn_N{6jjZjDFQc5uUa%+271r|<_aH8Vwj@*o%;TzD|MdQ)U+6EME%7Upj z@Qvf6QX8OHq=8#BB6DbUFwij+td)~kOO`ZOL_nkwm+8G$0!GtN33wSg5JP%kDvmdW z0$f+dj}$Y9d^_=kbe9t?iRbK)5;H*-*N684&R$a%!{n(rxVEiNDPY_^bdxou71MWl z@<}^JSJ2eiD71Y9@bMHed$f06F5_EMjhegW;B;Y4nK~8Ta8IeN*y{+9O9GR~_XFb2NUuwJEP|v*_ zsZi-Hs@CELDe93Nm=HFH+4R0jp}r*)+FpPO`Zu1960 z@Q_aTigc%+c;#^8B5OIoUK(c-2f`Cn%GF`tC@&cGOsV{ywBZty1y!>OAJrs8kNrN~ z!E3Llw->4^uPij)Tt#Z-D9(NsF*8ZLUl!^Ru2D7UJ!C*x(Vkg5aqJJ4h<%peiQiBe zjV7lYM9_nemeNuMahra4IcmoV{Fry{8~EIRQ&9{gr22ZpVWvFatby65S#>`$^VQ*- zaic|ZVV7G~ow1Qhs7De;0VnPjAIF z_xHjPX3~tn5G#9a-kU)$c)^0#zyzaY;{I zsq%)l6KfoIXvZyzH!#dLOo}6qE95xY9o0~2Q*X;IFMp+ZOuvwn=-$1fc;fXeWwZNg z=efdVw|i5G5frAL4Qh6xnuG+)Gs5tmrrEW&1l+p@8_Ze_Xc_e5?2$OX0_??Ld}T96 zYurCQNGP#UL~uxiK8}S!U;M9ztVG2;zjiE;^_XNxlS4KvI zYnw)o|7UQyW?^uXDkn^@_S4p+LEAakg&uQR45uK<#~)un-nPX35KZy2(%wyK6|uJM zm%omCVO#PJ)d~i=1l~l&z5DeC(Eeg1TF0EdFq{0>&>A$w@>Ji!#o(9`11B8Bl>D37 z+PY=Zr++!@wMd9`mJQ6xDn*hzmZ}p};oDe5PN3mz8@!)=l>0ILwbh)#e$v;{4QWzD zF`jWT>3r-<;jYx)EKkx-h0{LwnSrkT{Y{H%tHZ@Mf8hXIihH6XGL!|y&2{VH&Zb)O4;4WC(RLjZ>LptX=kSAU3sj`cxn#3_tU7W z_)5~E1ko z10lIU?$FAWrBv+(aiKH!uJn=Revx|VpyC_@-R-`OfhtV{7(?W`W+dfZ!5|;)3}k#= z2pD?dh>^f$vt+-hPcjvPUF*XNp|$~4CS;Em9u=mXBr#EOZNr_S)6pyxJ)ZW7d(3Z! zBC2t8zG4dcEJ&}i<3KIw6c}D9zeZmXkxK@{S}L?babk`1x)o!jcnoIIqcfgShl}5T ztaOu>Xc9!vJ04V6G!srPw&=)sh)_lm#oWp!7Cqk}Ep3)ioq8}cwNG+5Y9=X!s|7Ke z8{<$oJV|et(!vE%20Vd`6t0r`_GzhMmQ0mOmk|)O32AWo5YJSAYod7A^Q*tJ6s@*H zigRi7>SMLUXtk8K%#JE6pa_<_I+%8>#)Ur+1ND#_o6I+HrEq_CiXm2`%**C>xd77I z3XGc996oj!$cdF;VYbcIrlFD7a9E#Y^%HaDB&NF=W84M>!i7E7h|CJYJbOED*+Ir`wOGp>y!_ zB65WjkyvAbRoKG#-GshRS&A(xdP;=aF&&()Pnu6(dnGgasMGRt zZN@}*3?naa)ZTk$mW-uj@1*IKJ|;)o5Csut(RT^$#GKG250$23Evz`Ik>qQ0x<7!_ z#(N_rt;}X$+g<=PS}J`i9``iAPwU>%{JG%qeGps!SiNh3*8R*>=rAqLWx=X2o3W7r zqg%gW+q6Zmj5Y)N0G-m3S*Nk6<7iMU4!laH&?_4I;>6mL1cR9(G&M_?u$|Fb{lPji zYRjd#`3?Mjjw8F9!&;_#=9w050vfA0t)OLpKX@+8lmws!biUX>vAOJn>NwyCWW%h2 z99@$g_DyY2ID+s974LRb=sbQ{-*>(8N#e4o(bv;;BjT;C^3bBBbA9?$(dtfxLGC7D zNmA!SR%>Z-KVGnAi#7qCo;^q;B}NbV2avX`EmF}XkJ>UpNfGzB`8$vps@*m#-3oLLGup3=tthk=0%^HxkK{5`nCi zidwk=<5^%ST_w_37E8)ad7EYqg9(lwP#1>UEjAUU$lwFCWtdSfmV&w*kXGT`+3CLg?!ELa={mIZ_;>xWA{z)?^ajt=Mw9qCmQmjEo{F)@_#^So4>_`%3O++(jp`GW#nJ zjIo@dPc!COy^MJzv?&n0Sj( zdzL+JOo{*X#6M<#JZGQ#(e#Xh)P-$GHTk0PKkr%)wKOu}R?}_C$$(N>chT zZ$;0crt&5<_2c_jz-Rg%^e2g}+V5gAD)mb?3X|;&j(dV5s7o2<-jH1v<=Sx%pOb4r z@6>rHk9%#yp6vjLnYSW(4aYB{9m4fE6zEDoB&BDx@%R&?tDP#JqHMLw6MctSA*;1r zmm7{GspQ(!em-3iSBUk`xFRi?Q+ty(Y#`mJB&MujH)u@_CfIEH>8Y*la25UjI~Y#< zkuT(Zww+1t#=#rz@6U3m184}MR9+`%Eh~HSQ}mZC%A78>+9Q}y;6Y~EfPP3E_?jbg zmj{TzGnK(n0x3N1@W8ack4&;Je-1Z@p6JrSX@5|E2eYj@kBDlu&qiHQ{;I1qAAeSh1%h^k z(V_(udaCss(dHD!9tm#{>LbI6!wmlb?23E%P{h66XZxrOZxE3sdNT}6V-k^Lzz@WekDLwIpXX#Eynh;_+ z*3WWawJgokD5Ka`}ZeCKLJ#KW4&_QaexN!dA*dBdmFAsBN&{;X=<~ zkj=%?`N_A_F5gi&O7gXA%lRY{Z<`(a0$#jPrMMI|N!7@9eqHY3GL%!C=Nh8L7BR5H z1~zk_wry|X8b)I9BG$DAlK7KCV#@VXZ_Ya0zZ zO!dmhaGSH;Nol3WSx$pAr@zTm2;|1BM(Vgw)UvMku`V*>z?Jb6h*FSvYb>BiKH`r! zShAISoD2p?vgcZjtj)=SE4%g>AGR8l4Pe@UV89Ud%nVg!ZRYABc&u`PH8z!W>9e|0 zpM`Zax^;Qw^M>-hNe4Vls@(CVchnhOafGbV&D4tHl0c(8c z?O_xM`qzCW8p}xv*7^(WaV?aH;M$mEp@)+IbVKA^_rBbx!!pXXsfNw`N}>>@xQ=T< zOCueFu{wqW&E>BCdXv|Ekgsk1d*3}x{s2_OmDPodp87}%wd&XQeF~|Uv_CR0BF7}s zw;oYNW$NZ0WA8BZkfov>A|_(^1Aql~j>kZyu|c8jS#CC|jWATFcft?xs3(5_Thaa& z_NQiwoO036;^(z82fle`WHXv;-MhTXa%6EUD^{t;!E8+uSV^Y9IVD@zdYe#zgHGEKm)jgxg`I1HgT$%Ne{I3fTm>74N*oWq}e%dGFI=nDye+8H{hT| zWmTTGkUnx^r&sTgjFN0Peo&rx{UBfct^I@HFh~yfTC3N>NBWKev;7$0rQosoWEPVab}a zRU~BIH8?T>e_8)%Yu34Lbiu znJUlO0-a|yh>4#Acg4O3@G$G=DRSlNiC}e$&NyAW1P!PBU=NF)h1PnNb$o*Z>1Yq$ zjie;o190tif=)6d9An6N6EGhCq58lQZ(tiJXdFwVWMB(8PVk=Wk^dE`A@?MV^6Saz za{WYG3I(|q!i4lVoWo73U&5kexn=U&cTi6xL05yIYB+YH>~jI#i*Wc$TvmckoV+w8 zQF z2L$=yEC_XQa0anqf2BgtNwjA2ou(16Xy~G%MU+H`oJ&~zk{$Im4&QWyY|&qj!cjtH zfJAqKoZW_02$+!Va~v9MrrcX=nfAKV^E1-1U1fOsnfo5s%H5A%_nYs79>X$#UV&a~ zaPj2<|EG&`-qw{`)^?3e^9&4{4L4|q9cr=@PEB)0Mt*@={p2#y6z~T7MvSH8<$fNQJEx+$AL7#P*g+b1mW9B8VM$tRN#DL+#;;Kt;zPHcxp7bu z4R!-QR-#m-r@gCA%>ic#wlL!En56V@2|Q+eVW+C`5mkKLeD3VG;8~Yp(3O=KKCo;k zpsa169HhQ*W(?|POJ_2_U0yhg&RZjuofX+BQa05s4Ot74VL`~XnNUEG3B{?YW;x;k zl!_Q+q`DfLh?a6UI6b#w>LzR8)kbzC6ayS%%-PO?t+Ctx0K)9eJ%sbg7RU~aEpTxg zm`T!rNT!;(nS+ANAW0hrs-dGz{O~jQS|3~g0EMU^qUZ!s(<@HdT${TjS@|((AWC;{ zWhqLoTvSFhM`n&ZcS1(TxX8oc1$Vd*E&g4pH3AgULjCW?BYJ&;*z~hr?GdDbga@7gdPj@uUs|Y`GTEYZBCf z-X>M9wcqA+-C0t3Y`yb^3Bz0!@$7Q)#}%Nn0}zrNQX~<)Gg(l!j60~{^&s^Jde}EN zX|wfd$`jti1l$(D;^kijOC=^W+YAW@5DrOb5M_QHw1t^)g3Xlyiw-L5?sBiBgr~)V z?V)z2*Mn1Q#KBA`qe$D3hy7Mo#jFM@_ zkB&XHhd*uE3GSgMVALOTsp4c^FZ`Zko>l!*?K!)5mr;pFmquZ6dvTuMaBFvHrHI4$ zIJ=-7sMC#0K*MmVLRLg>K4&OTKD9*QH2(lhSFiXapw^)4wp@XjNIz6^evT?_U0pCo zkL$P2)NN((?W`WXoSO2N(+%QSHD&(->~T^=V&Lk~owppDh? zze5jxi%T?R>!o+IjmKaMn2z(v!-ZRmQs|YbJx-$D#ddbSpR2*A_pLgaOioCa%(Z@v z06+qra_fZ&_wxRaoBsgl!-9VQ05eorloF+A8I6_O7^6*&c(@aw-|_BblAtH!;J$f( zv6bl+UF_VVdJ%DCQU>3OHFVj_3eD1Y82;?}7`^&D#stQ?aXh=c&zlo`#h(~&>ZJOn7`g$Ok zi4-LG`0)pezcf zY7I{z6EU#4V>q;d7Sc7K8~96?n6uKR5&+#bfdIhRE3{Mr8(ceCWs0dY1vHURW=QZN zv#h?1%1cu8;?||cc|+>z@A=?0L~{Ox$TomTfDb0(Yvbd=qg_SRD8f~|M((r7TvChw z048OF6oz>osUIE=-<`I8srd0|(!Iyy#pWaA**uP1xRT{ZXs~k`RcfBl8wVz-?xvgr zM#rw79t_ru$VjQ6%UoH!we2xL=Iq*{Ph5ohDb$r{Bw{*!SthYfg~mB5!j3I<6cyB&r3z`Zoz2?j z+hs^jy&cP#=j)6>r$9cgu2l~c8jjL#9Lj=}n=1h|B!i%hf=qFT5f>4E7vi6+L?ub0 zK_&?VNFD~RO}jP_mG?xTsUkbT6(>Mo5+OjEP_l~y{IFpqCIBWON|nqB#Xl)D6Y`xK z?Dziw2I;T@FWCk`)YeJVM$OGwUs|m#5&mRM%)lUluWRSdN4+Vz6cmr)Cr=isX5Ii3 zjYNWKc*P^$<9Sw(uTvg@<6YxrH25zG@uto5FL+NAzi*g26q>3w$?@FkLo_wRF+$xh@=iCIBj|gTtP}e5g4n<`mqxf_PZRc zB4br*JXn^Isi`%Yh-APdoFsoK0Pffrt{f)NkFAMZ39ia`wp0tCNsX1IX$?k^bBH$; zYy*U?Mw>PgPTG=Km6RsxQVkR^g?Iu?Z>v%$Ql?=FK$Nd$!9T<^tOTzq0FS{69x|n-uhg=T5{K{k^br9NJ$LYI|xmM*;Fc2t(*YcHF4%8)Qgfi zBW~bGRCiTCM+T{7WdH*^BZE`Fl|`kh;;U#1BS4Kiz-hlo)u)R(tss;gJT%qIGMWmc znLQV2TDc<$6!Lzp4J3n22Hy&vr#jls8Jf>ne086asjBcDD^BS-Yv*@yP|$=Pqjr z>+`ARRzz2n{C^i%1*VMcz$B$TW6s)h%JoNSY^-W36oa{o3S1x0$^QT^{x8wkXMy^v z?H=aKA*nkGQ=<~qO#_el)Ox3>jd{x;)hp~zy|S}O*3wo92SX2RQA!+r?jm!RfTkq! zdy6%@w#=y}2O2zR!cF4>H-hMez!j_9Nhu@Mx^2rA5MVZ|ZlPI_w`B zvErIc)5%!ub|ewDvGLgCX_5%nXOgkwx!Y&=whsVZ3|EAF4P7fc-EVF z9cK)l!s!Il#n>SvPw2p8>$7p&@f75FuCDkIyl6J^J7D-M0mvJeZ)OdoVh~`#<1F|_ z&yLp`^{}+zCLvwuics$ntJPLXP@nnmqc~MO4(>>0PIlk*RyQ*zEQ@^{iNuq%n&H%n zCIGI}iXE2)A|hy%%*K$CU~dw24&S%kllwt_X59S5aF20iP3uSf!Po9+*OC&tB!sBI z3MUmK=8{}=iLMjLzmdx&qFlQ=VQ~lRB>0fYP8@~vD7Ll80dzY=S9cS`S3MbEqw~s2 zWLA5?fCRwQR09M!xK|Pn4!EIRxD6ii6=Zi*Wid(8C{CkQ2^<|UEou3V%`3B%mzfP% zZHonVlR~7A5*}l;t*AVq^Y<;>@gOBf_J|r!^-+Qf4icqE_=hEj;%_PKM^6Yli+vrl zu`im$YlVHkpk&6e5WX76o$oE}Hslu(PW5_LccT8lwu)6TF`iqXzqn)3WyR&k*tax?oKaht_^YT7LtI9CWG~s zY*;}NG2I9Ba4n}X_0~K!I&Y7AlmzNasI0Qog$ydtb@py`PODax8g2$TU9jzYBdV`K za1j(AHB8b^zeZA#w^*mdFcT>$fv7(BQbJ9z4QNk;hzwGgkCr|Ip-of2Y2CrvWk>|X z13Y$)!0ZkzAZrGF{^kJZ39i!sMzK@LUZ;p$+qwag4$bU$RC;Sf#LDD;!44V;P2+%q z6uDgN718QeX*~efK}kLih*1{|8V6M=B(66@j)at)UbQ;d!AL9T{%FP6!F5LvPP+V6 zwS1&>YOTV8Ipu5JW4imo&#gJ0>pM%np*-Sz3t~sYprICMJVW)a?+Jlw!m}h>gjT0P1wN zi7T=Kh!v3&J>N4Lt*i4YQcMlC-B|LxZuT2wp>a0z3bzXHBr0ROute{8uu*j@PQg|+ ztO6EyM6z1Kq%ATe>B`*!YuK+PD8iBz^GS#SLTWxCrsg8X8%R`%?olBi+ z{t<;+3^*yvp%Z;_v}0FtD;SDXS@14U(fjJlFx^YjhSoz$Q&Y2OUW)uDK3I?lvP1eDqfp|2oq7jw{<}T(hS6ntcgvt z1c{MXI?%5o0#*&LjqGW;g%RTHq>hpY$-CmLcN1(XGJ_l;Q*VN0sJNGH57; zlmTC@vB*u@w5)y9%?=>tT2%C(8{KUcxQZCQCG1YLzO8{!I7LTlru?O4iGO;2Mmiq8mkDuEO_2hMolf=GvGcEuP# zpp!fxI+~NS^GM(b^24XAwoJpeAs`xROnC5?26$4D zrEBB5UJxQA$Uc8IuK3dmIsi;@Je6_rNQ%&d;7+mPz||@wjlWoF@LeW^ffHHYEH}SYt}KYduognVFcJGEkdjoKT>Zb;&yYXaY7j z`7VwUQk5AA#jY8yFWHPRM%J!Cg+&vGP6N*BQ$S>IJD}|Ca^^Bz#&VbrJRvJ0VKjn9 zfU+S4E5f2}tVRJ-D%dAMvwP^xu8~BYdn3V>VI38)5)4T2Ga?ejWF0X(i$X!9#qN}MSK&vYG- zm@iUsEg*r$D8qiz@2Mb4Kr&C32GZr(w2Mf>iKgiU_`oh&B#qz%gN;B;d~Ga+C!3nU5s8#0)VI@ED~Opm?$=7)lT+h^Y2Mg^1%&6^+eBYm^no%cq~r zz^%mvDG7;%RPPY!eo(0Tr^Yng_bQBKOIG*;v4Y}_)!};pBAb{EZVv~ERHy^N=nSxK-lZgK$X;DBm$_+Lrh9&2 z=5{^R%NAlt3)!{j6lxK160AB@4-H+yib0)ZPlv&BF4{n|(&DMZPYuifEfko2&*GpPJ^-GRG1ZGGr5dmQ5dQ=If*1H->POBa&8IH z&a>Vr27t^kB&-FgMDFVjFUg_R$y#ACwxW13eqCY}s&(*nioONK=n?BsvRb);P)%cz zg#;kM(NqkVQk(%}Qed`QAmO9@ldKMrD_^K)7}s$|8`w$oE}|`(ZDYB0 zB^{0(q^t!J3{N9czm-a@f|CZ1IFdw$;&t~u0mw+Ey2m6{^BzDjgww^e?V?BoiTE5H zhOh!iNj2(q?x>Uv#GRrv+VBjdMGZEVH$p%dL8krhL6@MjdOd>^v6KYR>+#~i1gafi zmmnAf$d8v!KdY?_orARdm;r;^;K(3O@;27It%xnhbQ#pdP>&V{;V}T#37#8#UWhP3 zoGXtJ$79oq2ZRo-(+^TWP)VdnJcD}zo#Le*n|b)v(xgELVUD(15|J^jY4Ne&<}h{o z%m~qmMDX~~L%sn?+aDYq%@m?2DmXEPLEwA5lc;(N;oR!(lQ@L$rwxvwQVH2JIy?ny zBy9;*d^S2lyP_tO+B{i;ret@%VWyh}IJ`>ALTOX43Ese{0My2yayzC1RCEW-Sd{=2 ziqGVEfFni(P!(}Ua&&hJ zlqnnb_&eHP;gPhLp(75^qgaEz;08!S42tZpTLMf1ch{)9n`^?Nt-4j$NE|r50c50w zlTaW!&?Q|4wGkTy zel{>B-Y=U>B#kypQ^XRxk)=G2DwFsnLt<11FOQ4U5Qq7Tl4~$F>D0^l<)8vmj>ee; z8dXPM1|{N*Cs9%#PD?MyFC6M;>KF+~IFe+V7ro`CCZeoX zQe{-H7l7M}qZPs2LbsPJh@r29A2*$~CIKltk3ks<{6RWH48_Fok1Lac1jq#ZZ4_a` zWRo5h6WHW{MH0J^KS#gPSW$}>G4a~oC8=bQ0P?pG2QZ}Iz&3d7X-En&zP?Rbwa}AT z@o)-AIJ6Gvv!@D6qZ5j-`Q6a0CTP@hywRJ_sw~>+Nvdn*Iw!=yey7aea_nWW-OrJj zxGF6t{FMIyG=Od%1{1s&dRAGTE-3S?uQPL(IM`j=8D*^r6EFrp{#a23XZdHfpRQ{#?9^p zidI^1uY-}&I4U(DyCjep(8V|m(6c2_iQ?lMQ}vls-2pQ&z?_?yrgu9g@#xehV`qaM z#0BBQLoTTu9=odRWYDMhcP#>;P>~zugr-29C5+sp21{feOTt!5zlPB6=tqxZzFdol z!c7di6(gfkO-T|zLc?ic04K4<^fvV^e8QJta&h|3mQtcZRA9_Ik&5tLcpd%MMcu=6 z!NNp5HpQE17BK@{RUC~t-B^(_D@SDAQpB3Y7)|6{6y@LX;M8r421+oE_1f^9APp#V z+pk-&3I$OR6{Bi0VuY%6IB@Yh>iLf#5>!$}W`1wr>HBZM&(p* zB96qIWjIJt6f;zy@LJy zbtAC7tG@y871U5f3af021K3?UH3SSq8tGrrrsQto&E5pa@9X@@V!fEX)2t>8DuO() z_McCgv?~;lJA!xUN14>EAdExft=}l6^lB>!1~gBH#@8PaTB?|FFq3sbN=s_*DLX`o ztE$pPRl1)N!D>_#Nzv#fTo@D;!-Jw1(kv6l$n&I>C~u!5XbDZYiq4?Nncy#Q$y8T! z39jC$@!3w-SR&-sKTYg{Ctly;@u7f&^1>Qwj2W-9kv;8$+^g5NJ#} zS!6Ea%0*_Ufa3MK)W=n1oQ;sR3xrWWFOTTfmm-lYiAoZPn1S-vdbhg;AwvnC0wdn} zB`nP1ZonHv+no<4_S0=xHD(t4Jg&nOeOMa}JkBIaKvF_P&dG|g!mT8OLoA?xb{2nSy~z+g&>Xk&xXh*CiwHG2!Us2#d_ z*%Bxo0y?dk#ZLn2kQYTyTn`62>P8Azgs^4c$kX`mZUB#P8+uK3GpsPAr32-ubF&Cg z!2=e)XD=tsxxR8%8wepY-&(0{J|}75B=R|}QlLk=gme@Y?$#w*hON!tc5i2Jt1_k8 zw5^(5P~Zq5Jfo5N4pW)3`4lBl(%hgbRbc-BK-uH37Nsyl>%+jUWN7kvEWSd}vtrn` zk-Dl;-lZC!5?tWNi)+*=^z z#Av=cDEiN?47*yBGTP{Twa;I5`Nx@wUE5~$sipftO1Jw|lpYEIZYA=#lr?{7z#ycF z280kQID-7+p1$I9e>hFdpZ*==CKFo=Cwe}oiYPr&;Nx8j)aIs)mWXA*%wZxia z(h{4f+)s8aVV$`>&!d#*Btk~Vvu-(e)XNWYJlp%m>Lj_6Jj zPWG;gsJeqAkm%Fx+vy1a=qH1S?-mlXZYw2Pjgr`okwnn!+zSapu>v9`+^4)r`0ISf zln_Z#ts4X_B|27JFvM0}uvWU(pTJ83^W1^Fb~ybdIFmEf?ow1cqk}NFk6`TghkzMg z6Dg^Od?Ny`e5X!ER3KwLp9RrVBn>IxX}7U}NL~Vb4(2>Ws=5v|cgCWSl@KvHoKb}- z3**S}0#du+1M?DmI~i@OrMRS!U~R2-JRvE_qZ+RCls9)aCb zpaD<}c=f%m&6|ECyhxp7yM)4ZrxpaeF^79iNf!(n#dk-IT$J5`1KjhY-R6%Y#0-f@ z5JGp=iCcmAHD*%b1tVbC<62Y&6TerWFM8gpq22Hp^D7DnP8!aM8)4CRF+-5JLQ-ZP zk_McutSKm)cMM}x)bAA}K|8lSwQL}+Yh6uGX{*4x?_?@5tHLA1NipOu$_b4?_}pAo z96MO=5|Ei1%h~~NEk_KOVoGPj&cSh04n@E;8=GJ)Z1)s=e2hVA)J!@dHvmvDCrKm7 z%n+no!eYEw467EU>DWaM_%59g(sA;mXCY$6D$)qkbsePFS~_7XuGR$>Nji2KyvHa~RyH4a z4@y;95#YkK$W$mAX|>H7gbY=H%%Yo`(Z@K9gpD=e>atK^NvMa_qvCe(E~tqE3EcWj zm4TN)QmC~=&cV5+k^fUt0hHY1}+s ziTSx|4Xg^^3dU(l zRznq=-QbcrT)UuSC&7!NNjS7KvrbnK7YQ<_mI%aY4pxZdaBYk#?wUu70J(rp7#m5| z`WM|L&Qkh)TiA~(5^ltC|&&Wq{bLDZs zXF4zy1nqv2IxP=6M59vRuKJJ5PA)#CuM3n(_cJ8XH1htAwiZH-OJl&UNE zYN~Qa1Z~0eP z#^r8MPZh&6IT(prtNXsCc`*z_Q&GaER2oP{I=jqhgyaK^{Rd0-HrLX*A&IlvR3 zlL5y5L^1z&+R?+y*FV+3s$_mw>Ovc9t~!gza|^gQ?%(rX(a`ARSi@QqTq1I(K}oB48;aU{E_= zpkDosi5RA8ICA1u^umTEF08E(R!VqG#@N!ob zDTMxr*!kK)Qw z48lg6YYYJdoxlUFCuc6#9Y|HPL$gT$xz^w++BWxu_!8a3px#P$kxsXuiEh_SxHys{ zeO(o4JtD29Vs)(Yxi(3*wG+BJ+Zeh-c-0d-G-S;p{{WB*N)n>2yMd>RNf}#rE(Dav zE{$gSP4xvJ;m}1Z*G5$3?R==lPpwBh#H56u0NgdrAA3SPOL%5?z7)|?ygakx6uRs;^9l=O;SJNd|m7Jb2kmkt6jyNRI*fj zE3bFR<&yl)Ng0qk*=2XP^lPUqD-VR1 z;>96He^Gx;+_!5tX62=fNw|`g4X{LQe>vuIx3ZawwiYXGYC${T(DI6h&4{U#iKQvy zPmPR9RvL=yI~?+wcYvMY#>29T9uBn2BR3yHfk2FVo?_NEHc}D@4)*N9#A+0tx;Y&& zm69!7k0Y^C@gG%HtAwM0L=GX#QpvU>Q4P?rk8mrcL~tE$TI?w`O?Yi_LaSYla&GJ` zBNq{2O9@CZF&ezOr3%c<2f4+_QZR^sN#$@en8LpZlMw^TUmnBFCtE8)G>H#y5N#M8 z>Q|PoGOmTumYo#oO;{Jp;TG$xmMlwxwm_Y}EVk{CgCe!~;O=$=_U-b$GAK#y^D|%&w6m-ZUwbJ}b^)2& z<_Q|R5hkF4y#AExZBc`|e3E$pmw=)~2sGhZ`JK#)O?zGh%p^#68X4;z<^_1^yMM=1 zYF-#sH*4;CloPMVB^}lZ=qHcFJO(8-n1_IcEF?$l>FsS}#6gTx8q-b=*6*hf02hFl zbpQ$u);mgsm$w#3*v5q&tP9*+8K$At7Q3NXc2Av(@oCx(IVc@ZiJ^7T9}0@_CP1c)F^5phq=gRXug4dl0+a)w zBWhI~8G^znyL0j2?k^zF2;_P41=ODCGq5>1ark_v$Q)cG(rD&dKuURDFaq+xxl$PD zMDn=#uvK)%;J}3`3d6H=0&L3_>>)btb5w*T(vI9D^)z&Mrm-=r&|ca~rW&q8oKelD z?c0e`!A=kpt|Do#gNI=_ad?LhoI&V$MD;r&{{ZZdsx@fB=f3i1vvQOXm$r5z=RPaf zvkT`6i85=&fs+sq)Gd>;?#?zhgq9*4snDpghh{W2ws#o@3zGE19g?-%P*Mp>fsmw_ zAn-dLI=v>2_nSWUU(4aQoCf{9Izj$u@t6zhJWKSvrh6-&wbEs-rs8(R3V10dqrktN z+4A{|*;|6;m8#mticJrP5xg2aqQxpcjRe0m``eQ#zi&7(4wQ6jBi2anl6RET!7RMn z_q_@!x1L55^vE6G+fL6E&wzbjIM}t$+}&|DmYbM{D@sWU7?V+5EApMdx~CTJ3T;p^-N~gDsQ=S_xMlE8A9uvaoB20-iA} zTjF#JQWlI#u9CtNiDUU)aFry)YR!T@Qg)dq$RY9`aT!+0(0q9ufP>M&#iUR@!y|4H zyW$?Cke$3I`1J+=(vq4A4Yi*mWCI|4ax#Tss=FvpX3occup>EA)B|kPQczyQNCdCU}0Xs!VF9sZk>eNci%wDy1%9 zsUkOijY`zu9vzK8qBiP#&T1}+tMs1@kSi8Hi$1F}j}I#Qa)Q?O7>`1R@LU{RIw zyabzvq)r+4YWB$JLD0-~HyCR1B&|S0z*w7vm{1eLRy?(%3e&}s=;+cJY%5+y0Inn@ zLXvhc-3Jgd2Q6HS@iR1yoUSr$QgrgYT7ge5ZvJ6H&R)vzuc9|hF2%5vB%v}*`?iP7 zw1@^wvVVnhnpYBK{?*UoXt1Dm#~Vp9YLDr0Hh-uOo6G zw-l?3pUY9s|&5eWq0249jpEQe?N>WYRra=DyaeKd++oZYlmw$M# z?>x(e)9>yxA{Xu(ia|)r=FXq}E@RO=wT&}2Gj*hXS2O6C`Q1uN>;$Pvk#=yXreS~5 z(K1LUH-~Fy*(K#r&a)8;*f&T!t%=#zGqV32BrMgLj^RM3SF0bh>{_%g_ zad5-`0MEO~WJghQ;)w3q&7c1O7d7aZKNnGJHl81q%-8<__`?r7sYtnGB}z#AX$ex3 zqW=J-@o2d25BF~SYyR_`u;B+!H+PW8h3Q$UtY*po0NUm~6Lp{YeA%z|T+gC(=EW5z z@hVXx3}RB19m~36C7@foaDaMSz`*|iaby0cju25&`{~}#A(42ULX{)@jNkpPXVEue z=ul7?lZ9AR7L}nuz%Wvv4ul;aG;Un=?*8$A-fM|5n9j%cg+|Zz(BmjWD{D5D*1iH! zq)bpMEj-fnacU`eP?BVa&Q@yFdNMz!%+sw#Dv5Ge@}5^7)~-ks0>;7O;pfcQC*x4d zaN&aD=9P}h<6=vcf3kic<%*khWC8kxhP$>9OD-64G@xV&CE(S_3=vR>tC5Fu8#)Z7 z=_&;du0c$(DI`21tQ*H?N2JR&ERQ*6`MG%V_f(nrk*&3VGITyVjfJI4w6GS}d-F@V zcqAhlijA=_38}+738))1ar>WIM-b!Xu3%gp6x0-iE}>E9es&~s&Fp%VzN8fd?5OKw zK~g?m!seSiu7xA`xFZOvRxMB`cw@rkC`Y_8Vo)@wdT22zH00jTe$~XGHx)|Z18&pF zqmg11v>goSrrDOniK5z;3S=yO|V9K#jXyOcqRSPndhlsZ5<*#M?>MzE=@SGr9}F zVNC>3ahmcyt+N4DI+!u>$jCI7U||@jPV)A^F@h_Jak3yds$Sqd&t#~?nyY0U)Rd(( z0f0ROsL9wOPnncrN`ADTjR9fl4+k>Jh$C$8W&mIxTe>>kkd%W!M~fxpLi8RnS5Tr} z0+$ZR<#;hg7%x!0#zmxn^#V}Q)L}p{bz+kN!xdjE&`P{zCchHLW~g6^Kk8&q?!2@r zUhn{xOqkw$rMqMmL1t1$60Ne}VFg8y6_?&g15@vI%iB^{B<%CJ>Le?X<%_FT!U@^u zao`ELkts+T%ZqAj$j`KL`5lE)qq*k*M_Pz8tC?b`k}F?3fSSzK9HgwF6yo)oDsy*t zF`UFjdA~3z{{YU*KR8dypXCqGvUk;D;;satWm#K*g^Woc<}%U<(D)iTLpN%FK>L{-In$wkrU0$)w{tN#GLU;bKlm(nLSZb$wPHva(G*#7I`x%-{3r<_%E+RLqX zt))sUw~C=l6q8BYz1OA&Dw)>}tTRF@g+g^YxkC=5othaxAn*>@rmiRW<^KTe*)QA=d(ZL9{{Y$YU$`Fk z4p)deo_8OMXTtaXqe2ckx%V@23hd=pM8-QsQ6@}cQI1DX@yp-YvR}9!_c`I*x`V9d zvM^>pXZ;snI+F=cfMFc}0Gdb3XDFZLwT5U#aHwfCWB&jfyfn+?hyMWJJ?|bL*p>9N zNo{|g1l_Z?|nRX@vX4A6?*YP5>Urb*>p#nT{b8aLA;moZ0GhcN?PXW|*0E}9 z^C#0O8QtP6?wiW`Vk#Ia(}cd{{T%7k^cZ3zx|&j`+@IkeBF6j zyLa#0fTiA1IAjlz980cvc_Axz(1iL6iBO6-moBW5CI<>KC*^TYrSc?JOf^aHV75EX zM0a@@t$geUam84hcQ~ZSW0`ifaV~tkj%Xa%wlTpGdyn67xBvqL#C7CZw zMcHlWSedMR$H9q(dLEa7=v_(_0^_X(%ym|e4HMonEsU`e;I(UO3QAftpO&gf*&*}e zE03ch$;w34CJPKJZFuc^l35GmAWyN8MHZ^a1Pz}YG-&rMoJJxfzNX*GkL4am%Kcz? z`Q32Uu#4M8+kcEw2u_o>Y1J~uI!w*Mi9`}~B9XzaUaAYr7Lb7;Rvpk;m`DHF=0VTC literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000018_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000018_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..580ac6413da73da75b045efed5729ce2e9da0827 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTgb#YfqY(7_cB$YF=9W>g$G z1Te!8S#~tsM#E<`Js^j`XnH^njloquOaa!QJ*>I;rNyaOt6?T!H7wQXl);4Tg;DlE bhrro0XKX=kpcPGP|KW2tC`pzk0tr0;T2Ayn literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000019.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000019.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7146b37faca4c921ec3549158dd0e7a168353ad9 GIT binary patch literal 31525 zcmd42byQs25tTmuBR1PL~sd*6NU z&RX+ZYyO%whqdXV&+gh)^;Lbf&pGtV(#sYATj`DB8vqO}000C10lYi|uvE;wEd{wb z`8avFxdAV$09gPsDk>T(8pdA-HWv2(4qRMZ0s;c)Z)|K_Tzq^20wN+J=o`=%5)u+h zN^){KI(jxXzL!BjH*^667#Qdh{~T}#0C-p!L?mPw037thzfZxy!oedTB4Gny;QmO+ zNC-%+n?MmF6sDEW(i~) zv{Q>3`{^xrcS1e*8LZ^yn>@V+Z~(Bd062IQI5;?{vOl)KKo#P`@!)C56q;YS1<%kX z!E4szYw`ANze1q1NG5RSyZp5C(hI;rhVh3Df`g%jS^$8<0lcgL(El{Q&fEY zn$63FBGYiocX;-o>|DRRx&5$nk>&kx!{W)h<-F`} zi*B$)68OOVyxA-6UX4h4Z~)&!o%!zC1i#@fpTobIv1n&!p}Nyst2fSxsAy3zcadP< z)Xrn>eU;V<@aOt5!4V0tI;T3lK8_S!ExPTJE#dN{x>BgRJ4ncuBC1#sXX7~k(}zF? zdv)3vBfI6j!^hGJ@n{aG@^?~Af4OwZ`NtMgho&=>|Jxh3fb75Q%lpg!!m^}sfS=RCWET0GCOgG7w~LeQwGjTyM*r=IJ5bEP_V7cevd$IWHE z>DDWRCzw&!goLAZY$H@)^3CefhHKyWa%bnjrJ2eOY5^QO-Qj29!X_zbnAx^zl) zqh9?lF!W?S7?K-PoFD--x17Lk`)Oh$-L~)yfqr=w$D>l4E;c4h_J-xclD$Fa}*z z#Yhp^&-8Haowl4YiIj$0q}q!A((T?hU<-_ClEQZ0bANbsKp5lf<~Y~M)A8X)knq#L zHF){aSqk_$_)ddG#Onv5TIz{*U`@6|QR!a*S6v}eB(Cf29~NS~orSC(oLz^3XyjXH zDsP&CwPD8ouBpH4yOS=ZF|9stX_vh_xzV@U)6!Xpl$b2B_D)xG(eQMZi)Rrd#jU2( zdeYS|$eYnwf^S!D$$>?k*NNiR{oSfU{9zbWpR;ubKeEzMl>vx9tJh`RNfCUe-iN zrEd11{+{nrsiorQn%^%crcRV5pIF9iszetmIl#LJHQuxbG7v1UdI^UZCxtyc zW4-07gAMDach82KjUt;}^>ZFqvyZBQg%7pd&>70D!DEaaUmJa=y6canuIB!yhbpS} zw@dN3&`O}Dh#W-k#zo||MFOfN(B?Yd&vG$vulQ2n1)%!VH~c~UNS+yV^gQuS@@k|C zO0_x<0{>VeHF!@|_5{gvX?iYkKddP~S@bw}`~Z>SkZLsevn24b=_}s%fb3fn^Paii z87^??s$PG=WBCu(rwN7!(6uDx55IctlbuRccwdNEL9}k5z_my?ieuEuJZ>}kCRbK$^r&}KCm+ZUiZj+siyVdPai??yL zDMz!y$Vi~-DeMU#CT2Sg#QTic$S8^U^9Fg-T1P_tJyaDK8Uq%WrNcjY6>rb&$A8+*jgf`#udNVK$nq!)Nxu)f( z`yAhWa%%L%vc6?Fnnn=AOmH}W&Zf8aA2g_T48ZXn2PX6zHWOEi!&q|Y5;G5V9WH+` z|N0l^QvVala!pHbE-A0b)y$u1Ky43GUdvwlG4Is~Z$Z>BZ|2nAYA$S&J6zQ^EShAj zEMq^E-Yn6U7Gu(?UkWFdJ^h2b;(sNDwWBAl)<`H&`vyC{!^SIU-KW`3I@KKPr2I{- zDDm`4+3$uu1c&V=J#s%>G?969q7!~ABjx(@Dn^nuH&S7mEG&znIc{Gl#hLy16(hK9e+uU8i~_~dJuDPCK2y-wCXK5;M8RKGZG_$a@m;Ze|3`o zKQV5ycJ{$192!1`Gp~{Mv&tOnktoI{{Z5xZ#rg+eDHhm+8UMX6P7Mb&2Ycmrzo*bM zR=YAP%UZsXe{%TzFP8e7jHIp>o|#6Ugl3>T!&Q0a{_P*WntE)~{YW$gW2>+t; z($!vQXEFWf!*>5unD9To|9?B_`y5o$KO8>QC^FRzhC;`|H3)eUIt%$&{)=-1iM1K( zn8$ybHnflZXO7g+pE+TFI){4F54$qZZ_800fr>IXRsLy2MvyH`8FWn9) z>%EWxf;GwqXGY-iRY~`X224sAmU1&QW5H_eR}a$~)xhPB51Q%Yf^kHIoFV0dzO&8k z9OtON{0Lry-L6|p*eqReX4?)}B;Q;D(aJy$bTVZiyF^pmI(J-qJ6Fzc!9NO*;463Z z+v10~e`)@BJ21zkkyzCS{0{q_cTdr%}R3lZOb?T%>JrZ2Rf&_vl7Q%E{v*Cn09d(%VvR`oqu`;`vSr87MhRFUI zi=16qu}K$4*m zFWfdi_T@)-o^gqYD_7ieN{3Rv;(W?Vy?|%z2+PfG2rMWs%yJ{75Ly#RZ6lK6)qmnU*6|@lfALssY(iRSf z_?Ax=*+J93H{-7S!ewMK%>8WcEzFJ-2-l9eM9dlrsRe1S3%?;NgN&+dbmCm0woO*w zhOzx-bnBxmxS5`@ab2!zT;t=a9=u))uCJ;^+XxF64ipC&4E{N3G(k>Fv)osSNman! zZSfzPW7|3IjYV{-{EOCa=T!M4WTCmoQv$KsfCx($(v4Lp%8A$z;W73A2}Zbh8T=z( z0I~0j2*N$?q|TlQ;x+P;4DNUiAEk7@ta~>|m^*C}p_|jy{?wdT78JC)Ew514AFG{f zY{ip>aS4T%=%J2^6MBj<7x)(0cYV*z(%qOhLz;Yz?4 z3UI4rF~urXk>tu|$wnH^eZFk@PdSTC+FWdk$h@nek_-|yB72c=XkNmp3e*Mh2w2S$ zLxVQRzgQN;4(B_Ty2}23>=%YDLkB^2k4gyyFbVE%hHMl+;>UGwNWtvC^6Gy9fJ9*l zojox-`ubg)1qvBoxi<5|XfLEl6R0{eA;aMrf&hciJpYsHe};v~NG%G$<&b+DcC_cI|XdpLO z?!vvsy=&5whWo2JYfalzktI%ukPX=GbV`S<2rV?XISo8-Q?l@Q2?l5Ab=X;QIA}Hl z27YDj=lZPfY0U>~Xqs<*TBf@sl3$d7VKrhH$MuB|uOZ(qq^q`;5a7u4!7rKa0x8$s zk_8b_2%jm%q-%`Y=R#3^Yx|EEe_mhptR`?HAYK%bsli(m>^>&XMtFnIKAMw%S|QD; zVA!!{*fDP9XhMv@Y^Emp{6*bad%P`VSwIOO9mc2e{XBBcrVJGG;RaP`|CU8*%sIxm zizDt(%*hiZ`~_Y8J#K2XK6*vP)r_aMSz9B!6?k=o;#P56iLMN^Cn&4$xxF{t%#R-G z-Z$(fR?{=6YrQgd6n%?`nws}G7j@Q4wm36I$E1Njr)VkQ`auH0WhcMa*Q?lzdNPay z+tjz}fa%kxlYtzT*Tf5&v;ISw9N@`-(^1}ff!X9Yx_=W@RXu=z8>fnMn!qV-O0Hr& zLIs_};N#6z`o!AM@KWocRD9hut13T1(e)JWZw=oJAg@feU=mhaKkx{JTLp#>KNAT0 z$|f&=D}~V1*4H?LXQKG-P(8{T{7uk8V2HM=)~r}1!;o6hj%_|GrZhO=9*Gq^m$6eX-} zo37t3ycXkd7mKJ+>1KmXr#Ft9lCBcmkgSA~zX2M?H5V;5Zj9rS$7{>#S8Z#UbRUY= z*OJTS%%X@+1=q||_S)ZQUhQ+rW2e#B>~Dt*`Z^xWuljy0mKmZj{l;SM0yAA?+;A&u zC{0n~mwL|i)XiX3Q|Bi&ka#}?2I~Y4wa&0|EE?Mq$Nm}vz=K#vOJlrMA#Zy?ssmY@ zv*Jg7Ovq^y3_wV5gSRpNoaP?_1O0PE6sVigfryX($8U57Rf0P{$By+SA8KxZs6OLN0O{Vu*Dk!SNKU^9 zqJ<55AOd>2=-|d~`>>L!rZ)Sw{2*b9g{-B5l*5c7s$mzBT)IvJkl%aotUGtfiJT3sYvY>XuNVYTS-2v0NZW&MqF!o^|*oJFPwIj-CTuPHu|? zGUuwP?4DpegP-zvkOZRQ3V%?hfpZ?h)%LD> zcOU{ms$xUccpb*Z;E66%sX!jj)wba^H%CFTHx@q)6;mD!e{l(!?%Uc8bFxfK*eDYm z?zkW;87Iv@u}>L|8`Kf+1DDr?vmGPalUFQ5s55!EySi`5FQw_$nzzT#x=SO1;VD1G zSiPt0qWJi_TKwMqr1vqekxY$vu@zQ6q z>W_);%Er{S*p1KT4FQBjw`m*fP~4ZEW89S0m@ak?@vyhb^*&*Ake`leAW9jo5P|&OYPcc#O}@Q4 za3&Ob{Uh(#0HggXn24NX!aH%*G za>Ve~(CjmIEXaFE!LIt1$4Vaqt7A4YDAa>JTJ4-gqvclk5h=^AGw=y7vFZyDu>?Nj zE+$U~y{&CQ_b^7d0a=vMzF^s_($er6QmD9U^z+liQ2BUeU7|=(it`%6bT7OIKI?6* z#aXMG-#kN44i%pW@7?AhWpXRo$f5*mjBCnhTDAP=DeP>$-}0pr%~J}Myx{!KW+$)= zsLh+jS5QMg#0%0U0Q5X+rTBhmqo-i=yu} z=vJt)W^Q?{=Rm$It+c}zsN}xvH`VMsS4%jGH9Xz_9!Fw*n`odX!}`o|IkVL_gZ!ee zk&4t4g{5^_#NV=wSm@3nI3AjQeS<-eJo41^{ z_jsPfEL0xtWaNO=isvK4TMYvNQr@h>+aE@AX4Bapml%7qnb*7Q_3s}R%;Ov~hmTHJ zkv?yC+`i9|_Zf)X?seC_xOQD@8$bfU74+ZVtg83Kf04cT5ymH?BGT9NoJhT0cBYv^>bTy~yC}M5i`Xf^Le90)#<~j*1oo+wvlDmMo6M+M z4rV>Yv_?yIV}_!nkzcZ1``I;gN`4W|YVR9MY#X~IhtiuTTBc>Q2XlSWly8<~{nGI9 zWC-#t1lGW#i!^dBzgSiJ)3USCE+o*agQBq*J4-{#~hvIs#VHba<+N*CeqGrAYZ*#@P>PRW4OO=|`K}&iVr{z{e{uQd7Ch^Da ziFIT|po8|DQlh?b%rB0+;B(zo>7Zd>&w>oR3f9jx5*@GZNUf@%bwYyD4`mWIq;wQ) zp`-)-cB+!}Ya!k(?iK4Y@W#PfuOc95X84@~eWrnfI%7_$!Eus??oE_Pep%L{v1_yQ zBw<^Wa1Id3205=kX17NuU-L=;9Lr@#5Ra^XkhGK9 zh}oFg`QfzFPq6)ab80xO4Ov(f0CPg5oYS4Z zWUT6;_ndU+l!aHP5miDZ z*Qvu-to9$QwXQ8A;yn8~9oMIWt>!Y7u99o^5#hI+mlN`WL=@V+A#FDv$V^oJqG#q<^|K8U&tynTu5T z9D$rimJW@k5~q4n3IE>bLnqN@!ZQ~R%LnDf<$}tfJ1(NHI|i}SF&X(@9Rq7y%QZpC z@R)w{)ECACu4m)&qTUyCpv%2yjTeA(e&^|}1o0y!-;I1%%wkmOFi9!-9Q_Z0{rH{$ zFK}EearUGImgcX(ViOtL>82Y!ow>^%N3XrKs@kOR)}aqd zbIs#NT1F^XL)W`Z=^pwvPs2%0MOJQs-jQ@j^lI z?&A(7$5Y;16+y^x*WEH6+;Um@oG40%xHbZ(0Oti6H-~$Y@18IoHa4@pZAsLxmEy@X z85D3P)+tS_$pOH#CWEwu&Pav4`F86`uNK4>zc{$`5EGm}y86sW+mmBu@1Xz-vAE5G zY5Iw#qLTG17}sPQGw5>;)FE$bR53=3bs#h{&Q&WiZHrJ|Yc!l9uJhiQ0wAQ z`X9tcrlygNkn0+7GMVw2kn3`oGPM!$LVZyB`4DjCLT!!0NW$d2>SDsgnaA-_3111~ zUzBHh!s3P6s^N{Z!l3l?GokFG!lCrDbM(jd$@!?^m~+#^QiqIdQ-+>sV-nK#yT%X; zL{2xNvIELu&}hV9V{mB90ov0Zuan<&x}_VMy7g7t(x)gP?7aXK_?*TM6r2!R&_0$a zmpFgIH>J2N-*42@A6sbwf~^24^cF?VZb=?b(x^SG&>fvn&!Zz4!S{ zAM!v?^~obgl@j?e#hqZG=IR_l%5U!m-Ira^z#D!Kos3ObRR`SW9JV}+GQdTLL*?<< z<{a{RU`3ubBL69naLF8Iie5;jH}~zt_Gibc@@ua(I%4(;`nr|h=SdDbhZQ2q3(l8B zXLlo!o1B4c-7|)2MevZwK;T0Er&agC0Q5^pciI z_3q{Q^c?TkKD5#h@Q1dVo~<_0r*D^c_DLt0n1Ax;pLUE9bz zyD?DFEPBOYr7S|+IQM#UtJ$Yx{TMgoS-B*gA9248(rk$v+LK1ZV{DL^&rvo@Tui)O zcjipI-n6V{b!$v_`xF{#oDo^+dT?G_a` z((?1B-YxC#lkW>kb2uVqie~c3qb~A8xd_7s>u2Bmq9P|!7ptwX`?1OU@q54DvEEK2 z+26hVoj4ZSEhaq?n_$+@lI10Q+3}N+c_sO8IinWYQ4r#SRHj>kyr+(moNWx8&ImcU z`4uwI)s8Yq=kzzaCd!>XUwhQeN?)BQ8L;h-x;ekIY&w%DY-BGa*fXf24}QC&;C4;U-HHE-G&@sys*e5LYyizwDyNs-@klb$#u-Ye;a z{<;;`uiw|!VOS!nqg*}WN@+|kJVzMrBK^8wF1}uCkj=Q=OmpFVM*!J0F$}y`Q=BAs zZ3G&SrE8v8y!oy8czz02^_6ii2SAZaW4oO4ccp zT+w2T6E%R8{yZ%;kC89?t8)yJl|`G1^b24c29CpSQQw)I(^p^~i@IE_Z81S1;U?t_ zndB8sm4GAkzW6Vp5WX(>xBHfUz~!pmgQa(CggxN%L{dYJA1qgD#TXb;$y)89`!Qq_ z6E6V!W(avb#x5tHj`yQ-%kvKQ=ujVLIwii_Vdh;0)u(#lc*${-7r-pR>ybtbNdw4- zx;AGFyQzS^Qsu~7m*3f+hT2^nREV7zMdw4@N+;I@MDg0?ts4#oRB6?|zj3@Y_wcL! z#N%0NY}1+O(#FjzB9>gxV|XCtsT+A7zUzujj_hbRU2*qb*wSk_2!14$N!xJNNbGr! z>?1@{AM?gvS0uPM>l4#WbdvKzc}tlPDh=!mT|qJIBFKcU)$!y~T=`wKps;Y898CF- zcusSQesYJsFUWpw!fOP4Sp@r!np`KV&Pjn0)K@569HP@ zJN)z{1!|ep{GToYzFDi@uSWzjIMG$VZzIC)W=urSYt(oyFZJ-}@NKfyq(CSQhuhDD z4W|_ugJ2Bkr@c$oG9vdp&iovOlP9d}1m-D_ff2J)c;agv%LOIGX);6B_z%G+kuNphjac(~z&G{|o-L$}B>dG{i5 z&262LDYgGnphx9)(#cH&DXYev#oGP2#N{T}lUn{AA{1B|TNiKi zC`RUtLLb$tg_D;4J&V{bhBBB?cxV>Xf*N8nnNQ%;UKSZ z4s`ghCX$RN-C<{Q`eZzUk!!7(10dW<` z*@&Uwq2gY6l{UY7PWgt;JO1iaWlXaWbt0}r(#+x$3nQ&>DOF$nFvIvkbhtb+{PBsV z8`c*9Y;&B|3xG7q>G#0O%JsP_LV!KJ&G%7RhBRy*IPys@Z|<-_`hHG{;%u`g6JR}V z+GW=BLSF3)0Jp%^vxj=tdH_QRiCT9swmQ48mPi&fgCl@r5KY-EQSIVs4%-DkoU&9x zHj#@?;o~=bIo>-Rq2fM>M-k!=HwZ`S5~pO|eS-4>u>G-sWeYA{fOo5vJFNNX*B+xP zT=V+Fmruyf-wXz=%j`%MA$lLCsG{!ugpTFf@q* z+okiy{Wyxt$3un*xu1y$@I#_A@pZBXNvHFWbqOK3b5oE8cY4Kvv2#|Mduf0vsTRzq z6lQY*{K3J22&ltUDDThPkN8aw2X|yDFi7H(0e1$@xK6m+}e6= z1?@5-Q}M*R$KQkp$LHX#%n=EUm(B?;37a3E1$&L`ny(*udUB)UP+t%ysxVmOH>&IlQ zvvRu!S2f9M0q?RmOJIsoHj}>A>4LcL}ihBaikA?!4NnUXa%_LlBc!AVKHuDX2#24?)AMwQ04y~ikvhq zq@yi{<^I5m!Lp7jX7VA9M(L~v86&A6?k(l>#ZaZnT=XkWqN+R;DUBm5@a}^p{Vpnc}+!}ftpa#LD zzd77Jk$zMGWnl3K@tM13h&reL2+xg1Z>E~D@Yec8AdesU_X z5b4kos_H2aHrw1(a;Qoo+~kc$V0Z(>g&<*4hg|9z;abRDe#N0AOYHXX|8CZLHMK0B8!K41k#qnEpS65t~7p&`ymhKn=uYl=Sq;@y$ zPbApW<+-)ulWix?UvcW)YF)9qz?`;BD0Z#{RTOrjoX$_< zFtPhsrm4l&*PN}0%NQO&GzxB+63R4tCjrj-HpU#+rT3=FXU)rxjfhodL!Edf{=%Hu zgZ@4~flCE7gov&C7maJ&9)?>eRyP#d zkC$W5%x`P@+OC-};B&#XSEM;>-rvePi{IhRv<(S(f^Lgf9_6{m_bsMdwhXLr4+8Oa zNDz?=vOZU_`NX?^@4~nxJ`ti4Je?o!{*Jpuy@N5hpC`u!=YfU>I*|@R?CRGiInKCB z=bdA3Fxj6^*&L|qK$rK<8H;<+$}3)Fo;_NAB&n8Kr9{`e*d=U43@DVkX(z%o+*^CI zYhDuPW zJ4hPWS5p|oP>b|i{{?^!MsNQXQ-EuD+5(3Z*?WIS-_kPRt}4V%)ZV`}NQ{<^$rCj% zj!W2Ig-}+VObyW~%PB2qiz#1?irv{G%o}{*`%0zYbXQ~D%DL*Shlm*DYyoeGqpU-+gr?$GG*P{0~73f4|+?{nz_o>R0DVvtTu;IlU_$T}eK4zCRiSJCg zIF(eRH59(43RYR4O>iMa?xRtJUSUTxdiDIwJW%D|UL6uU6rm&8Z(dzvCb&A(Eg4*R zE4UlZj>~_bworqGNZwCpfPC6?Ht_jft9yCkvt|ukjUq-tt>lSY5l1$c*P=?3f^d*1 zHHGes9@{&cUS(rWNdBLu%6syHtL>r*g)km0 zb5~%c0$iianExSAIF@6`fbVO3u3@D)Lp@I+pQh(#e^CeX#iQFbhMU@RZUF0AZExAN z4_*ilmP_5|BFF?~B*$I#Dq*IhT!K(NLhGgOr|c#+{>YK>yy5UDdL50aPYRA+M}^tl z2fBr2=2|4;hcQdBg(^8uSsV;~9S^LPANTee=}}8=)&>o+c%jdjWCP z9=}Uct$mUU*4Iz)DbJ5!NGmXGF`ciHVqO5BF?{{;Gi;ue9499fu-Ax_YmUUttP5B}INEQ)EIn zJFVEQ`Ofy;w)6cpp8NYVA7@vQ=60vSTeY?<@>bHCmV;TkoO2s@5w^X8CbE`GVYhy@ zTp&d77D#x>KUXnQjEV@%X#8E!QEuq4z*g#@)-rn(*!LyYs7{RRJ>_kad4}+SB8IJ$ zHm1f(`kBqi%A|b6outW@hpemz`nXMQkmNhEz^-rnfqaiST2=T5aR5FH(2m zGghMQS%!I|V;f2hiP^a`&q`aR72iIuTD?g=Z7Ddr5Yab(rsro~ONsSit@U>xzbELI zCDRyGCk1sB)M5H%Z zpdVR)ZO9quFKc+Hq`wDZ!K;-E_KQQYkAsUzuB@AiT? zudRx1Gm+fq$n(MoKp!xTCsfv^dY8^}3XdYDh9JIw1i#McB5adhBddD8`aW^=%Sh;3 zvAzHm4o*w`h_~p+Xg__Yc`YUNH+7$n10z1lM38gvqTG=(0Q!QO2Z%1!jLe3*8uDMQC>gH&eEYK^pY8q8$@AHSE^5&G2Luu^Py^Sh=THC|i7gIZHK8w!b z`}G-sH&RVpW8<5TolqZxmq)2MBz+~42}_}dYd2oOZfPs%5o!*fbsuI%L`gmm2g1R0 z82rR6>N{mWkkicQhQSyzSw4Rkv#_YLN~VsE;X|)U^(Mh+s*P@KL<^ph7A14GQ54l> zL58HNy@sZ&gkm>?V>Cj*8JUX!r|{ZCzOuH2Pr0hPYSgzMVUGNvX|tBVVIuvpBuQh# zTIg{GP#}0_vMh{=>s46Whv#H386a@}f!hK3g2)Z)!ng9WX%8nd3UIvFuD|EO(NBJ8 z6OQHhq`$YM!cxvZ28Mo_gDA^}y3HS6DjI>ku=ugiP9E*P@v@`|AY64PNH z&9Tw;5a%t}@QGeKEwVWf~RuO*=V>e-q|5+(}-&ZxmWO%zkWqli-hekEvG|44H}{_*gca|U1h zAfmJI-7d!m%LvgU9~Z^epFPPgUBmm%ZQ(KWG9em6%FXs$#ZIwm(WqIu2qI}1?J>ep zm?=v~4yDZL4SH1_ry-jo&1UeT@EJk$j^lkB*lA5IrarFNHT(H?OB9at>f<7=hw={B zm@xf|#o3kRu;R_y6P;XMHy69+4yC7lm!kr~;d8VaYVMn*Q~rJ__Q|#WGAD9GcOgqV zbIw>UE%@o{Z8tj@h;-OLKgip?1v~BYNk{NujFF1Ixw&=SDVI1jZv%}8b#5SK)l^df za99=1w2CBJfxbRxtu{cclR2ob!#NhMs7ymICn!a*P_4Mpi zQR>*ME3x9rgp6inBBa*NS4y8)GU$2bIGqz5#A3-vWD^kEy;y+7=?~z&A_Jc^r zDFmUNkAw1i@~zOt_LAHS;4Q>3y(v!=pJmhe%A9?w^PNK8s&JmHRSOcz4NLwfk6kho zI5{uO5;k?1txJxg!%bp}vB=alLpDtVYtntUGfP(@QS&mGwPIX;tJGpogmsx1*X^Iwg`<7q#OcJmYh!iqE@j* zzu9V+zT$-J8C5YWl(~p!-4dwMUXZMh9+f459OsY|_!*frUV#WLj1|C(WfNVA(PL9n z-=FJL_|Dv3s9KPBLfoK&cXL{vZu^PItya9y zBAuY&8=iFvOZR-UdPwzlLRJUHQ)!36tN10!}HntF|imH64bOC8)_x}2o} zuqBu%y<5!!W9~U~*^M6cLG;P|;cqrY zc1QdA;rrdx2vpGwba!s*IC%3Ic#U*5h485QgZ`IDC@zD(7i&s>Ubt}us zHYM@-_K9m??o;wDMwSZHJT|Ak^l@CP#CJ#6<*aG2n-M+NaM^Cnr)7JPl10s;YE->E zD(ee7e8p-zxBgzm?*RI|Fv@+yGn1Q%JH&7|0J7LRV^|H&ThBbr zCg-sw%Tr)8*&B(J0uN=+Q)NnY%=IabGoBLRi(H{A<{pN;`kb*<&;Kj5HcQDJs69-b zL6Rp;Ks`w>BbjxlmVqMSYkJ0qdoVY>)t*+N*v!#P4E){a4>5%9JdrOd1jTSHy;I+I3C4 zXQ+ela+KTPx8)Eirx9Lzp+vmKHx>ZG){xwDsn&I$gas!q1_0-19R{L7spcz1O<-P5 zwl4Tpn_F>m^{ZvP+UB<5IGW)w>U`V4M!(?MdlngZUAJ(M4h%S5-Vh`QoecQ}R+fr1 zM>9MC^DQ?9ghy|bDc)gK(q`v~lrpx?B3m-R8~ie2cgDSFjzV~M6zrvSjXPIL{3YFp zM%x|pG#;r|r_X6>Z4m9-Yg=xBp6rq7RP#D~y`@-XnCBAa2p}GOr+@CAwKy@GbBdvcbi+UmgdeFEyLwSJH!|UR^e+$DuEwZE_%hn+)gsu4tn%0SE*k!rt_@HVz6A!n;V)d9@)mN& zEL+8^THkU?4eqtCGmE)SCBf6#-BXzG$!RS(U=uvV%81t0qmVE<#CwWSbz)5>`VH9O zAmJQQj%khstZ^iY2zQxIy1?5{HGtP1fVn)q!frLfPkRb$*Wf)hpBKoK>lvoV*qvr% zN5L=tBP@QqG*)%fgwv7tE?7p@wnLgE(Q!$|d&U1{k{7@u=&SWY70Ec^YEsf{&LpfKNGCaDM zO|B=QyoSunfU#(YTMiucJWig-MNk=Wxm>-yp^Zf=^@;Z>R_?ao#M4g5s^@!7DQ#}B zzXB_twT8Mz1Ja3cztj(87BowQ#d|+R9^Bdz>f$Ci#Wnb?ZLn=W3B%dguVHRoxns1^ z-hL;xERjht&_r_(HAA>AGTzvhy}v61wT)tsId4a_&}@b{;sBG(d1|b}(!py`B>YdE z7CiO#@HZ{7EO&InwV53sqPv_L-JQPZ270~vm4)D2AVDFz@S+{9BO^7wuZQ$s14ujCV~;M@J->>-R3*7flR^u92pzyH5zCrVj#{%)6{$ z#0Vr1LE<&{saXqmFI}`4TOhQ{oEj)V=c(ML=Z&JvZ!QmufV#DU2%2>yXI{kO_V+;0 z>HL*N=8uR)-HRFQyRPOyxBjHKlk&D8O+DyYJh*k&G~H`oZlG!$T5182?kEqvcL;T- zVH+m4m`v*+YwT6Y&e#$h*aoC?)7zqq6a{!42^ied1)=5JM=-}^%=?j=(lah|O$YcG?Dwi|Y7f?*DMFJa-^52u zDHVrvhCAu*Cw6exlH0f*&gjwbIDr8CNC;r*1N2qIL!HirF&DT?K-NoObD@N1 z+s6?}?sZm2x2aG7<_mzt2v%#{SluXQh#GW3y_9S2F@$$*?AsIbG@kcdBU`fDdv5P^ zkXp#rk=lCHmOMBU&R}kud(i<05#+;%-RDiovYeX+)|V4Xkq4JmD4U0Upu`s9_rh#> z!g*A7(C}Kq(&;}`DInLxm^6bUiW?|HqK8f3B7z7KBuuyA>?m*Vm>jq0kQFX~(afDI z)_ls?+%X!jyd2;h60o?Eg7@=B_K$EifI&$f~>yG^wEkd2yAa zMaxMsOk786fgVHHkr+!(F20(1b|c!kLU;S4^-$t-%T1wPoc;;^*Oy99WzqnH!`u{| zt|gmBZ3J6H=zP1W$tE>BznTPB4CtN*0y|WOgB20E43^3BBr-;4y=A~|_oNeNcd;vi zO%p5}Q%;7R5i-ErVG*hJ1voZb38hgjI5E7mA|cMJ?Tl@d8-~GX`bT9KL5G zG2;+B!pF8TpGxF)RT#CP6C9HfCgs6)+t?;t($UvMgZM1V9vtS3EvYz#BdirAw;mqY zG{wVuQ@e2Bc>WXaO*aCgCbkKN>B4)*M1@xjR>Y-%F+k%^lxFx_t9n~ctt#2fw)3#Vfr+ep2}E_9M9u+SR&jYQ#R(`jAYIwE ztPo8J+=F6E#yWSK8|saF0KE1A7*78HI*K0rY=JQ0$gYlF2VL5}qtFyg)ok1o$kkX~ zJP>cYX>7N%-nf?AJk^7-0NarUvao+irVI`cryEAKQ1eVIwZ@<>g)5|VMsjTt?>`$N zcr3|~+stPwnf;&92J;1`gELfIpMLWI{x9~GtNpC#lfm~hro!#zw~LH!ZhjkGvW_ zs2$U%Bif^ko9*5WFi8>wnA43kQr}`NV~BNe2tgHSw+O;HyMR1_gQ{u84XDIKaU!&+=>w5M zV{L;*i5%ql()r4Pm>>bOYvse17R32cUF~K!U29IvkLU&fakvPA1!EBBs=Vv3w;=TP z`v}l(Kt1WNIzr8pV-^;kvQ2jyQA@fmv%d1cPt4dYdpD7U0V+NtUg+aI>aS`mA|*A= zXVhz=K56#@2=?=D`EA=~u>c=oP(T#~OlcDWRu1oU4sv|z9B-`6!^B-c zf%7S0zL$Hj5IyJKxo2F6*IDXPEi)Zn@oXNlM_^dEcB3a2j)ob~wwt>Wb-P(c>T!0O z9;i}35oAy}4wL@?l}1)cC(D9uV`T5u1MwEUFMWchzI7dy2X(ZuyA?h2Iv7hD_CdG5NN zQl}B+{FN_R*|Zs%=~^af8B-(c&q#uZAA`CpT6NMo5Ls#OoFZdXdlrq?Y=>wkaIY@a zc6KS?T;T>=$;N}QsIYc~3Jni$>k3t%*2~PmYlmy9JnF4;Um)_tLEl0>so7Xa8wqZ_ zjCS()K74~hqNsdl^1V`I~Z9wsG*CK$aMaSMYjTZtjRBg?ol&9w$qAGQFBc%wS)=p z3T{p=njBp-nFeU;@RXt9?E^N#G#HQlr3Kk?KA|34)U2JdE(1K5Lu#4Q=NA@rAc7FD z95${RQKa+*9a7__+xrOkE>3oGUMDL2M4A;k$MzS*@*MQl6AO3+fuJ?O3OskWuA4_g z)mtdg>_McNI9vYEtgmI5nDHMa83$T(m1587haG04txGa%5*sayV30>M9Q?}7#2n}> z<)?HG#bIRJx*Esf25HE1*Hui-gJ$KnmrTJiN=yxV7M6FZM!?;|HG(lqhZF5oYl$+( z;qK@KdsMTQwP@nJ4wMu#8qZ2j7o5WFoUk2Fl_g!S+X!;U4&EUHF%n&Mw)vS8{MyvNiuZXDreQ_3@$e*9IM4+!!FJnZ)6f3K-@&r>Ys_?l%%vr%pItT zN9wb8eg6Q#4qJNmmBkHv%bM%3aFN~u27-8_0$-P5aBIn(cgfP7h(V)hURoN^C;)(U zRB&^N??61abgcV-LT|V5W5LVZw$`w0o!)q)1Y4Hbeb(IKS}ph3(Lm^qAkknl*Ednm zkt(l;v8*!CGy~3)V8!}oxP-scuDj9)w$! zn$*PmAqWd}jAxP1q%4TJd!3;DR$_6O=9{pz(p8I_f;H?#Ajpcj&H(pKMiIQA3yy40 zAqq^05=ST-)M6PXWRy^3MC`83=M~cp(mT`eO0B(QK0~x0#r_n6(qI51nN8Oy>JcJ~ zHyb(S3`PCEUs(oZ&J+4A{8gG(e`n9^T+3gF4Dwdm%+VXZ@X12?%-94~(j%2RB~KL? zjE{$#S!T}h9JnPDYN$(j!yt_5iMx%EQB$cjetJ@QdV)8k*! zK!KCOdL|a=BVT7+tRk}>WJh&L?2m~+6cgD>ALZv&ZD}1`RvvSwJ_~h-r8q}k^Z}&L z0#-MWq2*eRf_?#CP(eKS6@*Xe9@sz(>!(oB478ITUhg_6B;uMzuwr^Bhph6&!_Gw$r?IJzP<2T&A0fU`ESWEx7rlj#cKMj(%4> zbwP;fOD-k3i63@7JUVxO2#VVa<)6!0NAc0`vCvMqcR>h_xZVe) zM|8VG(xgFfjwj2hWHXn$%__F>16N0u^d!z#bidJ3Tc5-$uo#5 z;KNx3T-RS`nN;Y2F|J_My?XoBw+=dad{9c(z$h6$R+1xC8b=f>hKQgi&l;*cbAXb& z3?tembtZ|`2$LmM?l@evQU3rbNC-)$nzD(Re(PPlg}2BeI~4%#9>rG!fGH6q;h= zfZ+%08RUjUcJV1+EtGxsi*7Ogs3NBc?#C$6M7xxB+UpxQZ36G&B0%9B(z_!v2lZN7 zdFZ7ZcP@zyFwXb4a+{iC8I#LlXltMsQVii5@8X_v+n8By!I{;e=B8o{pL&%q=b$rP zcGx1|+7m9rx)iLgR9){)8esCCwA+x(I{CKf7-~yR0$y#l*kP*tZ?Q8d?gS-07>4e zMCk|1nnE$Ag~DKPm0nWfhG*_@*!xM*1UOi z!l)eIKy(DS`-i<-y5aK?)PGcf(=CCBN!R7~#H=Uk;`r0`SZFi^aa9iM-cSL>Ig|V{ zXsAPD5d)!6OSC;AdM8DWsE%V)0U^X2Z3Jj3(4yeEWVjCTqBXjZjO(ShoHY;=0xj1# z3|znnnQ}1`%(u*ee3mpb(g4qwV!jr%0u$r|b}Luia**O%zLk`joFe7+gZ1FfwA0v? zuD!0XDOhZL5_T6k_MBSc(B}865D&prUy*T^Qd@<*i}63c*9;@NBt?0c_axi6ZeH$; zEhN`*6aM8Z#E80Wavl9YWDrDGjQe4oZM&V%t=^6cEn1v!;RO6;KCw9RDQZi_(*Oz| za+1gY092s4q0+{{m?%tb%P#K?ML2aOB%iU*si)6@b~v2FtIVU5E+B6AC@pvON#0WC zOTxTIJ&N_5yY5?L%cv)FWcSYI%9g&yQ;*n>RA6BS=poohEh2;w1Rew}CIW*X*p;x8 zSwUfx7hEiSMJclrpfYN#JK9wM(Mroz=d;=s0fZDcB}yn+sB1@ROK8Z5*1ludqGCq0>Z@Q*Lb>vl zp~|iz6bJ?qs|d0akeJC0`A1+@O*JA0D*EwK9OeTC0-A7(M8s2GX($nLWKeNYv;_-y z8?&LEed;?M6Es9%acwZi6&C5FH(uJNfV-r+W}E7!I5ofq>&^~}4Yt$2SX#@3xZx0+ z10Lerfs@Q{HI6D)BWQ4*HBsEq2zl0gv|3&sWTXXJ;OCLo@mCHFo!x4IzKDce0D&jW zfHGPHfhV$1ZihTK+vi7axM|2KE?@(d0sw)6X{6KGq&bGr)K}PmT`iTg@0O?8)~4my{jLW(S_Xhqaxt=4()l&gAbT|HPSwX^JcK0V zCc;@QwR-4|bUws8R*SD*bbQj^_NzO%;_`J?wR$birxf)*P$V)HD9n}Mer`GQRTl|W^SHbxtB8uyB%@b>65QvNyz zZLG2|jAnH-spF)hy9jS1yPrg@BnIoyJc_7*BWOAbiy^jXGy%fAq|d=w_<`>U?I+k1 zYlKmg<+(8-)nK14)P!YQB*c5Ys-!u>K?jGIooqZAaOo5Z;pL)!EfncY3?DCg3EUFfQicHGKOPHVcFpZxXE8E|CH_%6EIyHF7{{ z4w(W+5zRMNIc|?MthPuZF;8e$(39R_=LUVr01zYUfxc_!L?X2E=~R16^X;m*c0Zaa z&kTG}1t8lX@hGnE96?g-LtzUU45DZZyWhGe(etcxvB2Q8{J`!}%?yB;5}qR1<|#$C z-TwflxFgf;(QBh*+*&~QAzN^4MxI<#7@X0HXQ@Xvc4U&%5g^tmWF>sV3!t!RGo^G@ zyDrONH89zYc`Hr(^cM*@i5A)(=i|`H>UWuI9MNz$>b9Q5FOb$mERfQ8pV4e~rk)|M zTEy~Mq)8i(1$%f$Xwh6a;StuUZqar+)HspCsTP<+bG#1v@7Xeu+W$Z|634GWOqc>3D{;Nzm7D9Lb>6rN0XLIh^q)3)jKR z+NXKnJLAxA{%2E|mF!+0;n`mfvMt!Q8|ggN?)~R^0JU0qwwO;+F~J;1gMACLYYp!Z zi_h0!5A@o9h>x1QzFyVoEMsl8t!rA)AT$900iXktvanqVByu@92)-xanI9By^{s(& zAYEVr;Nd1a!*P+pc|VDL8_)Q5CxQ5j0=rfZmOVC9x3o)luXKfW?^7)`AzMCSX~9F( zg8I6Av|iuiKSXj~CpRpA=`V6EAJTX4GMf5Liqy*T9})165%7n`YSF_m+pX_fcVox7 z(Cy2S>9~%QR*1k+Hr#MoiUJk42@5tzp|l*q>Qvf+qq;&1%@lA?B{WGe;ddZbw%KE` zG{U1V;nj^jKov2!?4Cd!@TU+i3%SR|M9haeD2n1W!tSp!qb>~`tNbF;T4qbtYJtdrt0^Y(ZuO zM&_7XZLV;aVG%scQaN?4y=a)i>=Djk8#d9jVb(P1TW;H(^8wb4l*9_dOBfnoj;6ju zsBg3ju(j0UEAC2H7(;*@HQ~^x%MKms1SfpJhA~?hK_{G)O9os{M0(0oe9+akED!CB)M%@Q;m(*d=N6eNw{$mm!= zs2*KPqQ5hq-pVS6H2k#v`1mFRY|a>--4SbwKkN}6ed}IBSPVxh4{D=a46+)!R1>I_;*o?8bdgguz0HjYIpJJpnu9i#v-4Ru7tIm$Y)Dm zGy#vS^`PO>RHg#v$)^5&3XUxr3`dD<6wnc$G{YyURl*?ZemxXL5A=hF?&Ry+sMO<-nd_s7EZUg0aAG=~^af8P{+NZ~2-L82luJ7dPf3Py)Wl;|6Hv^c8RJ zbthtnhFHCt%{MJ#9dhG|0FcoUB$K!TSF?Cm(LA0*`q_**tM$}>Nrt~8PN(>t+5lc* z^gz##rVeuH>v-%w=D-vcZYW zfA)+MT>xu<>N|?_n=WvBTfVE6!iJ%7Y*E7q5~b*&^H4!9(~JtMmpAKC37;rbxeX#w zLZV1-qUw2YRujiY_D}kdcTopuLL*+LfCd6NtBP(@ zgMrn9sa5vb$igfDz#@*W^;Zjk{IWM90$ab!rD=N(v`9g)%}K8y7Ks!Tp1zm8>P5oV zfPt|k)E;Fe)E_LKKcYAR6idEUdUHPm>{c#l0s)@b_bZUxAjzK15(2mfGF9ByJ4k`e zKKxL$^dE={q>M4oolmn~v;kQnln2t|(mjZRS~>IxLjkQ2Ly6>R9&~2L6NSW|fb61R zRK1{SU)ugiZaB5&qNJUEZ!S6_I63@-N0&6f0GB=j6-ClKv{G%bHPebpzi<~41o>)& z0GJ<)(Mq<$+RPLZT@Wd~R%3}w16^}wwLHoYxLhHk742XL5tG`hbc0_Ymrxh783rol zn{bqFY*eNNXJ{eBh}@h5 z+-|&dDvdVIxVn);t+uSiJM>ZkSlf(g5Yh?N#a7!fOp&J~B3^g)1j8L}r~r>}CuhG^ zX0|De5QBic;?@%aes1pafFxZ3qFQ5$jNu|dswGVWWDaO>9B5;)GywFBT?JL+N>LHi z=O}KtV-KA$1m_5W`YmGvg(aws7h~$is8<~A9y$WCgCVUnlhCPht&yT(t+_yB_m$mg zqe$sg1W;QVD-zf8Rt;&~RY=3Z$&8Lf14JV3wbsZ2gdCG z00S>Pb=xif0L;DY=)523acewXt|kDrcD+m4u4ef#b(Pho?j=g*$tA(MsE@ww8&7%( zB%zg|N+}F0^`t!SjO20>F5Ly={U&#=bOQtSp?Us$=~iYKV z1GIZ3y*m+R7tXnV63%4r8}@I#uW|1g#2@?N(~GTocZGc{%Vu!e&H7nU?>7XM;9km0k?Rg z0uWV7C zp>E(X3$Xi=h!Qmp>_}?udzqnG&Gb05JOL>l$eiLL+` z#q3mV%dT*w4=yR~8!+1M{%b^s0nDkP63m&Bp=`&^9>p_c=#o@3bP*DS1vcC`qhWq7 z5!9nKgu1-7+^4z13?K>z!-ZDgd3#9L`KXPnf|@|Wolp_118vi|gm)ZuB}OD@4ORkB zz%dQv1oZpq?^N$PfKyK`6>%yXmF9pYHKb}f1gtJ30S`S<`XYi|Y@AK&IEeoM$`wSK zHPi1&fm8i6GSfV(PjL6Ef=rVhZk1PIZ*}gkO4gQ^7R`GPxgZCLkpg=w?^+tk^g7A* zqKht$Rn}E-iKJ7S0IBa7nszEX(HeWz#Yxxpy`0dGcaVU)L{Kk6L;>7Jdx)KqrveG2Vt3mp|6(QG4 zKp1?w5X@*mWy3tw|< zaDSz2hdO_V{H^OVxq04y={|coEo5cbX%^3Xjs@rc04^2C1z;AWxi(u>Wl0LS(;y;f zO>~rG$FN=S-yP3*VZ&|vOI+k@k-+dlYDdz`LyJoB&kOohn#tj{nw-s{0!bTIj??q@JEeM#OT;psA#Z7yxF5y3 zIlZkl9^=U)RMfVMjk*mD7a{b&DYnSlzBS9-1pejD>Hh$`0_aflHva$-mc5sCTXBwW zVLBe8Rh*ThJeg#iPX=|i+<@UV>k1w0i`gZe@x$v6Ka~{fvCu>iV2;H#uMX3paa%`q zCLpqKggfq-o$9wAhV9Sacf6`N;}Qm%tE^5ul|OJp=1M`-Nw8;uQX6-QZ4Xj5NVxv$ z4>e1KIiUrmD!|P{)7C260I_LgXsKbu+fp?PIx=d2=$Jd!KT@g+1S&e*G>|H?%V6R< zscvv+s*AUp0I*g`G500CNgPQjLs2Ri1Dd#jF`dv`TJ(GrS2@K@!lIf=5EN^MA=*9a zZRd9kB*xAlrtHhaxzG^64Uh}KJQR!J*hUbl#<7m>gC3Fg1tu73aj9TOWuip7i(6oG zK60ev2e{DhY6&0^17pjYKoz1pxLOBK=!Eu$kXYlszUT;@z}k#QwgBkW35;$o^T6Hw zkOsA~VmjUBD2He%?h)}!TbD>AH;*!_aFAk3U{TL@fv`8bRjnXYyAM?SP}<^!GZn|G) z&}$mzwo9*Ew%?iU5D7KWDIW*%`Tqdz2S5J+bC4td0PL!L93*BuKt{eBd>`-f<#Elw ze+z%Fe8>d<09NPtmHJXxbpHTQtF)^M;&yHDpC4q#TVMYGQ;`1v`6&-IW5q0H9cty9 z_}bRF!q{$N1EwV9&BlXG2?ugW+uzr3fBV~?;$2_T=Q`hxUl!cv3?htko1v%5c`8ac z;j_0EQdq5+i0*HQJN+E@_ZR;FEl$fe`&s=2?Kq3?apY=82ZkI7UUPq@v;{{Z6Ve<@eR+~(Y} z8s{_-c5t}2NB;l`NM_~Fy}kY8=}+EOC_*ltevcw@A$}g;mdUJC!@(Tsn#k1f35)r%8X}gvc&3O+Z z>nFAC4HK2l6T2&!{G}1bG?B%^qHT^T=Q;REeC6MOn<2F@g^U4 z?{5D9c>e(LC6`bA_zb4(xQ#B)JS1spG&bty!&wS z+b~o1+BEHNs=W%;>vx#f+k0*`)oYqT2dpog5~fGRGoBp$S$l<`{LDS;ZtZEL?+|yT zMX3vaNG*OF#P4vs7n}Qf{{Y?`kwd!D&x`o2uNbpo;@P);)+W8Wg4b4B?Zjwc*P7-7 zi1~UK9MwqZOcmB*bBHx0lf54+A+83M3h%UAT|wc=ZLJS#hie83o+x*0u!$YglFM+V zrx*v73V}o+5ajVq9oD!Y5J9T7uvN5Dux8ln_o)ds+lxgMmfalavU^TBZDAd%qO{6t zNENQjOTii_LDdyY*^moe*tyUFDy1B)iiw!Yh#XFdMSvAY0SgflgMlD6r^~HJ*%MPv z)mub~iYo$M=q8B)X4;)96L4@KA3wk=7q|!p5Xmxt3v;;;I(*2WIl%*D9@!+Tv7iYW zsRJ2}gvUN$dq|#zu0D%gPcn&Z#o@%LqV>Z`gaNH4YSZP^h~Uxk#Hl5XYXuP6(Fp(s z7&7-d2P8_}lL4(7Mx_ezs-#0F`6>eKYXiBTdX;uMB4R9t}irC(M%bPXYboSo5VXnx!IVt8?~8y=fJRq!q9jT{yW; zE?Oc!>rRVL_J6bNK^B^L{5;F1QP^zbm?IttQJj0^??$*bT6}*KrN8&R2j(v1paLrkf{{Z9sFaD0t_)ycOiHmg4E!6(2uKHqs{N?H9&uZzLnzbYU z0LS=W{S}Y!t}DlUTK@p~i>|uXTI26Ag~j%RsvPl3(s}kG5!$bi?%Fx=Qo8R66ukgc zxBZuRu1Eap@BD0!_XEy~-yY1{oHlZ;op9R2$d9Vw#HxmyjFYmMW_MU4fJOPLp#K1# zU;hA)lK$X%)_oYK#_SUkMaD~!VB$XUlSOu7etvc ziN$C6<^KTq`7iDVom}+pTR`f(DmHij0B8M|S3J^M8~dZ!g5Mv_Gv&3FZ|>T|G$OFz zh5mPc{yt0lf#*Zh^d);NWXf?%-ZA~Ho>ag80EoGN^UMDL@$z5X4?45Fdoy?Evz31g zR*Rxcn8f0U>C|P8+7Tukm%J*CzrPRrF0NDj?*9PXWd8uT9&|zUgEY^U)>Xf|XtroY zaJWvT5YwiefM$HH^5CqYj~l)J03Rj&!1JvC06f3{03Rj&!1JkkPl(R1#p!m>2)+LR zh!O_7kR9XstC5Pi7I?m-hqCn}xwE*<&T066Pli&T>EG9sdAP zi<$obIlcb?A0_?6eCcz;Z@V_j_pU}K-Xy?*I9tfd$*tTa&tRJh>}Iff71-Ko1SzJi zl@4kbKHD=BbKfAJNh2k%5cyf!l9~eV(*1=lMNxwZ)f_Eaj2LKAMu-UGL?9YC(E&&r zD<~csAt_0n5+;a>rCjqvC@9%hLrnlkw(JjSuDq#4t8&PFbTQ>A9W7e<8`!yZ{{Xd{ z={mGF>DfgRCwiWcHxn`G;q;rJFL?3N<~vO?#z|>^|g}*O9$QJUG!qI;_^p z9g{=BT0=^Lq_M=cG1*xHR?h;3a*rr&B^72;hBktSxu%M3m+cm7Z7~_ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000019_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000019_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..e8add791a3011537ba316068b26e5642b8ba47fe GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m#?0X39G&Yn4A3v&NZrH69Z6AE`w Kk}ORG5_$lk3JUQ6 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000020.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000020.jpg new file mode 100644 index 0000000000000000000000000000000000000000..30bcf1b7c6c69eb1c8a99a2e4217deaead7c66a5 GIT binary patch literal 33493 zcmd41WmH^Cw>H|iB)CKH#u|5myL4!zafjd@+=IJ&&>)SwyIXK~Nq|6tI|KscWAFEz z_uXfV`=NW*dXyR_l z$HB(U#>v3}__Gd>0-&O!ze0b7@pr+-!v0Ue#loD;}Z}N5fQ!Ic=;kB zA)%mn^M;ld$jr?BXAsZ{0Kg-_y%fNI7eoXk06ZKr3Mw4ne>S|R0*`=*gp3V-wBh%dQ4ALM*N`+lyTJDEV; z6_`@c-ykI&!n5Nt5ZuU1XIAKj1AvDIARwY5!^6WNBK)NZ96Sy+E`ll^r-X@9;CW*H zEFwNvUmcW2O>!H_`67v%pngv3{hwX{4l0~4d;kI*4FDbofPe$|vj%wm(g8fqi!27X zPhu@X!S{$D5TbG2tgehRpJp<=xbGAzGIwsSUrGuAT5*C9DJlpE8&NT{~x7KhkPEdrDs{ac+kGZ3(EbvxjniRZr7VPZyxf!Hg@LMBBA18g~6O zwva1B*gD@ZK7aQP>V>bnt28b4UzV}`Wtm=0<0;AixKRn0{f}Xq+I#;$Y`)nNja7y#rkS6Xu?REs}R#wC+RWAYHpXb!hHB;vLkd5aYEp35QXZtZgBR+-B3L z@J^92NC6t26OxJqw~2^QB!BwJn8N38J{j!mUoM^z9f;Ock50Y znNZq+haLhf1@~Uj@AzbXX|N`k$QkExYjZXQ(Ejt`!*jonE>Iin=HSZqMtV)cErKc+bwVO<;`Ota1yiK|CKT8-{{h@kEf)l-A)I{Y4|# zYy(`q3QZx*BZwrtaGucMdSQkBKQ%G#4z5kDqO(P7`#4!y55tyBA}pdf^Tv}XGH9D6 zLGT`9-71@K)^g0T*-6Q~8>Zy>Qu;#j=YNn~r|r9W)mJFj&M7_to$R&(2&#a~5ePTJ zf$Z`RzbzdBEdO>y$KRoV)uHzP=gr`|Xvowr9^xis-lOmmZKRLEtn*rSYpA+N8#K6; z+KNv3Y09hFKj!wHjy@Nk{aboqIBv2GY0NsmAx!Vu;JdzYUK@GUgD+um<-wiKS&fJa zmJQ%{^^pJibNz8$#moe%v0v)I`IF7^c**5}#b;RzlJdD+XAy?u!)|cc_TXT7{G-Fk z+o(c)i2~*e%`Nu5V~E}T=^}Jf!kJlXuUntp%>gai0cqk@QyjkcHH!K8_4uRzLH*Xv z^qj|WnE%1$Uy|p659^q%@;i`cx5v(SZuiATk>3LjQR>0?lKm?W*63DgI_)S$r_+qh zx`F*G50@6N=s+taidizs`_J=D5ci^*>_P-(!${u@{!AS$~S3eiGH&YI{)nx1_jn_$=nAvL$7`L;f$! z|3=#5FZBPGEPiNR{PqsJBvZ!zzZ$|X*b~3Lz@M<@X#we+JkmT}ldLU1kkAHr)RHKB zMq(@VQ`Y{b{QThC(&ciU^B>8LU=1d9 z(8^F)-k8ekd7X++MvJ}=iUfNjZ{Ec}^7S<6Oc&Liv0rcc(s%p;Tzxbw_-g$uAN~iR zI{EbDOn82h3Rr>eyTOSQjv4F%XR9$(W>cP0P(&O_UmTPZKmU|D<;NQD<^M80D}T(b z+@(L*{Ze*%bR1vcUo%SIxLV@ocW#kc2lc{&;PXJ7s5~mPvcC=pZI(kTA%s%4#`f~$4_Nw zVOJ4t6GT26jcesls7elek{U99qOgv;wa9m&w=>%@zPKC~tBEN-2T5+MSmFG`(9dm8# zw~IC>$MS~Qbzv)@elv9j{CS;phX38>4{z=iD!A`lbzU{b)Qh)(20JB%1_MA;vilV| z?76LtOj;H8+Vg0VvSLi)^!VO;qyP5Y|AkqvbkmxcWYr7Q3t`FC@$n1)FYLKI?NR%e zhJWXhzsdSnEX3RJC%Z7r@1B25|2Qx%_@w+V7Q8h5m+RX8i_t^gVZW6x4Sr*>9Nmch z%d0QB<6m07OwWJ$m*B-6Vxu0%!VNVG|8DB|_qhKeq1Fq)e*kn}YcH#NG;VE$uG9Wv z%HL*czf2(e$G;rA^Fq~R0%qf!<_+cb zBkK{KygBV|xEC8DMzo_kVT1XiPyVe7E;e`@%H968_uM1aMM+qddi)NX9IWDzgX+eA z0JA#gbVM3T-yH`_pTDSBu7| z6)64un*vNofF1DOchBK-p{)-F%pi3+A*_PqN&VoHGkOY-6~nO9+B6H4vx z+>Xghu<{XHhJstYJBRAJD}QMmi!uJ~!(i%^sU?A=2X{ zNvY8FK6hd-)N-DWMN`NPT!L*%Hi}}@#1_XUGgr6I%D63SBmIvW?86z-?`t4yM=6-w zRek5YB5U^Ms9!S`4UwpYN5{ZzDPS8g6QHfdmA6QqMKaY9lkJtTlYum*tuEu|{_}g4 z`CPnEr=P2N`@!K`8g<6_w$#jmY%xCg1egu=bSD#>I*oyxM!on`b{MuNP z3FFb$t#^6MWj%4wYtgrR>ySRx-KJrK@iKU|Es8WbRe1*<)T8RIn6abkTJ(QZ^Z%&6 z1UF)2VEpF&{#Ax=Y_szLF&M0r zO5B;)xNbe(eqz$t=wE_itAX-pZDc>S4xX`a!pCZ<*A>w{cHWef(Qo7mbo!i&F!llc zq6^yXMX0BVaI@%F6X^ZZYeyr4K-Q%nu}TCD&*T$r(Q%_~bd*THd`ha~vxB>jHiv?2 zCa7#Im9y53whnEqMcTanz)E+AQ=(suAh3bettDk{fR$C>%Six5G*kPBN-YD zTO?x`6O|1`l!7+}%Icd$M<#>^&vT0-dE)m8FxLM7=oi)kVZX>l-29sC^pLH~j`3@> zH?Aw^=mDY>)9=z}-lh8M`5WsifNP_sCp*6_W!gVJzqa1M<(W4k<-OEw5HAp%x~TPZ zuuEr{rQ{=Ir?UZk0^2Bch{t9Tzs(A?Ih{nbZR7j{a1kMqK}&8%6&lWxh&tRk{R4=~ z9}=Byu+02YWF{B8S-Vr#4~{6Zd7rg17=<4dOmD4BxF89y7|4_?X?t-is;@qQgtzB- zwDNPVdAuR>Mmy@Yln)`u3wr_`hoH znGQdXh2Q}=f*PZ!M7>*2me*uE;0)G4?sXp1MyhI9ol{gAxO4!HlS0Ip+dU%^jWTzdqGCC;Th}wpOTfPkYgPR@v+QwETT_4f@^x->X6Y_G==0#`UqH` zrcUe8XA-&P;EyPh{NN{Wybm?4KJ7Ne9o1Ns%f3t=gMX>(=N90!{EtB37e0bCPtKAX z+YEryO5dkX5GcK-e4~EObrqk^7%X?*(9PX%6sNoRP?syX#Cjf`g^9HGW$(%QJ9x? zlNqK#6JEqb=^%L>SxXO)2%$@@f)Q5kF0d1dpmoK)_!TlGS6G~+jvME8s0U8%M!Y>f zF=fUEGOIjC&gi&4jo^mw7B)5>4! zF&e+1MO;++W~rHa-5iNA#N;FbzxtQ1^Q9#Khf~P%Q{rxhK5RTB$7*|Evcl-#bHxPW zCr8&&=LW*LN_%fdHzT=N@{z4OEON#Asy?W(;@fZ*XOi;i5!2Bc6!?8)%Qxtw z99U>9KTJTwQELwNsI1MPuhW+zq^55$7_c;aih*K+;+1>V9}HQuqV!8Y1uu*aa$?XI zr_g6vRzB+D8dW&EvoRauA@CCx;kV9qh>cf|+dA)PDHB*-_ zEhk=5<40l@=vDb6HP6K_M<){*aI$iiCpeXZBYe6eL4i(|xv()wG7ip)Di1ZMu1+9ToIO{w|aaODc(uUXJ@iHHfO@CZe7?foO(C3(Wik#VXFj5&%x^Y zG$6fjwt3~WvdB&E(M1Q+33wA*Sqsrn`|17MZqsup%WY`~+Ufj8`*BHcKU1_@^Ca_S z`pRs(#%

    w?0HhcUI_9++j@)ruEs|a3UpDmIv$}HQfxAx^Q_0w6DOJ1S7ngRj|B( z52xN1y@8$@ct{shMGCq;?L6!(p)csWb7MN6&5X)o*(EL%m+&mVePKH=M4>m9~CSxtSVkte% z?TvdaR-aV%hGq;(HVzl~T;Owf_SsB58I7B}DCjm=L4qU`LU#=2;Ba$y#0Q?#mkQ!X z??`L5tcRVR)Y-Wv`yYicCxLluzS9(%p8}qWHJ_FFXEiVR>G!RLzjkb+;HsM|oRvy5 z-pOx%Gs>M>9k1Z$3!xQ91eo?Sugmr(M9{X^`!ojDzZd+PgCPgo1YR+a81bwwva*bq zMh$sTpfXYQ4VBXBt2Naz1?zq~G}}UYrI?vH?KoP;#|#|QavZwiuhlU^r>kE*%J6y9 zh+#G{*NJ3i7etICFsc)&@$Q3!R_U+K;zJ8sDe_pO#93nK$aEK(r0Smc&zsE=3}R>o zz1ww}k;usbBPr!**r-QO?H90wrxHQD$*=Y#qG;pY<$Fgz-2j~9`^GW96z5~ahlk&~ zA1dG#)-TdDh2+6b5?>F#_Q$_dn9vX?thl`8wP06^LsS6;&?l1xURXLu(qf=}V{~1Z zS{mGgS1fZza<8zCO+*&sQ{o9V$tm$;*b{7X=5{f_8abpRv{9%I)?GMw0Upfl#!kY9<2xwgS*)n!TvZxZlYcKq=V^9@A45NMx=Xlf|6Uxy3SCM zfny;<#!{{F)rWO2YhZe34}BeS=gldb`3UT6F6m&^ex0U}3d=k+k>8O~2_o^9q~eC8=r+ud*p0kulN#S*kUN0KK1TGt zW|7m3JBp>EDcZMnhj!(vZqfGV$NTxX?)F};qoZ9!Ed#fiH+GVXj*{m`5bXiHOd&Pa zgNb#)F@4kB(SxD%oB}I-$wn2Qbi@^W*iU^C)=*YS>mjzw{sjqF3qszS;IE={o!>p9 z9B)DPk8n|>sg6$Fwjpf{HA}8s6dgID#$yig6z<&S?m^i9urcif947#}lH!QT) z;C#FEP%>QQXjqz{Q^lsyk;x!jeT5JTLbw#c>I$jqU z%ts?N@oVP1YAzZx1Qs`Y;v%p_)Yzh*G_LcgpF1d7o#3uU4Q*O^&d*&bDeT&nD+i#n zfmVy&R25Zy{19(2janbFJmuz*6w7?nR5l9v7M5m6jTrE-t>yM^xo%-omxpDOV+zu~ zo0xeSZn`%yE9&U5oO`O(Tlqx{*CV_HLFXmxMy!jtJ#NJ-Bxa}4$k@!^`4~@l-*7(P zL710iX--+aKV!sJy*h>dmtSP50tJOw(UPwfB`!SB;$Tx4>lD9*A^&UOxAp}*#NeWyXZ^VfsPMmUh z=hC^;myLT{EDq-al6+>28J9440vmf+3Z!L|)mG)MM6bRoL!tBY{leU)cE|Jc?+67I z*1MJU4Ehvy&K-=aJH_~lHdO;dq%O)zM6W)9>TfLH@0MxDi=RoyBV7`fV0 zmWChrMtnX7$?X7>B&~1)vtyphT11!92s#MPH~Use-qBSUtDoN-j9jm9NN>C5_I?8l zmB7x_Y?`lj&EI89>y0xc4%`3@$lngmUmaS337m{^86~Koh!?;3HdWR^Q6%)d_g0Ix z=qVY9SOZePFM({6bo$AD?2pNj$YG2qY74uk2R!LT5$ivq@Q-1MZ_Ez}=IK+_lil7)&xmfIUJyXbfr_2jlrDH-@JdF`eFk1QpIN{+_ z?(+jWKVnv|cs)>RB7&&CY{R-w{NFUqi%=Vz$m{_S0nnt}{$vcDxYTJ2HYVFHC$0ok z2}VV<1$qXx%2xjqo}_zoAC4NzbFE&RkE+${=b|&ep2=MJKXov*eLNfjvhBnSXI7}z zHJq&u4!@rKZVUiY+sFqf00N8`axC5Bw0>(F3Od)^GoO>);??9Bsow5L?eU34v%>^s zFF9(sii&CQ@mh3f&@NmIDCgT^QuD1f)m-5fLlcaDf(S~~mc))5_3M;4*Pn5@KaQxf z@!k5TO_z#_(K#N8Y8dV4b~r*fDlvXTjoscd`@NlQ;pxWfvxfT7ajqEI`AxT^W5~O2 z-xX#We$^B8TfQ^XM7qViu60`zElh6)-Cholpc!x63=<6H3R?=c7xl` z7~jQCr5$brzNflz8Qcu!rv1G511M{O9i41082K-H3%|h6>i%q)*3xb?iuF68#pTib z+o+ro9CBWefM^$f^0s1BL3{7?)?44S_dgd2)1FvF?;+wp;LHcDNhM*XS zM~+=D@_i+Hiz{}9m;I^E$dN`C;KI9s8YzxEJ7yKNqn+PhBs7{u(#MNZPAY&v!6GyzHlJ<5aE9w+^8s1DMzVMBbggX6vj*? zAfY!jGH|Djuc<2K3oE<+13=EI*5qVsfJURpcv}5-)q&6a19&$r==A;?n8fX13GQFQU(1?XVv>nxEh$(6cS25Xm0a{^57rCzmP` zmL)ym7RdW?1^>8_F*k;Qby)*rt>t-=#A~A8x}U7b zYaBJ71etc`B`Ecx7j#k%J=gEgdx&|jIc7B{;fw_{mgCXU6qGn@m5iT?tj)w0Wo&9K zv>vpHgfC+^ZpcIBsU;2EB2ObBlL7)1;3WKy|FMFcL*J-&nPs=w4Kbd}Z)Nrq|mpriI zy$w)Igd;*LIhIqM>B+M6Rul5_(71~(SQ|?Y1wmER+Tmh($!n0u@CkT^bFCusAXJAp zDM}Q#tqSy(8sf?=wij+s&(-Y;3(!Y*QoLV^ za)HJ*v`!p~qzY&7z7~|E75tmE;_wqwiR-V&P1ypHBtV-6If2caDP`y(eJF#YSVYGu zEHY}fH!tqAXcnFlMP78%5P0h1x)St^}8t*77_Yjb_q+L6TaNu_c$TVNB#AHcWRtN0kT@IdD!W1xIq ztcN{qUHF#)7tCpc=i2W9k{?FNJji#Y223-!-zvX;8;>qaT*SO+6bc%At6oaNjW|+L zqeCvK8}%g{oA*Q=0iTBwS-ew zEy$DzUXgsjY>7<64{aSe=7^nd2)F~B z8URPk?m1@y_!o|beNFq?5Ca(b?ejzoNAim`8W>YI)DMK>RvgClR0S588X(r1O|WDQ zZ$^JH49N2UGzzx4y!(56mr^~T#IIxlv!=d4~j7ai}nm{d#J`ptSnWmaBNen{j6|h=_9y# zW?MKvF}Yf9<%{VT!^H~Lk>b$G%FRt20$BwJJ+i$rAj*co3;^UQL5ZJm~?UohIE~A*Bdy+yCcs2SA1NfbLWf;89 zNe1S_xXFEdqv1wLZ8x7mDUb>+!m@IWO_-g^@Q9dhW#1n_xXLCoMJBY%)pvU9x8rw1 zteLOdO@9C=JLwCz)A3__qV;BHbi&^%UVSQ8hy9YYoO!}D;x*+~k z-Po&(CS^yBI|X{tgbv&)?$bQtG5j2>}Wn_!h|uszr=D5GC? zH1{A>`2^nZOb1W}^$A+Tha)hMSg^vJHoa^T87{7*MH40z;h%F%eA$FHd>Z#0W7BWO zM_0!#$d`GsUT91v6UohWCFw;poAia*oD7lLMux~7*Q&L>2aCK z9-bb0mI=&5$~~Q#isULD1~LtfpVnWFjP)eM(J~QU-hxh&%1^&N-fhM<;rfie{N%;c zthjS9&}2?0a6O2$0a3YSuz^=pYx!PQpW75fQoK$2H*^e7S;s%=L8YVP>uAK*9HnKa zrP~ylXMTh|X%c{HNpa+cXq?Rgh<<<7?)Ww`dTs8|s8xrxzzJ}n%tokl;iGcqS$F@h z^aNP8*dKuI4yP3`5mG};Gqg)u>|qs&Qr#V{;e#V-gU(LnVGW*5DBoR=DD_ljSVPd{ z=0|-+cuS82{QM3&3^asE7tsXwaxKE|xZg!2b^_}hu=W^%6gQLai$DCtuo8(;#IwR< zve*C4sHU^7{dJ^W2YKeL4{+wry1U)`%xtwknMo;{=SZIPAx%Y|rYVTFuek35(Ba@B zBRAO?3s@#Ls>??N1i~2MYJGfl%8?08r}~6`5|2tBNi4RQ7WJ;7Ingnyn|@?E!ai3d zrVNQV;<}76(S64GdG1<9+eNBy0Utbf#bx}>LxW1|d2Mf$$SBA#&{EPvk*!%i*5 zdz{x^q1|Jl!)ANW1whCzLUh?EptsBGJ;(5rHRxKW`b0ud^4_lo^Z!;?+Wo-Go?Z zqnZ1*daY!<87vYx;i=A10&jL-DoWri7thvh^QlDp4*mhK6NWH!i;%fF)Zt2dPUkE` z+FO%p+`ceb;1&*idwfs$BDu+j1F?d*h3~kG4Z%~uzbOnjY`l-gz?u= ze?FW*rXo!06O^^SGMsIL$%dY9_t=FxC8PCEMXMa8#N{+S4CDGmsD^KOP*$ZxSyjCP zA+QK)jm={_zN!eJJMY;q%s|*EIH=X1al&Vy%B7E%)zbG;pwH*alrR8 zzyUWUB0^pewM2um?&ox+2acW2q+XksHQt1rXXX2~1YW?ruj2<2-*OuV1ghfH5`#BI z4fgOHz>8CeFT%mx9cjDvFyvZTD`UE>q(qq1AiQ1~a%~gJ9$>wp!nXuMlIhK%?|d`H zMziBYEg|K}MLF)m$GR1i7%J%}qSiF5C+oqOT580Ep9@_|ux*qyDx$n~ex5b>qQ7Je)DU@_yzM)Gn7pvZw2uz4&EGXhgtx zcf|d1TrExOYw3ea&EV?8qFLZnZQGhEB1NA3ZJUt4wxH+<{UL@i6HbXLm6`&3z37?b zyFUP$lYui?+}nrQ=bDBW9PnZJbYPB@AcYO&nw__c^tTo_3p7rKt!gx=3OY>JQg>af zb8_64$!SE)KR{44N;wqY$a$9w=1~hqH&A3KMoy4-^o9S5PsHIEM;J(;+56-4I~wOS zjB^7rdDkm>boD*~W2m25_*`m)EOC{>*@u}n+O$qw|1|N11^2Vh?+6qG%jn&zgeX%Z zH^XMl-vK-%Xa^i-rhk4+?3i8cb28U2N?^x0C`F&SPp0zZOvBH82*lHk>O0!H;Kh|l zTey>*U9 zGy2%D%24tk;uR;yDd8Dq1`CmYE|lMS^j~_HK{kXwz6M?fZnW7WbVVgbc#Sv?4HZwl z0m90xUyg$;)Gaz?LF0J1=}aPn#ZG?!3OmM2e7Va%4t<^=Rsya4&C6Ka@>atFPai`R zAivSUzi&s+sDj{=Nxh0CEcejbJ`Fj^fI$+#a&SsfB)wuX9^JOKNAGS&ERsjpdaYNW zyy%C)RC@)b@WJ5+Q4;VX>8OhL<&ZvXj?PL*eo(#3JG>~~myXY2uA;l%b6Y32)AqzN zAH0uS?lJdr_;daWFU3J;{L2}chbD6~9v-b)b)#p=*NKS`?L5vh%$)%Sm(_hF=LKd? z$ZsC=5(YQpZ>U#Ju#8grL}lCgN;sr`by|aP;~jS~XWAA7Ko|)lCwI7(wwrEFP6RbdF~maa?G=` zIt24;cKZ%zT2!)~y=uGL1x3QTDJH(2P?8TGB}!-lv=JjJaRFuj&I-P9bYVLeHa#?n z`lUFG&RP>>rNPjsni$EHmytY& z102A9;jk9I4ic`EYSvcih*m)^Hm&tt2WB9Ezp^0#4)D<$r(;@V%AqD%i;ghq2d-tC7p1$S!#;@ zwD?0De<5DA4@UP&1DCl(^S+Y5gj5q&tWty%jvcZM%WP7>Q6G8bo|bJE<*+Dq=T@S2 zIL3-p##Astli9aC@@=b;X0C<`Een?)BJE8fPgqH214@D~tCG*VnXP!`Y_mmf%r}+- z{(#4z3NQG>h1x_jb*xoFJ|YUYx_&9z1P;g&$t6*CvGHXh*FUpr5F<&L%*`T6MPp6$ z@(Uko0e_j}M*3}VfYO&5%@a{J#@-Un54uWk z!ECjPY4BGZn%rM#)(=cYEHiJ2AF9zj29@^qv1W9pfE%5K*mYrFj?8n zjan&l0(|2ps*HT4%sj-^%SVIOM7mV6%eun)DOKWWpYA=bc#coP z=!kJtV39QjV1lfgQf~AJV=^5j;&IKC#XWE*|CrsAH7fMsqdi>D{+YtHFR8p7cc+Oy z=F~{TV`CD2(TDt7lz>JS@X7K+;?-+LB3@&CreZ-H0VkYQ>|OtPHYj0kVXPSZkEz1+ zaJ%XL_|{o>p&8komD9lG$2?BP*v!X)9SNnkoT?O#eMv#lWz^!tLdH!obhh9CD!5uo zTQm!mY_NJnUT}|QKH?t$WrhQ|ul4233`dqDNTc-X*MZ6!LYayRXYG*bZyxXb{Fgrw z-=K6TplS5F2gnep(L%D6;@_`*XSCxXjvwS1ah3i$`YKFS(&L~6Fg|;@!IC~D9X~!YCJNZ* zFLW29D=Y=DV94&F!sA|fDS#D3c4BG#GXt>!Bl2O#$-rg8E1rRp)TeZ~Pm|1?OcXXy zq1ue%HN{+ZqK2{7B7i1qDQpl(Op3Lmy%NUl1`qFspLyd{J!*nq zjykaZ!}qItQR%`00P2AetW+YNP)Rk1dSvCXU|F49-6-^g1#;A6j-rgl#^IJ4nLA90 z;gCRI^Jr$HO|)WBPf$({nQ!*)W^s-|?{C*q>*!6-jG_7|&>>K2OP{HJdT8bP`;wE2vl`4uhkhJSg#3C>ExuOIjmVcHJ*~idMwVVoUxx_j@ME^SFx;yJQ+>VPAc} zb(vQxDSf9UV6GQ|dqbx|`umM@lBr7BqFccC%Ltmqh50E&^+(&fjV&aIvE8~x#}R!#SmGE? zbfigJ%(EWR1gx1`eW<6I&{0;COF&kNI8fx)EkTQkO6#>ei^bf7b#AH{KE86sSUI=P z6Ne5T<~{kQAB}H5>TM{81h(7!rIY6Z1c~=Ed0p2v*`C1)XAK78G;v}}aB=0b>*1*N zpruW-188BytRqZStzb+U7Y+l2$Truv$k&IjSeiLaIG7O1C_C3f;x(J4&T4|4myVxB4N7s$#6OLqB6pQS0FU z08r@)4s?{`t+@=7@=!jd9Pq3Q<`}Yr#qqf$ikAxt4K}ceyX(4LzsgaCRv8C^-*7kS z0x&Ok0)qwUncvXc@LN3*ze&{z0BZ$(Cxx+EnrS8~n8j?UgwP*^GD=q#mTrFzet%*h zvq2TlL;lOukH77;IAbtr`5w=Zf%n?!dw$#~7%uz*A1ax<#-3+Jd98knmo<`hvI^9d zM>J2WTr$kEL)t?IHCRF%$q%bJD%~IbL+>0C1jGH;;sM{7OB2tnq}Aphzy+_D{U5-< z#NzC`fP1RbuU)Ze;+iJnXd(WV%R^tbZWek_-HBgWymB(tob0aY_L(3wGQA&1nFH57}0cV0Q%*)T|7qJyuxIqva>FK)m zt=CXfrNaBJZ?ZT5(gz9&SuQG6vfL$CN9eb3DqOZwVt63A@r~7FcVj{_4ZQS+_;=C# z;-v9e{rIUbx$8Ys!DYe>5$`_sgdYqR4~<1#i6}!#}=Kq19`0En$JoI%q#QY z=eH$&xpv=4(68ad27AwDveF@CQxy0m%vW+%H*ekWkR@BZ&m9@J;y)&&Zhq#IY&o{x zXJx=i9x-)la$mCzM?LXUFQc(-==?4(a@Ael38Db-4pV z@_YTxT3|W2lev0!ZL(+_;(Q5o)L;tMxqjIsrUnZ$_V>3F%zW@h7x~4*Nw%>!kT@*> zwf|7yrAvsDM;C1_M_jfn$Nb?n`l0AZbYJ#SB!x>JGCiveZl{Dn!kvy&%5MaEKJ7sI9wltV?U|Sc3Y_ zNh$!K)-BbB8g{@)uyo{SL+Tc~7DsstJ!y4vP2_v*b=(ZZ+|p;PSDN@m0+P+7$49nc zbCF(gk-~74IExGJNzY>vXBsF&6_<#hNXQj!JC9XkZ4oI6gakbY@$4DX<-rmKR0aA=Z@MZRzY&!U`@Y{%h^6X8kj?JVM0$Z*}3JrT!7UkwF| zPauD6J6bY@;3I|3YVHT$y&)Z_nvGM%MBJU55GGSHWGE(Pv{_QigG61?gvwSS9}#Nx zU7iVDC3FKf1$ftdaa4(NmtCliQCGM@_<o)DvFU6LolJ?J$<(+*LPk9#?fQ^od`ku#kW z)X)y24tuY_jo>g%t~(4t0s-;pa+wab1HJ?Qmkv?R#)cJ>#WhB=N`3vkwKW5 zIkj|^KXnoTES1KCp}1{eaRCJyUTja50VYO6J+!jnfIMpHzL`695ndO5wTU(hp?r{G z78-X?Ff`sNavjtl6=3Wrn)WO266$Nh*ASh75uAo71%CkvB3Tli(6rx(Nv7zDLse)vVi};eRc_ZJNrZW-n@qUY z(2-J-X`ws4d>a(?ZBg_n88{kS?ZOe3q7Oj@JPaS_s7QYGfRi+>iPIR$d5;PYYhlwK zXd<(%gSemOqti=oUwN)#rg-3I<3u1~B8v-6Ah1L-se=mCo(>-vr|DOq0j0$9#aSjb zC?AllQcZ;|tbTA3q6fw*fMATlL{GcO7B{FdXCm|Lf(r8c(Gs{z^wYbQRC#ioZVXy5)g^C&)rbQH z!=lSyLMFVr2-_USoOr=cz!_DMzQp1u_6m6EUZpo8rr&o{@YGnwy?X~rs;e`l9p+8D zFJJPx#!rX+>3M}X{%1DGN;wlnm^NNq? zu5DnG%uK7JLz4}<&phCjZ4e$__u0#wIC|*9PZl$!yv-}SJE_u`QV1(VqlBT!B9&=d za#9lbTDP{cGz*YZPFK{M%iQEAS%W{>hR5)IWD@xPs`k!xwQ1*MAZie6s3*dD$^cF~ zFH-wplC|1P#Msg zFC7=%->QLyG1`_GYN^%~Qr3*8NL#Jr9v}}w4y=}vAiU>9!9Sn9+#6(6A0|aB87dPX z_)sMoBG=$zbG+V$2%xEB=o-+BI!A=;(L~Z?{=63nh3_^|MdrB{tzrtw7Op?kjJ6Jf z6Z%qJ6|s;t^7^7;Z`^}55K;Z%kl5UerHn&4Kh^%7X6BH2%Ikd^z@U>FghbPXhCgHG zkr)a6<+v0*2k?3Ha`6&h(Jqh&gVz?Gz5#TO)fo!nSoYIi8v0JmSRC9bkh#!c$Feo_ z#B;Q`F@`E$U1Dh)T_Pu#DjXl~E=FG%=^22Hyt=;dosCt1$N(z4IW%AnPMpYyJB#Zc zh73(M!8I|g?1R5YWa^xMlQW-@`R!bG1}~>}9cqoIwRc%&^cG1n_4&YMtwmUuq5@Jf zqm@J#x9Vs+@x@z{C0oj3gub$0Js~gmDjGRHWG1)O9d^WcEV{OY&j$0jPexZSPLN@0 zHZQmt+)xED!SkpS3ubemxce@nSN`Ne1$wF;osH`BJE$MJTgC1ag7|AZVt7&XzC^oAA>hy_z*I*j=(to>P%UdVhYA0it7;Di2%$ z*j6vQ&J0FtR#o8QAzF9khAuJ-CP?236d6L*)!^ncGy#Gs$ z^84?*!thC+T`^>H6X`(glu0w$dXHxPA@P6=6{hN6`9idiQ{a9vkVw{x2?Cd-lNuMP zky56ua2gyE5igh}iZRC2D{1xd%^T`z;9-)`tAm4A#+mH3zN?ARxTe_Py15$QmjMP7 zHXaL^US1U4#u^@p0R9sTS$u^Z;)Og@)QhhM@gQYR`6M;T0$JF}k{cWU8BNaome9KB z9TnhHlxzYK@DCDfC8b>mWZr)=2dBlYA`N4S|KMpxH2R zM}tS@@S3PBrA)fFG|Kz7)xR5PLm+5(-|hFXg-b1tLyFUu|7q$g!{P{*Xcw2@E&&#YB@i5fLy+CY z7hl{hxD(vno#2bR26qSq3GN|4fZ$F71jyswcfa?&-!s!yQ{7!XbE@i;l{{)(euku(VGggtC`Vk~<_UNL!Bd4tKjfCkN4Tb?(lA9nAH+2M$F&*qbqz1{8)Sy4 z`G9$k9Fr9nS@!ODv9ZPqD{2P4*_|G=WTdATL0?U;1Hkx(L3T>VWuP8Jtlp<2df58q z(~*f6IC1Bwc|0cEmohyPS&9@#0GnF6j_qXVw^DDK2?x&y?tDu((-4sz;PWfrtz z&$r%Yhx@P1bt%$5kud=T0Cch3t3>%XI+$h2J9n<*?M$`$hE{S;jGJQ~rjYo+P&?IJ zi8ntw7^KK?CVqT(fK@FC(6OqhGm0Fd(K9IJzZRoDO=Ab2o5pwjwORdR!Niu%6$VTH z2PkdVYXYM>85bD}@`%kk1*SHB(HFj&Z&n3c>bAXu3~kr_1CWWgCU*2Mbp~9AzeMpNy^>eA z6bKp4)~m(Z-h~)+ zj2G2$c73v26oHnE7;$NcTDZvr*^>O3!#E42hn8VCvHh!d+cBu{^e~6*7LO+0LABzS zpVw(uQT;8}aJv*Z!@u=PxS`N1BO&$}Cs(4Pf5 zx#Uq22LuI1mY=@+m@BecdEKy}XMuO=HmRfjTfDo%41!`qUqmZOsJ*h~;lWD+jJX?H zOUsI5&x!^peuY3&+1Fd_m)~|ZoP2FZ#mYadPWuX{&`uAlD_l^FPTS%8e&qI!h`X-2 z$VWAp4;uQ;Mh=#gS_+^YEmqdAe=WFVE?e;%W4~h=yZgv77X*REg$>K5lj4bV@0}#T!lY40`18K2@cDh`SFZ83Kha+e2(T1F zXKPE7?+3%;C~4rrkJ}yAS)Uj)N|>^X#N-$i1Y@HI2|R*Q4HZZJ%9$lQ1IJ||3rIsW zO+3lum3icLyl>6HBftwB4K_r~JF%&CtDS~*vacUYi7?Sb4i#u72dkuy{ar?A?AWi~ zO|phgLE@ikLv~eiCSHuI;>lUbFe^0@%}{l0ZpRh_woD`0ZfF@?VymsBE^*BL-9fSk z({iz3sWiZ0Ku00D>)v%NyJA|@?nlkZzd5zo=z3Yq73>ghYw^1p*xZEJ);JDAt17}I7I_VkPBvK-xDqHn{9uxhdbx~+Z?zZQqK zO&1|x`YC;}oN({GRy2ggQ)e*!IDJa2+j$8#(T4@6&ay^*CYC`>h{zi54=xwZ_r5Cb@96iM{vu4u{5YGbyBib1?V@ z2*aJ_q=sW#Fa=GxI9LkD#Pun$zh3i>!OWB`@;t0`^$gRg-NJ=oJob!%MV%wVJG^*6 z4jO-_?6F^yox;gNyp``>+fis{Gru=1#O=Z_x};RJ>+Wv))tPUiTYO706JMu?TDG^9 zN1}ld=#K3n{P)vjolf3&5Yw25mTWS3zBWeHiI%3g_a#W^5X+Q~quKWHi;YsA7<#P^ zNt!{f9g`ia@25(!pfLUO*Hd_$5t29xX&xD|A)AIA(8gE8SX9MWKk$uMWnun>CYA2$ ziCCBAjcJ^3k+t|@_=&^fNz6S@scTE2Qd+%tpD99uNTVVw)7Q0PBk?NJSv_!3-#G^F zC2B3-Jp|Ri=|-}2t18FQ#O`)=)Nh`I&K1RA&P+FMw{>OObK z$s#?DFAsSxHG2exaEY<&%#C@#wTYT4j8lrRMJq48kW>h&NTpGtA2lwl*bVNxi$c~u zRt;&Xbb2=|b*nQ3g;JXYs4R)!xziAobv@x#trkNs)9uU{#O&X;5w*M}p#a7+Bz8}D zo7i|I%lk*^Vr{T4w@y|vio;`Sd&68n9UhPrG(Ycm)ZaIn)KXg+;{@Jp!BUYUt%??2 z777Qm8j<@Bl@$!}b7kIq@BXK!bz3#no}C`z3GBW7?XE+2k*Hy;8Y5 zuQlu1;bfEcK^(|#SMZKcCpTPy)qxx5Rlw+9?HEp!a%cOpg!3DJ&Wugc*1V~`)GYQ; z3OnP` z@3-~Y`}KzJV#7^^y;agKd|nbEi8MOJ6|+0+xpHA2Kg(AT_BnaEk;;E9AwoG+EeW10 zA)ht8Kndy-Eyv>#CC!j2vr*YZ@{K(k&XCHj*R*bo=HUxXZmqVqs#Mt@= z==vBrRd|g~!2f-@W6~WwG?gTMX8(EG=LovJ{Rivo*57Mpr=T~6<trepgmRA0fc13j{vO+nudJNY|;&YXs{`vpp?TlIy96c?qTIL=*hJ218xPF9*N zu3KoClU?c(E7Qg?Fx0;~4=J<{i~7;R-+>>zr}+*#ScoLBGFJmse-O?1#f3y<53{X4j!V_Y{gZ{=fOcCvJClTk_?;<`qWZo0vql3jW;h3N5<%0QKD^P^0mg!> z8jv&l&bD_$)dqk;Y!?lR zepFmcRzMC2KTSQVr6&m<8d8M-pL)^i%IIG4(HSZ6_hA_NlRcx%O^9+oenn&{?L$M% zsae16Ea8u7cEwpY!aEz`@%h-eu9&qb7I2bE;sOb?7uw1>lhlgfLc_$JU9v7D0x112;5T3LMAcp8INiP-oL1+QjTt6#U z5?OX^^({B_izjKyo*?S-a>#?Atlg%o5ND-h(=h(2xXXo-^W-e#xa8?Wn7EQfgq;$> zpXwfznO#fI2F(*|#Eju$*NRyU#4L>>xPp$Ea=^T3#+iFJ5mjPrbAll$Jch346y`S(hCM1Fl~Cn5*_L+Se+~ywrOhD+->YXfJE6$hGd`yN`^c! z;N030s@9B!b#{A-$E>}8Ca~s7&ZRLSqbU$vuHm}9WQ4dVsZJv3)UW~C@-Lcqb*jj@?Y{ri#x~3STkEmQxb@Uea?$`>B5lw)FOERD=y4)&0qaI+ zqchiTP`+Ii?{q}0W=pG<4SZuW>;AJh`a5mPP7u2^Im{8Mj6l@OLn8k%J{fHR2c041 zZ#Gg(_VXO&2$?aHzA>NRn9IZ(37g z4fPa%Z>fTg_$f5bf!oy!TwuM=8w94CRmf4GDl(k{4Tw8^btIjkq2mxE0xDT}&B9^K-ej z#Q?)5;mfa)F{8BP6HDsKA?Os_)kHKGE-~Ym<;olHhdbdSH$#Q&L&tprG|00B|Bf<7 zf1+{j0d3ccZM((okToxnH#z^v`@eoDGqX`$+oG(?!RDWiY{SvJCfrxdz$EdCqa>js z-p?w`dW(77@i}z4kV(U<#Bdr5OLg%uO<#Y-Eih{Ky(FLO)#TR&KSdrPJP@($^e;UM z4Hg+R8>QPGW-))s)Tm$V^fW)=yu|+AiTFl%aI-S-?;iWE*(=l7-RQSgPxADOv_&Jl zHVmKe>os+SUnryS@IfWsW=p1VJt&6y2BqRW|8lU0rxN-KBw2viQzu(GN zD41(1(?}1w8bcBFj)?|0DErK}Iv=8viM#F^Y`BiPLUQ;?l_#v-R@&3=TQ79Re7DAiqkxN)0Rhy8I0;O*sw+@{JT%Xvs2v#(I7oo_9P*^DG12~LB*+*7;Nu6;}RaWPp zlXe?%&&h*g7F}e8c+Tvoq~2uv#P@!u^KAdLO0TZ2V5>_sAM4^Ehm|)iSwhON^*wT) z3%aKen;BY287JJB)dnW2%=?jLaT@TUK2xLSOX-HWr-*ngnx$}5S!tfkwiEt7M@_pT z;p?s}t=4Tq0-?v1f{)#{j17M8mG?~bj!LM0aDy~c)xks`(TrtcR^TsZUBICxDS7O2 zMqX9TsVGHN2S?f9y$@9VHS>~|WySLV$k@}ruBic#`K)39X;ZDOFs0DiO&8s`K70D=3r0!` z3F_o$&>FE?ae!(F6M?&_k9rC+82#e{<8tk-4H#>nerNFzip?OaGkuG<16A;{uMIYk zWoL%cW8pK*&dfXv?PP}iF=jz5)rQlsGe}G3f>z_0H18P=1#Fvg?bNguy*POI%CV1j z!;2H|;@UxN@l_07R8$YIC%X0)%{t8qc*>I}DAFY-dDv5VkzRXse(Ga+aNy`+7Ko7+ zk<;tYD!xi}u?U~k0H1uEe91G3X+!lH^Sf{KFVk_gH}hJu=KG*e2vFq`ggl#ASe|S* zG`k_EgVgx1ydyWhT$2Qk&~2s;?3}EI`N)FT;YyM0YjtzPFo@g!LsYIE!>am&s?B48 zj+ckRPF{FscJl5$rj_NVL6HoU)e$!W0Rm- zQYk!6n-sQ1*_LJRQi54PgA3}~KV^fIYy^6`Hz~GFs8q*SETzU0Xic!s zkrd5~dO3{ZZa^|}7@2c?%1eO^!cjH$7Ev%R*|Yyz>U)M)10cy-U%ii@nzMhudjGEChbobH!(s@Bs<}^WTvHzOV{tAD zNfsAmxUNb^&z3+-vqLE-s4dX@FG?)3UdG30#>DZIEflvH?18_==v=ykQJZv5+3c5g zZO$1#`%xSTgYI|)d|?Oy{i;Y^6QX(uxAfb$kuqDR=uZs-tjn=mYMr(W6d{X){3RXw zsUDs?g}e>e_?W~c*Lf^8LM6T!Es3Ew=acSjg#LUt=6^?)3MRF4I`9wNSpJ*@Zv5i1 z+3IAr%^iX={Z^ftGK3>rBr((~2RtKH^trI=?4_kw+KrSk(iVLKJz$dtE#YWC?-nA&xEdEK9pPv)Vm z5NabfoS(2UaGvS$*|ON)FdpO9QF>4c+Z|VW8J57$D<{GPxxzxF;O+bFT(eaGWo9xa zVT`eimTNIJ4&KbLU~XDeK`7N;F+($7q2DkXMl2im>!8qIIFyfjuGOm%ThZhkJB6C`6DY~I z;wCL5IV#&$3kOB&n$<9uf>DlLX|8M2B~I9zPH~`ves_?ibGD%w@+w4%r(rbz?ZWQx zja@%4H2oAusfx$lVYpl)2E*$2Nk|8nj7#03D(69tRwSi$VUn5S#EKJ*;R;PD`SEpA zN{B_jCd@|HuG!t1(3wSzdpxJ*szb7IEF4oXLn!JWpipc*4Gy$HdhPBCeKNe>Li60o&@jZ*F&~*|&qNG4 zGwh{D2S+o})M&D-#SFHhhaZ!89`^Ig+f%~Ewdv*e^dYjP@@57k zjxUr4k%L|jc0b4H1#Tl2&GzEtb?ruBB0uL2f!02$q_bOKHCBaBK(oAWfSpKpOVbJ? zG(6@9Viz0mpz~k4%(fbP@J8YO+4sV=%QP3>1?3stZnM?`!}I74R=1JVm=*DWq}qv`GTR7;=gZc*rEW4yyk)mSMv3WYU+nV0t>E<7?M?fM2Yp|* zv)o6xO^7_{2dC2)CI&lJ+6Xa(oF@B_1cfDiLDI!}5GnTuY@9j~U zYDcS$$*yWo+^So+!^(n>6~Zd|y9mhxp!~O%$L`8t`PobCzxjy{{TxgU-ps_5_%rQfUy8C zL~Y?dXS!+*hFRj&gU~M#E#oN#)B<~7WA2-R6MHt!@CrsZ!TTYr?dR~1uNCA;K}W$? z)(y{&0eihWzs4oLxDHx=dm;A;{`F=fDGOkG>I!JL*ht5uHW;ehqUw(3NA%n1eYQ1x z1b#X7c5nq>E2ufY+vpkj0RrdXmh2dDKNV2%CWtv>V=GTzfg>>Pyv~SBQSH3uo$MI~ zNgZdPVU!Xr-aWQDso|pgT{KmloG_;xa?jINdR@&_<)2&ID*OldiTh(1>Cx0OiN%R9 zQi_PgB9I8M5;<0pyOi*F>nsKzat7L9ZRJL8l$2wUV}fD&2;NnW}kC-nhTOb;}kYAp|UHJYJQ+*w>fC`aT=}Vk>i`g&K)^wV$hpERERSRlR$`YTUdQvuKK$jPk2e3vM~ zrr8!ElcY;*WES$oj5=tF7PC7j6!M8A36Um?S1UUB=X$?T+pu$xJW78x?6jqy@OzcO zcGx#A3`k4(eoj^g4SSePx5pi3kz;kz-?Un+w%G@2=PG?P(v_Hj;}C4x9~c z<6=VkM7bZ0v>a?$w}$2Ax8>u<)Q`CgvLVg{R(0yAnGg3C( zlGf{-o|_}PFe(C}b#YhmVw!&9Dvd49`n5UW$>Ih?e)KTtkxh$x{ST%DX=H*@0O46; z>R4xV31d!tWkNCB6Mc}A>NVi@#p9)CDr|!<-XahOA5FTqb@1`Wr-9TTMGSX zUx7Q2>9-ZtOO&J@$B|K9fhwjAxb0F;(K5oI8VllaqE>t$RpZ8ES|a(`=i!d_;%yO` zWgf6L8{(aR!drwIxFs-fYf3Wvl`49iyGiU(BJeNliL@U|x?r4jaNO>O?pgTveL;IS zSS?=GmwLgt{nxdz`o3I0KaE4*$2LI@&u_W$;C(Ds9iEJFeHij0h^4Xc#CN)4StFz= zVB|yLxr~3D{-4w0UACn-L?t^|&ICaB=@1RgO435YyZXbg#FKH&3PTkpO$dI38o%Aa z=Ch%CqxIiP%S~yr2`4Ny=j7eGl?%L>KoKGQoe}{%|LD6*3+c6!`HvLm6x4F7n4-adB_Aep9MV_&ZBtibI5S zhR%0_Bv!(w)0%E}^-mgu5+uG@An`3!k;yN4!Lh)EBX@WBy3uF=*8OmISUJ=~Rv;^v zF)hdzK;E)G1jC4XM+=AA0Drg2VwOkI*9@2#G*_eub%i@bzSejBQG3 z*&Njnu=AZ*_s^;pnypE2R+5y>Vpa=jnxuU(u5@~*S;oOL@YMh-xKR7)IU+d<_&8KE zBNG_nFRdY$I>pBrY7UP6}~V>e;)eRRr=Fccv1GE|%>)=OrDeB#h~6Rr|QJb5J-^ z2XX44@0p=1;0lYXJ5VaZ*q0{*oFV=07OMO*#>>7sM{>E%Za}$G`*A&C{Ie>A2S|b7 zSwH)$@xgkQ(^`jv`9+OF)TT(qydmIfdTj?YtT|6~K^QGPp1A~Qo$YEO%i_OATs-kUNbU?yf<&|SBm82_MlB|N^IELA;MU9a-5`)r$jjrkIp6LdO+0>F4AV=flKXQWmqohF1krVNA|= zIR5~D3oi?VGqzyTM~eW5RpI!QF(a(pITD)_NGI6z zE4A;zV~t%VXbx%SHtgkeIktDT}FA0?^~tB3>EMR#}T)oR7QpEoM=*xhM+ zPtFDy!I`KU3|0OnG)#$J(-fACYG1#*klZmwT?vd5h0Bmqp+4WDEUoB;YIa{(%FtZ0e1P!(LJON%fC{MFy$16sfW!ztZhZhgW^ zF+C(9tgxhyblMELMuRe-R_P7Kt;s>|Zd-0;um1VQnPif5+Ha}e!qr+H-87!wS-85d zX$G`pPpZwFZRu-mD*)lQ%5(}|mbT6ELEFQmSQXm}$~dYv9-8aN3$!VMTbD+8@qk5y zY&CYEyX@_RT1RYG;3`(#yG7YPI|a%FlPizfOu!-Y8B;lsjHZ< zv1n9BiRDHCNYPGaIujzKmQ{^)XC^MzR(v6j$pOT;b5&T2pjhYJU4tNQ)3WTbMpMyP z-BGS>Ov+dyjGfsIP-dj5<32=rUD7Z*D4LjK1H(}yy0R8XDlaYBsjGy^6ZF729^~)2 z)un^*69cA zv(RFOBNU7?op3PuL(zzR0zQg_X0JsM`~>B%HNj$Gqg!;9)V$JT>k52kIK|<3WGXn6 zra5c%2HdEB=unn@X|p1CBj|+)3%jaSy{n!{JR|{KAWj3d52l00ZY((2g&a@2T&w8X z`N2o$PR!#`*0bjnOm2$m6D5VmGxm^-FGVC|!;`zbd5kuzQ+jf{C&M7Tn+ll}Z$!wX zN}}yz;gN*UT%UTq<)zWVEeCg=4+qeYuG5mTHo2sxws3nU3Qh)It#0LN-!`6oIfzaK938JBeiijMDFH)!U!t_e^C zu_LO~j)I2YY-NGb^l5PVfr5-!$6Hs`6 z(q;QlEhqwyC5|8kIUT^3*T2wRwTUl60_$Rj;5cur zlP!F#)X|Har0}q1C7ih+T0wXHDw{F`$s!e_=0w)NbbSaMh$dMl~PWUaT~Ec*|=>FL`Wsj+jr=oE%Y;`J-gBLIXY5nkl%3X za*0Y#Ok1fv#+7}0DJL9{@R%!TMkXd4qp`XpK2ztY1y}NFU1&1)Hw-^9lGl)?UYd;3rMG;^Zx~$w4 zeZ5wCKHvRq#~??Gof3?y^ATX#preQKCnuw+?x@X&b@w`!cnz(|&T%3au`v?##+%N)$&6wlDEr!74a}XC%uF--;hF`S+Y!$aAe%aC(hs?MES>Ab+vQU z{Bmt&%pn>l?Bp84rGpYw)B$#J)0%Bo*h;u&NS=3h8X6scL_gk%^x~I_Hw*7_qrtM; z3gxwcAq?FQ0&)oP{$B4ZwY#}Um)_P9@hxo$MzR)Gs@G2gBH;1;J%$%O^9<}a;>)7S=QPKskV*MV0qkT zKh5oNx0Hk|!V+_xsl66C0-skW=$K(nf097`H9+Eie_t-{DXqt*CH%Iu{uGAu>7t?F_m&$E?j zL#^1vLp(h7d!Y%1@*=$!V7k8g^$;7WzhFb2<(v+Xg~kIZ&WniNs1GA;;JUO`DkgI< z1&%^CQX}sIxO4%vRY_yCro=iIF%tMqLepipfA-XwFbijXN2eYnw9y>Dy{)EXMAKxx z*@huRP~^O|H0Nti458gOP){)__bPT08$8o+sEu%24NFAW#Rnl~YEdd!BGZJFZ-)m*v4*N4HNXc{^tP0zp}W>nGQ9l<6Ch2YZnS@F&ol`2 z(!45!HpX+9JQTwo!H2|Rd15-EMRC(TJ?@1hfY!jdZgNnWfqbZwRYT=0{n^ zt%rGZG(B5-B*i}+p1p)haHs9QY_+};Zw`}R`-V}e%ivSOlQUsS9P_lGPq$!_53utM zy^nA^R`Gm{1zOH`d!2n8Rxp#^c`TjCfQ}~dI=?2DGIrHIWG^?SrR4wymK^e) z`7ARXko+drC%ibK#O1y(k1Ytui zNo=~A6wPIBPW)X4I)uC$#t%>MUx^xzJa_E%p3XrgnY{P}FCEWII*V4rzLEcy&8`R# zI`&jOHzX+y=9s83B;38Pz5whR%%wSS`ARB_-FJB+-VMG%!u&SN-uFfFV#w$O%oE3dRrBq=RhFTdRJJ9LX-1NX20M}5w9 za;6Yef$#mHGNcPL6o3c?b_kJ{Ver)qb~a(B@mr|LxeZP!Gy}>kpob%C$bJ6Z} z6unTBr!SihqgXqwACQ^V9!AQB?Q$=O(v;Dxb58Ba`*`cR=$v4gE6oN?N-3`X<4`^g zmja|%X4Sd#d z|16R7=RWp%{-yw~sXm`M2u^_{-kjDvl3`kGcRG!0MO>gvu&qi0E(N>tY)oS%BY z`H`T*_n>UM@6hK!0Z2vQ-8|GRqzk+BeUoo;Q{W%-#1o$ zdv215&D_rnxN$Fu)16iUf?57trLQtxv<%5@fyTIZe8$2fCmnmy>VWQPAE0%T?jHbaAU0DX z6Azzfl*o zE-#gAcime$Y|AjVZDY^K@ilu&7^_U1jRYS!6L`ceS4AG?i#m z$#fz~X426!0LEE^d*E0a!f;=&wBw_rTcB}yqvO1`LTHMR;VreSzS2aYajfR>k&GnQ z3KW7S3+J^r@q3edy<|!QVw(F zbyiUNcwj`)lQG~jpJw54HE+6#%rqHD z;TC4H2aA5PEzaWtDD6dXK#uA$;s%|QSKuGm!d{qN*OBB~T_$B5Sgc9f*x9vEJbp$< znKFHXX8_=D=9o%vW!4njAwP~UEKV+5!cu-kQIngD`NCojP7zE;G={HCECmzB9m!sJ zH$_ApELdoDG_;XYnL=5Ey$y%jdcUMj7z-qBPAE&m3)3ViT(jTbY0jG zg*_hx95ZGa89glm!l$)&z`DaJxQ{Jkh+8VYQM(ssb|J#|tn%%*u}u7cc}qp7gzNj8 z7aH83Pp;2zS$BTTe)`P#j8Y;Pcu*5)d6%DeX#W$T<-RsyV0df;i#bKpAsS(t^Xy^7 z1KkzhH3=sv3%+}fA$l8(F}40gZk+lLpx^WSE#)5o=(kQhe_!nuv82VfyEfF2I9Sx( z6_E!q?>8qDR#0=e-5UM}FzEZ=y_=Ft=l=lByM1GHLivZMD2wiWcWqYuJum3UR*-pq z!4y|jbGoK#o=vp|>$G|zM#1gG3m5(U+bfKzLvhjc725LRZ(clqB{w=BkHi+954Pq+ zKJrHo*@N=#987Zv4M0@Nv@arXdGG1}0e10og#wFp2K&XQZob^L1$zE@W;K7|wm;5% zVSc*arU*WCfL;JM0!?gLjxl=;KoX^Rv|b3sYmA@jrm|cwmx* z(@RT2jOxRv{newcr;BYSJe>FQ_obs8dRp?~+Kq%PN@nWu@V@^iL~nc6_+^JQ@~MX3UPgg`*4HP*ANfp^A+Wa}G%jN+>V>P+xms?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V=-z11vp_h97z1 zHyUoR6f{bYh5-2?K%0Ct1z3aju;%8M7N=sZhM9oXuvDj01{12o&z?E67bHR}8r3%H a^JwMn;bJcJLmbsaeBlI2lBJ12LJt7I&r&%6 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000021.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000021.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ea20d860387a6410a325ae54a934a3e65409ac4d GIT binary patch literal 29167 zcmd41g;!hO7Bw2AxVr?3Q=p`{LtC^ID^fJjA_am&Ah;Cw;=!fG-GWoxio3hJ6={L; zrN4W>d+&RHz#H!{MmX8$WUal{nsct5oc%cWxCS74E3YUIKtTlnP>_Fs$3FlViT-<#kdTm*lOtb=h)77t$jHelDJhWy$bZz- z)b#W~ATu)y7Z?9yH=q^i;0X!}(#3y{Cur!XC>WSn$kBg}|6D*$2MzrR1`z-S4HXRy z6AJ?i4HFdu4f$bIVn#F)UUYTi*H2>kK0A^!O`d6t$K_OYNNbXzdjyBL<-UHH`HH4ebd4?e9#GZ;5#sNl0I#t809Y%`vWW z94G5I<71N6e6r4b&QI=d;xr*6@b$4BK#Yarh3bcf!i1ax0F4;%xB$RInm{E+p1%ZK zvKd$tJUb`ibd%k#k(i3U*v&n){5?&X2XU^eok{H#Fyj?NZ*~V;O>0g3c<8GSTz^W* z!OBWbR(|Z7Np0vPvDwnN-@Neb)Xu~1I^1x(Ot4v4_ZqNrCQd4cUdJ^nwN>!TK+g1i zVXM=j%w2FjE||4?zZAGh*a|H&BHI%yGfO;fqjgU&kbYObS67}Jy4I?% zfPXF@lb`4x6J!|_c$bvg%dR`)Df21_oPF^nUsr--_mp!h7B(~>T%twvwoDL|a-}C3 z$BC_jH*iT|PdU2nQI(Z(qnx~n`H6XJs_~fay^`pOhCfCV3pE~#b7-FJM=I2BzEFbi z5(~PE-nS6_p%MHHTgI|4xf+fr!BMf840#0RBI`eAq~^ z9MR+rwa4iCGYTnZAenS-^{aC_C-U3&9L$Vw{aWAB@C>}>sa6KN{;4u?;p?>XxsbB< zv|)`lEMxnK8>L&bb3ST6Vu72UTkb)rqUE8!;YqnAsC&%p)-^o`rSE@3GmLPjRIpri zxhueRp&Mj7ZhprTV#2xs+!L@23nIL;3sLR^x7^JxJiM;5oa5f>-WZthb!HGPzu3(` zwfy!EJh*ZizmDsT*ByQzyMu#_5SDn{C14q))OF^-I;aK$ocOIO-+QlR^3zOEuh=nM za@ZZvUmwxz0%6G(RsCN(>H1m>pO!j5O zQs!;r9BwLjeWFt-!0*f-*6C3D6e~Vu zE~fU&$rVMWO~l!O0Yldplvn1Jrr{_^Ui>$h;%(ZQ(x_)sdR3kpeWJ7ay}^hw<__EK zuazHTD;22U{c`+vtR0~{V=}qmXl)Smjcgbd1@qrD0sW=PzF%Cjk|wr z{40|3=Jx~o6yD|2-@FVb7j2hRp-Ol?&&J$#$~@z0wg1dodS@?B?B8|nQbgD}+Dvxw zHhJv%ZT>$j99#$Y7T6n5aWeG2`Yl@A=|6cC|0kkm%nQ2U*5em$wu9HUkPHTY;Mk4G zt=GE+yyA%T*lgj_>g9tB1*bdjtCDD_^(K`Eo7`(Gf|vhWkhF61)a4s~RnWRI6l_Nk zw<{3Kj73D^gfj9!b?2yM0RFGvH0l4JUq&iImJJsp02H7ysAIE$=lNub ziRMUG**+(c{cNP>(D~2wDBG7VvC*>0I~|GCpKAUk5_yW>N1Ze_Vfalh&dU~wrwf7H zM&&!@JHqXs@ht~(BnOth`NrGd)MfW~?lga0uk35c^Zk#7)h_+f(#bou3oE71CC~Pq zD}daIS?=)eBE{jUmbC9>e`L#CchY{B5Y$*r+H6vRkKBfUU0&Q@JT_6vo=G3P#J85J zQ9FJ)@a=4-=1}0z;(cn*)6KlskOGCbtD6fa?A^WJ?hAY?UY7}jR;p}xk0>3+M4GYD zyzz3qjjDI**1d;gX!tBIH{fmfhf>vj;`gbeT^^&QRU?l8w&FE*YEe#Yd*8|aOZKjw zM)}!8Q8Q4**-A~+B3juW6{(k;R-^aw@zr+jg41WyvwPkvyS8P*zgtEGaqI6H%YPpW zOOCST_|VLFn`dSGQK7`w*L{8C8E>uPQ^B7oebDyL~{ zxHc@viGQ#_?M}X&I-93PdJ3Y`mk{0U9goso36C#WV25LL=K~8e%Mk3W@ciYysFl_-Ujz7v)ynZN=@YJ<2qf9F9 zl<30`{l4#;88DST1Lz3FZB+T^y$K;K%<#OCchg`3f=0$!&l^1Mvi?XdyAf(Tr4PYZ z7WaenllQ5s{{h}?=Q$nbLTm2=T+#Io+h*I)r5R5D!mV6HbYey{uhXx?2CW59Vv#5~ z!<+QsFA$(78XeiKLB zd+VW*B2Khqm@P}t%!9@ekYi}vK`IJx5b-iWaP$7*1p$5a(H=QV-mVVD9$bSUGYufKh!WpYp z8n{F9BsprtcVr>r)?ROae{}!lKzT$^`C0y*PNUFN(;LNUr=$wq_Qe~t1-ekVW?O#s zG@8jC<83-UXu92PhDC#8=fc8I52^c`ye~-7msKHITXrv8@_YDYEs_FR*xf#h$3~mB zO1aU_2X;+eHk7^`vgB{pUm9&UXG%9a&8;cH%qx<|W%A=az02P`&y$ z(8-5DDYR7mS;6I;j;@__UkX~MLH|i%PeN*uCunrZ>{yN>Ld$(3EZkOU$440ENbk#hbeDMnu1*YPa~RbixGKw zQT?KMm8KBVR;8Lju?SPGUD1(NH+}rZ^meW+hP zLFU@XL~cZ52JA8`cFuOGKYMGS9qD&=YkFfW>YC_Mf7~>^IG23SxQSnB7}#HE$B_@^ zP(zB=zjF2xsdN7*W|MgS$YM?xF^@G|E&66(OP-GcxjsK zxpdPw!naGZ3(QTw(EV@eN(cw_ClmkQ1<%`YDl~j^UtjufDEzI-KRp3*macI8efmTN z0fMItGQ#lgi%J_4A9$9)za9Z9FK=J|TN?GeBn{6`jkXsw_kq%s_WvFK&za`>!B5X| zSncmO8}FJ5eDr3}=uXnF#kPF&YyLJ4o4-=H|Bts*C)=C}q|6um&DcNVe|t<_ACgE# zXDtdy{f?a}m=M(hmHg`lwF}F8!P4K0f7Qi{)prr@u&BAu{ExaH{u^Vt!WZyo;`hVi zp5VWT3n~MCybE5)QbfwTuJ7~1G#RIM=9{doXQ6MP)HFmE7 z^a8!8bmF%{it+Gsm1lMA!Eh=-W{+U1jyI>>u*#V}kmlPhgI7aOwkS*C+552A_kn8_^cxL7<4Z!z7441reaM(K|{RP zqGM9#G_^_f&pb0n^XCS*6rp;^H0a^J^DCY%+F5`GhDv!(+4Ug$4>c?5&XYM1j*MFW zN|i1G3r~8AfpuZZ)r3i_?!dO0Kcr-ugAq5afSN|bGOZ{?Cha};+sW24b_zMQ4y)a- zUN_GV#uFSe?-U|{@<(Hy{Wvub19tgfdURA(&{;n_@#V{KnGwb62szH!#SWRe^n z*m4M4O@1QVm~Sf2!MsvjoXOeIk;9Aot~nX`;@AC807OKAjIVLE2KAi|ngUD)h=Nm@Quiw9!imVTz2qp?D3oDGRx#eDa-K+0C6rx{q1U5JEJ0 ztb>d%!P4D`wcFjFi2-(KBbA5_iju2U6;eYH#A}b~EDU#QcM^hk~kt?s+}C zL8foCkkfvSnV(U%V6!6~*w@8Z;f3~7l1m@$z%NC8r}izoweZmx5G*zCc203BzTY@T z_e8}vx|GY*&ny{#JXCkpzCN)(6@M*bFNznQb|rC~v~Y1{e>wNaI$ z^g@uLN5RO{t+0HWbR2jPJt=~nqONUZ+*LkLcq+LM=RF>JzQPBx zj1yFda8Cp+*E^WKUXT4%SVA@yG{=Z#sGNcoRQT$>rG6mBbM5!9u%w9ycpH{z6iM>L z->#1{6qdp$11hJv>NW`{aDxpTpF6HOj8RC(qLYmSn!f*p3;M;ax zc@ve(r=l2Fg{9?(3?GTvzouz2Z3~(iFAVGs_0lYS$QhZb}yjmQ26gND_5iS8E~_@Hzd?)?4UzFWG>_8|hEDVmeIA$w7tsj9T2% z<9JB$c5_Rk5g|;_i2rH>d5m8%Q6T!x{=&6CFvT0rqk^TZ<)}*85E_q*ttjCcesqKX zZ^r&NsYqwI|M=9R3MwH&rN}`Em7c<2v?8KHCJ9TL>~G>3ZJ3a6%@~k8%|>U$2}s%o zX-z@w)3m#oXv}KkjT&xPbeu*Sd*NuV82mdpBf2byr)cdw_$wF*(QVW{j_Gx}r^5q?{*l*bNQveAE3^OT);|?Ms_qys z6*;TQ9kCqj}_Ek zkQ9b)zqD8Vp!xdrt@$EDrzu!y!oc|R6Ztrr1)k7$guyVU#veqvMN-D!_qflmYVD@#IyPjJ? zTJ`c!eF=s|Td-4^yRvQ)sx3YDx0wAzLB;J^5A`n%&AHdod-oh z+;b~?sYB$to(y7%N`py}5&kXar|(18!oi*YeVP4cLwI7bqNU=9is7XX){T6vmPyS3 zi~Ga7Rt6{ez^8iz^%9SOmP~v0y~RfWp?z8Q{U^$E^7@g~V_&9YIGg)P#3Mkb;1RHC z@CfLCpAGAWx)} z?$Re`TwP5#K#~p%Ji3E#xXhN~KOW%HliSWtK+1r%S?~YwL_a)%Rk#s+IT=tSNFG5s z?=d~w7}e0%*vf`zk6ol@*Gay3$)P_++qy|Fi7;$s%8>aq{BbDb$$Lc{XZ-%ITZ zJTtPR3sd5MIlxQ!u=0Tdy9^J8(L;BM#T{H z{ArJXcu5!-ZhRD8e8}i?u{+FJM;?>czCb=!D;=tXTyP*kLRL(X^Uj2#lPHN73n?^# zobRNGf|X_efjrJq9UoG5a_|EG%LLS;CS51Ya-1YV<_I+n|DX``VJnJ`n80vsYIPA% zi#PK*ID@h=Z7NL1mRQ>~bCs`p(th+U#`%t>jK+Co0Y-tfdw7)U%%pDxNePrqa{Fr( z=)LK=H>$W-hZaqQg=?0+v!U_t*G@GnDY+t*VZ`cl^#UOCz>s+ur{J>L(?-Z7n3mA^ z&R+vs6f5ap#JPU1EZUm;<7cOaI~Q|gsrt;)NcLD?qLh423AA5x9~ihux?J_0Wcoc9R$+IELK1TVOVd1*Z5(|Jkt z0scqlLDzS4XVe9cI#imnN0K3_k*4YHWx=1cI6i|d1;M5nIK}7BSG?6C6$E?+%Sd_AL>BhhIJFWY4VPnybzUu|g4=acOVcxpgigH{n@>fx6`$qC zRl9Td5LGN^40^U6rnFy+$g&%oy;uSN`sNV0^*rd+0{J>o-cW)b&RMATS}qg$x*2Q7 z`|y4(^7nxo-HkZK1&l;x9kN`d+S?phe}NYhsuL6gXVjJD>LhP`E6r$V0#4!#vXq8X za0Z#N_JnimBS6p9aRxOudML{D%R02U7(p6^+R|Wh9ZC>k?uiTX$%ACO+XuG|^0Y$d zsVjAP>5=5to@O$PypRd{tLT#xY|u?#&nx}>pr-ZyPJn=JY0xzFr+1}+jL7QOk?hrR zVFi^KD@Bk?*K!9oRB3T_l5vNZ202Tk77xD5>p&u+;aO>EV56+vdl6@vFY{;_?OwAf zFI2ZH9sy4a9|*5#T6#+^pANn}l|c>&c|Kd~z>@xjnLRYAlS#ECAEQ}JU)50dnTPiY zg$hl`>QnVJaPb3M>47bV=v<}@n1ISFyerk55H-&BHl25)Xw=`g0X_7J1HkLY&wzJa zL4NpM``o_l^hBXe$^F(H_l-JT5vm_)XRR{rsi>^#M_JDwmk^)Kf;a&S?wKnFer3-y zWg!SzhUge%Tr{J)!KAHB}&^*}zd%qsQGCg^&fVkk7k0Na@&sp zry}1vp~D-{uv?Zw^>JF;WJBMd!Oyp3M=!G6*%RmnJgU6QtosS=C`nK(WbYZv)5fm9zmER9( z_7({kI@b{&(rxEjVRI8V4It5+qZVBHj`*%L+xcCz1*4SNAc<`*SdEy0_?#YY5aQRY z?E3w(R8B~!J?wmB*Fb7}h4;9jx3N(K6FbMB)rvev=h&4wD!g-u8u*{^0S z%jJY8%_xZ#*gz%iK_OkwFvLV!Sxw&XEXYj{2TUg2!bILc%lb7is8C4vGA7Go~Pj!A#hmi>eSVF;q6XGCTM(dBmPDf=URA<^FtI233 z@^YqMHY18_9Cz75uw9>GiB6hm1WoWq#@8qhM%CkYG8$2IF82#!1#U5sY;j?!O7|-z z*b!s*>tQ=+5dVkrL~t~RI;0L{(*{*mUJl442?=ARna#m)3e4;1C!Zl5Rq{<^^w4`muIp|51^F{{N%pftr_`^ zcK11Vs<@5$CWirwi$moTTDj7y^7<7$$FT57_Ovo(kSVTA<`~#GI|etp_gFIGYr1}u zt2|I2X3g86kLo7${woE7WInR9*b6=22e5JAJ|hYadISizZ3|F}ftRE|zEdDi(TddB znvknA8G1Wsq-cHVQ$5xkR@S7HOBo%!MjJ>x+v;5BE3#+5^6C4|LtTTP(Gfj~TMW>P z*>2$Ld0*5w)*HyOIn3FDUTw@^d-M@N=Ucb>JIcs7?m$!%UUt}3Tc>++6yrGoM(a|; zpa>wXH&;q;|IU6vZmQ8VacWt3Ed&b$lc(05=dw3fMxuBvX2_Ens!(xAy@*q9M-3`P z! zLM0)QV>I@ZUnM_|baylMw2Yd$Mi&=X){VM-a=%D1|!jz65bt#Qu_Xg5|C< z#4j&@jO(DhQ{*!iFf>4O7BWXf~IH zK0>?tXWAeELF!*ysh;QZ?)OOs_P~Bo$EnMOe7$WtyBLB^O^>BBLKvah$pA2oSg!&H=UBZr+sD`OaH=U$iR3i$djo$kiP$NnQiMvx1!;wd#Y! zbpk{Nq13i&5c}oI>!#`1d1vCNXebzE^qbLGt5!o(?TYQN+kFjh4*x!wvq2}IEdg<@ z=<@x2voAI=<<&cj8+(1rTUOB7oY)q*ukFy9caQY>B%Lzn}{zwzp?Xw$apkU zz8+{x!Ef6-kU0tNr+uLO&fi-8r{QW}O`p9`nH>hzMG75kJPbZKmq z$6n0m^F^Yq=rjQ`O8cSk#+#x|t->_JEY*AxrOUW0-G)P+G5{hdT-qQt-ARuDR1tAK z%b2eKZ_%s6z7KrbJGBL}ceAa9bpz0zN2)3vI^0{8+zxWM(sQ;*rO%*F_Sf}EY>g>K zbwFp@crI*vP+mvYmE+x)$KV`$}@q$5Njs3TlvSq2&diqB9KEtC`EKmO7ZU1eMQkM#$wANrEZ_8 zHLg(A@KEf&GHEjT8&sePCmZvU=$*JWMOOcbWlX=hYej?}Fc)_CCJSoF-ODvE}RvzCn-1NF`Vh&2-kvW%nxg1G&T4FZbj zuxPq~)Z#8Z$7n)=PW~`ny3eo>N^Mbb!K*cBBALtY7lR($swH|3MlHPyQ%U-t3Is6P z34g0Ne<2g{XXa}K3Vn9#{gQ|fvr(ueih`>@xb8zOISG&W4%ZHx^^ z%jG7E%aD$a5#q|uefQS|I?GAPpFNJDrBnqb^5$vu>~S`WHcxj5=<-EpcZ|f(rM76) z4~F)B(I!$2TW5G2+RStRR?o6gwOUO@ls0>+e2P9>JLOe2IYpnW3V6utwz8Y=br3P` z#R?0RFTTo79m|$%y0YrXX4~My;?Emf zwD3&~z&Ug05kWd=$t|ypYj}}j<-WgyfNRiQ@M*lDji5wD&$|NzV9q`!fQ&c_0r!Z=yS3KL_N^-R?iD@mpAN@Mfv`Mu%n?Cl+ z!IC1PMA7tV0L!(0-+$sS-eM9cp z**>eNu9D5g+N&3mMof9&ykH@|Z&+(Js~hi9a;aFN*=syvE!R;~qD8Yja;bp|THBeU z8|5Q0nWAQjkQ+Qv`I2$Y! z6w=hlM^BrEO?>$sszS67s~eEY(gek&i|TwP`tWx9*DKLpP276DE3x2o4xOh?=o^4z zfc=j4-mb>mMYrZ&R3;;lYa5OQb3OK8)F#q$OEi{=9k0JXS#NhaQs>5xc7~b}!)dnC z+mEWWa6-$(Zm;t1DrcQ9B{QrmKIDv+e$#hL-4{4wN$RzJ>(KqBZ<;b(nZ^V46xg^0S#5m3RlJNCs7sR=OdeM%u z*mgYtV{#W>CImF>TB;;mLwx?xwE;x8UpJ~%yCk&y#YwVLYmBxUSK}x;O)Q?kN5DYu z;-{O!hKJuQE(IC%$t*v{-K*jo&T#ng+cfpj@LcZY zx1*t|RNTWlM2<0Z)y*^21N`9VQ;j6fh);9qE?Hvt0qMIF*!Y>#se_C4tzL%9rLv2+tvHPk($X#{ znkJX{8kEe6h#l`Pd!cM}0*Gq}4%*{#`6;lyvzZcD90x(U!JWECMqfyTwOH8>sRR3+ z+xN1P-YV7)XBey75B5jiP&7ReM!96{5w}}2hBxsSvScp#hro8dp+Jl=`_0FH2U9{Y>~7l-y4C&9h&Bp)@o+*wL^_aT^g^KKaUD#~R|X`qk^5 z9FSGq=}5RolMYR_(nM^e`={7qF6Knwtf++SU6^_~?x>?P*7~cHD5%5%O8$q2`5aPm zbj|8Ek5lKT>2~{6+y}NVHK`>;1lP2qdnVSa({wVL76&Zt>|hQe1<`5mCIkHq(<6CA zTdix^IaSXSzt?|@f2*4j!n!*7VcXRz>&J^p(LQQn8U0=VTOrSpvr0Ua?_%a&0)`gc zPJPc7ja$Dpel<^~#8{>*TXC6S{<`#;+PK`k;;Hq=^!EW60rnrn;sbHt?X*c~KcGq_ z0#aCnvbS1^DxMVpXG%|DS))z8LXhSQN!<80A)`uLeD6X1U@&@GQoFG2tr;fY9DdwK z=#f!K&NeqC2hU3t$BNv>R?7MyedDfr?jSdX>s|ppAq2&#iRrR~t1AuFP8h|22w0}N zdv3Fe*FyaVIAfy*wv+x6xdsS;kH^=DBu)UqMa?QjFAN6j{#b}=Xml~}Q-R@JPBRtq zH~JG!b`=`eNgKXUhy6h%_Rc%EFoqbu5uL=WA#nAw;G$id!%YLzKFD^^jqc?jl&4oB zgP)nzkiPlyN!N;Q+YcJQPLz<9mVwC75CRDu$a$>ey$dK0${D}nO5;1C3p_8MkQ%dY zuC(=k(?A#&eX7I|%`CWDqnFzVnN2IKJXGEXQ&Q>96?LCEMD|+mmVFxhU8cEOM0g7P zn ziw3^QA&8fj6{qktX2#%=d&6gD90Q;}-BkJEWj49Yx>0udX|7<{HA_vu(drKExD`%i z4(!7e=m@9C#Gd;gI5+DQoe_z!)^m^i{k$HwT~|-qEF&Aw9;D+jIwF=qpO|F7zX-e2 zvqrlcQt`y~(0Kz5ZIC75Pl=&)JCF5sa;y`-?6Ta-vIaeW^9<2+_)W{p-`&&)Fm zuCrpm1AMpcsZySRZ>&f^0;tvg5PWU>leY(!V~f``Qs7Q+q%-3V8k)Z!?_WrpIWm|w z@$M6hqMKf0-2$*IQpX=^hVSQL8?o0u%z4ueF=PGUjmTtonMW1eS&4Vzj+nVv)o zbiK4Yq$Y7U<)>5z?qJz&9)VSisOC2|N*>*fg)6emC=tFMIgi>GE$bXdgE*uNa6sHm zhBLIo4>OqGPE2;~FL^UORTEi=G05}oU>Kkvrgi)PUd=?Bq+bfk$1&&_jMf%5E*@jTw52){F_|=PCs!<%$ zzw#)%WO+#^53;MQ)=)u*;X>(8{UQZdhe*MjG zoeKB&$+Xupp9{Tk^Qp*g;blxyyvrv^cCtppjV z#V@X^2AAoM`@k%N#VSu=V{G_UijKtHC|VUxD@i7HJbu^!wZ0Kj0=pz8P`oZE>K@%r zT*u(nmApx)DWL7!4M}RB$QLcv2w>JKKY6_&OS0cUX8sRV>o)<+q5XMfLd{1<)-;om zB#t5U#VX7zxfO{`M@ilVf_bL+hK0+L1@oRuJBIkEyb`ZcXhT|6gwv|}+(@K2?wXcW z2IcB$!xCb4XFHur&*>D!M)FP~pO3hWdA4+}O}fV(E!pK{mCKAw3${t`+F;!l(-}PB z`3@t(yc<}&DPKex!RKd|s979JsrNYE@3PU!HT(QqPOG}LdBol18$5#3rCn7SOY>2u8O=;Y-jGqWI|-5S_kzY-yZ=Q1$kDA>{r7H z5?Rng8nGk-)uxw2m_?v6Frw<5bf=lJ(od9cJ*)-&(|T62YP1UDv14m22G8!(+$34| zK-({CwYwm5LrCkx;9TV0JAU=%)`Nw^{N)sgKNz5hrsKpCD_yE-xf7&rXEtJxbfZ9~ zLP)9>=cyLFo!`XIa@^kuNBgLoQ18?ql`b&gM<6fThivkncDqr?N1(u!uOx`(nGG}= zlydX*4)#@)zOD?#A6!$EMbFQ~_LZmvXC8W$3e8g-0^8l9b8i6RQlgyhKlU)#38~-K z2;CW+V~cVSglG(k$q+|S49-+yWQGlmrgJoz`p?j~g|Vfm8EvC1M9Sa+zf1sqoPBFqfq7g4DtgWwPBOzx6kmzMEGB`?2swYjRng#MZ9@S$$0+M&ia))!f zcl_nBNDq#6`vJ8Bq9O0OB0R+PbxfOWR?z*1bUPlnRDY!S$Io)LjnpLCQO&qhe!IG)}5UBUv;@jrHXB>!z9-vEK`xLleICV)DpT zW>%+BWQ0CpTW%v0`!?!SmZCGA3c52MJ`?C4VQUsAgEUBe-Yqu_`^dA9%ni6I1SPZV zzK{QwN0|cTY3yR4BW*(GlA6JkRh5m6b7B!VnbQWx<73pWXKsnUB{t~qOd}kAF*hNM z8O=El!DmT*Q}#^$oO6^&z%O5Dk3Jh-|FO=xoylCKE1X3y36LI#2b8Mdo2`)`}2Rbi2E6Pu?5(4Pe% zWNNe-*+&BpUAaKLein)uRmGDdVZJp}oBMjUQo$KpKWJppVznzxfkm#+WJ#LDtayz^ z79-CBq+yK{P3Ehm>cASo)b9K#806?#I-w+UuIktp#GgYPm(~}EBkM5F+G?>8wyZ%z zXbPFC-I@npFbKcep+=($5BQ9LYn`AGLPYb#ZA4y0hMnz(NOg3)lj70~#Vj2+<|XDs zM|}{scJ7D3 z%lFc@@X(X25C=$>TWY($g4~PO$8Rtui`1!az!wYUB^Q%X8Qj;~FXG(fiH90hPab-W zF@$#2v%X1)O&~Y$ZB&n}@6b8;alUc+h?lK(TpF|zzqIlQh=JiNuozBRfADNEGlVhV zO{9>1_^fOtpf)Mq`~p2znQR1_olHHjmPGJ%d6d#k_M9x*Azo0-|E|n|q2aS+8f^Pq zqd3@Bye2Is@fueb%3#dc`KGkrmFD+!mI=3pQj;V|u;Q&hSsI+hPGTx3pF3vntO?V& z^u)+k&6Bq%a{pYn4$JaI-7L{Z@g98?{aKR?$Lo`{!355lH-YtF~B4Tu4Qh@HnZ1A!g5kmqi?J#>zb99jvfo>rQ)N zSA@#CiXL>*9SUeK3IJCt(XHb_N8f(5#rXmgGIot13aIUu<9Nc}qK5eT(b`BR&2eIV zIfh5HK!hP9>ut5T1oJUK^1R@H#f3ioOe8;< zB{c!dE_1Nu)?)uYHEZ+BiriN;v}(QMA-#)s5CGOF1&y;=WMW1>{NqQwC%IJFmY;A# z!{P7h!($_aEda88#3?K3eXQ*?F$=x&&+_J@lt0@II)t^N3LVpZ-{zZ+fMF_Cb)$5J zWSKP7c5QH@pQ5ccA|F>S38XXe75>5YJzy1nu6t`u>p+Zd9LGT7T3n}p^sx?Khz;bo zfW-okwb@k@?E1RL6taDx$0MTGE>6q>!x;9S2wC(JP{=-mE!SMuQTH%- z&vtlsL9$b?;Bn>3zhPN!qBW~xyVu!~jw^kV+qH@p^OJ;nxp3tKk9U(;Z?46r2+lS=;inbczmA{iC~yHQT86v1NZu za_6W1M8NnIhxIsF+tSN~!G1F|3|I6jr{!5W+dgW@OS1Wi_>}tJIhS0sdGi1X$*kzo|DqZC-IJoZwqiE0esI*{{E{xhnQf2zm4301s`Zi8J=Y zhp{I3i}G%ZXS$}D;@PWtyeb)OELlWz#aY)%$sH^`KCuUToz8oY0MBLQR;q%QDT_>Z z>?FB~F|-e~sT!Ic)3$?6&)JGz&#(M<4g+l5l(7t)g0^n6T><=NR^B4AnTYyxqOUbymNG}Q%D zaP_OyfTWGA`dm{ZLxOXnkHI~{ti0=}7+Oc*{7dd=k4ruC?dmZGV;;9yQntnvBmWn5 zYjpG7Si_aCu63w?P-M6aakr!Co2!xItUfysFhv8-9c47@6em=oA%m zu9rtrE6=MIC4@{yi;F(sox6%$ai64~A=k9w&5cQ%oa%b-HN4{Lht{c2tOk8d6q`f# zKHa5I7vN|V_wJVpOuAUdX>!+HprF!^VdoSLXg}0Nn9Cc|HMB}c^rzZP)WAsYc17OU z-G^u!^SP)&Hi3Mb@Wyv7$=@K#e@u$vHVDVdr0xqvXU*azX+&NVuxQK-+}t1i7WO?#-{Kat198PR6vw_z<|& zd++>-O$?0i^nn$of|W`?n%+^BUR(%2O^8z+KVS@mYl-_WLmfTQ`Ph<++Bv} z6Fs$(_Rq1q&wxO=*L%mgezrSQOz=02BJ5^h^wd+x|Y%?ZPk=* zFOzk;XT2Fb{`|y9#ac@_1ya4~r%LsRzY8B?VnSE$M9Gi&@SF44+d%ZGMy2Iw_z>BT zm>^o~A(QZ@;mCax4JWFuA8Um;_(^ZdD9%y_Gn7LGEI4-o=c>Wi*EA(eIf@x)ZOI%jud(KmIPa|sHcF#wEzfbO4$LLOvrUZlf<*0crg+|^do@4uo zQpW?e8}sC-s_ubVA90$U97lR=yEww3oHIUt@Q<$uS={`~;zLDtH#p0QZ9nH@p+flO zR9^fX+3-;U&avfSz4HrXM4`VG!6xVmF7TjrSR-fqZ2KcpD#`MXN+otIz*fgmA?THO zQodYAt3iw1suNKZPCkJZ8X}60;w*KX@7QoMLAd5LaaK8MK737h^p=gzV-X7VC^>s$ z*QCg|VdYkM;^kXiUvmXYdfsekgN6_NW<#iN+tkPU4QW?QceU;rM!Yj~ zZ`{_q6U`xm!9qZx75yP9^EfIm%>^y^VwB*zo-u=tF-G1e#0s=Ng$(Tczz$Q~K@kE` zd%-6^r$UPsFmYk&cE;qfP4d&Oi>7w8Up{tjMo^=4lg3bI;tG!*@-6Zb{9y|Qd!vO$ z4_z13rk7l^HM~?E93DaoDkp*Vd|nzW^Hf<`ZhT=%D@}hXjV%BaD~vDSFQ?-=ttY^> zz>ecpjs|+X_As`Gjz2v<9ol>P zqSaP0!;K*!pC+i~Kd_oNEE27uK;od}iREtf4yei+C3XVfq}r*ecZ4=;$XCLNp4J(&Y@Vv%9$S-|!fQr-eLeMyc;l zuTV5+1aGRUmQ9&1ZdD^+OeFTbqIH(l`aYh#*tfOJA&M@x<(f7BO*GDitL^i5x z_Oq5{q14SQcG!Cf=t>?bfnX}lj#=|mnnRBRX02%JSf}#%<=Y>H6EG9`lBT*%YwRQ{ zXhA`eOh~KzS!Ax}NuROG09>%lI0Ey<>xKN zaVFRw^eXA^Oj2|T5eTfN8aKHv?Il$@bRUhhCnW!nx#!m=PAh<*jCTnd*if9}>QpKL zfw)o$$878({m)m4*~vp@8?hjR5xFSK{)P?Gp^F5KcAE?62K&_Oy3wB|2LDe0=q(r0 zKGCR}2ebn;9TAd;-MZ5)+99nPnGYcJcaC(`^8TkiY4-?hw{(XP2y1G0PoD*$q!U_t z(xuS%H@jnLrVeYL>0-;mPVMRj2eqdYNCuMOLrO;pT`%BjHeg$FGM}YQfmOF6mEx6c z+H+jnY`NYs?g8bZL>dtzE_CZO_N?-Q7|{~b#Gj&gnyfi4>zv-7n2U~1%28jund}O7 zqStwU|(kW42(MznCGU-`ce{hg)mYNe7WjZfD0mf>c)ih|wbO3fS zGC(fzNTf!}Bw$T2^KM#6NnDyTxUMx*5Y*U9wV_w z1WlV4KHHaru6{|uthccF{*pw*;6Q@|jZ;157dWgsXqz^Ujk=po`b2im%2R^vw;q#x z`IkJ`w*Ulyc&p>%vx1vGMl`(q4v07$h&;<THucyi8;=T^_^DQ2=QNXW2|Q!g7F-z8PIg;7`&NsP+LZ?7*aEVK>YIiQx!*r> z&4^5G$3BGoK0!3*%lVTw?%N)kdABV&(~qc+n3Zi}jyu2vC!1S+X@_34{{RbI&}*5G ziYwhjw3(zhAAwJBNDdCoKV_kg+@VY5QdIb?}Pdfj{7Li2U^Df+*2nB03E8V zw&xm1IfVsGe_zW}*lwss?8>!(tsQxgt2W;U}X> z>QY&Q0S0G*^{CkKaoyClveHLlZUWPEL{in(yw*U9d(@$Q*4*1SFpM-fbD-u-a!~_P z?VZv80LiJ%U4r9z&8aQ~9Ei&*uQPv@pKXp3m+6;puW+u%wU&kuTOapxrY=3ET61|Z z3CB3OhU9`aB>RqOyaf1h28inUu1NEtUKwuKCNdsypyPGB4|AE`7Pxp15a$Tnz^jwt zdj)7CY}b!6ZP$&0z&ujWUAhD+lQggMBI~Yqwk=y|@zPsK0zGMsBb%u$(><`j3GmY! znnrs@-Z@oDex1&}-eart5zz!p=Uto>wzxL%pq2tCDkdg2O5NV%W04%Vr|u zjuF5g;GcV)82Arb13)Fw((~e(zi_yQjJ9>Ice%uoVc>!zrNuX7p7@%>K@--S2cXfx zJ<*UUIjF$fiG9;1U{b2l5k9YyZG$nsXr@Pn$B6CA;Hw5TxVSB2M|SSDYj+!3&gCBz zH08#XSp}^>3*e{lJFm_YA+u;Yv#@rnvfw3=N>(_aFpNz&1d*Q(s;aW-EN~;t>`wwM zlGxuo6gK!Rnao$W+^8P51;?3;r+lim@WMf%6rRGE5Z1}gcdP#ZQtZ3Ygafx2(JTOP zB=PqqaO~|vT<1QcF)Kmy$!Mm=?>?o4fST}Hem;J5Sw3=v8w?qfBpDIavUV$BWoa24 z&#gG!ySRc8m2;8A!U4B8jcS|00jDx1PT5b$MqkAZ5u6Mi{{VUcf-&%HdC@*aW3Eh7 ztzQU<_4?3+eM5V=W6mJqBn}{N&zco=jSte=V;W#QNZk(Fre6MQn;ZsNt1*GCY}9O} z5laWXW>1jK;f?IdrJ!?}Jme@&{%)b#OO#__(mGiOR%Ay~l*y1=z&E!gm#y0;?G?so zc5Lm*RP6k{OWDww#<*LN0NL%YGMZY;3i!U5wrv)z(=lYi!Aw|rr( z1*2tiPS|J{TNKX9_&U)3=E>)wVkExy8)l zdp$`zNpZH$rkT?(+EXuEeYOk+4CZ!Ma&)w69L5@W9B4PReAaK-hoy*~Us+g8u-3lA z^-eBxtvKNm;W$7b8R1&gC=b8i+!ru32&fcFcfhdYr(h+Q1h|~*NDnPF?1LC(BZ0UjLJW3c;N;@mNqiA_F67J)HD zHuHdUc2QB+M-0#uEqwG#c+?OT$L|{O*j3$e_ZTB-HKeHycyKZV@H|w`*_p_Q=D@@W zB<2;G*EmaZ;;loK3VqP#)zCnsaa6ic$Bm*G*$19bThe%8_p1!epg^p>657XdGk#97N_ua*##0 zT)J)B6_O*dOrOl>WYg~@hYki-7x#Ib_A6+#S2aE01UC#!@132AtCl;LxrM>dV{%zt z_!-mrSQ_A8J4q_eaF~(Kt8YzqPG#*q<%?X{CV|p@I;vOmS^^}gzy}+E@>`Yy6+Y#XbpmDXXR9|*Ne7g zz_Mrdgc@+%lbNTrZN~00_Z}yDorY}_JPkV631)F_=8QN9W>ryzmb_v=M9AtxUT~E4 zo2_eH9mlO>110S`k0vO$7<}(h*w^^T8Ts%|5OA^w?J7d}i(UcXvGQD90adMWtR1uS zR&yNek2-*@$GZz=KtzLav`;;Rv7qo&7Ubk4b*KtB;JQ0e(#H8TDaww{Wb8L6^WPxK zgd1T6*^psQc>=`cJb!|Y8rH+0PSNUxV(e_fpc%F>Br704!gP}xTu9m46^Dc$;1=PJ5I&*qLBwO@ zA2mviXpuWNpX2gYtnDXf-WC{f+01>wg;*na?DGf%#M))&+JQNcNZeJknSyv!Q*`90 zGBHEGpnH&pmkL%#V1x{btdqx%;HJnE_4wCC>g zl<1uE13jIKVt7I0$M`I^1{-e+>p->5!f}?D0l501Js3tsKX(t=&Mkv3)o#c#&`rjY zWA{}x7TWu*2DArePTZPq2Ru5yeM#3gUypQdR|1Y|UJFKfpHgTOY1rMs7LK?-B|`4; zyeUtA+@$J&?wJ*W*^Q7?@BjpyZUYJWTx;*mCh~>bcwKF;B#_Y?=3^x(_sO>qdnvVT zy*<0OT<5Z_B!Xlb3N0t6H@lHmSe&4-cG3Q09dEQm_7k}_KxJE5_a!Utzo=__$4(MX zKxg|QCYD?i;p+?5o>|!TDJy4U?0(QHtXSMqYl@;cDB-3FIh?mGZP*SDp4Cmz_`|vAdLCcX*42W@mq31hZ>j0+P2GOtz!(CfCSUAa&J8^e=Wx+EbRB;Y^*ZdE_1=O$+Soy&p$}w z&Qi9utqowP4t^JQ5K=aiQ#dWOnXNc&N;>Aje{Fh*>Xf>AD z&_x>2%n}}t1Mhc`+}{Lw?^9Ut%Wdx*1OYv;tD@42Or}_mZe5EvvXPBeQt=HlJD&HB zCuiA`N7OKVvW83E*%Z!+TU~nHt8TVrm$q7STF0NT#7XYI-JOd`zq_7BqV4%(e1^F} zoz0-;_)Z`tv^gFjBDO9zT<19OB$M%1(^|Qzd0}~@OT1Ec51LkJv!a`KZTF;{K(&F? z*T|UgQXL$Y@eb2n-Zav~L0vAE0{(}zUed#^8H8-BuO}6;UIDIM#$jK^NZUzR- z;gQT$sPHjS-47uMTn-?FApy^Fjm#ID5zzUS64S9gl=?&BpVyE1gV;U_E}4XCEZ6+T zf8-abFT<$srD;caKskijw%)|@7liVhNg>%#@Z6%}B*s~)R1uwcCmV_2fGaEM+Q-nk zDjiDEV76Qjo=ST(q1_`VvY#a*2EpgsE0(j2s&XPTPBu!JYTH_TOfL6VojTibT>EF{ z6QQH-61mDu<95xfsfP&W`7LfJfa+}i2zz$Bs=l~RwWM$ApSg3n3Wid$2X1Rx(hlwE z*2K$!=BG6kiEd(?)$0aQ;MRuLgPY|<5Cf_f=bl^q%7E8F6C`fq@l7LZ4u&kxBA6Ew?w`j7;M(ardL z5G0yYK0c*soafwq=*uAAaYC1IICxLqqc!k)igDDbnDf~RFd^3iRRzVu=b1D^1mll9 z(^v_(=oC-ewpblnoyviR95);1!8upG!D+J|A`B-jv8{B6Nh6~p)_zC|Op2832oYaB z@T2!GZf+g5`_z`U;AS5Kr!&PyLjjl{0)6XAE!pFXd0wiz=x zPw33p2P4}b+U7vt9L3AooYBt?emo#+Iw~3K73NU zn~X$8^Co@CrrW9ThjsnSdW~I(&a&_$hXm%6N#B`j8Z`+Dh^S#NWE~t z5=C~YCw)8eR}y#Aza?@t^NJceBm?KgDO!2K5#p~hK-FA(Sll_18}a*+@VgG`S!*OW zA+F-yv!_E*NcAZ9oYq3~X8}Lfb4QmgCKn9Sa_hW?zW=d%Mo!1Lh~MTjb@t^VmU+(CK88q8%PndfT!BQpU8aHv^o1b z00nwnu5^=f87eH%9yKWqVPgn2=a}IF+(?RiBGF`?v zG;@!E3aoGg*3t8-p>7d1S#Die>ooZw9Udbwgl=tl%ABee$dvB7k+EF~P?yN6(#6t(R6hzl3|% zxsN75uECcwa5%+k?##DQXccpe#F*hfa0>=G-Pb4Pu(*Th?T=ZMHH{*CT}40_GBttP zehH?rfQQMSa(AVib)S^{mg56AHQO6Mauf|fe}|k2>J?f6cZ!W0r{+mpE_W8N9g#lj zs!sw*9f%81)Bj6*%&dTYVJ9)GzKPovskk=aTi3)j&u=1oh9r!dK1yA2XZ zJ%pc!%{dI%S-z*_pcrebGaGVOe_zD>ekr#Lc@B85GSNII+_Vko=4W+EeBq%D_lV|# zqn0_MVt1=UMR9WpDiphcj)^|x7`0JJ;hF1z6&v0-(Z(ld#t=^S&gPTvC_=3+Ad=z+ z-~Ry2t!!nD;&*$qLqkY_B&{b02|a3p*a#9#$2x7Y=Xe94XHyEKl4Q?1&`5U9#3^SM zlQ2h~@SJd#_(>aa826yVs59H2%{JD8c1IExNL|Bb5Foh%k7Sqd_XdQAlJm*+fOhm zt!rG;NHPx(xdb`BA;nyWJ)l5t(=qZW3~(?eW8SV<;~NKHce<&pC5Oe9-2N*A!E};( zh#w>dx~~WBKLw2h)u%1~u&jJ_BICCX3osW|gaNaPps8`3>)Nayad@h%(CLD}dp&89 zIG};pe8Ql;$TSlRJRzb(fyzgmr(cAQ4NTz3;$yc@#bg%^c4nhxJ^8AKPtgq%yq}V@ zSVh-7EaNyw+*m5IyqO+#p^B~A*!B7glsU{6fDJ^IM0H-`ALj!IY$hM1vz!tK6ByW`F zmfL3GAQOu?s-YMiJ5-Bj05r_vQSb2sPBD4FWlw^L?)wpm;~6PWX>*H#gGlD$nqe++ z6H%NhnQluiE(98aoEZfi&pi~CgCK+<03irKA##SH_U;CUIlcgCLpYBSRNj`=-9_f# zc_cF;PR<^1o77u=uV3A^JS}c2{Tx+4?JkbpmRPn}$pT5|KLy#f7^S_P+>~IFO7`Sg z2twj;5-LB-9c28ULL0;8vPm*WK1c};4Q@_sb^Eg=H;g+s zAAj*zYjIWq>QyjkFt^WY$Q*ZsW*l~AoADZ)BXm`0F+*?xyUZ%F&5{~NYrT0O9B6yG zO)=U&aIk>m0s!7*`+!>j^q-5f%qC@^^k5(^=LQ&UehT=pL4bG75jz?>(|Mi)uu1{! zcm_qiNuojeMoKWV={!!uB=cp?g80bueaKR>evu=#eai+X7!pSM4|>cFCP{R8@KPd0 zXW+C9@UYYm(Sz?=7{{LrW)mUc95p+Yi2?wkgfaw>4>TZ*J`*E%ol}nvwkH^GVXv)K ztjFTdd6`re7zfcV=7iiirbB0Of(NZz!0wNSX(X$)z)-q-2hBhYE^}qjI|O_M6%HOU z(XbL{wL5mP4Fk_$O@i(qKu{HE6GA$~tGSLGI{H5~W{gHdfQ}0d+)nXAs0_KUq#5g! z%83cpD1-Bw=*Y~PX)}Ib+Y<7~G6ePG;cko&;2Gyjyd#TL|%&C_gJ%Fe| zwig#1Z6|(8&bRwj<0|2$&h&@9Qtb$W4(wSd3^+i53(ssVcXt#guYxh#YzTc~N$C5P zySBSB+&zcns2eR|Vew%rB(zMl{>rm^s)ETKrW1nGS|=m}A<$%k+1#8y5k4yMJX2R5 z>NY3OX-`etcJj|?1sO`>Yx9BF7CpoW#2dIayz5zW4KRe^1<>aI04(;XL7^k1C*quI zN%1v-@xLXm1?EeGy~;iqaRMWZ8Y%~K;5!wcJr|pA*%^gE><1u1a31l{#t9q3fNr38 zPf(dvXmQDv+E1-m64GRQvRjN!WCOVYTp-X0J*wDDL(KcnI-s=RXy%e~h*vBmkVA)J z5UHr}mqz(j!U!S(J9wu+!{Ed3Ru){@H2I-MKb9z41aSsI=gB*1?GDVhWPR$C$2{PG zQG^$qxs?U(JBMB-E1~kfa-Bql9y&noVjU0O-O91`Op;tQ~_X9`%-N-qhV~ zt9JXxAEMAtUhdOD#dmr?%TV6)SY-WOv;P2_hrzt-QG}<5^amBD(&{y@iwrLGj!hk; z)6R?0IXm+GoiGl*TYcHYe=Ku}cV9ADw#Hg$V;UIIkQpi9z+TwUjB|Oiq*vhwqf9hH!wj29>Z1iXvrq)c{54MEsMtSdWakH`ggU}jFlPowJ}B+Hu6WSmjVhTQ`uu7w@$ z%c}2vRy}}yBh;>S7X1R_y1wj_Wz5X5p40LPm^Gy2KJ`~PjN_hGHKyyjI2+79@|-Nf zL0}HwNI+Gfocu%G)}9S$^aoO8+;j6QLq`&z9l*}nO+A=*byWriMDqG|Hd#j4O%5*)c9tnLCqh4sqxnx+qX3 zjjLq2I53en3>Ag2@i3@5CPf~O*Yxm#Qni!95b1Yb$ex@zBUqg4htY_5iXG}6OwZajJpi)qRcJ_ z!~h%iCvKkyfl`c&1-lf0#|BGG#}la1aI@B#w% zz%+>7oWR?1ONTOT*odH>#6zvdCiAR-jCC%i8Fw z-;ppj6_R_&Xmfh}`IT{9#WsNWG$aa-?7C6ot?M((gn-2hheI_4(E7PCR zJ%xF9<+9JWvpikf^In?lm%4}B)@c!8IE6<&t<0}Uv_8z!#lf{HDC#{LDCNb*NhJ;G zIvW~>65;Rjs!`2y+GFA=S+^YAkZBg1P_$5})~~hHCkgZ7rDlf)tjo&fIZu!@n(25Q z#nWE#fLHe0K2JJq2_EG3F!8YLQb|h`Xz?9)C2M&&&DP5%1;*8!{0`#@4^=KU z;=L@om(cD3z=r_fL=bW#cq_s=OGoxTRMc1k8;H4sh6nhX2M7SiQu+01Xw4aDP1Cxj zcLgmr)(Y)TFEcU-LJ$HFgaQ+sakz>ZCY%)eH=l6Bpum~q!9lpv*APJ!n(PU$}-L1p&@;%hb&# z2(|@F9vd8EYOAfe$E71_DgxM<*tZZgjMHJf1SbWO4~gTjC89z=HZaJ+quVSSt=VCD5a+1^Cv|=5Jwms3{U>nmf zxcP%$f`m2iFWR-Na1yI-zV zxLY-ndec#elO3w$v{n|j?3kET7HDW46#M(4(S$lyy5S7N;q%;!Ol!e7z$39|#1Xs# zg;L$O2g`lw!ENqzk~Vq$=y?r>hu63`=VVZ&;4+TQP>@Y2u~>Mh48>KN;`umadG#lQTH!JrwYMUfAQ%4tFP8rRa%T?( z$l81>(=J|Ryxr?wMae|Bl&ui#`^>5STbF+%nO{SP`+ELJ?FigZb>qQCB$|{33stVt z=)~W=%X^9O9>$4r>I=|zvP8AA8VMu;#%R3zSNXA{!=<%|`gdqLn$tY)9gB|Da*nfG z{d>=No`|Cae&7MUaVTd|>UaMDGIDl};0L<j2HZrohp3g-XtnxIP`BP`?l$1# z6mXwOrlYNV;-zWt-qPF`Y}|FXjEhVj&Esgu@l=XO(nUzMseQT=h-~}Um-1KV2EHwF zgK!!FeS^dLY!XLneUQBd-Ns#GZd-d^*Hk&AI3BS@4cSOmFKf(%BdI>V?}Fa{`WSX)mLsD9g$EuM^E;>CF@ZxEyUm2ZAU&`Yn@2yi#Xg$`DVQBX5P*d4=wtx* zpL%N>_;^C7Bo2>RTURchS(LGKQm5Xm*1g6<2Vb0{78oAknsDEWIAEsv_)$)vRn8)X z!>}gIoT!~^7~vpB^HNm|0~IRd3N@IFDT32lD9Jv1Rx(;L0W{+G9+xtJ(`=5E9m>#R zklp2DXdo)wW&w%B?obytT<#?N6mJ7eVZ?U-02OB3#zr?`VDrc+EjArE0YAKa{zwW3 zK>TD)u2A{t5Uft*|mFCCUJ6pqIZ8gOjn<_?cxXXZy6Q- zYsdS`x{P$D)!t^(x!!+nclD3;tlFCC zcY6&8=!2cY#;^XDQRI^@?;PTO&#M0brPO&PURQI)%TVVwmwGO6n7PLZNSN;ZZQs^E z{^hwsoL#>z?_r?}-_81sSN$%h$tPOp`?avx3pv~Z@PpXHyYu+WZ#I_rPP)&Bs}>U@%`yTWJe`oH>JN0LZA<EDHv0eNowx)8W8&&M9!aAKl@!zl2yfXhM!h{ z`dv?wTF7(Wz-Za?cUezE@m_bTbri~W7x!$KGF-iYpX zasL3d)OjQT{{X4g{{YhJe3Gvw66W46bDa^=bDbUwo{=dDw=O_E<=lmP$L#G_MKqKA z^Yj`K(F1aSrK$P0J$x)1^s1z z;$8lowV6UTZEDKc+kc@PY)QEeGln*m8whckJK;T7D>m2$Pu#fRb1j4cEhH-9aVgv^ zoZxXer>mN&@?G0sZM0nsBOOgXgShbPOFF;Inhhx-jyi`97Wy;$d2>qP z^=`#w^x(OL^f-U9`ki9zj=Nd;lV7R?G2drq%Y81l=gkY5E-oavl1^mQp3+qU)#w*2 zwKiO^xeSi0g%(tcc`8!rS3$~pzD<5DJ<t?Kf?KkEi?@5ZNZ8O>VFY`W=U4@sU zO?1*6f0pC>uQ8vZ=fVd;3BMcJt96xayYnRBlqoRU_NMBC@2i z_AK*U*C%q$i^mOdd-+#CYyng z)RNa_`G(B0&GGb4cJ9Yid%@>iV)=Cf#h@^Ny%l4jm7@dR zC>k0!=Ba?1hT=TxGn{c}#ZL|_V?$`UG7Y#~XG#KXyOkUK$9Rb;Xj)~lMvl6rz?NT& zHFif84Asfxc>%{=eE3=h@~BJPJYFQ{6z>9wT%vP^BiN@h(^%BYQHaP!LdOv%3=N9F zz=czi6t%$iHySibZT4=47d^mkA95`?hlOjdC8{h@2qO_HQySUv7zwuKw+L{q)wTz2 zI?J1dp4ahIw&vS2)UNh5fIw7m6j|I-?$rM3y{CuCbRCpX-*Icf}0T^6)0!b>E?>OfJbw2eX<3kNLCWp;XEpkyE z{<3I4x9V%h`r38p>;y5VaQH6clW!e1rt6y4I&`-hL&heMZ|SoXp2I|_HCE<6!O6e$ zc7N?%(!3X0(k{oF$z4C@$vnA!eIDK=bd1&8k^cZA$-F<{>}~y(!*c%sENTZRzUjB8 zd+xEWaec7rhdfeECy!$77-alSUhGYgNasuZkPKfG`Z@l^)ye+=d%gZ2Mf!p3PBZfc zt&@h8vxU8b;F)hP>Y35CrEK~(u<#>;gywTq{`Y)S?D{X%4_Z2ZidO`{Qr9fj<&G4+ z{-Wl8zF+?U52F1*^{YFV^-rs;>0Cs{D@ECvJtq`Y?4EpqC^)=>Gtxb#htA2>LD)nNk$&)_lPsod))NmX#Cx-SJPe=&Anz*dDc? z-!K0FhtYnZde(MXyus9R!2WHX%fIR*LIR3;e^maJohzba6{6^qC#2%D`{n-t@cJ** z4_e&m-dKaFBxY4}OJFdIR;LwFV_+HZT zwvOPjahC(z6wR(R7@L?gSRgMpAc8^QVRCwHP5mqT+czC;sU7R! zIpxlF8VQ`@PipXSDs?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u>zPKBM7+JzQWZXEc09 z!v~fYK=f#OpiO!}4_6o;M#JI&#)r`(MI7c&7>(|K^2DbAYtSCn-2BqwRIJr76R;YV l>U7Frf*61H%$dC)7OiQRXND?G>+l)Q=7W-CX(Eu&0|1#=a5n$| literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000022.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000022.jpg new file mode 100644 index 0000000000000000000000000000000000000000..331aa6e3c6472f4a606d74ae9d10e58e3b4998bf GIT binary patch literal 27977 zcmc$_1yEeu)-~F=6WrY!m*7sY?x2mkySqaO?jD>3hu|)aB)Gd<0t5(d0fIkr&bjxy z|6l*RuU^%wdaJ9t_g;I=xyBr8&armw?tWf+{tmzb%YtM9Ft7js%*zY#`~-NVVB%@U z$H~sa&c(?IcwPm(0U)EIqM@Q;{ME2tVf|0T!NI}D$A5Xp!otD9!^6iXBqV(4c=;eE zCZ?buBcr9IV`1TW?gO*}0I+Z{FAebT1_uidgMf&H{HOO%`}c(xIB@U?h*$s^I9ND1 zBxEFbWH=-Q0Ne{oY-%_hcrHm*moo(1B$Hs8y8K!0uhYp8H7No-MA}VPQ=W76lsBpM zGw<;m3f#=xdtQf1({0h47lzFBUSI=YVF7S(=&&zHUc9}`2Lp>ujRVI8FL`F7ngj{1 z%g4oYnO38b;zro~dT#nYndkWn02>*`ANCy_3=IGl8vut5cwPaZzZk<}zZ{DLZj;%H zk?_1D@r7yJ*Q=`sL2kQReE}!0_I32DDw~S(QAp6i!OX`T@ztMa-=bZJLQ8($w1I$3 zWyOwF+YM+2NE1SDKMHlxOnlgzsEO=46}6~ZZV&qe|0`HKnTeju@b*&dPShgN359Fq?#d#?<Mtj9e zTWeV?`45_gqPEM5Esg_X{i>X*)W1TBE<5EIyu8+&&t6z~hdYB{DnBFb4c!|%$-8J` zOp>W;Ji7Juo0-d#A!wd$nEaJ4=HV}sOFx-wgl%Vbowxb-p%v&4I^Cr&e;@n}lc1TC z_%FN!?tfWT#UlQF@Gor5(nMfkR_5`wVax z+DWWPPxrmk%HU%oMG&i0V#8;=c>_28pYaEV@4ltCV#xk{-*ckh<~#q1v!q-Vg&7EaG{u{ONdp`tlmU#4o9=W9;Z>1{N>_5i8GpZLNDHMftv25vGKWn z1}cZZ`UAzV!^3INGdRWzdB;PtPgZ^^E2vi@fS6UTYLh>G5}|tMUQU6F{z~*e*XK^T z_pshgYNY=91n3>58_o(uW<-@dixSG63`;eV%V=ycHwuF+BjS0Y!tsWXY9sHn`yc#{ z`8(SmwFEYr%q?V1$77aLf#CL3I(i_80?iJH9|9lO!#7$w8Aa*a2nj7DFr0~o^mwe3*hsb>}G)`{VY;}#@Y#S$nT(xSz( zHlN&b1ATLfc%sEIgU?i?2Z^@KBaRVxZPonXrTL(&`VaMsj!8bKwf5Oj8mr8Z_j9myFIf@WuSQoZlKxi}{-Y5` zgkzpKO+{5E{2a?oAJpF?L!{;kH_xT4UstfZ$x3Y0$9R2qBS$^{wNMQN)x?@yDMY^~ zq7eU2I-q&7y6ac8?ChJ85OqhqdWaGeR)>KieuA2L^2-zl-5 z0knuFdN^56w+UxrZYs%-@hiG3?QETb7EX?XTb>O*&`fB?BB=x^z+v~qrE`2a{=>Yv zBxw50hp@9*4_iYrCS2_84RMoeS7Oq(`*gN^PZg*~PceIWX`~c?B7di^%bZ~vL)KFX zFE3s30cduAPn6y3T*-{rM6*gq$%qIDsl#W+Z1o35od&Yq)imf#7T2BY|FXa5&Arp_ z?suE+S_#_T%SX&FP7^WY-=+<{DK0xvYdQZFy3=-Xl`XY$9MdX|U<%~A-@sSSJiqjT zwyk#3e3u9y$2r?_4 z@6=vY2G$&Hxb55fzkQjTKj%)566cZh_?SOo;BIkwbz%0H-C!`)>?+9e^5a{xEu%uK z<&M3Xr5D$$D^8SgS2cBqHrMIw+ap|e-3>l}V&Ir=_?JV6zuhcdHQv8XciAC&H2RnE z^0kxTuVyj&JAToU?(IM$vzLiz|8IQoJ_ATE?kSo-+EAz_%s*yL1kfi$1;z~?E&L7J z@m_KJysPP2)In@U1WH+c&-*6|{7Ko#f063w_dg?c)_g3*)voRSOwW=R_wf%?W?OCl zTR5X!cL{%6{|V=xiT(=bp=+?Jc;#FK8qhI*sClv?Ra>$zsSWV1C03MALUMUE^#>r^ zAAtY3HY#+o=?LWCdBMlN7&T-hXhi^x)>!7pcX%tHw#C=^=3s^&8Obhe!g*XD6}638 z%hv-PSTz)@u5_H`DYm#=`8VCme?z9fPvI)of^c$|g`|$2In95AE<-f`DsbFN?89JR_rI9om~Ls@5dVwZbi2My z{qbFPY9@o8qI_^`CLES-Y?^5lUDh0wzim-Cv$;ac$rOCHMTEs=QpTMcrpTLh9H9JP zITCmVoCtPzJnj{X0%7R2)ebfzV+Gtva}lbv2#P;{hD4{D(q(0#dWb9kJGOv-k=I#r z{`*_{OXffg8T0Bgp=s^x%bEyuSokoik&*?$B>XFT=<`YDG(X%?AywWG=d`?)Je+;9d3&>O|I`X)hfRj0(d$dSk7%m%-^p%; ze%9DhoO8EJEkPAi;ruOk)O+~f(`e90V7Hq2V%}53hi3g7sw`b3K=t*kIpGR)GT29B z8O#Abc9&+~H)xD2rYgl|yh%0rce%VM%s*BBrz*NRUJBT=LkS`{DG-xO1>O2NYg-Ni zNx7qf6o@vYuc4kkdtCOGsl`C88Z=3M5zhG7ER$X=$;e30uK$`%KlJxlclr$=H8@W6 zK%+2T`Ked0!o@vF*)OX-ADeWcemI1otR~H*NwEOg7BHZDFmjUM@~1k~WZNY2MU z3Of4m_SgT6*qAk{y<7aFpoWZTW>?}bas^61G%ED@BcPdqO@9ff$=@=Z@E3}ITpuU& z=*%Qme`GOS&5Ig@+R52d>fSTY&mL27B`i?(3f2`L)t$Ccp5^=rJ%w%2Z5n#)kvIOL zQ*u?MGd@-c%O86(``&2;G1Dovh>}V^@aAL?y6$!D6}mXC)cp1iNPwW-(hk@C0sZX- zw3aAoZB5T|<6#iXT18%5XLGm7?JG_4BJW*stv{92=Jxc`9db6|{tU25Uq96U<#=xp z*n9XQ{Sn|7QI-{Vx)vAzwQLl2z=4!y{Y-5KiaM+mg*fYZe7!(Gl#-| z@$E(35B^jLyV1**olE$8gb>^!{~PaX~U>3h-!vLrnNFuvK; z{Qjo<*FNHUBIchc8%kKqEUIDeUb)c^D|^uO-qJSFTO2r-CW)YN{QZ%GkF_w574I2f zSKW@MHI8r_nnC%R;%0U9GWW^xtLuo$r+ieIuQY1;Skl>s~xR8yU zq-$9rV)PZaWjn8MVu&O}5(#BR=LUbd8~)!QpWc}5UJB;WQBNx>JZavHr)>`D=5WL5&AEaEdMhRd8l@4m*Jklwd>x+ z?5N$Qpg?v!O;VctLYI{_Jrf?l*;hUFK*Gb(5*r@1TXe#IPXRl6Jow%CO8F|Jy>+5oHEsX zywhrZ7edJmE_F#cagh09pSmV=S0qPimiGNqDiJ;^d3DwvteoA-08yrG9#ZiGm*;Vw zJNQ+F2O4Nug6q{;#MRal$9T?G><82J`DJhqBT5=QC@C}vk1h&J+N-<9!cEi7vws!b zmqG?Hn-`8;d)oUwE`R4p7;jf?|0=tq_qwrmpS3PF$od?PXaF$46&&k3C>d);1eXj^ zHx_{2sw-5qZ9zsVXEfD-5D&~(Q+Q#vu$r@Bhmf@W+s%?u*$1*E@cOtz;@98?!Xy z8BjU*d8|7e(N%D)-z#lmbl(A_ujv#N%PBwnHnUO`(N^f<>S1upTt6$q$7L&LJ|RDH zu&WdQ!cJ|L&DXsqexwAlOqtTw-88%f+oPEsO5!aciw%C^JHsgpslhU`w(r}ck4H#s zXrfkb8eiVoIsHU)JjX}^PQ-Km+=Re`KVyWz%MdC*Cyabn#kg=BfZtzAM$DO3P8qOJ zDEp=&RiM7jt#YN=DkxOg>6*(YIRHor^Vz{&h$}(+mlUh3_m_VvA;1nSAtA4lKJ3SV z8*gsQ9)JJC$dEU;C-1t$7B6}3wpW>@Z?4#Qfa^vuIR-vSOyaCkIIA8l>#o!9Hjj6MCf4z_Gn=hfaAdaUrIwl#bQ%v7k6pHT;)(07F93$|&nfEuHBXx}xS(l(BpEghwbv^e zWs{-WmhpB)VdkU|CeTbJ>4eHt+txQ8kQkX8HU~EBuC>iDrBC3dtb~A&?PZ!cBnyKB z(6ka0;Y}{DI!6>)@k1-M_mAyM_0l(f_*SL60gx5n?vZrJSoZgO}+69Qg}n2y3|^g)L*4Jmn-MjmVyU-N$NL5 z0|GGdDo2d$#aeP?$y<;L(sb+9wYdY&Hxw-lBl86YVHWbZv`goCEK$}4zsf2g$~ekt zp_0)a)7vis?fYdwH2e)5(KSUa%^(9OQ4H8Hxcg*DIt|5Rrin`B;4ImLe*1d7<<1I!i#)F|p=Uri3rrL5j}SERKuW0%ti`<9Wa zbEYwx#L0~zb;WDWz6-O3$)vEgeg>ciMt6VX%5)!}zE|eZXSn-hBRkga$Z!BohM;6! z-L#h|u0|iLUes?#IiaJOV!XENs6l0mV=i1+;9}(iiqR>}Jp;DX2puUeB3J9oEgcH3 z*tR8FrKjOzbwRUkH-e7lyZCI(4LPbR+7LNKJA6J9Cipl$MczP?Cyow%Vc-+ij_D6J z)D=7o?{3-n&$D0m2fQ~#FmY!l>veQ5#m)we*qk@&Y&YjUr@EuKkGhv+%IOyI5+dkk zOJ`n$apo^&rRXr`G|--w>9=Fy);Ech*JFs0@fFpmh`Q@ED%4I~o?)hW8$bH)gCN(z zSdv*vGDPE(J)oHnh^kzP>}>Z(pb1%5LFf2h!u>X~#piiq+_PiXxp)=9t9{A5NFmzo zQL%5*GcPEt^?zxB+;TS;1|-^^=(`lPN!GsM_VD1shPO8B0G{IPn+YD$SHVJ?PSLOd;oG{><-Q%z z-{Fe&eaYLa#P3`QhP5v9 zSPb5|#TX?M?NizBaPKR^sH{Oe%_dJcwc?Vjj=AJ4{20(A&mY10bTFb3A%(oHh%;rb zDzM)ixqgUwju_+Jb_5^7*}i7SGc&6%dIq4{DceZ`-)V?xwV-NDe`SEF(Ke81##?0=|d zE4jdTfzeKU6^U*MoV!20%EAyO@|XMaJFDU*h@)?SE1@$d)@*?{gy<^H{Dpv@|CA3L z5wLK9lTNdqs;u|jJ6pysiVkN?)p{A&;7dOPF9z-}*e_Y5g!@Uv_R2V9(5guCsw)0~ z2P4h8Ag_mhn_uzwZ<2xr1b0tg;w#NGulaWdP5NRsmrW1wF$6gISAgS{6Bb1jcib#- z9d^5}G=^y4Ne%nWZnC4C^bD1TrN8EgkEcNO@h57gJOjs4JxUKuPdCRg-X-BYQL`iC z!AjqYk=Sri?_DaQ!Rkb!HMV31wS{@CegLCa(yEeruEyMcbIfO@&<+sj$hxf2wK4zD z=Dz*<%W*nx;O6SwEZrgfrmVj%Z?IWmbFQNorp&G|Xd1V#{m8FkWk`E}@ydj@_H}Zx zda@zcTr#O-tnWV4i==25zK~@4a`*E`XM|odB^J5U)qC@$Hx{{#o1zunY{{2oSQ0&g zLC}KsN<)UrZSLCfuwubJ7IQ&TqX7C{8^3EIDnuaEuD8}ZGeBua6lK+y@b24AO< z5kEzJmqQYvJAR*DA(i?Gh6E#FDz`AFO3avFQswd8s__iDX{M2vV}2f;^x=;E)w|KY ztM0H5NfU%H!86pxT58k=zYre5lwVR@Jo7!PnOA1Ibx?uAGRPdq`ZL9AyGQjOz;g~! zbi+aiibgBX0LpTk-h!Q=*WUaY_UTGUU}8+?!K(y7t2v*DJ68CcY$EJ%L9R+HvmBA| zdQ}3!-XYJg1cGmc6OB^ z_i*G@(McuBikL29bQ5Xq9|P1x>9h`S)XU>dY)P0Y^epfO1({dQXI`?89A0||Jpwhd zZah9UWXaX+19C6*c;e1`&LP)uVdEIG{KO07db0c@F8Mc0t`W6gUdDLuBo4g*2}TsG zTZJdf$=FE^S>)q=&+2>iRobw3$q(a?*^2?!1V!(MR}-E)G&VHtgH0n7RG<8O9Q<4a zB#u`+xtPC(9>HV7!|fdf#fd$!l%{LzY3&l@^?BO6)+RfK+|T&guBBMxne=O*^uv3^ zFMAxb!pS8cSFjpsqNT1_MB_A)lztU+!_Y+OkKU1z%nbP?datM9uHTxN)#H5r-f$aq z;~+SS$7S1`R!N*#*w9o%qI|)&9A+|RQ-)NPhg)~$qA-#o^ZIw(cpfb!?!6QDVzz5t z9rm1j5Ge^Wur%#a`RpW!%?)VB5{ov}zxld<3(`UQ-L5&|rz2{1I**|p++?Q+OWuRB z#w*U2H`I?%F%OiU3)0OWyvd=x)Xl`0mUzs>7_~$8efgY2juD9`CQ|u7_+M60CBvMr zmwNTDs~6;TQk!v!7bL^d6mJ{}Oq>hy6Zg;~?6JZHdl&8BFYTd+!&AJ;;U>b0NL)rY zjfqHH8i1!h5)=)mMm87hT{31T#z-8p0}{M37l~jkqKqDIQu8Hu@q z2RsrAI??~uBAiP34DeH@#Jwat{GHAo+u8S{2LD^?rmkuf$~kN~KfG_lC!ub!lCHe0 zHju~mnvG1ua_>?Q9CSit+ehZ@__xeHnyql0KovY#F9={txMOwFr{jZDG1T?i76$@> zj-Jco3)I=vCw!&6P;q^#V*mIcr>qQjd^Y4CZ?oqoa%kZN3DfyGrjJTh1hLL9JErDH zn52w6Hcw7UF*M?F{zw>Eg#u6n<=mn+i_q9ZWHSqQ71m}1>s(P)sOkm zPy0$XWBXuY{e0lnFz|uy-Z~y#jSHYuhffbFh_BwhNrW|@+0A(f2)cZ%5#o0Vm z)gTNWOg4hrdQy`NBmDtub#7AU&YtBMsr+|*{rWD2lyXr5O}(^uPH`SqbaAQxN<#68 zW^QIF`F*@fNNN>#WP-MRW+~FL!9Bqk`*9<-T9gg>70344en!L7c9t%?p00X60wdn6 zC2{)YP3M-b$|ITM;rG+RQsTEa%ldVq?cF##IKg3P{OF2xM3T`Xu?n83Ek6YBIkM>a zYnSvh(!9;C)9@E^6MhVh=?vg$T#~$+u5Y1tgHyJ<4ZJs;ZHmDBc;_eD6{Ft#uzXXL zS_YO^*`N`baDGD8vS5BsH|uwJ86;vf%xc`4Vz0bYR#SLx>l|6dWb|45$A`gE{rzug zWLE79!$uq_M~T;zaidSbdgqwyGlk#D=AII~3*ntLixv$8Jf8id{J1W3a8f7=QDKf8 z-%2}d4xRy8*FJ_hg-xorJ8~#6dZp=h(KGl&%EX*U<+w!Q*?TKfpCJCU@taz=A;(?5 z)p=rYRaFF~lnX_6C5yWAFptTej&9T}Vr+75b<^0kGUkVk$h2LR{oD~Tg`%}0@h2KK z%_tfSfGj&kCnQ5uHPqR3`&Pq930z1(-VKZ$XON&>CcH`u!N2F|=J8_}5uuzEPdSpwgeIxB%XJ2ou%Z9hnHhkW zDJJ!J-?3&sf95n!;h??dluPNeyWuf%S^Nnqhx*i1Ap82pfix8<_u9F8l30Idt*Nhi zq@9;`UV~2Q)9t|V=}FFTu9m&M)urBoa~HLtswVp8Bk;&jhAtdM?pg z!UoY6%wB7Szi9}|Fc@R12unQr?h`iF6a8BUbArIcz9)KW$kVx_XE_YNwgIgnc$cuU zCwj?n)049St?i|+qvvSTl8^@{`nTafE~;K}LIrff8lqWDED6ofIH4J;MrdQ~6NDLh zTue1Dv&6D~dGU7k$I>UP0hPtXT$spk({}>Q(o4PEGsbR34E}3^&YoQt8GjCED3MO^ z*lvAVZD-Gsv0rclA<&RGZ28L=Mur88iC@^R&f)D_6hw1bHD z;{DquVNia~S|F?HI+vBpN^7W-gI*o!?}YF1S`lOw>%1LNq>*ea;5`b;GlzljKNWOY zU3wi`-SxjkVfC4)Q>XWw!R2XKx}Hw@c4GP=v^keL^?=qy1vkw}uP$>RsTM#^YRDo*_h4o!hy1P4ot zm0)IF9AAw=GwetYq>0WfvL8m33$*t0S8Xnaa@u#TN@JAcDg(_Zxt|K%B9{(h1Sz=< zuA4Aojrds1?u948LgHgXjGVPd;l_3-WZ*c&Wrl?1OF15(5AS8dM%PJ&_8ow8OLrz0^gTGgF>Xp0;GR{a%E}SZbAbrclz(QpQ$ zkLvmO>gOwUHTk)?_P!>mv5`{4mhY~vPvcj)hLxUlu{qKuv_2%aw*>Asq*td2T$c`Z z$SCJZo1=y|rjiO->WgVY8_FWtrqQ^7%-NQ}eJe%JC5YO=)X`?U;hGqBFEso_-G;G8 z!1HwK?AZPDK%m7i-T8ty4{A$fDLbit%0Ys;S|o){VOE^S>H5$u8AhD}NvQ#Pw9|_1 zqZ98FEp!~z5#x#NR-`RPpCw!JUg4fHLqpj+y+RE@GF}WK-Hcb_|}DRPF$O*GFkSr41S*HlL6W-O~VM-g7&~MsaS3BP$D_y+T#S z@aJ#4rigVg-w3|cC*mc7)ym^MF53*pfxu6s%oy$>ejgc5oU7pRiXaR|8d(OGb0yb= zfk$bP%%Gzmp1c(d2z_{orYKWh2e%*C-EA4drQg)L0pVGYBYVj&+eIcd==&CrhspJ> zp8@RHSD_NeDJ|VAzi-$$SRomSesbowYwgCxzrtvMn?-5?q==?doDRu|mjYG^h@fgW z7H;ObZTLqA@wsyoZd*$@UyMXe&Ix*QMWpC#|6`WMT5yv>-%yCw@l@8hvYnBvRb+}z zis9ga_^ZpXYNtt^HrK5?0Tn|mTU2?xtak+{_(v6&Lc>_X6v!Vng}%+imR$~07M@%- zgBMhFx>AZ8bGRLT&u^*7pG_UO`#*f#QgWEI#7%F8MzvOI%Z%D8u~L;S6ucuhd2o5T z|KO7~c-w2|j!!SmLPUo=t#N{pZ=?8&eHx`hk|>&hsc}=Rx*t}~@Me4d8$AYC^{c^+ z7|}0WnhS9WwB|W^#XejpR>rqiQMnIq#`VAX<#%U%=>Y1Zd1<0*G~!XenRwE=IXbS_ zaPat|DgZGz>1|QN)@Gng9bX%_s`-3kT)i`d%-oA75ll@5qiC~W7;sfdtNhU_nz1A> zZs;;zEvJh4L~~C*wCqz3vvMTB>=GxXAbfB-=;Lid?q$wgSD1jB?bf;eK5l6^1Bl4| z)d>O50{f=2qIKC|D3f}kawx6=X2NP=4Y|zZs7a=5d<@ax0jJDw{b#_on8R@OPAbFW z&L<|=P{zUmKAfgvl~wGLTKp;(d}HX5Nng@R79EQD@`hF@GX_rcBU zVnV6E7Fm?4E3GW#b&tcEd|CD4@rSOQprnb+t5)wP#_R{D34Xkf(1Q8kcL;EV^hN@7 z$jh4{B{gwY=gK15?d)X3P)f-6RFXI;=i5mC&W+qni;>BxdIn&xV@^zd-sFx?`!F=K zHQSo}K*Jp@!K_*(ftYX^ACyzQcUdY_IK80?-rb&QF1w7 zH)Td&6&#tm$_$Ddv_5{dC1`(b$DAG6)ujnLRE*ygKw^TOmPhf zHC5|?)KWg>Dtz#uzB^v9>*|QG%5b#Qq8CjOaVkoMlnLYQ)f?|w8~v!%J9MNBqcI+% z1lBM%LHqJ{_|P@T2uz3!{4eiMC-n5UEZs3}Tf1f@ihV0Y#=2n3#m~|`!*o%(Gq0B> zePL;Ya~_-bSbWr1AE4>Q7G922 z8n1h>vb}L=8i3pnhF07i<6~ZU#bqg#xpuJbIPNc7Qcum_LPgZm>D)_5zawq6 zHJk5&u>kB)Mn)H4K~)X4eJUN(KR3(c9@bw~d95-gbE_{!@6(TI*87=fj^DKfGlR0Q6xE6ax=4iQ=DKb8@@AEg_K{ziJXrDn9u&+P zje5ys*GaSgUXP-!0YH+KbsT&#OoilZdN)hgM_G3laPQ0*ioD89p+l1Ham(lCJ zVJJUmni$sTsb5QQeL76OjaK+J@5+&r-PvVf9J+dL%8W`R0Rt$0 zW5Y=F9tekm=z$T;clI7E6^WNf)wF4N-Z0#2r#gWDx=Li~(sG;q74|Shb{z^sYk45T zuY$O-Vz+A#%cg*;oZ8M^$W7aJ5H~`k-P2gE);nDNu_C!st8WyiK_X-@8-hU$S5%?zQRNr7N_^{--G!nxA)gC* zcXr+8C=8UuNRr{KVL)q*o-0xq@wqN|<=Q~|=@QT|QR7v}&a&3Rvrnnwm1OE&F zdUI)lcOy)ay1hK1es5)ld*aOc!2=T%vR|^(`i+9T{JdL$JIC&aEbgt zAB%uJB3VuxlsM0$)xw|J_cN)!8Vs}tr(SVIpNq;;R@-hcACZxYnh1-3coS{Z!PCEPs$BdP)#Vv;5G6pkkV<$dJ((M=mW- zEy-LgO>cOy?s~ja^d7H#Eg7>jr0$q4Y;+X105fPqk%kysXx<*FO=~h| zzQi~tF6fx*mQv|ABg_B_l7=z4I_LPTWOo8hF;q?9xY8j}%B)RAJBFVFB%NrE!fCT% zv#}nag`@WVyRfL8S@@o|ez#K(K0w`jx~20vzN?^L;3j8{qp;FT(whZEIs*|kv8uD4 zJWUk{?vtUTQ`Udy?(VBFf#WjhtDHz^f>vKNei+Tzfaeo3cNG90#}8kFg3<;fp8=zT z0#P$K(k1R?1AGE8w#K|=SMOi>toEQlEn1vgR#*?`W+PPQ8g zWU!W?z)8iN-Gcv8@_5qd&W|03a;6R!-q6wSN2zVS_mmQIs1$B}u5hfk*Ky#0Jf<&G zhXMkrB&@%y)vJD@omi`ze%yIO7+OiM9f9E9l#3p*$3u=fz+%JnI*FTi&LMA{OzhA_ zHB6KtqZS;Q8=ksay&G7ahcWK@-7%j!$i2l=5&D@WGdVHQcTq~&#>Pxr9=8C=tD^FA$%lS6vkDhEjp<6aY+Gt^;K!;j)Ws5( zJY4hSGlQt7={MIyBYJ5ts92zz{iWF!Xr6;L|^dB zk&Ep8Ma;rPh(7~Mj~D|$iJwpYNmd_@PYRZVxOkWLtHFCplAkmMv%uxlsUCXpQ4?x5 z#@Mi^0_DN7^g#13C;$-M44jpmi$;%RCEY|w-JPLDOL|N0(C4d|fxU06#t^5~=6Al* zN%70IO+>t6uZL!MT^tH3dOxPajjS1Rg}=t4q7;5FtvaK-|A0=W9vAZ2ej_QV*0Df`%bV;`lh5R)rSxreUih)hLd+*}i?mM^z83Jy_p3{=pOE z(q5bz^VOOuR@+p*Iy#p`lTOXD!whQHrz@K+{0tcBS4pO6O4e{gIWJdXndj1WRn^3) zWE?&Uz^lM$ELmoTt%Xqp-TP!|n<%`S@3BL!?R)>!Sx3P1sL}2bquhk@ z`ygGPX5r>}V7k(ZLmR8!GoYvQK3$~%C(+M1{S8i3#`}|RhNRI&`#-+V)8&%$mS50x zvqCi`H1MgMT$CFUcMyh-PpZU?+{n~QYY$givw=Vmh`jL1yXjXbQ}OGJ(AhBqQXoG= zoq@E+G5eYP%RifD$V53y#}}`C$yU1d?TVBm$GjVBkX?s%)3d8ydA?#DsZc|lSEDHo>GAJmbf+7j|&w2i|%rx9vwp2u4t zm3J&kRd9{mnw!k3@WagIl{aw@h6WXF9>o0)=F!~28Ypr7MitQ-DYB!M-V{cKS+31k zwp>I9*UE`)Xdn_gAJO++i^4w?gru3D3|ETi)a^+-<~`iK}pnY0^Av)?}K{ zC2|!J*(8yS%*tIy*jZU%e;h{H$d|g%A8QE~Ukp|7SfR9pnPcH{!xVlFE!m9moGaij z`*;=#MI0jJL<2Dz;iTH^UWlbZ`Bf5w*f-dM8C5$DMQi;io&hyak0kB;gCBoN)uTw6 z|8l#Cb$Jqfj4$w&8F9=(IYv2`4Vga(5Y`0G@{bgK)Uwj>eyU+NcVLn~qeLc=>r%Rj zU8jStUTf{MnNVweu#?x)jQYO9zH{P@>j2gYWN)Gbda~^#r7MJv1-RanG!x$1AXtj3 zin_>u6qa>l-`ZJ^2p*rM$ObA%sTE~0Vv81`UNys~epPu5u??amC;W5y2g9f9p^cW(*?GJ1YZ%mvto%8RHj6 zb=_icCB-#ARy3uLCW`ap_d7Dv)3)Zu3ssX7k<6Kg3pub*ky|( z1GaUIr1)s#RM-TI(T0h-;A+jk)iJ}TUvumIRO7n74q+SKFcnF&)@Yyun$Ut^VSew8 z-Gdqvj(G&#Wv6F2Do|-Bz8CF8(w+jvp8@o9j!XCDr`((65takw*cXC!qbu@Arv!5f zszmz(4hgTD5`!DYj+4?GrINPeqGgG5JwFTO>b1BG2J?McO6|Mpt|%1M39=sT>cAdQ zZ^jBuk)Bk`lJXP7KDA};F-PUy4s49Ksh=O+C<8mpLL78?HXsFOmYT{5g)`zI_WRIY zxVA+Zz0_1XP2UZ}9QH$8&Hny|0{N5af$syGjjx)GgeOajSuo(y#tYxY3-YeGGW07x9V|saEcESo z_C)$CVIE#5)>mO@5No;krue~eL&Fgb^(tn{u4G}%g!=3-w1Mns)Hr=NLZ@$6r#LiJ zX*ORct$6EI1Lj8aM6|RRT;~IOY>;~UXHe4+=v7!+l=3<9Z_J0TSeE>M-E`cy^P!n_ z|8iTNED?dmRhhSO6aaO4WSOJNnJ+f&07163a(9zy>}QRQu{5+^ml!Og@>iY{(RVci3=)IQGk@?>hX zJRr>hY$1?Hw}9$?u(sn&r-Z;@15p**b0F4H)Lke1J8@*#9S(IC~hq}L*6E& z-wUEP7g#!6|4zIobR+i56xOVxVnLOjHTN7C2u`LuFl)R|Sy{iaV+Odl6PG+a>63M| zinIE&ww+o<+zY;i0lt$w{?O5vZA>H_U~FSBS?J9WwR^5W`Xi32}ShwEq?h3AA~ zH}cNP2IpttrX%vg&Rb8&u!f8|uy_=1@6kQVDaO9<9x~&iM|Mmo)CI4Xwm1KLg~X1-qCOeB}*^v6q;wJ3F>O#7tJoGL3o$q8}fOK(Q)cjw|cTVNZi>lM6a9oJdvUcR?6 z9y^Uwil5JBIH%^~xf-)uRb7NxSAK2j*vCBLJQTJrAtE_j-I<_ceqJhFsbuSr9o3g8 z2T&Il&aTSWoRQ2sllxG8Cz!&gcDNHvDt2z2=6vq+HgmVjxkW0JBkYj*LZ2<|msp7} zb;#!4GayVVs>=5naHp#YvpA0X!)tzAWQV97;p?pu$JyJ+7c0>DN;{G?>S6mWdXdRl zCXJ*_tsb+}QHzcEtQ6~eg4*U%sVo&?2gLK8Yd;3dsMH%iE33*MPdU8?Lh%L*g0VR_j z&3CZEqVADMV*u72BuaNw`NZA-&pg}fja4pdSpHD`|&RmJJ2^TGPu5^ghJxY@U0w}n$4$3e(7rW>1IZZ7CpEY z%W^2fMj`8#D-~diY%A;$waOXQF<8xreKLG9NDb(U>~q>_S~6hBiMsAkW#F?Ll^cQB z_j+0aP^i#VyUks<&ge>r!*nUC4MN6#n8!cnw0d?V=3qCf{0vCIqymar)?ya5$k0R> zarg)EQKNI!X#LzSpLu1)W_^7kB9Q#P7z6(B8|p_^G3_zKpN=&EaI$qGNJSuYONr#S zm9=lIemiPfE-R@Oo>l~sh}!FK>$^5B%wJ4ES!MdEpX>16r*ws8g8>Skl1^N~ zxuW^ihzuLd?=Z82_eHCue`;uV`%=s(SmnC+{7 zS~yHQfu_7V6^G&aFkebq{i=(B^g1!SFM|;Ubk#ge6<1ez9Dt@N#5zB1ZzBEjxETJB zQRmQiB%MNjHbF=T3zPO@F(i>mrKE6uPL@2omS?b2ZIv-#wK0r>OK8tuGpO*Kqk{FQ^89J+|L|WyyW~55t693xEQ~rpDj`Zm!Y$W#1q&?TK-YRk~=z@J1Fk0kf_<7tW zKwoC7=f+wx1A@RZvq#Ds=<<7oIEA~CL)3@SGS<|#D!h%`xXR_P(oDJoq=7U767*$zq`j(E1s46dE;_dK_7*Y9SZrlv18yy86ASgYuE zC=c$u!PCQ)V5yDPCR>#9{F|IbYYN*E^Q!vOj+3NXn$Am9*QB-&_&4{v*kXDIW@n_26u?EtTlda3#;s_?`To8 zm4Qdvo8_<}f5o_YEf%GPTVyLfPExp8wnLuP-h8~qjE4=X9t z?PjBkI#?eZe^{LoE6q%^FaS!z~pJ3b~+DS;ocIMBhs!UI*D zg_`Z7jDkuD_kw#sS|A3)a9V%0VW%h? z|AWahUph``sxPDQTU zm`J!iWmONUDW%Vse%UG0SzFo%ULSe|;shdr`|#NJ6Q%1)zkD7i`t&~P7+P_reyr5* zjq~`w0tqPg*B26}-BdnKx19yiv;JZ4c1lK={Ke_Epd=9;@0-Yy&6lGZ>Lrf2K}xH! z^4;Yh33TE9RUU(-Yy)Tr+2?T<6C5?KQ`;1_0i&AD1$Vi`g1t}%kOMJpHP{BHzp<%F zB$K;r``ay?AwVRo!67N}D4&!hZ))!;xQP)wzgjtH>)pI^!3xY-zJYK^7%<0VsMn}N zg}lc|p$LR1cup{?;hn=u&XL;sdUC5Zl3Pou179;>RwFw(4PmDaCp-kEoILN5gGzCD zsodrYO!of(Ek}{06%?N*3VDXtIs?ie)OpAb0&|Htw`;z%-`3(5n5iN(>?2nTn$V59 zLslNr^a+wwe@7(Z8gFq1KUns;$~%ISiLEG5D$s&JB$~SS5>hb*5U?=G5@4Mm)3wcNkP}h?nuvqeT#QfqdidNN?gZpS6~qe6EG_dSRnvXijFspfPIS@nn-`r9J@enR2roq@M$ukp&V4iBxC^rN4AmeVli(W(I==l;*$j79KUzk#bKXVv1rBX zrFm~l&sX=kadN~BC_8_dEYQ=>`W6Q4r=FfvxQt2UeSk4U6;z(#9@mG(q3enAuq2~R zN#pM=YEVd+>XSbUU^o-EybL%J| z>N>zaXmze-GV4rXO=s0qAA|^dPAI769Np6u*R%P!o3{?CJT!LCH1;M@I|Y$%Fv-|VZaYC1IVZk`qEemPy9S*%~lq$K@OUERHI zQjj16$TgDnJNbEDoaZHsn??qBf1gvuLE7MS*Oyb&`TIYU=0hBWj$2GP$9CRO!eP>a zp+jnP-TG~6N>(OSBb9V+)~8NXrpK~2iSoUvQrNnej?h(IpslJRlP9-vF;8P?wQU1@ z#~r@MJSbP_^dJqdUaYFNE@Bm~0t}{Oi>fW%B%c?aYe&=rd!7XoA~sd}7WI{X{ItGk zS2@VI6{4qZ((I!yk_UI>RGK?2?a3tw$4&8REPRPYGnF8! zzprbbLOy8ImsMNqwa5rBUiNuX^b^?0nRt~r=xgZu>qB?cifi<5Vo%G7&KVri$A>R@ zTrI9VjRPy1Bo2vn-Z1|FiUaG|Qmora#AAHb-WjDWI;lb?CG_{Q#{_fsT1yXCp0Kmc zUCQRIRd$vE+Z5{*b!z<)xKtD)T13!x@1!~kv#b_Z}wa`9xYusMgEe^zv;`X zm_c7tylkU)oVKRgQ_Gg;x4-GNgk^lW&J zc(H_3gCewuZn`cq2ZNtGU5BTPe} z2^*W)59gf-iF++-)y4^Guoo`g5G<b?$V_;zSeyuq2@;G=#o>G0TgJkXY!qt`5Fa z9%mDEThrVp-rGSXD;APCRqhf7g=6a8PpT zQh)#|;8aBS1Lb-7Q=u;=+J2yP8bMLj1H2!5l)#L0-3}9r>XF{z+rAZnI z<39J4(1F}fxy-8CbxH3G2&q%I%D@z_okNH7XG)!jY{!Utq<+Y77*dpa+Q#rbg|DP4 zG#x~%*c{4Z>J!9()&*)~hakXVmQ^lyS!605I3fSEV~mSQA{6^?csvn3kP` zxI9hV8y>AedcYgtus0G_sGr#p?Rt_GNP)8wiEAoOi1HtMX(>qY1o8fDi!BlM?wYegd?fcDbFQrC?gPc^D-?8!aT-A7!NB1P@hv^#o^bgp%v@I7JQ`GI zQuLt8b~71^2?B^$?Ee5D9Yp5^ui&EqQIj!Vp!v%_5y#lzr2v85*IwOP0`##vo84@`l(RDB}Zob+h@@X3~^fLc|$SPy@)K?$=9a z6TaG9IF;_-H?9kUgAHNUQB=rx^4SZSi)CqA5+tS!?Cx|i_)20*Om~OXR>XBpY+MgX zmc*-8>OEc_M&dh1Thj z=}$B~JNoPFB!>)zcTDiFnVhuU>}RdRi9YwR06Gp_3&=KTXlu)md5Smte5|hM?x&UP z4jx7ZADBsvXZeEiaZc`BOWk&Y7I^h#uG#^SNvj9e(GK zY=c;i`S-oV6CQU8>Nx5_^0H&Iyne2MfQ4=zK}}p>6HO)_4~v&m*#PKATY%c_aUY@J zT#24sgA`15Zm+%DD!`l&0w$jBP{5@kn;b#cWQ|^dQK22XPZqXZ^66v(yihc*hh;qf z0IOn^gp(8m0j!hXz2v48DoL2qJg!!-jB<7e20L_c$)gH@1n)dFmbj2k?|a0iG*q3V zYl^V}r|QHedp5@LsH>9G6B=^;TG9!H5fefDy|Cs^!L??4jdTtag|*3zDK2w`Q@R+b zNjs*8eFGI3aS$h0L1?aRsO}2yeB`A!qYX@Q+SnH&2Mu1N1lMa2sV>8+9HhRCrHHqx zfhWxMSc&i^H@ecNE|%u<;2F`bgvz;gws^LxJ0@H}Qis+q3h?XYc?8UEjw<&stv|i~ zlln_qP9qfM#n|-uuf19$E5GioR{{UC@a8N7Ti;45F z6XDuP;-emCles7zus!9>3_n`X>`b2LAIv%>3F1jEf*lYi#Iz2;l_>iIm*t~qJ)~F6 zPZ}21(1ZMrCxFF zYPhvb1=Fj@9L38pLT!#Gm$ldE(=#4dK*ZWQtCw3^l#PR}%*)V(5hRnrzCkNhs8}3z z>UgxSeQ_MNHNsG6CE1dZD+%ik0;0mcV0_J}P7&fM^0Y59lcYKqGo+YGde5+X-tx8% zqOfdAkCEBMX(*CaJ;EQI>8>5H&=g6jm#l#7F7>3DCU_1(*H&>PLT&Z?hrQ8OjtH#1 zQZz8#xJ=A&qsxmiDkSkOMkVdz`F&4m8cP%w)hX|IlPWD1j3m6as9AQln`>_|aay?g z$Aem~fumBCc5P&ckrgdOku-Zak^|n1r~#qc)RFmw3dFsgly)AeH$ch+l?qvi{{Ty? z&DO2nUBa}5P;Kq(YJzOw8sLZ;$sL`n$SHYctlmnjf&dD*FeLT@sIOz-_Bz^gSSeRY zncLSg*Oh9)=+gmMMwQj^ZK_FD)31Kl)6f2G<8t4QfJ$m9!j6J6>Ro7#FRWcfbefAw zJm+&Rva4wVG^@4@9ll8J6m+d1p$S3pVH)x+@=2vWX&%?+mebEIrJAZmc>Cg56m+E; zu@E)tb1IM}D&xPt95PJlq;P}h)y8L9^?RO(f%SU@Po1#Iq?5%Wj0o(J@Aj~&1RCy& zN12H@OancU2inC`>JIVlQ^+t=ah~6JY$;StG^UM4q6tXtgYK8UG>XYjY!v(083b-~ zl@JUHe#kyeTy+yD?j*Sg(;qJMurp++pER&(Ap0KGKURNa*V-P?*AM){dW!=MV{c!z z$sizWUR$c;>lr={U%L+zSs3w54IZ913)-bc|Gm`8tm?t%BMkxi2Ye4jgi|v-+KuLfnC~r*ckxoKs}$UidA2E z0(#}=h->6*36##g(+mY+Dm14qEs~+InB~2wd|XNXZG^~zY1bWYfS%6Sta9&rh|I>F zT)W;c5Ap8@--@z9a}v0U?fO8n^ky& z6y@H~D|B%TzA&vZyV~`{reM3Y2EdZ%gqq80Z7)zBvk!8GNU9_>H_9rxSO4Ur0TrwMM;d!6-|B zf935X%F?0_s{?zdbk>p@h0HY$&@U99n6%(j>fl$sOiztiQmEqX0r=LC04^RiRFYc; zT{@MM=J&a+NJj4c90fHM2RmuDN$CQQ1Ix64by(dZMI%a2@@e#qtJYrCDcp%(2P!b~4G zz40Ku1>B;GT`KjM;mTS{)T1Pm#;+x+9+XO129B=RBXtR+^)zFbq>Dw}jR{c1$)l>7 z220f&QL^RLLVNv>Y^rfIV(EN6?=DIpB$2)wCI!pGh|8zM^6I59)||c7*UDjti;BW@ zc8#c0LV+TI=%cZ73OgNjDrf_zOr%t5-ECk_Cc9!rj@ip8LbUUix*~}HsL{^PhUA@G z00}XoajD)M|H1;md&uc^*Qq_mKJaWTNU7P9BUu&NkWwe*m;=avr74BzZ> ztt2HNCr5tegrxyfxz6f3Lq1o9LL<4$vo<2C6gq8&zzIyi<4Jf5lR&Us<|?Gg>Um&@ zBT0zagn2`1Yro=$iG!#}ZB4TVLxxC13dduba0>d|c?Sffl}v#gw>F$AgG%eL{{V%7 z5sGM#pnqmZl(>?jXnUdQ3%3Udjf&)u=OIZdp5UL+fT<^7@0RRxFr)+7EeNUG?Qflt zJ-A1o#NZ(qHAGJ32Z>CKM2OurdZd>mf|xL&!Vl`;GZ3!s2iW5Vq!4@klP$}Y=n`X6 z1z&3=%-Njo1xZD-9yJ0L2ju|xxX?Qk<*@_LpeU>*0H~S-fh3;5{l7Q<8M$<&dBf2p zeMH7JaWx4a^KdWNWO=VCkWb|!#}IC+3eS899fF(RS4ZuJf=l)&tgcqZRUsCwrrLIZ zHQ6NUc$&UHIer<+X6EIX%iTMZHr2&DIHo&6euHbjx%wO;)lq2pK7X5YoZ7F>wxA6H z#FQ)jRUO6)=P4kGAOpHJ?-S>Lq11$33PRGArBqT>0ZAK(G>76j&+v_#vvVY_jau|d zC!;6#RjPOy93PH)4Q$wC`A9o~_BM?so@w@bTXPv(`8$T)>ua>J^;85N3Q2>+OI22y z%y!q$68zOb(2!{{$Ct6hz>%+4xq)biHSO2R;8V3eE))Em1e8=n8h5zqs6VwnRt?9= z?-1CufGP;?2gz0f#uV>7D&|s{kb9(gHEIN>)G+7aH9hda)TC5TaRm7sjGqhy_Kru$ z;&ALzmBxQoz!wvQ5$7Dcn1YO>Y=gS5d?%3o9Mmxly}}PtwQ8Iu2~dsc{aB>BG)%zm z0FJqAxo;2zfP06yHRHwa5IliCNpp~&SWj;uEKpt>L~m$0g_<}i3R4BX&gZ2>9)%55 z2|G7AbdjP!aKUi>So3*Vfk=W(_HqmcVp-+xYav3WUgKHN0PrlPs)OWt1iDhMR@p8p zRl)0~2JS`azO9heC3=@vz&E7a9`wl?f=h@`7UV)ltJdft(ydfVIeD9LT0r$mmG!vw zyu@OV;H$|-&i6Kf-71wph)*sKaGk~T7o9-*!~HI*`NoixPwjiZSEUA8J4H5>e=}&wzXhjz6R35b$dE zl@#kMJ9&F>@Z~Oz#VKC0iQ`v;MvLZwk~E`;ViN3!%t@$y3_>2-qRCMUhOceyb?z_3 z=Ve;ESQ4GVYs~CAD4aDXMb@3NisYUB4q^*-6l6N>vH>W`6ECW+lqtlf#u~x}; zV-SI>s6d#9(Fjl&q9VPlRT>r1V64u@)K*EWO70vkHdh#=?r^vrFtC)z7tOYi7d%@{ z#?>GM7}w9Mv>U{$bvTHFvDvqlNC2w7M_7n+7fCTRU~S@rEg%il99@%ex=Kh|g10?c zU__d=1*obW%XUE$TGlXMv`|v{)>c0(ntHPOH?E~%U^8CdXnt3; z@k?toDvq=nGcAcqqDz4)aLFVHGfFoVNbwH%X-FzEWcEAgNGU!QLyQ&45HzP}>f0v` z9MF^|kI7sK6v|)V9R!p7<=bqw12O?0BnK53C5>SQ#*8r@i#@c}&kZuMLY> z(miDvN>jyJPiDQ2f(Qe{Gu-}-_)TQxvnkzndeT42_oo#(P#=@#47NxGtMjgptgldT zOqt?%(f4<|mq&%w0w2$V`ldNz?|+(DGSO7kgBl|F8^W%DXCBAc8w zlcz!wQIP59o49O;SbMTQx zFM*mj6-KBN+eVcbGLTc$_z=`3C+i2L8)P_(V&@*hkbW0AxRHlYDN*m3a2S-Cq!!Si zni&U)Pr25Rbq0FJSS`BhVDTh=j0lja#4$$t$C`|iuUN?KpV7H8krNy_2USS{TX?z# zfRU${-u4!*(YTT1ql#1phBvvZ4)mYdqPmPQ9pFzddx5w%I?0u}M%O3|=?QAauGhGd z;#R=l69u1`xCP=MJ6Jni0`MSgpF6aixKPAY0NqB93xOn-1>8AwZNz&207qG861*Y? z!Tg$lQ7+;y3rUEs*ERt{#l>unFU{u?t9gZ`M@`+LU#DJWYT6R~#&DHiQQsBr0rb<_ zR8un2+CxUy=|+hL&?wRfmt~nnslr^BplOx}x-@mK>}&>&7qkqvnugN8j@~M2={>92 z+DgVOf?lA*fJrV;QpIZlrCqHPSCU+SC8&`^%Hi7yOge!D#HgLFjHSA4Y{y6rvNY;; z0$y3sU@iu)rc?+i-XpE^6d=*r#TD;x&|I{Xp`{kew9$*WrF)#M-3SM{h&WgSQ!XV{ zhIFKfbg|(w*n#VHMB!oTc8qqbZfMft~dv1B?0-K)5R^M0aR-v zwVkHD?qnpI669^A#>%ibg7l;YT8I`p6T##NkO4QhP>@03{e;+Y&a!5$}Hc@SWew%^+-26i@915Bx9Z zrwq+Y_aB7RfUM$lF=+QBuipFU{{VDUeF0iDXwVYi-0wxdp!KB4`gQl2752lNzNMbrh`Qf?&GDoF$p z#5z|@^l4IV4+#a)GKCtxYE+~d3L}HD)8qBE-r>dRz6U^Fkp;*)K+Bj>40HKDC(ij_ zx$}kl1i}Kb+t<`rhz<19hLvnNiM1TPXjw0?6Tw`$}L24pNFbtWHV7z0*hMYe#IN>1mXk0L< zTS4FsCdopnF!3V^kaXO@#9qo}TQtWmD`)~u|XQ16r z><=k@VS_Y^ID@KHt7#)n*8yu4PZ#M*&ag;|O-C1h5F1@?i~wCuw_Ht9k!t0@T5+%; zOQ3dtM{_EmFz@ZFysyk&sT{nnjhkf&Kp+?n`Vy5dK`+eq+2oec7St64 zg62S-ba1|vr$o^02b4G_ggmn+KrIPP0X(+2!fsS6xulaDp4v^aJ$mx%c_E8d3Kq+2 zDH}u7i*qI^vq0olo1PTWffyGJOK(LZz(L|#e(s@C^8%5*yTKn0C|H6yYVuwhEzQ;H7 zzIU4!HZFK(w$)oo;+YV8L>|}0NtVY8XW99)@jg#KhTYi9T{{#fvPo)u4HQPoKMkWo&K%5HKhdlY!THoR?LLFbfQPdoX8Lp--M_$!Pli?C}JQl$1;^Lv=ex%P;a3)OA>TRTUv|+t)`jpdDw19t~Mp zX$o0Ar_SAI0X!ar71ip|gh;$i$-#=~U6kB_Y5~IyGZ1ML;x7T?a!W&H2u8xT`r!T`H1;aYLlDXe9|! zQ6*QdTTiUCh?%SH55qk0J#3`^0P3-5*YmiP6SKg|6XGp$QTV-nP_Q`Sk{?|AC-v0ojYdK8W{{Y(N{{XvR^lz`vYg#T@2~v|H^x;uV z#n0U#`N4Mg(%z5yi>Lg~w-~p1yQ}Dz@Lo<%@=X2PHc$Q9{{W+C`=)pLCU5(_??(IF z&ZR9k@hNB3f~6@@;JNb>_z1alq#l;^kUz9tf98g6DJPCU96O1bmaXM88ZR%KrtG!) z{{Z^k)0S4L%Vx*^n=$C0EH3BL-u|(F=6(KmTG4XIN|c!wX9|jE$NQu_ezHye*jGo< znSRA^i~DlZXx zcjhNoBM3>Pvt#k}bI61g-&|s|oTvW)ii0=Mx^nBmlglK+e+}}bCmx8+$;PD zc$6cRwRl$A`Xr-Bwl1wBcyU$*)C?ufN=k(UIHd~LW8z@c^uQ4~hX#-g*GFp^0L4p6 zPAQ`U6i`R22eH)8+_tL(1mRSNaK?^?clZlCu>SyvJ;VD)QCxvh6nXp|Y+6u?3Jy`rvZU+-b< zdYiVvsv~$hIRRwVA`b1G47~!?(G+z_0v)Wte2uT9^webNtg>!_qm5NpT@@8}w0y{G z81U#=3oAS0gM)(}%?%P_UC@x}TlK@bUp8bYA0M2e5Mx3^>W9bYN?S+mJ?Y+Lbc6ko z#VZ_LMX1SaXjY~B5$k3HQ2IoG>4b*H2yP~_NpUoGEg&ma15reW71oaquQYnR36#ts zq_=EnQ^BGH;c-fdEL}))NDh#gE_jPnJ2uiKz%K%n(QuTiA68#B;QYlN)~g6D4*F6(taAuD=(YOtR)JZqLM<>i6nIPH5+u$(fbjavoZeP&41kN z{{WSxu;ATE_a7`W{eR;M^X2LJfAK#~jLoV40LXJ^_d6fuejlIt+WL9LS52(7#oo4s zsIJgeLY65efbHJVhSg?n+Bh7g`hH^uGXkGiU;1}nng0NKyZsv_{XqA&qo4CgoHnwl zop9R2G$OcECsOwfI+DcMT?4{64R`$Pp%ucRI-a(UoiywdK=3BJ9Oc(fK41QilK!B3+&|wh z{{TnHe^5Q|pQQRqdY*WWn$HWr;}3zWO1b+raxvPkNd%gW1CH+A6 zzLV3rZ3nC7q{46i0H^#HU!F&*nfXIV=ls$=TUk{8&8#y*D}_TzFxUR~e^0yQzo;Jf z9-GLO?6MbQbM!V)lbX$`S`5O!z=z~N-tYeaN6CLsJ@3);MfGN_-o0`$F8L-XJw$IM z^OQ2_%N~O>*mbW9Y13}U3wPBquVa6#nupCkvEuxPBI1@!S_RbwPLEPc#Q?yt6qSmI zE{=Bx-R`8$gRB{B8=Xwh{F|H}qy3l42bC^C!0OzG^-h4=+5ltFAe6~=aE6xstY{sW z1UjzJ)49^IL_=G))TStxh3mq=1Jw&=t^~?dt+e*OoG57g^^fBx_M!LEOZl6^ zsozP?dr1WktQccsMcqklvM~it$H#$zs9qO4iVIG#T~bJFwgRYkJb_qMx5W3OCq`XJ Rw6YT}!hMgfoO3dt|JhL72-yGt literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000022_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000022_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..7025f38d062e16f56954073493605ab93a1a4a1e GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTgb#YfqY(7_cB$YF=9W>g$A zeZj&9M#I7g#)r|vT^!wiFg}b%4~LN|4)gaYJsJXoCj_PdYtSCn-2BqwRIJr76R;YV i>U7Frf;jo?nKOGqEFf(=N{@yEEz$ufNtPx82|WO(Y>9UO literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000023.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000023.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f481de76c909859f3c7abb7afd422e3b0b5fc2f6 GIT binary patch literal 30427 zcmc$_byQnV+cp}UV#S>nZ*ec~6e-q1@!(KggBCCD6p9pgCqN-U(Be?sB_vpJhvEf3 z+TZs)?{nUD&i9@_&RS<@C3|NwbI(23ecjj0%AWl&|F8nUeg#g2uZlYv!6b$?z$kr0d~3fB+N87da3GiT=?C015%%VG)3X z^2edaYe~R$5(gxl*fWAeG~abK+qORvH0fFIZ;QR7Z^%(4P+S{Jfuk10xu`=4(c=Hs z@$O(%NTK(KlG5wfY%PXcVLAr!nO>>lfa_**qvts5CBNG_L%Iw&(>q7Ge}UcEy0S2|Dihlp)*a8jY`-Calg>;*?o&JR z)_)shXSC(V6K)kBSyw~t>P7qf~2Og7d@ApaeKxHFGvH|Sm5@3)6t6@&p2M+ zDOb1N34TWdTXXh|TU>f%<{_=O{>R9x%@9Qsi-K_foi4b-r@n0{t&|`Y%}_`l{^f5BYOyhew@kW zZm4p><+KZlom?2vpK;fY>-&0{f2NwMPe#7i0K63Vi9(Oy!ksBh3_Q*LYe69rH04>Z zGvS2Ggs5#X+ej5}a07EVflz#^#3#bS=yB56M(VU?f-4RekJV#<`)DJa&Q~>wD=VV8 zfv#mVQam@b=ZkvuS0%Bn!pFa}4m?AvdJj*XoWkWOM#6c2_8iy?Sqhv1j z`I~zEzBN zwDn}_H zE59w=Jlq-t5O};pi)@1FbMmj&UGjozSswseHXJr!IQBAZb#!fJCz+&{LOG^e+wdP-10qeHx{YQGUe#~t{55YqzVYi zFuLkK{z`&MD;DsVQ}6zwLgEP=5okI5kZIV}wb``9!8B?^FON^l>5BB9{L>f@wEB-m z#Pzfk|A{FFCi;J}URmb<-^NOa2_@(D9P5sgP~_cLSNMv=!_R$s~lcW!u;W84hfc6XY`#%8E#jKG`VO~LO$ zxYmPtq7}`34*>1u69?(fp(CDMtKa*j4daLO$J(ZDbtG{ye$CeXW9;3r023L;o0HDU z7zvg)j*dk?;pbNVYqO2#Y6V)RvsHSIIpiFmS~3wlFQ2N&EhCLMS=705n(<1? zQisFFAM2EyJ`UgB=MjkKDUci=^F`>D%|)g{w|_+ssO&b?DQ_d8P5n=A^A})$N=Xho ziHrV652p@<;=}HABDORyIhSRJ=>9^{R(k;a&F`;2>vs_U=zGbc%X8eDYJ$;{L&Ns) zWSI+`JGKgF@*TIR;iG{Ez>B^6Th$@Z@vJwAUsm^5^^amQWQ~%A0w;HdJb%JmF8XT} zQj9xp!r%x7VD-N` zxm9CWZGRj{(czKs(>)IJWtY(w=JhJ*7Pe2{+;@LU1th@HTj(rhMNsD)qsUR*YA6Is z5EZ-H!VwMm2lIeocaE_k?=a=UoxE`FvVV(|ri9`Bb zR0`|W$0hHO~g4-VKtNK&RqM)-2pzDwC6oU-+%KmNV zpV0ny_u}u@D{j;$eKpq-C=Ek9@W>;&!|6KT`oLWdeag3lpndd8Ww|Bvza9YpN7Tg5 zV^!fC6COh~!nuLlcrh}#)R==3bGo*b(R8T{-$7Y+%~K&AYgg$XJcc>v{?Cycm9MCT zeC$JCZhAJpFT(+|NspPI&DwjayE~M*?%BF0ff7eLJ6CZ0YYXD>UxWHL9saW0&xj#Q z8IJH?FKgbztug#v+|*VTq(E~nAK>9V`_+G*0$x9a&P599t+wmWv;UInhCgq?uKpk3jk~X%0lIHB?>C>l zZfTp1QN;%o*raT;K|HHbepP>)HA-DPj_{kjb)XuOcBEH75$cNnAHdNR8PgrgrfoqS z!)&W;Y|LHpZ0yO`X z)r3w0RRQi0>jAtY56Lhh&U0|XU88=B57F}wG1CU@ny)0dz5Jt=ZV!OJD)EO(|F@{W zn{&^ANGez++^S6RW4rFF_b{|7BIlzzzk$x)?8TdBeQbco*39yo|H?!$?QhUg|97$v z1r!UU;you*69xX0Mr1e3<^SZ-58+Xt%&iZ*$t3$(H|pBsO4Bysk7^_Uyw8}qJBTyL z35yN|9~b|LVfuAEd##VWf8uPbVq5VaSO@Y_26 z{X_iV%>$r)>W@&Wy*9Sn<>Gy|-b3dTFu3>t=$LX{`putupE6T-f1i=IB>4cKWF;|r z^Qc#u^FE)~6WVk(+AJOb8vTDTF}QYmOkH8)E4u)j9$&$kHR0*Dfk#cN{-f6Rk4nz( zZ}3M!{}3?e|9y?S|7RP>(l)$NrHRzS?*w4Fw<()^ELkd>+6v!p2 ziQZH`oBE65Be)%ZzHGl@@FyySW6ULujb+{m2(J(>r)x>$%bPzlBEzYF$I&mu z%vhJ2s=AZYyWy}i7z^avicmss=uO>PmV!;X?ILU3R9lODPchubl@;dhpO7Qr2_`M0 z=2KAb{esCL5MAba&QRy&|FlA86O6&k% zQIkC^y|3x?KNO9OwQWV6P8TBV_$nF4*`VG0q8KL9=`Sfw5VS1sP+4vZx8(4=4|&-&XruWw^e5h6u0BIew0l z@x9$w&r4uQ59+&TVq=UIH9Mp}9hiwG(^Ph(iinw08kCC!=EHPo5?(Z9Lrj}>GKV?S zXExpSW~8w!i(v!IdDy`%{d~FX`n7K{wG7M5ZTLpc-iXveTRY4UkrdO7nLkNZM0bd_G&dN&qK;UuHMHU8qMagmnWn*sVhrSwVu)q-avl&9N`^F1#b!aslVQj>L z+5_>`W@76cpr}7tGv3T`Bvi!qbfF2dR!u&ww5m0hVNo&}EPCRQ;V{_lJ(UnjZN`6Ece)vPAzB|0d5lHt$s_n5_Kl*F(msYfAOD* z0?5on1^yEfaXWPRIfV^e98LH+>#xmJMmV z)@pLqd>K{-jII!NTtC4XO+#@t!U_FP(251sdvOH{GfWL+O#`#__G(s`jRT)uq8&IT ztiks{h0-oge0_@bAjQOjdTK6Fi|+1{!ZVorTqXY$hnx4;>$T%tkt#zj2+WQ<0fw8hDMROt|t-(qRQZ ziY8jL=D~vdNN`l_0&;WjgoysTr7~lA-|Ns=7lfTMOP--e^2*y!6~b|jCRHpi>Ql_d zLpmJ}2iV-$Ii7^UU>R>_VW3mO;-Xf)DE(%f$^IVSixQEU+@*q9Xmjaeu+X+9U7al5 zYY$aEQXcGgn6mhgzd4`#@4Wvj4Uh8u@`;jg;=7byha3fDb@c%zgkPhtq}fKbs9t)b zR0$AN$5@nyT~ulDsDMbuQ^1?T3sxzj#BXr%?%!g2U-oA*1gWyc|G%}7F#XLVql}uLUD;5=}vYT zTG}6E#Vk!t^RysVTx)7t{_+I_--7+rcW7AD?g`H*qAW9G2CIsXEWAm9Phj<$fsV}%$67X1qQ^ttLTq!Rh!X<; z^?szn-x(Z&iCHFthO^mh-(iQj0{kdp zTQpR>xgu^C2l6iPW=f)SU4`l*vDlIo>5;b$%gmG~M9q|H{2@B<5nGwG& zX~snEdofpO&&)i(Q(0%s%nBHqmN(i?csqlDTiP6v6(g@PSO&CQj0fLmydhrC$ww?(XMwg!n*HKt_qy)2Pri^Vl>N8VPPITJZORrs( zlnM_UM0GRSHTAHTLhu_1u$Bm9GU=qGv6M;B>1<~lWnOCNhWwqdvELRAnf;vyY)R<; zNIwTU&RsNquE^{mMr^0B^AwfK0YV2u*=0gyuPStuGS!J_yb@QnE;C+~?1h^0@M0nd zot!Xu;8P)Vej+SS)8lKM(3nW`m*cC$evxUzhI!u4jTYk~=F7(nHa zA$~mnvzC~qMP+9c*{;B%D%W{6tTCB-RiKDUqusLnPP%GZ zyfalbMTx@>RYcIt;{*CJ$jBFZ_=3Pibg0}s%eOgausTuj%r8Uyc*S`5VBC+*;3h-C z2JD!2HHRww{dBO0(NFJia4gJ5_3oWrx9Kq_^h#a?onJ)IGtx0^jHF2@aSaz?j?mUIjz0+U>@) zVB&jH@#$wnG1Xib$~&pC4GsDv)>k-Q;SpFK2*ZjCR)qL^wk&OXR^gt1!7iMs zVX=uZ?%1+1wXK_XUlrZUEXwlrSq?S@C!9p1t22Q*dPb%t7ZyVguk|Vz(~cLU;7{q^ zR=qQxIRqK3F55RDuTcf%=0vacN*jmixC#8uDZNrONp5uZS}xK>J@0PAgpv>Ep| zQs9Z0;FXulsu0g8^*fb!x|wSgTOuRsyM|1C7{VpJE~LGArr*=jjtu%oy&vyCUBCMM zg6pPeEw&5K$1J`;hR#`}T5r}fvaxcKO4n{(*+5tm?}d^;X>sqi5-y=4z=!hvlWn9tSiQ6s+2fbI(BZn!1wRcdy!AkmEc9iO+Go{utgJ(K?fqh0-2Ik?kTyTKpGuKiZ4@>G>Hv4lfiMXkL7ZCDU3k?Pzq_JI@Np zn}Q%{`j(;4s?sr^Z?U(YTmubxq7?7bsM;S;(m@ zS>r=6?w0J{Ch0l1KC!Vz7q$ACX)m$_d=-s;?fBf)vJCHVw*KJDz}F+PR5vT;*4r=o z^dTqv^L)eLf{$%SF9=Wq+`*pQ1vaDAixXj@1b9$-(P8_5={La|LaO2`^iWKAqx*X$ z5Vev3RSf6G9T7@!%SaNz?^<*g;`wABxn(SeL_-DwE(-ZhYQk>45r{S1#zR!K(lh2` zdfEud96*0F!E@BG{+xA=+@bji_2@Ncyw2Ai_(HG-##b4^+e&?_eMk z*ozPFjDR2BN&0b5vjm{u;vNz^2c(khnPT|D<^D{>Sv!5sp@L{gBZPCwu(c}n-Q{mK zOk?Kp`xW_U7Dx*h-3SPTVwM4b$wL-p(V1(qTb1} zhB1(s8tIwncmq*QW#~RM8Y?s*3wtXkoy(wo&yyVn(%*`3WNzvV(okG)t{Bsa2&ED> z7VlesquCbk^l^{>BFkf|tCKT`e;<3}rej)AKqUI9LkhQElhKb4y}z~=I?%mrGz8vZ`w!nozM%1usN7K#~um@ z!4r*1B504~dfKeAWFAA>Dmr>a zAljysC5HjN42w^w+H7ZrWhcNU*QrFEk&5LGuA(^C0&l~@G2{Fh_lzN5KW-2WMvsl1 z3C&4VJzNHCxaVJ0f7-uWg(2uRrkF1u4|Z$rTx};mJOk3eZN|Js``?_32TbeRNy^?3 ziM=pxa`@hKI!CtJWjV@&5z5-$T+O7e#ZumKzCMv?XqA+bStU@CdqWd_;m+P~yYAS? zu1eSaE#oS7@VM1j^FC|l+`k-t+cQ~FSC>q#HW zBIovddLrintY73cr7{-mAx*>jM0EINXq74Z?QsTEa*v~XkGJ~ zAu>Sx5SdJRk|sLLP$4-MXBkq{$BKUnE$E4nSIsOqLJeLPF$u}N)WFmXt*IK|B*1cR zAY1{`6VchTxJHH6e8o0Zj;403+ru_AC^^Z&tB|svnOwo~-{x&szrQf3f|T>Q>G$G- z;^-gmqcW~V^bQNYIKTh4rRyHO_&ll?d{CS)5t33P={ECy*6Pz){t&vMz@Wcp)Y{Un zKG#dJ_IRTG@zr150-3ejW{10uhRv7LW}qyZVo=E4vqsILR?fvH{nac*bW9-Oo-+>K zA4!6O3b6InDOs*}46>Az5NdbqpNkWJYc&(RLvReDO06e4P4rXz)K>Z~L8y-O+^l2` zf#D?{ox*(3TEo8|`@7(0J ztP>L?oH_}#<+d7q(-^n8r;yyipJDvuI9lCjkI-%)E9|mxR3?v;y!v_h#G=2n3EAb! zj>O8Rp%T%YU;S;?rpwORQX#xQAxW+@R5ha`6|##4@H&vuUrxZjJBek=L{mp6s6ir!g}}g(JDc z{Oc3%YcSkAeOys52wH;hi4_THan9B|ze!(i)`3Q||o53XQ%TEF`=~WIZ zwK2?2xdb*gt2;jHIrWGGe?!EEswK+x)B>(~>*L=a&A>lgh=aTF>@ftu4Vw=De2_|T z%coUXvy;Fa_2CI)j^bE0Rya`$-rz>$_K0+UKxSP@Ha20I@CNhpG+fxmN-chWpxd>$ zvscKBxB}xzr#{V|Tg|NrIjzT7rR8G9XT7EsHbSZyG@`MQ?L;L>l*83)q;hTX{ zJ^~m+I+kIZl!YvI%|Oz#GI7k4XgG6Hk7k`%LgR3)?cRt)>Dbzm*N<;|d1Dj`$TUNW z$_dNiEusZ32U;ho>9`AiW;jvaGfA@}y){)^MQ>__A=$?#r>Hg9!)KV>s3)X z&a0)q_d==}oV<$-MjY#r!%A5z)qXc05T#9Gd-azn!L7>VW}BqV@kn_ghv5_ep@HPR zyuMM_`pWxAfj!Cbb&aSf7;M{TiH*BL7i46#yvteLQtc7pmDZg(Mto&9Ttn!CU_6Ye zGk&9$hizCn_9|@jf}}|+hK(EC9BE3fZK};(fR&zdQdkvA{-?CcmMQAE4y*3*+q6X$ zW~e#xFxD;yn4Z|VLWtOzxz?xcbVXIJCQa_eiahEpJ3YaZD-s%yhJGFWlj<}I z04=mX*dpF_xSTP=o8gV12fGX%n^Ji#_#2m=KXY2TR$;v;cB3MCP3zYCbI9ShIU|Cw z@AX>-rpR*Q`AVZI(@NN1DmsO%{JeCRnboz8=ZL1!f0%xgN*=ANvK;ZKcq30{)0@v~ zxq5(_Sr(GOhiIu1O3L!I+?q+Eq}02X1mgJ@@nbyt0t@NaNI$Gzdns!X6flZy^Tl9`-ebTa1l$;kkPTU55q6 zrA=D~F%m?wW=wgt+%F9^zanyGa!RxB^ga~-T=W*g1JEXNf8wZ{_0dVMdjJeUg#C=K zt9(+KM%YP)19i3vdS|1Jv-Z#3er*L^_#?i2NiyK5WAl4jd+yk__k<{#*U zSKI$Mlrp9nKhHu`%!+KxdVzg>U-nRiDI1E9%kPJi9EiZq1!771^``2Ln;3-#aZD*< zT6*k@XU^CnwYB zCsn%~*k~%>1xL_xQ%SS#E!UImOcb%E>tT-k_#Ax)^f{j|EEp}Qi9(jJg+#CbuLY_h z2sl8(Uv^Gd)`_G!q*0(YVcECL8SwH zREyMB`Xlqk<^9%A^+_8vF z+r2}cwblI$@~S?4s6|#v*{?C&m${=)!Zzdlm9DaK27}}#3Uw&brjWJlsSPU@n=1Bn~{_SjsCdaf)1`|R?1V2ObchN_S(B6=1gK| zxC_Y(Nj|w@3YuT1)?r_a*4lWcsb^E9pLt$m?KJSXzG#N$)mc5nC{MAYgf}m4 zMwqp&cItPx@qA4 z={qJ!C4ChXw!W36@0h*^auaEr{E&U$lTv@|JFsL<52jWF3}6t;p*Oh&W7#W|@wC&_ zC9RH(W|y;grPf;**YAF!t#1+yTP0p?V?{`AQj%!|HrP;XEwT!g3m(9sDrotjlvH&t zsHLHp;@JYIs_ag6cJY(>+dT&mw{TGICwgkd?yn0}WLb7K@i8>l4l3@^DK+Z}D#yYK z$X=ne)N(v=qQzzmX1g20n=p^A2SDxUpr!c{jrtrzB|HofcNgto(XgKJyL|@*z(WLV zLWUZ8&!yNei5SqBo2$B~!MuQV{;P=-;n)YY-|?A2w0+0zuYbilq(}@l?>RtE`1x^6 zED;VS&S=VLv9b<=YOvTQW&O3=*`+z5K&DDk-A&}(nNbv`Lm;q7UA1mG{nwP^Sv-qv zhvT>Io2g0;$(Gsl=ymQ*TLE2BZn!zk13-nV=sxF8ML_q!JrqU?1RO6NZoa&#g&uYR zZ2Z10Oi zm$`0@&jntxF#K|k>6KZc^Tw;XdtKaC)Ntp}XH%_&XKFXUkVJ0LZ1dbeVDNGF5cgUN zkZsu0|4=Wv^Xj;T3L8n1xQF6L#qM2?o2}~5Jv7@0SnYVW3=asl-5BQbM~BXnGaYrk zz&2>>rxmKs+=&z1!qcLRnKG2-9OEDciO|1C|>8VJ&1nV zyaM~M&@;fyG@prC#Y^XdtE1=cA2%24^Uk3%7nSgO^TP{|$&9%zL)Fb)cf4kfXsJ|_ z2HJ*sP;^ZV07dsnfxNVjT9M~_KQi3|wd6)lz52a_lV>Q#AxJ^hNKMMw`s1{N{9j7? z-nXZ$ozw-yS0%BR_K$co?55+@kGp>jm~e`2Cr9hXp4Z3ghZLT176Ur?j9yg(+dRID z*!gX}8UmYlvA}J*7LJIJU*LfV!ApBt?r^Sg1J|vI20L{dK}Qr^6O<0nux+sScIg~mX zwKcm4x=Nc56fa2e$9Hn^`3zZ3q8wh3R)^UX5EfYa6#H>uaffsi>h&0`eY0YB_?q^^ zO2nh&XIt5_%|IM}JKIfxKHOLr8YA?zdnl(v*=nv<(!B_B|jy`%)zm2osso&y_ZMBm&XYsLmmn>Q|>uI)9O<&juR(@4`%k6 zgO@$pL5PSHZ>iqu8(Ou$e*iqGc6BPfsBUJzFWkehK#K-Zqp&CM-Es~JA3{0`)Ty{*KIL!j=dFJF znSZb_&_l`ShgJ?xzqxBb+0P1v#~tM2KVH=Dg~t~(%!}`(0%Cjf!C9ZP@JxL(sv+nX zRSo%6B@N_4)P|?7B|WG*t694x`tI@i?ghbzf`BQ=N(R$N1Dl3Y(}`%QPHNAV2-utP zNK8~GyUo!8rpx5mK_D;IVyjR;1+A*{P(7K2-V8|>JJ%5L7k-?qPyevAdmtN#)}>gO=12{7 z1`X0%@1?c|NE}=E63eLOLosaAdJ|AR%5a%Z*QlTD^DO}L__jTzkw5DPOH0l1VJ-U_{E<2DUBR(Ko*cp>^ft|TnoZbdNlHVxRfw(h_XQ=|6;bOs z4&IH-uq(r#zc}M3(9wwk*ShKbpDow2%u>@V9g+W#LiQZW*LM6`?zT$iXWdaS_3f9a z=NT$xFzME$VOqD{8(rL&<_~}fX^P{e(sQ3Go6f$?QMT|ko2#nTo5J7I@`cfSk>5mD z9MQwQnQS*-u+`;v?btU+W+L$qH3xD)s}rpcStDC;?^~4c($RZvk{wiUW37=7Sw_hx zj6M|o9`#i4-WNY2!_H{NF56YYvu=Iy#HK-tKs5Y0Z|mT!Z6+RD9PP!^w}vzlWqEmX zoP;~{)Z0k4R7uDy?o=+Z^*+gc+lPl?`#MzikgK7{V&yb}S15{PFO`HhT0V7_eKR3D ztoE9kt#cD#C~!UDB-cMA>NJT~cd_AQv}B0;PR5fSEu`49Xo(1zC(?V$;UjBiatUv` zDCabQdaPdQ86&LEp_1J-ihPrPz3=Ef4X8I%0`53<0*b6>hJqT#{E7YQN|`kL=DpccM^el+*BxWg*3&H_43{BEu^+ehM6!t`eVM%Q z0l|-r>?IBTFWAx)bDwh;lCL5d->Vb!$BoJvbSq?Y8&U&yR-txFvcO$rFO_~|*B2~< zn$by68!+m}@_AjBp&r(TrE!}tI+6@oVq$OADJgwh4`gi%r(P!=CsF&j0vngjcS24> zXFU}Bo>qMNxz?C096k>rnh^F*E_^1rAdqIkf>h(7Fu)SXA}s*_`A*xLGcG=~)ly$= zeN;>NB8#zh-arv`S_A5k%q+Ah@ljRp%ev5x^0M*rpj~{(QEmItY+nJH6>SNl# zp)Z5+#knA=jkG;2B}=~=$G3i*9kHQ9?q-7{-hy;8YMGf^=3YY+F@Ii`t2aG4pe?GZ z9}3HAdjOQ-BMIXfEsJWJ7I3mw7%wkOKy@v@V>dJi-M!~_ce;w13{OHetvnP({u!(F zW6jH@(8ABuv|+pOGc}IYXyFi77aoS(MT8u)u;zDWNUK(GZW|#~`pifz+J89Xk~qdjquCVQ;0wecvdJWU@}u6J zci^Q>B-%Q@y_znbTu`=eKYxQ)TqABl5Q&f!eLbN+M+yRq%Sm z0|1&2E>X2C=xs7u^9)=_N=E6n+0@~f!=`x^$)o4oWfg-gZ#Rh@kDH+QBXlHpwOtdt zFzzaIi|BEarP`IAMk3DH{sn-?Nc8bVm zU;^o$;A9=OXL7vj6naZ=y7vFPFX?VRpM ztqD*x-2P)Ec9r9V$@fVr;|P;f|535T`oL(ZK+q?7P82bEh9col+jb9twEa<{(4SAg zYB-%PX|yOsr|q7-n3y~ZC&c3`v_9QMz0)6)yf7H?InzQ4O1&X7EydJLQIMs# zFTRbn*Uf)0PONZlE&@ITub&GFENa(7F2JwW9f9UBp=bpB75)C(P2oyMn%_DLB6W{9 zgQucBAde4wgir(TWFG)IBrc#6NC*UK)*XiKT^;dB|BSZZ z)@F9wsyus;2x~ZieLtm?&G_XhypSmew5adpGg>6ElZHzR-PY;+8Y&%m0i9pS;jeD8)rx1-nHH)hDR&4#Dl^o5LKNe4sPO(a>3WS)0#DEGPIr9Bm_EAHBx{G zm@Q1E7Le;+w22KWum+2+S&h-3X>X1eEm>Sz2JIG>vFf|)ISl`7M&Mi?$g-~mciUvM zUe{j^;DWvpW5nGgV|!$eZ#RP0_@i#3zV0lIK98%cA@P_%8teKQ*19vq>De<*ljk-B zT=~A4!kY^n<@SvFy4tLzz4PqyvwWs8h4{dF+v_kV?j_B^#2aPsqC<@dnNV8cxbGM8 z1&}%Wb-=d+T&4^y6cTzDBC&qkA?+-uRX~s=Q3kLaBTW5GUuE_b<+|?~P07>xwJr-} z?A(F1XeSqkTUD3zRoaQd%njb&p~l*14=VsDi>u)W3USYBbo!gvp@!ht#+}1gD~!T3 zLob8$)$NZpfhpQ|Dt30}1h$BL#pcrs*&wu^aTG)sGn9VVlr)i5?rO*KYiQ<;G34+1 zOT@>O)%t-Y^Of~i<~YzPJ!ZdOx(;8zEY*3aBv+`}$;yifko#&Vn4%RDbk7!YLJRr2 zk&K-O~c%xZhKJ*fTZ=kFcqR3d}0Gr2>4UHn%QWQd4zJBQH9m?QyL?YkOR3VXN5% zgJrXHHm&d!xP&W48X4AI;5m|**@0oYmJPB%lexHF$4a@0YzCO*j~*n?f`o;1W7jj; zkERPyulG*o!doA(=?%S}3JPjr+13ti6oXcSqHUVl&OYxNw8f=S=M}V^C>C$u z+WHWxHV#^JZeCa1l(Y{8>jSMmuz!=pDV#syg!VI#Fx5^Fwd6Ux(cKp$W7c{#ghyyA zsF5^UwbTkrIW~={5LU=$b)14;vbq)-D8^Bjo6B`(#2KL3OfVU~gI4sy2#YGzIiI57 zqdxm^@ddUR;{d|uPLTf%{lc?UqfBYSpVbMwDt8n@Y~fpEV@Q~{835DTN_54C>(J*> zFj3b?n4lX&^I-d=m?`~qnH6-D|MeMjuUosAE|f~i%t^BA!NK0;|iqr zC6dL!yZdq9q5hGtqmQZt;hd>JPU0?bLKORTOPr9rS(5NiPsb6pxqkhyHU1p%+WORI z?N%M0Q3AzYI1zI20b5399H*047Ms-5hQkXCau;jHq_pLfMJ-*2=AUC0zEFJFFkaU@ z?Pa*c1K?vUkdlUY`0J>3Em+>2R}IJh9e52zL0+i`!%vr>orH8rsCM!TQ4SuP(Ui-) z^dS_I8MD>z(c|hfKYjU4P}gbR&46$lHw{JnSJsd42?Iy?fJIH9n2g2N_VoC8--iEs z?|KM85%~#)b`6M-Ma`Ww;)H|f6&xvFiDNi(j3~W1`vH*m{_DDIO;Dzv2}2vTx$rwY zMj^S`j6fbWjGV8!fs`Z1*4X(9H<4q$Q;bh5eY)g2w-2Y^9wu)KM?c38v@FKc`)MoZ zL6Qw;0stO;cNowhogfzt1EDOnjxB2q}AYLmhP030XyNMgW5!AI*lth*|m(+;q4 z*^dch-e0^uCZGWi>acfeY?JyeCg-3+45D-CPp&ZPp}6n*lW6EZDflmabiAD1NzURc zjyda7#;SAk(S@F zX3E3;A}LFA#=!j;MG@Z*o*9h=)1a)5L@EQNbEiI8V3i!}L~sV@xH5ZKPJ(rjyQLTv zm+!99i!;(Po_+K^(tdolQM%d#O-fIS!liI4j4`|Gq`QduS2K!ip#5-jabQfWw@_Cr zHBxTScPtAuQQ{JO{>`}g{W4+pgYQuu+d@@GX$~qbZl@J%qxd`qzFDVu2j^R=jg}3= z^;laiWAx5xM4J_kmZ0__l7od7z9s6HKgM{jiyr{5oCh+CKM&JM_Cyx<7H=+VELJ(4 zPx-yv@I18i)Mv!~Ou}V`ljQzKW%2nGOS*((;e-9yTr3>d9m0QB1iuO-Y4Y z!7j?Maf`&RStrC4vb6$(JK{uh`%DRUCMK--HzNiC%x|atC||DSMti;b+LN zD8L9{4uG4-W4@?$1-bSMwoYyvT5&u86x~fj!WHq?zuB&d`)X2{rVhCGtvCzh9yu^Z zeX5ceGDhoC>&Eau>C&=GH^{jF3)#Ey?6Ml&%OADS>(r|>jZ+vRRoF^L8xOP4a*RUtyq>9WG3bViMjMZPfzd3!7p3~e-bvQ z1<`!wQ~CX?{3oFu!FbiYY6R&y`h2jpec3X#x=3hkxle=y6)ok~gmA^dVbsxenfQw3 z$UL~3K}UEF_kag&)YO3&-%hl|Ecjg8_7U<7If?mBLnK&&N4DEG zog$UAfv@>F154~yt@LbfpKw@<-)|_R89<>RVa>M0VKgY~MfO)tdGxioeS#i^r`g-m z)}Hn{)`%2F^g%*H#@i%XNFUCI?tCukEsj40#e0zAXffN5iWX%n;s{6b^l1&L6<)P| zPW~uU6q7Fm5k$sHKQ^DVPZ=Y+8(4^hr?P1EP>2IvgnMGcCEu#sd&06E}E(-t-ruIC|1@>xxj~0 zaWt(_2yKl_($GUg4gXgF1Ty>0y1XW_G#>rypj=wgYlnu=xtBENn1CN!J)UT^v&88Y zuNGYoW3m8Na*FX&#Ob6Rvbe#HwcV4H!RP+zt-5pblDZ1%$mqcWD4Q{oq|5WacJ%Rd z6*RM!WWa+WgsdUldQ?1SdO;pNuSh9oKrAszCLBfvTvR+pdTjQTv9SGS`#~(hhY>~^ z%=A;%r!C?+R?~`R4=H-f^v>|^3=R-*;TdjgyT*N6R_X1!{lYt}CZ`nEhj?iqgOv>z zLjAX2czfJja+9CE7A&&c!@T7S>BEPdo^WYxyI5`9FB^H1{{U|^9xG`{YfQ%c;7)sq z+N?LLG=n?WoOi%uTRi3bw4Mssb+6(){{U*9J6?(N&b`ZorMdd}7}PH9?D}0Je*aA8P5#hRCl%T&irA=H~h4L9}9JHW+uiqwPq0s|!|LkRyZ*=%*C4Eg&MCZzTx23mmm5>t4!+sO~d^suyK1_(b7FiIVFomlqD<^Wf?%N6R+vt zVZazNCWc3=HC(meOkq_umTNo=wAN1|*Tch@EuNi=Ol$yS>YhGn(i^-s%+4aT%p!+lR_KiP6_*dv~( z`${g+U6~Jbx(t&+Fgw)DA2dk19vBIEA_VjAJS!aQtZy~uggbU)jOE8QTK9nV_s(!k z9pfyxNv|rZPWh2bdN?-U296`-R2>QFz?x zsy^PKz}$67aROs^y)E6!I%oH|b}Q_;ZUz@j5uXquV~L3>E*gBZ11@R~*c4d5kW-5J zPHA}E@88a!V&k#2{{Z3Gtaz52@Um9@A={fB3bx)>D{_*ufp{4*Yv-L$5ETvQ;JFUG z?e2~%m3rrUHxhRJt16w&e3@;?^7@z^3-ltySIk&BX~aW zwHY`SNyjF6!15djcPvM(U9+gvI%B^aXgy(BcUG{@6NK{a;XBng>26y%F&8-G0FQa! zAaR1XsuJCEe~j>1or``UtFSkH2`bzHhDS>Cwm+3{vhl$i0j)MWa`cRB+jESMIj%Cu zY%c1pSxDDqK`n0T=R)&@5ykVwcqNL($jMH|~WqLBu;>jNg z4yNog2ED1wYcZKSV}#eIdg;t(USY$rZ&c#f{B4@FxzgyNrd*|Z2HNnzmlK_L`jl}7 zz-Wq?=anfWcaB1?=%ZmoDIo2{nBJJa<@iOYhUUjhqec!j$opDrRUzO7p8!x-UDTHHPu zAk>_VNAsMlcGwR!4sFr|r3V7BsNI#i99 z4g7NLjq9O=jE3JoH6Sp9h)zS#z8BA*@()+yvcJ`uvC9^nz^iSx$Pzg6c60PAC%sWQ z`R2(-2^9fwtH(}%*YW5yqFB<80nT*i6dMN}-0QQc)PZlgY+^L}^pHmLm@ylQ^I*Hr zLbluYFuH!Qj?D6|>|D!-HdA+#yZ-^113V7k{1>$5WphpTXFy|8Tm>c?T{uS313mEu2~K^H0K}RADL9}Atp|O=4+$%)g@cA;lvmft zqkx)j%h$JR-m`qhc@pS(`q4qRZDZ6IM%9lNq|zEhaHyV9Ud7rvRvf_o$ndK)rYQZR zC#7qchcn3J77HYIk7B8H)H%{ub^1|s33eE-_WKe)+pQVz6X(w&DWhs}wrf6viDlj3 zWbfXYb--fh%*yLGt8I*lA0*HA`$Sex5Pa`i*919`+;D75pZ2bW%`>ZBJYsh)lQf!p zCB>uC2D^^L;>ns!<~=~HYfoW0i()*>KxN1yZu#W-^Q&dyrjaYAbhe&l%oEOI9QJcY^@ z?OV1xtplgmRP2+8Xf&+?90Zdt z!-N@CGTi4}u>Sy{S_TWT_m2GCNh5URcqdyJnR3~2_^`-30UYYc?^9%`yq2=$?}*r$ zzgw2(hmR4hJDeON^@O(fezB^HU7L3mH#-Hl0lj1i@g?x@s(tF!$KZE}+O*Q1`Bhnkt#=X>iF2(+ zL*%W2W*dE?Nf1fh>Rh|n3_~D^lf`r-TWbquNN7BYnQw>-f#y|W4t5MgH;e?kwq0O} zuLxZMZH{jITsEtETP|~7jBkNFHZGd%bd2cc$ zmF2Bpl_isBvxniO#9gM6PRECX5}+{^?|eHXlXZsmW*kly_A!C19EdN|Fn4s?PHh%prvCugy$=p8JnpXr*|*f%_h)Q@#l@}v0GZ1!YHr-r;Q_c>Xx25Y z0ya0^yUo5~vO}E$YZ%|YAUL#6LP^@WOO2bsCDwF@0Tq2xljySVxxv4oI#)D4ABY(N z(^%QQp;k8S@~gU#agt`PBu@%=c)-YSfl^ggwb6PCjQQtH7)}wq<>V|Sv;dCX@jOrfoQaBKuS26nOi|om!IsGUU22Qr*xE{=%)-Pw8{b58gK*Vv^3Cc=JHe;Nt3l~xXfo> zPH=!^3H5DR5uWg4nMHxr0ycStBG&>&MhqTQ2IrRZPg3OFwnNVnal|f$PaE6`%Fg$oeYMgM0=C2@H@Cb+zw?2ZUz_z+*Y7;DX9tRWZSlA|MOrmMf zw-@s}K1c>!X?vX+jP`Ijs9L^S=yA-2VTMJ=9pH3It!=G)OX8!QKscZ07Y+a_g4lMU z#O74)I|#+Uf{Pt41cIn6wZ0V8>{L#c<;&klxZqk0hPtJ#9;nS+)MC4YuopznLV&%5 z7MU53Ar(a0VkjWT=4H-eB`3hs(R!d?G!5<<@+CSN(Lare~2w5u5Wxj8$$S&nx7@gA>|0Xd!1 z{no@J-7afot;JD9GCaDIzCB+x0ODh3o=I54H7&!kiIGjSg#n%DJgQD{?!15j?)#Mn!6Oep z1hV1|F4)^8PC<8If!mskVZvRu2R7}ppfaRJWS))4icZ|O)Iu8}ll)ZOSm#@cf(3Fs z`}u+b0-tKtu61zm&u-P8e8K|)F&Px%HX?ca_90M4YnEXD0A;zYM8X3orU#$wqAF*C zWkE5NZ`u;zFxEKA$bv!676;7F3QonP#m$(js^T|JN&|Bw?le45=0p=p@*-LqBhfn} zC`=WC;wVVvKqRs<9f$*Fet4(Q5(&m4j@iLZ)Jfn34aF`5wX*0_W)|FVN9?k+i;qoHdEj3arW%OzEge~LzUX~s7Fs0lfT049V~ zw|O(_+@NLH&Ql!SzUe8KZU;E=k2{iqlONlpstA^c$Hm1WI}ij=$Rn~4 zX&JV6o&2FjE1fo&T1FhrHWBW3Nmjs-8-X<70O=H2BnsEIKHpNer_)~6T*0ScdEXnh zl71d?zK7MHr}X+bxp+49+8n!$><2&IJisl&oTY#~uU7RIr(K2jV#SlbJI?*q!1j$e zN;<>z{Jy?u%P5V@96glZk`6YZm+ALrXRCL40un`fSc9gc&s zd1p#c^YcbdDRksdJAO)Gz(~(*fByhWFwey1tct`oo8~=hkqtP8W1tR6Q3DoypCuw6 zzX=;*Y>t24pZQFv1(@vfw>36Yr8u3`+g_R}f<|O?dFfOPZXyrXS|y}HOc4>?=lfPT zIGRslfK&efnRt#dvvRRO!Z;{+M;0j@ss*ocAbYwNfcsZsRZIoeaTuAm)sT z-I|vJz&dt`Bei3kE;^)TyE=K|pnsVIhn^~x)(}C8Mq*GF`;(h_ZOUV6V2{GR%ommN@)>>J}T$uYQqJ@`lENvR>cNqaOH#rv)y>)BM>4@2JEFs;`u3gs=_k<$^tGfimeSiWSFf~+9+ZK z4T?-~gH8F+pfRi@!8Mfunrw#`5tzyjd{jli;AuwWGn4D-Ku6D>#UDMYU`|UWIBY(5 zDh!T2i(-_|u@xunOk;5zK7B|9kD3cfX|EZ{IjxJ4=Ovd8bm2|KKq~6>){!zn%nm$#OIR^K24SX8I@XDH z15naSw#boD!2yjVjxoR8&$VVg^a=pgA|Pk+!>4koan%jm2n&)M5fn4-RUG4S&IY1o zEx@+m-%pxj2YJ#ktB@IrUeGvKrB42D&RhD}~Xl|YkM z4gIEx;t8pa&+e96x>}9jF%{{YUt=^urkxZnInydiW~s+|q8 z^eu_$Y+2LVYb`J_ya0*qIg_osp-g-K4u`*_^h;Ae8C(mD!m z0pPX3I2sUYBWO{OQDk^e`BESE5SXFu=2~cjkkjY9_bvuVR+>S97sk3 zM2gQh*prM3dh61xQ~2H3ePepAUUC49pV?p$fNFUnQ<=~1^{59DDkq4Kc!Hwt0iM5k z`Kk#xl6@ZBz3O4bq9wdxC85VKc}P5p&JBZ~3F#5*KqL%jdFJ(H>r8oQ7)wt(NItR< zT=s~-BVduzQsloTiS|^mCdRjOy~`g%m^(rF6(LWG*&9;a>cl|fQaV%waUKS}D{$75 zc|v3kFnELWgwVo8VcSF!mj<<=%xU2>xj=)%wK0Xw7)cebUt*RVP2mT4pFB_%s)q=V zKBUHxCP*~pP%jQGZteYr0PHQKZyhNyUzzjLq+Q{<1w+0cv1^M(H=3C-QMCaH84US) zR2?96saIrq;G$z;<@b`b1e{Ln?x&VvgSq>;`0z&KHQt!VNR5UD*iaKMYa2o`jysau zKpZyW=1_H70RUE@Z%_ok7|0QlfPO4(E+(^1$x?M6O61~}JHRtB01m|Vh;>j9E*BgI zSgFY+1Q`wk;-zW0)h^&ZAuX;Se5)9Z#uyb5%QejahhzX1`rmD8m4nL+0J>8T*Ahd4xEaCH_CQU2nYQc(PB~VUaZ$Ca?at_XqKIJMo!n~eB!4p#)`?(5$ z!szD_%bEkm7qz~76dW-f@jSf?9N3u5?#WPVOt=9`n)cN*0CODC@U%~orECU}O+ z2y~o8g;Pw-Sg}(i1>yq;iej8ce@Z~bh{wWcPQ_O3!V>`Q)^=8(GzMU-ny`Af2tM_r zj5a6H9<@7SJ|F~oN+66?D?ly~Vw|7ZYy(ozM$FTLTtlU;H1uoGt9XQn4ZYqm5d;R-;Iz^3N9F?oTW{~5kXa>!(~HDip&kTXA~X^gGI9h zNINDHnBWd&G4{hMPeRWQ*{fY%MF8T-sH#b}=*0nUIH6(fYEEZ(RRVV!&;iEKcnMlA z;cgPy;9wG?g|abXBeUk4hB9+V;N?KF0H}EEgDNG^Xz4}{^qxfV-FNiCo`fu}THU2pxpmIi6?8i& zmS1((Gd#(b3_qCs)i*t?X>*(!(V1}oa+T$EU#fI^TYy+$)VLn;2MY&dpyEA^ji_9g zk@Ut~doncOa$OrY*!mNM@H^Lw>dw2+^`HXN@ZE_vUk!NU8=N+`c3>{+kdlI*iRjff z-E#>BpcvY@_O(|n(D$3b<~h}uTxG*8Iqo@xI5;?XmrKLMgUq?Q&acyJt(IP7ozE0| ze|6sMxP>(4+~Ts;$#dX(-8a!SH>@~w$Z7XR}_C zN2ym2c2GGZroObw5lmx{j)XS}qlY_5`&0x*%tMg*LdX$V*_zM@6ist`v+m3w6A|_& zum`Z=iQTzLz#Bu(O1mb}oyZ2cfR>E#{71EDXm%;xF}sybt_8S_L!iQlsXtYx&h!OV zlG4x}9GR;n_?ziNy~_o*H!LzZdeSjg7&~J$1K8(GE;?SD3sRd~ zyoxt7C$y_zKG={^jtQDdU5EEvp>Mq;yCkqIxHa3B3er6DB{q9g9Mcn30M`sc8*ouT z0`1{O!)sh4wMo|+)7=uFHZHZ$Cc9LFSkiGBCu)c-bxjJ^fYXZza-h=IG`sz>N@RfSEtAZ&j7AG#P%je+pv1JcT}pPs zV?eS}F6JF1&cp?QETs~fWRP4XEI6FPkO8#{f-@!-n`Od8(K4c7?Pd|KDH z3CLbx?*7{aM{8%WDPyXh^Xd${6e_@QIWA4Rw(m4twC3Dt$;~*LdkSzQZ&1vsr+|*R z()}+%df2O)JVsKunZvY8dDq1*2N)sk;Y|8h-kW=h`3G9*o21s?w}-on6C4FFx}%dW zW#i%_gpWImSe!;0ca41WsbV7|K6a=aF5t10rfbM#VZ=|>=8#s15r{&Ox^n{Eq-A=?ny3-xgnBNAf z4-vRH8LUZGh?6RX(&hnQie*4rybD+`ie<`@=z!Mezt8ti@dg3DfkmQ3&oTinfCdo< zdTqFS%!&ca$xm8KNqxrwyRw@Un(PjD9ww@RZ>yXdVh=>C4L!mMs1eYRZL|{dWSOU# zP!J|J=O$%P4seV!39fTZj39}P{{T2h@Z94tfGge*fYxqo)3btRCd3yL8-k3r)N64Q z#!n)a2C}4y;R*w0M|Tu(9`b0E*3|u7nWAwFPIfKDZSo?Ll^jmtqhNH^7gz0Ty>j4Vv+VZcHsBBSd8Hu2iF}Yjt!DO?46;X|QrlOqc0$aWi-jynrz>hr%24*u%BI^W!=dn;s z0?}BNo+Hf`F~P^aruz}z4)d_AgB{_7D1cib`8 zwL#Pntyq2BFybS36eYysBvlyhH+CSH;=)Z*i%#PxTa7*y!C7iRb+I7g?O6Ly6$ExE zi(s~-ZS7jPxVC^y;-YVby^fuQIN7m+So7KDs#scn69s2iYZOn2GJ65+M6s@xf@qfe zPR&Bt??c@J6$QT*+4{5ElMH*iq_r^E8MWAgP7DsARah2W#x=4|QC=xedm21OG-7GQ zDMQQYaoy1Tqn9XFy`KAZpHMQ9ciKK%`M9ecxBhL{{SZB2VxZjREGd-O2%L= zRUxV^8f{K3Z9Zu0fy9v2F;JZ5+aSdb@sZ?1ATO75##Zk3EDnw;^R2UZODzo?QsAm( z#)O42BjN<@8L8m6v@IkyhEJ&g);q)$>CGkdqlq1EPz4q#*vIUEgXteG3 z6|x^upSy)r9^mUlnwXhL(+4@E%tBZeF5%OF=J!j81ooY&Mfgo1lh5`ldaIbN5%1ca zZO*vHxB(v}t$>`Dd_~-nPa=?k$Fe8y1~;#@nvAc(!eZv94oDCjHxO z_@2-JB$-x^RCTxbGIRd`>a-vGVmg--Z)+=E1u8&tF5y78>9aKHK6|h+Se-etyZ4@Ri zi3M>D>zVl1zs$M*rRhIaIJ2)Wh8HwpZ5rn{B4gnrB~)P3c5A<`<w5z>{-xaN$34uyiOzS!7@X%va_tMqsm zLLg1jXXyH`_+3ZgDdK#8`ue~4T}R zvG692leafM<5&Lx3#t4yqs02wHGJA#PvI!_v&$}84Re|ZyEt51F#iDID~XN&0DE>n z_>ZUkOHzarv(?lzFEi6R^V%W0eOW*GZB03CC9&5G;HZ_0n zx}SZeFA%lV(`wKDms9vl%^Q~w{oB9hKmAL$)ov~q%be#3Y+`er9m~FvDn^|-J0~pq z4QNG+kHlrVT~Pl3(&|46PRoED?LM#{{H~@4C8g!ra^y6dw@D&r#DDshm)!f?pXNXP zOFAjx`TC0sFE`8=DUw05pOjQ;>P0s6u@!tJ+CUp0fb=DuBM>8aD1 z!K})G&bIp3h~l~vW#fA~N4;pbfMI(<;Cpx~_nO84MFQd5A`61mxPo7aaC|V~z^69l zx6$_So+HRq8rQ7HS_*8D23}2DpGvty!^fmK?L-fT&ztV-On#Y3+j@ z(0WyOfC+R}HP>9*EQlC|PzN%Wa9?rrT{Mp$bK$hRn87jyDfU zS#XmXb}hI|1r)gi+Ji?9ekwN|SZSida!zYOgaKyQd%MOx=+qlv_bNvsu*Qnha;b0( zStih5>VNC&3$p(JZ`6PJwCloKfX4O(?sOW5Y46^9TE`!wTx>jIX$Jn7!5JRRmrc5> zQy)O;&-gn(>0X4d1@-++#n}0(^*^fr0Qu&kcembGpk}h%kNv}|Kj5r?rFgEt>uc@n z7u`0r?|tSm&M&u$q0bbPN#48088v)SyWO1GUH++Ro1F$I-SN-z;0Qh|u59kk?kNe&4@cJ+E z2hDTkJ>@-DTz^`}W$*t0BI1mdH+BB0=3Qq>=$OT5x+KZ?oK}CnU;h9PqW=IOeAh|S zys;tbx~Pm7{{Ux?p>=uDAU2zVNbCJmJ{vkl{_U(YLMw&CDWZq_)$j27FY*V?M^5lc z_*@r5rFQz{{{S}6?%wA9V)Ebbm;V66=)cGxG}G6*n}T0wO8ovSSfWjHaq<<7J4Q4PQztuD0wWV+F+QT#=xLhYPq$#y4+!h$q zfxVpQlBtfozy2RZ{zHCif8Q_v0Ef|kkUnc1zc6}!xPf}l?%w|ZoGv9|imST&RdzAj z(!M4OtzsffpNYpbzxT`k0O9msJ^_^E%-~FHYF0KW$-EUx3$6xB1@Y>S1 z_ibUC5nL`ANrb=myWipTU*r#(I)8#!!sR*!A$OS4?M}A8{Y$_6Y+gtG?)Uh87x@F` zyWLl*xv6FA&x}vJ;$lGln{pDk&Ke;s?}3TnbzNyo2!$K&BMMd`S{Ib$IJNAo$Wg={ zm`E;hX%bdi{-7RmTChC67O!9Y?~iEz!bu4-(t jwG%3(wWNwLTf{LnSr!~dAV$%=DOjN?>r~^eL_h!8w{v!s literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000023_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000023_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..13a7c92be9dfc67e9ba945ae3059e3d050504af0 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuSUty5TJetG}vP|b5tA>E~DWv8V<-g zWHdbtrSvcbScCSk=H{0cr(&&!nSj->RHsu06U0+z&z!LZu|{e9;Q$G(p-a*se)mGt O5e5xPlBJ12LJt7VNIV<> literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000024.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000024.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a85a3c261be322c0c753187120ed7ebb56f29ec5 GIT binary patch literal 30925 zcmdqIWmFv9)<4*|Ccz;Pyz!vH-KFuyJwR|8cM0yn-6cSP#-(vcaEIW~NbukmJcJ?7 zeeZkkf35j4UuMmk>Q&vlj?~%u+ow*SdRlz?0l)#tD#!v55CH&$=MUiN58#!OsfRf) z2irR~P7V&h6C5A~K*z+y!o+&<&w=v_=RXG?9v&ef;qx~R4jvu>0U;qVG4XT3^Uv$o zuPG?ryrHE9vaq~+>H{=C4}gMz@I1u74t4>i3FkA|H$>ODtSq_47%gKxFV7Fa&-QZQT2#=pkSN&dT zO=P#rNs-j=|12b(s zor+iK|J7e@WXFETz64!<%1l#B&f6~_zyhMz39&{Hd9MwO1j+BvVgq_^x-O# z>0fr!iZ2+OV7WO@06o_koj!wqNl50n>>KRzeyLO0y3ex7?lT%320}}DFmZGQ=G1szG0VRW@?~p}M&M7J zH||LwkP&9e*!<>8m)xJxovC$|-*p8jQ9#j6!p?t%U(CUO@{jVH=M8_&of9v9X?5|4 zy)tQY`ocL|taYXCzKD_irrM4lBct#q;p=YO zXkJzq&H6uDGLdqnOKM0~p7Y=EWHaovog&fDEDKGglaw7+I@qynhccMIZU62~KT?Gi zsRd`&PpU^qbR^cUm{xf2l56qTyBwbdgKn~acf>APJ8w7JNoD1AsbEqKLG)u1#SIUq z!OmdMmbGg~^IUD3&1)&w#Z1yHJT;}05ciS*?1P2rN#5N2x4rg7YpRFGv6x5UEhU?D zwj{O1R5?s&J-`D`?zctv9sMQQG{2eChB!Xgs=clxS^kRl-<=QI2d<#B% zTC;`JOd$C)13mdl5}*Sv%uw{V{KH&P3LQ!PbqY9TL%#o=_*aSl36+0@@uzq9aP(A; zNjoyA6}$o>M`;rjKa7adk?#~6472ST>D(W+NlM0{7-6(Iok&BLI)kwOCBAdbv=LAF zXOk1(G}g}?4l*S;L5>_y`m1(1LJ3mD3V@mAGCmjpe@?d7w>hNUh5|a85BE*6T>A;QT@wi~SN?=K;!O z$Lg_9oB%}~Zb>z&FJWXAXTf9LJQ&mBQbbDnKR5|~i0jdAOs%BfdEl)dNsz#l*H7^i zWmHB@b=Y(J@ZylM+2`4JtJjTeH&e212PoZ!%BX_|{~IV;TemH$*1=qxCj^9avKtD_ zU-TJPvD4TWq`0m*cf$Ke<+~JjM&c3XFQF znfpTrNng8s9NQC3XFoT5KO7225n;FGL4aYDX-u;Z?Z)!l~k z&bvLcJ!1h8nT}RE+~MSu>=7PN)r2ZbNN0QyzD15w`It4~&t|gOdJp+u%Kmp;Wzmmc z)vQ{kChuO2GpP#AwhFtt-r>1+>$|>&Vm8^1xZHjNrAC&kx%X};sb+6cYFyh1{DO(H z*S#!rD<1z29@E1GYb z>^EOK{$+#iPxj%jJoDT2RiE6OWq%9!;trX=$x_GO^x|#$v!K}%{;cs)0i}D>c25AZ zpI2nZqV)o@>pK72`#(4Qf7U{K{U1fN21M^Y^YU+%tSjt_#s2b%z0TKj%V}^df8fcv zH=ul0kmFxn+50`4uPb>hfb5}0y=&%>ZEf%^{Ge-mPc&)J_6eZ8w^X?|t$S>;-~Ox1 zyzI~6tYhQ-(2jw>>R);Omj5$Q=Eu1wK;mTaCAr)b-?IM)mq`=#e=+c^i~!4qf9a?F zAFarht=qVFS+PwzzVn%Dhb2@pwW(Z@rUpG0pGHbV@5{Y)ue$@f6N+7G^S?ha*`%8iYZgiD;%V%ykC>L z1L}TH09@U1grol{)hr5_HQUP9z95tdgX%b%fzCEbSU>67VwV8Md9u2) zk34)IWDPdk?^V{#F8|*1w)#)<#GXdQxK%uU@)JOP{BiG8#O6@GHwM#|qfhpY;ETeq z`s4lOwiPKwB_xQT3TtE!(aR(4fc=YZ!u4s-2mCe4Wv={Z($$j<+saUdbindj`)9j_ zE2i;um02YEG1$hzqC4hWn|VV#toUyRwS#lWsx zf4X%3ik1H#vD&Ltpg-xo&iNZMJD?b&sR0EbvUO8$Jd9DDfE#bORQjf`<;WM2csX@j}I&k_N(fRXoz%#GKN##S$c>O6qrCs)oKNWmmf^LEl=1E zQoWMh$@O=bg5#yJMR6ZE=ARMkQMmiBH=i!$+)V2}N1>yM;{QZNx*mGW>6^L!HT`N( z4bY&?sn|?D5CvbESHe!m_l%vnok?cjsJepKt$JB@Izb~c`4Pl%c{J}4kez;8{V_XB zw9M)UMfY2#eZxnFOmBbd;}D|`P?OtEz5H(Cns=YkA<81VW8sV(KyX#@k&w z%^9!D0;iN1cd#O_ob{m8+eBbXpwduRKE_WziW)q_q4NJyszSNO(Bl)MZT_6`iHiwC ztvTj%`U`OCt%hbS;Oy4?-mEg+cR}dma+OhM6jl!r2OLFLDLQmB%Z}@wz3%P_@MZKl z#1*>#LpBR`>cCeVsrw`+J+m6bJY^Ev7E|rRXw0-i3MaT|jt!sA=F_Ni1u(?={ zY#WkC8bnjMG!vO?F|l8Fj?d`A_rDbHo=(MsZ|pO7pjquECRxoQ&CkWj)%gtZ%@36M z_kPX&uD`PXL%Bld?tK2XmU8_6cfF7!OibgSNPb2K6~RByJsk7uQ3@&7Uu$XM&`0*q zv+>&G=1E@nH4S>}#s4JHVb!^}_t1HZnGtAy>EXrl;V-uu@H5=5i%ETgT(5t4_rSLE z@t+(Q9-z|B@|=MgatCTON}ly8IEV*7?i%t8hcB5to2T=EI{)6j=2`V{XXxC%%$ihG zAMb->{p-`xk-zDBP-o6>v9LQ$IDBuv{^Gu)VBq47`Vj1XvO%D`Wc0e`*XqZy-9PE< z%iS}+_ePFe>5eXrru`J*a7gz|{u@UM*w=xz4!0LD_;WtYT079nsN-$to@RgcKl7!41L7#$p*h31;bg=`$roS8?QSaX%2sV zq;IZi5G;b>mHX}TIEvIbmwlvN{?*a>+NbCj?E5h*NS7paS5en1#U%?|2ZKAJkUg_! z{me}9U*fsM9u@9L2qfVgpcmA|soCYIz~4J^{&?l-<)WfSuCqs(9UDfZDR$sAEwFQM zo&>YBdTkQ;*+z$ePV>UbNfsh#%J!a%PZX>yl42u_#S3@LoN~knqMK7h$J1AakFWJ=@{os9nPE>-pMiO8Q& z2<%$ouinlN9YV-_;7I?R|FJ)xTOlZ;(nU~+P8afQ@S#77hlqSoaDP5i1iYh5U-)iey+-7-e;9JzyaRqpc< zn+SUbcGHUU7e?xd&baclIY5muFFYQW{qgQg%E|K+$6Mm&b*V-^owyoOc7FNJ{_)1~ zyUzRTwk^J)d`$W}!#39FmXYxyVX8b`n*%+)r0+A1Hij{VEzoeQ;k3Enwyc){Mki_a zby})a1h(2Ig>FjMqDc6{^aJpL zb*X;^=vpo3;r!z37rpa~FPf|BtOwR$eVor>%NK9({sZ4DgZEm^mWzatrMW=(U)%o2 zuFrw&_Ts43!>0*dEo8*Maq1Km<%mbyca?@HdLz;6hb4!9+QXiTo$g8_O+FLxDJZr`~6X7&Cnj!W*p+09H z8Y@|l_t~`V)#<*7NgI3RP+;ru9Xsw_^QR{{K=BZ0LqcaSB6{SABgohL+LYO25MODy zerNZ|4mrykwo^6coBk2IE$Ly1tin&~yXK>qq-FI4Xr9{l8Ez|(K#U_FYfGTlINaXr z6*i2Dq^Q#<)LDr@B{ z?#1qNcwtcM)+)cpuvlK!=)>LbbACtFKp_vzgAb?$9QrD|Ea@}$;4cy5Us!GRk`@mj z->}qQd2(qBX)`!%R!3fC_j@XQf3kJh%g=GX* zX5y#;FHMHed{Q2j%N7&E4C2k9Sx}u>TRA(#DkjoFyXh=Hq*#Ct+_5}J_Bh1{ld^$g zKCb{Zth$bQny~C6;{gwB;)E&kxD`uMqc~3e(s{l7U9#*m99qggV;N_2)qkC}D;$3v z?m2A)X|HDg+uvX)2EH$9|K*8B(_!qOYrmDoz&eO-5?1{@E>u~*+lJx&#aqR;CUkng zkc%>dCaq16XMaD0r3E7_wU)xJ?rE{hc2`p^o991bT)0%~OU@X3vJxfRCs7L6vZmfF zI&gmA_TR$EjCymd09(&enTUEZXM=^~?@ONT^quTv{>H5p0H6|qWPd1x!5v7+97sg>%3^oQQYS0b&W${_I+&ILT z$BZ`1)FEl7sL;|~g=bsR(Gonn$BuEw$A-WIxc$$@rv2%izLr3*2`EVRbFr^(*3`32 zI&p>^o^$H|WXTs^Lov3-!G?ut0I#_>@!3f-BTFbSi_jOGl#FIRD*bBGw79b`!Ed%L zj!*l?HVUm(%JRih-m!3hmJUoG(aRf;kl!JoVtu>g*Cd~Qyd_@_WwPc=b#eMa zfN84`(qZps_6N30=uBd--_5BCB(RE`A*NT$a%fpVR7kGkZ|DaotV8<>6Xnv zOR1}^bQlv8a=8SpHz#>b=;_BlotF2N79apfdL^)4vv~9DHrfn*))s3==s`H=kL_mF zDoZ_HqN5rWPze#m*{r$>0V_0;Bp&UoV=q6Ynl#{5|KrSZ;=#=K-9NMbWb-S(QkO3) zO^SULp@TQ&6mJ!?p<?tjn0xJi=dT(K7UKU0!_B`=?& zlvHbmF~3p7*wEqdk41Z*$9_V}NrTUAE+z8;>mDY*uJg@l-WtIQZ0-a<^@Qe*#@QAvzxvrs^;X|OY zRP?|%I;`HV+=v2(W>mdbQtY&$HJREg58h4%m@(|DiJit>szQA+DT-cm9}i*=eqIYd zRPaF;hMVh7UrYaK`k2&aaOmIzI<{zwvF5;F$Sw@qxA1Gjhzd=H4pw!A%Z}SGt9D?R zd2K1AWD5ky6)t%vpap${Y=A?T{aUI~qsw!hQOgPm&TVtkM_Hw*ph%F_WPw&RM3(0AA8M3een< z!dlb7DOE}xh;BYi4b3+VF$bh{BA<@!O%j&TFgx)VXM|?><4#7i(+N_l6imRWh|ZPt z!T3OJV8$E$^)(0b)mRomu4eUk-fTX+*yf}##tKkW|HsiRkX_3+(8?kc?tj<+_u7>`*Soz(npD?rla^wfX!UHJ>{BLK({l!Zc$Q zgruUg?dXPyc0|uPr!JN%tfp#Lgm|9-)_yJt@J7Cu!LjWf;V1?kkRt-f>N7O@z+e9x zM4939i_b3npYwShrpvjA$BgkNp3DxVu4Y=t#!K_Fq{0r}mc!KFUVnRpKVz}@zawF) zC<%RW+kbcO<>2e)2!iXP((kmV#C&msjva;@i(93!+OYU7r9T>Q5yL2JIoo7xvW*|2$2EJ@gR@k{>J--D}M%{34Zwv3;& zr2E!Ofqt1$Ox*?$*H}6Q@If!IoUwHtP{yxAOSDDi8S(BE^veX}N>9-kbTJ34;c<$B zZ^+0FWnTLG?v&^+J7+8CtDYFiB&jeReTZxHCl&F_WBx&dh{r#(*aJUK^sr*FCLH^5sA-qCe;W)JdWP^rMR~q{(9%0 zHK6h<)e8`7mdi%+uCn__#|s}3y@lrTC2v;aMnV5qx{$kdH-Gca0N$J~^8q%TP)1~6 z7=A@$pp^u|JB8p`{MEI&D$tBE{$bJC1=c0S^~cjOempRmPs@x`6s zbaiV@Cj00h+F*4>R6n9}mO7JuTk~q(?^ggj>sif}$;1gGv{`L?zorhuFaIA#dQha}kWmKfUZ; z;XHo*Or(MpLT5-Gdmsr;++pvV5;`~@dbnC1iT~Zd?1xs5EQI2D)rP&g6Recin19>k zCpNV28rt>K|83j%2Z|vDcFIKfsqx)NhSphqSzv8;cusiM+_=c$nkQP^AIe3LI$Xgw z;OH0LpN6yES?I9s=#|{*Ek)wD%om6^^Vn65#qjfSWv7j4;H=|R z`Zv7i3ExOH<3UB`UD>crI>ZQ79NhZeZ#_ikv3MsuOMQRuTEs-Uac=Yp@Oh)*oX)cS zw8LRV=l9%f7(RvqI&Pa9-j-gsyMkKlyx}=Sowm5erom-1NDTvUnm1&JxQu9HtSsud zJF4yYpfK_PC-u=hDWj1E_Q+Da#b~x&p5TmhvGtb6-9A_Bq1~yGdeKKO*BQrXjX%kr z_Vze!onjT$926yqAhF^Bv;RR2xSgv)7wK+Yo+FY>A}EB(@0zvLLnF@d2h1Bg{ldYe z%T=$bHr|SV;}kcl-Mmpc%{|H0PoYYkI0raq`K{2ZHgEgCctxE?7V+S^5ghTU_JW+? zB?Xt%7+n|vWU1xvL}|4aOcpNhjV+n5BGl{jwV3ba*?Ewz19|o{R`|eSm>m%wiv*Bm zET^xZ9nRp0HPlN-9E|V|VV_#{W{*`at>&tZC1-(?yHk|yw}sk0plZfaWv8!6?`8Do z+RM5UpmxfA`F%+Q3@IdjeDj~a>OcRX`(ZCV>TjU0mB^T&nF_^O6%@;T z#re@wL*>t#iuNp>i1mV-QQ3}09u~5}wTWC9+0F3|Bs*ObFXdRsB+u$&gC$#b)MyFO zaVCPSPY*NjHI7P~ypYLflX07$0rn<9SPu6u2Iwa(wvcJeq(o*8-XznK)zgoswX!VN zW5E_^m2rk7vEGykt7FDpmdE{UU)I!VOX6OpTn5kw90Q*KrCG-8*P@}qpLDKsZ<()z zIu_5B%+&w%#tm#Z^EYKm{tA`mCge*>Rj7cOj>>lx4Jx=aT zjvr2(I8rF0O!-pp{4%d+xfFWUxU}&EP-LsQu5?{}-+2`y=CPcRSGp4KjbJJ@R{2Zo z@*UFfTGk6u6@Q(Ky>gmI(by+|e7donh==nUxZB3_h4Ap^Cc|scu%>LHa>YUj2hYxz z;caX_@!5@C_L1#%UXQY>6t^dUHkbEkW=JMD`fV|8&v^;{BX{}5_7ytHorA$4sv|>s zlLgV+6u6S_NUF>4g9k;C$JHBfDwAy!PfsJ8Tm}lmQ1`UANB}#zaE1H$8qq`LZI`*C zzQ)+*1N4p1PvdC57q1>Y+nxZ80UK+NyaxW~t2b&T@OJ8V6>Pxd7IS&_wT^*^3#mVf zvlk-dD#hKK{T|EuzHq7nv0c_vTAVNTs^}KvI|)qvH&C)CKt|>fy$?n6y^+@wU>M)M zd@g+{`c>5ZcRwucw_BfV>BLEj)$*PnlC3Vv8|JUdnC3$jo&XLtqe7$HuFch}+IZNZ z@qHN{yr7kecxCw1J6gRa~AoiBdOw@Rq`rxvYZ7Cy$`?H=6Dy%2kwyJVnC-V zQ_0Q=U2aiUYnm6ClZ}&GIjy1!4JyN{_&PgnGxY=zfaF`DJVcqT@r=x3kJwT<<{D=! zOeKSqG7@G7WREa-XWG6vWK(>m^(vS8Y8NVz6$`h1LvMY90RxP#HhVLl8-TSl$-o~V z>(DE*Zruy{5J$pgO$iAFPNY86_P+i8v#67+O_~i!vj}_g6!>^YL%^4v>55f7Oad)@ zpd(I&QnyNe=IrQ)BBCMeU|Klk+q;YLR8HwDV;0aj zq);U^N2LQ!^DTT{Nc^-&xYF(c<^SxR;c~Q!PC5b@>s40XJeSZf*gZE}s_K(KrPzEp2w$rAwY!dV}4L5tB1 zBQJt0Dd~fYPE$HM)xZ~ry~dyQtpuZAPOe8jG1=~L_5{C{DXJ)Qcc0A=3CF@{0T<+2 zvNR15mR$>XIMaFl3COG<0wc<3brC+aE`4hY4b4$FqVOFQij0ZmLks^LNt$OAyWT$k zhK}F-+-@ce+JNCa*t~d;M;7kK)?#w>_I649n>Y5^ijbxJ*~&X9Fa57X8`)T1Mq?m^ z7rneh8!J75&DbplJ-epj)Gb!wf3Ky6tRfOlOvRm=v8N6h`$KU0e-7mhfF*;ZoF-o+ zN}Ab}-ne2Sy{zK635^7CK&{nhGABpZh~10w84r6@k_APWsA^2pv34I$g1VDWC%>8@T=+l@~0Q%t(nT3ebSpA zwV$7y)i0#o=AE0u`{Tnqa{+6{VuKd3Oh+QdayX79gFTP@$6FP_L~u;IuuGhr68?a$ zSzDv7WLgt4onUf-uZyBxk%lkYi{9EWfCRD~O;J@ZOtSX(R##W;=gQ8s`p=Ag!Hp}S zHF&7sn}CVyOeKPH1#0HRmbqI`9Z{BWO-+R@y(;EEs(F_x zOg8;9z9ycV{*+9gGk5K*=v5|fGRD~kkZ&Ot$9mP%feMlINlLCfFSk!a62{6_K(VRC zVRS+-&^7T;Xc(NWi>6Tv1gn?DK8Qs&>1XAOilWUp-MX|@Q%1I~i=6)X>;o>LAF+Wj z1mcw-@Z51D`O~gudiL7?Ek)S z*vADYZ{}vNbhnf-i6>Nm(1V1Vg!Gq0@pB*Aw$}H|52|1F`RcEJs)(Ks+dKjQEm4A%veA#pR42Ajx(M{pOgM+K6~m?wbbeqh>+@3B49Ip|G}aPGh4bs zp${{Bec3A19#*$oV;4H+!hl1#iN1Qswkp%hPxFH!D%XpIy`Q>XY zk?m7`buNy?606b`mo~#~;Vuv%pW1u-*Z|gx1+ZEm9$v<_n`#7ZIUz1gjz;FSI0D1D z#}wx{tnbPKr=;ANN~;V9u%L=1({ACR3^wPk^iij|<&zk@83-`6O8o5v=_K^Htu zW6qYN$+0w3r89hZZR#-+b%w1E-jOEYk`qdg|rv=B|a$(2j0LOgzC3-WQqYnmyI;XHP zpv2lu@y^o6oR@BNmWi5&az(4xO0Brfb;H5x8{(0(ht=a(KVQBHd@oPt(Gc+JRbW|S zY zEt^O;fj&5aS@`PMQUwtMlVEk|2a}9luzOkNbqD=RimVe`EZEedW&_L8VGD1G84leW z55aw0EYexUtYn7^VOx124kT6mYk}ITrqS)MbWHm4T73xHs^+hL(GcVo+Z}FCq%13d zC1+KBin==||qR-MN4c=jrDapulgf5_yKWz<0ceu{Y?+HicK^q|~FTITx|4Wi0QKGX}oE2WP z@03_$Y)YsQ=CT`b#1y+NTLzPYUXR=y@$Vg zrc)DAc-L@t%td#Hv-%D1G`iRf z&$Eg-pfz~r9%dS*KFn|7;nGOAtgf$NjC*>GP0#^lMhI^AL2=aSooVbhbPwe?Y`UO3 zPJrpfYF};M*2HiTftGMVKJU79{R&~GJ_K6a&~fqf?K@S% zU*%f-gRg-H9t*x*=3|}8FneqzL-rj}Fq2*sle+M_*XUivLD_P};~ziPIVVsLV`3K}WV!8AuFL(Wy0 z&l0QJ5gwVI$x0jAOy~3P`aKY5BT+LK@>57@&HuFqz&$!2Pa>F!*8JjmqievPP;aFK zVt1@oiG%OY5DoR&x~-N~(v3)VnbCIaEy|b``z}cNu*5k= zW8bdyhNQyZw53Ymvui~J+Q_|k`unS&6!kxwBwu#5-$*UbS2LezDKL{sYxss+y{#Mg z8V0r5cVT4(=|qkpnuZ|ychz5sl74|t*d4i@mun{1+a@5;C-)|F_U;^xoQqO#i~8wR z6*F%Fhx&p>nIjiUdY5ydf=aRdOhm^&bk9ByXlx*11`v~f$_Qk@(d$X3xGBXd27IC# zzyZ5g;|P$aA!)z~Xw_g$(`^(AGV1hh+?6$lQBWymipCf56DLu< z(JzLmoG$rL^ASG*Tr+nWpB+`0aU+Z#p}z*V_D`=@=;7n@+E(f|UT{(aKlZ8b{EKNY zqrW1WAxEOP=a4yxm1g9RaO7k+_b4jHuR0q-5)lR#b}Ui+Ny#U7whMuYG6X}M0IepH{>^A2|MRd%k0HfU0j(>JuqOH(-oUq0JDGmu?tl=lM-eG+}h zGO7E@g!Vm0vgauF6JU7;%(~Knhgmy_QLv5+#rb(`2{FXtJ&~-sozejv6Og~D^z)Y27KJ+rNe#a3k~ zNURc~-&6^H6j3loNrM!hueQMB6xypnNid{ZV0cAvhskvs|Zy&5w253rL)-Pe^)*xEa zPv=^KMpHSq3_U<=0|j0-*&_GHS+^#5{i_86D&!6I<(J*G$|T?*j}l#fW|ej-3gf|M zFz#=0cVLGa(T^~@n#wk=1;;)b&2D!tFgLZ3Dnu8YvPo5Ll-E_bwz?B-%`2^w8!k^L zWmIJ#Lud$`g6@r76J+FewdW=zJLO}aOjCSQ=ye^PltX=^h}}QVDZWh!Y+yL)@l(`~ zn|?JK@=2{~VnbMn{9Ss}N2ISPW1Pi~czjMrANT=*fei86SBMi!7jMq%26#_YR)kjA z<9`eXFGGsUVr+i|xMGUyQ5zZ6wH#i+7ta`}hI%R>_~dVO`=>#X3QGvf$G6!2k6qHE z66o|VibR;*ydrq#g>FH0C?Pe~l^6rj^zv|(OdqvW3_Q>K(XdKipAb+=5NKi@e1fgL zs>O4~RszD9v9m$OHnWV^ef4qOWfz5+`0(4}8{AJ`aS;>Qh2QBI*3gJ2>nWlo+JW}) z3vUw4*B_6XGaFMMZ|L4v1Vngj&Nyka{3fJADm$09d&#My@Y=o5oZE2$Guh=^Lp{Tb zCKhE=2mi{FQ0?=M0 zz2}WF!vL@HhkA1}M*SCyT8a-Cd9I~fVHa}mtSgq2=eJ}Ui_n=ddH5#-5t?y-ehed- zU@ESo?X_mn5gKKH@C~kAz!s!eRSh7-^zTt36>hAI;@eKds$duMGv7Gkthr}S-552ub1*}?-y=NGY;m=mL*f5xynUJ$+$F|S|oOD7LVlYe9oybM`3x^KgC?0GpZGrF01^hackRtBPy^j5Nbvk|Lh*%joS2Ll_ zlBS+$+6kY#kfnaf8NYkvGP-m6*^nLPBt#EiI%MS&Tk~c+mL$&VoTK|t!tkPhIO;ow z`X#qNTwQyt#4P!8vfBH3hLclOFVP07mZ0s)LeBcBFf|0jM)J3)RG_MoSrka~=rf>X?rq!502KI#BYmY|A!Fb*8(abPeCdSI zGu(D3<_0IZ0+Nfc+TrA#eu@>{7FxN9qCg6iPSk%9xAed`cC|3wM#me`=2;A(I(Ab5 ztro>}y8U2MKO+ytW58dhwb@^&>pV`yh}qX9tfY#}N-3M@AZ0zO8r4E!CQ&O7LBf-W zozb=P9_~b|Y->yndoRezK zVZ3mji&SB8bOsnI9b5I{ByT8G@qnT2i9_VeH6>H=U&Yi%AG&qWphi8(%Di`RGoO3y zUTL(?&m5v-@heecENJuyp`ExfCzpNJ8nb{@A1L=orsN}{pzjpsrAM9oD#1KwghCiG zxB>R7I+cSjF2)T*4=|^_J65!IX0>AQgaYWBZ-nG2gg@l>2bK#9Q~51cTO`hHO6ZBz zi%(mcyBlzA+8o#@qOPr-W9TL~EHo@12dUrM4d%s39<729qop15)2bbRS21x{#yMgw zO|s=4KbT1?1SWX+i29nyrPZduTWSUa8A<1fc$FZ2*93%^p&@0*pE&a78)JSvlz4$Z ziJoR&t_@s~Mk`<eLhMJxoS@WJ6XcKI8d{*tN>Z%*sGHUfR zqGoABz~Imczdt(w$oz#UZ~n*I987vzElP9chvNzfojmpgSLf@G@RGjd zP=mNSY479-(f}`GZT$)E1ZQSumBbbt!{9c}!H0F0td90Zb#;;`z%T2Kw4a$)T@Ad| zG(Qd+5x~SGHo%^1Cp&Mrk$yrkj{PdRMF+JyX4!&00(9dnHmKZSd`%x5)6B+%t{O(e zRBSi&O(j+@BowSWEo`GK;2~t#)}&)Ojv}0SWT43y%T2*1b>x{ie{4^ozYqnY3g-Tu zc_OYtz!<{dkPM6c@@?yE#*)shrI~=Bgi0K>Ai8&AhOZf!bf>>`S%cnut*>HX!7h^6 z^M!VUQHg=y(##Kl!jT8YA^z#OTk+X%6Z*z584pQ+aUrA5ex2Rjpjm|wvDw#Jj^u?d z!Uix6cQcOmoOC!UczL=24V;gGgu7||DlD{`cnc)0^de55tz2qQTqI>#u};>6h%WWD zAdUvvuVrHcori|k<=0n8?ig)qlxZh;J+clS3&zXKVa@#H?B&s9=mP-!OCI_%Ek zj6jyOK+*#z@8Bh#ihKFb3%-$Px7%0Uy<|Urk9KRBL&n6V)nb>M?9y;ud!D!7?cld1 zERI+N_J-yLcYX=To)4fj%^!r{;6`Ydm@=1+yxw*3sjPP&$cL^BQdLuN3QK8QP zh*(V$X=B@LUCOhT2R7?K5m?SM5w2_sU6Qo^B#!7=mYY(1*Oic?F<6}0r@p#SrPuF_ zAMWgxAip27y~#XQoKo^WP}xgc_*!;h+HlN03&6xVXs)cae4HVi;+-?M;>>2%r__)l z%e~pB5y+$&j6v!Q<@(wP43XHpv8f5y6j!Hx5Bxrg1?bC)o}`~3%X7CLV}J_!_Pwrd z%5wA?yUQHw3w=`+kioP6J4aqJbwfRB`5x%$UX0l@u=J)&CD&N@BXVTvH?e#Y-^oo! zK_g2!Of1~+Qrq`0g=|_l(3>MoGoos0dv(C9@gAthYlftnWDDAvK{0N}>n8NdLK}@M z`ks*a=|Y<}+KTCh@SF|<-j{Urk)g!{(-;>u+3Lzsuwp{|&4j8B203u?7a|(8HsEpW z!>vGePy`$CG{$S=8UT+vumTU8J^~|S;o_w>7B!0sE>^LN&ym?-@Q>)+AHSPL!tgX@ z!*jWI&IQ)J!O$?S9Or(fRkH9w{61l6HW~l#x_ftR$sPwI;HH;go5!tVH}p=syAlwY zsT^Fmy4V-fQtvNtnl3~sW}xVwuw3R8+QYDMM;qHXaW ze#PA+cw9e z@PxYX&0!7(0yzquIfb^{JXAxfO0^-Y%RAiQ2~!U>Rb4PmaZs<*tHKd~i&G)426w(; zWCSm~pkhW-F!tTC-%biL7LIq!W?gBwV9`ae>yf2= z+0Z5-M46erWAN%|ljEZ!xF{c2J@lujem7`tp(3S$p^3*TOZ|JmJfQ(aFPok$@CPb- zzL@!^cFeDs;0h9MowUM{J_oPjkkTUBxRO;eV5&+{#h%n<$COv;W4wxy0CNe&N@zC@ z50y)+H7t+;FP7J4T=o)Owj5cz;Y)g0(sChF0rymtzy3JEJo#WQV~;X-v38QkNkr+g z4!>XVeE#uI&k&yWAMhN=`Smcw$R_&_^QmnKfPo55^rO zBr+{^3BGAs(iY&IRa0$C6lz>3fi8~*%`-7FO4bfASI~Qa&G4L_Qen{ z*s!P|Lo0FBfkqllMr)w2AuZ^abJMHPRkIPUbWL!kj&@RQ=qbM? z0$eChv#xhn>1Iu0WM?F}?gas+J03>xhxj$q3M``V-sP;Xo34k@ijA zyLaUh(hh{%vFi*2t2sHjXUNI+G@F=Z1!$%egYQF`P^hI#8xHu|7?;E7Sx2Ds2d6KN&2u+YFn2urHP*JKIt#_MoO9CHwRDGWYaoR@W=r zMC{Q`>B%zU<2UB^6jfG?Qr$}>XD@_Aj1>@=$!i;GUDpKD_EGUz9yU{Y|tQj(48O`ppzNguy zcLtV1vTf;`IEqhvl=)PnDu<7MPx!J!q{FBN$QhIcRx4LpKkgPuYI6VXv%?$B;w6$8 zHE7EPjZ~Ms&yq%PSW=S6WX^jbLbI2-&?i#e)J(v9SJR@nM>uE3GnJ?;VEeU_DLQL3 z*M1p982|Der(W!wf>T;K0Z(qStZbTCJ0A{`o$l}|xGLpd(3)OBFpu@EL2CpK0ht|T zHjI!}9H_7X{H4fsz3UdOO+Yvg5W;`)!n=vp)V3!FF^UFwAI`&kp8dW`|JKQu?o<;x z@<)`*X&TSj#kcOIKBskfidSl=t$U^VzFa(}LMxg0LM(Oz{7~()z3s~kr7sU9$_fa) zL?vcm$v|M@rm*Y~R6u9@O_|bx7GP*agAgCq8MnEo5k@=MBj^qVYsJ-r9PoPpQ5^K$ z4AieJ;K`9nkOia5>GO_97_L%yqkEwij1vJv9*_6*!^lqGk6ibZZ>o;N!(cXX@DNU6 z!9sDX47Cns5{lWY6s-%FZC95@IU6-mEOD~z{x z1^n#>W>4z-+Ao%QZ?w4REh}#w3bs`Mp+j1K`^YB{3I}F6n!Q9GVQHKMg0#}tcOYVF z=43LO>|V>`uf{7xBFMBKa3$rkYqXGGqqb)SH4E7uK+at`6I$>olzF_}6F>wQbQ&-g z`OR6MzO=lO6zHvGv#7qE9F8>Y`w|<8*4li}RRW5Ht8_jFh{u8}F0wUW9%t)!Yi90M z3Wxb5u+p(V8IS*vmX-vm(tg>t(@O=?O>I`^8DjZ3QXaJU%wUvJ2`S9tuTfK3nll0B zquSr?38LVzhL;fJi>w6yI374G5b;A23XjLym^d9uahdI_HgPJ=n>vgo8y)uX@H$p~-aFcIueuwYfZ50q(Oq_dP(kv|XUuFNUU12I}wwH#eqe z(mTHI-8=DBySEgZ!_Csy25}yF?{B_l|QG%3E4?tQC&5L`5ES0?$nB2fS(b{`uV(;va&o8NtA50ka zI4Pjgc$0(7KR|5D#R>4N7F=5!Q!zvD=hiQ9RMB}`j(!TvY|@pkG0%*5RhgS>5%pLj z%2gh@t0{lz65F=&lF^~nq%U5A6%iGvBpT_>1h8_WyBTYb_o(m$KfemvI{Kx8QHA8-zI5kC=oH3dKw8T`j%%=CNkV{O5H`@lh($gpYnpV<;y2ah|H~ z{}u5w4$OCf>)W&1^O}ZcC#KmLaosqM$ra&H&ZMbH@EHE_(~ie=SwhNAq*re7I#y`S zfhp70cj=>!L~pebQH>%Mgq{001FM{j#q71KqSA^Kv=FQcQdB?&1ceW87Cx;fO?00j zJp-1G25%BArwWKFPyh}?PHtD7tSm^j@0=KY+593v{jGI^W0s8R#A2F=Jl{7F7V}D= zI7a!{X0ankJDj!Hx=~w(g0VZ`8ti>qM%oCQN%xfmzypVCFT1~QobUD8(Z)vX(PNL| zrus+8&$p9JwD{JuDvjFdqm;VVX1mD&Nm7#}YMtBp{YxkbObSFK0my6Y@?ie}PPGK2 zD0|hNF>>o4o0d?NMP%ZRl^S%N&kl^TIk|$eU`Cv#k5O=36JK9#UDbe0(t>sOw{pc} zb2@E=sKR}Go7fSssY@!;O7`w#!!zQD1Z%>K;uHy*DzGDEK1>T6xD{~Za28S}HJ{Es zVU0Ghr(W8c!#*j)8kjqIPcy2I;+-qXbQ?6G;^-nO(vj95dm4~h_^Zr9?*EO!3|*?q*vPZQD|Crftv=bi_#y7*hnf|&x8y*nF9>*q`4RZ zmQ*%w;COA->U0SMLPE4MAZpgdkx-0DP)I+9U=Tb)gnn8}uXcAW&hpGI!ZQ}oq$d%_ zBNbv3uKi*FtDkJeW0rM*BbnQ(K*XeYRZwDw3D?Vpj;YbLD&x`li8pe+XYl_3mQbVk zK|88vXs;wPC%f#Zi3fUtB_g1W{Dr8{Z953$pUvro4S>god~E4+J0@quhODj|VJVd- zvLuprQiSt5BOse{ptTdA3W^>uC$PEok>ojbo``plnkYf_+FLs&cf?0IYl7)guq5)T zcu#Cj#-I|RubHhRxdh_(k0$uoJe^G-8i}ZR%-Jr%8)-tZ5S&L+C`}=iX zD>Au0HEK4WYu4?TD>{WJpI*YniDL%9XV-)0)0k3}k=81tZ9Clj$T?+O!*b!pig7A* zjvQRd(?q6@fq(wER@hB@8d)k<0+XoIi)@dy43j+_EnVMb2G$-0ZJ~ zFlc!4y>bTNDR~g)wnK#=mjx(`f^=(2qE8`cqVkD?1$aHKV3P(o9o?E}_J>VRVXS%b zy~D?V^6A#ClZQ|pCD#;@^rRn2HNf1PdReh;%bb? zudfZ89%A9^#gr2mqe^*mJjR(mL)VLd9wF`Xcc$2NFq;XQ*#o zk+0|Lqo@&5XZLga9#wjA;J~>6k6t`B^XlmV*9wl&tv!r@Vb@Sb&yxUZ5`|%qVuy@! z9j!Q{G@FD1qA@^~cy`52Jh!!(PsLBkus4!LPZ8{TPKqSky_j5*f#Ir#s*c#N3bi2- zS<`e@U2vovLsI~aK?Kk4J2x^&_tU0;YVyM_0G*q@bF9^jVKk~vphlxkd1E+A4juL0 zeRjG9%ub+~-={X#AaJCl;pV3A-tJYPiIRMy+TwvEo%>#tuCYlY&46CBcFw~UjW&lK z;KziKO5+Bj#0+qNVEHlxEdz`}+g4u(x-^li(!dP~2>~O;6sguKIX1lT!h)*=&!fjv zpwa12l@qAfe<0w)B|%iyJm*&|wjP$00O;5`HV2DTTuMwR&MnaDl~Wn_ozvs-VXlUE z2Lmv!P8s)=KHYrTF$AeJnNb7F^OsA21ci_$X}}k15eXH>u6GDj3@OA8(yS-bSBRRO zyPG#o88M(8t-vP?g>B@A9Ym=~OwelsSacGR2}lCI8n=dI55-=_DZ(XGojDotibn2m zz^ZM90Ax~u;6Tw{=&stkA`+ZOY?ybPAgI)hn}Y&25Snk}v}H^v1%4$( zcTF99B^-oQa5CTt7>*I)^3ztx2EDt4G3uo(Qw=*=$q=%bF|%iD0uIyK*u{%Z4eIO+ zItdiEs5S7qrBb7C>F%CS-ojP}CGGs7j2{U|^lHfhUHl<3H9862YtF@7DNw9-#QRv( zsVD-YTJMK&L=nmdnV*wT3Z;br6ZnXOx?54kD^u=Hm2uRei&y|i<$9X07Lc8rRWj5A zYk*MeCTc3&;an_yHgR$Ce(Jeg)t?FEc_5Mh z09a1Hlda3kN{9e-?&W1TMdD_^lcm(9Ms}B=ZM?xN6b(nM^_G`z-W7*2=7>kmUI@ZU zX{P2ZrR!I26yc8vI`VOM%vu+KNhCuRn+dR@xhf<_2dpyplu5S}N)T}ONY2Lz~ysas;1y6{OU~bU4qQ-0)JwY3RT=dcF z`tw+TrABEq);(dm@2{4R5;yYxP2<7^1j4+-z%(nx=BPY>C!s{?0MBO7@hSi(O>4^= zE4Ad~Bxza!`7kaFi2&`^lI?_6kaizS*Ui_g>dKW>M92qdEjeA_gB}Ru%3Z4zPQszA z)5+A&&BGnw4#V2ObHM%?{{VKoH$7x!rYcG1=ysnjh-pJOR(v+e2*oaWy+8Ix$92arV|kxVzqs;_tCtpT+?4JEQKBrtpWMGy=6T@NjPxfC&)y3 z%d(X=&;=+Wc^@MI0zg$_9cE3$gzL9{uL@L9R1JJ1vX4TLlx>l>i-aggh#=I?*Q-<< zTNBUg4Wmx95=w+s*bHQYStp^=gAi?z5LCj8Y6R`WE_zgyk~bJ&3QU`5j3eGW&QT)>+1bB1WD0oy*t1njmL zFz=Z$X=HG55NLhT$N(J=RAn&dFeYPE;&Q~PHQMckJDb9okDs;b!GPgYJ|-Jh@vbT& zT+h3m?;rv3XzPTstN3&$vF0@X@YV=76bOKJytD5fLcqv@som@^ry`REjFw{<^X72t z4#*@CS15w$igG$GaI2BR>y{#;q1_i%bl@>p$;T2A@7pUxP98e@8a2$N+%#Hwk4Swn zwTo zTM0D^nsT_Gam$?(9%trD)LKg$?l);|58d~=(rFd>HS9bZHd4|o-2pS=l4tfyS$t;g z;9^wX8w8Ri;6UZo*hAVn3@HhTqQl#j)kCxBdYlG755Uh=|db`opA2#M^-C$ZF%6QFltK_f43G8cH`N`O*g zV0d_I($MW+>M?8Lf#KTvETYFzaBp4Cc88p_lpBBU^3nB+woW>Y&3Xf} zOsK0$2<$GPfH4icH@E`cNkys}0yeSRUCcsY2%{~-j=%$2bFiXK6-{LMHy~WA2~j&u zS#2pPLX|q7lZkyo3KC_JI8d10hiC_Z?~^H6h#%C$aYouO`?d2gQmi)9i-Nr6)p2uE z($ozJ2MLM%T31d=v5RCL9$zP9otp=r=Ju1}(E9q(&<_bxxDHoQZ;sY{BSm>}@-%oy z?2j+sE?6mqVwn>g{{SZNlt!Mt4&Vy#u7bU#{{=bUOY_gh8b8{^w9ANUtWn zf#@3q>(2+~dK;0vc5CNXRK)weACoD~~7RJH2^b0+McrRoqSm ze#z!>F#_>KU{@h|0#j|a`kD-lj!rS7nP%Dzq{=~~zum3z9j}#{AzN3AXU)y)4)!aS zg`p{v*vE-mQ_|&FDNo$oh%|a&5D%#AW?B-XPjqVNA_bTtCDlZzCTUkI4)1q;Sd~R@ z^67d6b21{a4-`%!NR2&9F~#fqo?$b$rP7ergK~-h#`gm4ke?6$5gHC5t>BGz?w@14 zCvi*dI2H~?>l*(UZZ4k zwh1<^Bi%|zAI_|Tx_tC`sn?^{TcmC5c`|RTdY0o~V|19`!}6oP5jsEUpd2 z;v!>rI~q|GNTVXH&ZN_X{M~y>-X#}Q2%w>j!MzhrU7~5QdhC}VpcMWd(mE*eKtpE` zIi1J?*x?LiAP`3ZS9I*f$0nNQgSP317Ue z>C=(PN~X{!d*;#RlroA5yD&;ZyyK)Y#kHRb3Qz4%L21fX6r+olEZ9P*&I|PrBxdosg?vgzWMdo>LNt%cDOuFsRl!~1r;YByNAfZZZHj?@iIA`815?y zK^o7=z(~f$aLRPr5AW37@p3lAI>|k6=V^<&tQ?{|il=7!KD-_% z_(gg|HWm^pAf6thnTlF~*JmHdJTbGTpD@}}K)Lnvmr|)eys#dgv_F&3B-c(f9W8;p z&m+Da#3!AI31Agak6)W1kZUpPUEPg=gqlmuuui*=ug$SicHNme&XmPz&47G?eO40S&=G@)cJcElX6jm%eB6&C)OdrX%npjj)f?dX&;~D z_b{@kuJ=?^&zcLg37upuLV4H_Q)H;Bv%GDVJ4eQ;XN zK*iVF&?rYowW8w%aN0-(Prh6Yq{ZL7S^|r&^vQeos|QsM!}1TAwzJ}r?`={U>6aEj}#;XIJ(0C;s9>-*k7QY(bw2WA@k9a55sB_?+t7f0`P zUAhf$yPX6IaUZ$RN5mx1NZGNRkWwL1cE*FYj_J}28S{n-NcU5S1_PBJ#XTJYj>OKO z5B8s4*Mkl`N5VG>RotaynA4TOQn3-Q06CpOjXNrNnQEbgV==5s*yTXPf&S1BGXlyT zT~JkBYVjr$02Lof4`hJIRRC5wSXD{_E9lpq&{U-I6UX;9L8hdSAUPe93Pi}{I=V(9 zj$m#RE`v!aA_-CCk85irC}P{Bx`0WMRP7sB&9GPQ$0=ZHqiz*&ivDhsNROO$AN>=7EZ|EN_!2{NK=5&Bl`;5x9WhS!Y_3}V za^SaS+O8<<0%Mfuk;IVwFS|agTeS<$tE^VNQhmY7sugqz-Wo8J-Ux@6fUzDWARQ)a z?PY?m$2?!YUdd&1R(hM(P`hJ9#7FGmf}i%o^PckhseaBYnKc{8+ibwPKV~2=pza0q zmWz(eLB^XCxUfM<_Pc_sX9)!aP^DhN@oOvCgUY zvXEmGaEiJLf=}UBWW2Z~F$8VWy)NQSQtT9AXUUrcmqI`Uz$1%cLy^+rz)ts&Qg#QlVCw_8AnkMy z5OxK|ExSORlE~p8)t+3SCo4Z?AwdAh+I_WY;{D|WgsoNGE@iIoN4&DN@=-qCmaGw# zw45l89-uy#70S(EBa1nOd+Sns$vp5HifyLY_pU7e08+7g0Ow{R$9(uhJ2Ab><%u;$CCZI^J!RyYjH8Ce( zt1YIZwRoWe58;~aE|(B(07;xq;W#e(L)UXDNhMMUNHit;FJbF+kqDrQ&a6^Xqmgn< z4I)~?JF3~Xg{txK06+jn`uc`)irPm=MYC;pln|Mn`bQVi*~?q7h}BzEcKLj&?QY$? zzl9&fVuvO=S4HFi^eDzFBlC)mt|K=)M~u9BGE zS@8gma(v;yQ6T`FDkFiDEHFu(8Nm@x0QjNM+ff*Tq;DhSz#yCYkYjRKl>^qU*3)c{ z5bHSmvE;-GcLj2~3gQUZdR_#bNJ;+ybD<{Uu+Edn%S5MAqB}cX08QL90y!(r>jhA8 zj@q(fLbVJk2ULjl;PeVB#9*p@csidBqip@_H!LNw0^KYl^z7oXV!2tmzZJrG81Rk>0j%28J;z6h z;n{&)rcW=Cwk_DUi)kB$N#3KWJWHDCiIMtt5{(Y;2CmFeBfLACCEkesHMm%!ZlVRm z`NxQGREOo=>Gfkd-T1B}{$|#=&^$>W)`=&~N_f9(OC#q!D9^c>XU!dAkRzN?;;v&A z2es;ya#-K-r_N${)J&K!!d zDJna>oc9I3?Q_@ExWo~wkQB6NOR}%2TH)+^wg|yc%a4m;Ae}oI`lKjC5bc6`RFaTs z5c%Vpwv!Ne)Auob<1vX=t=9p;iZ#MwO!!2$5m<_yT~V%Ls48=Il&m2)Cvrgl05Ef7 z6+%=$X!{9zi)gj(in|+oXrsnYvF89?Ap6)wIK;XDB$5f`nDz5`7DkD$L&CC6R~@IB zz*zA1bG%Z3q?&B@2mm^hTtjDTz$ETGSOrccetw`a6-17mKQ?5LgD1vEn;txBoIc*h z1&ZJUGIt+YY)J7Gi99LL9h8bmsorP@AY+1`cqopJcLA!Hn2#U_lr~gIq?5GCJmwuM z6C@8^teS#C`(}Ba2FwW>?vGxF9ZD(IdAcT9F#;t-dh)SdxJVuzNqR^Y>kLw7Wj`kq zF+Ebjp)ou41aG>5|VskfAC`T+j7w(Z&bPr>S zk`1{p5~>%hy~bQgHuHB0AShFS_B(y<@iC5khq0wvUJ|X3QorJMyOk@z(}fBOzjJE2 zzCq!?bJ8v{l#?1dl_zUaiX@!H$fAqX=Cc71E-DP|)uyRuW{p!JZ>?ON6I08vnn@L^ z7V@_4EDTCgj;GkoacD}g;=>w*yPGKjWkANm1DS##c%#Yrww%HR+a_ZP^niof?}U-z zuPekHUKGIBUz^pXe~5z{oFQb}Qb915JOx6k>1}sL5CT-h4+1!yjsSqqgb3oT7?cQ9 zlQHP%xih-kT-(!2Gp5)09Jqw_A)&X%%HRQ z0qkHUK*U4QK&c?@B=OkBW?;EsfGbxSP)^rMGC%-8JYudnTr3@g5X;!Ypei^Qv~`55 z0D*8wX0ohEC+oX&Nyi`GJRr=h0>;$N9% zvvMvggOpNL)c^(^pvdy0^{=QxJFfL3{%+vY{{XdR`3)8;`=76UNHN`)xaODt01DIT z3}kcFODWu56AWn(%DS+X0ju<$M2oY9gr-dzV)Gu4YmJDjQpZGvMP||uhw^ffc`D%0 zHo1V(cQLG@;#py&BKnc7LsLb>+!6~MB^n9{6m+|77Z9W^Azk!f3PDyTe$Nl!@Resh}1-N~$?`VFNGirdI$2;vMspm9!*hx*}^ z>#_*Hw!-RvrsO350DA-S>S-#qfm%GL!Tm3M2WT9I96*xp0e(vFGHTPfg7qdfNOlP? z3Ro6|t4cbMp|#j0+X*{Z0EIaTxo~QnOll-*CDbf(I0?aNR>!9ch%2?`H4T#BBz1gQ z(yOU8gX_J)>*HrnKsp zg?NuFcI|CQ3HMGd`v++(47jXXN0y$(2`Zs=s-`OTF+yez-KW^<13nY2cSRjiI@J=n zfaUda0I4pQP*8wWl)>FstK~%PS{j9rLD@(@Iz7Myo>Jl3DwF~E00WUCK2Gh&xC*E#LP;h`o`=UO z-&VCMbtl@_@7ub&me7Ti4-%9_S3(B;hem;Z(!`yZj8*i~y|#DKS7kb*aZ(IR8q#bK zCS{V^umJ3!ZW-MBo${(98!`C3dss?JYIZp{4j>@9ecfbgp5MvRG>GJq?mTm`l-ekv zAkBSf?5he;6%OExiF9mL`Yrz*VTPh;rShT#y9 z2JcyZeMxdS>OMu%l&ngYczV^ssZotM>kdNv2?AJw6A<_uL!#Zq2?NX!K5e0ECkO-A zmDdf#*Q_z(wsN$}UNWF}K0#4axw+S6w}BD;#q{7D-or9xEJdmRm}HlPxajAdxFI0C7Ds)uQC@zj@KBdkyZq1<7V^u zOoC@Ic^3Br?;qJnl^-DeCn`vk+O8tnN|vOdBULFt0owfQJN3K_NXgnfE?wsc9TIs% z;qzJi-gQ%c^6qX2-5iM?5#${*rO{4DYH4H{{TfT@)eQN0-bxe>vmf);LW6E zl`xgnR*x^@N#j)Qk)!fAjAcmrWE7|Z41y#P1Rg^1Dk(HDUxIgMerJ2EfoXWHO>Hiz zJL76k5dB-bKCtAw&Jq>W-TEc({)JF0%f-Qtwwc&{ViI6$S|0L3jw{;}5Fu`vWiD zH$Wa6pj`v@z3=Y=`GaqCzmp}684N8Gv^uOQI~@=djSD%uABK4z0T&pZpmw<_KvhDg z%kyxTaSAj^9B1VW*3uPJt`6vf&i4Mmlx+nuD=ZJXV}{0W5FVE;J+3*-EhedWRskDo zx9dlAl%INz;{NwV0H>f7ShjYbCSSW-Bi>g>vBn-9tis`HV1l&uiaOV$$;!5ngRpTm zK0)eHO87$(h8+o^*G_7=U?S25NrAlTdR<9PFq(3F{{Sa!IQ&tj(sk{traUJ^kJ$_f z3brmr5vQ)IbSY5Oc>`XDL2yhY9#Y-|CP*B*^sq3}f`s}tFASq`JYKAgj}kVKEDc5? zacR&xDty^IB*jXPAWv)APvWDBxKZ$k=U`l^2v*f{RD7M1s3*cq?;fL}(IXcQUt_Ae z!Mu*gKukELK#p0f34x-AEHRa!O$qafU62s#V9=?^LG`B;>Q!t`swXE>1}=m zk}Z%V*Ui|Jqunv<#lcZ2SOM8sjbZu^PaunkV;6{~CKWAafOHp_Vx&W+LoBUo1aNXO zDuPUx5hRh}YS{}!sN7c1ca`E!k(o-7(ByL0@BaYH<+A4L!WDI1J1D}Cq@<{-{mW_h zmXH7>)u<=Bd(9vEH6BdPLr*N>W$u!C?B^cu%zwtq7r~3Sp(NhDP$&4S+y3XuF@{vy zw)=5Bsae`9x;x26e&(ai(6eb1!vvZrIGFX$etEP@KgTF()dk$p)GzgTeF0 z_1${w-{LR)&!+m+qK^B_1Qj4)O%kOfP?+~f(__+vB%Tiszvf(^P_?3eft&vT+U7q= z>aJZi`@V1g0Be~2C(m`4K9=wJi~j&KUGB9NBK9RIUhEYqO8ggel3F@?a%p^6_<9#2 zESzZ|iy6890Q_9f={IqAm;V4mH&6VoXY`*wo3~0+H*VHH8vg*y^xV7jw|~S}wamI- z_vj`>bm>_Lj;A*GxdDo>rVfRt%=bXlozxxu4Q>?zIw|*p#F0 z#w96H;JfJ;&!X+(2k~y`5AzlNX33_Ke-<~uxe?NZEnfYM*&9FQb0eD5WkgVA^MCtn z$LT(Ith@BLf5cb$n&*3!qUDg4DIklo5~7*${{T!fhMhO@Vd2l%nGtyZ0Lx~N{{R;= z`b;W9kGqx4kNz%W^vgt2?vwo6zv3@XiW>KI%9DJlN>BkYiAq#>E_y_}7(!0IEPOo% zMcz{d)2*D}C*5;Dq{1!UtmX4n{{YJ7HU9t!qzkt&gVx=W2mM9=0GaPwcj<2b0EoZy zHADW`czPEgt<~N74A}nw<7Piigj&ocQ}=Ay{{Z7({Wmkd7PMTl5~U;_yWLWgFu&Zn3y}(HMf3jvrOl82HU9w9V(POq{{ZUd{POqz08OcP zQk10LB}!C8Vp5eK!@Dtvm(C{j*axn?Fe}zv+y3WTf=T>c1j>a~+srP9DVrMq0O{}k zn}n|pAqq;8s8CW=g#!S=3JE$8!`56M^=|(Fi*x+WPH8in&F<@u{fT6wO{>EUYnf8Z ziAW0rb0SU%9$k(yMii@+w60aiTxY3D$=3n9hIIE|XWd=#lx*CZnWvna{0R{LH4zWm0_)70-;&c?6frscTw_mZd32 zQb{E8$zV&ohw417wq59A#OaK&2}HxFRJijjh}DXNJa%6*p+*?WTRn~JuvP*Py>a&Rnd zN{^%+({)MWADH)F*6()Tcqee{kUDN4bD)2<`W>s*ub^DCwA@0pQcfY;&Neu22^_c9 z`$Fbk#$_kPZAA*?T}NRb-hOv)G3h^K?+mxzvX||8NHP0Dsp$duf(2^)I??!%=bUn} zL##`%STu27XkVI+bV*gI;8=x)&Z+G_$EjL|_arDC3xq6=un!Rj*OB7X54@7}TpLTY zrB5UwVG~0R(Hmuyq#A}Y^onf5rZ=+iiFyH5YT}llB!FIR#Iq@18Rw{vC2kW2!gZn4-sJt z)=PzPN=nrP(6A(}7*t5paO7}Qr7Q6ma}r6mNv|*E4iVk4)B{=;;z^UcS16AGjm5^Y zkrTFDtG?o!m>}CpDvi6S%!Qm)A2%5)H99*J8afF#ZWM|!p+!PTaaN*Up!f>{>LV2y zZrwl_X1vWi-dr9Gx3x-kQSOROp$Qw%J$fHBv;0LXVJDrhpq~%yj&k{&(LH6}3s?Qm zdV5*hN#e(fS3HhiSv(vB9|1hB=-T@pkg?TxwV&STqm3n*Bbmdy{K7qSdLbh&lOYML zsrA4MR>D#^dh@<#AO`SoRu9(_l#3f;~k_Hdei{=YH*02?m&P<#pf zko&IhzH-;DDJepyD{v68i6k5OdJ&=IG;_vk(fiU1J zRqxB-?N810h|OChf5`W4&;1>P&XV|E{_kt!yq9#^%UoU5AxbN>6;P#$Nu=$4VGJ6+ zMy}YLQyyMzf4XFg;6}=S{{TCWlm7r*zx|&j{s8u}C%^MZoHlZzb&EwaLMw$rbvnBZ zI+Au!srN_hhDQ_qa{mDKe3$qG+UfqcfBQa5`~mHF-98}d`{Vhxc`tv&Xah<(>hJv0 z%&OK^Q89|qR7sQXD6#(lu3rAllK%hzdtXWE+_r$@?YQ&YmZI1C3`HS!^+n?=lQmOX7~I@%zv%l{?C&C z004Vk&wpla#ydGh#KtRFqD_6}7LJuEE89SkQrG*Yh`v8({>#2klm7r*zx|&j{s8u} zr|Sl76Nc7VGp-wDGeRqcLUlU34LWNQ7eMfdtpl-#8~two0QP*B_ygMC`rYsB`7iJX zwdZx-Q>O2z>umnb@A!eBY+_%3=B`FNIYq?AD_EjTn8c$K!Pot6_x603_ygMP^zK_h z)b~fbfBnDk?!IVyspNQ`{{YP+-L;iW>xR}Dp&x}qNifI$w|o0GOZ)-tXQ%Kbd_ZXV zHEZ?F=Klcx=l;YrPxZUs+45iD4{Mc^zP!@zU0elRSt`OHd#Tc8#|)8pnU=unw@Mj! zi!@O`hyo96vfX$I!Z{1(xE(Z}G-acR;Xb?^OYv=FJ0h17!<_zb{gl42+I64x0u)=8 zFv$G}l^1&I92@YL^9?-@NSr{%-+? uG+R#RQiSe&*#Qfi-N=_xk-3!=E&@4A#MHoQxD&a@CxCr))@e{!pa0oAV8Uzw literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000024_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000024_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..cfac254a78c62e341f892fef64148fd95bb50f5f GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5B@6YbBq(KYPFT>pg$={_Gs}9E|{e0;_4L z0gfF90FE&}fTKge#U~aXR(J1+N{Wi#xdS-r2RsCvW??zYa`yc1<)@23{ZCC}mn$B!|e zW;C8S_K!QqPaHdWis>i7@ngqNoH)(Qbb|3Yb&5gY_=!v6CvUPo&~bhbZMQge>++{x zqOQQG63Z$~dDU#Z?4!}>6`G4>%Ik*(9r;1Jd{SeQHCC?Q9!a~UUizKdDaNEHPBNxr zp!h-Urua!6)(7pEVbJ%VZm}&}IOqK`dP>5wqk2VBh215J_u=cKuYgOZkNF*cb>i49 z!0}6f6PEx-eSmW(jxj2XYh}Qi04(f!XSo*8lSm|DwmJ66iGr?`1rIy&ON_jc&Ox_6 z>wD-&kM*h=WyTwr9Rb23ns2|K#YRP;Q2HKa4X^dpvECSYmnnZ6zut9v3wg>PL1-tO zsTSBl>1(QbEfr@I4mld1Hd}*eS?tO{b!!n*2d%2h+UgF;8Z4M`NN6TCU%!}Zs7oj* z>1ob%QBGoRO;^SDzyugx)U!*GoG8aYGw9mOUFj(%?|$^yL+)W9$sfxR%|P9n6&os5WLJ z8s#mE@U0@Fp-rG(Z(5Wl@;bud1MYT$wzW*Lt!1SqfANRhIE;7Iu=km|y1Q6ddAXqL z&!?T@{zV=6&&y&3HM%iAk=AmNw>bz9OiW7`72Syw35dSDh$|V!#m)X~bR}jQ}NqY>njW`b2e*bM9kd*{$vf0R$f2nKDg| zB5=UN@B+EwTfQqZ22jWb&zsNe*(bjZb=72&@UKNJ`6B;fnMg*QkK z+Lg_AtR`dWB5%j~^<8=$!5T`K)LLUN(^tHe|9~Sly3^r}Q_~fsx%N4_7(>LvxC$-T z^?k`#;AUo)*gL2O{`j46=vd0+4>w+eq(Ls5yb;2yEJ$M;* z8wZ{;k@DIlT9vC-{iWd5^)wn(<8IO!z|%kMI&V^UhLX4uWj{XyU(93|Y{NhBUPbBX z`+}5xv<)S~wSk~GZ~e9W6%CDfaEjjb7W!G!;x8x8{$bS$#W0+bc<1zEg%lji%cjy8 z{Oyl~NuX*MUTavlGajM}tzdaKl{P3V_&ildu>vFCNSQ@y^c?{fiUm#z{XydoofwKW zZNb%Ul;bqI^<37+p{r<zMd-Mwh~B*(nT#Sw%}8+YsIEd_-9Lje4G zt8A`%e&qZ@fr-@?vJRdkxL~vvWec>oe-4u< zq3BdOk+e~5JBmNo=hKn@Y7qGrqT?U&vL($R%yb6@JcGZg?g&Lv>}ua$LnXHV=AKcL zXliEFn5ZnDsPUzL&KR^}=kwzDjeo#sZe^Ho;ip(V5vFl|opd^oKQrIR0RPJAAHCWD z|1Z`x6E|XzUeWF5YzvWAd0zmm=ZgJxQ?*oH>Ir8jby4!iQ@=KDxyr3b9Cm8QC=Imu zJ*gtW=!vXtT}iBI!*o}PnBtH9c$hoMCU~*ZHXU`rP8ucybM%12HJ-(uG+#ViCJvIy z`#e11XmcvCx?gt>TvZm=`Yv`e(=Tsv(sc}RkeMvp@F{a-WX4RyFk8flRZJJVHK^HG zHq15rmE3YD&}5xK#(;D_6~?2BusEC2dz?55;T-VZqRqYA=6!_KhQH|LvsF1Jo@AsD zl;aZD@NB<*dlk0HX!fl{QP`WjnHeM z%KxF~-TLlfUowTS%877Q=GtAS@a~wq%W~1wdI-3&Qg-L%BNNs@PswD~5R&rmG67$w z9U9sc(mY&NOem$aJxc3)&XdW5O;g`KEojpt|L&TeS`dqxl>h&!1IJ1HVAEvPH_FJ6 zfa;`i`{|VJ<{vXN{@P#ot{0S#+L7E3!t}ot&A2Rd%^$YOc z!qh$BMP#6`Z!vZotwh)_@Mqu*cu3SPFWQ<693FI!QVl199G8tu5>bT;GoNfK(=HJBr9&Vd4Km6MqY)R~% z>UC2(7MlsybDRZZ(>JZM(z@TISAzeVthfysrZ^@WF#U_wT9y z!nzqB(Y$k^;@;|-;BQOz8}gXjnR7Y>@Jc#0${ZRRit>9~=-aLI(4nU6kgtedm@3qG zKrc*8MuIB;T^e}|aV4xWgh`+J{)yJyO@Q1JPi= zj?0{`e-pbr&M(a+!q+8JL=|r|FEjk%SU)^g?#~FYV7KX1_q(113{4iTZqp}P5n4b# z8se&P*z@?R;}d%n8K@otL@fQhI_1tK%fF7ZQ#T7owU%A|gF->&{M4UefRsBBAixkw zC7soDG>#xo#b}nc>&_~DAZDZ$=A$%e@XW2T7`R(1OL3yIAfeM0&(@iMa_)1;K^I=1 z|2vrcNjPu8ZtDh$p{b=3wMg$NURaZ!pP3OZ+mageGBh-TLMJu#@vOzA7f7ISpiT_% zS`ynwLUDRa3kB|x-`2Ec_rqEA?isdv6eM33V>qT&U-$elO6G@n$V!ma7G3urY_;no zz3&qD0pf$jHhW6+D+)+6X z^-O3w>|;aEJQQIg4CyC!5^r}HO^3QG3@byQqO&ZJODQe3z8<#mnlsey$4=tQoFvv2;A%{hNzx^)DxIlOpSFCumWoQp3(7T$;u5moj~ zKEa7Xn+c{DMHR?i!16u%W8`uCZ{!APLBidr2%+$uV#$)EMWVWH(RiKJoT_=LU|$fv zBlc6EF=p`M?ev_NM7})DHwsp&V3uvk=9Uqy0-oWAcajoVXH*01nAGOB+q>PF z^dd5SK4?eG2`2Ji=Kvj0RX{;FzNCIqe8pIaucNEvpm;1Q@|SgEYG>`L5S!qR?xzcv zG**|U5Nj0o^h~g`N7)hJArTh(TKktN(hFKD|JAy>&1~=MtBupqc$O?{XQ*lBgs1qR zn4ym65#V{1hWlDlxX5!vP=%P&#kQ%$EBOtJ>JSd#nFoCub<4=6wxv>UOP7h4$IGG@ zmj-lP9qNil{q6jF&(Ol9y`O=_UA})hAyEk}GTn|z1irb`s0Iel3`7{dI`;D)Qu@cY zC^Z2m+swCVI^WrJGP!K^I#>@WcAegCig2r@SkFwRdX;HdaiI2`B_pv!L9$c8)}H@G zC|8j&9?01&%@^+@wS22LXBLYz3&xsF9ww|V0@a`3;kt>`yH8%5`Ga ztlEoH4kZfZjv9RVOiZqV+%%D(ZmL-O=GBp&#=7*x54O_Q-h)Q4RQ@wr(lZOHDZcS! zYO_j5hvrOFU7!j(G#ISMBPVuc&S`?KM85Vb&0thiV1%IUKKef5v4m336cU3;MTOR_ zZeA_9zNkxtW|nWMvtKvsD00>B99qH0UNMG5UiG>py0Ul#xVeL-_ITxl&I*L27N*4` zXR;)&4qa7^A$Um_2w&OgZTuP&t5Y!@Blf0Gg(%HlUe$(sb@2(E5&Hi6Iw$s0rhI`_4F- z#)lM2N93MUDJcL2m0WEV&-%zEQR?$h**h=Uy_ondV~}t2=#i2?7+vx>@AzZ})vjZoef7bsWla zzb6Vzy_;b^9bjG|1kESxKi)hr!*^*2*a%Dp#M-p`XfAyB)_b9gi?3?taVA{#*CgfB z3RtKp|FZ7v@JWSCoB}y{h2vtyfxpSO&n)h!68Da;5!A~8eH1kbIxf@=41ZzJkQSd$evi3Nt z&`QrLX*lB4=Y(Hoeh!}~qq4XU^Bn<}+@qwHa+L7Fa5SMz!~d>iTNtg2`Ti-suK<86 z!faDtQ{6$Q4ZxJc$hK)Qn`)AYlT%(g9uXIOhxt#vVYdP6 z2Vu>v6n9HyDV)0j4NvSyNy@Hm{XbttB{?9Y$Hi1?+w>)N_Uo-BJ{54tsp1qx(nZ)n zG|*EItVtmM_pSF-*bD>=hmNiT;l?w@qcH&Hbe9uZ>TZ4hLBU%OF#=y&s6~X)E}bmI zY?}w+N|~n*ir0vpOXKjY43OIpM%f}6`RJKh&iF6-PoUmfM_BNm;qTpw1$}!JWj6h3 z18#Tl`M`Xp#d$V^8Ya_?n>R7^2SvFIHnZM2M(6nmc*bCuyI$=rLIzkMigpXA}>AsNH>1>MSccpqq!SXuLshJZCp_7Vcl*q@S{)3wjR4WpQ8~@mI_b9{aEc8(4$Rl0UL{4qf_J?Y%QYr3nuL6HVkCaSi355817 zt?ktVaE*C$tN~M#P}gGf2?I9+MV?-+-0FOHG0Vt3eM-jTfY~Ei3zS5# zxKY(MNznH^6^Ds$>1-l5&)2%?&u-->xn2Ke!O8djTYvNyz~x*23i!wMpZ4_(0JxCe zb)f+u)tF@M`t}>C7RGskRr6;6(+9>wyq1I;@1DF4){1Z3U+<9k*zgX;GsY<8sS`P0 z!@;68M6NnZ?^?vHW6L%VnAf9G4gGOKnH;;4+KUKs&Y`d3Fk^m8!Mu3HJ3~KmSAli&U1z6qZ zOO`B`LQC%hpay?ROB!nhRggeMjda*`*aLX4=FHgDJs5NY9s&9fyua_K3Mgnubxrz6 z+6q;CyP+I!7+-Au#4Xavc5VhlOBnRR9Rb9`T0m`K&IU@siU1#*pBd7|3Jnhb=6)?p z|1l9dImSOjh|AeF-)krcg;_R5}Scb3O^65 z&{MuN=|Us=@s3H?Ey+4@g458q7!X7>)U?ZZ!PKm=Sar^Z*wA$zjGbCk-FP*-%7y1h zUHqx5U=hJ*+cDZI1hqs1*%pPzA=aysWbzR}S4K0pcunvVAza_)&FD%YdPY_RCkybv zj{K>gKYrW*jAZ?caV;P1#mn_s_Q%4@x%NJ@ksbT9^Rj<+J#jn2n#4SkfEq;w;6L#- zZiO~!!LZVvofdeu<@NWsR0r0}^*~1ea>e<>kSpnITio&bBj#$*A@DZq(_>>iBMd^E z0KmVlH`yTT2eq$tJ4eHa5sBE-G0dsx{d$!MD`R_2)PdKbKmHS*mZ|BUeK&X2m4>TA zwOX8()pfnf((7Q<7YUZnHQSgIoFbm7x9gzFnwYq%AL}3u?r@laagR1R_RZVK7+#2s zb-gI0o?$U-4)DxnqGj^XVRD?%{27Cp5xp!>ep%onpQotTa&H$OYP7Hs$Z7OzFM~{Q za#E?Bb*5ykUcNl1lP0%@f1Q~!ci(!GYAnVg1dLm|2@{jYAoc|p>}e55tYu%j8Fqf7 z`}o@h&tz8H=lOgj!J2Fj1UujM`>cmWdAI?|g+=NG!yRn$@`DA`E z=-U>V$|38@&UI3egTt1j<_tWMKP2RMNsXl9z^F{Kr@L@--?DaE@hWiwQVB0IGM0>x zmCp1FA!))`LKT!k&dq>t3s%MI)HdljaujtIP6^3fU%(jaW@0T$oMPA4qcUp5Bgh)+ zXC3qO9E{!+`WtJggvQulk830>mLt>#KOX_gXzExKW2lyNNoq>s3q1obdm%KC$%bCA z;nF9@?IY$&KRD-CZ<%gw%!sEh;=%9MM{pITxrGtyN!$R5OoScmHTEOTJT*2XyG|M` z(THCcCkT^UJ*{V+#x!*nRn0n0_+x`e(m=PxNI8&{`SK>u)jEu5S75pBa|$A({XRCuE>& zx&~VcVyJyo$gJ7vp0E2g51o$fCB;GOlyVsh=2D_>HrLJ-wKId0{tpDy3{qLW4XSX3 z){D$(Y2B<@{9jWc!H*ZKZsT6xr?}$IjFnO_HVxn3{GxJ7&0BrTTrNc3^{rqmAzk0H z4e zIQ8mJ5&ZQdXNUItGdDvgmMitjXmGH)mkr!ZK|DY%u;$C^vS`|@ph!J6kQ39=^$quY zuMAlhZgY`I9I8c$eGLr#yhbb$ba0SbsXTNk!7Ldz)auFQMD131-S(Un(2F0c+=wys z;$EMvmo*xefIibyjtMsxX*FG1T$O2P?}Pc)-s@PM8Fb(qSZ-f;a`gXZy#-25n-!v5 z3~Ox3k2rg|JsPU7{`f9T?gOP!jUztLm21r|K*fP+0EYH3B}LqwFKLB`5fn`~CBAF> z4gARL`s^atv@s?<3>J?k$2UTovZ~H)X;{sT?_dC631?63%h$A-9@zZWRD+NG-0vlw z>0dm@=M7oJ#}?wplT#EEy)ZA`7#TFp)1}`LsFTWBw$H22NwSXCcD~lKW!<_WklwXz z*}dzvYN=6eDxaUeSbm1gqItQw`cCU}TX<*&*0$O5aPA18Z}^i}5*9w=s|sHp%0(&pky#eH=lfAVHS}ka?U3X<+i?? zd?E&lURG@shy%Xs4iZPmd%tTv&tVbnqLW(K0DCPlYn@b@kw$Vjv|qWf{Z@0U;DDqD zm0Tfr_k{I3;DD*dorMG1Wt6I!=R%nv^sKZ)v0GH=XhmfMqR%_W1IJPl0u$5x{^kgP zC%FfFiz02?Sv4Hg6Geuw+$Je2hx+E&N?aj2*VE7M;>S<%gFwfJAZZIlJ`|@CUe%w4 ztxp2k0xrzn%2g3nmjp&szBw~{R^D0C8}sX=ASG`zc0KU<)AQqU>P+p%sh?wZbd$_3 zR~!HKHR}OYzoI=rF1<1>`F4m<7bA)0_C41fJLL4u{tDm$TEpSlFLe8(D&tvwUr&eLoB(YEzB_mxWp4*dvGvt0OwLHbT(7e$61#VP*6M zEwWa*8D3fm1ohoc>4iI3yt%FGOA*so8vxcAxMGy9Uzozi=@v+5__OnsiJP=OfIUjO zm|9i`e>4FVVU7}pm+7uX>8=2k0ls>kY@T5K({#;Ug zkTgAaV@y2^UOqgAiuK&`C&Dwk7gZM*{L9arsc$JkA|YIrFVdwfaV~>k-Ps0f=df6( zX?01ZPJxQdi?(iNio3U-vJkomTD#_>^bv^LxNy5><8zA`u(n~vUaTc3vevM}XpxjQ7;mr&&BiVuEE-xPRH%Oxsylv-!0CwY>K_ zZTLx7T0TXeB+{ed{w$ zNlK>G0yeDJ*`GTX^#geLEWWz^kB|JgVyxGgmJ{BTN--IYRLGM+mOEj>EtU~*bY@pn z5N}xgY?;LA;V}*)Il*}_>P^*$OrzDv(+;+bjb#qift2i0T{>UB_PCUWPy8cf@mLs@ z0BZ~uNwhj!uiv1d%x2nGEsZ};-Pedu(rL%un$?gkZT#}0OF_Ii^o^Ei&9%(!%0x`V zjkXTGjb);Cv`>ZJVEX*NxPp(Jg)2fvsO&KCeb2mOlt+WEu?{xXGi$U8o!`}*O)LwX zc9A+nEDG=b%)>mKr1A9!D+)GN!5(+OXAEDU@JbZcFi+G^LJ|~e%$y6S3%T(bb-yNy zW0h~D{q@aX{I_G?8x-B2BjdnP7)^pAtb_7CI#*IN22_L5v#pV! zo98U)pg|z5A(@>h0?&HhEh$&`X@6;+FUqXzhru3^zAA7%*qbqusa=O<*J4j#N3>~F zZSswW0viH)Y5O{MBxFhZV97(Bn2Xie{!T$Z)5|@&L7n43U>%pq+N?YO8Icgay?cGp z*Q7!A{CdQk+P6gsxaf*~%~5?{iBHb1mz%CzP)aF_VOgQ6=V{VwXB;mLskyiHsWE?W zZIVg1UP&&Dko$1mw)IXYlT6b=K}3oXdP(MqJ?gHfq~4IA5x0+7exDon;Fq5;dY2{T zhP2EQIdXOuO%f~0zquFndpcAsU4|j-=H`@q+&*bMtKz?$kYrDIo;7JnvmJ7tuEP0j z`4G@(0ZX%P)nBsBr*+F~lPtLXM>Y#1Ly9kp zCRt102_Vv)v<1f6?_~;Iw{UbH*D}-Md$#jd1c@x>V(hV$y<@U-=Y1n)NqpJJDuWY0L%YfR&raCy>I>ZqYFFT6zq3<;C7 z{+|u?CwkGmdSjP9d;nsSfTA|dc4Pg-~JK6HDWDjMybhirJ)+PCGVrH zFb0z>1kokg1@w$+;I$ee9j4((+t@I~RXMem*;X#st8wLnCbZ~WJz7*nLQ$-NhiH>r zb4E*2jS+EoHNZql(eR7uqS?TOU9U-ZWpm`7I~N&^R9S2m?Z2l@Y972x>I^jVmotq0 zyy+>bl1fzDm%aWCaBq9(D*!Me|MK_(kOUPf%wa3fTp4~13IW>8is63)uwB90N@uDn zi-WON?x`T4khBtEepv+Xd8?$a{wMhhR4gI8kVo48sgG&^IXfTSx^_$D2gw6F|GPD;02ST9yM$;eg1GJ=v-83UU8n-8jLpe zLG_a$exxdc*kFfnu=J*;d~^ea$5q;<#(RBe9OMzwc~l%^+YweY&X(No9Ow{pOoFl2 zy))VS{Ay`((fJu*{*ztpNhH@H%6_2TSf%1jJC(%!6 z05{C5zs&3R2+XM5W#W7Zu>JU>`GSVv?WFTkTLJ3#nZvj0Rq*b5)h&r?1N`B8z(abK zD=)+)p&;P3=k_PD!5Yy*+-Ma>pzxEgzXT~BL+UI`;YGe2sDh2Ti-*fo!{BE|zpYyc zrBQ^0lnSX@9vCj^*QObDI6YhzHG)Zn=-_d=^1_r-%QX3nzbnHW>EL+M!j4P4SthE- zIclK4A{LP+xRP~d2H0eRGm^T89_B1F41|d-Q%c*En$wZ`+`Pcd5^F=G)7jL8-K=T9 z#c{!(TVkNb@d0mckTc>TV(gohr%<*lyw;`%g$wQkiyBSNLj5ZzTp5qDO$E*dsU2DL zVa?rh)05BkbUx-U>?sJg3P|#B4uZ(N)V|^FuY$dVAS?=HX<5@c}}x ztZ4g9Q0gh}UyFJGS1vIHz>_2cQUNXaxL&ZRyhk83GuHVl))jOCWRUftgIqm|pD+ve zaj9O5jnE_mdnDcq6CaP}-n4*4M};T$nhBP~HES5n_ES=!+Q*C8!mC~o(?|tf8( zJk5VAJJ&SMc>Y3`Bxdy5eoHeI+<Da6vus?eQcxUS{!PZfXfr{>5 zUS1jN7C%k##2G<414D0Y$L0p+V#o1&LB<=z&gyJGO*-C=-{sjkA32m_s(h0Tt!L0(xs%--4-!dpQtU^2+8)a9samqmsMBXGW^y~$D=yYB*bjVGdx0I;gmC+h=r$AzUMz(Iw41p8P54yxlE z>clxqsx?)nx}5s3h~3lyk!I}c8mpBuD0-ERLj%{}3R`!^J4jKXkk-7JuUYV=@;X1P z3&{bsz`Ib)TJ zUsY4dNJk^8m;;*WC}Nw3-FoNOWH)oUWIJA~ zcQgzItrMayno_QD*Tc|2^K8R!6nX3QqUiiP(`~C0gG*m2fKZ2cMx?6+2UtsueePUt z!>3e^U$xA!pa?O)DNdUS))WRK&Z28YlWeOEah9o^YC;G;TM!$)bRgM|b5?aS?CA>_ z@!E&m=2Bt2r=B}LUGO`r_x1P9tYdHAXf)UwEpWZEP$R^k74*epV^YS75;laD{2=;x zsFy|UiR)-PeuR6&?uvXKIxq?}{$8J%*H@^lVzEo!B^GcVC|lnT>6WT}$)I&1-SVe) zujG$S;z-1}5r#mHD|0-O4}=*e6t97x9R@jGKCf~m(!AZ;{*=&rcNUE zwsrPEZf4u}1Dc4xQ|-bhjg!01yycWmA#iH9rlWA;sdokuHpkljhy%yogM-1XH6_N? zm(Xav{>MPSOA$6YWH7#S)GS6<*@1;i*}I=(_tyH2UsvYrzCdetOL%jn6zZm-!O}0X zI6@(C(FX&o#twsJiKx2ljoSx8TdZq$>0}-hs&g~)}=MOFb5vC5J@eN(Y?+b9Wxt8d4%1Z6(uPyO0FV<1g zv~sseT;pznwImYD11@+PG(IOpFXWZ{;!>^M&zvh}gNtnhg z@EDsw_1DqcgJr^6FQRHCH1@bFTR83fbGx=(uCRlQ9bi%U1BvaH!ANwJ=C|UUp%%!g zXzWyLyV&CyG|XnSvj*G8V_nc=ZO>$n zwH>?N7`VKA1V}gnT;>G^@`{c0A2h8o*0%Gx`?0-=Qp(9A%|v3DTWE)HQH=2NMYa0+ z6#}Ov*>A3@Y;%LN*u|&pgVB2^A>L^QzTlOF*nTT|^*e3*HEl7pP_$c9#>Pw6l#Z|hFTS|v1Z{jHod%wZvue*#;11U1==WID^EK{NGGPLx|PnrlXx4!D- zd61i{98Q};(3{W2Bl>Aa04)tCQ^UhE_==EUEQCgL=`9W{whF~P>kY{Z1axZEjba-^ zk2~}9!Ok5_Pmxr|cwpq^{tE`v_qLZK3`Rc#7%AuToe==wWzXnG#=@fwAV$p~46Rgb zy$=MwG!SdU7vIy?aHog?eU0;k0*p9gUxcih=+iJ&KsPqBv{5X|aX=PxO17LYnaG_V z`pzfHdgS=YW`3n#2hsz^JgYutfuSB(8Zx)s=qryx zo(YR%Q*m5Ub9U~x23Wk$-4`$G*uHR;H+`9Wo~IEfBz=C@2lZ5(8hUUZYVEW*xD1yh z=&ixevz0oY+0NZ{$2@Mi$b>?w3a1!e*^GTqQwMGju^WL|Hx>oWz>2P{v(YE&eHNrQ z7v$ulk`((x`Au#ugbGHh8e4}O0}FPne^E@>J(#jCKERM9+ zb&IIL-GiYI(`68fMtQ}9AXJ5%|JbfA`tpkwo$9J$i=3ERHu79s1Hnvc&-^3|Z{`B8 z5`U%^(>AHc8y}Rn90vw#st2gNa>NOY^htM^oP+hj=|Famc?)gnc1_HO^CgyxFVVKZ zq}y`rr>#phyGld#B)J$TcOA38o!q(=HuMbeijc$JC(y{C*n zJW|>Ckg}WZZi|@(Zh#7!`P1N7LsY-l2g+p*xgl?1RE1A_=-|mEnMTgJ-5d9OuRPA+ zG{i0%^YfH$SZH}vnBOwvpH)w?sbZfLGLw4@x-6;;m?sCV?};Jf}llut^fp zE>2xdKTm4A9sy93OWcXlr&d6QltTj%4r1k_`)hHi-ingWENI%7#-Xlm$n}&SJGIBb z{laZaAauF3@~>56rU8{M55N`j?d8`3ZG*_2f4yl)?OuJeHD2UXAo@O$6n{)HaPFY=M9j?lVaH45&D=vLn|?V=wBL(r^+=9xyyZS`a* zW>DIDxDIh;!5)!bG-gvRFe3&#V*IaV1`nxgR5f+1Z=DZv!1XT_7?<=$-c29y9L*`- zr_ksmug0`x{mSx%XOBwyHZ{-PX<-})^L+-`8=uU6S=($)cs|`iGBgsOa&m)?HN|6Y zf<;}U!r&q!k%CJBE3sbP%S9QrVCV6}9gm8EOVwGOB8sw3im^Mpym;*gT`_Xbj1w!Z znUi)ZF_5<%n5Z>VoZrh(TfRKnhqMJ^Ze>Hl;q@GXwI4~9mZ-g3245Y?quNjlb5oRl zj(D3kTLZ%`_BRR%M0Ki2O{Q}8pW9>Dn-9MLVEWce9;P=T=OX|wd1TK40B7@vP+5uH zW~G%o)n`=1e&OrYjj&9Z7aNkSPS-h0o3(d*LhR^iT#93#(Krw{$*2z(1j9*T`1pb7 zD%)Vv?IjO;U7~&Y6JMO>=KAK#?so{fm*jQnDV`$$!QXmld-ri)%DAf&>$ePDz3fH@drx7F8ovUz`rGf>kla*p#Z?rh{48_Wg8VZlYjn93N zbt}`RR7EaO%Bg4JOo7pAfln%jDl{`Ovg&m+pxIl#I6jU{d-K}B;-R_M z1TB;pa$PllWB#-=;vD4tjcB>5uG2*s`{v>i|VUvvbr)rEPLJHtNsC@{L|E`JCLL^d5d1y2_KU8G83OS*S1I) z_FsruQp9m+eSt^=iy`1<5v=ZK+qD&Sj1odb^$c*vaVT~}o}ltHBe`B(vI-?x5_$eC z;#1CbP~@R?;Sm61>U^sOipApiZ*Mph_0yeT+$C<_HoguCsa7|dMv(p1@AINbR_eDY zFQ%nxGRsw?LHR?LA2hhdU|I6oNLJh38-mYIXo#AZTPT6NZt%2bWDP38(^w}F+?__8 z3JodCb7miH??8IkkR5L6T`dVEa!S{hB?N42;T?MtBS!!=Cv5l#KK;W%2{=l$2}fS-M?f zk9cR4D9~%QTRdCDd~38I)7ZGrsB#F1muocIESj^7&FCByk=X=J-?a27_vRfiKQFtB z^6*rCST>W1NznHQgO=mI7uH@;UA3K8FW+k+(m0lNql|_%1nPjETlP7JDZE2GtZ2OM_ zSrouw4tCRYQ4x=4zpmeex`y4tViThtx0l(a@R`0P=|7Ux10u*`tsS>JKPc3^xA{0JRk6A@d*tLHBqg!7?J77 z+#p^sNId`X4uP!4H?d+reXC^no8go4z*k+CPA(m1543VsZW&g6lbf&8kat=Ac|89k zQP|lRB2$GV6?0?U3vHc;1F4Zo>RL)b!{z+L9K)nI#Uc24g0(<(T^|^Ra`wAkD#=>q zGm%jPU7KlcSgZj#_O+^AWGPty^_s z$DX7ab!f@r)3+)G6)fsVZ^Y@__}kGW(9#7IwXdO|!#h>GIAJ%zxOhz<@XPGo2F4i@ z)A@|~HKp6iC<&+!H9N}3mZNnHzF|@wG8SAc2o8&Pdc06)pO$BKwz1gH!QtB#L45Gm zqO{lOPRhlt^AC%&8Uxb0A|mc#51)^VZ>MLcS2;m0MYL4fbdqv1J9_gwx{c5~hkUKg z()$9Y{yL%MzE5)v4SJRR2d7-R^kZD3R&N*5tk73ZCMAzKuBH=_(s_kg z4;#uT^1|?pC@@HszD| zw^AcGG(!~^T8%<$xeZkhU#{_N#gOSZS)JF>+3PD`W*3Q>?doQ-mtzz3J7mn`rp!YW zQpKYP#^P}yv4RbhAP;n6uD4A7fTvpDFe`FhcAYjkGNt2_Hjup(^1ZS=wLF#iHGDS~ zG-*+M&25#G)mY;iV6wWb6E3GB(l{x}W|fU+%kICEh@3n3)-LcMU_LO$ui%3Nq}cJH zRMin6$6)nWhKuZAq|I4<4{U%w8@8Qj3%;l#l1j{pua~G;cAjW-v&bLT?wV-Wsf4|C+Kchr|8nSrt`i>K5?!OpkeYyTE_x*l;#sCU~ zufxlnICY2m8(wW~IH1kP%m-|W)3+&gF8nu|8JGhH3X+=4o`_{IgE3aFx`pn%{PqD?wMk2lNTvY z-CKa3xr@nskzk;mU#CRwk=ujK-P$0TCOlmZoP1Qu)_s*hBkhOqm3oVh0Ph(@UhWMG zmRMJn*aT93vrj0^=y%An?BB!ODMMUOR2Lu!(mF*v(lCAWM=_N--^FSV$FCq(R|AvxD-^-NuH-s(s4y&!b&*8v7cv14Z7u@mhB-y>)OAwN_YDmLGDnqlKAJ zd$eoHL~tE^CE9hN6rHAy0L#8Ie43GFZCcu3KlSyuwV28@Tjl~`&lSZCUW=~%T*Y6I+rPH%C`*5n zr=~hnrxWFAdr#AkIo= ztXOv53Dh($`URUs6IEhkp)$@!O3jXwA_zf(9hYiDS6E|BR{So`VbF}&EuOC-EFo;D z*1gzg1GDW5*F%E=HdKAZ+*0Y2nrDlfi zFt+c@eEN&ev499tmFZ9RC_-x4CVhpe6}ACpd_2=FrAY)cgM-HpJ3iOcKg<=fe77|k z-}^$IgzN>2?_$Sa;M2(lC7OY!$lmL>u-Rr<`pF{GJPC9uENl$Zeaj^hDk?J4YTdN8 zWf8K> zndW0p>Os(gmYAU~-6CVs`oZ}=TX%o%Kru31=4yep zL)Z}Lkb~@)>cFCC!?%urpy`UzS4L)>Rnz4>+v|U6KUpJ1o`^&+X>#YlgygKf{h`OdV zX_CqR7rg^Q{B0Gm*cXq@ZM3@au@#q;Jdr~r5+#YK4)7c8uP=u_B|R{ z$H(qMEn>O&lm7tkGdJj~ar&8Cj&?eK?EAZS{{Y0J$NeN5zkA~`4)L8`iV~GrMpAO8 zeN?2h*Gum?6UBWQ(7jJ#tg1#Bz8$4kyX2;-sI@FV?CNnLf6pub0O|5y*envA#B?Yv|UtlH$o@@+Of`hGrWJNnp2j7k_gHD=j;8u|@uOTmJw}k<b2us) zM7t=ivPoHHV^^^t#_3-|quvhHDe_h9!lw&cLLR&OGi&)dPwZD%mw)D^^-{{p*Rz!^ zrmpVoc|1%_ixvL>@nusb^i{Q=KSHNz3jTeW_&HDPS6RG&5a#2erv3tN#F|$o~Lfy2{#bj?rTxy;WM)zKcu$07CwH za*lU5vabB+{{V(K7$456{{ZQ-U)V0P+-uJ@R;6jE?NpO;wu|Xqtb>kscgWVTI^{iI z9>U~o{-v&(>RykElfOy+MW>=%tMV#jB)4tzM(;<;YNI7P+Bhs4c=d1KB9_dz+W{TM z5#U{nEsnPQi&Yc5O9r&;E$z!|8|u+{;<*e<)V`?8S)$LE#L0{i#BPq_SFe zjE%-V^j}y4XG&#dX|kqZ=6o4I*z_|O3-0xD`pS7w zP1-!4q0d)=2?)0bYjyG*b9+rRm#23(DalFN``1}vNx#Ud#YBfEOKO`@n^t&!n0>FI zeL2ySzW3Yt?=0l6u?Kg<(9HU=SHBf2@VZN0Qmf*0Q(QKq4Nu2uC9@~HciYxp*;A5^ h_x9XlxThD&iq=+?6*#+Hy*1s)TL|wm_S6s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu(l$9u*%AA4FU7Frf_UNVnKQOPycfh9 Vr3WS)kQ38zWP_4qX(Eu&0{{XSS~~y$ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000026.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000026.jpg new file mode 100644 index 0000000000000000000000000000000000000000..870dc14f3e5e45a58021b4d0f00e458339846bc9 GIT binary patch literal 30925 zcmd43WmH_vwl3VbJA~ja2@>2PK#%|df;$A4#0RB7!a6TEkoAPn6 zbF*`DZ~*>50UrRU=;#>e7+C)lTpZm0F8KKPL_|a{TU=axd_qDZB4T3VmkTdD5)u+h zN^){KI(ils?mq*7P5=NN0q&&${#_7|5aHpFQBdIk|5W@(4IDfIA`&tRE&vVz9svOr z83hR$84ey90S*96z!(1X! zKw|-qI(?FhSx|qWDX&ytAj1G20RE+6BqU@cM1+?vUvRvf;^EV9N*Jpk5;*zC&tCM> za;fIm??|d45ncw+nd~|f@gx-V{pkVVp~8KI_d|fAeL(;~zythQ17Q9G=jHf4;69P9 zIh4mUj7Wsmb+x8q*wbanW5DMa<4{|lt%|p#KAIGB%Ab8xlh>xg+T1rEI_4aknUs(a z5uxSYKK-etl}g~8-RtaEgfKLxy^FT=D-g?ma!J&WzXN+>R)Q_!=QLO1BU?ay#DF2h zE&r)sFS!PP)pZHGJ~i7?wd2i z#wF1mt2+u#bbr_45`Tho8?wdg!VHk?d!mct)Hjngc5-E{t<7#VZ93njZP+7qHiV%p zo|*pzY5XswuKeJ>)xK5W>)E`CAC)n49lo)~b~>GQCvctH?iSJXvY9oWiKP=2i9H2N zEgm9mU+2n2(Aj{NF4K3Zq8?wRDfXugw(t}rPlZD~{>KG?rRRU6By6Ha{{ODpu)XJW zr9E_*@51)|&27dm92>vF3WlXJCo00&3u?=+(jFbH%=+8j%AMZcAjX}q?;$SY0~jkF z>yB6xQ&+c|e!Fk>ZHI0mS6C@+r5_b)JD$O*yYRXVm5l##?W&hc+WHyPFu&Oa;n>rK$e`%sUEL2<;zib}jE>MauL#Ff#GQ zBtGT?JZt{5k(u@0FIC%6x2-at@zi&^O~{&yH8jPOj`4kZLY3OJ=)EibVmF4?4absu z6z=0Wuns>ny~&NXri9NZN&iEiksxFCU(rxvR>)~%uoLiWI@!MURa6rvu34|4U0N+B zC6)WKv-j?S8N0PhD9ammY=kjX6#4jl4nL8*|IirG;d`8}(LoIMerW3fjNy!=p@}1_ z8fA?B$eo#$?x91vG?VsGC}o6WujeyK)$3h9ynoD*@LyW?s>SQH8R?6X&Lpm9b;)+C zvI6SPiKJ}mvLK_8Avz^150k#vuxgIBuoW$R)eO<){xn04O8K9B%$XWl+y$*b><8Ns zx?RMrQq6(NZ-yuKn#v+UOzZt^e*iwAR#r|U$N69-gQyr0zC3h;e@xL}>JLEYW~n5? zxA#oDWkLe-D(eGM$A_xSrBm`nL@SIkxVNf5Bct5~2cQtTA8hT z9uD#U5aUEE_>0c(cFpKZ4;(wNi{K(JqG0pRlK9~`p(kg3(y>0ckwxw+}eHHCxe46~~my!T6OHHVLQ|MU(py`;}?Z`f?I zkr{oKYd1wQ?r)j}XC1Z__tsCO_CJ+z5`#VUYDgF|rQ?lcajpAfYW<9dgP+0ZqKcu4 z7g}U!T1CdVXW%}bwS5OTnE+?Db83z zyN2nFv|d1Zviiv(KiTqZiu=^@4}fU8Qg$QDa&70CC0DbJX;dRewP-6et=y#(c|)kx zzH#jCQtH(^Vi#f;;cfAdCaan3olQF5euul3ypb_jd%{lLFsJGtfV)#JF!WcAkFMCs zp4c+ZrT*_GF`24@?L$K$-sGkH1I%LZ_l|d#IwQRv2=2Zue+L085tjRC?O{s6;$mv(iN3 zyycxF?rvLJ6QZufP2xh~Aw2LGC4Az*FYI?%=68`NaU*#y$Lv@SqDQE!oef(SBe^t~00zBo)e3s$qo1Z&+J)m#e2?|Krx zbGUm?b-Usaaj-~t_U-*?-4@Ywh1yKzRky*;LM*`mjkho4*B?Mj|BJ}wj?Xy5Pc3A- zke?raEZ+NoLRHC?af44|cW`jwz1QFT?W$74;Fl%o8u8A7lE!qUI>c-G>OB!mSrg-$ zQqj#Tw_SUyssHhXJ;!gPLq0X+zHJAKCHK020CbmzCdX>`UC*+qlDBq`Owr%cqTUTU z`|dFMOb&Gj4g(7&)8{=x$P1Iy(v@Qc5|6$-WXA3S5rv6&(yZ~p*X zF8=@=LS9s^Dt551VYq2}(4L{BfCq--OLXT4K)1Jf63+{Kcs4!Qe4al2xlnX4WXZIt}`9J@#$$0vs+ zrsW})xMP`|{Ll`}(~63W;T+fjXtW&UT;fhKdOzUqVS(k^zI4sgyGdO1|A6+bz5hk# zL`dX-zNz*@@T6(BQv>psI}-Lud^z57O$Sg5KY`uS+2+LZ{sfmENkcla?uzaPLtjzC#>9_8lU*Ojy*<|D7j_kd&bN!`g~*6CZE zy^ijQNgFE7q)sWL4e1_5_%n|9n<|0AboOh`&MiEnO1IQuU{4<#7RQasSMCvX3y=;W=3RTxVWeh{qV{U6=CoP`Mg z%R4mw{iShZr`9YI59dcn54JT>f7v}0g?}yWuYu^hEel;TUJHZA^+-fNbKT3H^xfpT za&FlL|Fy>D^+#Xm&R>%#eX*;5DKE`8%%n!_dc)novtVOBxyvF9pwcuSM8xYcO!~-j z(wtg++g?EB+66^aZ!({dd$z5m+^eB_wBzX?y<%J zMIx)t)%>Y0i2x5J=ksBZ2=rw64D#=_IukWY2&OSmlj_eelB0DG_a ztCr&>ex=szjnn)MbJNZBe+L+!f8YI2pF}VU8(*%U>z?ZG)cy^Bd85j5P5W2wPwPJb zg0v_8-H!fhWteP|tt?i5v!^BMweYnB5OcAJ(xPTxUfD{P)gmixE zJ79Ca`&%pAch=}B@Y;yih~=U04?u+KH^U#mVfK1=--Cobbj!N;#0i-BdHYM?T3zri zC|kDU;(VY@ZwPyQl4=V9wi$eu?v$!cW!HP^7$+@Zgr*9j!XNnM4IA$$O}4!9*J#dkXxtk>vhEZG1gwj~CRmbTW}q zM}w+K_m=1cL?7o9;p7ZIN6=@n=M2Z2lI@63SXouS{^}I^Y5>0J*ucE}HN^J1OD>yhl#S68N=2!R=co<^^V3Osgu)fI)rERK(vbJZRDlJ*T7;G zb_R2GYlaRbsm?0#Z%r&BA<1nFuU=PoeEXl7Wn~q+@KzVal2NooEd3U zVTo-GZx8Zi4=uOf?`HJxgsrqRDSMrYYi4d@C)tjF`xm{fZnf*sgA7GMdLLqjzb-cQ z;095{p-KY&=Z-KrvZ`gQr{ao|W)Gx{bCv1=W)yA_Y}kVS^rAmeN%18#HhcuLbdg| zzduuS!r}rmrzL&<03z)@`#Gw25)3~91v?6PF5rc=axUKy51OvSL?>SP3+m2daaaNA zf2Mt2J2bRm>!p8}o6B!#Go`f7V~h9wVd*b01$^}?N2KfE1B52 z2YR>_4!9r@*M~c&QqdB~?ka<4&db{FRSz|o{*GU#@wRBUn1MJ^k3SY`GZd?D7FUmGN9tW^pxi}+YUBkvpuuemf=gnf0el{A z?C9M^`2#4qo?%+)GV98ZCI}wzy;y%fG~zJge0Z5nGVnjwo_RR`7Ml-~Bo1*-AlP1sD^~qxsHww zSzhlkZds?Ir01RQo$CQVNb#sh@ja{#d20-rWOz!xV$X`VQDLvC3c!+xhiJ5i0hvXH z(^!-AvlG~wBC)-->66J^@DHfiP=qQDATT*)u<`Ud-*20?WPw3L8AWj3v39~IJmI14 zj0Ab}66lOuNsV_{$s=Nw_<=VV+Fy=Zr%xUhQ;pN45xrch475{&<9;amlyWS-vB@y|>oG6Zj;Y@Q zjb?L<({OKHrAfsMl+hRQP2QTUdtzU-GoQpks=>!XeFInY*4E`DD^@VQke$NU{Na{T zv|{(mKg7Xwz3=IoG9+6$x*EHPGRt2lCDP@U@Xc=(%E!Y`RwvG>ZbVQ?S8$F#GFJ(Y zo%chc6TFgL54OAyIr2JW2RSy$XE#YaDP_=7rFjETj8yHlY4=r%!kcdx7Eq!=g&pku z_FOBT-d75xEJ~_U{hTS)^6?WQmFlPu5v47eIc4j2S`h$7{u-m+P9ZZlpZC^A@s`7M;%9 zIa`{vhA8r;RB6$ztq?7cO(L*}cx}ftt00TcPGjr6)zQF*QdqrbakWm|(n2(C>Gj7k zRd8Ey^K6Eia_0P%hVF-qGv4Dn7A$XF#5hr2>8MT1t+d;9E0<4!1#3lr0FNG-fxqpa z^L{GVpxph|SiCDu!iX$WM|tr&KxF?fEv5yJ(XW+njC8zRIQy-FuaT~eI*t_@4Eq~X zq3i0Bo>p(ni$N&HK{TiU<@zU}>^<~pJ1*UNm7Q}4qQC4)BKhs=RO(XfB>0m_<^bMQ z72h63VdyxlYj&_Ylu}=Y237Sloz&Z3QLCu%mL@_I%cLX|TN#w)IN}7G5v^jq8W1)eURiI2~$@XY{I~ z*c{|IF!}0X#OOuSXjm{p%S&ww)E#rgfnx#3X}NpW4(8-+1XYPoG^YlRx`joMAyG#2|1 z{1M@?8$W;UIU*9-Dk&^lo1swboO(704JC2{*wwI>>A5YpA2J}pzM2mC&KGR7Shm4n zwG0Ixt8fqVwHLGA3{GikuBo<c{<>Jeat6;HHzh{((4>*%5ew7{_@ zoi-0=oGIfmnxFVwkLJTwraDGql@*URk%l=V`jt8hBRn(Ilzy}=aSZ9&B{$-!q||55 z+w1^8DW{gMso28cF>hy4+!9ug&s-H{5e|FJPZ^b(Zgz8Xc`+;-o%O;H2eGFpg8_3F zv+=|kzdil#i!f--(u09`ncxQZ;=5?V$^P036d`PqOI7dN_+Q(#|F-0~IyL6LaDA6S*8sq&9^S;?0+Rxxd41 zw4p!%b+EoAWX{DHem}qg+gl_3a22;22q3Aok1+5NVJu0SAI;$@*j+a)B_5|m!#D!M z_`54zZa1!3`kwa9rT+l7{s6xJ*#GI%7Z>y{Q&{O$K$1>F`G`2t=c=P5oqBC5ZM~Lj zTNr?HA?`wV`4fuNCoETr%8)(eTB{n}__5WR_S7F*iZ)f#OkCjom6ucs3_}cft$qwYXzt`)Za? zPAPM?GY=kXy?E$9I{F8`cG*y(@1`noQ8FscXsC3CZExl}m!=&uzM_#Ywv%^;4A5ZY zjS%R$ozCR+s~0b_F_l?;NVDhM-kC@`LxPO6Q{%#Kiaj zN!+xThykEM6^DNn=f@SSIwg$ebI)eNX5jjX^VF%-Sc_0bCAFXa;q}Sv0u9@_c}Z$# zZ#5Ca0-SaT9xawk5arvbsMicr6eGWtIM;6zZ{D}BUA%Vx1HgKEsNA<66I$0b z%?EW8UC!>NtPDZj^5&VQ1)K7({6DO9tUaER%B{((>N4cCFD_eGS&t?d3hupwxa#(q zptzHA3$4-`!Qv=+@$lg)Z*U@NzpVcJ{;kAoGv-p3!^BxF*~7#PXU5zSExZ+J({E8lRB)fPPmq zKFj&}Vy6i#g2Q2xY0%QCXtILOoONo*0{z6CrT$(Tu%x%st<`&)i&WpM^A8~78%2zp zqTt~~ZJqfpaPXXvO*Vd~)u?>y-&z}_}Hcc!+RSstw zF4o4wc?&Fc1FZSdU$eG_Nb`9i_VpE*(*6E~i+-N~etwR`yLzy(SP|QV^*}L||3>7CSnhJ@(FEjSJQ&Rmxp$K<} z#2aNbz?qKrmE;Q7GGD3@&4thY^b|n?Tqvg}ZDv)>_AVZX7By(xKba+cY)cqo;NEVU zc&(=GJFv;xe*kKoVV<`uNFi#{^8XC$pgLQw&}4h2?@2J(9|Ee*Bfq!gf3JIQMqnhK zbSyCyPY233c>miiM>FIx$5)MNd+sGIPFQ^-Hsy==MVBSaq30T*ysl0}3veH$fI~^^ zjYf22L^A4Vb-k=HZgL$)QaExHZKWZ5Clvn3a|%(INF5nGE#IS}JG-rEIM#h|z_y-g zI;v_p9lFeMyMT-A40gnHNt4(|5m2VY?P}j{9dDf=qi}|1CRbZH%bn9SyUBLB*D9nO z5OZ9#>Oc*A(#bXMJ)}*Pj-GPXr`3KpEJ3CSt=Q=u0SwBR$w~6tZ2@VCXv;_WE=5yBi|?A-T9huONvkmxe)YGBMjGswI#&-{<{89y@ELW$ zJd1h2K{VfY>32m^@y!w1TjRRzq?0!5_%L2Gzqk%LJ!}M84}g@-C(G ziG8QuD?g%HR=0lmaB)n{?8EM7c_R4+zg#Dt8nK>J9k&-fEPD_Co;qZS{R7Z_U;6&} z58#sb&>ci2>8|=UubE96BwZQPFW~=_|9q@|jTJu*>>S@J)T@*yP|zqoTV8=VHTpU? z9L*PeZ4lPfQc3?i&o;;SC{7|d2epH&!K)%*G^s|lz`!!fy2Wm>$O=-v?2!zrXG19s zEG2qC{jXD4r`nRUzeo~L(C3)*3~pT3;GJfWr}(zYtb&Y5Y*M#}pw@ z*lW|iS^6O_L+o@DO7JN2ndBOq^m1nqq&%>4OQuO;@jyOIOjZC5Y;@*AF|Wdv1EdKJ z5>Ksvs4g}DXn#Fs@BTe$wHXJN3h z;)BGUvTU~RzHvOV_ozj-OC1R|PNaqElFpL)a*7s}MXiPe!_U_wvik`DPjxk5$4g^s z4cnb9j6BI#GTeH|yuuY;^0VbTDlEk*Yx{k=u9Q=gS_Et*IkF#QjFk(|blk{g&b@S6 zFMjxKWoTM{O7J3}$Yfo0m+bS+^gyCX*|jE6RA;8W6z^9YOby#dnenJSbW-5Vn> zYi6q2!%MD) z2|@P2wPL&8A}eNrjA8WZ^}1W>{v<1EH^MXCAb-2@7;o7u9xYD_JS7pO{m|ApxvY#2 zv6*?^T9!O5&Q7+Pg+WJx0oK@_rWq2BfP1UWweaOT76`HT_$FoFc#%b%T>{*)=Fgj8 zkvtA9ZX@n7EH-K-gRVibx*SMcFRG-UNc7xnIHZ5#$Fr%35B$$Gp`FvIY4MK5nfY&* zMK}eZ>~GGcjxm!a>$brzlZ5g#S`+(S*H+qbycmKZi$$0(85bP3$gcG>WGT^}o8ru+ za5~Na(?;D=Q&wkdD(7gxIme+Oo7`B%CT-Qy#^D$itZC8C$}4{&04bR0Td-k~GFT}t8tk}90CG~|rL8QS5YMcQ%gc)Bu9!yLqlC3$q&rW?m-w!0IA9MO^zEIj+lHj_ z#^60=X2C^Y7W`IH&UK6yvd3!}J7$6!FYK#@@NE2SoF90w!41ASNN_@b^Zd>q?0IVh z;+^DhL6JIhL{w}32<^7H$n;?A@MQ@Mo7ttohDBp9?dYv=#D{-+R{_;6L0nR`Yt3pDlWd^ID_*7%=R$+`b6TI6NLf zrLg8#Qd)^esVQ_)_Km&V&+Cu^Op^*yuMBx|w8#5SBUiU!k$4PTHYjgX68H}OLe+AX`pDE(AxprU6n zmZ@LW%X23{4g)L}G)hIQD7?prUGU8wkrA5lztVnzGAmxn+i`53at(c2G2AcD6hJ zbHkffFBdi*_#Rq;lU+zsYSyxI4}3IJw2&tEE4a#Nsl#ELOcGrM(Lui+QBkIPwnlMf zf-45LWkVBct!8^^7u@7v3!V-veIcm}jYxk+KGn>8WQc&5Wjp!?gsSRdP|NG8i?lujSb;^&!2 ze}5*PTDwwl9-sxSG8(ZB!2*PSTm7mn@3SL{%5r};Z*kCvj4um^+|b5sTT>vufH>4+ zWsUe%I<4va{!;03M$gEAA2mlRE-&4f7n4QgT#Z-89SMgdZr)3br@F@mei1VhviRDl zRD-rTdOdu?R))ABkrbRArCfwX^2>|(vPjGlNLSy z_??!H)kvr=`Q-f0ebtUF-+Xm9cZ|hZKHG`r(+c zjv(mVAWbAJ|H@*kj7FuWiNGxt5J$R`rlpGMv{F(wvC1${0J~Xhnnip+N$+e1ccqzQ zI-t=)Ivlqi$diWDpUAYCHO=`BsfokJ@_QS(5l(1S?3~cApVb!p7=G-=0Me%A5~8p& z4iGRHwufw5nptcMvUn|}$RD`4x>f^g{?ublkgAToeHn}ZQE8<-Nf<^S;hKTV87(OE zfO8ZG>A+kJbHx{$0M;vpVB>gv{Ln9>|7;h+BZ;^s;`EH?O#$Q){`|vnlaW{HwexNB z`eBRI<;?nz0 zh$*xY9jSdg`QCnYF|#I5Fy^GX)s#p{=Q4l{j0P&uds==dDXJ-Zq{vM9(0`Rq9hRNB z5U^dz^kz)e!$E7^MUi=QM^pc0nQ|n6h0Nk~j;lqN)-8<7~DW)5BQFUD4?}T{f~)!s3&J$41HWh*9iV+)qr%ce1z5C$l`|o_T&QISni@>Ejwe#jD$qsxvh@O zbLZ>|bK@fb^H!?4enP)S#+uWO?Y?OUl_m(K^!OMGr_%EVDXQqw@OlRPZn_~U|ZBCtL$rGEA-r<)_nSao7) z1}ghkxi%G+hywvwN}e?8CYw3`~9gAv|ra>fA0YDxQ2 zp0?yOiq#@hx>lkS4zcZA8v1;uU4$!exQx!WKeRV8Mt^vCrt!6F*Qu^_5Hoq~f=S){ zF|L23JsfsRLrbU>8lXF?Wew1jpM-VBVzIecbExN#UL!r&kB++NyQfvy3*%4NT|079 zm*Re+%)eUH%r4d)2^$k%*1oGu+3zj11n2m_qf=_|dCxmB7&?B4!2*+90QS4I`SlQ1 zD`z^Jo+9%ZH^|bb=kF%yb1!I;`n(yo8D%)&%M^HQ)8Mw2AL*t1jQDY9F6sEog7^`O zy*1Hfym}4XM5YTDWiR#8c(zp$arYjQ`zPc| zML_||{9dR2ZpR4s-VYb#wVe*r7c>eI=~TQihruFf$Q@f5BZ%!GnMB#zYLBI?x|nVK zj4pQhr6##5nnPL*U$Xiz?NslT2JrkS~+L5N8}!1<=k| z%AEAOZ9Nv3;BJ0so;WMpyo@tve2xaZEEgkQxWVaR|CP-$ZbK>#`O%X15?{m-KDW$Z z1a{OdYfbGBU4>dHf}L0ROPKX~9Q5YbfCzj)y^Wg#$HcNU8xocItw6U`0TZ|D!u2^> z@k4rzcA8Pjw?1ED<8(=J%qRT#Sti_a%nO~DU)=<&O9+Y+RFi84o&O4LMu2?rK)g7l zvFd)~hY@O}Q7QBX;QF8mO+4g&mVAi}tQPFidTXzwho}UEW{TE5;PSVSErW`C#UQz> z9-|zr6!IhPdMV%W*B3eIq&Jw;9WWo|Mo{5P21ATSE2m;W^CjNsriQ27z}_vqd>siUOP_M`qiv$_n_y3~ zim7ne5S3nrnzyXE!9w9hHjS89sM`u;aq~Mq;66-YT2iN57p3i(;_SJRJM8Lbb3U5Wls#aGc4SL-OR{YAxT z>fMN%4BLZudJ!)vTQ)Z2k80hw8)g9MLb*A~+LInh+Y{~Kh_|fG6O$V-J!vA!-zPml zmc!;RyOh{QjLjUzny)AXU)7|wcnQfz0`);aF5WU|y-ueGHvk)0YEGB+n2_Zx0e3>K z0Fi0C(2Ugjc&7Fs>~Xz_%}@ka5+`QW`dFWmK~n4^iaz9gz%yv7TVm5bb8)fUlrdLg z!rU;sDUe7eUVik!E>)w^y~{~6Y?Z!PKv*!}daPsxlk-CS1muQypXDPlt z2STPFQhU?QddtDVPT3Ogx=q?cezmC{-c5`usxWVb0J^4WHOTxS&BK5lvnHTpU*=o- z&bH|MP<+8ni&Kl;HkBp!_;^eJEokC#LEHt6W_21Hzx+qIg3DX;R8`}4Ub6CvUf|ET zO@MiN43SohW(V1qFwxTQn$e}glWtSC>EB0}Ei#K0dvEckTDIR_&tgc9OW+$6<$gT< zp{ZDE`I5DVgX>9YpTd&D3Q~6rIyU7FJJXj-6U@z9r4Y4h!}0K!SO88F6io%1`!Gvcw7}0?o*e;}&ll0Lg%w`!bleqhM-m(6;x9f=qb8U=V zc9=srILhnA@1^OZPUGHMgBwieR5L>pmd5SjY7BU1E^9V^3N<-~Q#$Hj8VC^Et`VBZC~{G>T#vGa95ZyvD$5zh z4!qOr4hGxIWECgu&dtgkLKj*umI9mu#M4%V;z_+E`yJ1|sp@}e!!onx!j3=Hol9_T z%1%u&RT_yfZ=l9((~J`XVwrCKS^<3q_A5oJ&pTE*l34PUSmduM=|mD3?lG;Je{0_-Xuz#zUfPc}4_^^w3^XU4JBgnz~W-4uY zwMw-Kw|Oc#u-ORYPE^(CZRO-NNy|c=sP{Q(n{G=5UUFV;ZiiYM#%Cn@vhdUyfh9Zi zb_xOtc8KZp3KgH`JsgMx;ufXbo(4rqEI%mbWX=np#Le*_R2#f9zk?$jPi~t@3FL0; z8tAo|#S*J*6Q{h%irZHe?E=%%+`+87W6#AsN0!x#EH+M zQaxr4e+jX9?iYudgith(2xvs>O4y_&=#q;;<;68s1M6;R3?;P~4^jt5x&#yeiFwz; z<{FV~V{4Ea`cW)y3S*@utaW(@o^hZJi14uFyZR}!Ix6%C1O+rAhYi4(s| z#^*?-Nbn;jpq>bWs@EOt7>Z*h0C;QPj~bkplw{fF#_oDq;0SnJ+}%sN$BJhr>2oZB ztE6t;asPhsBLTp)NOt3I1pff&t62`6?bs3PTV$#&)xz%B;U2rMW35$82g#^YU~8iA zpi6SrxML+0UAS|!%2R2kwgErWQiW_by6^Q2jXlq!w``60!*O=zpjeK0>YvHbWzr$Y zHFDAfpI^Ex((0msDVys=IZ^eRA{sj7k`umW?^2mSh)@$1P`LS5|Orb}(uO>2kc zF&Xe9W(JTQ9&}a^)-W@(%`@Q>`C5x_Kjb~LH6ql{fQi~6EkG$RI=})RI(|AHh>-@KcE|m@W-$(^IWYX@xp6Mq;c_-tgexTdc8e!t|TqD&6{7G-)Y=;l$(~ zadP_WnYTN0yQkti%WGAU20L*u09t9W5Q8q*exp6c-5n2^NnB>-{wOnUy#V}sGwi5^ zoLMRSKGzYjC&Gv%^tM6y5~EygbirxkL<^ZJ(r^{p?Mh`rhS}*W*`Be=_8lyF5>i@i z7INq1Clj{;95_8z8r^JR{OFM;&@U|hn2>8c5gC{)V7i=cnV4Ca-+f(DL z{&dSmM(k%Bhyb2nX<;>>`=AC(BsiRjiX8b4Kl}2c7S?Z(@fx6goH#-dkIQv69EfuO zfmHPk6!Iqc#8m>1#Q&DAL!oKEgnVX{+P;eMesg;Tml;6TM)csW^8u-v1C!uoRbZyZ z!~JvtZpx?aVL_1KO_9v2JJ(QUb?cMu15KK&$@5p+1Co*QFOc*&i@D=P0WNRvI|kt7 z5lT*fvwaW6+MhqaWj5op*l0m%L&SykY$J%r2xT`b#~n~2Q{&;$FY!AGFY*gCPX|!1 zj}MhHC$=DepAc!?_ef!sShhNh1EQf1CT?uv?jVNMq6+z+7K}FA-Nz;g;h@XmiK${? z;!UB(wY|)Waq&LSxKK3HU2D_?%qy*3X)jZ~p-0D;AVe!#SzK+mJ=33CsWxEri#w!s zo$X{XJ|W58Pa9jE3+Wd8ynoI+Cx8o_e@MxoL?vi>(+}Q9UuU;DR8Tb%?bx1^aw+61iD6%19eK5Ne%T%-7soAh%cUS#{Jog2w=z(yRt;AmV zOrIFebzvV*V)l)_Ps7|t`sE4IS%yV!Xi2E<7~7g%Wi^TbEokKEQ7Z~;ZSJkj8fL&Ei)FeHc2)e;-4XLjmkrI%Nixi+(9u8rkv(a!*>(1IM|M( zu$wGIyl`?7TTwtWV8%JWMn_P*q$rFS_H;H$IHOx{(x00_fODT^0;@K%8#^TZ^mdbu zTV%e?DMpq2E^;A>>xW2#iVzV)JN)gzMNluT{aaFg`szw84bKU0Je)V;t^dXnZ+e1d zgP1({NBsxQlie+^pQyTSq~o;pXHpVKdHA$uy-S(RIv%Npfa%wPD2g|yN(1Xi6Lw4- z91>MjIg@WIHe9Hihm(*^5D9b`0~mpv6~Hen4%}O7KOwOYvBjZ|{!f>N_U;jH$Ppt; zlE%In+w3Ml8iQ0jpNo!ggZ=-`NKhoNwb(sM`6*4;cxS!ra3G$qMmfx|7`q`rMMGH9 zH6t?HS>Q$&fc_Xp{%L=Oce~8c(G!uIhIFLXA`q;ckacd?6y{gS+7yFG48>0#FQMd@o^1P;ERwSGKk+H*i{6L?PUs_D#J`{}{A=~mcI$lG>e@qqMG zv4Mi|FQ=;po(sX1f(3`W^a!QsH;*Y?sja*iqg0NFfEt|VRD zm27n9akcF5kI|r=0n;wjOM{puvC0GXn*Xa5y)vK+DD)a3a(ibDT(hH zhSl)1^RM19H%Al8m;iEwK9`eS+y-p%i1TcqSiU|wQP=z0b!M6(6E7;-X(QcvXs6h^ z3QI#U_*lMDogrM(>8*I@vtN(YG_D;-k4+%rUYd%bF$Gs+BlX**0-Wp`VX#!F%*0>Hew$%!j#vI1Vxv6*M0RT!%b%keW z+hjXG41BOV%wN_4Qe{q=Sdg1KU}cA=lhIT7^{jqepa{ zvioua?{QMRa}dw#GM@OFjRa!u{Yrm8zh~#>fF0#r^L&cPf{@)3{SohcwmR0k_T-uX zuO}V~^o=2py6q2)4X)9fiz&u62d4gJfwWn*iiI}IEM$Xy(D9UkhDn=w7^iNI$ms6L z25%b50h;DXM*!JJ50VJwCTiZ^89hAS8Ux{1c7Nj@8MBM+nj@4RAM8+jz$|b!Z>QI}gs+Zz6Z$-J6)1o0TgsFF7oz_P~ak zpT~oBDe^upJL*1(^K%D?#* z+~2Cu)~RD8H(pHp@~SgK}J zZS#=kSI=cJbJ|&yFA24lM^JoQr3-Q3F2mwyJg{=e90MWLTSq{#+A@PITYgN;&R!R5 zV)(g#;mJl4BFY5Q&VRYQs}aT(DeJC`CENMAv#!~yzdMJBmk;j-?NjTRvc;FlK+|kr zQbZU2QN{)0m#S?wko67D2}YIN6L?um{*ND?U3HYqrHYwXTAJ= zhTIJD|LW;21KNCo?cw0=#ogWAT~f5T1d0?1R@^D>Qrw|HaR~0xQrz8xOQ5(@T1wyi z?!E8-TRvoWpUGxt=Q%UyOvJ?k@d<9H2kk}qXfeT5PvY1)#Qn%o+7gcyy3tKE?L$$` zIof!eMzt?g|CiP4&Omd|8S4YZ*wkqJ#euHyVT$7$C$0oh^#T=8eYsz2(r@;B-;1rF za5&3&9s~_|vvto?f^JM64zKUyWtLM*SER|bX!G}NUhx!fV}18Zt7f{e0oHDYeZaR7 zefKiA#$23c$7W6y7+`q2{txhgA)HIQYw}7&g?EGcOz?MRCbPj)bm*JWuGVG~xHxoV zO2BcRhX*!gsf$dE`RR}v+mTp;N#(CQ)rbNL)x*h=*xLp_~X65|B^iKWrz zy0-6Y6k}0YN*uC;bgwli^vL3y$Rc*u;Aj{)_S8q;(B?8dH>WLiv*sl+(6DJ%End=i zfGZ*Ae!(wuKl8Y$G$tr5cxP_A;uZT3;Hs0*N|q018}g{rcc&T4HA zAA-iFR>lNX(WtlL&nF@2{qiTm?^vwZuLWRyzCYUVCU2D|ohM+_YI-zCmne2Xk zbffpyr}@3@5HCN7*`VyCT)7U5R5zjuZe`rK3L5)=1Wn z(1R)t6ko3E$OR)DH`Vl~zNi<=h=<+oT#x3T==(WALCex*Kj|o6UbE~z^A_E-cjI%3g3<(ekTOrCo zQ*!V!xBd3>%akY0`bp`K0wBJxsih|e;bD4amM#B78 zn>TwnOIqY<9Cm)F_g#MS6-nEy|C=|jwABh_j`o);JS)XMt61)HwBgY~_XOcyz2zj? z08wT_1cLgJ_$50zg3l=32nE|S<^!apv$ooyi3#$DN72hovTwckCEQJcYYUpHD+>A1 zF0lSqm#!wJ*ute%2;)7(hvsOO&c^|~kS6}1>u2^HQS3%&Z|=+X*tAn2LUkpW>_FYp zW)sr=q7~%(-T8fP0V*!^&&)j$xkoPlXmh{<#46ed)Rj5qJod3AJhgMt(rUyCxvhSi zUAE8qK>6%TMeChQRN5{;nCao}D^btv0*z$H_lu6+L>n^o>Gn`zghRbD2-MR%xbfhR zc-EC9?vG9W=0{sR{{u}&pE79hiYz0C@Dg`ck-fgDgHs+-!lYHD$~85J*vj$XM>O;& zpaK-ULe7=DO>DC*G=6)tt0PJOrP9p}EaNgPwch1yP>Qzj8|ZxZd<$b#WF}7<%_7Qn zv~IqYHpPz`@c)0z#s48NA{$aSyKAny57BNAR&A#|hMx~FqI#VfxgGlEG(aEOt|gCC zd2ZW=$fT+X>MM=SAVD%CJ;81Qn}W3DQ$^Q&XAH?HKDG=jR-$;(wqKpol*a@%e(<{5 zEwtiHn>QBxvH3OAL0*59l?o(pg9;9+8QSxs**Pro2_?@VpS4Mu;NdUPV+p{qc>rlv zchC&YyUdZi6Qi1Uq_{+qErQd+7e&iu0?wV27{ob-VhH^h1b2pw?ZdY$zW6q^P|5T^ z=bRx;uB|y+R~moK+dp$RN}`|dlGJOhD?jE*B-N!)rA8zE18~`WkmIx3SP?FJtioEU ze@Y=cki(!rb4Q94O=CzwLu~sE9p1F-1^}NcsSH*TX!I(&?=jgDZE0v2;`7!FUGWIC z*?wY7^;_Vp=h4-@R{WLcUv2Rwa(=DEzX2!n2DTD4n7A2TUE|8W&qh7-6PMGU*|KXp zQI*%BA(3{RHT*~>dS5C^;`j=Y!xN{f9W0l@rv=ZpzBL~4imzV6%ZtVs&Q^F}*XV6~ zoS;qb3l;MdpNXe^K49#@b8hKi!Yp;(dFkHa(5hUBb?+|6Y!{LO=t}&v{*rr{lNJWE zhl{83Xdluu`aeJpOPnuKOY-94pG*~*_xbE!Z4z_c77bYQy*wQAQR_@XVys=Pp#Fms zPDIBJEKD6yHs&=cS+foGPn&+8RKDc?}E_L9hbtmg)%C+52+6K5Iq^}vG%Wc6RZ zQ9VPO*U&nfi=#lrp_o-5h0l+3{ro6TS1jXj7ZP;wD1RB9U=!jZ!t!3yH|HXBPd;CK z4q{Tj0*N8YjBOu>uX84~d*Z%5nu}6lHIt@b+gj33JF68Z#Z#-5hmEUfZisAjoUbZs zoEPZ%{OV-DeI%TuUr{Uo*lgn_GGBq6quaec;&Mcd02I~j*r)&mXVf@KUU}7a`>yTt z9dEnW1z6LC)>LyAzIy1b=Saw{zs39^fsQevYYHR#2cWfRllm2%<50bz24OErUTz0T zeK5>C{hXKjSL||4?M;y6qZd&-v3+X`=MHG z(@n_?hM&mpFyrPettTV2vnr%H5*b>!ML_7N9B2~W9jF#WrW}wSSR#aYH@D!Jg?*zu zemF}lFmC??!_5RWEc+ezs=thhn%lI8)+Ksl%*xF0t2(O71h zp)S7BAQU_oVo9wl*TjC|;g+MPfi_+@!J0v1V+6!%kdcU%kCYTr#xq7W!RTiprdE+b zS(ALx6d{Dh9s{f1h$c6M0e;P3(@KXlGzEP0%=Ar}fg|wK+XEyrK~3|j^K;DaKIXde`*cc3W#fWSWjWb5^&t8++=*TK zyNZcKQZTRE$>ij!>07>PNgPi}j}9J7b$g-yDjo*2+`$N~+J#>1h)>`K0Vh{Gs+>BM zFYg|6ADH}pINsZlY*9O1{sTk>Nn+R!4zD2XSW`N0J0%Y0nA7OVS92ZsQ@%d;qP=iL|8Aw1*IQ zv0*x3hxlevET5~zasFe#oO*&;U5q1!WOBlI-W3Q8d{3+eIo$9+{AREwk)F6QP?y%S zO@D!^6{#wr;mW=sxkkxUi`gpBrGh&oZo~$EJ>Ze#V z`AR7_FNHb`Eo+NRCN}?=8cIRPz|hV7osLr~TP6>KpgCS|;2L|x+=&N-pK{V#$?@&x zI~xTYg&8d)llL=EXcoeCdnj1Ge@Oi=DGzc&_F6ZFM4&cllWwO573TONQx_Jh#Gj@o z8!p$ln|72)X(Y$tNQ`+V1eLBZx)3a`(tTg95DD(65B&SSmD;gQpen6BOC=5&y`c?b zB@9qGa7sL(9)FK4r#t|2gOepBb1$+|93FcKS8)j`6cZ_8C92JeSd#=l3O63h1&_@7 z?HQv232=AC97{ptHD*DpOs87iC!|tUBObb7+5Q4=qd}3#rn*gT6gpp2%0$UpWcHg2X(}yS}wve)W_;)9RcR-1zkl?5`3PM3qLL(BnMmZmCP*yxwOlJ}Mj}Rd%tZ zRW>m;@4$@<7Jt;_>@j$^SLcPquLBWzK%X>Ru-QoAJ@QQ}zLY0Xb>8yz!)mRcj^7Px zO@CYS7#8?Bvp!7?!Mx3J8TK5}f<#bxnBJ68UH?OrC#xyt?P}108}_D1VLowK1ZlGI z80Fy#kWQnBn2MNoafWHb=1fP>75Qwr>21dXOw7>#hSv4Zty4$nMk}v)e}8(vY8U&O zy;&1@x9`#ol}?U;>1SE}M*E|z^n@<5gG%*>OXc?L+@0=n2({`qBvw*FA07xCA_l}e zfzEGOA)0YbzSW)!@#!}x*+}3yu?mTw;EM}AI}+PGo{{_H`y{#h9fNOo3;Jt{q!NZm zKa8(jU~$qRX+8*(p#l0_b{}tldvmMp(G)gjhPueQgT813)u%*$`=E5uWIUBy6)qF> z*&l|DZsRkWS1D@^&UNY`-r`3g!XaZg3osNV+vDxRWEX1_Kvx03l}mthj;cEY>WW+C zw1F%_O+0xYNu($>P9E!4_OhJ9%JxQgoIT%U>JYFLqjq~$Hqq)_kU3>FCE(3>9h^YA zzfRun_V@H)6I=d5>#1>gbB?11nlEJ-*tC&`mAR@2+=U!bk)(Cj8|cvbP*6usbxW-&8-QUqfwE&G3Fn$ERA#&E4cSz#{^Zd6AfN_ zzn=`)xP*i{Z#B#FSN(2)vZIp!&iCACnGwJ|NP@Kg}LeZJfQw&{ZOml=+Z*`Zi z+=a^W0t6=SLJnr{z?9uYZ)>)cR&a!2hQGKL+EsKC;@h9Bzv=Cuak3M=SY$k?6vH#5 z`a?8wyXL53C9^Y>P?OCg=-uJwPhhB{AI5)CV4L~wk*VZAjm>vR3P3G13`~o91o=2S5Dtwdx?rKk|~^~Y6d6ea_<2=-hn!3cE335 zE4JhBafktbOj*mI3*Y_$96jKP01WhTADzo#Q);6)Dd~g`^)LrSLnzP8+KIj(Q5vl( zkpgan--#mGiho=@jFQ=Uj^!@Li!k>S0LiNI{iPhHC+HXr7{050nC&&rZCA=Ux=;fn zUlMFoiLcOD{j%XlhlaL-qjd8(rKYWs+Q)H0&2{@5WNX18p+7pL?hl%{m?9^RRd=QC z0-;*pk$a!PvmRa zSGsGGd-8!6@psC-xS1sD=lVh7EnZ@w$YhI~QC#`Ey6u#G*^0cvD^O83jHj9lH*bWM zLJrj-(~qXut|fQf0$qk}mV!9BgPai{PpuRUnzHn9vcvr%=n2D~m_tg?2KWaP$f;1c zp%P_Os+;*QtP#u*NbI}e{ap&e2Q}AR=1Q0iG(|IAbDOM5w7L=p5dpf!Cy#^*x2hVw zJb3sf+e|$R7AK@R%CJRq{3}vRCmUj@>7{rMy|xE5>wQd^?#o(;3fJud75ckgq(6Ob zQZeQ$ew&2(iTrkvo=Z|`ccJTKE|SNS?*ES-C1?hk zE#^l{xVF|L^bou8qinMv9lx|#-{n;{8D8a4Z4fHycvUtYHUIui!9g-fanV~-_$eL) z5>yV`}+oP!R1gDbC~7>!r@sc9-vl!XHU+8YSYnHn=SLkuft$YIc`In@TQm`1zDZtuOKgv+fHuVhb_LYaiCY zJ5_k(Z@rh3ZL5F0_L7%jeT@2EPZ%>DLBlr04@jz{Fw)XIfweqFQU&AP`R zxBQgYl}We7+{wr6C^5Ghb>7Akdg0ajBf23Lu)84*{DMGXKuMFDLJJalS*A4nvtT=q zu~XT}LCrQ9`<`bY4);n8Y0SLhhs+_IyYUtBNiEMrElMkuuf62B{WltcBDjMvhRU zKPb(P8+ThxZEH6%9G!P#NG=*7JKE z>321a`3*7h}~wrCfryy(1^L z-6S!qJ$qTwR6oLUNL`koe`8+Osu5Y}zb(O~J#0thk_^$1W^z@UkE$9(r@gw<<@{IF z<5mwz8Q}Un)XuOpZ$pDXRs9DpRd}5dJ*f$QC2_vLS)S7BV!Z9BEoN~iZ8<%ELg$`a z7o^YuH?G~3Vos}t(HpaDo=4ho%*TWhIlv1dieSApRs7?kqM9T}aX-FKOG0 zl_+D^5%L~rKGfSZ9agn|We{~fvvfbli8Ni4B7S6d8_a5b(8@C}vT4RRrPYg0wP(=h z15$EbbvReUg?3ZB_aqT7+_)XSVXZV!nN+f4HQpmw$$L^>Nj)vb?Ta2`6(T@$tq`l& z8yD3C61+u+sSP#uwnk`_1{kb#g%=|%;ol>+(}%69p1+zI(0g2}26UMZJ{u$|e%VG` zFGKD?!_q$BmSy6`C&f6NMYvQ?&A2d+xG&?->Qo-1V#^_MdsK|rLac9aJ+iwfy*6DC zB9eJ6>mfx|bukyE8!wT51KdQ-7G;}I2A>d~i?~W9<1;Qcy|e8QpTUV|mz1vqOIf!m zj$vN*cqe`nf$6Ay_P>?1+Ol7==)v=y~? z*zj$(`eK;(%p?%9+0jMDauB@lAVyJFk#nmJI$I z7{l(RD~?q1GqH!i(n5gVLw_B-1o8jM+5ZzOA&n06cJIdR?{1+h4>5GB;}8vVew)%u zCXeL5K{s9v>R z{u?3s4^+QH#V$18@=V-l+6t?M!9jnsDPf*@9yh^GTw0;gTV1oawA*31-RBj)VytVf zPM+o3$l0l--uYok;nqEW$IlUrkjYXUy;dRY0=w0|h##n{!KLtNin&bpi)uFxmWoQ! zywMdDIQEe{T(CdBd#w1Ti(6x`OKsq>S!x_Cv7D&f-un&Wwo>*+OtEOO=TOh=sQQl7 zMdGerIuu_S4?l-klS1__kkBB%8_}=Oz}EJ^lx4--L^m{Jx4I=0bMpk(B1~!mLx5NO znAh|jk!M{{W?hnNDEUDt0AdPf{LLZ8P0Vc-uR1zObtwJrofOSuNv1Bk7fYO$vc^XO z?b2p@l?%U+6s&jSA+D)$N#L>omL8cVak-1ojm-;hSUk7vc5R#c*_ ze~x|_-|Q3NydJ?Kjv04!yK+G~eSvOcG&n@{vTL3KrOXDJ zK2hE5n2F0IjVvdh?fOb!CVfmTz#~pKoOB?;im+IFh%DbP!1+NbU5g{WNtD!kiF}Ip zR!$PTf8Zd`6Q!r&ZX_H}_ENpmc#NAs3ayK|gW_l2WA!!)129;mcJ@O%dX0vE!NCUN ziJ!h;3QDC=-0A{m_!TYwfmS)U_e)97+E<+34G6<@iIkKUObXK#=liJh^_L6|ttJz^ z`eD`xn>*i7JxzuA*3`O+0%IH3!*0f9B;$Rf(q1%=z2z>2EiUi5)#wz=>~B|lDO=z> z(85clP|LfqaIbZ`8Ix08W8Y5bG}E}0J`bNcH|~Un>l3>aTgcA~d~A6st$JoqcBg0o z(YFI5W2%?i4?L0tc4r<)%uBJtMHtaXsj47LpQ(PDv5l{JKML?UyO=9{H#v_Ap9s|y z2>%N~;amPchsja?FyTijT}fv!c#RxT+KH+w}cDN-oi^H zm>A>Rw&ddsb?e1Mq(=M@^bri2SN#h!s8wuo?%uiXzTrl`Ow*PavDfd!iWVD3DinYw zz!c@TsK3MkH~vn z3B9w|O+$i5Bm<%^N%5j$(7!xcKlNr97~BpcIXco>S47_9v>TO&Dp z9FKj7QbHN8Hbn%E5L@bx;M?l_QU9}}o#Y?cr@%MX8yqL4iVrgn3Rh>Jt0eS2_UqWV z)FjDeRbM}j^FGKQCXA=4ow|FcW6p|ac(!}znM>`!Kp zOxZ|&xtbJ=mIu>#!ftSbrLb09>&WlP7E-dK)02acS;v<`Y~nIJsBP`Jq0)L0j^eT2n)7O%SYI$B!X8{)l?a@6DN1WZ>uY&tbg@`soX4}Ko9Nq=uhpJL*80i)d&JQ0 z#{}bit=7Gc@lArJArxDfhH`lFh~^{tNGp2$; ze!VwB>fFv_?vk;Fi4-{}!e~lbYt)IE|3VMXKIyv{U^F-fog8c8uA~a zPJ+Jm3thi=B)=!>k@ZPEJH9A`lj*;Ht~eBRC+HA-8jYPEVYHZWD>psAH`v5rD%-F# z%k7|GIv0R$ZH+Eu+D*^0W}r97+7cUCV8;=LUf)QtTdMBU&j!WO&oFwwi|TYW2gI%( zwmKecv_lmZ2S@U}AVibrx=vtsbcNLaJG}q7ZTEtd_NI?%5-TeoFuMp0fRGJjcOzG@ zPzlA)8pQ_6A|3fn6k_^#PcNLLYr@F{HaqW9zo{*F7B~-H@5}AGROv?fF>Sj)r;}9w ze_~;yo}Ln8wVqSP8-FcjAwcE+o42+ zjfUsht&xQ>)OBKxt$M>*CQzaA6$mUQcL<*=s9hIIiXTu4hYKSnjE301URlTM`QSO(7iKq=+N_>Sm9YNBwN2rDEtohqrADRnSjq_M!%+FZwsw(>t8-_i#Km2zb52roVs(yl>f7_9Mz8;;bCUxvH4q|I% zZ#Cm&ckP}Z2_()aRTEV%dZ=S~8|y=VSrM@qufI}F8u{Sj31Z7zMspdGM|s~7OYvqr z14!IWTSAxL{P@IS!_NyH9Ml}G{BRGIHQ`~KjV}8T~ms6g{=Eq0Y8P`viorTx7K+lQkSsi z{mDi~ez&a7tCIA!UO?LsBIy^gJs+m}O~`7IUc|;uG4;G46Z;X$U5!O8PVtw0U}YCbj728Q6m>)X}B|)ShXM zl6r^lEzk}(xzcY?s{aoboM@AecD0A z#nEzHVXvl<3o0;D$!vC*j8V(~-_c{*+FL0|_YDWsbp~4UKu}|Zhf?Tt^0=``1Vkfi zxkYZDP0Y&;JC4K)kiKW%MKvr*Uy_F+g{ne;Xd>cT4l9V1#hKPJ={eY#$Ti(1Wk@;z zT~z!YG%F=yWoj35^P2sNZ5^XG%~~bsx(w~Czk?lx>sgQ&6f%3p&)z+;XB8NfrmXa6 zt+Hsj=+&){Pb3dpUD)Z#iH5R84>5-m>E^+vFiIW6{(I7d9c4p9o@+J3XmkZ4w%`t1 zTy;@06Ls6l_5}KV;pKBm1GuIKdyZP#a+7x(LyWf$tO;g%;eS1`t&&#CmRyux%>59f!QTx7vAQUJgh*R$rx zKCG@h!R?YSZvmkcU}!`9A$bCF2=3qlN16w^5FoE~J4C;C!^fxr{J{%f9ONq-y&s*b z{X8Ukvg|L55%@V~4a@@X0kAKw-W1UI%VsS(c9b)V+I?;a)e0w)Zi6@e*HJ|D$X4qb&UE3|XY+US{7ao(u^Y3U(JGTz zTB>4Xhj}lVcG-wR)ZO0YZRH#4YHL<}zTILmIQz9(12m_n{W~zH>bVRyr5{xCApFI5 zlwcJCDpH?9azftgn?H$Jjo01$zh-|xQ@)BXcsYj{$IY`FB~F?()?RQN$;ueVxW zYXKqu01(y9zsdE+cLA^hso4xH-a#E>}87xn0)MT^JFom1HtpjGATj{L4PPK0+^w(;;k!0A5T z#6r*Zm!|*u{Ha>?S8WlrT@eHu>k$W;ef@RBLE76X|H=*5z(4esg=y(j6c+JTrZh4x zW0ldK{I@v2tbX$K-)LUnR~FXOOj>Nug30xQgECU_nb(=CI(zbkHJOqVKm7XtBX9eZ$kcn)cygkGlQN&F5E+YuYIo3;SJtt>v^^L2NySzk%p7@03ZwBce=%R|As z(H%ZN9axSQfk)q8)cxzfT)4t>d2k&+=IFfV*Z&^8JX_t2dIWx58xcJKQD*b3Zp@8A za&Q~cq)aYtC+X`w<*)pCJnd*-%3O5%4n`ldzfW~t8L;Uk%+h-I#`k|#cDz_25(DYe zzGf52Uay+_+>QSOj0}gh+;@Ji0bX~I3fy=W>@x_G{I?Xq5DOi$q=CFOYFaD^9V`+g(=&NfH(tib3ya zj~YDcB{!q}9ugk@`UaW|g!5iWQX9OzEZTrGZais_H5{nh56F<=ym3kJ*yL{U`h9gP zSmp8AJ~QZyT>4LB4nCle4X?E(!r)XsF!fwj`Bn)gI^w=|8wed`f1OT)1|wN?y;H~X z!cyS*;my{|1!LyNGgY&F+Z;+8J#5WqgfdA=4IWI;>WLL%bVueb|MhS?S!9A6V5$$0x!a3hXXt|C4qI867Ayu#Mel3nC*S_k! z^kOU(j+tawQRXQzRq*F!E7$e?$Hm7evzg|aCgrcu1q1jzo|F5PZW%51W7QCM>76Qu03c(zPL0{`ONZ&+i#Vf_-*hiS%SZ&r_jy0 z8jd6#IpoI{yUO)vLF}t}_iWQeae*S6r6P5n_HgPhfFbJmL+-jxTTY=z@5g2KsU)S3 zyZMyymV!O0+S7dI4& zj0G0aCeCtM=z!RaA5~)6MV`DwO$bTPDI6y>_AKg0Qkn&Eavg?W=zCgtR&0FAJ3G;ee zc7Js=R^p~DCh&?s**&=UG)78SlQn(;e^ub&gYRKwFN<0)vTF7#_3)AG%2fz-_xl*l z{m3ibaI?q8K->zq?XEIhBfio=r(!9+bGju>63x+maxuR4wd<4Y&&KzMcmc=1x32yK zkv=1BrJ4W4O9-(C*7@FhB60|+Qm#}b@#lx0oyT9iw`hI( zKetW8&ZGoWV+(gSyNk`nKb|LT62CiGtMG8KGW#RaJdIv&gi~!la^*I-8}kpaOLJc+ zvYUC|nb7c5-9Y{dDS87JBqs3x^8aQ8Z_CC7*|kJ^Y1p;pF^wOIofd#M1ZK?ud!_d> z1moXdcP`B>jQ+yC)|%x%z!=R#`>$8_f_!7cnJyRGJOWTXQS~yLo+BKmhSC^51f>)6 zANax@#ORNZ6zP+H05kYs4J7su+!*+7VNkaL zxAM*Rhsl3{|1}dV;hR+)@Ewx|r1~eM%UfPRpresvaP&swaATn3@qgE)07icQe^#(+ zb}|R^psbe`AwR9bcd^U&iLCd_@2!7;|NWW!(4zNg#y9j7@=s;?04BY4tY{9nZRq(gS32FQ2 zPfWwR0?j}FYBm^*&dk3gSN{WmQ=3Vb+-SHXZ&)-FiX0#vx(9(~!jwZ%*UUXewBU-j;>OJ^vjlA_YQeXD>pQyARpsjl1DsbIdqLjVg$ti#aFI&VHoQLCS8@kcMoDAM<- zPxwGVW*m}w^i_FIg9YXDK8{DP=Wnn2#SctPhi_n#{NZQ{Dfs6tEddnNGDg~t-D|km Uq4FCoTdn%&rt;Q4NdMOVKc)^khyVZp literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000026_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000026_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..54e77c560f1c0d5dfd343aa955f374c6a5c8c6f2 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfm|l^$qe zCbir)YR=$K^T=^D%EpW@+J?y#U=7;Cnwwu*oQky?W&&2jQk_m2Ovu@sf#K|#GkZZI Xqcrw#Kn<-?9w{LJN|L3CKtc}y+xb2<{LF4udB^AV|;z z2^^B=eedV~s=o8@RGm#VteNiKd&#w~wY#f%SbEq5;6voVasVV`008On2k>wYc&cRL zVJ67U$;ZjV%?)@!0A2wwu&}VPuyOxh_)qcwdl3>6l8}%*{>H~ABqSy#At56pd;H?@ z4>>tG4Gk3)BO{QVo$p})(DnF$CrC(-kNDpUe*`@#AY$OH{|rs+-I~=fB_&3aW^_3v z;Y*s)3@&U)eg%{l-QOcYdPBwyq}6eI=!G64XE0PwH^z(GNJ z{O~AC0M3DW5WKQ;d^S&+twt*SIKsWo+K5{NTDXNvOT$89D9~7d4Ye}TYrdi!p7J|( zT_f3dwKzFh6WYEEj;)77ffdqAI#_WOrjmAzPiEe} zT}t$V>E(Di_7B1wkXrIsNu6_y_=!l$mP$y)0@x7Y+!~qu4MMU;IzQWdR~zsu?$p8O z;MaS9rYS!o0{@-dk)88)_mvN!H!P5R8>uJ|i<{?(Kk?bli{{C40Wvbun)+dEv50Ef z&$gnaJ{AsV_kSlk4z!2inuNcuHmi1tcdb$AB2$FI}OoH9- zxx4%4Y{eI>%`rJsoO%2es?zUYFR$+1DdygMWY~yw-OwI*jqodVQvDxG_5WBZxP&D3 ze+)22J>~qb)@JsBM1P;p@heUI(pNU71@QmnbfHBL>JBoO>#ysWVfThJnvzV+`?^R< zTf6juE+3huyp0Haw)KhG)Wr5yZhgFgJ3bPi23T$)n=GR`?^nuRn|Mp?Ms_EAw%7bpVeHF7_nYC6MwD@Y7D(Muo zNCNJy2Y}^Qml3V;@h(612};tK;5<1N7*^8HSAgBle|gC0l`B(XW0LZ`zlkTOewWQO zg@$HXXbO|G+`iKGj%5csi`nyz&)&>q)z}eQ2sYis2BZW>?eK~jurXEX-yrDy$H5nR z{UFaKhoyE4Mdj>@D2=bvd2Fi5)&&tE%-Fl1GY@;0f7*Mo46(t2i&I$N>VnnOXw)Qy zHRC|xNK;;agVFsrkROvSZVt1j8>+fGq6`aR1(gtK%$a!KLKPM>=`pVU30<8AuD)`+ z5G>I!q|Dx!LjcXcT~cvGxlk2dHlb7R!y5+DhK7d1L5!pJ77TU933YPUPPfjC=ddWX zWSj97b%+L>QF9UnAoXvA_%XYCIC?0*Pd(JD6LSZmM#7SdZy+x%sNuzWqnx`2+IMyz zq@`oS3@|^q9m+VaH3v%nXZo7$&om_)tHoQIo%dD~lnK@{bb1qPqg@W4=hfr@6@i5F zT(8?w-XtqG*^Dx8zVK4+=*d$E0}c#Io8<&L`~TaBnn@3sENxb^)drc%)vPBpIWdF> z0;K-cVR%@T(mlCCn=a(i9)Mpi%dnEH!GAX4l>31jPvYOW{;Ay{$fw<@Hx{Zhx)YlN zZZ4};BW%W{Q!jl0SbQb89uGenU#QY+bbd*ARQ^gwjTIUQ;(0dn zCN-wi=l*a{HXPOIOs)_p)s4Zy!_mHF2ibCX?32Y;qK&qt7&O3HIV9Pk90xxpw0IQ( zU;ht$8Gr1rOQ~kE#ccPVtZFF6mrf*w({g1JNrmaPFO#FZ9phLppYURD{i53jXa4A1 z`k9vv!>M`2TDL(nt7;U#-+W#Zj`NrtT_Re(bHaE0JKFx&YwnERm4-lK+ zPxs{$mQ`%?m7wwOPuO6EaH(O{6drNMXg}4hs=bw^Up0Pu8~r*{&Et34FRKT+5Eilm zDzV9H>$Va)sMm1yb&c*c7o?O&pxI;Fws7v_r`Ujaes-4lWWID2@)Y$PX#bC%UB7H! zT(!Jp&x8H^vEX{-@&F*2s&tTOoeRxQp%iN6JkR7x4RL`fUt||KR^AC^-|Ep4-dA_` zdrvjjpK=|ncY`Ma^ezo{?`-_w=Oa6menwlJf7DBRG?Ma@Uv*q$0BovwSEBl+=EoO} zBd%O|R>Z*r0C(@@!tg+MrQ?rv*CRV$#haFoiQSjo4u8flCfi>IgrDcxnr$1+xBhbc zqaV|6y_f%Q10D@X{WBooKdNjEm9EIMon9J+H|ANJJ$l=&|6<5I?*0w?oCtSm+ci#F6j=~EYZ+@u?-IubQuHLjd_UZIL0Mf_) zSmnfaWz_1`7QgPSbagINt=D0VHIoH@VvMvqZ6UxQLmm3E={|8vmTN~9SDXC-Fyr*c zH7vIi-H(r28&0c!PPdVHf)2|ZGGvgxRTOSsh1QI=RodvxiAM-t?VFUQ|RH)AEJ9F@K3NFVe%(fh#M8Nn~g_x<~j@+OhvWD9d-uauiHjp zX~MChB>dK(W(W|pz-A;dmvWQaTX>E3j;i?z3bx z`O%}xMj8lQ#GZg)o#i5s-&|nkH_`q77oRO(WDozvCG^NElDuYSau1)#e^Xbw8Kc#n z*@j0N>e(q?l*I;(+2F1Eg(ae)XLwSy5l348pS1de%10RfMRMc*i@5LpYgF{O?=K+A&`N*E9xvSr0^#$#t zTZ~reb6~7Cd(Onyu}+fx1ppKREw0RF1xX0XUZuKh*OY5V3MFjSK+``T2bGRVq|z$? zl90H%c8uDa7y3tfu{nR(oolxJ%6o#ViBFr%+opr9Gr?hhCXZJJz?y0D0b92N5oxI9 z0ewHyVMu`eLE9dg4a$E2{Jld+wb6V10B9Ze)qYvt{a-PgqATruLO6cimGww$oAX!j zNmjpd1)PZq1vNQ&pQkC()$Zz(<rXd2_2b0MkxXcJx{q(Umj{Dp4jX;BD5-7o9W+54`mrw zf)mtoG?XljWdE`AzZUa_XgAz@zayj^c>dG6lJDA02fHb{L9&%?u1i{MAdrntZiqdn zHm|J-q+MyRJB!(pP`ggYOzgdLVH9%y0B}rwBwLMu|BAu_K3se0&h(wzu)^kY2rrxv zP6f<%x)$^6-n{$Ypl$qD{L2$wf5ub#rA*DVBfQqfozC@dS}FgN1lkfKC;OFlAps(? z1EpP5jgIAaLRq(ZRFw|^#*KdO?&kWj?lZo9+bf_dMV>WP<)8k{|Mu7Z?H9Uez9ppn z-~Pwk*YD<-s`UPY!;FAGBn+=z`h&hla^yU{wEoSPtsdg@{Ke!{O2deR;;!nw*(3if z+>~;qRn>=FozRj}ev)lnVYx{4HcdMla9z7z@%)DbvwzrJ`HBt?gr7&)U%k1=b$mKu zA%FYFNulQf2QC4R6(Sq!4qI+snE9DsAfjt!^Zx z-?tt-vWjZcM{lYu$BH|?KhAk1Sj0cl$_?bv_}^;(xlfAyWASj!vrYR`WCQ+@{uUrj zs=tNq0*0{v{o+eZ3&QvrVgZ3d`SvtW5Mlp+Z9=?*+abkU2TllQ){;hwBzY-bDxAp= zwzS9AXICmSXEEZOKRG__P`*WcqZ>kafWwd_=_o>UjH9}287AS{5WOE9w*}{?sk^rQ z1bVhcY0bw!6An13Z$XmD5o_JM`&!YhiQ6NuNSi|Spi(oLKX3^~WiRt3!w=+Ck>sDt z3h1;*orOm^bIR{JIJsz~NzV{lb_t5j?{8>(E8Fxa*;m{{*XKQbr~{buyt;(zNyA?? z#KQxU#D=WN)l{-wBRx82W5eO!@saO!W8Be8GIb_N77^Q zrWl40_DyOC`xd~TwzQBRnrJXdA$$9gJ}9t1j;vbvz|tipu-d{rapI^QBRZVVhTXDS#GCf{MRTUB*H8k*$8_*b%*sk#L$ckzCJPe`Z)bWSE^hMhLH8oBU^~ zTda0ceT?37%MBh5rp&cLhM-A>0~nP-*wBi?L~V#BcrH1q^QAmmz0g3#=W3waVDDYV z&)biw#R%e+9x4q2&Xpe{RhROUK~-yYocNYPJUE5&$6RnNu;rT~Zq{)UAo<{uh0?b0 zxd1-htHqE?j@?Xrgnetmqt<^s^0%QX4M)f&7Y|+KY_9%PWz@IdCvQ9EOV+@jY>r#0 zkv@EHUO}?6SESkQL zRl#`C;-Po|C)dFCDnC}&Ty<&6)RU{Okk{oNzhs}NmY0-6)DsP8y1qOiWMWUTZ>@zY z*?+ft%t(afKN-2+@6_|`?O}3?8uAXy^XgqH2!s*bAj0nm77uu(*`g`Su}}5mQgAvO zU5}op>m1_Qzn9Pu6CZy1-LbA`XuYL}=YN9oCom-T%PmN-wX8?sPzVQw(g(L6@Cf67 za^K8R&kpC&A873`&TrgKm!wFj^ONM73G}jGgs5m{zPPBy*{(8VIU4)*zB|Tsoy&pf zfbn(Hmj0wQH(E_A%24|WnUaBANNL|8^bGlo z-QA>mT?OS9#B@Rn!VanBvgI)m!nXp=98KDcDG+s#WMfR1&nfByZOO?Hb-VD>s!(jR zvA8sEYtKedB8khGmsiC6_p~%}BjOLb{9D}-l*WWsnAT~HBWN7SWk>Py)dlNE`fHIX zy1i#}d*3vYVjY%68NQ5bCVo8cZprG$PmW1Ad}AQmh7%E3@Qj?Ff9YxS zA|t|fEG(=STI9?itm*74aYno&HmknTIx9g`x&O6j;q6Q#%T>glGgeeR@FRwQ3AzjV zt@gcTjfA>~L!Q7Hciz_OaJlfy75$Cp>rll909Wh4P72%mJ8Jgf>Ta36KiQ7;X!W1; z_zS{ENIcdF^Uz3cz`9X}Ww+Lw!`+i*+0L3!{kCLkA>zVh)<8J%TGSC$TyC?Mns$<9 zd}wLVeswon>&@_sGMjo6a6COo7t&gR`9a#cnXB#a9x-|qfhpZ78XDl%Kv=#}Ji%%f zl>3y;cZZ@Dl9Fd=$e02eCgyBGIM(dGvTs=ftQlk(ljHU;JaUwWeV`DXMQVX`ZjOp* zlI2yjAFW2DO^a+ncJ;=G9lDA!yiQ?Jf71;b`FA|l(m0nPXI?@=QCD0*`7V#Ofo+RE z4>cl{>{^;r7_Y;Qo_(t+PzM$k(q8uK>i6bngf}G9VW*t``LBiFnf@+X%g|MYwf^?XG@F5q&i^R%!M2)U0z{k?yi9 z_$y~$btIc=Kz-Q!Tv>|_V9@-bvY2_7@7gsLpfX~WHp?Iq)d3!!fz3Vjx@yi3`8 z`$WwbGb-#iYrleOc^Svw2Nr5Ur#Y|I^&DVwyi-gCv9Oj{GT{g>0&2W~^ z257VvLV?i51M; z>`3%e6tt({%e(0o!`p{j-fDmnX$RV{x(4PAeBS9xu*2R6cw6p_w}OXfwR{F#T!o{w z-|t`s&E{Nd1-Lr0O0U^({BZA2I#T4yQkA|la#4`ep%9{$HcJPHt~_omXztdO?L%k7 zSVMoRQrE_aRL7&C$}N1WFE;y__xW0*78Wq(IL?Jl?OgOE`1;sQ=+?98I8n@uQV7rb zWlM6+^dyH&#pYL9y3K{pm?-Ke`wJ&KA+4pNLr1#^%S_V-#UDySYV~)D#h*4rHT|L*M4Gi>^TnmE7=Lkb{GiP^j)i_h?#-4JW%r z6f(64rElCtfvWJsYk^nZ(%}ax=tz|1aSCRb;biD0Mxt2C0*{!$PEmJ;ByMGaN-YDMTo zf4Jr2Pk6h0PmWKTF<{3v@D(o7DhWwa68{YD_aPtTt^lxi8Tbl`Dl7sQ1XR0w?`nAr z&WWHl3wk$G5aJ7VR4}t7NFfaQDZFQv!Zn=GI7Gy{M99R@`nr=yhlio)I0g|=>Eeb9 z1ZSQoEKk4mD+z-tC;@|X%@n>2)?9h`@Fl#~dOygGQ+a^Q2%O6E+X+3B_UoXGa)Ci*Jef8Yc%LyPPIR1_FC5r+6v`B&x0ls{_31M5IfqT zTaiecHusX1tPC!RQbnf!p|$s|*RNrIuHykPx~Q65&XR3^`1aHL7T*CRO0%yz>X|wW zZb2>Br|iEQSK5=av|ZmrNdTp?^+;IZAb?hCwy-L>8^M~CvuEZU1j!e)q>4fQR5PAB-X+@o5qFVIW2A%*`4zgZmG)U?)RZJYva2e8f?#Xw3={@Ytcx%t37K< zUW0PKgDZaUsGt3&kis%ED#VfXZdMhIF|D*DP4@L;iGrc4>%kA-edQZ_e2>$R7$xJ& ztWS~asKihOiA#g6$RyOi#w%cUI1L{u#QjRf`08t9yWWv8X*ln!?r=1Rv~w0(A^6f= zQC$Q*?Sue5GVOdmz`0^~jxg%G&ab5_&{^`FMh?QKPGo)Ns3=z#S^ALfiMFP9+ko-) z34u0RA)L0e5L~jOa_HK*tD@NPJHokcdVPTD$ zKOK-#gU;7D2Yq`0z${ib!mp z#FxTX2o?%yzg0EafSEP&E)v7w#APJU$08wf8HwQ4yL{c^P&T|@-)Oa$)=F}xq>omc z(+vx;kcleo08WeLHF6 zCS4sfT1KkKbiSz-mb0{|I+!()I&%wiGef>kxcBFHj73u5VAM!_Ua}ZVLd*Jeet!KC zJFDY)0M}kQxvWf|QmH?!BFFpkU_jq9wxnW|(g3gnaz0(}y6u^F zKERvkTJiR=YTOzl24e#WG2%cGxh^PYN{bdA;R<=Q9PWa>*#d}&pPxbLnc#pIpJvO4 zT+~ylV;XvWCywE)`(T7M7W095jQpfPFlq$K2k%pNNmyY844SECPtX!47DXW$%smt( zsv?Yp8&vM>LR<^6N~DAN$HA{X12*?5q$Twocm~+tejx1;!E0c3XT>7tAz50cmmyX= zS$>9r{%mChA`Of-LBlKW%ZKH=w~+BDh^QqHY5=2V19vx%)rE*cD=`ZBWA2W_B};#L zMFuKt7>ZIKGj9}V##2mDvQ(=6=4Q2<=1raP6U)A(O4lU<=i6aAx{?z8oEiUGH*0XL74VAf5CI zcz-bTbh(3Fu=7gplZi2tB33-`DO;4a!0MeGP8!bBhot6xy(AiO_LN^>xC ztxz--ypH3!&}73(X;q`7Ef{sRiO$S=Ylv;?wQ<2+sR|t@S~C3T!-!a>`Ua?atrN#I zrn!Z6p^6pKd%@5gpsdow#y$iwcy>d@$-&cd*$N+ctcwM!Pvq(@`(U=~&Yw1% zQ&z9X7|`H|Q(~D!(tLojjAIhI>waFUf5C+9(UV9eJ!$+zLAfGNVtXlrWcD-ucIylzw@G7nqo0Z&0NHTPG}q~9 z>)~Hz!};XszG8ds(AlbmzJM{Ts(6>2rSdH-c6zX3*#F^RrOO2T`7XNkw z-%bw-abrPJ%d*G1*S)Mr9`lHrt;B0>3UH~aSNE5`(S;r{&LtrXaveZ4B9bUGy(}3~ zKkdIJSZ*hHKar$Hp_XH{{@|ULboR56lI+WEM;_J#U~K<>Pfx(=jbEH&Zf^}F+3x=E z!-D@%O&V@rkuwBXm2(q46+p&A&csS?(1!eYjDScj8!({sg4B-QTaCnl(>tl^e0uSX zAg71n$aqP7Q&qfgTeiSkc}#!xxxIQMa^fnu5OGnE`h{C8kEpl)piUjB#ulTJHzuY? zt7e8a8682{3h%GhgSx_DWy^IRFR;_-|9ZD}4TdnH}13k+Xo7|YxG3a`!(?i}@WFh3*?1R~;oMO;I z3`NzPwl#9LUOc}daC!!fGDWKfedm3XiLyH64y*ds(yo4y!Y4c?8TsA!%=8FW%*;)V zVI?*?bh4YQEY2aviZqnN?Zmk<)68WFc*RudY1|vCEM#U7LV%uG6WLH$2nnRJY>uH%+bQ(4c%s;F&eYI)vIAYw&>69pvYgVIk6orqo&xozPba-9u9#?tC z;Bu_nLYLFybQTe1GB;bpAtX)|`e{$Gk_;6Xa~>w>;3nH>ry{n#@-{ssya93O_-==} zqmEL^akxBn-2f5=EyQILGMV*(SN6(7f;NbR`?I!LCy~uZqb8yfMJf;@%0giiLL}w{ zn8Lv0RmuLELctLrkBtznS@NLwH}fMCw)XG}3X_&tM4@oI9}a&B5tdf5Bzc0_+wf$a!iA@H%=Sv3(MK9U56*Y_^d^D+%wE5 zC+7l@$cy4Y(}tvvDLDuu@0Glg4t1UA$?u?6H-ipmVv&DH^LGyc@CsF81_s z60g)6z1baRjEz=5qYyKXSRM0fP@K-7c0%9A-;T7 zQ5=XX$z-&cQCH1?M#+Qb>FQ(1#Q!;;-dYJFxi+w1{DpHS9UO)_mF2v zQYFq6t$cmJZ|CtmKSoxuYS6}n*%ti$9cM`}$VC$)5D-C}Eyoz-MCeA#0eXC~QuX0! zBZGp112V*HI7>WOfR;jDg605_VM`MdmV^EO1Yj@5u@veR2l5h8tzRc6RO_2 z@-?FMdzDrmkxyryYjb|MB0MG4ByU=)-rc!A`_Us>%EhZCu72opLt8N1o*o5I*y!+I zoOJbL-5AF$HvM^4s{W5@CrLvqMtXtwJ7*$+t8Izbk?mOK^}m?mm#%PPhk3 z^;bHlrZ_g%*g%43U)384cECY2U@?}Jqy2~r#aDCkuQ4#^ODBv?4@_71%bt)I<04*I zjN@C$^pi9X$A~89LDNwrr!_MxYz7LElNds;Km0iHv4@TM-Yf+#n&eKH7B0W1=-tl4JV8sQ4c{9r-X}qp-131hsV={4hC~6Ww;=; z5YMCQ&qcWl2Hk?m2mEs~>1(v}13yj-hz#8|FWwR8*tW9cBW=V1{WuJ;zI)TZfO7!1 zl}2HdSHd=}F56Lo6qcn0*^DXUDms#(5n^vWjZZw*EeP^skIq}O=6WsW4`hB6rn~Y2 zt@8^`uFe3To^xWVQ+eJLrZ2jdDP8P;xBIMfd3Pp5lA>1*ZU10;;;>t=NUNA`S6tDY zFWNftrH(90xvBVD5XqO1$t0DWDuk_5=4zD+xw#8ZR+W38+2H-=xpl1M6Q1^WauJ$& z+LIk*=V;<+i&-=^xuy1oth=RP_r-0sS=FhO#I>fZNQk)#4+{~c-qXiL?-^Uu`ZN)vZ18(XGd|`qp2G4!y8~N-uKvoDtE(P zug&!MV**Bw#IT ziAsUz9bx=<4A>2%%ZX?!E-UR*9$;5PpIwM~8<3z@AyJaX2VuXis-O6AA>MZ<(ywqo zD>-2a)eO^Z3atW>sP_f98>i>*S&3J#RCFX8BslC8c}_a)sDFh_=i8#em>I(QclYHu z)-1~P3Yw2UtCt|^x%h6<8V`#pAwmx3-dZ`#OiamOo>*SFyM=S~L?V%kVfPnr@YKX6>rzu)!FT3UtZ8_nAAV z7hD97<8EyPz+x}3$1`@{z+lmtv3=h@hdmRVaS_pb5j}r|O@;*xOy*$u$x3IVt;6@E zZ1+~399xjg8OtcDFb{-+duevSL3v3be`TrV!$mMg8gQ%gD&z5xL?wqRtC5&QasV13 z98;J>c>K*PQ9X9D>cVY55;!|iT)MK!S?PFgw=Se(#5Y|25@tr1hY1@p!OUND3PXB^ zf(1b79hkT;6k7C_?v?sLoIs{jB1!l({A#y7ViAsQ1ogflpDo^3(e6>6C|O&bT%jx1 z)(XYZ>Jw`7bg`BSdjdSXujW?R2u%{cSq4Dk;FRbMwUZxN9N;bV$2+)9^ri)nS^ib- z0;dZsr^q)#FyG>`K&PZvO(>xq*fvbum~*Ijp5XY4W6+j@A7KF4qhclnE|?$l>Q}G= zgRTxSVG47dbyK(P*j0(@nc#_6Fo8Wq0!WDH){i{%c=PtfYiMPyUUQ=ke8v@KZ|y7} zRa@Wq4Vq{-N0_wN8d3W3Yi{q6t(kgbzEth|H%QR|gsR+r?h}_;KVijT-t*3{(rJr@ zud7x#^N}K7zFto@R^^X6^yRcU(*1>y`0+x|Y}Hlpmg{Ag{Mo)*0guPXXMvzQRnQis zTk*F*;)MC3p`3*XF`UUnRQ^m!ZdLS1mX7;aPbmqS zE`QcLjAq+iz95CcyxxzgUZuiUH`G&B^-}D;uo7N2I@ZTewmDj6g%s7%IrVy}pQnRM zIW)9we_fo2+<*G5=(3C=RI!ar|B{fX!g^P;3?tHSNKqnf3IRP#!B0Y0i_<%`7aSOz zkbK7G9FvZu5k6MWJ|Nv4-q-<2+PT8tPLqI=D<1C;&s8K_YkEdG`$t^)tko}Hk=j@M znq!cLHg5y9K%oBjF|I9V)(RkKs`buv0mMdtUmX}jeqwXb!J3XMV=eO*GSqibjB?o*Jg<(SaM-%!nefC za&^>mJF05IUo0k@o~POJUe4;aIt&=Ii;>9hBHx7>dQ@BK>Ov1hzCpu;DyAxJ7$asw zFSggXpWS3OG(=WWiMucB)4x0W$%525$^Pu@rD*lH!|lN%de&n%qf;U@#wFc|(P~L@ zK!&oXB#S4%{pO~=N-)%RygApx_WcN!1~GD;Dj6eZ7-czT90cNt>H!D^e5=|FijaO2VC0>ct zIS0lI29|IUU;=4z0(d2#lLv5PtBLv1H)br+vqK3n?*)UH^d`9}aXRH3fwi2K8bLI= zxBA~up4og)e5snLhN%;mg-gZDSOvlo`OVR2xP4-C^SBO8l;n$RYH1T-3iXCz z_Iq$fHh&BVsU(92yroi_&znFZA~)nV@6t+V?Ti8iRAEn zQ^QUb!)6MVLMh%c-k0Yr@77;V6|5XJB(p|2hQ=@w)Cojno1eF)ze;`m0BB?AZl*g~ zo|lthk*1To&bb{FihiMM5hLZ&B?IuBa;i*vd!tk;9n}9({_?=t?g7x7n<180FZ?_M z?b|%d+AKj!siPQA#Q_v!0dqf`D(TtgG!Skiw@6nC`!cBtUr0Gcf4Y;dxeE&}U?_?D zzP)n1F0fGj0O%+dE?Lf;a-9Ja7-oMueBbo;qj%`(vCjeG{kTngg*&v1^&r`XL>#(< z7^@1x7j`5G86B)b-uIOYW80#8D)+JL*13o`!3h{eMbv?8YD?T-Tt`^dg``Aid~~0< z!bf&Y&8YIrg#~29vZ-cqkpmI`k`%zxSCf`Vn6TJ9`XOtP(YX{yj*|yc99GU;4BTcl zb=o}mZ2{tnPDmTiXMxs>1s6+Qp1)|d%g;|ZuE5yLpnl^;Y^}(sHm60LC?lN(b!Ia7ZLYa!B^i+Jd}1MC zFnnFdQj>(A#s3%>%XxKJp{nnrOm-m8q2I90yy3A0^ZjGPo?# zgY(x60~F*0N`7x`9$x1_7%9{g5Z!Uo`<^_$#Q^Fy`6++M6K2s@45ldF*i&mUxoW;`&FlFJPqf>)^lYz8GrxGxO6kj;;@6}jR zI2Be#`a<>nc35Y|_%DbIW$IXRR}>18u9D1)$D@@QvU52iL660&cywEIRc2WP+@>IG z9iNr5j@;_@l*AfLy0{B!D+K)Rch}deeaq9xf?!iB>tXyHgpql_28m_ynb)B#enR9f zPu{NFNH}}4WY7V}O!1b=x%|3LU9*$&#S0dpC6v<&u}9p-PQ zT%GRz4m3^~!464AqUh7qVDQUY<_xe*salc!8WN}*9OYZTYAc5Ft}Lg2kTedk4*3ih%axz=WQ6;-lpn7o}k#q`;nsL00DiXnK+>g5yzV-GD1l}0HK66iIC8+xukS%^oJ`ihj6|6yxc6-<+u375 zRYz2Z==^n7ptzzLbXP*tnT1|cH+qo@D2&iEbOgPEdVxKfq$h%bA6X0QnGG>R4DlQ7 zk58Op!$w*&bt6$t^tn@;cyzUUVlpt}jp590J*$GhwlSuU9uAd@|aU z)jaQePqslCR6z@GCD+B{NOTR^qr#f{MefKl*a6#HX0yVjZA&pvap&AzSD3}?hwLKJ$=giRHQ z`;QwT)QO+C2oyOyRydK}O;BU4mKHAH)@Lwikj9}EDR|DiQj+3#D1_+=33t^ANjspnkJsc*UBd$6m0kgR$#+7v%YF88o!fF8C4-< zsLtaxhwE28Ga&kwpKIZxxnh?Tr75v0UX5tqDnTZ-?W8rMpCFXz(5APNO|qs?A~Ndy zg>!u`Qa_;dg|XE28=v;EPV&lJ-1E_ljnI!KZwAz%okKP%z>^K#!voWSNM5+)F=BS^ zHg48aOuxkpbQ^Sv97}?!I{M>-q*#voK2bV)AK2_+eBgUmAlfrQ>r7PHbxT~aRM z?`-E%2j8v9BC+_*o3p#qu}(?hWfh@=C=DmrW{`MW4fU=CapnWZCpVZqUP=71o?1l6 z)KS3`SkC5k!2M+y@$b97Rn>^(WGNP!&7GY-2IXj5FoBd@@ElaLlup5CM_;B}1X%7DFK`{X1+3{F0P^l|c6D*D zThuz{86F9VWON1(x83!gsoHslAl7Vj-~(V1rfmF-@6byLW#31{jiabH5vK7m>jxwG zE?;}qOa0V~<5MIjk_ss3!k%ffsYsPe#j?~1!H_AQl3q&Pp|AS|v_C=F2C_LgLwnC4 z(^sU%AWRS`d`lI!OPm=5HUD%Tc`4qKN%0LF6@^xMz5^Lkq|_vun5UogxJuCm z0lo=T&jB)_S>HAzJ)Z7ND6wQ%hCmYm3eQe1Vg-cTi=>02k@N+FKl-=oMT*Eh7V5&3 z2xXC1$uiErLYophBCE)$^;RDpyjzV?{EaObCsL$UCE~GIkn@?}QZLvQ#Zb@vUdrMcg_1d)ne_zjf+TU{3 z1})v#y6#ehMDu=)6&k$>bsJ?&mn3;kkD*eBp5XCK!ed_{GmYHjWhuT{h6wM&HxB4Pv18a%yt>i?u09%tbKnNq> ztrb8Qf~7En^Q;6o>lM%9gQi(b;+-F2z|3Az*m2>%Ou1%XAVr7EuiH6#mif3NHtG4! z6u=6EfG;dwMypg%RDPr>+z1b&tJF>>egNFXIMwW)*O|2OG`Xvx20`;yjEC}kz=P7w z)}BYJDml85utPz+f&*I``j}qz2LKQ5DpV6dRO;cH*_kprdRrg7TdXq)=DU`)X1#v)GLtZ^gOsj&_i zz7S=@#1Fc@$y$)CC^05;HkGU8SA`Sj`cnQ7i`32}xs52O z`4UDk7gkNo7AsI3n|LTD%ZfUeGnyfe$TBDd%oU+(jBLWXGYuXC#Umj*El2fgyCmqx)?s zVVuDGt)`=zPq@Hg@+M~e_#%47BnOF?cm(=YTWv19Yk{`pRF(w?#zjpsWH6Qgv^K8TgBqQo}%9XH`zlI2RY46<6jHBTud z`aHv_dL061W_OfYDkETS{22dAC^|I9K)r>CWZf8V?|pl8mihC~lCb;;Xy*Z1rqu5b zDkB-rm;~>nzwSB~9$4Wq&O}5+6dyWW;IpXJ`gReGw2duA%k_8rPk2u>B4^H)VVaOm z;)E>Dnzgl5Nheb4&q6ulbKZHdq`8@>cO|Ijt3`YAgxU;(OKJFv5K#rnJZmA+^JHyd zAU=&ansk>2W}!s6d&ADxgU!eENz!e0gPp-YI1AH<186l5chU-l61}YrI}NScm@#(5 z3{lxU?FKW5yxj5jN_-J5w}!y<7|pG?^6lzs#Y~~C+Q6vk!}}`i{<3V4U4{9+aJ}x7 zB}X7(Iq~oEYk0VCsLJ+0GL`jM@9(Tbj-5uK8vU65QAn)Pg~A(J?PH48)+Cv|B}ExqkN{WTjZOkd|iivx*A)%)U2}?*;bfxkE+nj z=Qu0Qrjr|L`u9()lmfgJJXiP2`; z(mq~JST)UID(RbpM{pvXnH#7yFpSm4@_ODaP`1Y_p|aFRk%9kYg^3>PgXsz+9~=|n zFQ)fn4)g4$GH-mH9MTEAR>6JU zoH~_j8XKm6A>$&BWmE)lTU(vAGwg6nJ~*bl&(>|=-KQ~y6Akf*EvfIGdGDI9bg#Xe z>|;AFXN$A_U|&k%LYr~Tx7HF`iDRo-w2R#%*}rq>=^MlI065guIO{=gj8<9ou(M%R z5Zv|Da3Fv=Kl<=i7yl;zptUO>r>?=f8F9%(KP$=M_J6R=PUA#;OO%KvpDY)vkzJo zgv4gXhzWL>!294ZAz_UbnoRZSB1-mdz{&vt4HvVdH+*S{ZQSYkJ{ns?q6#uP+~6~G z+?tJbPP!>RvB(Gv?4?G@*)ifg+PoBQkFLsf5`gF zsJ6bZ4HphhDDGY)5Q4iy3B?jTSkXd};!cCMcySG0Jh;2JXz@aEcQ4YSrKS81|7X1K z7-x+0eXZ=d_D*Kjo@?IMt=`^T^;A`-fEcEL7POwLrxYbuN6nN!GL!mvI(w$R9cm@) zD=*u?GQyf7as->KUpy*BtXZVk%Qk7{tm(L=N|z%i+zzvr#^0^a*06$SgjCFGI@)}m zT{^9{msuJyW=ypO zm;`~d#{?>;gpvbBViIx&@p@6{?%F2$Kv5Q$iLKW#Gx6Jf@-7L(8fJCRcnC0?!a`NB zf-V-b?-G@iO`FRwj78~jIbu-q8h*s$fLoz++af z)boh?ft;M*OgJKmi^~J#%7AQhfI7{{CV{ojm~^nX7pPOZqOMG_Ibf1LI$|I4Fsi;R z{+d=0*9j(bkxq-4(PNeMc|W&n9?yLQr^ZJ2ZlNz*>Py%bsw}I%K3=L8@mTmbtKY)O z`6D>g5JeJ`NUQc9%W=f3_l3f@E(eBu5tP-bcs@J{e5PD{&pJi7@Mq0MF8rdsf%pco z3pK$tfFugcij4+n!dQnDiyehGqb?2^%kF32Q%A(lJ5Gc}Z$GqAc+Ww)lz6gWdy~J7 zJUpLpMC%lzp?!&i_7F@RRUvb`Jv?{#nx5ZSPI0=`?`&{A_2m({;Y z#5D$XE%fcf+9x|{e{`hXmyF4a;R{T!zQ4-wX8B}&Z#C48NKW{LE44mX7;cyo;Q(*X zpxP%_S2%U}-U_z75mvM&e|KFKM>Wuq%p=m2^_>SMcwGBj{`{6QTnpSW%@z8lu;*2h zg#Q4+U)VcEF}e+0*_#Yj>LkTEAS1A?w}0uo=p)4cq~v8M1fA&+T&i@-(mZQY-qpS8}Ee~+J3*TO^|Cw%x|+5;P) z1~>HDVK~-`Mr-nBF8l8LTqw%9uzF-KbvA%sPIn`!h;>9!S%_bnw5Fm?GeHfe#f_Ji z^-LdXrWG5BOeIBX>&*c?GMBUtZ8>KqDjXTV!TFZ3{T9JFZX!nNmMyB!0?4VmIA26L zL|vWscWF=WlyPL5MCK@*Py$x-?@NMF6r4*kYT&(sKktXXSou{q8o8c-VO~L?*Dx?R zU|0ysh*VtIRjD%9nxT_>oij~MlyP9?P;!-kbCoXD&*)=AAG^jQW35;#1Loi|U?j(Z zH-1}2cl1^NIp;BZWRBqud%^;$qr4ZS4vqvoUFE-h%zx z@;r)c0x4QOw2D+zBT&LGf)>_(XPBaart88&n;5P!)^`j!GB)o487|9Qa}kb){{aM- z%@j6!dX0O;L?1nv+oXGyTcWQpy0pM?<9>zQP&G(M2#hhV-S021W|hjw6`)aeoB2cZyQGye#wf} za-wPX_|_17l@uQ__t_520VLZKD(5NZXxl|l3HEN+>e|j35WXs~$Eh|9-Qz#f?22qB znw_PQeDG|OlX`{`G$5WQ+*j{3CHK8lW_5E($7;P&H zZGoa{8x9&$OQ#&tqk;}SVIn>dlaO3#-@&V`RsYNj_g(&^Kt&Bd4Y$;|v(%2hZ?O`g ziuxrG9t5U>a0$t3=O=ehnrsZQVn3+sNlckVQq&r>I(L@W$oTyE)?sq$=cWoE4}-Fv z9K70crI0(c2BbEaf`7;!IA3`%>p!_JW&7Jql&YZLF14j}7{WF(zM}b~Vn#ogg%wVlB0+KK^H9@exoVM%1n3|_SXns>=*i;U zheirbP_P&U2CK|!K0&0KU+V**z;E&o;*`+uM+YBJgMOA zZ3O%y6~>HNtg-U?FZ}QvkE+1kCPwC3uI~c@lz~dhIQBN#^Is+~Ak5bW2}7v0cD|%H zLRn!9J95tOF$hvWJb5oyJ(`nP@L77=KxET|#DSs*Kc|zBizNm@`5^58wOHwjAR^(U zY-NWJMsFSEufkQLUM=mVjES4XwO1wDKrTpG2D1{!4f78)g~|JX$9(7kVt#F99**L( zF2+u13Lg`dtSiWKq~DE~7%199$&Gmn6AGA9V^jIwvQxL#jH1FPz6wZRunjUjG3;K3>oo?#igpzMK-s8(a>&r8s|WqEhmHn>~@{`H*UR zez;FZRbxsbC8)+oBRJckcJe;}n0h|KthI$Ro9du2>yw~A54{TaQrIuu5alMJzbZ28 z-A)eW%EgJNgyr=Dr6)J4k(l!uR$$_SuQ@>KZWscZOUvZZ{*`I+J`sNA!YlIJ=Q)1g z;X&0=F^RW}M}LF_v~KoVy1Lj^^6R6ccm~W&<_yfZFj#PdtL|6h8!|Gh?66N zmjo`i;l=t3aqrEZa=)}+G}sjKQRg=u)bqx>Oj1iT%u%VXW*tJaxn=OZg)YXf*!XC& z_Pm@GD}CN*2W8x-BDj$kJ~)nsbJUo-_S^`-Epg|)?^#nxd>zX4%8|z-0|wIKwQ$jw z?M}!@JKZ_U0CxC6L=bP8;w___oq1l3JLxp-*1N+j>(P`*=HyFM z!{EGuWcFWkM4|C{e)?idqcrNx0H9so>Ywex76vf&OB}J*Ck9+L+(*=rSum;wrh$q7 znm?84%#v70smjmBSCYsC1*Xk$#BBegOSl}@sa+601MRQG(!ikT^i8|)1ff_o$k5-G zXahg>;w5{?zj12PfNeNp4I5-lmhTNkRBuOoq1=+qoS4&8rEw2)P)o8)z0+nS{`$pn zYz9oux1tz1hbArG#2J{dC5(-keJ$6Iq+r(VlW7|j^YuJJRwbnUdk|>XJ?F5lKvL5>z|>D5DSG zD?ior@WBLU|Lp9`=vNea9)|r<;VF8R+EQ7*GJI&$>PF?FObj*%^rO~+4x)mb5Pz33 zd5%s&dD8LU*B+CXwMqE6siSlIjjoECT~Y``ph*9lB88B!nsLJ`4-Wm4<4Z?x0!)yy z-#TRkLDY{osaHfA>eLjd8hswYT>ag6`cSqLP_Uf@bI6f~!|KO@SV0|0KL?pC+yFiZ2_-rD&3Zyt5=gu9gYSDvSsv<|I73`)ux_ zYyS%)1f;L3Da>6E?WE!eeiMEKD+(e7rqu2ZMRrohM00g_SrMc2+^K;r-DiI#NQ3JY9 zd>CU(E;W{zKJ47tE!BWje#L4hee!Ar_fzPP7*y2mU<- zCfeC5WK8EV&x~%E7bAG|naCyzE+&xtg|(OPs?DxgzL`vXc3Mf@cvchiLr+aL6{AT2 zMv8u*Qus}`Rz6=-sFT*Oh5pzGQEwVliL(6rHc(1!jrf=)naJhGXHZ0+Bt=zuo41CD z0fceMN6>zJU?86Mm*;oya1s^kSS=pzp$R4X6F0Ot@VN2QR2m#YfZGkVrT)U{y()d< zCZcwLlW&pXUQn5Ry|p<`yyK5DwAlq9-uob@kx71sOM58v0truhM@Lu87N+)^+}ovd z%vNwuM%BV!uZ7Ys9Q+hl1IMtt+^^*OIB4YJaY|g8LCNBzs6#FtkjK)4fuA zf=z}Y^jbwKv!7aks#I;RNHJbgrI+|DWYyum(UKxkVP z{AOhSZ$+Fnt=n?P=+6uf@$OMpF&zURqFww9bAlOihaWc3pK}Pp1b$bxR&UPJ@N*Un z453D$3y(9$W;agJwDKBZ4pCW>nSJc=YHn5WwO0xYEnF#1uwDs|6XE?-*Y!fT35?hV zQoYY+`oEC^eB9D}D{A61M4*`IFvYil9SbuA=}>l^FufCR1cm2{P6aEc>I*RS4H0}; zBFM9`h#dTcIb{zfwA+#l5o+=}jt}RLT#~tQsQBUYkXuoy%D$`*qpytSCALq6hI{XM zk`@UbUyEIgE#IlKAn5{teGta)rw(&bzsnEx4ZJ)K{+PL8{?ZSZ|1zMq#?*{`>J$_0 zqoriqgh!co;Uw1GbDgh{b#VK+px4Uc%?j>@TS+Z%-k>(s4gQlFw6j}7jw!9XyyHW@ zc6&hX7unRmS`#nDwB6fN*b^q6S_#f?MTn;q{*B2dpCBJDgEsC=vugTrzqV+`n&nOB zr!+MrOPQ((lS{1tCoH(-Z68!uj_+er4|wDPOg++n9cRt@I>`LI-?@PT^gnxjHSYIA zc5Lw}A&|39CMGKcIqVx<4&2f%1(*5yLSW%fvNG1?f6eP&=}e=?LUhy;{Y*cCygnN2 zr-%APVX#lWN&^B^K-tb2CFxh@JicxdWh6vYf6G>OKTsjP(fV^Brp%vtLkuhe< zponmh2!`0E_SvS6VaA@pbdP5;n4eI9JYam~gkWCOdw8&ZLVAkfHRnAYt%DYpcD;F4iM0lD0kHhUPG zLPsHB_zKUKv>;DRmhHWbWFZ15qryKt;iDy*&{~Z%Q^_p(p=GJOpW>U1%Wzju0{fpf z!g}iYww&^vUz}Q#SF^Yqtr+kUSRCM9Uns&aSPZiyj(7O=jTO&k`(HXs&I+4yu*a2; zh8*IK5G%17lmIzLVgln&>#;i&I2|RNk(XtL7yMM-YKxlNnIL`@QpRA>For@i<`CVc z_f$xDiW;7c=TyTXW+_!nog__7p(RMR}E(8c?BrW(tzIk?>Dl>6NNxE(`HRNmnr zM(i~)X`QHO$e~Imj@PlWB~$m*tFLzw;bLsJwI|bHQf)QQp{3{__Lbq~N1rU6Pk2Y? z3bD7N?%KPi_0Sh{+?>afXKP0#Fg`&i*!RX}I|)&&O0C+X)i4vFcV@8zRLtNlamuUf zC$doPOL8-gwVI-l*bv9p$N=7#VH_3DWcEjNE&)Qtk(5>~W+}PchgI>su|hT6OJ+4n zrcsHI)rR2(AEh3eIgkRcJ)XUjK2uyJJ+Vx#o-akndB6^<3h`M%OD6R*hV>AJ2#aIY z?z|#qFx>xxVK7TZ&6H5UO#O6R$d@uN@10A9UZ25-)r##o6D;qLjstLEU_*xmR={+l zbw-ubYWSq6NQF?dwBKvNsv^imf@;z48szs+g45s1-Vvi#?pw^4|IyW$aT8$q6-zpc zit|$y5aj1q#0~?dwb?a0s-tGh%=KO%83o;2_zR3&zs?a!Y8V+hpQzC*TL_2-=L;u` zbSN~J6mL)sQ{(u11D+rYFSp5wJ-m)xse-5OTM{?d(zE?~8GE#j6m9P7w-W<;@Xdt% zCM)fm67ll0JD9UFIk|SgDwU@RVFBE#>0Bz(ZGTN>Vz*g}#hd=_j?W8dxme%jop`l7 z0#dodqST(NH)Dqo3`Y{qL=14ecX)sK`Qu0N4rVXoKa3xEIf;L-NU1k|l{Dm*5P?6( zmz?AVn@&w~2eEey!gaLK+Oz`;BS~U7^nG?HMvz8_k8aOEyJm0DE2%u&D1`+*P*g8T zfLD}GRY6}UcCJ3z)aBNivtiWmNG#>hOE(F;XwA`0yr_zOZhJ*@U{4uoYFC2c&1|SGss#m2WIq4vzYNqiy@CS|_ z%j=ZzHRJGbPz~t)=^MY|o^RA-C6t`ji2C5dCPIhE+kot1O)8~O_S-ox72XW0YO-)r z+&);I`<0h-0;ny-yt(qM-V#tps<*b7Fv^`gv2CH4fvbXG`nON7@ zV5X9gT*}|GGes~<6We2DUtD>4ik-*1w&FTrjsC!*-}6T&M$vTw8+c`3ZAP-Q97Yv? zQ?d8%U_%*hB)pBm$|i)-aDc4|z)IdhV8eFq)T-4oaBb?bA*Nj7f-L1lK!1&bf{T!m zlNRh%7HhUzg_aUl_0(=sI-TNb@S4!L7LwEt!A_U18$ZOT!CXI%;WT@Mwqi5WiPAsc z?iDUkhU>_DWcufF{fKhL@Kw4PoPxJPaV?dav|q5e<$bDOqTc_Q+Hy#tyLc6O%<$*O zY+A6~M(H1~HVagMOpptPBKZ8eZ&59ZW!O|Odbmtx@~~_)*a((7B$RMqgZ71kMwz}v ziw*BR-kojQ7Z()_NC;bF4oYPH?#i%a1B_BVq?__dZ+&RlFA?|S!{OY!#wFHiDn53T9-aM@(L z{Fg$MQ01^{`pbq1wKeh2v(ud1Z*>Rp8NbP+REmn8cB!$JPmM|^KEY{kto=56i4~MF zKSw(GD>n*?Ui;m(XZPyjc0KE9B-g-V_}wh3jYvs0(58)Yzp1d$TrJ93@1IG=wl<{V z4x)X*SuH+^NGY<(a)({{G>*=x)qcf{sGg8k(JV+JzFHrjxTd9Dv^%wUroK%4Nbn7K zVLRQzk}9Sac5>k&_?9_WTe&JU`LcL(d`%|1lewjbiiMNg|4}m{xIDV_PuHW#!6k_n z(Qg{C-H2WJg+gfvIfaGU{#H!VBU_~03svd z;srqy7b#pWyS%+TQAs_f!g{qP5xi%t_yH2cqEE}9`KDoz`cmbl5mi}KYFGA1-{Fr_ zq6n!lcz&LFAzO#X^ZO6uWyTggDz3SbeVAKK_P5W=XwR|p#8D2G-ya4Md+{$RFiCot zG)=aYr*Zm@D9ZA^2uKlPG1Vg{CkJ+QuhYSLS5l_SenvkL^=#QA+~~z|c*m{h$KE-G zUpUKn=i%F9^AX)**6XubIcjRd`~06SnOg}-tJ2TO5%o{N9V)7Fb``Z{$I?~IlaJU8$&1(?8pyX~q5dwMU?!O(=#Zp0ObqD%h% zIdsL&*4G4m>hsf}BMP~o>#IZMAdpEZ24QgSoC58}%e5;7k~ z@-v+RK>ek!Vs2v2kn}CzwgqsKU=?g{eQ%p_1;!fY&W#R*T5HGP)Q*mFn|TFSFpQV} zKQnLZpJ6rK$xK_%W`*P6W2-}quIGt@M?ZPr1m5*zPm&nTFjJ*~iDm9BV%)Bnn@)~f zf^8U`MyCTe{unTHTB}x+M}_*k2~e^V`nzKk@DquQ2@oR4$B=ldEb; zQjXJ<$yD;g&c2rNGG(8X*2TpgZp zAEm}Esml}tb5t>Tc?ARdWs9Mp zFZyYU&)u8C$Alq1+$_!WEQO$ptb?sZAxUICgmGPFqv4%++;ub*-ht(-wP^4LPv}4- zEU)X0ojeWcilMODUezo@+(5@5LP5G*NNdywsyGkEk7WC@WEYfBaAJr`;E#9B;&|6W zR;Yw2ab^wo=FtM3ubjq)Sm+5}nQ@VG3-U7c|3G6fGU;o6QKR+l@iI(3PYKdejooD{ z#Y&GPU*AgJt0UOjjrfKR!A@mz1DTQvF)aTFh$I8gFg&JLLxR38al=^c1YMzkFm){@?TV^Xm)dm&($`%8fA9X8#@d?WS8 zLK{tjG%0x-Mr|HgW77^ex3a<{lGj5R%H@`K0S-{bQ z=pZJKnJM)wc=^S@iJecMTnkAq$5^mFcl%`}qmJl%;qGN}LE$nnI?TC}9XrD2DuaXj zkYFU8#|%l!6q!)S52l~yy`{o_!*w*$%ef=3Gt}_PyxtR8N_owg#}O^XAI=@>QH{JO zaz9zULQ)eAmB^^kM&)%%=>t5rf82`kCfcu!IBC`Zc;l6dzp3@8@1YB4qS%KWdA?H5OP z&G12(S+Rchg0dOIdQE0$GmpUA{^ls|R@d@~e)LJyAbvUD3z3zcLjUD9&n+&=5+Gbp z6Actw7Z_I0N>lI5wlMWn_i0kqJMdGc!7t7;*6Z6=97S%e73T{dE{H6uP`=gptG^ePWJ8RR zYoGo@4#qZC&GIY1z+SiId5gnvqT#xN3|%l?d>rl80y|UiC6prtQ=6hWCCF;0IVxUc z)|RPlG$BwsKv6UcSAipzKT}0S*3tPo z&XuJ&;_3jhWpLFWnF(D;LY+j-62@n1*Wnl}hxqAUZl5hBZIYbY%KQDddUlx>waZ>zn+?qrt)Fb&v*fr z7$vRUumasdRav9Yroz>BT~G8)oLQMR*Y&kK!{XQ(+$_>OHyh~^r8nU6028dYbnMme z&cdbpD*47GjUaYnm@h^igZRS==1=_U@#;_P+Qx}`n|D_2{{%-1-l!rgf(ws~o(Hm5p1`NR2H8zBrUaQo^;q-Y`r;?*I>^z|9TY~~e349k|#szo=} z^N+fhnfqQD`&s(AOXFlOwDF3PMz|Rq`xUThaVmqU;9g0z1jxP<%JVnY?I~5ZD#}c( zFj6c7<2c!Z>!sa_=kG`u<+YXB=2plbM5?SCT_Q>;%SF}>YnRJ1Du%_UZjLNsR2nL} z6B>?=C@A$Wc4Nhwr(wmLuGlddQ?Rq3N=AgWxkM#5KuH4k0S!m74Kp~XLuWZ{zvZg%)7=oX@oktKBFOV(*Co1$p2`zl5JXpxYt{H_dq;}4 zj~so8JGWFE7-EcyG6(E+y?`Mi|*snO4g?%b=Z3%J`g&NO>M%E_-aI1F z`|kixJq;nH;q#-0>3k-ya}aSCx!*{i8D*xzHpXykwt^NWLdvumnH^Dj9Wlj}7iaa8 zGtdMu^P#Q@jFIDuN@@4tY|UC`MfR@dYxU_h8*(Fr^N)1Y)j|~mf zq_R!ysU7obuu3QrbbPPU6pToc_G!gMM~je3ysSm@&W_1xe6$Iayz2rIT!B7x9q|I#poYIL z@Eo!UuI75glS1THGg~;69B@F+eFdCshUM8;0WT|C*h$@oy*SF}72V5aFT7|M-bj~> z1cQuphU4;%z2=YTmcA>P7(w!WJF09$!g^YT)2rN7MdU7&1XoDYm?CyESlQfVOfXxi zm^1YA4rS@cXI)w093sK{t{YX(fcxYsRnvrcH6b~vEL25wwGepH6R+#a9t#3N_&5qX zs??>@^D({D$m$j5;=&wGL!^2;2~!*}RI!JBIQjGO)SHg~TX*-0^MS7CRCHyhHFkL@ z1dq6FeqV;+lCCiGSD?}{!uvSPLRhsTX4vV216ev8rz_kMo2w64*Tq`5FWeDaypvkQ z0x#{B&X1OuT+At>U2b0qn?$L&)NfR)timv$D<*UAWwk-dxog~(7WN{y?29x>6 zisxSa{nvWivXpFYxW)Jhj*;l`0%a1biW6s~CN#8_$}j9}Bu-C>~dyNh~KPPxuC zC1z*A^!Mwa=Tc+Gx$A1x`?Rf>N9PP%2H`cgJsGd=iwAWKKeT@ah=LtH0FXotX z)7sKBCwfpthcOt_B*i64W6)qu2|c_%CjgYz>+^}2wF!Mu0zr~{TFe?bOO@dOacppi zPVv;G*t-ZG`~#fW?*fH%A^J?ib0=sf$&guRk#9Bg7jGcBTV7q5Qp!AQJIA>n%kU(} zkUnv8aqk)9sOD5FR;-ms!^6V0eqOjn5*c5U9W91@q^%~{GG9AJuMUpKOt)i2 z51!~`NG>aWL=9KO9jUOIq#7~umY75?dz|~GX8>lql4DIW_Ic}vR{e6lSw9MPd9q=I z&hnkK+8JlK7&!?MGL0^r`j#gl+cSjk!|4FYcR`s=`!}W<>e)}c6MI9AmWUE7II~^^ z-t;byWRW&1NeVJI)`hSCCb?aWVh$^X*F+6VqqoH4JaF?ZY`4A5wTPoCKO0o-FR4ny z%fZS<#%M3|%Z}86UST=d*t7p!k zzk7E2QdHP=A4Z#hBYy{oCIrBj2p$*&>xyfs1>%sJ1fz=byhZD^@h zOj8_57;FplX9Xin7!!p@iMXAz+y&rDM3I~D|F)6;rl_GvgV z+(~0c-T5uMg!hrBXA(hPVH0uhVg{@^2;=`K7SHiYXlg7x2`!4YWT2lYp|`$xGgws3 zZ2D|5L}8=Q;=mwGsOmLsBnGqP%alYV|2FB$Wkr=rNt&k36zL=dVvk@|FvI{jNRJl;T^%dH9TVoChQlKdFkBQ^CbV_j&nNKriCg>Q zo2vZs^=P7?ENzw?`v=>G73)_4zLco=!JhGISg%#=TJIIhRZBC0aVxbURru_YEkrYH z2Pjkd?n!|~#SI+BoO8DIlDa9~Fvs0S4HopRp3 zwc78J=O=fw2$g-Q0QS@i{*gK;AHyuC`P$}9aVXak()aX;z*nt982gO&GtG_oOBEgS zmNzR=?PAqzh4co}1sWRGs+(ohs>4R9#R$ie=1CXhCsG_o7*8~)Ca_zYBpZYU3gndD zPT0y9FRGOpxFeqZCzLXUW7bHK-i<6_SvB5?F8n(hkEYrCSGa6^K4Fn_ONk+G8byCr zmGzn(FKkSIO(Q%u2!7Mrf)T>xD?4LwtndP8^)WZNLGfb=v<>M+c4hUWxI?Q~RWl(b zSD5BWMWb!%5Jwz=^#m^+oQ?0%6Ls`q`&@@^UTtQlY1WGIGiwB>g+--%^Paoo7S$8~ z`!!=0O6Azx|N>JAh0H*!7dRa;HuiOPu^SEM*3CKyTd4p*hIf#{ZN1g;v!hSq9^ z@)c{bnVYh?W$CZ2s=Oc_&#roxaYn>5h@qr%xmR3{A)+S~T-V4DYBNF6bclCw!ege* zJ_&;jBz=B4E;>EQ@mvGb^G;cgijfg+d+GT*=% zf9eSc&XrsQ1B&VZc%F%7B;hjHjv53mFO=r@b=AtqVgcOx+$?%Ep-m#=iC&#GYl++q z8#n=B@j41MwAkS|0zSBxx7&E%+4D;n2n@3$c+%xGxGI*G7!Kr<_loPeso*fX@UOfM z!!x&@qPS^}CTl(N;GxY1@To}ID&nzC!>i-PoB zIhc}~9HKj?HY6}O|8aB9XM}-8U24rZ=TurS6L2QiNJ`qpBQd)m?`;4ypddqe%z!}H zSHzwgOAUGTm5nI`uKW^>rso(TtdI<@@GX*Pz#9>8bsilyE74UimJIe&a{St!a)__$(&@1 zbzKJ)&pz3c3@n6z+DAL#^+W)B7!-km)}_~6mJAqF0zf`gv=>3 z(=9j{l2r!>hal7y%x(_HrMlX3$l1x{&`GQXdmn|In(TV73sR~4w$M9?!LN*tNE9=3 zs_bWIuB6T!w+mKKImtU*eIFy;B$fLgpkMW2=%-(rBQR#??}zV7ao?Q&1B55u=goac z!J2pd!_O$CUWIiHS7}`JFz|sqi4#~q7b{o_epfQ=@WCIvakFXj<6)4b4&E!qr9oS0 z@wPXk>?l7u&XYD*!!_#rb#41-B|KLISPjoPc;MxT0(?2;$fL_~asjgVAm>5V{M)u; zkomk9sD|J#RX*g=KtHSxA^GtjJo4MacxCYd=;M+pC_QWG!S< zT|7$-P=YU$oWRlY-!|k&U=zB@iNBy>Tb)aZGRapdurNZIhY}!CA&C3P@Gp_TWiS-F zqP{1o4YnpL7z4@Nz5CSP*3Tl^$<-esEN>NQG9=qn}m@bYA zrO#uCx_IIS@YG1^+UmWB_F9sXfu+dDP#p-EpWvG;?Qtm97%@-4ioU*B^#ipKp94_)-F?M3lD} zo*JCs*o|el1z0_^X|;dz=|fdvCgC4t-N`F&R?xk7aT$+9q$WJv!?z$LGFueu@l-8T zTV^_~e3a1P)mmPgwTI2Ink63Mp%ZpOdE@GVnJP&!`OjE0q*T8N?Cvqh8OF?8k9TEJ zP-Ic3*~#M5z;gd-XdSK>mm7ykZnHC00@6{Hv7f1sAoQH7h_E0hdnsUSS;TGRakQai}nTL&s_J z)SIYXTb?nPVi~wDQ)MOGf{tbhN5m>SJv(by7(JZv*)n`_^wKKK)22k~!ZXPg5^^j` zD};O5vZ&Q|uUynwxfq$(r6a_pfpDeGeY2Bo-+9CkN~a#8fKqUq?LJmsd5-p1)SKCoWze+patrao0=WS2=xM>U~@< zVpMpw{Pa?kpPJ#0Ym1a~i2NBKRkIR=eMM~$giWGf=_@TONxl(3UC8~E)P?(}S>n}#$C?743 z6-q6#DOJmsRF)E1P3yD;TMHj5q&QAXu|ECT47fXqDItFavbnBuF!L=bp+|Lh@6JN&)vZ3+s`xSqm(oWX##L3esR3=W8nkFU2a;&GsVi{f@I9(LP)SeMc zTM>DV1FE7WvD6Yi!Oq6DIi{^AQ2Yk8fg$J&i;{s%uA(zFAyplmC36fiQ#H%t{Rxyd zCN5F*_71IXgEEx;|2(|mGeS?hQ=TOzC?X5IVDb~by`{Aac8R%&5SJ;;a!>uF3Axmgmo9hMTS!^Tf@6Lb|-e^QmtX@^|WuO;S7a?VI%Rra!U(++ERIupvJ@wSj9K5eM zDADA7vbQ|;0{;iNkw=3=nce2*6S0v-GGE)s*uYV_LmbnB!E@2G(@! ziNz1Pp8SVe>yw-FPio@zuC~8J{{gb9^q#yL z{{t{@+bjN-82(qi^mHw+7l_fF&A*ImA5J?GE3M8v3q42Rs3hg(nWC(Se>3zDlgEXp zE&M}l*@MNxK7so?{~jHC^EbZ`^9irTeIJd4Ze07gaW$()|Myw15yQ)l+%zo^__bw) zvhH5j1$A(22Fr8u)1(*YJP#A@=#&rdg6u`v(H#1*bTymzW+TrZIVZ9i=nDS>oY=iD zxynA*zH9q1`1(Hk(5b?q!-fN{#&&wFt8xE$OsId{yt$s%S}@(&U!tx;E#kC&Z#FeA zE4`@aBpT9mnU#9dJWVwI@$u~97tNzx$NwT3LltAlr=ZF&&F`Xzk|U%o=wGgcam-%ax&8-;`f?@Z@$uh3 zBX|3!H=F1eYj*#8R4?>T#6~`ScFq4|HZ|Azu5*mv1c*psJ0maQwEEy3U-RT`{$gYO z_xXpcxx%Mi^uMl%LZ4jv5~UAV67y*j|MZHwHM!Lqvg;l*L~gWt(o483IN-YU)(BMH zvKNtF-_>Y`@Bczc;@ROfe38|LZ~UT5=6rci4&Nd+8}RVi^~0^%@Vpd6ycVogf&(5a zXyzk;7tqc$hjKytu0r=mYR13p2s})>kO~}yxktAo?0^R|pywi$2EDjh6F$t!}-RtnYM%icnP z8h^j9oo8G{d%SyFaQMQ=)`dw*pW)EVtw{z?9R8So@x^u$q>cIbVfsJx0yN?SECOama$Nl-?7 z!M!i7O7iQ?BDpyM+9giEM;Ub!AEfgWemiBC`j!9d4X&i0Bq4oxE8=*L__r!XPh9|{ zjV}B_lF;p??Kg#>SD=g)!!FDJBjh_9eLV7WqqCChN!3`wuzh)*MKS>Y8L+*kOm1~n zlvEyl2nt-Q>cl009O7g=F(+vPFyN^Xl2R5d)`lq%RU=xU;o|{CtbBwEaf~K6Q!tJo z8NVvL)!N#+OK~G&RTsb^=37GbCMaIz63Wt>p50lk`JpO$KqE@fFj$6viIlSp2^Z!M z8JK%H0%VmI)Td~F^AjDS;F|*KU3kKkUMQ)9i7qWs!V47IzLf-)|9JA<9LJ!J z258ea`*5YkCeh6#AQj;8<_Uxx>%n=bs4G7+oxlwRZ7OfV^dXL<6bwshA!h4S7cN-! zU3Fd<RWO=YcPg%)0bCUZDgi#Ryc zCX08(L@zGy(>bf|FZRUA3KZBHKs4Gxw$C8h$jMiduutAgsl{oW@rqo}OSeDg$F)e6 zNZIYw>*--){|fZo?*OLr_q8kTj2ZhImvgG%Iu&DU z*u0J~+(%O~u>(;@EHOvxOQxKs;?LWMx>6o}GLLt%E6?@*RQ(6IcuJuePwC-vG+meH zi=T4;8)W@Z7kRd^@L5LLNVB?SXhRVmbB3c2d6VKGb33p=IxH8lGlC0C?^#wD`pNP~ z=5c><^n>eDiAFo+KL8-WSa00>37Hf*@v7of1i-Otx4_L#a8d3wRsZ1HjR)o?!zkb%m2+- zfD`rcY@qRg9Sv(m*hyo8Sjoj>NRl1A~2ESIB6lrA66*HH< zI2(V;%9pRVNStQ8FL)Jx$Oue-dEC5zS|wSb?7&>V%m>j#l?4Nt|J`Kv+j1*TrWgPJ zz-pvj{%@IO?*s}Ja_+zX2dH@B6{4IUP)Xh-(uAt1 zG95IG7;p@O2j-mR7iaew=XOQJ`YNv^wG6$*DCjj1Tny!uS|LcJ%{MiUKvLzjVucQ3 zV6X(8j)_!_%$CUujn#IBIy?|@63uiCGl6kKp_RB1xW@x7h(e43TDbusc)8hYX-FhG Ig484b*?z<63jhEB literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000027_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000027_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..7bf8ae6847441126d0b1e3356960b69f911dd8f8 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8lsnYM2RF4NG-8WiY{v bfzW5qoY@OvjnbpxK<#t@N|L3CKtc}y;vCKB literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000028.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000028.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0e91d67ba936e5d63861174e2d3455ebcbd4feaf GIT binary patch literal 30879 zcmd41byQs4_AOYr6Wk#|e9^45|VL=MF1Pv0baF^f~EO_!rzTds? z-q+p#_2|)uaq7sfz4qK|%{AAlefYijdlP`AD61d~fPn=7U|#+JzkdNq|q?mX;(U9>E2TMQSKFzFS%`?TnOm zqjXpkoqM>Y4o@G%!)gE<01FF%LqI@=`!m{~!C+vqaj4+lYH;Ghn@_ub2tlC!QKZ#? zr^&VTMe@SU;`e6&HZn{AY%m-Q^~=ZrIBdZ0WdJ(dUl`Z`aljS5mObX1ZAzoFn6V6B zFa@0&El*?bhtoqLxw%mV1sNG;PiHY=;HvL% zjMkVpL}PpQa_tnaa{~zS05zPz^iX=y$DufwmIubDUADwVR%-yubffMtoCw;Ktf4-F zL$9m$!JjZhk*ekuj2(AO-P5k=FuaSGAxF<_Pl@7#;N8yUHy2F4PCsLBS-qmvYN0*p zy8LSb#&Edk2H%8Bj=Oy{cV|59t2IGvY^=bY;Rum}sxI!Cj` zpuWy|LN;5rp6Qy z`cSzcH&zy?i{#S1&#l6-o@GHra|plS1j6@{!J) zq>U&IimEKVlGcj?W+Re$qrvfqkxrrimp}A;=F6n*WQ!9}moo$>s^)okYwgJJ)`9GZ zC}iQz>92Z8#C7DG6W+0kG7vPk0XdZyDX{|nYjofPZ)bUgq6gy?g>Vj>bSE0{{XPW) zg*C0#v{-rm+>@iQlvE;;4n~#BrIqvAHyE_PK45n39ev-_-e`K}m&Vwn4w7NO`l6%256lQ~W<8 zaaL_>3##^M&POjPfszU2Dq#mw(wz}WG^+guFi&ycq+@aj_eicy{e1w? zf*XSN@ef1)omb3_LPA17l5-?tYe#&NZ}ugmu5F8-7jr(X0TqFuX8D?7bsiPjj#@&& zFJ~}|*=mH57N;rdWSRe?I)7$_zbM{zzfk|a zCeW&Nul!-EHdLlD;dnRn+T&{*70mG6e=uCja~!Hww`VtK#KLBuAUR{SmqKd}3#%$2 z_@6%G3x_NC4`tD^{r`J&{wdx!RSpNvcjz`Ns9z<}K0>|cfIMLoU-Xb8I|*<}9jq0e zn`$1{#GAF|N`6i>oqPP6Q`7CnJ!rN`gXCVG(`Oofbv!)UYA(xbvl^ z`po=BQETXiO-;lwa3J4#FJHDQ46IqaCdL8Ww_x)i`|+cNq0BW7{M=mgxJ>baqxI+f z{~&58J6Wa0?Nwcz^`{%b@r_W)i&$PkKZ#HPEX6-C5l6SnOvMov-G408Pyx=h!RsVdAz$Wbg+3*2MYMvkW54Zovc@?7TyH(+dEuyb$ct^YyFV1G zQX5!!aNe2idhtJs!@031|Ey2#pvISp45+(OaIje`vakdy#2;!bMQrWnOJ6CkPAr^i_;M8G@yGQk-v*B${&ESn^2l_XPN!1VHyz3@b;?BLUU)5*lX6Ep1JK=JWF@p~V zD3?CRr(SpV>qu*$gDK$hj*M6Zl%CIh%WWL`j>K(_e?6+4YTww3r}3^F+jYN$Mv$bp zBN6jLXXj8MDW_{I7B<5xaMV|hDzF|z?kWUSrn3~LYj%^n*~ zH}k)|a$j=9;8EQ4cG>x5%BrfL4t^Q$F^3+@dpG2r;NpbZIoH@KH>wERhJ2k(&_xOz3&a)$d*gN!z>Id1(e0$v~ zCgy9royRdfjT@t*ft;Ej8>|ilzLe%$uMfjN{n43!QtiKz;NCmYEA#Xgdt+UwDzWBP zCsWt9LNB5OR~o>wV%>oXtlkKD(3`erRyy>N76ZTMOW6I`ip}tlXRh}T+5e>XUlsaS zdXIIvLwcRt(k8CvxhkTWE_@uzydaQytB zeb1}R*^Su$RwqG$Rh<;_M>YN)p z`Z<;9w$O_s`}8Xlnsc#{#yaMWYW*A5KgswnTsUVk`BvdjlwuO-ya^H^Ty!I=-YXcqpH_Jv2&|MukH{Qn~}fgA2w z{;Rj`6I}Z^`(&kgm--z^FQU`^M|4X56kByp7`H-{x4(yXl>Ms)c!3SFCzNp zZ`pX^)##tH(Hr+~EtdOJd|n8=`%m#{F7*9VeAEX2x16W_A2|O<`@whJ|DN%`ajL(F z?B8(y!3lXX`L~?^Ck{~WA39dNd=+}(`RiX*V_H3&9a>0+6;26Hg^Vsa%Y-h875vW? z46-EPdMT2hOvSmhR=-f863;=u`=#3Gi=@uip*YbMy``Eb0ttA4<@CLIJSsgY zK*HE22?A}cHjI#=sc`nU3$YqFSbqic#&!}U};tc!F8k$6@Z__)iXF_ z#wqkr$1-_yKhp@S{^V)f3R&6HNExb3LO!K|9(-7MFDS^ag`Z#_qjH7+Sj z@QcQ^Q1dEZx30V%HOaSH;}R~r4!>r%jrpKowq!KZJR3ijaX+MM_R3fXId_YAV^Z zwv28U~%t3stW@Sbff^lh9$!T<8IHibG@7Lb|o>?TTqx#bs5)v9VNgQ6ug(U1! zDF?1luY_LO_6-(Cr^`8Fg31#EAPcE2O_1)~XqFRAl;{$GZzE_rkXg5^RbKf_$77-4z2_#TmQpuu;cU`0y%?9CZagGlRGX z?8uG5*mbO2c$0I&$=63l?J6P(+v}3i{Ax30+s=szBg^LLtFFkxtoxcPs&SZ<7W|XT zHRX3U=CVUUiql=CR~B-#J#1ONu#&rbHcy za4LtdRLBzl!)C=hpFvV`DfbC#%YuvA{QTzyMK-~$VIT&nfM|E&E%hvD{hDT;N((L zsdyrCCN?L+Nf@A*-|XfC>-Mal;s5`nvk{f%;&3MFcR6foZlyoC~;W z{NUZnTsa`=KLp;z8)4(8El6*Uv&%F@mz;}}Mu=0d{KKXB_Hiy0&0g)(Qv@l=M8=FU z!NpdXr2`$BtkEE^qx|B(+iw&)4tv;rTWQg#l&zUmZP@};4`H}9YVEeTo$Nc( zzq?x^10x#$DEJ25x!8MJx1FXOzCSjqcK$IBgv- z_2~(1BtkSxhEnjSGM&9M2RD^pY3DB2N5)%k>R5frmi>`6Rjn=5o$^z?!Hg(Fn-N0G;hlqCltVEA}T*jld6!i|b7IS1pXg6UI`RvS~|b1eVW$QoUFIUSAagn=UvJP^G= zkWHXUk|Z8|Qx{NfPgGaA*F5h%yqUr&JnA;z1(|qX+Ht@v`yVY$(8M~PUmt@0KQd8bMfQmFB2 z;#J>oOtX2?${Gq-@K`orh&f9N zf~)bkVOZMsGpX)^@|A4jvV`OG}i_pbv~q=E@O zR}c9o3l&pj!HGQCd|fT_y|VV-kNu@gyo!K9LE=xMN)@rc4DK>!kB^TwVw?45V>m@9 zPvF8mm~F3jwEGlX`VJb%T0+zOhoYTQNc#uY!VVJ8e;EsfadL}>1b~l3<=YLN?pm`C zG!zLkgFjT2KX{tB{0M`~R=a&qMiMCsHB&W2z60}0RqKVaCW)o}f&}J7@-0s6rd`(V zs?laXvrL+VOEwaotHo=e9!O|dxLSNH9`yOkwLc|uP=X3siS9)fbick3h2`Z$31N_- zV^d9DC0W)2t64D!nTRG09RGtnSK_o#C^~|qr*E;`R?ACL7?0m>9plqsGY`eqhc|4l zu>9vI_LFaS_o*BTnK|zfMMkO;%30t&LUECG zWLx4`7B*L*%UHJ-Vh7{}*4nJ(?|Ozu-~-v!mf>NWMILN5lq85`fKK9_H|> z2#Sy`<|-w#nvo3v5!D32RG@*EBOajnZ@yp=HRzTM@}(S*hF`6v&+xu?Us0_Tpf8Mi z8dAX7F(VCVO_6VCygWMYmoX|BzEoLA36xDUB$&6rh@n)eYtMN;SojTK$0eEX&(}Eb zQ-wpQhZ!((V*k*l!227(F^zBZpyFxqerUFdab?}8GnV$W~=VII_Gs@>cpMa$o~AWHlF1KogiVZYzyk-zFC zG^wUDRk*+0IU#?j=}HEavL;L_uWS)S`WvuIo~M*D3p&;lsM7Y73(`m{*vnp5=#o|Z zl4$=f;CT1w81(W}PH=gxX)NR^iGvKRJzLnr6^i>#Zn90~rsatXVQ-=cW1?%c44(iN}h8{Wb8H zNn7-IjUx++!l)9OmaK=^aqRN8FP8liBRfUB%xU*tX~quC7N4N=g*wAe7&WRLZ8j1q z2}BI};-$-+S!?#W;*h%w^VP_ZmGtk7YuVQ)Z*zsJS+s+tj6d&Cb-B=*vQ^g%ENBAm zsr^!UBKW>7ueGF+iTJ;=?h8Mm3cR}yQ^Kup#KI^NM56*M<0eOIe_h5e!U`^>0uo5V z5lBL)@e$0enEPo7-JQo_E7<`Dw(2n^~=Jjuxg@}5X`ZfQLKyWBdnvPu$n-khHf81=rT(cDGgA8xL`0YN@1I{Go`wnwl)3NOovZb8O;cIDnd9vM*`QUSoVX@_ z^qUR3LL}6-BuyB%d!NVAW)a!nTZ!e1URt9Bo=#MC?Oq77*mTI&EwP2Pt++JsAFO?& z67H66LEain!ni=(lU}~)e}72b(~yp~CR2)JQY(&qc7f79ia92!!an4}vxS7;>RZA4 z!!OBBRkM~~!PR$Pty%8GkEX2aaAh5ziI6N1kux$oc@G^9cLZ^P_+2`WC(urGJJ(K= zC~v_cPXd>n#>?=mYrDLrq0Z$Nyof~V3PCb#D&wr4UvMhORk~7*52OLN(F}7x8dmcf<@p_Q(;XZYK!1?eiea+_E@z<)&MOhy+ z@;dRFbUzf(3P%O{6GYc@Q7ZOEP9evs8N9W(%G<`;+slO24MSs2Q3lMoFXPT7z!}UT z;npH~2E*|JdeM42s+JQz3qDurLUUWQ%D<`Tw$E&yNpc-UV9%`|cg1CmvaaOB?pxG< zI81SDiwjd}jo>K7BONycs}u4z)^a*4&#KR?2!x&vXO7r|b?jX`%HOQbl(Fvmv5@Ow z1PGUGRxw#rN?J7$U7*!Z{@5#_AdOO-f8F5g%l<0+I-DB!N3`jufdD_Eh~&b)M*K$g zahC8WmIzmZNOLQOEi2||^#PXfYij%fMdFC$f{5fgDk%XClsf~I=rnHR@Dpl+8uOQ~ z1%27PcKq19jPZa>r6@F6C(jeaOU-d8A2wRfx!SmpF#?}ZOj!+^E4*gik`pvsd^6Rf zH}|?Db{3p33JRoIQNw2Gn>jYIfEJ8>65q95Cj(vHKP7D0IHTcCMVrKNaW#lW@M@L` zQG-O&hWXq0_cXs+ECNTJvxQ=Y>le*g9dOEE=F#7qv+<_pLH5 zy#ZmPe3hn)B@JhwvZ9M+M57W!4sVch3IF~E(u{W`0!2a_2!&=HHH^733F>&IL=~YT z*0mrKPWvkz3YmPeeok=@{^;v`P}Lo}PLqiG z4%&C%!ncf?wf=CnZO3y&M6M2t)I}^@@ZhdeNBYHWFNhO#I5*3$c;vRc4 z@bLK0F#-87ek2cC=(-<_RzGhxH|bZTX9@F%Qi{t3J5N7xXv1VNx^W$B|BqjcNNL8d zYjYpLdOAFw4&}g?`VocVDFJO{udmA~nDR-P?q@ywrC@!_O^%+MA=iNHZiQty{ysWl z(%TxC;OW)2W3Nvk_%+$m%M47uq1#e)p=lUS7!vF1lV{fHfhZ#7jV)SG=^5+8Kk>}G zJt-W)$0~^%9dQD2FKSbl?TGQ$XETv-2yp8sOZdrbe}o|{RrPvTIOMZDwR0S{e4FBgN=ux8K&&j$G%*OB0Jj`jjz~hBk ziD`DnvVhAzB=&M`u9$HiEIUnPBb;14d51@_Uas!avX@4(*N4_ZlFSx%PgL~*M|K#q zNpr6QGuzgNVb~nIn(+uS#2w52W#z! zCMI5T1<@V>nB=_ukbjJ4ze7DIyOr^sGsovv`DbB>M!v@PEp#qnoA0@u=-D0fR$_(W zlCEgi^8tz^eesNok`{Q}5m@;VggRBw13RCN*o4xN{+K0&)kN(kizst*&=uy7r|T)Z zUy&J9m>1&ei!>vwJjdj1A*3~WtaVS&_2t+afBV2ZSu!36sGF;6W_GVYaF^S$$M(-A zeKl{^gkA-O=9~DL-uqLYLmZOj7OF|{Ff{AF8LQxxcU`URzX1aFwzJmcdQVEGIv;0! zqqOWeEZ~VIW4uhW{TJTKsV=KoHe>qmGMc#uh^TN5Y#tKlh3MQU0-anv=7+m~^d4kk ze$*|s7YcT{AX&A}%oDkHUagy%x#fBH^~3w^L0{kV#%IgISJo~h`?L0K#?ln@s6LKl z`~%jWeg#*z%|H@Jbv-JZJ6uEPKI+HRka-rEtQ<4auz-K`(fR@LLicgc=T$MS_Y1KeI!^TAjJaGV3zFmwzb^UFUK;O^N${C z*a?$CM`#yquY8aizf7M+N$83?yuKW>-^uVWF%)G+9T&z?dK~`6lxo}Fd?id zq9sv}8d>S^q) zOe8rEdhN5!I>jn6<+oh^F3B0pzbwxav|h`Wmx`v7B9!M7d_8p(eE3vdG|wt3!gFqe z`k0r;a`HHQ|whnf6=7xDTm3$@VGcU1!Z`gZs!KumQP~@ShC~~K!bOBxtX!%XQ889_Vx34 z{(A1WKXbdn7;&StB2ATfw;H)wT|bc5rO%n0aNa9KbZ;{geSR~`*0 zL4=RcxTm*QTux^@T^1zsqq!8$D-&6OJQ_XX+;QyW&@;h~0h}>;RD}FjWfRHrTZ?PevsdBev?YMee0--;1oYxZqg4%J@{FFd5 zm2`D+TiY4gqsT5uO<5y?u5+_R`qH?t2A-;?2sT&-^&2eS|M?KdwWF80Yo z7d}%_u6pU$`SkKP{Q>9QqVhG%K1Ew$&C24b-H5 zrtRrLJkw@QaAd8ULOHmYjuL`MwX? z1t zwCkwdB$IfI8C}7{;%M^lmX7>wB1R)!#IGaV)87iHs_&aVPKgTQwJk%c4h#Cuks+49 z#yRTgg{vQSu8_~1T1+L<$~mBK38bs2O2JvG34 zJQAZ{OhGkW!QQ%9DI)42a>Dl$UZLGIqy<5J{`CO)Bmu6yY1JqVob^DaasgGQ(fBok zC%b_DE%IQd$mAOhyNn%J@9*43XdE3aeMlC+pSby=r60`Q0yZjueo(;bU+29fU1(xDQYx@ntpM| zpx3drpgDG2&5}ZReROt?ca8;02M3<1Q*o;M+Z+C4pT{B4CFR4;1!I+)>gM`A(NlM` z_!VUyaV&m#3nGhQZtF3fbEGg$%1pGZVA?2oSooC`w{*b^0qZuBp%D?faneUsJHBof zRwosEl`n+uFE5L5dv1G9$HrLYz=Hg(&#k|5!r58FHo$bn7n}>-mY|LzP6F2gf68}_ z@v3C%XaF4cj9q?JqS4(%Si3M%Y3Bo@vQ4ivoX2?4W|vU*?U%*q(Z^POcLUu7yYh2V zw6B|1Z|N#%?)*;D9JUlbM4@1^_S_R+KNHw}Nt4vBJQ7T1rP$%X(1n>zmXFNu6gyn_;`UZ-=J2ekERs2A z=LTz*>2l7#8Oz~I#p&eB1ka5)H9kB5xMIqu7K|rC;J*hPe&hZI!sq-AP!|dApU~+a ze5Z{aI)&EQrZQErdAOlQ7)_?Yz_JX_F8obv$0ob0i6UO0-pYt?yU}IV+$Q+fYn+v* zkx=t-krF~Gj8$QIRoK_+uaz^D4JgWRrfg?CGSO6;Rx3xNq?D&B1&&gzTXD7`zkl$z z%Bxmb7pNqol)h^01}fLkCJJoPK32HKgdsSJKJne<VOUv06Y}fIDYf0S^FSK!&x-*mfNrb)1L$c!%IE8 zQpmq8!U=DYY4E1$iosiZ&Ih*1&;`bw$(0Zg5N#E7OYP0hU_T^>j)SO)oNy`{%Q!Pz zxg3VE8k9w{oCN~-u;Yx2&#-mM)yx^${n%Ob;b4O<2p?auV(!*0zxw!4=gFx~&<2Ar z;b8jAoLzhWGcz&(gSkuU>@JB2$<}dF->9UQ)qeSP*cYB<7ubjEoVTn4B(4N8oX)i= z1Wg?`Ij~i3e8WbU>U;3dL$bSasU<;ylkWSQ2x8(X5)CkZ3`|{FME{f=?R;W_F{|eM zoQtg;S?OyZiYd#|(&cm}o$2mRWGa=Ul&?J`^|n1hr7l1CF_&Ij$L@X!aDW@uFe$1W z?jFh~qmQPE$T~aNr!}UYEjEx--*kaJa>7Jpzz$o4zo8T5H6<~TmWlI1<6WG~dFObJFcXV%1zBz0b$|of)t8?O(7BO?&JZm2%u6iFx zE`S*YP~mXp*-rRQ9KYNyhlDyu<{SDc# z-}j{b!^a6%4j0(Bxua4_9I(R7#;>Ie_3lpNraG4x>}1R`)={VelE=ZBQmRCI(O?M* z@1->RjG5OYo9w3&Y&hO_KxYS;&(8SqD&N|Jy65j1=_4e51H9MzUfmAS-^6dRPL~sr zcB1UJf5TaE^B4wsK+J9HDPdacueNOA{Dwrnx;>(`_5;C>3CS5jBj|&{MndcNdS!OyOQD zpJQ+roHBytJMXP_Sv-igiRE!Jjg})KCfA4J+|U#50a~mAK!I`+(v|Q;YTt9suoJ$3 zVcA~_jO{*@w8#3}ui;%9H;(*nO_9u zhb-Sus*5u5pC!j=X89X*5sL1qzAf4#X>jigO?RivuFjAFz>+~I-R z{dVyj$ex6$bK+;1>5Lzv#Km2V+)00h$VH*No*}+=MO>?r8gDNBvAjOQ6>J-7v=@!F zz*{a*hxz!7mrB%dxiO6@xaASXVaPA|xm820ZQD83-s_qP)xuq&J%pa-4K4B;_kKfS z0J>v~sdoRVNW=r}#$N6P42rJN*p7ofF)hsd22r~j=?+XUa(CC|T=o=Uh1cS-4gya- zQ76D6<(7{S&VJmQdGn;fmEOR<4`Zco`wMC7C{|8C1WpEPs0dkMA>j{Gb#it)l7Bqp z;ze};Yt*o2bDB3bc(R~d%5Y*3*oyR27k)HWxHN)ZwWG+)ZA4LQIX;~=v5b$j2!HB3 zqF=1@R%r3Rzo8W+cBw+ETU-@ur*pgQCF-Lqf|3SLa-*W<9Vz2e%5^};$y4jk&@>Fi z1mG{}HV8Q^klyNg_~?yql(2`!GX=AFqXj@M+EVD*_O9s(6f;Rk>}BMzBR2&X7P+t- z>%+Uo8DcG@6b_uoFO&0rqQ(D6_I^NkS!AMwwKT)NCjBt@b#*0YlZ|s`82v&aQ5qMz znCi(&6>I|EMRu3e^@@4X>s){IiE>^5_F0oZNg|A<*mp{nJO2Dg*ii;Fc^rugIp;%8UY9;Ii1kU`Q)0<;Y&o}2 zb!Hv26k;qd?5MqSIXS1lP4d=+3abva;V?E#0>(HkJ=W#x<7oZTh@-b`y$N)mRN(f+ zs#N4+J?h(>PBrwmq4Bb=y)*c&-k%3kOx_%k?6si*ef=D?J?ZB#?4>NGsfD7fOWf?1 zAY8IEZeCBgpQ^VI!v>Mu!fBxIxh|C3TKi*tEPxXA+=VMSZT7&U`gbxd`hBAz=l8IK z#)Np@P@%Nt9A4)6`6{S%PdltpkXb)(`#NYBaP8noBuZRqca256h9Wc`Bg2-2$yao} zWoSLoKSTuNQB5zC=a}VTzWiguWLCsGArbyP;Lcpm#}uGxti0XO%Fku&%|;^_4_UHH zk7%wyz7B`Q?xn9wlYXvsoKf6JHs*~0AupJyAe6!9t!R(SF$)nstXtScHu&EFdD4hW zQ8cs|Tevr2UEiz6->#8=tTZI8kZG%Lzf%gPMrxy$Aoa~sZOE`CML`Nsu4TjYH*p{H zD>3acOA!`DEo(H?Ux&h<^hi{jIflED8^32_^nfKX8_(|)!he!?_1F_URUauJ<#OCi z<<1nn2f#b&*#r9~2apBHhqzOFa8;;k54dNm@V05Fm;62g|G_^Fy2p zmFFUN2yLmlV{*bvNwH54vuD}|PDZ)mZ%llC|^-c~i z`$TK?EgM;Xl!O;Y*{wn`Z;mrTg4<422_`slZY`438(&$m^mK4WLnsxpOt zpQ7}zN!e#(7whLv?}`G=yd&HW<{(5QtL<&Fh<(tGsg9{%P`dmExdL1XMBXqJWd~jN z%bwM(o|W*C4Z)dHTdv!3)6{>~a1pFfjFt^QGTbH_aJ20*NQ_h1ZDvGOHme$kHE(M9 zyjo=z$du$?LR@Z{LHLS8tpMjXn|pAj2`qa@;xpQM@%lHl+o*V1(uHP6*`{wH7QZyp%gYrY_5$ zRKOK+nJYWtKEeP1dEKAq=cH3f!iI91o$%qpq+~7G+#ZO2js3Gck3?dna2jD#KxV+X z_Y24SE!!hWR7fu-5#Jc=F^J>3Z6t2Ee2_@!v)T_e6#L~p(fN~&2J7z>O4Qe2-oy*E zoFBAUTf{4Cit#h`-5>+uZ8gp6_qIO=~|aXOmvMUIb5x#gG544H<1&58qslo zWBgIui$~oU_wr&#$?$chgG(H5oxloNqQW`Cr+Hm_s$d?1&j1*VwG5CF33dkPcvk!EUn=(-wK5F9h~t~?vB`D}vXtORHfiH)J=XYRJ$(l2CEy(= z>A1?6PV6eAT?A0Zvrhph8V%$&2$IWYuKL-Wjyue?%XP$#>t2JIue_anix%t*!So*m z`1ufeNTClPE8Mr6I7?g-UEj?V)uD)*r9()n-lAo?xg2xvl`nV@Mn--t)nBy-NkxQw zWQ7B8^-f5mdH&TvwNH3M{}xwBbegp^hJpXH#lse z|3U$>Z2v(&HyM0f?6D$X zFwoUMkeL6*xX{Etz$?FBQ1B=1d?erHPJ!idPRc4lJSXYqbr|9^ID+(h?f1w9I|vMg zM_^It6e|N82rjE{n;zpMjQh=@YCjC@J4{zxr|gsUa(F@b$ytdtGu@n!Th2mW92i0j z{)s*>>^6G+p{$+L<|xN=CNrrI%c>WQlO1bMxA2w~7QmI^S;bg5c2~~P`K3eD2{V>F zRz#uSfNIeV{2Ju0#2I;iVWhv3LFMqAX)rew6iz;!?e}<5>f$WXJ5zBqM4aK(&79UUqfnYYD^PXB z^wXXfhBka677AqAvg(^UoE8`v5d5C?jG;2*8Tf=IgfwD( z#79wL#fAA&Gzy6NmiQnsG-3(hqY5;WP1=lSn-f)uU7UKZ_f%%zHFHNC$@nT;YA*o1`xib9I*jHyb-tK7o-== z?Z}=+>5RFJ5D&O>R!s;VRlsw*1Nm=1N)VfhHB;o0jZ3e}S){)$&V`{j4$6t0k>OGp z3}q;-aee2=RYx=)UVdlIfcw=q#aPHEmmbQvCjh&4#dj=gW!)@gvNM@pQ=(mYRaLr` zgh{%*lrF6O*4_+|Qzl!oVnMzA8}P-b?E#-tt-nH>cfykVYer~_?ZO4QM(OP4=tbA* zV!2NN8AonnX{JfZIEe&lHd2su%!Zl7V)jgmlGkK5tZ8;EIY^W=KTY9}aNRamd2Hgh zB@!_@H-e;iVZ(~Yp#U~WKSVN<%VvIQhdST^Y^6Ha#krwMygVEsW&$qZ>CmLeKkD3? zb>u%kPDl)4HzMm?v9DB-E`uB%5Zj)h;)tp@*3ZX;Yk1dO7S{q-|+-jt7iHy3O2GF@nbh zvxU$^1mWW$*nfrrwA6$a$uMsvIOhu8EY`%}}X|I3o;i=PQPzIBH7#hUjU)V`#XC06{V!9r$yb z7%D_PpQ#ta1HvGS5YsVxUKB=BbvpMW3xTn@4xHvEDc+^{Vy!^X;tNhA69=~9%Z}5H zkyst4V5nurv^CE>?x<;!(LtaWH;B~ALC@#T@TdO(_T3J{m7@NIFxqC8CjwmJBM$FQ zm2p?ck)=8k+}7MvsP4i#_x7wbYEh?%<-3^^+icQ?uSx|r7C>}~Aav%ZRcL!@gK!&p z><5=Hx?{_K^mgWO64|FUsG77682m2clNsC_-zZKr3iPz*mjh|MxXkms8~oI|MCtf) z4!qa^vYjENq$sv37ZZH5>|F8o_#B?4jaH^zP2LOt0Gj7wZNqJuhg+H7xkV+jQP?oR z9u0tQD+p*jGqEmc`?1=(p^O04Yh3Ok1{TbF$7urBU0rX>UE11Wi)vRD9sU#Qy{8L| z-WnS{MR}*y>1fGDQg;4cCtW=~Nvn)cBgXLYxa55-oGycSJN-HrN!~rrsMAy)C^%X? zHdNgByo8oCfO>h?xoW~`H1g~<&BK#Sfr*wa6LMqDv>ZT-2qcfOO#K{XZ%vCoMD5ZQ z#=2au<$iiEk%1By3UcZB4dI|gGRzoHb%dQfSW5iZ>A zz-_(M-8Pi9(DOKK-IK*_>N%kGR`Ld&xq#;fiEbU|+7Oi;N}UE&08azpa;=@wBw;4!es;1=4RCPhyq;=H0~}8g7Rz?k-1VlI;b(~FOYmiA#r2hqV_Q!fU)tC= zzcN!UV}t_W zWri)KRpp~68{{Y8ocEd!UqDu@#G{B&p{a8W5=i>=d6KAoFeXZ8MEc?HH% zJhF?iPa6sMlZDOZJ)nLmG5A{M?enHy_^+})1!D<}3|KF3G4xr9v+qy$hrpzUJUQeo zF5IpWgjd8lwDg6uh)1WzSXOe7bJ?n__y9=MC5N<(iIMeFP{9K8xNk4n%*nOQ=3xJDyG80Yk(8#_iYf@ zI2TR#=k!TuOhCj$R1}&;h#sA* zr!eTXSrILd{!_sM2*B)Ty%$Z#Q)E zka98(P*=d#_Kvis)DnVMZc5$xW#D@up%+-EOR?qPc#Uliw@%P&>1)w!z;#M*rrc0S z0J=9yj@>hyH}$9AOIK9>AI~=Xxy~$cIEcseEnHrc{}*>Io4>*+mw1{r_#{)aQ4iLi1QS&PLQka z+A~xm0XEAmzeV4e+2_XCQKODa_@gXWj5?X%<>xDLx*%rf)ovIU?#^T?yC+u29Cg2q*h)^jPJkKNPxkzK2 zcnMoA)7k+idI3{M-pgxPhVS3DMnJQ$uGO$NYv`;>Z>{R zABz#3BNi6+0!#%G+w(>koQdD*bVawXOjRnX>9I~U>JNJD-lEkOkHhKlP}MEl2$E^udR8{8rt>H0rM8Bhz;U{5*VL;?wBL+k zA}p#gfehd##L4DWL#k7L9B1I2a~i?`HuTuFmVD+Tqa|9h(io{;tecm70&1J#{{Y@8 z#~Ej z1%~RN@iY{s%|$oZIu9a9`2q1<*mSo3Qgavc2;>5e1{`SVNpR)R>G`V2IoTmntoDfp z+>(oRnPP4q^po-~A#!nO!7v6M-Gk0ifOC$~A1O_Etf+GY?6FS8ta{_lvJ5R~%=&an zrwO+YPp82;T9JZF$hW7?6-Mj=meH7BdJmSj;Zud#pPpkN?TNXex3Ee$MrY2lW<4c^Ym^h`Ua68k~=)JQW`Tqa}@za+Pi1mG|*=u>~#U11pn9l0vqGU1i-Cm-dC)I5r={U#^ z0vb=0+~OCZ_iuO8?zcbmwcDzw-~-ws8YXdJYl#!IuMbhytrXJERcTYMJL#jYo-J0E zOU3t`k8=J|)9UozF0I;%M;RC}wwZ8%LxFo-NWUz?=+EVz;-u2RDn6q@j81V5cIki{ z)eEU^svUx`ity*trw+S7BxNHob25;sX*F7kj;l_yTE`h4^oX`td5&w~ORVZOX4}-3 zn%I_0i_i^5Hbvuh$rP7zyt%jR0G}CQTZ0ClE0k~s}B#`PXXcLYbEEtL1 zdu8Qh7piOZP0`SOOjPPUg^^60&LO)w1Bh99$9s2`UiCEbQ*ALz91btVhGd-m-HX$q zsjaO}QQGtL+pRuR<#WCw zBc0(lE;f*Uir^Tuo@3Dbkh_tCB24h)NHT(COh7S_`w@ia1PR}NnM0~F=F%*dV#56L9iJ6Z?drXktJA|8 zYH>>2;pLsbB3XcW6%o?v-Ac73uWYNaz|>Uu!;|7qN6lKRoa%#bY*{j8j(=yxI47ik z6b53P{XQwcBv=i;mQ4l;iJwi9o@+_S#9qy9kU1KvTbeNo>Bu(M_-{#zH_*nMb*eBFnfQ9_^P}` zq(|4LB~V8skBsj6D9_n{qLYVkJtE&U*=fs&bbYY4ib8Ooj(%XFRU6WYZ)c}kY1HQ| zw*9N3G_n)gL9z6b6y}*|i(wsUJ0ARJvwf*J0e7 zW6KP4-khlWWKPwiB|!o1lmNnAIpzwFcOCO}jU=1)ASGZe0E~*bZahbH=CBxOY^f|5 z!5K;vj7360Zd=FLs$K;`d5BVJ94#Q3%`jLnHdHX<_8z3^VeL+0$w8bJ1w1NY18U72 z95eMFc_fX>ODoi27oOXsu;&AOwqp51SbTAOR<*Sa+fa5)e9G5{|OisGrWZUxFddQIh!ONj&NSVRAu-H-6SQ8zOh(>( z)@AQx)6EUmS0OLjUuBe zG?>aK(_)J!#T4Bh>uxjmLh>lb8&s8UJaclMt{KX0I?CN}BVi5Vu#QrGYTh7pz$P=X zfTdJpa5!&^dP;&CWC#u0&#&OUnYXf!U%hIYt-H(O9KM8nWmi3+N$^}Q2p*h|okPWo zGDm9YXr)5f5o9s0aJEW@o-O?$N~aBM3++F_0G&wjbSE)!izmYv54#<4^Pj49pc}M& z{O9JEkP4tcRHj@3%Q97o#hPT!Kzegf5?sjxAp{f`Kq7Z2uP%2rt@mz#YM7FKiPt^= zY#qNvX0SF4=iBo^Xq(@Cl>@YW!|Ay*jnypKw#9N<8xUKtd}UpjFdix2hEKxE0kBT_ z=~XaFeAohFI}H6o1H|mx_Cf;#oY9cKYS|#+mX42zTaHPwI|D(Fq5*Fx=17zFCMuta1Jj>RmRYA6WKHMDbPa)7sKF=&v!wkqpQ?u!(Lcm@nUy}gj?eKXlkWg5+Ns1 zHEKL!X#z&$oTV34V3QJ5A*^%lfB_w=H1~L)wAbpoPL*gWYqbhXPANGj6;*Oeg;6Be zYcVv9nRi$5PSd3VY6FXC2jZXnMoerOm80{1(Ds|^ea$x1J>>FU_lFU(T2B*O zBb=>n6Kj9FC#L0GNX_MwkitPfMMwrr6SiaER*!;fnal__l0s$DTw)!%1&~Wig zZWbR{Rn4#uMCI^6Xo0)vek7|pNd!oXZiRqi6N%kkcvwy#FA>W!pd5)dGJ62{!l;6C z&pC)!0F!x+(LYiZAdq;7XgdeWR0a_2hVGxKD@pQCZ=t%-Tx}$dVtoqSE@jE#ISsvt z2JlY))YF7^DK?&6zhDrm;&EotY{#Ua8FV>Opyea${{S+P6Svc~QW_h*yskgx3qX$M z?)e!^5gUCv6r-^HRSR41-Hn=yq=}~M1#K^K+&c`JX(WPrT*=O~O zF42=aMs>tZ684wOM>ST~f|OlOd8qyj8Tc*YI&&gMkyMz?RxpAep5*U`1dvubk#un_ zS#hEu&gxL90tLdSgmQyaM znr;J!X&)q;QQ}mP1_{`(<7Xy?hU%E^Z9QY;6U{iYIK4Tf29)+UHzNx;2}0}*&V^S8 zY)OpNyM6wN@l-iu2qa;0d@%2)J%kU6U{fyw#CEi!!Yt*`=qn@~bIVK%-23La1SAkvmDS`&PD@+*=z8aK8@-Xzj81 zAR26Pm1{JRWOR?3tDM|iA;Y8(H_ctZjL31=R)I*Ih>p^J&#D6eyN)60$d!4*-X_fD z4v(ZttYP@DY#8bbLt{7_e~N4dOo{pZ6UBx1GI>de=&}Ik#9TQ{dDWU+ zL;*a2fCp@UCu-WfsH45{{TppkZ7D%BpvW$>a6S{_^rxu9<)Etfz~qEwwP&j45K6V z024DwxVZD>D!OCghZ1KA!{%9%_gxv(oSJF3Dp_0LOsXX9Ou%sfOpN8j;+l4$nUc_Q ziTEhylh3~R>pvx>8d;f($x%N$W=AXia##Rw7H4X0jBZ%wIa}rv=1XwMJ(nnS8nn!j z%t0tPMoyl=f2srv83VAbabR}{{`3HU-V%Ts(nOH_J+Pf3r{O*_@U3XayCeW*(PEjZ4jgY$5;tF zh_LChfNUMO`eW%ASCiq9^o2@?3!vGM><1R))9I36 z!0P@gdqFdVb!GKTJ%@Pne3r2=Lp0!L@lo8t=s${5dkkC#?GTxBT^5D4B8l#0lw!;z z6P41aZKHpt^!}H-HGSbz1qx26cb~%F^*H6}3!;&<#5N$J%``zbapR7Yykt%F6tZ3lGNOA2l6h8 zrku8&dacw5J|m#8o<)NpmQJ?$USZyy$KE~BRJT(}-Wf0|wp~fn$=i;?0ocBkq+!Zc zN_6VA%&6AAju>3m0OpW)5N*)3;E1&y=6sOW;uuY$RIEjy$1UsF``_{<4L~(}x#Ld% z0J(gcPCM|Mxrbypa)so2%?`6cSHRU#eHxsdksR7YwZm&k3yhOTWhlsq(lC+Kq*HhD z%RsrGl7~1=sUI!4kQJX|(IEMgzE|dcr`3@4IRUz)3iO&#rEVMWbJYjRGji7Bz zj7%_8j5AhmGk-#ys|Et5Ft0Kborr{_)4~EzH2EzKBP43hbB_c~-mFcOM%3mMkg(1z zGz~XSR3`~dJAh8qc!A?Pp(^t>j-st240&kfyQ?IoHLiEQ*1K#Zahi}sEI6W3&^Wv6 z>Ah=e;d4f0&F%0~DYQR3z?e}U1~xj`!r|CtaoBH((eY8_Ev+0k%%}Vo^D97owfIH1 zZ(7_nPA6t9X;BFP5LKA*g=T7b_EE*{7&o<^Mv7lX<3D#UMQv2xyXDpdGzcz$r4bh`3mlD9;m3Cw#RfxLTQrz%wt=CWkm@{pC= zLlxXQvr)fD%YfKBP1f-(rWz5>6&@R{UQf%2RH?7y93>5;@N zJvS>huw;xS+g5x5S79~=eyah(BIsjDm8O{oh}l#Dgvj1Gb56d9ClKJ-Gm>i*S+SVdw4Vt70H@<2Vsadi>h&{QXM(Q+fY5A<9LZw#* zQ=wDlQHt$jvI-5f;D^PFtSdNkHNtkO8$2jxddsjqZz z6qs+=_YaeL*Pv->QlJAVF|1}#&;aTYer6Q9xsJrp5WR{jB+ZhgSQ$)8d65f6NOh(N zN@*mxH{Kv68dV!npn@_(R6g9(8m%UqTTjB&(q(3HaF{&7&8KJ}r^0>^!&%wmUHADC zplhmkYmHI&H3$_4`X6SvY;+x^c_Xy-boCudvmVE>#jb0?Ynpa~zQ741y?dLx`{I@j zT@dmt%7ccZKhp7d{*x^`!sg(g1FSU}@}Pk6+A~sJN6G44r+ofQ*Jv?S($21k+yvHO z9M}gJYkFE24@vk$b%o8UJt^O)>zdHOV(kDZb)Aa|8>Up8LDRP|nQ+3T*s9tvDnv?c zDSn-=U_R;kC{CpMXNX&#VKQP!qfOK9aTygyQ{``hDL06R4a#A}b8*^kGVofc0S4+p zUb=168NukhGp!Ss-I5@P<%A0}IKd>ICN#K{%!#|7cQfbs8xj)jI zjxch&y4R54I`&cVBZ@>FnR*oFCa8tq&Lnb7c*Q6VJnzw8898bfiYA|qyW*_v`)YAs{w?Z_dp*B zPPMKXo}|;9`ZXq4Pg(dO;M6#YK4b%oY`BL2IesFW6!4fhh}&Zl{W{6fOzUXd;qvSW$}aD^h1!DKWBo^!oW zXN!hGl75RZmyDax6LS_OdltfgegJO^ugNup6LJxRd1M|-9PGFt^YjP{Rm4CN3f8%p zVkO;*k~g8ahq?hq3%09TW1mOCJyEt0;QfiW(rFmWpv1=IACESePzB~8X%MB7!pykf zUvB;X00kVd)W1TYyD)wsWI8_#k+`N9?kuJ^CrEKN;Qo{a0eJFyQ}X8ta~TT`2u`rD zwl1I=;CB2w69t5vp-GBcxVuN#s_ao_pH74Zm%gM2Vfdi1jM}1;ytI?E&?$^H>bDre z=JNbV2;@h*!mzcj6Cf31jmE$k{{Zg2dh|b+p|mc(#3Q=ZVHok13A zhDj#q1rC@;9OCTBDN9WAghinw7d4G8aSa0GxPS*qN&0WhJ=QClCsNrl#{^TJSn}zd zJ7o)z-P6=XoqefIt_~K3&jK6)5JAg$Ju5@p!yuu~P`()pwIm4}l^I+c6&0GG@edKV zci5n+K&Kwd7|daii6I z3AE}trpjGMw6Ntnl11mGN_(*|jp|XWF2-Xv>A^7AEGYx9Dz*-{c4 zP`bn^3<1Q=(NdEsnQr~bsVjU;rA4Yy02d2FiM>mE%bG%nsAppsYV;@170%?_@P`TM zO;sSYM8fcHK6zOFXF{i7Qv*K{bXwfjU^}|al?6ox#JYP754AOBn+WMoIgS~hs%@?y z!*GqRKuTsT$ZJP4nrf{;Z8q$WiVhKXI~n>X8H%BXw%tH1f+rF8N1aIws^JcCjh6T) zszo~j+2k%$E^TPNhbi-|aoh|oU=R)VDuC8pf>p*Ajw2^x(I;R~7ih%eKv~#1ldvYmB5w19a^9{TdqDdAO0%icjCCGK|3;L&$=A)^q) z1O;9yoFWA8Hcmd5gdje4dK3jB6TEG1Aj#U8Xp54Zd)-t-*+4VmV%#NLeaRh(%5>?R z-q!4qKUGIJ6R@)TRRiXm4o!o!-{zVAYYm&8+x*kcX{IdcDj}%;Y5>IN8-Sh+bDf+G z=%n0s9V;p^pv0L#UF~S?=A)7(=ncc2&|r%NL1z&bRbex+S*Ue0e)ek0uPh>3Pi?wOt3Y*~a^eO_h+lj-Rl)v}D7)~OBYnbZs33t<;yI5? zARU@!z%v(=B71sOu()R6@xj(%q%H-8k%>9ROoJi>1g7C=7!6C(cr zG{ebolO;%|0U)~k!nIq+h0+i4R0-7DAKC`h>3z#p%+NKA;TWHN_u9PlQ+!Ek?mLy= z@T3#&)OW}qiuCD!KSQi($mkmBH!AC!)%X5`L`cWtzsRKy>ptk)dDytUZ{~e=rdvid z>^@E3((`X*4%O~D+DE18wVHh(#a^o^jKrQJ*wHRM0dacI&3U&;OJ|8w;J|vM^ezGK zeL`IKIqo^R#FBbUsw!d3QPk`8I@e;YRnOi8@Q?JgKneoHNJAHAOWmFQuI|Db$9S~2 z(x_#h-7_pbmgWleUhVwBq3( z=AB-T4LwKvMJ%MB=Co{av%4MEy+=ddT2Aq1Ej?Da?_sms{-$O=qiV_5CNDCmc4_Ge zF*?eHsBxB_?<35=Jm?RJLss0LJQ|UfCh;)OHw;}3UR>G%&Em0~*U{bcrEhunZla5Q z{5a{2_}RqK{piAW74w7ML!?{cxKAk^bfs~p#SE6=vD~i1b+0@G>`YUa+9ZIobtVAN zNmo(vLb);!4pwn6s>aNwi6?c!1K~x$aU3m+s(=XrSiVFoVTS{?0n*jZV|II%Q>Tsz zj3Dn$+=3Hfp}zCF0`Z?XyQG-KZGslg9H*@gsK$!|G@G)3YXcGIV4AN89>T5cVl8=h)4yZ;t6vJ58HKoPXs#6~G+8`OeN@4Df z_Qi?%GJ7>T1OEX1YDaWhuChP;cBB6Qdrr7U)T{vhZEU{r{&nd}DJw?Ov$S_~_nH3y z$-9^T0I1Lt{{Zt82eP)7qo`HV*A*HX^42hF1Uh4jO6tuSWa0|u{A#xTZDaocsQtgx zx?jys!{3#|Noc^^HO0S(z~MWUQPY%P6U~n` zDOS<3x*C9Ln^K)xX00=#));GQ&L)wki4BPpmeuDBeNxa(TGs&o0Eqtp`IOofUQQP} zqqoi{6Z2d(H1y=S>MFaeIqcev#wz>v9=)dT9_Z3muTqiLG*-Y%KSQp<9NbqQ@b;Wp zZc^#@*K|@Y@Y$$lB1@yY^`c4MV@-ADTGD(i+=QH9n~i zsP{+wHmv^u^tz8!xt7(cLuRd8f&Tze{{ROoHllk$Z0R8;CiG=7&pSp&u zfAqSaRE8Im$3sWAk-SQ~s@49|r~J!rP7!r- z`F9K_NI)L#9cO3hbv~tFtxyR4HmrgF0Qt2(sYmX&InAtG_qrqAbDjSHPOSUH^!6kw zb!xyStx!x~#Hc_0!5lQ{KYKp{;W`Pa{{U^P{{UswdZaR`ad2)Xmbx6*G0bo<4chjD zkqH3RsrQwCw5k69GFfG{I>^iQ4*vka#m-TN3sP@Plw9iNG&&}>`8hZBP8F6~E zr>g@`Q_Ieak7>I)WPehR`NK#nX(;O1#o)UMA|TVL5M*+6NBWl;td5GCJdip>6>3ck zc{xZSH0JL^T^X!wXlH4Y-%LE8nE0qqVPS*(REEAbsM?oH+Z>h+vBtEx5;tyCk_vxJ z{u|orOX-JSe>5o(hUw@NDjjNNo^`bA9Ex!zoux*xn1Pe@g%3G|si|loK|C`HhOBT& zpFQNjGIl4awUaKJZrr0Zl{5aD;K0RdwF$u1h7QnA*rMW%^Yd9$oKUoIfI{6;AvTsK zT!xRDVgCTkbjzHXB+hC>E<%(6ty>@%CKJzS0Y|3LNlGZ|i(1DxV54bY5*M5U>pp^nxm;Tmn5q= z-BaFhl-N@#%gh1NrML=hfH5CTP<=CzpH*r=?xllh$fMf}T z^+%od*pr$}=2DIjS|;{Ul@x25azn@h!8Q=(NXX-X*(nC0ExNTPg0oU!fQ$+~&fsfn zC;tG)>N|hM(=HyF;csAFk9F5}n^9HmYaU&ts0et((hYqvGA8|o(3RV(r!o2t?BD+Y zlcT%)tI(nDFH@p;drvkpF1nx1NB;nh>LlL|jFJBU1Kqp7@^pFs0Htvq``-1( zc)F>lr+e#DW1L@46~mq_Cey#Mbc`~2@uPfN>`Y7F>CJRK({We`i$wnb^6IWj{PMs3 zzKi<-=T0}j^=zCpw5^=*(!(|)_*^G5n$&dMB(uN#G|Bo_incTE%2*9+L;)EtTzB@N zvxbI@ZnY8r05U(%wMYJzllu|pTqV;=P5a2H(b{WA^J&j2zx#X#W7P9(Aov98+%h8jPs!G;IuRSl|BuB}9W=zBzA5I zBz|GliHdqrImI}e5qNN%%53-UpyxwJ{ebhWJsXS_{{S3bq`p3c%apm^{{Z6A?5&so z05Njk=av5e?et&R4?1z({i%PyM@aaX#Y+}uceuww+SAGAV3Y)<-s$%%j(C6YRr4-W z{O-5?zKi<-=T~ok>Y47+(z0{IOAOeH;c%SJX$?AQ*=&%nnR*NV<=XJmB^k3Ky zIzP_q{{Y+Qzpx&4m#2F{)px)6w0kRm?j$gxJHPc$_vvY06Bwyti8E|YF@8#)=XL)8 z?et&R4?5Je?p#6E_hFXTpXvVq!BunY_gmN$R`>p?zU>_=CpWx6%IqU_9%h?yW=hH0xG) zZYS4qFfINYGLlc#ah!1J%RV$>Tm>#Ro_kc+7Com6g!b>`qd#(iQ{G-gtl~?#gAxzX zHt`pdq|^5s*sTFTLThnu%QT7JfUwh@_@km1Q-Poo2IbB7wp4$=@Lr84@2`!9k;EcLhW=j6|VT3SB8t?;TAI zJxwsFij7t|j;8CkhLvQq5 V29r-ti*YnujLp`Pvodsl|Jf3fbyolY literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000028_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000028_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..ddac75ce5c874d228f7260a5ff106b27114d1b32 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuOMWDJu;7h0mg^XFj*KMMvruHnEzok zdbo^=!$JT;kA}}^I3TBp(eyBy9)@0em;$UpdsuVxON&#nR>MrdYFMh%DT4{(wX;YrfQ}spzq<{bN1PL?X~4z`uE)LRRFG{tb!~62^j!DLVN*!-ve-z z%{(mxxw-hbc(}O%zZU_o0O**QPcWZ6{o{p;gZu9nK0ZDX5fS1iE-pSkAt4bF2?+^e z0P%~AjEtI^l9HaDft{W2Hw@5}hL=gXh&dlYjMon%UFPxru;#3n2NEn#l%4fbt zR6D8lo#B;q9)v%rt}EY7z*4)Laq1%;02v7}9s0vmhy@X1AP(_p@p+`wQ3*c8<_HHgfc5mLSE zT48?I_aKT$jQ80d(LQlw#X{Y`cF`+mCx0&vf(a^A4~^Z8R})^c33sjPae$O$EWZNKQ-ISe7t~kSC zXH%7j{wpaauGNa>DMbn&c{xWkDg-n_Z z6eb{^b)JZee*BLu==BB8*S|?P&g6k^c2zY_qs4Jb4{K=TcF`r~75@_KUBGwQYrC^? z*0cR=aXi^loA~-P2-*H0DuHn6>+{)=SY3xhH$L%?lUqbtN-O8ns<+fsfu?6M>&zFN z)OV9C$6fI~QdQMc0aHTd$sEq5EU3kQX!Ku*@?r0qSfM+>=40(i*C*|($rDe>>W6$)pEgtxPHh1snz<4dIJ1g`r;kV-{A4laT~>vcy(LMW67D zkV+n0W@V|vh>THKkDY{Zw0UanR3eG>5er`Rq#289p*7#UL z9ht5cPMYl_*Faxi6mnR!)t0?u2$PoIO~~WQ6r)wSZOmc)12F6!W20GQ>&~z5sCp@a zzq&@thoK!a40t!jR7jDbZ~z!ZUv$SMUW7Z6SLCeit)D8li`y>jsWfqcTM zmNgyjyG0d}?#82@Wete(Q@aqn$Nu)kQ0;{G>9+o+t@cjCym4WU8a!20Y4DaWUeECE z{_Q!0Ej~Sw#m&`>iJ^bmiq03#s&AAVLC^pgTFW8H>_gvP4eP77B$kC~Nv3>4RC6+yX1{Ak;B zaV|YZd2xq-vyjr);&ls%2lYR;K;(HtJkr5p80y|R$je2Vz$zW5UVw?Z^#8d> z&Px9zhj(eE?|9&@?y|uGoc#-kb_9yvdl+t1zH1bj%o=Nb}c;a(^jcx`Xx2Ui%pSy~p#`HlAp%B5G#A2QOi{XPC}@j-=}KmWNx5>V^z&tcG3_0l~p*qCck8fzc{a{9F) zeyv=_M9uEG6K)wi3DbC;^~prL*>{@n#96Zv_!57-%Vd_oFXrgTPPMjbJ5Lk?Kj&Yp z1R9v{eMkX#)|2<)v$1&MCjSO-|NQhbQZ*qP!)BO^@tzL8u+ybnyTr%2&TFK2W;H+p zGD>huHYXh@v%M46G|gGa&-?`s1gcIctau;7v z{i|U1-0}R#;q;#MPT;MVPn~WQg^Xi`&eQ1F^`?=LfzVb}c}49=*YH?E{7H)f>Qm&c zEY^3ub&tmem5dr>c`MqEr&MsfR67JuBqo>c6S8!Y5Go?NR&qfYOyFf=b!hI7e37Y{ zTH}0*WH2_6J3T}_s9+O>mMcP@P?pG$;lehwq;t8|%zvi2v=0`G5?>9QIQk;thV9$-wp;kvNb6X=Md3>X>i^u?KR``T zv^CE|;r2W}Dr$oqivhe}=;yA+`1-F)suq5Yx3!c+_muD~DW!S{)*12EO!E)~JD#4#P zFTLVCN~;g~yTezcMoqyDLy>yDOiU=WEGgq~JE9X^vJD~5$Wld(gfX-rqr*w0T4#6> zw zvwUn*_j)ZaZiU_F*41vd0q7&$oAfm)K|L&{Y%LR%`8N{&i%)w4Z^H+7&Q2{=N(`mZ z4mdTZSQi+O!g7jw2r^8r2f8$TiMOiwqv!;#Cz1K%zL&XQH+f%1xfh>KJa`l7-s`o0 za+rx5WQSFLMuSO1L@VP=FY)F75{yd!`*dUb*Rt>}_FqEt>mpO_$KN@EO8*-EuhQsa zX#OQ+{`T_D0?%Z2_doft_TPMHd)@4Pne>ZUg8UyCXB_`8r2bC&k|2oibvew0xMe+CR_STLc3tdR}ET>(}^6PfK(G|keO~erjt;Y;ZM#b&$>p^tiarD;O;B7n6 z)HQwO{@Ofh1`V`S0hqwpf%d$96kkS}Rn2z+UqlY$266G8YFBYlG`=0R>2!*lt0*J!jBN=S=xwpu z6a!}PZtK@C$m-TL=J8zZIIm@lGSfoAkB~FBtzpAVEa11|t0lUV-CWv9GiNWFO@5{X z(aML>qD!I!aglL@070|>Rs+wss(dk~Kd9I9fC8!dyPb{Mc;=js*)?f{LF+hCGN-1Hy&l2GIumg~T^GI3lNO^>CYc_F(cdcK|Ql8OP;x}aVcWb`Y> z1?|-OtNO01`qLOx`7H65id9%F9~)xUB;h<>5ursl3SJGOHDy~`X!4-qCA}weqA}hh za(TPecG2%e)pzjaIUI+e#Zt0PO5`T@+eY0?>rVrCRBicyczWBY&$!APxi&>wbd&j= zz<-(JA1eAU$}6O8F=`oVO$HviwoW`4SU-4wUHP)1n5t;KICBep&q&|neD}kc zW_o@|&6n_*@oOqgnAg))M`J>I5_Q&AuCwOG{6J_nL%f0jGhZq@Ztw+j62i`yh7zn0 zn*Hpnc1RwoycT&h%(87w#NeCQTRUU5Zv>ljZ3|0uUfnqKRYZ;QhP0AHM;%lu{dgU= z-!US?TsRgi@2BqKZgexrH8W}pPc}V7O3HtDC%9nK4m~RRWV^$0@zb$P$-1uR$#S8B zMs!l^m_)(Se3(JT;h~ksf_RF<$~%cf%YvM$g_K2_llWvJ~ctE(H|8$uDYN6m9+~5){g9o;G@)W^1rhlKs|1S#q zYoMn@(i}GkO{na_&9uP=%vLd6%Ma z-&E^+eC~=YD43)Q>Z?n7n{XcTrm$%;KAE`MFl$LgNZz;jpoZ&5XP-`vMW%gdw8pEj zHx%KvAM#BvI+A*9o`}xZ?k6?P-z~|O6-{4WMY8}&s$39Cd#n!0g@l!Dq3c^!h8q#X z&G&H=;_(7_nV;cT4Tbb%EkehvcM%npjKTN3rY~U`tj&ZRUUr$3@9?O`SKiq=Qw^vg z`gbg7TO}q`it(lF?fDZ?T59UdPEgM>XlI|utJ@B_8kX2bB1VN}GGSKwQFNn1k1HIU z%f+0m?=5de7`6zIEuKkHL|n1zL~DCgZH#3Kp+Go+yKFp9)B<3f53%8YWn7}7N|X0X zAHMhdYMxWl;T2>ib*#mpEN17rO;xLARZ}f0(W`2Vs{rv`4a~7(Itk#EjPvGuIlyvyqQA7$5lOh$yc{Z}uX=w>{Tr~E zeHmdO?%7R!(c_8rdbWp-I@V>Di5Id(I|Hg*J#SXjXlK`vZP$v|p*0A4uDtnoa0>~< z4K6~)ec%V$oZvq|hVXj?{l5A)Ge3${3n1nMq972-fDNV1SD$?U)R~Q+)+(L^0b5A~ zWHGz3L|nwVi59>(MB;@R$5z`g%Ga@e&Pv2&jw{IAgcPGI=k-;hcZ$VVkG##N*9V%#+Eg#%btHpmC1YtZnTgiF75rJn^i6rd zKRw?jtsL#Mw=)Zsqut^y{`@qas$f!$(>35z7_!+e@t8=l=__dvt&a`MgC7|nF?|qU zJT0aw$P`5zgZY1|!fc)j7TAxMEoru`;>l%o5`Jqwm+)*xq>rGyqlYO;vz&bJySciH zr)dzaf%Jdo0KpvO=^?Rba!^WKJlYpy!L4XEBK=yJd!$=gFSk<@olD7+2T#m3B4ctO zcU;S%?6-^=ZT? zkiM+|zGUKB0-~eZA-M3-q>mYa>-P`!5q*9A5mJ*dwBSD2G9Om@K9UEE;!$@K%guKQ zSnm~X^LrF=dQ0aYI|nL5x_%PW8G^*QX5QO$8 z;Y^2rj5dM}sf6qcC>FiKwChbgkqA}+l|>5)`EvFag3W$mB!_&*15@FDos06s+W~`v zE+F>6Ho}bv>p?EA&LCPN$-fsv!EHdt0!}?+wz?usG-CrhvMM;i4Ba)1!+kz|P~hkC zfzWDNWu47Uxtx-;%$(!Pq|Z`x8Rs5gY_$fgmavaz>=9#hF|_3u#-W_uSc%Q1%BiR&1FL!i+fnuUog{zMrNimXw3}|CeDA5r z30TTQfaJOEwu$*fy4hT`>NVx-vwfZY(vhEy?5yhiW08`|=$$DiaozKGv$3U9WhYT> zqle=mq_naL?-aWdy>o9}XcmeK#lP-0Kd~Q^zzAN(+Zn>O@S(D zpi*u?X(&rP9aRp#Mb$A}w4aZFMiF|HN5tX)+}0>YK6zOG6|<+8WCe$#XA4iL>L5B_`PD6YHfDeI@xB?JFi^YQugiHN1 za6aIMNA^I?2aO=6q~EeoyNa54Sfv|caj<-#mZgnh{i_jtDPZ2)PTEa;Y$QY$t}L(9 z9QGkJ8bg5I=becuy!tV0^|hcziI{@eZOo6ou6puP0}ExI0~4FQ=?$HXTv97FFUR_- zo2%rR=rq_A?UNZ1MC2y@cV_#iM@+EFE%WOIO;IVzTGtA&2bU8YbmRU}4R9g^2XLZ6 z@?g)zAX|J30_=!P$sscJvop-;3014d{(c;a>3}w{nt|d=4MQPA1v-!@2(d|I&v}<` zuFQSl1~IS)M5Q#+(-=&T`+_Li zt~^EOzzCb)dx0W7W>yh&YhaO^i>G{A^dL{Yz>(5i zl~mX(+9T~9FijWxzAhSFS-7Ipm8o95Xd!|4?8)l@)gqS|OH{avaI?f{PknXTz3dpU z1;%L{jP}4O#1(p`v>+HlId(Y@H1kC5A+8|eB!Uji#~<`d@CLV?Sp7o|q?iMVztb9A zrjuc=>m-xwq`2*!0uX}BCAum_PvI;fgUn{SZjRvrXfE@*{HyX(i%M?mJ zos+JyQNB$HhZj{)2WXQi%8C4p;rIRdJ%tsn3lk`J@*UG++*_P=w}zXSp?e3f*=N_NU&#nr46pLz=p{!mX z%Kp7Vi^CvVkh&tD22WAgcn~{!;6vg^ocjk=bN-!-BOnU$XmajX`xx?L=qbNB;|cW- zp8zY#W{}E@CwU*xfjb?+LT#AZn4T|egh{tbA)Vj zlPfn7%deXEAmK4eL9`x32$-PFgq6Xb@u-?OM`#0sDN7GYP&t9If9DH-)DaJ<4`CpF zl4J#x_gVkACgBe>7`GEG!RRxq<$0V{b2CRC#=Wuqu<^`!d`7et zpXN*(&D6eZe3Y4$hU#?^zJm8bq-aU``UO!N)Bf4fKr&+)9%&{IRetQ@hnCXq@5it3 zlg{Ur{XKXEGhu=JUtYN*LOy}e(b%pk_{(XFHp~7j#(;ZgPcpd%B z6+7Q!S;j0cyH(xBX1`QB$M|VUI#u!*xz;bd5<@d5v4?ha9t17g!m;3wOW8lcEaIgD zUF46<86;tA9io^wkR_|1ikw1JyrY&8W>9m%jYAU-yX{rCsghs37c#BoruFAmSM`TU zEip5?(MFB*czJ37QXkfOW`?xYib+0SCv%B}gW?)iUHGXso7UPI>%vK^_khlzV`mV{ zbu2d8!2-P+)ljX9?5t;>otp3=IZ+%Q)dAhPCVymfHU`Qr>&u?Z@>r)4A_?YeA6+BH zvQbB><+!bzt7`h4VlIKx(gY<_rxjY2!}#& zBjNE;2k*q&LG>XvM}-|PUmaQORD~`^{AZHLT76TArL?eb+Z_dz6yyjRr&5CbQqU-V zowlB-Oi$7}ma|(34nF02ub)$DgS~h73pFug^R`HiOgSNB(iIaH5I-aLT(jA=&$|8H z=V-CMFE{MlJFuDcw`C&(&soYn7(7@?z9E+*ZXV{T$cut_`M)oHmQES`6#|w0zmpYtrTeVU?LF#RT1#8ZMqOGbeFIq+7qjYW$W?PD43XsW$==dfpzP z77_7PM&F&2@kpBo%l3d%V^v zU}@fRC>LXHn+nG0$4BQGV2ZI!QTK#2O_hbK1eH|}c(E1u8sEr>#KhINkpA^Nnd4m2 zJ3(?zvo}49dsh2-GFfD5g~v3-x&Xk!T%K=A2M*#hgyrc zG##XBB@;b|TRzOY)XOkyb4{l+iC6VVGb&1+Uck_MwlH1~B!LdI1Q-{68&QN5y)+)g za}izlNq))owpEmJIjWF)fc`5lu6=}C;uM->qHa3IXk9?j8zZgNLMnAbyCV@xIb#O) zBh@aR7|BOkO9D9VzQ}v!F=mA<`9iD~_V_w-noUuK<@;Li_ZH{(bX65HXD*bmVw7n2 zzYc^RO7F$A^f8j-woJ$)@;AVPZxQF3{uK;JYq4Y+yq?Lmtw(|Qsu%`)Y9&T>gb@ddqM7IoLyhaq-I(;XK z-J&*FHM-6p!LC_gM(T9+DST*p&q#<$Gk1t@_0j8D(7=#U_=0XyX5QVY@yV)@EPynq za?QckLVu*&BsV#ciQiVs#h%KnU$->I#aRJrLZskXQ?S*^J*!+Co@paYo zHNI4{`ajbAVZeTE1{_&#Jr-VK5CF1~;Tw@|Arg&Zc%=PrfW}4D7ur|C?RAQiCWE< zq;6$JksVuO>X+eBNsJUMqxkD=YNT(uhj>>K4^@LY!j(X7vR& zPwZ1&aZ=c4lM}gUo%ph?G;g$A$1_fBm>t9JITDQwL=v#&Jbb$Zg5DwGbQ8`}4H4vUDqL|@C;{*~T zTPvfsn&SP~SG@q30x-foH0#T(UJFiwbhus|em(5RTW+n?Dvg6Bb`fq6#t|RC!EO4q zUOn5a>#5RzDMN^!$FgW|z8b`-$qnPoLvY6=E}8U05{9Uw5Ur?(qKnhuGi9YgFruCF zN8h%b#GcU6AqKbMi6bKaG(e{|CwZE*%}$VYd=cNaMz_E?*L>!3)nF>8*Ll+#Kf&OC z+WyMnGrNN4wr|L*ij66f)SF;Y?5JXaeZ>hMu|t`QLiTyAfN(4lh15!e0Mz1)z@JT} z3-S!~7&txjuO~=vM#LioG^A76c?L+Spl{QnJGtbMk>Eybul@DSRQ4(EiTvDL59)(4 z4lg^q}(tU2F3W=S7l zuQucBUTIvjnyF$ggwq;eDAGMMWkd5yRJl!A78jxeu}h_N)26JXQR92eH`8mflKHBd zV8ZbxFg>cs?_Omx$_lClWf`BmS<=gmOqvl{5E_9x@^>{zpwQ5@9j9l}Mq`=!1UQlT ze7!76s-kRi<#k#a!fUmbkZ~#d8oDGvX(7bsk0hr0N%KdFusN zTrC(!WdO>ssT`h`u!(3EC#GNyjd3c9FFrDICRG~;Nlo4Bab6&ZWXYX4J;^g`R;fcco)J*(>G(X*a!0(YO;PEFWh z)z8!H`Wl)P6fKhGI;G*Hop=j*7b3EGDIP^hJ3|7v!LB7y=6*SkOqWKX>OO>CK9GEy z7c1byKKy1C@gzFIV_n9n%56a+PwWHIl#hs7t83RSZAnU&!v+fq_H{NID1N;5TAvN> z!`+RqQEBV)GUfc^y0v;2t{G z6RlHU`P?vx{G%Qjdd(n7^>$Hm=LRKoS^Xgdo3@j_l>^w!-b+bMsn0oT1W{+WvZo4f z)-hD36nqZ<6s}$>yBLL-2h*A|*FESli!m({1X%aj3{M4+*|DQd7?W>+w3?f4N}tEl zfGJNVtqvj%_|sSZRgV7AokF-y?J7$IS8Pa1aEaa@eFqS3NDksoNFZ)wB-S5IUaq9v zoZx9c_np z_X?z#X(P@hp8H+TZsfH=G3DBu(`-KDJUYL#Vo@(0%ami<3Vylh`}WHU@hnOi_5s%r z$VVYZUaVNA8=l&q@dDZ0EcF0QJ(g;9J;TzLy~g_vOvS~-6QZ01_FB1Z|S zzVX@tS@XM%yW#H2TCp1J`msUeN4~lA`m30@RlXlL1WttDRJQ`lIrwbq&DC?)m%jl* zdn0>nV>S1WXj+Ot_b~OaLASxIl)^0LY_@F#IqPflK%SyU#-QB}$0cR=J;*_aK7U_1 z=f2=N7nkWk?v%8=n9wb9?W!YRtgT(JYFrj|gLzBQx23b|FHld9)KFN--9=xIoAslf zAZDG0&N4IaSMUw-g!yfG z)&s%Wa;Kju~`l+)VE^$LCqrRQzP8lMZ!L zR^UlO;b>`X-^mTd0EgCP|^pH1iE(~*!y|2-pw_OkH*382xAJ4K#_YmAtvol3xgLU*ohRkL*=|s1Bp1V4 zGgikWSAU`v7eWgn{)mVb4^`8HH`zX5e~>Oxrvh=rvP8fg`vFdmPiJSKm_nTwu!j+4 zq6!0&OyXw@zyWc8ud4N%JQ{b@jB-fL!i(%Pb&KE*TMc#0A0vH?5+d&+ zOALa&sT5D#dShO5T!z$$#*F@A@aIhYsvU}J?jSWf7qzry&J*<;5D^65)#9qMc6v5i zi5C^Z!=RP2R!xrk6yve$Yv@u3$L97Lw%bmX0A6^~HBzuk@!^EmG;|h2rJIOyP+`w& z@$6ibUPa*3*-lQ;Y_+#8XT6<*wM!TD=~SVn4O%M{n()~x)h;)DK87Caibc57WvGZ& z5{)orjLgCyR!qWZYX7^c5eI0prI<-bh9{8 zo%DoEe9n7clAie?qXlk(q6EPhqM~(i_&R>CY7=Srpi7h`TD%IodWf%fn-Jb`rj?Kl zneB!8gy+6xZeYza@dSvabuh{xc#`77Ad#E<@SEu7OiH>u21+@XqHKfu{7kX=^@cNg zYjYA?LpK-t$mmvMf~d>ASpG{^0y4jtr=Th0Qb@FXOs4dSa4ZXV^g9y4v88YA)K9WMaQ3QwA60E!oiRSnh@M$K1f#w!Ud^&pVpH zhCcg!-h67o5sh9Y#(<>!&i?GsV4!R2Njx2QQe27rJ=;z|UynB>7-vMaiSH2=r-y?_ z`{Yl>m2?L@7l;`W3Xk57J{HC>7uB`=Bm)>u19XfnRjufj2CTH(WxfK_8I&p$bUX9% z1i3Ltd!%tG0LF_=sPHgU*G&#^OHNX|=nvW>wBY3$JYJmDt_wGK#Qi@%1Q|aCLO^sP z8nwfTKyKdHC^rVA8C}gdNFjYHLstD>m;Sa9Mn*b$%Ncc>Ph#0i9vDM?M#p*ws@!GY zoR+lg=$tOo+8ct@OUMUE!=MH;89AhEUa?q6J{o{KoeohHjiIQARG!BKjja(qSUflhRgk zD{-SxD-lQ3+kB+(xwd_X2uPx&@rTmQqPIDi5^6Vb9T_M$&?xO%)#Upurc#kdgpxhW z7?jVO^^yD%^i)U}zcuuVe7FNx8>UJ{|??NA|)P!c5lY z0vcMqDQgjK^)=RmyiYMa!N#9$3BtlcHJoMz@lv$r`CgR?)g&sjjF7e~)&{jjB$Uc}K)9+2_bS~K z?zgMKp&DE|o>zSXZ>gz#Q!SXyv6L5h@r1Lcd_^1>xGKds6^rZ0MtPP{4G*nisUTxk z?zfk-$Ug2(=v{`oP(%rR5E={F8PCz}QMs!t(}u|>rjMz}B;JK}+~MF+%Fu|AW1$n=No z`FTr_k3gg7>1h?I763OocMD_M+eRfo(h8R(3N$4{EW^c3q0^+`jHTyyaanOx)`_s0 ztb+cvW~7IMRGP zEJ^goC8Cvy#eW4~(H<$S$==?S4vFTWiNarr%li5)-VWO6@|8VDjw23$P0$J8E<{6d zuhls}PzzMN&SmDPnp5u{YEs)Ene4d5PCouR`V3|+-2o;5V!ORm^l0t~PLFOhghZx& zS-{jWJouJ+koR`2jkWmM^l)SvtN}^AxQ^rHQ?P^f&jy)Ajc2QDuEK~1`8;^PH4$0+ zKExg*P26dW#UiB(FzNp**|x;+WiYj#WL8jZ(BQDr{;7%jV*k6cXm4qx``?$UQ5 zD;ltJ0XIF?<3xEIE>`y+>76o-g(A{TLAUthY?hY1ad#qBSrlO|1;rZ}5Gxv*GxaJ%TmB^q)Hw8B;cV%mlkPdPh@*IA-V&NSO2>iB zG*L2dAE6A1y9{AmF}jKniQ_!ff`?f;=RB4lMK#JjO|urwhK$`4tg_NH92)dt$?nsc z(cP}d*@>N_5VN`i$n3s=?mNgeieH3>b%-8G1I z@zQtID*ocY5%S<^JQ8dFMtho6$fomrFUt>t6<-vD4>8 zQw{Su`JdMV&&-;I@_g(KX6nbBPsA0jy%T4#o9oG~gf*;t{O_i1}=s%r`kVfH#SYSv&esE%KCG$(k;(S zVw^A&P1>rC(k=w%qLk!V$*F)pOIt$8x;<>8zCc8(Ho=q+=3+7d*D_df@FKS#Z{{Wy zLp5nq?axkt(tKoCMghnLnrt6C-ucd^4|~v%PRGZ8scFP{>e=R_0None`p&9nOyOI! zMK5oL?^~+;j!(ztor$AE`))?g>XtCQieDMtJRve9mMOeSjU4fhLZ~@(NRN@B+TLvM zT5n7oZhbrr4u+Goz7y)6oOcr-$8Sx_@o>2AKg()RN#=5#i}jUWY(kEMhw+s>);+y$ zDEc)gc)p26&tb06XGYgP{M09*Fomb#$G^lHM6#fX{EJ5Y zaRV8-QYChfHKQ6Lo&|iz!K2ye5lB|HVBKbG9D_eW06p7QQlZWY@7fCum5rfnOaI|9 z%lq)oALW#?%U}ISDgr#^8l};hc*9BKYPAn~yr;o{-LuJ)PZl{VKuRNRP{AoP4 zeUxYE>wGBEnEzg{ueWRkCvw||p+;BXi&f&0Ays9YYGw^Ivk7FR8%|;^q54=GTRi2q zrr6)6k`wXlPkt^Gy>BE8;1h2>xfjV!UT@}p*Z-oZh+#`rGHm~uG}tEOqM+VmF@$cT zB8lNWYB(=)>j7z6CPbBh!(zX;;6#MK>99Q$ZS)s8R-5_E{`V5W-Z=98Oe;k=?OMlN zT$Qt7xhds)X#-UIlvq)uXF91FrvQB2@(Ei1Gb2!f=eWd)!x&AMI8zoTrIo0PVns%^ zJ>x070zxtF?iA@_bU>8@w& zE)DFjl5Gp`ORK{{MG4K(O$(@-LR)JGx*kYhG^=N_2ju8ipd8AO!2(FW?=Q|sES-4p z)>4%zC4I*}(0px}qO~z~$x7e9A9nLu2O8YB?U?E8+fv_~_12ksoUMhfRwJWvXuNgn z)R@8ZGI1z?oN*S-Mx~lNf`swS1SyX3YXd3Xr7X{<@*^fhG(I zBAL_7ZHZ{p)*&1pcF2D`!lKnjs;^R79g-8IK&nxXu`}+RBl1|{#iR1drc*%*T-Wd+ zBkQDDKt1x&I)bbI%Usdn-xb1|8i|FR2&B{67x{P;>{(h_8`fAui4JCP~SO+EV zf=jq}$w)WpLF({eN?Li8K?4Zgt_Gmuyim2b6!E+$$c#c?hQFbanZ6mHr7&-h-BH?p zUDdT>CHZu3k4i#o#O}<9b{3ob$I0RxX?dHYkHZ>{==*u1v*1T3{mzXPWMh)_3HTf>ExW%jrdsg78)?Eh6R|vt3LBp-ivsP1DsFhh~WyLrHA6KWz>9AllDobD2*pG zP+;TH{l-+sIHSbwc>g}&po=5bC3U2C(fJ@}7GNhSI13W4f0Eod-hI02dZd~qW0yJ# z4+^y?HdY`nk*e7Hm7(F?&WK?mcr?60#h|s9mBB*@Cj)zwjur|#U$EG*mT4x;Tc1P< zFl{ln>^eh3`69b5Emz)jp?+cW=UiyECi9fLo22hA=e`mO3pKUK(~Lu1Yt{l*NrmTh-;AIQadoaQge0B12NdBYX)bPvV0(bFlFto1UJC1kxDv z3A-ukG^xa@4XwX=mPqyT=NrL!PIOzy)&4T|?kxek)L^D&M#!o{@#hRe;VdSu%6Cmk zUb3teg6zxqQI7eWJCJ9hSW)gD7GP_Ks&`VEI>hnh z?w!*8l(){yfwIDj@y@%>FtY$=EU_e)hB6j-u5C$grijVt4Z4(Z2anhEa$@#YF88y; zd8Y>AbNp^HX@YRfrl+C50dvc$(09d^UoY6W7R7Bq!xarT1odgvl%$&w<=K#>5h;HW*(pk%Bpzo_XQQ3NGQZ7zL7qoEk zOfZjQPB`gDy#z}B+j%WY!Ngu@{n4J!fJCHMlBMT9m$4yQ_T5id1C}BD#MHuAo4@l= zgyF}Klcc|2^DcJtG0Xzl-eAc8+0VirxX;lF#gH3HpQ2);$P&3n>BP%lrib%vRn%M= zS_MfocWz~sKZsWn>t55%?N)zbJ`_1E-V$V(iDs;Jr|BUhBTa+PM$59Ydbk=r?^Q{W zUy?QTor2>tVDJl63Ne?wX)q`E=-kXZzzsFU@orWW-iNp%hgXLQk(#{l?2mo_7DT;J zrOH&F{q!~A(Oy#a{EO;)hI|?w`fmiyZofDfRkb@QJ0Ck{W;Bvh3Kd@(eB^#AxrWp> zwb4LZVq1hQ6u^kQStC04#)gzVh&nt>B0x>Vx@lM&A?i@N>Jtrq3X(L~u^ScLruN~~)k9&6tWDtKa0 z50leD^mV=lvA+T7Es-^#@F>XAX}B_p778PUh0r%_1fG9%5gt~kHFw5MA}V^NbNG3# zjankF*Z=~rJ5d_COPx?L<~MG3T)>6$i#FFltUfh!TOyH{27FU2^oX8%W0pWqI_Sd0 zGlNxtM!xrb&T-H7 z4zy3T9bkjP3~*yj&%L_tS2h%V*HG;VLJf7oH&SkCFn5&oOo6(=v#bSVaIK6Pir9YU3%7t z^7@elfo3q^+3JgX&jXAu>UNiys;e@9coXjsdpM;o{piq|QIg9+nE1}MB zw$k4$#XSA0^`mmoW!incMG?Pc@bZZ-Hj<5{zME=s`X`W2;VV(KAq9S#?htC$SB1Q6x&)l>iaC$qBKV8~*(Z8mM zZ>fN2rB7EJe_u8?fA`S>O_D^x@VYjI%#7o3VlnC zs{Ks_#0T@JQ@7f+CG94-DkjGov&QLIjT{ZOW7{+i)e@2(PU2F}y*^>oHix_H4!&sI zE92#%<3Sxq(ewGD@U+{}%=+42&e5~apH!J1j`*D$CkeF?x8 z#~w3-yVziFbxV<-ZM#0J`CU?G#Q=v|Af2c@l_I)qeCb?KuJ*Q!ZCl{i-WAF##0L?G z^h!*AQInL`NVRM`IX*WV`Z*W^#DjcBY#^|`1zgMnqscssGLCi9=kJmf0w{FONbHIa z&8@M^g%qb2Eh}two5!q7rn1*0M~jViq(q|!QxGoj+)uF0qHr3vZ+w!M4AT%;r2kZ# zRlmqeVhkB|Ws<9ys!D!5tf0GHE|l7)@P)%ildJY8V{c9@6_|7;WU^ zlY>>4%FmW#jv&k$dvWST!5sG;80(zQ%;i+=!{Z)XkH`wQGo<83hTM@L?yR&8(C_n(lXndH^^$) ztDOQw!N}`~0}WKHa!Nim#I0^(#BH1t5!I+@lxbZ$edvz-ljWxmwQk+(iw#Z}T5>Jx zz(|UKDIo$oL%Dhqr_SPDS*I96D+@P*{;M0}MmR^@K{`gjG2NF^klS3w5tQ07ZbG6) zn2prNyT`RvYEpK(>D@w?+*PPG!>#)u$~?B?A_k~|3~{FYIIaqGl%{g=rU0(3$%|{dB^ouLUeFU2(F}0DyJnjcHA^P`vbErW7@%#U}Ls z0L{Bw_<`D7Rn8(b$+)!!$k7q2nLXd-EPN>&;l_Db&v1ICc@S>zosce^Is8HW5{08+e<=mliln^vEsHPRm!EjXKs?>LK zx01mYD5KJ2_tA=DX7xbFOu-)ABgm9oFOJ%AF)C4s0sw=vh?Kz@#8(kqGF~kkcL2se zyoj7}6w@q;#}2ew;QhT$PC~sPucyb>QdeoM)?Wi1HMsgX0HJzesA@5kqARF@T_Z9` ztxtq%dzH;KNNz9pmesCpt-G&9_{vb%7Q~|&M+pP39k_^`wl8X>Xw!w;u|kF;Os`oy zmdGGN4TUg}b~CoTsa|Cg9&LZ^hXHBIY3WrCj4vq!W;qStR9% z60p8*w9+Ln`Fy%kDka7_GGe;oY9wfMk(EKisDc#pB#4|VzMEHFPBqk&UQHuXHonO# zYve^k;5@c$t|M&4!IVUKpvYmQ$b$@Mch_;x67u2i1R z?_!nf&(66g%tQ&CgrYYXns3`mt~pfANykQDFI!f}8zjShpZqY>TKsThbv#LM3XLZXCJpQtNs`H^VE%7ax*e7?mPPnRss zUHOuDqtPeDt)S7BNu43`Sx&M!4Z3%zCC4plIb=EnYcT^^LR$zl??_oY%+)$eRZO;i zEm&uP%9(CZTfhmuI)mGkusf-buB*NNrC(*2N1w$IHQetw6axlU~MyIkwfsQ!fMR3N6 zv8z|_-wm+?IT(-)J=z>K955a0)OcUr)-7o5#U5~gbmr6#aQsKS`xO+F?woM9SFtx{ zJKC6DwFjEj7)VrVjG}SwkQMP5Y*B|JBb5q$NtffL*^cw;UgF}ar;V&q>BXLR_o6`G z#yX&MGaLuJDoHV)js}=uQenAjgriUfdp(%1N#YzKjv+AQjG*8Y)G-7MfsR;pBrg%+ zw^nh0F|XkcMkY`oZYS{f2~xY4mtDEkl@ZrwVIsGZn3TwEk;=qS&06hMtMWlqR_0x4BN}uUK4MYLgStJ9&T2 zLz;!@IR>z#=NOM|1$U;s#IJjG78!CpvEA9Hh7|HDt;#JpYH7RcMNzp(8j+;ev^G^; zb)#4lg>WZ71-2mdWwRiDQYO2Uay-v;<3${$5DX{)u5rak)HIN5?+g|gV*;ZSh%2o` zPaXJu9=s4fP1y~jQ1bZ~4#*Hn!%3VPpki}O+~`2|q$qISY1A=SBI%rhYq+)fV?pl? zLows^?^2Eg;i<8_j6;LY!m+5X6vm)e3=)Fhj9%F_5f=$$V?!9s9bMG~1HT;c-jeM% z7-J^or#hPZXa7qjJFlrjvy* zBA}RKUv7QtZf?cx%kD%2L(jj4M#+KH9B`;yKd!2a_9<`C#HaLZbn$qS%maTqda=5DO@qb zI;yeBb1uD}JMs1^j4e&gawbx#J!&%`t3^#j0J$Qh1#m(%{{S__K-NydPinwHkXK;y zuEHV78ActL1C&Pkc;KOX2-9kqU^1rVm%)H3JoJp5ksE)>C?seTKR!Xq#+Vv|;gXaV zEYVTG+%~p1FLQZX=NLfH672L$h+|r5##I})s9NkY{QHAr#l!ujwpq36H$u5{(}tJXYP zHm*!fncd1-Vg%(u580@4rd1WdQkj~ne6X2C+e}T#pL#Bnq-J!Q4`=~-w5mx;_v`#_ zO2?t`<6_%wHpDH%n$DoqU?&YCl$McEiOB5kQ}JRgxFap^mne19U3@A%2?EX6=Jh)$ zPLi3fH79AVBECD)my2;Y@i?&J(ZUA*YEUQH;uB^a+Qrdtz;Ohbm8 zY0QhHVh%HPw(=N`%T8y!sV&@#JSynxfDvkK>C=pyxR#US@^FrtWm#7w(HqD=Zx&z( ztjW2zCPbhio_V~~gg98L*78fHiOHl}Pp{i-9KXXQUZO zHbiucYf53(s8f_44_EDf%vim}Ua^Jbp=};hZG4-wkK=^oCPhv-QiWLJGi~_B^^4AY zE*za30-Ta4bdYBtCIq8}0migRCT-?zG?7V4G99XnO>^F?+1t&C7?Tr~C^vgCR!XJS zPfzY^M`W$oHx{|ma&Uy98W@o&Xv7BOL~-13#P3!j;ABk1RbB~zK+`;9i$vmI3%JgP@*oiG<#J0fMFEkoWTRW$0^zKfYU zAwVnaR$+GIjT>X9m9LwC$6Yk%o=CRd;H<#Hb=*}V*o;I-ttBZA>Xh#LQ+1&~%f@s%+$4_bdsBV!h;ZO5K@o|-1nV<59%=w- z<&jBes|>NA38F2$UyG9`2SqzOCBG_Ry;_0XVWpurf1WQo!o0{|%s);sYy;etn7MlzX< zurk0Lb0fRTqK%`wfi&+1ihFingNIyy4`=J`>r<&!)ox9YvK%d|2HKINOa_uW%`1;m zxhVX$uxYA5SE$#@poyde4hEaX!htzJ2#U~VXvfF$?&RI)h$vc{7%_c6z!Kom6Hjswytq1^01I7&yQ+TG%a#7cGiUhkm! z6tNl%#J?3hGNM|ZtBX zdD`4sfYZZCrolso(NXMPqPTIk-HC&d5(jk^6QZ7&8L8!RGVNoYXR4S3T4cC|CgZw+ z#+mQH51}gWcr7F*!KmKcjz6Mz_XF)ohZ=FGO-7mwVrmbvi-u5?bw)HbHr(dq+m=oC zM2I-kzb|5U1F@vysh4!lrz4s5Cmvj>xkZLI>C~liM@M+e#49j`3GAE?em)8}U=0dt zVK9VV5tnsAs*tk-_*HvJP!{%LJP-3%lnfdlWWmUm$Fpcty4ZRGwd%l)xQtb|Vr3jN z7>}=bRVr(h0raU{!kJ@CXfp99)@qaUlx^KGI)*#8ZLLzQPBNu8sNJ_|(hQ=f7i~); zD;(QD2ip>ftuV(mWl<~(co{R;p$0;lwE>=I4-mUIEkhYh3XBH**wABwhq0M%@X_xj zj(D0axNt~-FxJZhrE)F0ojmD4xYQX}xDqYK< z)P^`Kc&-jha|4Mv7-v-(t;9?l&%XiXRi6f+;s<)aWIF`yK7_r2LVPlGj)NM|YSc|D z{4|bUDps9t%j9`(%2gGVL`+E_WNRF|@=r;eUAF7xsOqgquI%ZzC5lp(4^E?3v2a~P z=LaMET0iv6rs>;TNXkMW$o~KlIrXbI4N10VJPjggo`2?;_;C&Dlz|x2A-QZ)wbQq; zsbO(PYTP3M5HihJn{=j{4Yk;~w<1xf+N={r2YZkz+e1VINr@veHt*TI>`+P{FDY$^ zkAM>TYO#sdGV$`&M+uU2oN7oE?>Ksvty!dXB4=i;>f4f$g)6fq+l}@Ps#5@Z{hS3sbqIZUZNx0mDhdy-venw`ActfGJF#40dx2h3GH{CEW#v}j1ANZ>Yqtt2ABkGkSkUJj3WzJhb28LSX&+r` zC>5zdO|vh>PW%blxMCF*IEv-qRgZdd0}Vz#sUww*z-ui%%dG)3sHJE{Mit38GTfxt z4ggQ^*I>`T)U3M=5Hcs+?GF)>lGdfFk)LX#eyShk)Q(ps^uL)!n4U|O7>9818g})m zS*e`iID-%c4%)8bJ5G$dMEh}A!=oy0S-h?4$r@@gn}7-cM*Xn`RBFAMNK%nKM_DxW z;H}PIC^uAk5A#y;>y1I`OW51naNhp8I}Eg%31ki?fl<10cGBhGU}I5QS3jgHcJ2wu z>c*rRfOCy&kx??(cU!J}u8JPkCV*FcRMNJk(Tdx#WA4aVoXl!P0U7U6unw(6k=h5? zQ;Zyg#8MhNu&1e0)FU~z4QV5VGEziRR-Vf*iEYVI_)}p$bf=X@=;ZTZ6ki0@NMaha&zG1?HV#l*F!OC?eJcCKTk z$O&vY5#Ugy)T3*dndQ7f(6Zx=oT6*v9P#Q-Htus8jJv3Occ<hXIjT<_B@^}atAvG7;tf5Dn2;)4x*!0T*Vvpx?a1JL zK2*;kq|mI1p#@3C;=|^+?NAuhnq;HROx&r0jjRf8N0$!IWRB9B6`Vwd^&|A9Ym%>P zJu)!;Af>Ptd2&pTaPsqx5aEolIzYhn)hQ^Vau|tk7#{G6gxV{?n&*O-kT$8p(}z-t zg%J%7#)nxs*Kqpeu8~lnWNMV1^RSQXkEK{x3hqQiY_EdiEeVWdPj+5~R_fsm^%yZuKc>f!F0)q3q$~FQqo!LAV%*#8=zpPf3B(u4Z3aEswZFtO^4v zs^!-!T}4@tFlG^t=|ZG$2SB>PmC4oWSYb6vL`F#$mAbr}Bu4k`~837Qm$ z9@X5XK%{|{MwI%@YHP7H5*&u~m^N%ic6m0sGUARQ9d}CrZ$OdtTgDyFDz_Zy;Bx zOwu3Sz-iu8ti*E%kD*t@!V%w}eoJX6n#!Yh){ATlm$NWzQkB)@?@jHwI#loScqbNR z6I$cril=lzo)}@dNn1+-wB4SdMq_{Zr`AllFfzpH1mWD88@8N=ts;jkLYTb>#+BVb zD1L-s+j0bEPWhD84As~W0Uj!xhMZQdrk7gIC0J^co2U4^ia&PWUi~2ZO4~_|GAA=5 z=nDZ!sYjXHIAWa)M4>YQT7yCpAp(_8b_T7Hu%xg9b8bnZBOxsD2Myin$}()sVk=xR zR4+g7IxAw?);vO3!P)X8+@o6R!B%RpfEeM_vUv{`-EgmpfI5$AB>o~uJyO%_T*?|L zrY8`%2M8UeU)lv;aOsUu5Wqy%v?LmMcP=@S)<0@Oox^&m%^K!&sjsL7EGJpS(<*ww z3qd4p*k-er;`6fJb_EZ~TS>}N5pYI_alquH;uYkA4Jj~Bc%kyD4~K3TWJZL68iR)H zXtgWt@+^)xx2^?7sN1cPQJ9r`!kovMaC`oWu9*-fT8FuZf$C8;Dm@d`>Q;CONw!58 z=`KfP>ZK-L9(xjBcWYowc7yxLvff05j?I8Oo zxt1+c%`)`wQ3b1>+fb0lAM^FAyfyPTV-rry;r*oFZgsby?ln?c9$nxc#62NJEt`Su zPF+q|N3u5WDgbIKaaG9zB$GffB;WwrN=jdS>9e&47XJY1W;e$Mhw1(=(Ww3<>YZ}5 zR^eqbBN~zb8Iyn=X(Plw^V!Tsh0%usQgj=p@9nG@ity{hXmT3p&|I9NItB_&~Pbj zWA}_6#o*F#0oabu=|pAYA7aZEDGRvHnW@urZb8_IHz7Ejj7>t8q;FV4QQUWQ-#LlE zXG)I|iu1YCp9v4ehtRzqF6~GzB+n-1Q6A|!;ox?78imPT8cU$2Nuw5s>m)8gs0V6| z!zj?i({kY=S(J3qM@3JZ<#VcYgkv~@w!n?1hQ4J@P6Tc$%%CI)+v!j`Cfo^cKJ^?+ zlp3%*@%jl-)WXmJj6m?&>{}>AicohQiXi3-d9>WL8qD>;S25X_h*aL3nR=ww6B%v< zD^(cbP3OHe-Bgfu5)XNxJ_{qL9@Tq^6CDHInbJHCex)}uKU%8LJlADC&>Canb3j^* zL}DT`0|mpA(Fsdpkv3krgkj%Oi%>Sg1?NE8aq!-hsj|tuOwWK$i)IS96VTqma7}H{ z%fl7jQanxSaZtx0HTLY}R6cuId%L*>>X{ zg1*$ANiswSn3j?%TsR!TtWqe}0o z%~yZU^>r(-h|-%Lsg?tIKs`mxEk=VVk5EjJ?N_>mIf+()<>XPh2nZb`6HL5OI~qXL zbxNJfiM{;{XNoon_GFcJN0qUZKxoM7^s&Xr0R}uaEJVKKPPs2?* z)0Kz2F0WWs?*9OGKWoTW7RL!!yVMjOfG)p&d$jrjoD+o?WLFiiG?4B;UCTBA0G!ol zr6?HbI8gn43j6LyNb&L%-8#FBcLYWq#!}^j=Ezg6UK7^f1mJgdDzT=B1Ls51p(p)C zc2~-PmhLf$i5u&eQjMWHS4rJW4HS;YnM!WjtuS;qq^7`d?CS$uMs&ea6QNr?M2C4? zGBs+CaP*=ykkXY3lrtw`SNT%$>i%jTPiqo!?!)L)xusc}%oVXZTnc~7fN&)Jd{=yk z*ApT2@L8C`sg()tkSb7VDsGijg^2Da9|EE+N+e@n1h_Wb(sAz+3e1RY__!%iLIF4` z?LbtSR1k0(1MO2uu_6h~E!qaF?k#!5CGzg}cQ+o(WAu0T8C0g&sMZGsFvWU>jGQeq zi0@SliCVDIU1awzRt2j-A!@~>K?_zPGWDhH&6Kq>G6Bvf7|X^!T+;7{@#}l4LFzE) zf9~brNyEVVO7m1DjZVk6RWfl_bwUme#**+kdk>`{Y1`Q#pQ9p&(#t6u+6Cz_P%3<^1jV zOGHUKg;a^Y^}r$`KM}{kEY*RgM^ucC^=jF!WRG&JAtIiwo8`COaaKzNX~|$jVtrrD zRoR%MECyylC4!@K$)oPlDHWgRCe*e zpRNkxi(r5>E16L_Ee3#9PGv%(h9r}<0cojek9o~W%qkj$NGCKXZMMZg$He_A=CQR>xb1)jKVJ1+%qlqH0|KiT zW_PW0X{A&lSdph@aw3&hqb)EI{t{I+cb6=D0y9V@jXQu?6^XYYQ-Y&fnj-cvd$wZY z=3%RLB=o~&*>>4%MiG=ENiVw8ed)wd5^+{f3F3qP3S08Juw|C6r1L}G3Pt_*FU_J-+H8B#~%=spfrplDMu+94$0i5mJawlKb~gg zAd#6LxV!%VrMdV@yqa*I+}(fD-24>H%y;cw{cQgLyr1}7XBdGPst^`#}E6C z{<44GS!iGTb{mjTJ|^G*F?F4P?Jj;2$ZG%s#ohyTrN_eyjxRX}SrL??2y(h)W0d#l zDT`a5YUuv}{*%N0WwL}K^1R)i24q5>Gm!rPV(K(%gI{8+_Qt zLy>on{{V||@RJd7-?ex3lloKV7@XkaA~KXg4%bYy*SGv76e+ads||GUH!t(}iGg;p zKbO1z0HwJ2OOdacBgnhN*Z$Vy9Dn$df12t40Cn%yPy5ODh00P7>XfBH8fuiK_#N|! zSr|f2*PcG!10i6U0YJss{{X_=d?Xo93Ndst{{V%!_$e_LT`>f@?MMgrlh35LnCIHN z`pN$Qd1hFDzh{8RDaj{0d$Wi5Tc3oX8sj2E7lZ!*3v=aZN5$tTI>?Nr8uO|nBO1}q z;XUb-nBe~ab>G%cf9WliI!W2)?C>)p6k4W#*t+YrxcEyAoZlsao@Ce^^^Yce{P)n%rdx^J39@iN)gK3b6s0od<3IpOi4dR zE4lzXS)I8bSCp`>1R>~^Uo**%$pHX7H|x$Ms) z0D6a#zO^7Bi6sLhNNFdy)7Vszfh*T8IZ~+uV}2?`up*3VtY`~`I*}ujl8A*ei2)H5 zT&oL%4T}u10a;bu4#g7FPS;hYAkjp+su;y>lg0LHB4r?HBbhtvUj3_I9jfj0rhXjm zhqRogJ!2&KaT>%Byt>iLQ?#X&qnedM4mD1qF^~qHr3Zn9c$F+9Vpk4C#)c!sSmngP zNFNb`t>R&7OgAh{LG^kSM{T1RlL3gvio_>O_9=D6Ll7tRF5qH&Cn+vx4KpKQQR1(; zGiMVCN;?YWKHS128B-L^0oxub$*VSHi_OB36;|OP0f130T+G5!;VPW_QmUC(+K3tg zwOlDx1angrCJ9-oGLb*RIYEs++(641khD#<==4qcdE96fi)#y%?e0nq-iiUT<$emxO3bq5E|u~P8N~(if`VW z?f(FFJL~CIJq<4|DgutBZ2?u0DZV8+;4oG4PJ$y96q|>e=~G6P8^%#iiM{{{dsVGP zMd|_UfK-LEl?2laaUdrVngq*+hM!*=rT+k(W*7KL`FqVofA#kR4F3RqZoK{x>__%O zTyVRIGj#NXVJh0V2$-x$FI#3SkAcBD#_6il@*Fe%ji2tWg^W>Rc0AaF_1KjE0DC|C z*X!o_Qj!;K)PJGjyT9Pr{^~<<{w(}>jGZl8&TlsfNX?-vM5ZvQ!n^zw;T?U6UhHcg zDMi9T((uFZ=lc$%pY^xD!`XhIeJWV-`=;xqt60}PEorURf+ZfBZF<>Ic%Sojc1@1z!uGi~j(l{f9f!dIPf@KM2|sG2{18 z_gS=$?6s!25ssM*s-yn4{{Z-VFVqjEXD#GP`drfCqJ85Sy3g#r)$)lk{c-;Q@b+J* zA4=+6e(H48)vUj1tyHP0rDRsBc8rzvv4xD&{{RZYHy!$WzXPc${cZmMu(SO@`c$Kj z-9z1K)-}&dT5EwA>5#5vAx_OZf=C|Bd;*gKsefC4{5_ZI2hxx0Z-0lf{XqIve5bq$ zjxYx?p9jCT6BgwJxb@SN*43=@_nOt2B>U1dQU3s2d;C3@>Ic%Sojc1=D)?8r_vrrs zVbq_6G{?XwM<2R{bhT?8x?0m*2>db|N{jm2-{I`PP(GE8n{Z0{To*rJPs>B{!r_nt|>^yq{Wx;gT}O`VpRkxfK`=y8R1? zNi>t5G$g;wk~AG&D8Z2{iQ-6FJbH*2DDy6C_fLAUu7i=|Ch*K#`!v zrEFJI{p7!Bu427~q9%5^TCqsbQz#U}C8dV0NL?vCk=|i!m!e!;aHGW z(2F|~Ac8j~6lJO)?MbBfY5Ttdx$FMa$TIL1PECqOI;Qxi2L09NvL|FzDtyUA(yH>J x=e$-TC5X|AV literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000029_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000029_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..54729dfdc4a073e10e81c655deac10de541e15ed GIT binary patch literal 2866 zcmeHIO-sW-5KU5JrF!<@DYvvxdh$?2DOB)aXexpS4-&S?mKB?9*d#)*gFlOb4+lc0nvzFh?4r3q&2X%pMdOgZG5<@cKkz4Db)Jidi7Pm=Xq~^ zPx9Y=k$zNuNFSsRtAEC(`yk#ID6~Z_Q1Zfo3WYBMMOV}UB`+NKt8b8BP_FwliSXJp ftmS9;AnOyY5`JGUPl_jr(ue=rgJLBUl&8)oDD$8g literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000030.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000030.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d16b9bda039c96331a60f7b8454be66cbf93c7c8 GIT binary patch literal 29098 zcmce-bzGax_6Haoio3fNC=_=o?p`beinO>}kV2uj7pJ&W97>Q7+$rwv?pB~hm%i`) z-S^(z-|k<#`{a{MGV{!tGe^E>o->oDxu-P%j*^_B8~_d;0Dyab0iGTK*ea$T=KP!- zJRDq{oPeiAfHVO0#S8Qo=$L<9IM_J&`>3IMoIJoB_{(T`KAi~2TqoBe85S|Lt`^Y~dD z9e0P;I+8ZQxfH!i9-&$M=cjf6E-Ktdcz*;qy65o$2)KZ!1pvlh%shXW09-N|+G4#t z#bNc7*@lTu#p3NIo?6{a6Bk;$)k9{|dU@Y-2_wR+^`}cq3X^VQR`7#9f(jB7-)h0? zRy;qrH(oyXNItd7?}2W^7FfVpkeQ=p^Erf%Z;n(6@aC%QE*K>y7TCxtWDiq-)N6c= zz;*T%Rg+y3FI{)G&P)`Y^7xkeO@~RAu|&2e_OfEz8{@}5>W!s~ zbn(pos?8z!Sq4Wyc*x70EZo5!Pp!eAjaj0CDG&(Q)JO5^g07BOw)`aXIH`H?*Iyl} z7)RMx#*K3eC*~*SZAr$zbnlhLTr@uVO)S)UFaFIUsY1++-37CHxDIo!Y`NWum)oyG zEpGx~@5rvi2IwCq_=6X~Sby{LH(AwOfY5*45Cqr_cO3*nH2ZCKeLlKxK7)EV;4}KL$~BOM@2|F=kQXyj#$N8%XJMn6u(+r92Za? zend}0&M}^B-Sl^0@=U3Qpx~+F z!njQ;?b9mlC{{gYKYujaHQ}&!4ebd^L4jLFL`YINNqs|c^B)=_obb!`O(-QYDsFHd z?N(mmGPN=l1Y)}>4xK0f<$rlcs`JbKa%cZy+UK9cqtDd?;QCK>yMs2pr{fCABKcq2 z=W|IT6@b~BA-WiN0;rp;)Z)CpHsiKrLqH!jc3UTi+1H#msN(-&5PFIgzUfPo zX4^2LBW%iHxICla3JLsJuRbqDqMz!}V)rKix!Ty+X27hpLx#v#3Qxy3IL&_vvVr`* zcv3ix@zD}$w#bLP1$4$By2ASIXb5qBi~EHm2eD9jhJ$Zm7vDY=qS}a+MecCMyg@t6 z#icQK_=e)$e~Mt%23jUHTCo6ejBN6;hlP(=o2C|$VN*<3x#~u&&KzTxlQ$Vjgn>53 z?MK+tz!v=K_Bbfi|0xHs9)zDyui0=U#9(|shC;P2tPKb627^YE`3X?CM|w3Fay&Rw zZdmKwihm64B6WJ@>d($aI)S8t!uFqJJmlpqmCcRTHea>m)p}L9nL7xOk&xr-{Vr3q zRrS~>PCY~(Xq+|rzAGr!E`ra#aWv%<5iEy>^cqzL3M?kS8UHHaBzv`{(2CyJ=XM3U-1}Q2_E%h&`+zsc`+R+ z{Oys^M6CHw-+xb$daCN)v1~|as*PhZMe4vW`cQ(q5Ap<%i_P~Q9jZB&7|*{=dICgW zd4&q6Y%jR$i)Al zcA)ntFugOAfVZvn{Q`IKsV{}q(p$q__ma-QiwE1q0#S zCt}LmG1NUHiSd>XpIrawUxB))E`}?ngE@pPE8MP8Y37!?^1^FsX3BMjPXN*&yU5YL z(~lL>#D&W1M9=V0=`?LXB+c>W`>yb;dokLlo^#%z<5Z*Qs8JL$( zg-882gX`+H?nR-;Mh#=)E_Z+RH+Y7V{KxTv>&jb(pZ(s+mjS|&yKVCV;@vqLx(#tc!?rpO*UDobRnWW&)_q1y$ymghfO$%%C z!-9`4Ga&a_;d7=#QGr22q$~vz)BIagkG40^zEd)Fo6|h`F%IWvHWpxeak(dy@>6Ov$khk8v?Q7*@PZ{xj-}-u1Ai6W&r|@39n|~<$ zjgWulVde9AZ{zETcQ5E1Y2_wlQO8zU8Ci0eJ&?RViGNhg#y(m*V^{d<6qSG14cYVk zBo@hM3>m2d@c`G@72d4rRhifv(B@4va~*h`UV4dGp3pU;E``#q>73~jID2EL+dw1I zGWSTLX0&VXe)pF}iSmk?)}vQ{rCgg&v*c;zZCCq4(j-oysIHy33Jz zH+{1Ld>*2M>e8;SZ(q$wo1sJRoPB+ipC(-O<}6$FEz*dD=Y)@*FWx`wMPobF(T`-^ zh@H4bLA#;7pB4EA%zirl!}K1p;KQckvC&V3u^Z$9uOGzuzVOc`{;a*$aO!`vCKuYs zM?a#`LjKR~|BS{f)=@&`61=zi@7w=NS(pl!iLd4Ux&6PGvysop^%nTwH4~^Mx{W^G z`oD{sEN=Iuar8fm^hNyRn18q?qgtDa71D3P-V^xO)NI2jeIsO zxJ?TBKXuiQQM=fqebV^wcIYz|&t9dt1XQ1p?o6-7E$Ylr+7Mpwf1Q?z4(G$ZH4?PS4Lt3Xt zWnxs{_1vCfy%)$3CU*QOZS;G=(PlSKejF3Lv)@vd75@5g`kq?vtLVa~crkP3k*@Vt zi6JDBH^1FDf~#!1A)f^zf_H6nqIHgil7%x2)77)Cj&;`QU#*|})cb`ZvhiJ6fJ_iI)0;d4j?sw75Wg}W?m-bpg^~HzFl<&yGF7IYIte1e-9qWx( z1f4(G>gSyctE|`;PeI~^_37@N;~W;N$vuZ>_v!*n6^sq0eY(j}JaoCJMxlbVsBhqL zf&l-Qf+YBx3WgMpKO!wE;JFZ{vYT0>8yv>&1lt|2*SVt)_my~vE|;AjuX&v|Bs$(y zF-PV){-zJT^U6%L`BZj7IAvrM@(!{i7@&ywkkB^zIk03`gx_j(*<&;mQfcw{{juO? zS)D<%h3YDpimG6%@Tc2jNFA#zo3G5nYrZAeiWjEc4n{n`)Bc_jdnS9PU`&z@sa)1_-bvRn(8{cnmQVlCa&Fg|v0eVuIBzR>3SO^a zsF0#Xoz#t}-df5WlENwX|3}^b()xcFNRb-;T&Pkes8Tq9AX5a zoImE~x{`l7vSElT=vu=)BSpSw=Wf9R+Ia+G&%q5%;p^yW(u&S#HIy)<thQCAU-y zx>pNTEt-{&4Pi<=Eu$}$!fE|@1&1ny_^+A#uLU}m0wecE2`vac0n4RxtHWi5i^qi* z%B@tE@s-g{j>$gRSv42kP=5#~h>5P!$%@%U(r0i z<&?|TeCMIhc`q-$s5Z!K!&gZ>YlhiO2umx;id%mw{vF}UTiwYlDZlL$Am6vSR3nz+ zfb=&(v|Rr(&1e68LlQ*G`L_zc{NEWYTo|y16EQ!B47}3isdCS4EfV|g#gSsaCSNJL z@$$=WFkWvFvR+gxO!5$=0p|3pK#d^|vFqK2U-936L!~qlwE7@FHzo}zW`*}QZ{{Sf z#MOQi4p+%LVIktuj9oc^m}=GBW|ciHCabITf+|W}R@kty$2Kd}Mzv093&oJLW{URh zJ7v%S&c}-3i+Yq6o5qufq(+yjJ%tJNa=6xft}S(D%W)Vcj#>AU`{@FP-iyX7w4$vx zdQ($1VhRiM@HoMY|L3_ym0H0Aa1u^kL()bZd%*%NgDI@CEhDo=4+#j!m)a*+73)$J zD)b*(zWykn51+Qrn)VOWcu$eP(ukfGIXl#{y~vbF$}uYY1o%+D=o-E4uTEQ64m#b$z3@Szxy4W#6dw9Ae1I$J4znY|_>OO^vy@~dn*VxvdMc2VVpW`V*Z0kevh&8;JP+hlKwzH-fYVi@Bz8*~-`VwE|@|DD& zFi{b#W#&Ujn*l_AfzT25t)!zJLR#dK;CPvpHWhx3wKL_BkrJz_{;oshw);3Cy4FLU z6cLX*=n6*c*ltd7^1~zB8#s~4U!`&BGKa-ym4M+LSspf)J2 zbp^vuCP=Jpm9iArud+@B-tZ2583&2Mv`TTBO9n>N)U6kilAbX*;PXH6>g7VTH1AOF z@959!tnmrZp46WW{Ahc$MuNqH#er5|`STW*=vQ`qLXsEA1k$o1uz~VWzvKPaKADSZ zQHg%tS1>(})MplRTCUWRUQI;oFP@AF7%AGDD=6ziOYaVjULA6KvHVD?a8(V_`IY67 zN{O)YRy`u0vf=E?Y?@`z5;{;51C@g$G`_vyI8{Nn10qUdSkbe#t9@T&53`k)`{A8b z=#?RlJMvv=Mz~~R{A)?EFd=B0azwwEsUmIn<@S#l7Zd0+nsCT2Z4JECua~p`j1#O& z*zp%Q;s1%eNbxm&sEqVRO}hy|Cp{7&v>%=MRf`Umd8Isc=0}k0f+>z9MS5m_C6A)b z#y+hWMWLj(>;zhKs?RiX;fZqLpCS8C*!lz6s8WBC{0G(%aO%vS8)=@w@P9ABJ&ijd zDR|=vo^DKE3ilzSQ_Xb+{5;HCJlcu)p0s5HbEdJbR8&hnQYSxW7r#NMR)nJa!W3t` ziUw(xnAZ0k-&C>cnUmT{laUF)q4#dsYM0rwEZcWbPUXHdyNk?41<5P??Ir&IVuO%F zpxZpXqrIWf2rU;a8QP#~!`L`_t`{E|#{wXu0U)o{y~rqhW#~tzNF&uFwJjJmVajRz zZTIDg%L*rLKP)R>=trf?`=m*dul&}mVNw&lCPB0!&tCD5zc5vb8d|dyI?*0fJ_IZz zBxhYI^(>R@>vB3A;RLZU;78Z*ZZVJA)RkFm=6Dq5jCH`73a7akBJ zHARCVH68Xx$^WAf#tLQuOod*DunltUMIoxrl!`sx`VPlXZaP)<^?CG|V!{^=^9_ey z5Njo+UlhW4$qjPHJo~y7F~`A7mEVR{jVxW6(h#rGtE$hgN$3+DSQ2R~6pJDFwa`8O z*WVv%oQp!goQNrjE&FASI3k|dt1}YgQSC(0c4N_5kH?p=1vqiRU>-|uuFKUTM$CeS z(}E3X)}-oGF&c>}nTw5w>b8sWypy9dutnneZ;-|4&D15&&rrcMiek@)*Cpu_C1uQE zw>ZJwZtyrIE}QJ(K7n!<<}<0pQVOtmRAg-3vl?-W;Qep9)&rlk`%23jKrMo0Y3D&%qU=*%YY{9$lZ64-@=Q zM9W$t`s97GuynsJY+g|PjPo%pft`cIq??4qZRmPtUl6PZ>3uqQf1qTjwwV4dR$(|^ zNPXZXaY#6WM7F|m6-)f(D z1-lkw;N37vnJ6mURHlcT46cNYX!`;t!arZ8%{jWc6h&sZGF(0T`OgANCn=n7Z~hMU z{}BzIBf+0&@cjO3eUJL=Txh!X=-F178E-d>El4Ouwcaq04wkSOwC;jI*!uc!wR$9K z3hW!hT?Q+T$*0`MzPtzoEAu%@rRvP}ZZ@h&O`vjG)wqHByIJ^;#Mjc`T2yb-o!XtB zqgJ`@t2(0YL}rT4E+qaE3)ON|sai8z&eU*Wd(~l{t~Bdz$IEr7$BZNT5H>;PKDE*^ z2Ze4LbGWCk4&gI_t}~4W|sD9^na&K!c6N_RSSX%FR%z6juYY)gI@TB+>Y0VmZy^ zbPI1_f?lhmWQc%l_$j&wdT zDr^rmf7V1j&;NKb8((7>`Go)YDOf&xz>?+L#$c3#{3L|{(ECC0!wC5ly=i?ahzEf; z6{O~_zprETqctP=Q@J6Ecrc^u+LU`x!Lgy7lowtSe>K@xJ{u+B;>NJTkC#e;jEQ-h z;nqu2v@g)BpLP2T^8W?g8dt>CQS>}n-Ba!&bcWRH4K2t2Mv@y{gFU_wMJK8o^D`VGf(+jXR_$ zHLMG*i|H?9rNSkZyR9LwTcVq2Vm%dZ?3(81TNLU_Xr!M2RMWO;Ig)U-Gqs;fQ0y~h z`BO83Xx+YI88#kT06!5jF2F`MDc|0|Wq9eVE<#w6rpSg9tkkd9_&2z-Md1{n6Jw(u zUMAj5Q;?tQjYfp>*2gN-;Q5#~ zAxh+p?%JP}A)f#!{Qa*zSJ}_RBd-MBTQ>$&d0^)e6 zXj;LlWq8lxN(<3<#~m2)t_AQF}LLQD!yV4Kl-O4cdzIkf_1m}{5aPlj0=4v{E# zuOANEvmkC$-X>bLxqMP(XWD4@S;;gDt>*U$$IA`Yuu(F0;o6SU52Wf0->+E|X=TwE z4jd>uIE^>c!;cj;eWX)amz&t@lf=CWK8^)LXbVQPp5^?PAmL^TAKnLGVp{EMYQ6xy ztwE2}J?8uBR*Xtv%G98Bp6#XRkQ*w@VIT!=W%{9svOjm6sTf;2QqU8K@xW z-Je^*E18bi$9yFZOr|V0|6Z&ImG(&5d_5W+pLG-uX1G@iOv@1LZ>i>B* z6kb#Z&x;o_HtA6eu`NymXfJ1Qj*WMcMwV4)`1*aDEHDnTLwH=h;cbS;onf(r${UNJ zUhN{Iai$k8kV<0(SRK(~uXm#5L37+h!jGP*5?N9x_o)XDXEonQKVE6F9f+9xOI#Ww zI|*J=KjtJzyrd)P~A4)?6!)(<} z0{4l0IdPh@TEL{B%ozVhjJ*sk8o zmDP1=?MVqg8VjaY{AN+6Zga`q*bDoX(Px^g)kSrbre_oHbtF`NqGOiGd-LqMdSrSO z{iw5)o3*sGg4ZFb$lBIL3bg3j2sq_`<)}!M^(S_5#t`FG0MH8c15CyeduY>|*_#vh zjm`PA*xOJ|ze)B55+oR;Tap=vhg?wZwo2H&xT!Hil8G~OS<>^PCAqNcb{G3`W!Ln2 zqy7llcGgz5!{jniF8YFKHDXj0Be@=#yw`=tI8&}n?zciV^lu6kKIw6^iVlTcTx=+6 z3gfCT5ItV1*Lpbf?RZaCOmXhgD&+S6e5n`NwN(BUVB9ns-Rf6J+c4+ZIhf4#P?`2k zUhVoUtFX9Je@nWFP!c6#8f*FULJCz6yV-H|xY$mvsT1iJ5#0ifXQV|F2!zX5_BDF( zigANWBV5&!sK)wHbN*RJ)d3cFI~G0Va)8s$JksTwQA zFkn!7_uuI{nuDtaRe<%Bv3;{lh}lt68Iwx%$Lzw;(Z+DQEkfG&<}_T=)1qNgbP@eI43HS9^Qkc zqJS{y=?ovgy9#!#a|4-YZ0e9ZS@8fSo`Mx2Lu&Av2VYtR+scF3EHJr;&fi(2KOmc6 z`B=A1>-yKt3Fxtb_-z6_CZd`~h2I`z7du6|Up;hR>|Ur60Qrn1!}gkd=fq>`($rRe z(`DYLXfd1}-e)9Z!NN!q$wz?hbltP@oYP^LFZQ?4fwVMh@P|l9e>b}lf29F@4m`gf zUY?fihu&KJWR{d{`yE5>JXW>4R~XNcEte#t{?fiZScYMGtY=415S}nYS|%~h;u_;Z zT|!4lpwQW6dH8Be@K~F(8=r+U7rhjB!fd&dAbG!%qTB2G+xn|HKYoP1h_ra;c4s~Q zNpy13;xADtk&)^I8+3>WcIc0?#%s|<4F6B*80H3{Kdhqu;pS*>xC zbBZvqFx!|*tRss|=2dw;wbEdPxnl&%tw8Flajtw&w{AG$nhCdJ;6XbTZnmJ4_3UKs z(JVS^5U`8Ne5f{ zCkbnEaQp4hQih=ZEZOs3wxK`e#RsKd7E7j201Lh8cr|0v$W0|rSgv8m_tEg~>)=6= z%;Wes#8^%s3>7UirD(^@Na=fJbuawTt}aBG*y!~|ZjmW*)})DDj_o+m*Tt82wUs{Q z5NWark6=ufiU;}Psz{56S`|8kjH6vq8hK!#&lMV!TCV{!EhR4O1P7Cu-nd|uyh}U! zcgjuUET*ygUPE2Xvd!faprEP#t8a*DKdKW?U3bRb(-=7NPhRT%)xj!U#gNuMd6D{RR3m|*@e&jP1Tg$joVf< zACu8F$s(fi$YVt3T^Xl^(*7)REmy4!^RmxnOi*XqF;8e0mZq}8uq`oVh{HC)q8~!2 zr~JDXDPX$ZqA51Wmi(;MgNc^r(CbDrr}0paV%Nw7Y33M`0(e7LL>iZ|L(KK*D&*yD zYbuV4S>q<;Bm4Y{*nP^YU!}a?la$;1j-qvh22pQM#qg4rnh$kuyU<+SZcP*zE8P32 zuAQ9tX~@W+t~XBr&CALhVU9@lAMBh0Ij!%6;8vmrKwBa>i0e_q;(`ZJEy@EMpPXEO zB+Upky(^hk3a(ADer;6MGD5uSeL{7H*|~tc{=;T7>q6@z)fyMDKf+8xoT8-PHci&% z*U2T+A_rfaEt3jBj?yBE?8I#|Xg{KJ#42 zYBbcmX&n(3Eo-l{2c?~G>2aN`(~`AuxgPCc#2;=yWx0!sRnRYQ0G-*b-3DE|{L{MS z%!E_Q+MRI=mW&=9SoprrQB0RW*-pg7&^1~ME{Fc)A`H&LdExYZx(gnxU~9&qoY-g= z=xEGME`sd@bGn4gd3b5S^7Q(59?`UfEQT1@OIbmK%3mKU1}$;nswaMAQSP=X&SknT zFkE=3mlzk?O&Vd%9`r|g|BNhW4D1=J=h!F+{`T`2`7msUWmO5*36VTrvx8}In>0~1t+H- z-Hp2D2;w4|E;nIpf$SScvw!}(K`_;GK@DGQ;F{5YgHa0qW5QWGJnu*z_&H8h5$00m-2YTfJ2U29{YKCA9bs_k`>464}DSCE(hzsVpQq`OGG|8R_|8G>ga0K z-%+)c%otA2pEDN8K+M9Ie7d!V81C-x^1kC>BG9cn-1K6TqeHb>YkHP0Kc3m%f0a8d!t-?FdZ!Htn3N*p^J7Nx60%5u<5> za>a9(ZT^4*d!IyNy%Sp_R{)S)X^J0F7v zHoXd?GbD>Pd;&P1Ho}qAEQxx~*d#%&4T^kXaGU{x(51Fl({539re~JTs685DKdf!r zsZkN(p^GkkT=n(uBY&l*`B)#m6`+LXH!S5tM;we!if%8p7j{}I1f9lIb_&wb=16wo zfqgilQAW-z?VMZaa{X4@>L3RTb+f{p!Dui%x4oi>P3jNv)#!6x5lkCZVTt(9kB*SN z)8szKAx4f%CNh~A1~iKB)EJE8_<$097U{pI8STDJ*Our1I(tF{-l!V|2#2> zz7}itw)rZht8Q|l=1}&-Q7V41+(T_&5!FE0*%zs##u<&WfodsSi@d~X2>!d<0=p`g zq-66U2)MteUVlf~D?Gg%RBs|^_qmY$wa7KremL!kgBrbsE!B+#&w*Ao&@0RpYb^7& z7B!d6*I7TG{gsMT>H|-&oEL!<%?YsAMjvNVKG3yd=0K=Vch?yfq%!TlzVH9iJKi=j zJ><=|S59$!#bt5c$grQ$qmjxVMFY-^Lelg$tUjyFL{v$z`^!LXs%LU=yXB<(kuF4K zSs^>ivi7Pihs6lffm#Nnm|c2g0q}rRECjFAIe~KP)gWkYO{)T(@XbZ>F|c<@m2r$k zli>_E!&$V{FGcP({9Jx|_^bkt{SpvOgiVbt1;H>JG3rgd9BXty5yP3NPN}m3V=djs zQkmnZOQkG;NBs1ndFx$3wF+Qnf+b06v5DC;xq@5yl<(BQsBUz!g^Vs{jPRO|F03Id9g{mNHP#IeejN<%?vep^MsEnkYRUtG*~Az*x>3w;E@7WZF-xQ4exr?^y8f(yX|++ zKy8LD|8J*~T<5Q)CTw{ZX3T4x#2}fQw`n%S5&ZLCIpsZLxOnN=x5Bl4Hmv5h4qh+o zsz8QFCAyILRbCs0JaiVa$$b+1**qSs#H#<^$A&m@`9XdJJGF5}bIP3j3E+c!uo3@m z#5?^7u<)+=V2rp)n29*|^)ky7pq%L;CEGO7)4F!v$a@k)c$3T;&bHK^>e2Va8VmjFH zTiZDZ>hZBys4MY;BlND2@#jRswNVOE=hW(rSmteoSk*DdyycZx#Z}^0*K8qZ_G#CU zx_gyqjy&sy9JCj9u~=gs7&Q7q%X?O|&9&^YI_RUreRO_DOT&Q}|L%bixgR+=bB;`6 zgE>yB;Wh)*VqaNJ(s{)3oFQP2GDRQtZ#wE2^bg$8`D$lI8a(s_aN00zg*4&4+nRTZ zTk!etV4SYBq~}mSx6Y(^0AF0=>^(YF)r}Qbz#At1(!sTzHzV}ic`#yYh0^82WT+6s zHJo_GemR!{GdQEP;x5=GV(j^RnOHO>p~0PdZlkWW3i}76QKG}a6#~*?)?GHw&zE>W z>yO9jv5xwN0{c@u-{p)_=1ljW2ZmuLfc%Hq_Jx`TrGwWvL^cS8+)?CwcC-pB_>w6+ z=4Ob}Zfs#(!2IW3IM-qOnuE4a6o-Sz3iY z*FZ1%76oWju&vahNnk+l7lqW(AQt=DP;ON`7n6uYF!VVek$8?Jr_jwzRsm>chBi}c*$S{6HM9`cgf)p7|80TEC%5?G~y;~Ib?4R-)!1C_yVkfUa#F9aJd<0sRJ1e|tc*W6-YSQSvc zan|_DTx$7|id48c67Z!=FxnB)1{90>vqMjRht1}9PIO;I(~K36qB|giuVndUJ1Ws& z)*453YpNuLcl#D_r><5Pb)8T&QmtSvxgFWYbDYs=b{>nHi;9zx7U^!*K+b-Y74Aau z`JAXFy0kyQiRw(Mvg+;Nc_2Qx2M#%dK+f7h6h*N>W;^?<97!y;dVl)W;M6$>$fF~L z>3E*qQ84+Qoq?i;3}a+}v-w+!y-Dm>7#JXr;A@)#W4ylmU0CzktG?vU=`!U21u2K)LtXh3 zK~AT{=l>m?O<)i~x97QXp#!2Hf23%M%na@Zw>J?61_phmU}Yes2n#N3FQ6k3Nn6xi zw!`CG`a*@p)Xznbm65DJW7v$w`x)^eMew&!>Vm21XmOcZ-KU9%bZO-!6{!zR>1tL^ zybK|yZk7R1*!Pm_j5m~P6k(&Lc#x<~Mo&n4PjzS`r!FkLXA!Y8;A&B9XoT5>#$s0Y z3zKc*xXM8%2@E-TpkFm|<~+ml!vCMC4*8-y;zSnTRZXXx`2!0Im7+9$jW z4{3FZV%e50rV-2)I&;LntyuY>YCpklJs@F8?$2D4yXaZY+^Z47M+_ePz7vtL(p9~$ zi!^NP=5?1*B{XEqKA$^KkkOdZU({QGp0QaqC2HtVwBM}&5Ui{PJ}X zcZ$|A-&-ahk-EoLgHVQ9yek8cG0 z>QxgJ8Bp$3p|g5y!Yxa$*v*Boc~{GKv4uV{SBs+0cO)Ov+Zvt##BM(e1q|xf>1que zDZv6-wd^z^WzFJG! zuRN;jTovw^_0)?eFCssAMzpid^3Ynz#w-LcRqG{k2l6)DQErdozzXQB zA~R;cHRaoFZ3yq2`cx!IHCG_IB^qfX*}Xxd!&g<|!vml8v@AS@@U1Pa(_bEQ2Fo0G z%yeWP*99~dsus7lQo2p3NsDpKlS9#N7lmpI?WKQ-=$%XiC zO%G4+D>M8KtI&2H-oS%7a1qd8t3P63vj$Z6S?;4cu(|^j3NKJGDRv!6+6n%|VvuD?-!)abT_Q_%gl`a5oWWiQ zu=o5O+)B>Y`SDjpNWVl^2VKU71xq-ouHd-AC=+dvm(cx87pv!blBPT4Kz3kuOmJe) zc>|I3mp~PkJ+|!SYz33tiU8(ZS`bEg_{B>q5g98HCu%Ev4PHv7}Rja#<`M|TqSKb z$(>{2>ag&Gcd)#u3U-qBg-zr=XKPfGF{*jvvZ>0a(%zX$O>pquWH43J$P;V{;M$@C z)n75O<2myV=8Ps}itYCYa<+q0sCy8xYv#y<2K(pRO5SyULZ*uKj_M;4gpOFxF&9c) zg&e;m^=fvpqRf47ys;DW&ZRiUldrEjzt`)IlJ}~Jc5L<>OqXAUEP`GruwgVO;yWmE za;GxCiwd_&35%c+{Vd(0@Vw3SF zRUxT=WLv%Q4Vl~%phJ8xrr}lVmyn_ym(KQe`FDT$qg!k8Lb)DdGy;K&dIR_Z9tGoPC~>b34i5>Yp7MsEqhhDQhHRjbIv-2 z;^X(ldD}*YVza$^RAx*DYE8fXp>Dl-aYgyz!OBC#+hxKdia{X_hRBh>awh1c!^nssz{-lqe_WYx9Bq_u(>r({h@4 zYLX|wuQ4?^B9FZwsXE`&2WRWUO}WlM+|&L^?4Dq8GMC|%D{uT+t(6d?nux!V@$Vt?k4B|+sDL&%urG;!T1jRpk zOvTSNk_M4<3JjzT+h#-Qkipkwe$YWQu~4aR2vo2pC# z6v0sF2%+;2@OhHee$gFq<@yq?1A{4F3{M6j+_3A`ppDC&lBA`3BS?~T*xV*M2$Ee# zn1wE5GeqMr`IX8|vtKYQm1`!OgNZU@f&k>A7{2i>k|tD@xhh!h=bAMqW!CSv*3Jhb z!U>-njrh1|;rlf^x%(T|1G5aiwK9k7A$z(MpMRNDay}*}NXPY9%#b!*gMS~+%#G}( zbwN>KMxV9|iCX~^odcXz$-(5N4gFtEnsD}b6b_W?Pxy~BB{4Y~?US=FRkV=bF?s-R zds@UQPmlZk+zREHl}2^y?MLT!Jg(O08bS3l#PwYD!1r$*M>0$O8;8y|JKMwtYgwq!RJ ze*M87a!r~Qc5z)o8(CINK%k(%nNvVxkvLi4teSY8MG6_pn*cu9&jZBlSn+BkcU&1r zW+Y>%OwEFbGkBB?0|h;t#Wi&?7sZ|fNVIo2azs1W{X?g7`9hV5XB47$_)&ax=#pV^ zy|(0p)Zqo;Uw=CtD!4?CV}UGMK7G#3O6y#bCbVW#KBO0CEw?NrA_*(MhiM<1RF!a3 zD}3wLDV@}o34I^hjzuat;-{EFGVsk|gblbmojc{9M%$~%w}c}?J2pm2J2`);w#XjO zVw^ZQLc-iEdaD7N$3g&Op+)7N*G(^y>!`hI4AAi@Qhx6ri+^Dg+{<53Xsp&s(jGv#cm^ zzphG#cXbVs+0){#{YR@y@U7n3q1UVkmH-j=Q3fpSwGe*y%wmLeQ?t4pz1YnV=_MJd z{QU%Id5iDHG3BR!@TmMAcKj3VcTsnvf;oWBll{B#*$&>AyjM+wM7;Ftp!3&TRMib# zE7-Q71Lmp7j^Gl3JD(*F^+p==RB~ubC7Tt>9A$WJ;KkC(?8l_M?0Kp&KP#Nwu<>gs zEq3cdfgfeQ)E2EhQ|$!2##-)GZ*c-~EFQ&LE}YpyyJibNBoP_twz@#46Ay zw-N#Ct%U7&L#A}ALaR!?D!36((;^`pEc4%;LGL&i%*g#u(xT(OE`3)A?9m}w2uwyG zdZ^E!chq5ZDGrV@%zI@(@tFB+kW;iK-gK_nH~$ul?e;(csFR>MTsNp4X}Z!QBh)Uj z)G<{hMdu6LVn3uEUp^m4^X&*piu(QWS5ZjQqP}9hzsPA36>T{9R<^)3+7;8g0{i6en_<0XaP)|~oz0~&zJrxmChxoL4s||6ut*Ma zE4J<`E9?#Hq!0(coQoA6$K2pxy<;Q*Iav|H+h}XNPV8w&ceKED5zKgW?y>KU zoLQLv{W1+!U+!qt{_G*vjvGS26vh_iHzWXZ~j(SE}O=qp+Bf`-4#ot*!{eF zANI@4^`2PPqbt*jI2Ld7+gyQ%hDW}}%>K_`L};b)%7Ln|{t}yAXe$?G(43aGlRy+L6Od z?I~8Y2%JL}Rfos}O>;7a{9a7ozW~mEe1%*m6qD_p;BwbrRy=B^=8olJ!VYLn0Kq=! zgyu$gRIZtp3;~MG*x}XY6SI2js(%bno_5$SNVZdKpmM7^l`mjvzfWGtedLa;v|seZ zsK;{SIZ)&V%38c|Z&3aWRB&4`Ydc*B1TyD zcI{6+<=S8^vZFXTlC0pyh|6+{3xxrc=ygDf z;X1V&cB%`6XSrVyaHpQn#TR7WM)*!}f_-Vm+L)a99o(s~UD_#{cs2YZ3wl!zn=00U zSu`-NZM5d27MRRKjI8k=*YZlte=X#UHso@x#K78bOcsct{8NFrS{!;Piw{fQKLBE< z3Ml>{hVrL;HsF?%h~%4lTG9Y%C>Jm6KP}*zbN>LJ1k-K3$BT9+fFm|XUldq5V{iS; zQRk9l!`tptSjP9kp@Rw_4)|IoPaX&xg|Go=G{!8`!J;MwI^{A$%eYH(Rm55iQD!YN z7Jjtw(Ao$TB7N$O)m&*0#a5n_nA3QXy*54!UCE#Edn=m zpaZ}^m{Y>saV0hNma3goPzH@lLBeRP_u6wRU<%5Gj?;-aW>%Lq%`Cx~h{O0G1pP-i zTWG^%7au9vbY^`^o79)R_}=M~C&?Xyqhgds5?W|MsOsygVfAT+z1q4t}P{`72ngn$r&uI7vtpzu1%AXKCcBpRDyc)1lmEp5JgOL`s_MK_C@sb7? z7>S6_D$s09INQ4NTQNdLAAiYXY~9hUQh29+{{XSmYRfI#fF`lr2V~w)FU1 zL*D94S7{U-N=56y)*8(#UvjRY z0P&NTMrpgR#v6yiU>V`!sdnYfTN(v7MP);|Y1YJIk~l)u9TBr`_^bDMg~Jnufbrs( zb+%xi5bcnyYSZqyjV=eu0)!64p{TS2*c!&Qqv9X|pHimA^qZ9!>G8|k%!_HHH?o`rXJ@oHIs?YeForobpa%)4#FX=k-W6m<|na7 zq(Ew0Z^daL$1vRBK?J&X{4udZc>4aVFp?V9*9{Fn6AAnGNp-t5uJQ)O9`snf`y&7? z7~EyUJ>;KSwf4x;KWvECZribsPi?KkmM2y7u+VZSa?>Y{QS&NA-8dHfVCKNvE;3XM zEiIWj*UlF4I%4u1YxW8mC0VY-{<^624IAf?JROLVcmS-SKsm2*@d?V1sd2*%C+1Tl z3*200A`@=5B=?**4eAOv_OMdix?_(@+k)80l-nEeR}E>kRi&;67JhsaNRrQtF5X}Q zqy3RSdsQAqM5^I-@t2$`7024!U&Kb(PBFkt?t&C?-NQUiUiEW_5sy8H87M$#uTb2X z4r~w_CUfGegcgqR+praY2{V*a@jwnb;F*XU0+`1K2DH(mm)UKPxYBVG2N(4Ka+8$8)|sU&d;&wT?-AbW zPCgw1(khXyGxtY99K?jjD_d4l_7nkgUtNaW{D+ zb!3ndEVv%*z$8xmj4C`^E;BR3sb#Z^7Gbgm+`&}1%q}Mu{QUT&J;ZCU8=B7Xf@u{B zmq?^>-kKQPCvv1gd@UkNx0p|1Ygz~=tAdLT2252yFyV_RK*jusXgF;^4mr(}0*b=qwgr0A zO}H9MNEIeJmN=fOD->LhRdV0B>REVaR@Ow{`{>0X6fL zAWXNaEfCPxe^xJ%3jY9Ib-9MKcfws?^AeMuTSm8)47UK(QkkONCT%t^Y4vWsiI`gA z1ovCPPi7;Xk=oJx(hiCldpMeG>gk2i!@RH(Xf0p{=L=wT1##mHn;ot9rUoK@Fct7S z%N(g#RxEs7ohiGXC~Q)+p^nvTAO8T8lkr$>1OE4u{{YD`9GMVI=TK*VPHC51X!~3z zr_~Fpc9Trw-h(lkX6}zn%Wf=V3xJp{2hXKstY@U-97LQF+V)3QQdL8QL~NSZXdDc2 zOJeIpj|$JS)V|F=n&qau4GTpCQg=XFH)I-46z8*pPTmKPxoLFM4peUO`0Z1#Sis^4 z_z4H*qr@w@dImv$;!QIWWO3_G8*h9|2s4~S&jOZ1jmCGK1}huFfa`?ubabScXc)nC z3^B4Z{CY|Sp!dtc$}-*Onw4qbh$Qr;-bD3nMaQ9Z4tQQkE~a>d^;S36dTt)aXUBa1 z040|00PYZa)YsOdj%dA^R`5d2*0I+b`bZE)aynpiCr%r!hX8j~Cl~(!H6gB~@~%^s z_A1Z$d|01~1vnAdl8&Pr$tI;s)Ej3yleh6nklf=nvbBw3()5LG8?EAT#3a%@{OF$+ zyT)E_PXpGHSa3u@>;vST7ag86*FJnwYUD}Xi#V`Deh@e;F5B@?9yCbCLyYjR1!ua| z*5P>=bBT{tQbsH=RWWT+~ufa#OG@&#zo zNu_}qwp}$G6qedIGp#z1=fN%NQY_pM_jYtDxVe%e_*Vd+1~@Ijbg=hw>i4P4aPZ6a zs=3%Kow+F0u&!*cY_|!X9q4g_0LyBlEjnbi!(gwaxoh}AUtuBIgHC66D zXF$+#1cjJgUPUdu7UKImQ@CsB8DM--uAVek!r({%&Sz{XcAX4)-eF4+4^Wj3T>|Dk z#h7k8Ou=p@X1l7KvIU$(87|IpqJrCihR$S}PGX_2^4ha%=2E-AOJwI9OrVK8D5}Ob zz;tn`%W6wEU>@{VpMs9-_g(hR?9}OGipESeVUzZHgJ{QBsw=kOWrCJ{`5iU`p z6$9Z+!o!Am^{eD>5Mx+}02%aFw`R!DKYBE#sv>C!W0y}^a4HK3@ z7eUM*Qfrqgbev6&g#Q3{2sBqXad4_vA2Xm&92vjXofjO(MO3j6?3Dw7Wbp~yIfD%z zB*%4K&B?r5nck&R0i4k=XNL5yPc#1jsM}u@h~9UoTA42fdixa6_L53z_+2A%Xnf6< zAT`x7!8$I3GoJNu)`sL2$YW!=JK3jj<{`F%VEvuy!{R(zGCWhj2@OlU+sxX|axrGk z8*^LF7;TO4W4v}YcB4JP%v~Z-cXtTt+M|MSnfy;`_KP2+dAi=6%)T9d1}6k4)z3 zESKUehZf=8Neh~6vDExi*0m2~t^jIam7T=uUt!T3O%g@IoNF3>+2maC)mr@6ts_JJfsjJtSzlqPB4aip|3h^&~7s&pYFUPN#)7!kypiU{>3hYR-G z-Mv8(4IZ+0_Cid;x(SkB;^q^8@Q=*NQpg#-DcR50FhC*x_)&du_fGLV~CePr2Zq$5{p@MVw;`j z>>f;xp~I~;e$$A8L#Cr4OfpS5I;yiy%*rmzPjMT1e-zJ%8bOoO?kF%g1G=fpSPcka zA#QYk{G+0EvwfKElF$U{|qwqwXJG{^wu zgpqWa?h%A_&w4_WsqplLud`c*bFOWuF4lczzQ*TJn!(rssvo-037?0Z<=eO`%Cbt^KD|fk4 z*ETDh$7W>FF^IIOPLlxAY1LI<1LN;y#y=H9(1x5GYqdHpIOJf=^?oY$+fWT?s=|B2 zIrSyr8W6z_fM=JLRjwwJ?oaq;=0^w!@Q7e^j-&>ZHbIfMr6{)9wgGeyavAa(Caers z0MD&CagJ@lE||=6q*{tDbk|%R4rp$1BmvnFm>}F`2!X$=wM!1i+!U1=^l7#rS2Z?Z z;~!!%#M+W5uw5kPN|5a89PP?q$Z19N>IFK!fX|y~dV0+lTc)+OFqF4mpZK z%(oS|fb3TD;pe$vFL=(MX+7Wq((?VWABx9Y4GnO1d^-D6CrTo`I7s6{ngQHC9LkjF zEOmJ2)T1(DIaMNbp|TH6R``UNSHK+TCk;IKgyU_r65K;6%kfM!BM5P?5mVHpY1&;U zXM6tYXAT(?`0}Z?`_}0~Fw@GeN=nj-+-1NjMCZfpbSUM{bvIMbwNAJs3{=Em?1dQH zUi+<@;e1CfsLydy-GQVxX_xI5>D~idbl2gU?#ysXx?MMib*sK zuaFk&m~ZNATQqBj3K^5Xs3jXV>vhMA$vlcHTjE>1_tnfVZMSh@Xplx*3Y+@zA!6S^ zi>!OCUhsHZ01f024^oJ6I9IVtyJWz=`=S0L@W~aR>)e=qpw94g@Q*TAOl@PPF;?vu ztl@Hbt#JOYvQ?YicsZiuY8($4k20veCjBafe!<_L0E-CY+!V9s7c77#Ux_3RGu!h%!&s7u1G$JWugt@r^UoxxlFpxXP zk7|?}WVUjUcNF!DM*jfp+hKFwVWFa2XLKN~D?E^w4#}oBsk&_ruU&9hvu)QOoPphF ztz7JyHkIFVXXv-m_Pyn!F=3w({CRiKRzU zlyWMV9RC10a|Vl@8cW;ZUEz6vV*da)lsfCUuOsJg)7;_BZZ9qIZDcPu;r{?TekPu} zZY$qHxf7TLOroDauHt;O99RsFzc933JB09F*UrDAv|8*}bjRwqOp*YbPdi!m4%XB7 zO7gvbHEMaX@U^P8=FJ;(Hb44Zk?PEG3l|$3#?tZ<@H@AWS&l8aCpOhsurN`?t2P6H z6;f@b615k6R*d9VDeDT%*0S3$;mAkMO*hPL2Q?Y^jKDkckfJ*sd#>n&noO4!yji<) z)}Vc$bNEi`Z1?R*=GOOe6#bi~Ad@*%d)7VhPEc!-Nmyz&Sdqq`7sBFtoA|36E)*p1 zoF|7AzonLd;fEnK%y4i!93X?Zke|5F=9cB~jhI8Lz~&=M@o|d$n*$4L?Fo>&Rv@*a}a?HXgy!vmWjg;8W0t%ZY2BxatVmhxu2EE!ZZwoGTS<{L_GcsEoa1;(hlM`SCMmLLvvamrGjg`q)K>s*E$+=kw&%YWhPXXFnO}2g`E_u(qf0B0F z%S)VcnrsQHL|=Pu9`Gcd!T8Y#5SeAh{@n zV;dNAfdtPA4g#vJvEzck;0%0GC)e{@ z03A2ev1_eKiO5c0iU7tE@Ia!wu5jF{(JgbPB2%r0TZpV~^WcCSvP-JkM`Dp};5e5I zQlnbjGFL3ec7^~_G)sNf&<4VZb>o6A*@n!fMUc@Y)!i8&0j@3)l&j*x16y7RyG_M7 zr~;vIw{u|2p-_SN*;q{7bdfw1i`)DJ;(|xb+?5zfD05n4cNJ!|V*V7sa$sQ# zTKF*vtvD_i$vQ~J4-@i9j^yUQ!+1T2xqL~InSk=fo)s$Zw{Ag$ha;YAGRXL`Vh19F ziS!u=Rt*8>eh?xP0pvNrZCVmHaF~O(I}>8*h(*VDq|;&;K%^!~TzH)=PXZfWB~kzc z0bTH~OdDWT11?(dF+!7R#5WPlLG$fV8E_WC$Wx&^ZSvQ3mim;jk}qDx)?qdAP*Q4o z)SH-SU}YRSR3|&&kF!J}j69h;!p{)I)|E}&C9bYzU5=&_sE`<*WCKmhqvo6@-lSzA zy|i*?n-A_>-G-ei+IxdcUM@ocBMk9^ed`&!-FO(y-%{`+3lFt=)KA9T$MFQca(muT z6Nij%jNwJpT7Qjc_S=Xwf@%jHrC(6uZkKT^4D6X$x2nc6cejBfPDrv_PlUnjOg)%% zq%_HS$J9uORMWp5rf)d;2_B`F+;CX}VDp<*AY5<`Ynw>U7gQ|K*-hOVaj=Sdz3Tq8o*;8%*Uzmq$4`55 zrdc!V7=g>rwKnIpjOwXt1}9I=S=N~2JUIEvU{SXXLCC2si*115YaulDfIn`0`qYri zZf@A+KoP3y!RTGz=%{BK};@#mc^wzZALF^b4-0;{hm}5kf`Sd=3z$h8skEf9P;eUtOgK00=ju6 zw(%4-N=$fgkWfR3U{iPOTGU&0ySCeLj-hLWgVMI1{#wX{#OmR`JX)iEl2!k^qHxBf!b#{mn z@Mq6(R(7JpnS{6ycxTjm6e&x0QJR-Q$u-9iC9JOaDq+f&*dzEy_fm!3KA!#3^LRvj z2XGk!JqjKr}ZKj5x@e%<$Qm%qeqQk(8R;Ba2vW zi-gT6)hIi}AyMZvSacFk1q_@?mjiHNF$70qh|z|FEV`CG-YdYJiQA23xUZi=OCUDr z@mS^q6G=(d{5>de4+Q3Nns9|IJPsiu>V>wDTO^ZOts|aA$2^%5a6F@cP0k3@!u)en2n)DCICC?oUOs#;D9yG zkpXK%+{$*8*%ce0PTlHAJR6cjE?ygiH*_Z&;@+~DC5*2XrxXWo$sv;iqMOu5_a~0j zzEkYEbbjPgnI#HU>OkeX;Yl*nw5 z91BXTTxJ4}0jW}GAdLL`6B=eWyy?yqz}B?l6)}>A(}dAVVFVMoBGBM;RQz8g?l}jM zG$hG)7{a6BbUF_hQZ~!Wi=ukUZG=?R9tqb6mVq8&P};eA2N7NBN7e2f zt6iqK5eeN9sA_RW^_7V^&Pnm6xEu>dDzmKKnZSL1q+J!M1+V~~MDUEpLzYt>9mf5a zC+%Bvz?PApWmJ@!0{US*ee_YtEFySMY~-gMJB*hM$# zGr?mgo?oeyo@4L0X$51OFA}S%fr2*7b|&%nnrDbv8IRk-+kv70tUy=99V3M^{8LvP z1;9b*SR)QCr6wXiw7A&0Nz0I~b>c9^ckIb@x`*Pm12xYZS0tE~y*FjvZ%x@2w*XFi z6>e85EbyE-6;X;zf;$S--sPtVjw#V`G{zanyM%06a1$g_sf?2Injkw})gI_)A zlGh9-X~#7$xB+kgIT!`*Fz7CWm|lnaHcvR+C+_Z#@hz`Bqm2Ed=hVAT(yzGPb-_?R zuJO0~OK4jKEDvS5b&R1k-Ni#Uk5YmvWk0!e87Pz_l~og0uhQt7H}1CF6P zv3g#Eo%DR-&@HTEPaL-8_@10DmL9YDuo%U3ZnwcvXfYx=vJ#ph}DKM(t- zf5hMW%ZIV0@ImhrJNPa-nL8RvYEfIqd@C;C2UR^(S@l+vM@L*Aa(f9+0Cy~IWI#H5 zgCf)0?@-;1vx$yq^W&0ThwfT21D7f_cZKY7#m*THGOy!Ta_Jv#+x|OkKE9O<^L>{Y z65sPk<&(;(PA?Q%aocet#BnkNfwHA5bAF)9*-F%6-ZLC{dXsME_lmLOsMFq>Wt*06 zV!=Fj*RfM^YRJ1mt(Q6sW56VX-l5~Oj_lQDLabi&HbMMTKUW+_pGsK_>KzB(tma6{dOg8CY^$*d=|<3jqaab=&!sJk zPTV~D)?j$f2cJ@U*lS`3)n7aedG1v3ZZ)tQ=2u1G^~WcvW)3qQ3G)h78O<9{0}M&x$M7*&UeG{InMhRe4>RI zH)Q9Zhml%*eRmode($Rv{asJW(gR)heOG3eQ|N*~Wy{5Yo40BK{?X%y^(*+7KQ{09 zkN)yjj`{U@3rp!2a1;Lkx77`wRn>YDj0BJDHFN&}mDG{{07xoxyPPgr4Re|ZyEt51 zF>q&ugolms{jJ#l0MvK)ktjkg&CA_+4^K-v03G*zSNkrj(V`vs8l(RJ)zr^t?J9Fy zi-qh?ark2si=(-B$|(t#E|ESh*&u(sclVQUf=S+1ej7W5r^H`^Yv5|fR+m#B^wykT zk*a>*s_KXS66gF&pP6^B#DDja=R7&gxnwoYXd>+4acuYh0EC1nL8k94t+j0(x>>Gk zj7FbUH~#?2>OB&+6Cd_{P(S%iLH_{4NJiz)&Aa|1{poL-$#B$5oahaVP9{5-e4=wi zVF^1YJo;CVY3X2Dh<{xHJ0D`&pT)J8byLTb}@&5qc zRu8$Ci+B7-{{VR_Lkr8R$Xa|w%nm>9bzkh7pOuoE7F~+AHBtWnmDKtbZgz8;`$4XA zL|*t@TQL6s;f?7C-MT;CUH!N!_({%badU5%InInRiOzS}yOtwG%(nID2gbcHE8;)<$+$rz?=8DmknPgWkHK+2 z*lLIW09RA%@Jgt**T@0DvDjSKG1lA+0|l-Pc0?Z3gX3P0?QZ`7ivIxbB`rhT-CnN= zBln5_0N~X+V&d%Om9e+jhZ`Y4X~U_8J169;^=58wUR;Sex|_nG=ac3W?bTk7h@Q-p ze6Bma~k=IUK8^w;eAMI zO*7LqpQv;iSq*W%{{SZ4M{E8bi7SxSex}#zWLk|gPmx#qX0x@StBy5KFG9t+T)Uj> zjxKYHiOiEE^^&zv5D;7vu?)J|OyXH3F%GQ+w1No)8Kz+i6DaJtyY)T$5pPrt=2O4^ z@PB`{4%O&-U1g1JzY@*B_Uz54Xf+3|cv61GqV|>I<(InKhZde2wvC2~8Yv$2RO4pFmlZ8vBG*ttb56^{m&rYQTGc1xH^OM84Yz-XwC$Ot{f)*xNu&M9yl| zQ`l9mEd6wO2il*Kx7SGz}oXP1w$pBsqP_*`w@`YG;5$i#q zox45BvvI|Pg*XHN;Qo?(KLwH@QUQlz%=QUCGF4_r!6$svm08l$P0E1cHB1LhxNN#X-~jweBM6wq=hmZ; zDxHyT$#4RDencb&7V~rwD+z(tz(JP zTxky&nnAy&H3ZkF(M}qxQy);~&-Ob%-CZ$PiuJU(yB=}b%DRu$pZ@^7zdlIY#IB8+ zyK+C|dAI$J$M;tc&)o5-9OCF%IJBjNW36@ zoX_xGT&Mcw{{Xw_zrY^!;OG5QJ{wwA&UkHMnh{(s6Pe9aDcVas20Gt_bq_kLPn=Oj z{{URS_k9=m1Kz3rxqt5ZFYpJwXUcoP>UrXqEajE&`hg-w!lNF4)jqDXrF2YUv|SQp z{7x#r>zDrTqW=H@d)AYud15oE=A+%e{$5sGT>B&j{{Tqeu`oIR0947tXGq=nZDE=b zTrLw6MBn<|?izg%{{Zw4dO3dtuZ6@-&w5?wkL_&m?%w|Zs24B(xqt5ZFYpJwUHtv2 zejHYGuFkHti!&!T6h@ttmGQ6-?DejLnyB0P8N>eo1>LKX{{URS_k9=m1Kyq-{{X6I z!)r?4>9vMvMR2%IWhhf>N!$mSRv#U|qyGQ|d(?8C z@H(D({@Tw~@A`prr-Gxo{{U3@#b-+Bm@c)8Gbc9~>`ngwt={{-i~IraT27v+UjD2H zP0arQ4x>M_{{RKm%d_BY?oSSX)idF>rEdH-u+1X4TsM^8`pw_FXurT7^xPYQSHNYd z9`C_U);cx1w$67?{4Gh}>sP+*qW=H@d(}L5>YsA+;}hOV!$CM(zEV{5r=9Q_GRx*u z;KZm6_=>P68qY}%Hhsj|!Z9fCaB!lbahU2&+@iRw(Cv|h!5GwZ>ggoeR#jSojs>c% zR!j$mOI9}}sS;HSRw^_#Dd@S&RiJd)d<@GCSnjdEXsSzEfIVlM{{T?wHL%B@0_T$B z-EsaGW!PZuP(M)VEV=gYJmJJJ6Cg;)t{x|6!!UG;M zOxz&IT@H^+W%o4}+}RDf;5UJmVJ2&=GTx&LW-x=A(l0v|3`B=pB9gXCMNKvpd(;h4 z3yRkhp~YZGZNoJ*Pus_N?PE-WKq87e8V+l&(dlhkZZ?XR=4;TJg1MWGw;%u6hARD@ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000030_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000030_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..aa741413195771405dbdccb198b70961b7445539 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuMlK|{U{q00;AyoNe7gK10p4%hUcg_ zQ|CNKtE=nJ(w`pyY=|^i8i0TZ03g6WfIrWG*YY1-jd|HwxmY>a z*#Uo60q+6mn3yjxU%dS1zwpC|HZ}z0`c(h@d*hD;V0nDn>TOB$w^6R zXlR+4x&HJ4+TjDBARxep_@4&~5;7tJDjGTh011BKzpo%5A|a!oVgnG65Rs73(9uv( zP|*M0jR?S@LINUlh^u^b%*BmG`Al6KCjtFF{gYD_l>l#xX4Yxu!bmNi3%@SleIB=D zy|Xdxc5hEmLVn+z30)u$4ge7mfQ*EKih}%iL^vY|h&VtFDkK$g74|P8~t88fKe&1YhM4<_yqr=Cm zobPm+Swxx;l*?-C*q%r_sH)~~+*ldat#^4LhH->gOREeq>bx48doDMrFsmo5(w8+5 zOW1-06{+(CR(w40IJ57y|9IAzr>S>UU7B!9_wuf8rQ!Hj)iWB!%-YTT0l}wa5f_jC zrz9_$)_4b?fuXg=wT^@V^AN|}K60tGBZ5sZng6<)Pq}5>O1kP`oxzlhVeYCN{k2|n@%<^?sp>b`UDtp@Fp5?jR`zS$ z0`^xz|1IpP>Qbwa^kelpziP^%3ZL!D=OAVrX6!=oJV|+ezpYl%I((Xu3jeW(N0zOG_7y%Gy}H7p33VyZ zPeY|(h935*KJj>zD~oYg!_%jxHr!c!W&|iCLDYt}gz+^xB^i@UP%q%1?O$8{M@3!n z$B(BOQ*{-c?O`9NHFCo-B`|toyr~t7oh3%t^2c)3iqZ2~ccP!Q6-M^+%fUE_Zm z1X~%5WT7IWWK6r(i!&Ic2@MJ=1krrAHKndGjBSx#c6f56IoAkNPB0o>QGuui)8LIG z0mT2?A|BrVgM^Z;D-cnh|s8PbDaKITlMwwr=Va`%r@LlTWcY! z^%Bl`|-aKXy;kh#`8Iw&-d0JT~%0vKp|NDj4`gqFt55gjpsWH zpC9WZdyf)@c;Qwm^0U1hJzj_3`2X84*xEW{YtuSsHq@vml-@dQD^y6i`zMxsM2vj* zi;Xh6OOMa)L!;4pdgx)YEre*~vqS;{IAkG)>6^fZ{|2*8KuAFM&DqyCQm2XO!;pMp z$C9aUKUTVzwdJ&}n`JwKwRjc!%3z5gaVrF^SbIWgqf)q4(EYbn+`c#!!qAz~s7X&g zT^VH(M+Q+@sr&gFr#F3G!aI=F5Vl16N;>+H8BGj%xmQ_ugRdgMc zHELjmmXZ%v>pQdhY&qkJ)<>Hso{UFba+k?jAKXK?;)161d-rhb$}`%HjWdFI5!Po$ zvGM)I^bb7WK4ZvEQ*|$|=la##UpM6p^%RuvLEVVB@cKv3Z2IsLZ z{s8Qgguk3031rI%vjSJ#b;|Z`sBR$8fjd@(4Ho-DRiT?=j?p(z4b^dzPx;)1s!^4AuD)KMjE!X2-Q+UxOuu zk=wl!-r&YnLWdAYEndpAJw}#UbjJAl#QG5-A~GF~Q|?()ni)JD1S`-@HTRr0^s=yhjw&wBYT`u(z> z?OW_#eM&AJ>GW|DtuQB4On)n&);;SaI(X2P&3vHun~$w{-5|F5Ysk}jcdKWH9EI)e zyLZ>i27(7iE#;53nN5akqYsPk6vB%(Jj8!#hO{rSjobme0l6( z#`M=ocF`tGF#4if3H$uE>%4Td_J+coD;u2rR}9e!bJ2J<_;|d30G4xpk&iFBSDvEu zqf4FyG9Kl-Go7!OZ>Ecd}fzmHyWj}gWClQWL>6woIb)(Gia_4hVRd+&bP zvb1)BzA&f?8>upDZc=MN#rxT)wj2m8)vVkG2lnyU z*L!Rh{?!$Da^$ztq*?JRV2rVnDLdtQTw64)pLW@5Dm4vIR=XlZY zuF*GdlmyR@glj}mwqNBH#}8mVu0$wJc>jiyp;N8SJy1H%Ji*Z2s5aR1D#u$M*4Tv z)c7)i1glt0A=6?o1BSu)746tXiqGH@>-2Lz=s^EBy-xs);+QkbWpl<1?HdP8*tlzw zJ->hY9pVgrh*Y~Jmr5BhzHtjix(rRQyH7OFQRzurx$*trd_4Y-BU&)Baq8~Vg0?-#l!kjs_JJN=~ z^#TB*13}eC5)nlz`)`w+wyR1tL-|yutDq^2H)6dQ6!otg|2Gi4n(nJ^aCRf~uxe-O zh_yTFi4CO&;3;q~qg3d_#yr%QA#4!XCrQ>L{e~hYD5fLOFIAfO?tcRW3n^4ci9bA( zvVHq-)~XZ?7MlXmu^W2;_m^Q6>c=jC6E{YIJTIAI*pZe7HHh4u^n^HH?=mQlwwHQ7&jkDE(^}!hnruy-!26vBb zf$>cUMliXy&)5IrD)6tWGu_~z(|u!c6oy$t7YZ3i5+Df{wDJ0626?}OVXyz~uz1Sr z_ZU$PS5^9VQLGG=ZI2U*9Ewtbi|0+|zg9E-n4JCST@Cm7bY0!(_-ic3Z3i2l>Bsae z3Rqp2%jAs96GhgeP4nLlXKtdK#UbrO7yPcGJEy*0M|B(U4j%u6NPBpQ&BZz}MFC!JG2 zQKR#w`;2l5E79VEZG~Soe*o$ue)pM5^TIzCIp8tGH-4~X{mIvt&Mjs;iX*xXeqlR; z{#om8?Kl1T|6=Yh(~s`a9ba)9j2EL%UgMFJwiH-cwrkA3(M)+wvWQ<6?IIkR#F3B$ z@h2K!7aQR4x6U-@sMWVaTI39ina$6^M3!|gpfh?Q)9j*gA33f~2&|>-0!w}0ES<^> zCA1|m|I)M56%2;-5jT`CDAdy^(JYXJ9*(V{+jtI6nOUg^1-5ryx9T3Xqyp%4p~BTE zZc_a^${>!{Wb0iQuE~VKnOVI1yz$1Iti=ya(UDOqZLy^Y56~5Bh!9?AuHq$DBn+fI z3Ys@=>yrKsV}8GfDbl;5)nxVuz>J%r$sr_^#7hWzz%OoJn_X!vCj+s=ko$JkoF zo?jXKQsg*MEoMw~OGz}q3)tJU&uE;hHo0$oRYgNOY!t2s2`@I!55PmfN0wU%Ue=C^ zTW-kTmd>uf_ewi?7+(@;ZJ1Pi4XiQl5nCT4buqg<~ z%gv-o?elY|yva5?X(k$kbeDBv?D7Y&|D3JL*NQS`V7!OFN_b1ZbRs&v>!`ls(dcRD zL`O4%;aL^!^1FQH7uM5#W_eyx!aM{iq4$S+N>_Cntaqf*gPw$oL9_1AK;RdS$m0(; zrpZ0r=VgZY{%37CmF15`JWdREA>Ll(qj{^^t(-y+u4GxNq8Z~o{u!cgQg_G~${xsv zXwK1|#~$9FWM|Gv2zf;@)0Eq8N7fhW)^hz0;0-!vT55I6=zVD%XqebH>KN9$t| zvZ}_?PyMq;&VtvAB2Fi5F~$-*_QYrfS!O|MiUrA<`ja=f5xRV^*K(t$FXPji@v@_C z69)B1V7Tdcn&|~}EJEQ#HxX)1bnR&M#40;vaW9`V)SW;`Uak`c`0DxUh{$M@t*tsV z7JYfx*pxy;vT?fBEi9p`;MPxT!sIHe&SOP0#KuHVuN;htzA+6}lMwyOEjrw&5!RU) z)dCI&%D!ne`!E-Mb@nc$RExUxKLxX8o|ZQp&MSrd-d#-<&muo~6XEkhs&71Su);=~ z2X8yj;VQZmmpYg~1{7eI914 z71tD%nYuM~M0fE!-#E8RHcxY#e8bnQ%ee`+vhVKArJU@fS%n!625kw&%h$d+qIRiv z64o+GGR37$R}uLsK?595X#W}cuLvd;ojb{`k(9VN2L5}@jm|PN5D&oZsGRYLu8@=K zb~=n(HBK!P7e8aHWa08D!h`CL;2>Y-G@8=a^*65g>+usRq8}>izq*l%jTBmSJHmmh zOrN0XDE3htf;KZnJxPa!O_R1qheB|avLZ02OzFmS!C>G(({Uu*U! z8QhC+0+dp`!YY)?N}7!qodX+YEf!(ruC_(8my<$@I<>^;yH3egH@);|zL`p*hn+_U zcOtor6?-;G)v*{Q%cGuh$M!3uLDtaSDS@^1Jz#(i!7G#g*e5oB%WU>0`|5 zSs!=3;|*Gi$$&d6_HW#*S~9_^VTE1RWc_KS`c8{k_NKk55LYd@6O!b>-FE4(H2>j% zOieC8f_@BB2nxt z%!tSw<4i0A3XK~Eyo!GvKWbKA))wwQ+*eTCk2yC`)Nww%c5-{mM=-a;>{4NQuy5A{ z(=mspFC@uI%qae_o?ElHd#;W#B~&vjl6;`N;*n$It;|Iph8SML4tzvJ&#u>!_SZgY zgCVn*)L=`+L9M~)nz6rm0iN{Wi~uhFI{be-`(FqAJk)5RmyM0{eb5yWCLgKert=ar zkTJ60iVyX})Ut?e!G1jc{8!${k^6J+=+5VS1GkzO7F}XG5yg8bpNpVw3oC}*n=s)l z=JDj*Y8{r*W^oV`H9ug5S^GM1;lrq9Javt_?g47*q?@dFmq8z*>Na(>pvmRL8zmO*HUo}Dm+QAJZrY?{u4 zR3X&EN@iJ`Znr>0KFlSrpl{h>o#;zmDisDUOLzaXvpJ%%^3eIWw>b^6+Drh z<#kw;Cbuc-JRxTSn?Yd!bXTU)X|KNkpS+t{(-F3;P*3gk2M`$ZU^44TLi;_!I4%aR z0eFiDH+vMZmBY%%z|lRs+9W~dU>ZX=_odEHx+J6c!*TO3H#?{ARdIJM*}$~(J7+=c zDSU#08?G3tf6H_~^ z$U-tZBmNW;)$z^f*N=0v z2`dT(toq}uE0BbIjQKukyZ&|70Jk^fIt{{6!aei0R?y0=lK8H46E_}?3`eT|SwhnjNxrjex#eA}H>UX1_2qjB?CJV(8ywM`U+jYM z7!kNFds&*Xz?+i8!X78z)dIT^*b_xe-iWOM?gz*MsftZ2l_H{)qFzEs6=;t< zqF4fnR$3i&fm`&WpG-#smUe2mO&LpuU7;iMbWKmZTR}Zgc$~hWBsvSX()K5vY&|!4 z>JI-(N4gAQi6IGTDolChwvO9qdez)jf9t9OX-F_={gP6d%a5@Ko$=bbI&%^7@@VKR znp#e{)cEB0NpT(s$G0=byw6Q3@!{6~#OEo9;cd_o&Nd$vL`0I-!+^`LiEHt*!d|H* z&$k$WaP5b>Uq{cmC=*WJ1fdU^wTHV`y%d^1h2-xRu)6f!Z3{loK=Ff-VwFoT1xjEM zVHNb}@}H?a*~MFjgEYHktD=Y0qmG!hWW@ySoow|Q7%(Bp$68g z+Y?`>ZMhq*!}ljH4;CU?G6cNBw$&0QAM!H`_l3OpoJxKLWdc(eIwj>V;rVR^R)!PM zKR>aCdB@9Ydc_oU%`9n8CHjLx_;DlO(T|V9NafMWnmoPu>mKy>NUdg-8*Vn`T@s^K=60}3Je@s*)XTRyHTxE8(2nV;)`IzQYd;! z2$J`?py)<=+3Iq#^B=$jhf}G5c{PvTP}XW(#}nh({71`~E=@96#9>E-#$t#(812K) z)5cOi9$x{9ZVUltMHW%ZeaA*UpI|cmlv^APAz?$Je^Vs99{#@-K=5*CtTBZV+lrUp z#%VicHW#cNghK@0i}q3!oZcahP!n(Gh}Jrl{##B9Jm|(a;8sPwm@=I1TVR|9Pfzi? zafuM;zc4?Sv~fJs8>||iJ$HX^xluMw8otpzgC6Dgy$?Q*jMFAcIpe`w*&O}Wii$N| zF1ALlnnm^wR|*#q$PMufY$tkn3C|Pn|EcQ8@EY>>i*+rvD%wk-((o9u!^~%i2r2Cr zpF}7m85&dQV`O1Up$q^iNC|w1U2QI9mH6Im}YNHHn762G~L)3YN!z8+A_I zdIrr~LBuw@+ZFG<2j|}iWDH~Qd$MEZ`ETc(17&DnLUQ#uE8-aX{ggELxL;cGEq9&k zmgU}XU%Rptt$^czJVFOhW9#iR2G;7%PkC{bE}ri7#T{WsKUs9!Pv_zWI3CT3?z?Ij zja%3^TYR|2Pi^ix^24H$MPvk}9!E|-(PnB59QzN57`xIC-Q?aDTNP7=$TOXyOUb|7 z7x~d~$7kF~2ojV#*tUGxsxUKlr0SzF7)0x){@uW4FP6Vv11O!tf*f^3ZxC-LWG<4% zUbq&8fq`i8DR7a63oj8H5|WxLI=FS_TS#)p8xKnkxDXNP3VpVGV8flQ@7A6PGflh; zRwnvc;EW+WId*1d_hOV8Ffvy%2j1R6a>Xh@QNa3)lsR$8St<)XFSQmd1W~_+fq1rm zb^boGZ%@>uO)>~weOUryb3IRYdk!h6c$MhRp8ENhAHnYgW_HM1l(+qgp-LcHv>ELY zEom)02I9n~Fm`MFoE=#wmhxQ8rh=YCh-UiQLuwIM@~Er+$)bb4jzc_CfXS;LXhNq0 zn#}QaxgRlJsDUXsTNM}8(zR)r)TFT=vLMYjx10BT@~s!2Gf!x^Kc)jKjms zN&4LheT8vHPgXH9xBtq)I=0Uf8@4ggkx+E2Edf9b)xz6D%}DgNeOeel`7(sj9I6E% z5|jT{l(@3opj;k;Nd@XlrH`Sz$=>ow^?p`Aq&K8K#;;1C%yK&_Ilq}IU%hiuOvw78 zk@FBvmj2OVSVseQ2gPTkWdBS4=MSaFcBTvEj8ZwO!5A30Ey_G9Dg~Dm-+yzHHo5L# z>PSiMS(DV_?C3B`Gy+<}N1`Qr52KFvO@2&e4IEd3-UZsiT*V_i2N1MO;6Vf5xSj|_ z>z3#n$G?-7{}ds-@F?Oh*D2aGYnHo$+<=O*w^|7NX&=g49`k z63muDbbjMonHtf`tadj$)_0?7^OuY2`{ts__KqvF%agy!JKY+d1cxuo-!k8F=zZ?K zoGCrA@R+PfRYSNlv}ZEhM!|^OmcJ6!WMwgE49evc<`(O-ypk8d7+NL|m}K(56mw&n zYYSx?r7(Klw=P_Ya?ehsGtj=0*{x5eXpDHkBiU&bf;=?y*=kGV#M=T=qM!OI)Qgwu z?b$w+W2uO9iUYEwb83Trjj$!&_`IKV@>T2#zVl*cl=3i`OK2?HN-QZk;wBeEYUy-j zW*#ts(Pr5SMQd*@^P;yTThwhuf2@lqP%$@iMi%ELspZfNHrow@&5QR^evl{dR9U< zTz~!Rw(CD4e_amwS1rMWOoBgz2wuBBfEcH{eCy5O&y?Mgvvsg8fBoQNU2b-gh>dWh z*_1)N$>^+kM=!e+XY!LnBUyb;<_|#oVdS~caM=SkI7}_!rPNq0F(r#8B_YO9LewF6 zcWRlsx;z>Vsiz0aEBUMb0bF74et?xH2LRvZ5`F5Cmp;r7cIMaoy;uIJ5-~yvhI`St z$Fja$mQb|#qWis)>#I5H&TuFOS8wY!UV%KKxpNX1@4I3&)hnD|?|sOM+NG0}V~{1q zNNX*2}l$zdR zmWM+Jj1JhSH&hJecy<12?P)p_d9GW4;i7oq$MH}mE5z8 zVGa<03TC{1Hkqw5^fqC@<9cH`)8~1RD_4Dl5~IYQPg5t#Eo2l}V5I4vPZ*>oKa;Q0 z99S;4?pIgRpotulKp}g+#}KIA_>p&PMw&9O694!TaakN)@um5G7GhBYiPeohNL?0p zxVK^`|I1RG^N@?lW(%SVEv-xIK|NPeVz4EcKIc;i!+3vq7qNjk{h2YJEjk+Z#jMI# z``E|Gj?Bc>riu~Ax85VrXq6IDrYBFk*wY4n*qnXb!YcaCqv zg%G8KqbXWb$4z2WlozTX&gdEiQ3fHF6;8La-lzpkBpPQJ#Y}P$6-~qeq-rI_5_+F~ zq=Sqc5|ZRSV)5cK-;<1h1nC7xluYhZQwBJu4&jB85xm~~C+7_P%|dUmzE;40;7LhC zh{In!YV4o9gep>=^h(c@AuHotVU)ahP-qg@FY(tyvVO^`M@$T2LHlo*P&Ajnk!Ais zdr7AS2at7x5N&kwo_`JwpOFgSB{&Bue4ooetF8(E%Z#F7AQX$HRgzR+*{wC}Y+=l2 zKY21$c{OVvTwlu0ldmx5nxq6GW0@;*b8n5o@bycFjcwQi&2z;+e}4CrAPt(8MYge^ z#H}=l?R+d|X)(BwWBcM;a_#y^@@&^m82u!aF}UyvC1I%vFI!tObXT^-yK!S-@Nyb|KyiSqXjTi1LhP5whMHS-BL~5jgw6D*?s#D zWW=vlZEy;m+nEi(cl;Wkyy~@08LKg&a zRUkvTS!Il(UfvS&Fe2T`0R9TovxViRcVmjCl&vK+(sf|HwuR?;VwiOXqeHyU5qU03 zRnaU}mc`Ulf|VFSWpL-7J3KiiK&>RpeC1ook6#pB;ko#S~SYUI! zDuC_1tj`)N*NxqeOxJad72>dsfTRWLgxzn_IUq6+c_GhLauO4uhmWp$ z*YO$j8Od$51E*qgN2UQPP6gp9osv=aT9s01fi_k>tG)}&OUS3i6{KKG%y&_dNH#gB z++}0k>jODOkdwvbD0W2WjofBtOz!R5T33>OY@6sA)maYZ@wP$AVFuqhz>@FC6O7L9}E$Kdp{n-Z!yZ zU4_1z(Z9Z3y)&@B5osxWW6MvMMC_YJs$^PG;29yZABx zG~#z73x)Bf@w=h!P$<0e`>cWLfO+`8o+=g{wP!p(OI}bMQk)g7I-o8Rq_+)v&1jWnZ70 z@Wv`54aheXS{xS%bAntd9RMiYlIA4^>+|E$R%Ex)_V1!qT_um1z4RY^t0GPIPE2;~tHXuk7PrI!<L<)ZA_2D4;3=czK z)MvXTI!byd_sS;blqCjnD|wYRLrC`-Hz-1e4e~QF6e- z$tiiXD_2J+_8%<-BcZ3AekBUX56M#>JxxL=8zbSJD~wf$R~>ukzG(k$ML**f%3yZC zSzB(d`hb~$fURV}yls!JlQZrO-W)zRnjHGlAHOP4egq1lB!)o6qoDb!LodJ|@Yaxz zR6}IO2mNXXnq)PKphr?Jp&0GZ znkv<9)zXGg^RgYykFf7U`^>t|HV3dI?LM03;IcC}y`zFqqY=f1X$E&tJJKzwN*y4) zrZ@yvu^=-5&Q__&BuCgkzZ9o8*Y^ty^hAv4H&(gOHj=I|u{qhiZ%dRJ{O&1xA11?P zDo6U&Zj!z#1f`0P7g)-&D-rbvK;sAwD>i@qJ;-{>w}{#&Z9nz|GN#8C#&hJ&&9l0&!r6yt7T#9Tp2`vTe{R8su~q zvgn(kqf;XuB#96r)e$I znlO8dFumRqQh-8q>?6W^E$B0*AYiFoWzPFN)01q^wY0T!rR&p~-4s?z$j?HPQH-%u zzIQd$o;k+GsTr(^&Y^`Df@%gpM7qg7_JV*)n(jq}&s76kI`%#kw|0E}LGUEgz`41$ zZ%?ZMXmJ=f+)|;D4O1c`Er2hkSQkugRAng#zY;76v)0&o%-ZCa8!o=j>$ZB}er9NU zPxXK)+U#0}jAwdg53Q{FR6NvB)PdS2(CSwukg#=Bq0=Jr7I{_Bk(s1PB>7nAb-KaL zPpa$H@Z{2?a1CS@-8dyKi6MsGk)nJ*4!c{)^@f~+N^11AJY=WOB|J;I0?7BamrA-j zC)r+epMEON7GEU!etCL7!O31&i&~U?|jTq6*{BG{FIfgKq|zsJ>Sh$Oz4zqJp^vn z#*9hG?2te8nXgz+H0{69hba5dWgux?=ROdrF!$w$oHK0frsrLDniOKlOQ8rGoXhF+ zDc{n8tgZ{H8h3UT&lQ`E@lR6c)*iq6Sd(G06oR0iv&lvJaK2;%pG?Luunj#Z5=>#+ zotA*tz!@4-xf`!=g#;sQe^K6d206zJ9nGfgEV$~Q;wJ8CzE&(p&916MS?;-Bz8wgQ zSUEGp1Gb9c74q5x7Y3rc%}c*LrA0oab~2Bu20Gg*gS-4FxQJj_Ul&Z&qo#%oRiepK zzT+DY%O`9Dd4LQZLjSHgxf^1(i&{6=?|_oj3z)C$0#F+iXGLYA&URgMb{{;`VBgQc zQqH*T_Dz+#O%SNh_<$Bw7w zt!GH$4LD<-!Q(DbBl|`C4FpNf3HcaQIQ+6y6~N>!vwhjcRWxqeDnd8Cy>#SAh0ZU| zsh<-4Q5K-701G$5%fCFTc-jEw6$D73P8v5xFdgv-T5o(K?emRKzp$w~qfqqfCS<_i zQE158{Yt*qUMy!pZr?dH3@pRT`B~k$2>h+Y(9o&#^BXOanC`$>c18^yxa--KHz-ckMZ1O?Iw1?XFPjk!*g|5Dj&WI9}Zwr8cQyx69Rg6 zR!&|OGFq9*s^Pq$R>V1mw3e#aISu1?S#-$YFt>a2;j0HFzL8JGk5bgQ z+lR>A2dYhMAI9aWH>?PJ%C>&rM*I7OJply;7YbzLxG0J|A!OB4 ziUxM@e5miqH&)<`pd>X6B$0SEa=tIXDOgmD8gYyI-Q7zuP^?yjAY5U(8R>; zbVDA8yy}mXkYX+aObKLQN?-w|rrG12jU#~yS|9IVodU0mqsR8#wBkAjh{)}iofBp$ zt!BwgdfU2s_fCn1u9Pa6T(QJ0wN#xgJGs1bs1ZkXzC5a&QoO<*2gZgn#vOPuw9K1L zFEG8Mg{au(fZL(*4z<*5JRzneLJKPf(ooW)7zZj>YBCA-r+4AHfH~)5m#T;ukXG}R z)HE->K*9+?;Z|@zdvJY)y@LG-IkuG34~^1Y8rHbGcVe4>r4mkvo_W=)GD4 ztyaSqD2zB4?JMH~#~>LX4PNzrsgQm^vI`{=3Evalv8XC)l|oGd6+hH*d&BMW%yJqS z%S<~6QhPLUuH>s zG+KJ%qJmjzV%ytPgI_c(o2AL?EUYrU>%8|~f{8bFgBRLRBBJDttPFqnlHj@O@?&xE zV>yr2-t`CB?_tm!vj8h8YyC6~$NJ2`)L|L0Yfs`49g3;!&&hBgP%yl#(RzQhqg`$Z zCcmIFxE_~dO9w@0dR0u&z~EZkdY$^NqM1efgDNqik!{HgsF9W|E5`qUwA)O-D-py; zjb&l2s8MRpkV27A{%tj^fI(SiR4~lWg>1M-C5hQ__6Gb5MU{rGBD zecyJoAIvapp3=UYba7De(pz;upxXHCNkz+P2U79M$RN%1N#51Y5M+Vb63YGxFI6#4 zocgOC-9+b2qILsK;ph**aU8fR0DT$NdhNGD=I_p~cvqq#&K*;0q15rlmrPbV#k>jN~(O+P{zOCsxYZ-pOO^IO;EpQ^#` zhrykV8-wx>?yuKjDwYr>87kV){tDX&7QXuD8q$rAyj82s_M|-lhm;ED-eTJ`&Veh4C2vAw_^h(9k@_5r zsSKhtLdx>U_Qncxx_akTd~7@?u@Ifssj_cxGis)GqnD*&ECI8dRWe(S#cZoSoBH%S zAHPT7I)CTP$7C`+9K;)!n$ycJV5aou%tCv0Ibv@j?YLIivIg@TPEm2^&-3SeEKcSA zaOR43#m=lSJMcug2&T^;05=x{J8LPk+O~3PC3Sww%gMUo-=-3_dF$67baoDi@Wb1I zM$wZU!Z{1=!g0H_jKLtmbD`$1yRaLQbk6=aPxk6p$8Mv>_KvF2b@d|s3_ywI(34Ls z?XId}+tao}b2+gK28sF2T(>iE>19x5_E3ndMmBe3IQS!N(dDO3mNiaP zBBc?4%8U{@D}qKlQGCri=91%sZt!6;oo{h&?d+hn-s=7z0OIsd_s$+Z;WL-gjr#_{ z&50cAwbLMya17a0n}^o~jj_@QVd8yds)uHfiS5CBdB9)aY!J1?+W)x zOJWdm=6fSrN;*FE`W_cy8A2yQ2kbbWUB>_o4f)Sodroi3Bbd~%l4;25;%#vp#$Dx| z{YRUDylrb8A@c)4`uLc_k?1ZODo%KiG6n;8dy*|7QuRu(EQW+X;N`0c*KN$XSjjM6 zWUtl)i+P8F&>m4f?izCnwtO)^Q4`>BAYkCqG#+x4G+4x3vk)2A| zH%9<9GQN~wNOC@E7z(Gg_MFW*DXa<)gxsT4-cC^UMT8j59l|alXhksHtkT0Lo9vB@&}MZCW`^}U&<6Us`Kz=ItIDx)prymx|+yG-G1A!*HCGY zq-~~1Wv~`4zUqiBm2%Epj5xCD`$4k>6TD4MAci;wPrUd`d(2WyL|;&K8!0meYB3}y zq$D>OMX`5iE}l9r&T<7ors_ zG+ym8>F3VRZK40_ZBJ_1{ImRJ8FS3}n3Tz5H(Q@G!FN5iXl^Hm#kEsAoIZt5I76QT zs|gtTQa&##MJ9X+^}Nc>cP0At)=9Xbi_02l>*Ad(V-fC}_g^MXquV9d%nOu}z%N zm$Z&1|C)T-kJ)O&i-fE>a`uD}WrWX)IL+xJ!TJW6Qa&)CMC+CMshrlP{my>b7fP z_|q4}foCN^T?lXcj;eXPf{l{enB)Rx`XdUX0u_PGujw6Q(a%~%$stJa$Wlnodj~`#>tSG^L43tV*1T5O6XO~KQ;+3 zyh~qT;w+B*NDue{#gw$ki9>8!)Zz1MZ_-HXclg}D$R=i6wp~}9G4#9GVRzOZA)rGd zpoTg!eMDMgyrYMTZ!wT~x6G91XDY>6Yv`eK2@`x*^Y>6n#uDjxZ0QC^MU8z&4x9Ym zj`LZ`)O}0S>TFAllTy@uZ3t+5e=aL(mjZbQnEg6W=*Cv}a6Y9eT}leY5>OU2geosR zD~mAF?ISSO_Erb^+Wn;zDqgH?t^-hzN4X~kQbv2oo`vK(92sq3S!Q0eKE})({{jq1 z?6>Y-d+?o}WaCVi(n3Y?6f)x}U7I;Gzv8PbnyE5wiU$#?aw0Z;F0Fj=#c_>N@<=jt zS!nIZ+-xmZXFdMvO;o118l_TrU>k>yUzb}&O}<1URV z!iMo)xiygx`L(|%`;v->& zMT9TEUnUH+n@6K6<9gD`UkA4EQI7cNAIu019@_gCt)2Qt1ms`E~I z#t4_ex7)lPCkpcGi0nT0`Czq;l9-?zvf)DWk$no)=!}VZbk~&!@BauhJMA2LsnmlUAlBarpJ=Qa?l3-Ew^kVE8_1|!4<(*%(aT9pLvf#1sHoR!pThppfM zDmQcC+SD$Qf@*k@=y*{J?x8<2)Bc|qS))1#8#zh*8)mZtddrp+tRE>98#gc#OMum- zbaNzkOMe1x)@Y()mFsa77r6PZRh5*A$mXL5{{U~G&I|@+TJH{5Ng}%|UFHWXp-Pla z=`dzJt(8v>=4d?^GU`{wb`DSA#CXia^3kxQ&e7Ifn>l-iq6qKPNfH3on(D!}g5n0i zj^GEk-s56W6eFFhuPmi_WJ-KHd{rWxgPL4Zl}Yr}VCsMXCxNeXR*AUsF$?*8T~YWs zDPM#QNP;RSu6VXlZ3z$piJhlu(@m}D*btpMyy*TeN?noq6~`Yh1GMhqc_G-9)&zrC zj3ZuGu%e%s4bdC?uL0Vn;@3c9$kAT^0Q9498LV}eY(+5%j;<8nxwJw~!#p?j8gGs5 z$XN;kgLzN3pT(ZhdtcygGOCotc8MMBm2^|QS4}t1CkK2B0yQ)EPOm~1L`rnu-LNot zpt7R`dLU8^NgYz)sa`XsPbat+H=(GN3O3c3LGcoH{Gbgz?yiY*ME2$?YT%~$I}5qE z6H@Y{jm6|VadB@2i1~wu_kgMZcyjf)codqqhse@9SD7@zmCEE8328xaTkE)1AwKUb zD4y-N&)`h9WV>8CI2O(s#A9_>u9Zea;bs?O?%n<*)ac7{RP&@lZN?F6% zdzv$P7Z$>l4C!BuuKhMCPU)|Eio1fzjb-942vLSt_8U)6LSZ3R){#XJc`D=NfXkt1 zR?|`mJ2;0#T^JNykV36ujagZSqbDvboLPSmRAMVNrjYtZ5R<(Ph zJrY+ies98kuT*OtRzjvu$4g``;8;;n;Az0QOR<1RxS`8mbBik)UL8mAZC?Ar?UbhN zz7kYCi7rY7g)KsFB&HywhzER+5Ix|&Ydw(Y*1Mn~C@VN{C~Zu}7^a6tyJj+(bsP8z znrNfDbJNl0ElRB+NZ01W2e^aO(^icmI%gM)!#*Lpn`uXN6)70WI>AhQ7ccI2udd>g zY~V`Oge6Hn5>6TtPK-`lGzlxUc`^L6(sqmO`@kV8~b5~-73AdhQ; zs7j;5{`Th7hSd~NRuGVrM&1?rzwamB-w=IHcWidk^Fz^YS0Q=){0O)%>xZ0s+=P^% z2tFedUxBN*GGIpsGR|7s?Uk-LVFP~->b z;?(;y2)l7A5-?7)P@pz)mi4a7Lczs(M!OORl$SQgXJzaw5W959jYM@};LsCEv+9Qt z9S}NmtKQ`FHse+Gr}tY%#gVmrY5mqQ55<_%onH1ut=#JOw(O?bih(Mkca5FR+!luZ zw;)HnP@7A^s0KCmn0)WvuSxL-J2`PlSwenXJ@uXr8gi>zP;?47M3cIAXxF+La2Z}= z;*vp7;yx?Sdv_d$-;%L{9zQ?HkV_XOQLc zt7qp?nGk1T{yj~Ll_PXRk9RS7LZ)`;2ar1zoFn5_(k?m#>Az=pz8O_1DM=?05&TRy ztXsrN%|c)q2!Z4XNr(rmc|dlw0VsX6J{I%UiZeli*Mi0XR%1cFw{YvLqC!(BO-LK4JwF zDO%LaQ8+?L*0UQ75gl4McM^dllRrAlN~E0t*aY=B#bU*eu)?&&1GM3;ii#PZdy1tc zP$VW31G{|cC9nc!fS5D~zJh)AVS|Um?EWsWnkYZG+Cef;B{#;8We0kQ<#~u{6J7f3 ztnPINhKU?#PP+VT)w(pp+Sz4B8N*H|f#1Tc$1#N~4b!?Hwg`sE~X2^)|w;sAxa~SRnGe=@s(MyT`>H3B#$Hd=0eSt0=4x;7CYR3`JEi zV^Q75z8g3e3Bst_WX9PrKNmu+2*d{yPESLGT3*;_X*5 z9665kDMe(rIzEz`IOr=sJh=%f8ZpsvM0?gnpbX5i&erO|QSn!msMxn;TM`pIT$Mx< zVB8q7M|a}xlL^2%C)(|I7{!Gu39LZa;aMsR1w_DZ#Nvr5$m3D`&Oa~yX5GY$vhYqV zv!}hz7m}S6#;!W296xmo9NU6}7_c>WiraXA&gg_G-Rm#qSE+7e_e6D6eF=l;b6YTama(G86 zf5w=Xe=eI!NfHyZZ}_8)(B53&j5>iZ<@Sy|bUOQ<+mM*3kR8Te?xm*`laV0gP^&3f zQsp-1RK&#XPK^mAXjJM>>b{#XaEl8GkCzDVqff}?RxV;%hst6J+rE!9^mpWC3&A$| zau~{kaGsJ=+y|66_pS6R@DfsXk_}5$&skT5q>l(QzBc(9XcAE(M9@20>*!n~dgPxW zTOn@_601iH{N)D@zOfO`)UOl(rCPD#^^lT$jpf>1+`W|AxorcZOyepQHHsv-Yf|l? z+6zu2woG{^ndqLSJs4ZOvu$eVmZMD)Te(ZPfyZq}lqtu{qVNhB?TH)D?zZyc>TW4G z?afB2{np7R=DHimDNVJLxg})NgxKRl(dMih-{xD0DXC& zp%byVoW-z|q^u7OYg}Hm(`{xotLEknQhJZI;#4;|lvIS{e9oXKT^AE>c~VSLx0)^06!+l?dv{=KhcU^nUYLwP!FapcH65 zWY0RDuP|J+UL^@M)7aa{ZIGtsU;EL9O)3oa66yELW76`ov{j1QcLP&hq4Q*ki+1juDF7)u1rr)qd{mvX=M}mP z^dClOC3@^5tT%>rBs2vm6F(f#?QKH|9~|M?Rr5sq*M-~3Ge99xuF}*Ko3fxHk%~s_ zQ=#1_-1+4ZGaUZr{#>R^fxXL(h8Fpc=1Nv&Dwy&dMDy3aHaJGoqg0=cM=fLp#B2@x zwWKN*p#ZHYKkCw(h@{Yh%ff`#5g}Gpy&P&!v6vNyRMXkq`m>g?;t4_{02Kh4GBNUi zW}Bjo2QY*t76uTh!T_WN6p1wPNHieQD#bel?30sNLTj@Y-DiIzlyhj4Ls}n>_LXFU zBD(B)qLYhWsgC~uV@^7Q+wBz&X)!U^9{ugJm_$LXcyS)uG{6Sq@&1^HVFH9lbn+i@ z=4$}Ey5<*ftjcN1YIf+^wuw~~mZ@heX-X+oNl!?fg+dOSW}YBrsvNdf6uclxV9G#) zA2e(`S&c-TNIr zl&T87C`i=7*CRH25vU90YXG(AG_~6iBY@Tm(o0Ubio{droI8(Y# z{jQuMhutJixG|hl0O`2A78NlhO@33&)YPntX;Q3|Olm4qO`?B;np72rXhn3LggmiP zE^TRlL!G&7AW}@4Q1fqb)RhR3uN4zQIQPD>NCW|M3Bqv$09Is2D`7~rc-guMl&p;k zgq}?zHPN*)mgC}cJ|#edJpEi95}>0MC!T}g=`W^6T^&D2okk`3`sp4**teHQ<|s}p zsxVTI6{1PnCWn0h4flfYJ6MlGKqkM8Hv%nYBhD_t&}Mwq}@tH&;vTnRT8Gs z^49t!1c4ubyp2-T#Nm+n zUX~zXYu1Ps(l`S1nmPAk?)fn{Id*djjQ|6gso6MfrU}wVeOz_eMZ#25Gq%3>63VW8 zvaBOPunh;stj#1l#{{y&#J%LM<=L4MAQr1}0^Ow@$u1x$3LlG^fAj4s+6V<(?Kel1$p8gC0LHZ z1j(e4+;_)w!&92y7GDYkNFeP2Kv3?X+dJE6MWm`iR5tv_%O2)cM1>F{qyt{}XaK5{ zBX0f0)+upg#K#4^x>OZ~Q~PE#+(!N_!*pyDq4|lY9$x0if#RedcVbc`M|soA%uF#e zaZ+ixYpCx!w-azBK$L}RL$*W@e!$q%Zlh-QR%j|xH&o8Q#j9erER6lWj|j?G3IeML zOzW@8VBsTtY8rn|dQhb$CKrf8Qa}cjl$6j7pn(n8>Clu z*6})_;+C3|6?Hdmi?=PHk%z+tDIf>}2Ff;k2Nb6dAv?AcCI;g?uLP@2+W!CtMG7i4 z6!rng9Qpc&CSGk2q^g5MO6X36@ZKC$WpehfDRz2XF##*VifJ-M3U_FDyNgTs-g$Vp zrwwU9DtNbViyk&3X7b6lf~2I-5=`-(+L1A^LEzBTy?V8Mz=cwVm7O4m9A&JN7!qUo zL_K4q(zOjw0>;tt-1Nl) z@jy~mW+tQ^yjnhL{n#BGoJKU6otho3)Z0-tA{^sv2=Iv3AZtiG2!|4yCCwa~?_W(8 zl=+FRe`UCHcK{SmARfTy=Cad!772kQh^mL8CS(MF5+bKpby@)`I1D90AaN>E(oBP} zfN8;)qUbIxN&C{SNLo@xq!G8Ti)gOShRO!-o7lF(@msvK$?%m5?(U|?ic!~w=z1im zwFOy-fk_Ik zlik-9YYbO0VO&PioU-Onw1NE7khT6*x#b9Xp%$a(L~c z?P@dz2<3rI^I01ja*q*Rer6_{d2*m8=<`ppx-#}qX?1<+9>>vO7}P>z!lcDCTqyDR zoE!N(QlzOfJs3}=tO5-xOzde-v`S69#Kl`e>Ms8Px_NiGmabbs9z0$YjWsDWI<&Hs zg^(4YichrnYwCIZ{6?oIWw~kVQ`2 zcLyVcf49n=3wJ| z#nP-oC`KS~#DYH-o0zb0EkahWM_HNB(?C95&PsC23Qc?~@aggx+rYQQo0v%o1jQq1 z-LA%y!ooKmXDY0|V9({OcQtH^711dwNlJp!q{ut94M|-xN@#1Pek%>Jb!Av0PSF+G z?at8Lq)R0QV&T_MiZMiDufVVIX{)}Flc&cSY#iajG?<;f`FP=M&X~1}q(aj)p`FiU zB?Ty<2cK@T&{LdH#Q; z%_iqxCz4y~zLh{TN^7{oFtDO*9+a$*WI@wMMAXs%1~_TBmfuRAJli>0DXmLN#5ko+ zJE{=02n8mQ#6w{Ca^xe#F4XHOJK{|9Xx&Ljm^$xZG3gGdbX;XZVzD3vfr&pUG&SD@ zlG<9ZfHVhA9^%xlSyIFTkT?ylDaqVEkUtyVsiUSliuP!9R9n`uZ$f=`jd zvaBXlNY&LF3u4v1-d}sA+h_<>0p)Q8(7g-4Nrzr$Bm{7bCPxCpx7h5SX57yIB&rams>f+-;tC}(-g`T%;M{(KALx~3~S5>z+P!);~)j?!u0?qD=ZHx!e@dwPLewh*eQ13ZPWG8Rmy z0&3+E+(_ z3nVL!hm(WVE>Q-OkWUb6?`EtmlwGilO15Jcjao(^MPNqi#7`@-JRv9o01$Y52GQ|W zd!4Yr*NXzjK4oGCCI-((6!rK+ZATJl1k9&j+2)dVydbh8su+Oh^C-9qLR0dl6k}B% zpJeN#({pnxX=@lrAe5-vRVNZfJFB)Ou0-nh<$^Xnd(91d%?A>cp&>CCLPa)k@<7_( z$1;!x9X#h6b>6QNs#ngfIX$lzsg;(@AcP)>3{`?+2K{9<6RYJjs2O{r+6T*Ab;P_v zYB*^6kV+7BoLX{d*-{X*Dom3f#p?-SE;6H&wy+O>KI%2A1LZ2MyO`2zM5rc;-Ck!r z<*gq?4(1HH5-^P>Vd(c0H17quDWnldiaby~DJp^#0ZG~wuLpGXfMvdye*iR51`{n*42j5js?pF-Q_> z6V~cLB4io(=3$LNCwAe(M`4L)!i8LoL=s0+*Fez^S&B_~4SS)#g+Ta0n9_8U#j}@8 z&A^bPCQ6c3g+`k;@$PD)ESr}4aSB4#Vp)+P8Ew+8-s^ASbDOh(`0 z>~DYxK{QZ3-R^CKq%BHPK}?yEJ2v@S_bdy_NfCxbL`LY`%RyUrNX zQ(eqf4UGgZ6q$`%>r}RGTroi2%cUwwDi!xLv3vb^0(Q#og0p)Vg4qUhbxVGl5kW{Jz<#Gy? zXhyrJzyPO3n!_NCk<0FSa9S9r5bi!gz!-um0r&Ga09hMHSCqPyB-99>jpgB5ssW>* z9f%c!)aa-Kh#;y- z+YJudRfq+rL#K5MR6xR%^^@F8b;Hhdh}=plKRgFiqGb?34!#a_wNYifi{j2>kI8&ofIU9uljZUK$gNDatWR#m}Hj4U6VoZQE zuI;?8T8S+#D17E^#)NWL+U~okN`X=?1H^02-@B9xw{0N+6~-k5)Ii(5bbFn$r6SZ! z`cNlhN}UUVOn;ex;z?G+xYuDVqkh!a6t&LH?A*_2W@Q$p-$Ib3sD4^h4wJOT!6o`X zX8l^oc6z!}o}TQJiC-!w=+U6%soX=-8*V~Jm|U}d#e`-rS-Wo1{#BGK4ST6RjK(dF z58iC=p5r}F=qrIM1SkaqAcIgw;x|7%?SHBkGa*+pDj9o|JsPChQQ@gabfv&<+tg9;u%?g&3ue>hOz``*Uou&$S{{S#?(RN~h zl+O7OxEtw3cZUA}L~Yz%)RT(hd`VuCkpd|!%YoliJ>ELI&gv>WyQ?Omx|$(Z<2hpWqK8d&bgLdvY`jLrK=#d z8IWtb2bHMZ%vwn}0+SL5HRROrZglC_S?+e6kp1ZBA*&&HOc#SInYH3U!LZ>aK1b0D z$fnL#_)z8NhIf(-a^@$Tl5ShDw1A6;4kgq=g-r61M<|6TT(x^-P939N!^qzU0))M~ zqE#pYq6~qv__=MeTd>QEeFg-Xi31k3?rz$pvBku*D#QhOe;0G%<=x8jYcfY*0&t%c zoohy0n|9#PN#1p&^4dsEtC5W*CLn1X z?WJ8-6Kz}ut=zg2w2-j`Kp%r>e2!0{NSP(zqp&I^xgqx1-DwTVx6$0)NFT-8ByH++ z=%-ooHDbn!ChjFb4)+em<+~_J`AqzAZkuU5I{oa#L)=Ja(o?C96AsJPfxHnKdmZ6P z-`y^3p7r(-N=m_Mg z(Wp~S@$GSDV*wzIQ`WC@utyMkzj?oeBvj8ok-n5#j}nS_Re(n5)CA%k(eHK%D>MLc zHTQ<$!Kftsa=LVY=#kFQ07=ES+0{n+u%uQT8kGl3!tfN0rMencPU#>}0m<<$ogw7w z_PEy;5|Ab(x=(;If{PUxZCp?(ORg|AY3d{yJ8Iq%lp80NfV76%Zi!hYHV!R;ir|nu zTQ~Rs zIPf(b?%E^etd-hEB+e5eyTCj6?QyN)w{1#5O&5gmncCKD_-%|)NuA!!%|74f*hndl z5T|!@&lYUUymsFrW>?0Jk`Ii`Mu0$f3GfniB({(WQv#4Q7~K z;S5^Rq`(nFvNUW8l2$?nf(btM-Sr=oX@{Pz>uCurT`&b14DZ(Tk zM{^6EemPr|)=d0uuT!Bj_|e$I=Fx*#`0ArM1$}CD2|m|tn>cUy?{WY~H8}M!4K?KV zvlj9z%kTEP3InHfdl&_-O%O4Q9HI3|%Gk;vBF(UYwlzFdbv@(IsiKfRCMhXEl zx0b(y_k8zedt2+H!YKX(q^e1B#HEqJ2$3jQxA$; zPCDoFz%n^?72LFxfK)_Af_86o0A-QW0;L$jq==`AwF6;qB8els9fi_LiIIzVKIezi znByw;Jo`k|_`y2-#BKSFTz-jQRLIxEP&YM@lqARz!JQgcD~)_SuUttR_p6%r2}#5T z`NRa$egoc=rEq8F6YZ}nROpbeLy6(e>a4m9nr)yY`&i``CoQ+iUVyc!X-ONM(Q-8n zlx5PIrbm4lgoO-4!@uIMGX_ZM5DwzlQi;3E4!4tmX(_0oc%@WqZ2~2N6W#;FTrxm$ z&1JQV2?44CMbwonDDZO>QG^u2O*PT6aYGF1*-G^Gumo&8Ta?@bngTaRB^|}I?os&S z^=zUNLVhK?;hmjMMi|i`JP2OF)R#6EG?W9eUmT9b2^t{t`-_l~aC%3mHwvRuJ>Yp6 zWQnkN4!0p8KDj-|+}`O@HlaPlpCiRzM#(+E4{LYW2IAc(yWYy}BsOuYk>NC}R0HIs zdP+V20LA7!2_^wMB%gbYHG6RQTA`RPElMJQa0B@By;7YmsUlo^mqs`Z(9~Ne69{y- z;+G!slu!WG*4^HP5JALi{9P#MT85pZfPXRu!dy(srs{YB<#e}nsYK`7owlDN1uRX$ z>MUDLoM}5nb=%xs^9!5Vcs`YTgG$TX#eAkdS+6BGHC3i#g?>AW&RRbso=L^~-!?Ip z#4M)~lLVRNa&qW`C)G*(TtqA*5;ggUCdV(N0d&|C8?mRmf4OU=A&02V`3CnBw87QgF*sUBH+e8W7s91OZVe zUgp(|D^ZcN<5u0vN-D)ewD-44MU@o_0viFF=LjHD3Xfe|P^Bbs0nl8;wGHjmuV9)} z;;n&T1XN)>YvdmHLSuheAIGh@B}TF*<4T@qvJtXpfdF^5GNNLGT8}+L|R0;Hl5Pi<$TE~~kM^h6(KWNwS=6NSwgO)m_fLRAovA1-Ph&#i= zNHlc_DwLhQ?a&&f03*HJ>upD6TS`TB0w~>4gn_R3Yztmd)jPI)L!xC#29SO@yh@}H zcD*D-3NQ^(n*4bmsQpn?_I=WSs9*g%Z_5-+3jQzDKB=Wi?B(M0uiPWdK=Ok2*#2Wj zUw3DFXg5~lF5*+g`X;FlJM^ePsqSftzN4Uq#dT=Y0X=z@D8gtR(RyfFcy?f@T1{Kf z(Ghh>(ydC>^sbr~7`U`Ojtm!b1UR>aBsP(7rLhRTqu5zo_9Z~GaMz?j75$;NnDKsZ zp6txWXQ?T^1{HjZ2f$BM&jKH`1qD8P>kWf@v#u#WHKHyK@0)1!WWH_9w9|aX(ma(? zVvu)$VD+%439QN9W_h93iWO0v!cUf7fB>$XK^{Zwet8DrB_!*qY?0zT-bV6cKnItutipASTm+9L7zA>XQvpEte-~>b2snV= zJ_~*$WOsfqUpB^B`>{ z#Fqr!NGEHgtZ6nW3(8)NwJVFIvnd6&c|uu2p};{f8<7FWut6t-lDe=8gI3`)M#Ha4 z5v^D>tg_T?iE(;WuTtDA7o%VegAJa5p^7;7uL%GLFAzvgb~4=-qO7KJ(h{3!wh}62j_r50uBf8Q zK->e7hou;%f#sVWFe*~04JpB_mZEctwnDX)yLe(I2J`Oz4nl3F&D5f1D_%)z7k?!v zN>2LfX&&~LEvVf$4usNqA=AY#R?uX2{n>*WSQB*_#y9qvVoXuWYn%p`2vR-s`D zwU~e)V3W%{TXI^i!Ao&!g)26ZVHi`UwW~rlly2No0r3;~ZEII92!_?Gc!`qnPORio zY1$LAmRD}-lA=6m#RFhT{{YP9+$y%91Q{uw0&4YOCiOm#UR7z#+1n=9=GVv;!2=cTALI&gHdjd@*z6?*wdw}x8W~0$D$@rE5V6vSA zPMaU7b_L{F?9n)XIiG*nzwk-@wY~O#pOZ9#wslI(DQ8F=H9hU3 z+eX*VXa2DEeoMpBed6HsEke7BzEQ+UJRhKRX&%F)CBV&gj!z;c{nQ}p5jd8vM2S86 zKQPaIXU}G3Hlr+$fw$>eNych<}H&K&6JkWcrC0V_``KnYOfgUa|vjgXo29$PnKIc!|9bvF`rjhqND zLEuEU-_QA6-eVQ48;W#L;As6LvwpQ%y%zI;Szpo*?$ygCQO-DqQKDy{ zrux8xP~8dzz)F&^6p|E^1Suv#ogx{EG?s+mn4}$*3rdioi7F6z4ICt0-sFX1(6nUJ zM~hyvyuI6&d{Tv}J4C_PlWUD7J{cb!!`k|vtXWQrJ<|gM>Ot8UlR_)N1K#;>3h7XO zHW_|gl}VYJt^`idYt4(eNY+$6tof)`I8ggXwZIkxktsdVKK3Ol#k3wY z=7&spcl9UT?RUbp3_?#UU}@H|I=qwIkck9d#x>p2%nol@YP#v?4##{2ql3J->spa4LYA<@^{2Hze(E+n{fpn#Ryw z#$NCrEcHF@aVU*+#kC&pwvyE|7Oub;39i7?FsUR+Jx@rT8+C>86oTLmM+)k(D7`mM z!3}||{ByA&gj%~M*icF6>juug5kc?b(UsYv=w+vnNw7|ZxBjQ1Wj6PznhWD zH^s34NpdvES8RZ?3B+jRBQCYn)0MOrir`YW%bG9aJ{ndk%|@$REAk8twPr3`03?n2 zOHmnXbBK~owJi$jF5SGKi)I3fWCIjEOivbO7^3Y^sgMi26t(z_z@3yGPWj-!0ziqG zjnwLD85>5`m{kc;E+&5F6#*WM`CcjB;e<8a>>_qjv0S^r0oH#8v6@tLQL^As zqfS!|yX%kthe%TET+3pm8Y!D^ZQZG=jbfiCf-{! zoVsmgtefiyRmBW;Z3LkzLP;vWcG7-jqyPyub5CUUs(<8-Qxwf)5B~tGa(gb9vyr#6 zvjb<-WsCFV-MtAW_3D66=B$72eJU}+n_9jFsM*<$$n8;Rpk&O?^*Im!0CwW7H}dON zUiMosn~6$E2uQvmQx(da$p(gsAiiMTyRS`q`%9m^`fsf&Qaf(=K~hE((JE4OCO%WM zV`)N5C#JtSj}Ia!=?DD2Z~p+RnEfXu3tU#lY=7}{KcxANlI;Hgdw2Uq{`2TNElNNS1j3FkH$FH-%co6YYqy8T?{{Yp@{*#MvT~VLA=JtQe=6^|`qV3WF zH*VAe{o?-sdEDH&^p}6MU+*=CZGHZp0>FjYx|D#NzHeXYb04JMNmLKW&CLG*OPKv8 z$7`u!c0yF7f}692MKcTjQqcNKkbg4tf7&nio0K-wr;F3TFe9V7yQ%^#X6ZBk0ENu{ zla2+^{{SzV*ZN$~={GwqMM=GhN>|GTN-B@byXzK!i-@5rXpDwLQa-1itr2wuqu##@0&>5{{V%}AN@CF6>PDa{{ZS{ zNB)!M(RS_tP1~|S{{U#e-g*~z>2Ci3Xuo?^wf$b61z<->ir}}0q0R9B0K#T}Mbf6H zFPz20E0~UZO|I8kk#fjNl$u?Tl_@g|{!?9AFG+LhZvOyjbN8E^Hsa~x@@v4H2vSXD_m=zkT}r#r9@((*(n2@J1t4{lcvx` zTy)ITlgq1hO2q)6a)AWAjLk|+5|Sf!)}WuPN$mW;L2kx7l1U`Ru{s)fmC51dY4cgb zhkVeN&F5pMF}OaRkLBcCT{2#Cwe!88NWFC7`ZRb^-Uhw$(>vW0Miqejies-sW2B?3 zmI)7)1|;HaHxSszSjI47&u4bVbF)laRy%ubE=msguY{fBzF((ye{SUaLK8g|)BeeE zQ~ZjcS25uHVeGXZ?=%At&ZN-viq0K|0iNbpsZHe7xvIcAsFSZC();IqcLJQbl&8Z6>3^a z)IcYd>Ybfib8>qAJ|y`Gc149<@eR6nG^8lh@eiiS{{X|J@IF%9z15&@?l?!b#EaGq zgkw?K00GCerI^AtzJ*p1nI$WjO+*;p-(FhHiz*0AbDem(Qsn;tu%Wqhp0;!Mx>y+Q z9dr#~UPZK|YfX+2@oQMow=pc2E<)c)VHXj0P#-;>xTRiIK2V)G8tuQ4JcG#DP=Zev zlr)iqC5Bc($Sfk-g1jb4jj#Z&_N1IEB<_zPSGlArxtp9V)2XWbohBN1mHt@rGFjNC zt8ABGe8+ihW&+lsg)>ip;-He2qn8nHy~}j$fgNd8swh{j(&U>+r=(l?v-$r3>GrwV zKn;}8dR5!0cX==)**I;bH5*~4Pv&DjwDY^)Qj>Dz5-MF2RU?cy+r$o!vaVvio0@<9 zeq&GjY`fqPJSpT4zsTQKTK&Z(C{*Pxq=k%0Bd5#IiKyUc=O;C4W9}W9{{Yjo`>$xL zg8AlXogQnaA9Lz{wf_MB0CO=uueJ1O&D)>&9?jYR04HPjUk%&cuY7ij?weU_i@JoV zMRtO!6tPJ(ox9=b7--ezip_~+)|?+j*%DKHK=gG#$#wF-SjE8(3z9dMX>sxNX#vv4RJLat7zb>~X)xFaH2dlK#MZ-hYl?{+}iNfcL!nr^IJd z+aKQ9(@Wp=8UPR*3+?R9$SbpzSrhYGD%8pOly zwtr^!pZ1%dFa9}y`h1u61K#yrznWZPvy@a%&1)8>PO2@(Zlfi6&=R$M2W3$g$L#+A zlI!HZ$1nc?Pm=z?d)vde^GNw^WmEe$u+0dr6$#X`8+9b?2T}4TkT*7Ma8uiN{{W}S ze_%cDKgTcs08f(sz) z6Vane(dRf|jJ3RpovvPg+O=7i$Ye@X37O~9KyGvEI3WX$)M^`&wJ1`GK?BA_IMZD@q{jx{-&<9yhQ;*tcNz9elx`kW4 z7ZOrLR))HKz?Z>?J0`P?j z2V?V$P5jHA6~#SjbH}yVrDC4hd5D)^nuWUq6O0)WemA?QV$x@Fgmb!qg+d&R%6d40 zb=y{R0x1hf`Alw^=6aRJgSoqB^Ad4Y-+%G)%K(f_?5_`V)EGY@AL{lg;krOFlq-}u NB+paQ5yd2c|JjUIbL{{C literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000031_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000031_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..031a5826c74f6b88c4a0765f2853af61cce3f113 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;l3fSfl*)5BM-AP8m#)z&U&7%w7;{lpYNS3e$ltR6hm{ LN|L3CKtc}ylw+`S literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000032.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000032.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3569af7bc795868b5137c4dba30c4560d43d3367 GIT binary patch literal 30016 zcmd42by!?YvoAWhySux)g#?$uhr!(~xI=Jv7+iz9y9Wqv!5xBIkYGUq$tBs}-rsx9 zy?5{b&U4m0Yr5C!Syf$KRlln4)$@Dh_cj0%C?hWefPw}9pk6|C6jfZyu?NdO`;G72&Z+TQ~c1M}YpHa0dM9^UIWCMGsEE-oG(0Rh45jn{&Rh=`n= zl$3^smW74q_aLC-l>rPovIrWeUbio(p!NO_2<3 zl4=~_EiC%%s6Iz$E5p}ryk25qQ*q&_!HRz`NCKHO1Ub*D<8o8O z^*cB_^|n z)Jd0te!RsOZ$q`m&Ue7^1AB`aIz+h{Q!tVi_EnXJx&q4`SvLdXLx(&QV|j_EUH9$y zH|HbLiM*%o!fE?2euipxy?kkHmZ$iEIFrbON#ab=w_h?Verb+rPg!#4cP92qsE09Q zXM&f|8zQD}DhzRtg=@@H&%fw-XDWvl7|hxmh#1Y?p9%i@OU#PNG)ws`np=MEY)15xqTGiYC^!LH0OxLek zDMsZMF0C%Dpio+CE@tMU?%sDJjjh#Nt<8WuV@=vPT3O%H4^#8I8uHs}c6uUt25*_b zd2skGO&wou*nLm0V}#O0@4Tg3N7g*iUH^Gw7;y9tE_&~c{|`1lUBa}PUKqW}?&tjU zc?}o|TfE64#*`~E$ce6%>AIzrVtB)jjK@emf8*EF4RPtE7z2q-qO7`as3Hxi|aE zV$v$-uN`CjOU%pH}2ni@Al2=~T>W3KOt1g_e#aTZh(IxVlRp!x)~H&gl{+(4a#96EVU+t#`vcc^0@}tTR5f%UHE6{J`iZBDrAsf z3Hf#W^&H!Nx+{Ct^I4~aL_C}UQm4~}l>KT?pv0m6{o(;Z`|>4w*1~MvgXjeQY=!{! zT%=fAhDNf_W>Iv09FzutuF|lWe(4`r7<;h_$;LIe?$b(xbzFyUGi9DwFY*6qJ;>nVJh{ss02a2}LU5&??mw_7j zr$gLpHS+Ulb+n}-HYHDx3MSiO8B)^O5{eb6{Q5RZbCr&^o?fHfX1VLF&{~3OAyxnc z6c5_T;Te6l|}hKnR<rBn!YtSayP-!GuK@xP;^KlC5JQM;S)T1{B@8r4Io%9)+5tePe9% zDdQS5=yk7(2HAZj^h>7B5C30fnfgM*=J1r1PT-VZ!7P9*4}!=gg(9K}1q%JOUe>>( zkH1a`^B)c|{g=Ca!~gBgEaPn1x!6 zr#EnW(ynXOe*;Q4ep2PVoVBI8o^CbmSh1GCpPyd~ztJnak|-s)J#tN%vP{qJh_ecW z5feb5`P?n@8xU@ExE8_-(kP4|oV^V&=b0xxYcu##%`Vol%75(Ek zJDv6Yg!b;Mj$g~mg-m(R^BGe*8dV}%5$m7okMG{T`)1Ky^-;6` zQSwc|>A|1sZaUnLD_h*}WqWmYxDRnfFGn5>7Tq3q*`{mfG3`=trj#S(CuuP`QybF; zhjsN(g+`k17_EvPRvnp-7Msq(eeWB$G`>`sZs*YMSFEY5?;l1N;2)4OeR7R7JO4Fu z^y#&$9H*;4-iiKPbz}$O%mmv;|?vfWh zemvj0s(t?YOZKKeG)Ap<(;K%yziL9R!vkKqz1Fgk|Ii@O_Dt6q6!k0o#CSBvke&AK z)M&9i!1Cr(NBVWGpI*slO&{^zFG=_FnTmA=frf;-F9S(+KHorw9 zgG-R*Rr5L30apOw)horjSPOzLr!2$`bj+!~oBJ7WBsKma?0=4K(>cqFz^j^U4KBHm z4;{Ijkt8ng=#VuAM+t)$%&TzaX_rdIdUiWDrdo8ne6h76F#CdH(8N=UnQf|DCAt^IJ1jqXiZk>5~&1m3F9JYcd~k0h5R!1=dt5lgy>JrZ=*nx}M=>)6sH_ z_J1!NPo7YAWqmfBQE@6>2`6JwtLcaDD}J*7X8LbKwJ|#BtxNo4o{^dQ7iRPtV}_XR z$nrpNsz(?chMu!OU3Pm{rN*w(g3B_KBCc<~$wk7C&szUIe=ahAnymJ!35`{a{TNT* z3PK1al+?UZqVD2XQk3!9aWx+;yA+9w)OEQv5VA9NK!psM|G!Q7-wpu$4N$uK5dO!# zfU;3XSgslD_Z>DZ;Ns!}VK~}SR7YyB4v!C3T<f?2IKZjF>U|}wD ze@(gRNZlrAtSDa8Vp)B1xBY1REc;VgWxVT7g=ww=!A9mw+gMr|k!9{rW#*$OD|Y77 zRIg-jCg9JCN|``U--Z{?jjr8}^&gA2=Vx+;m<{18%me0N2Hbg_^p%&8D~fX><*rZQ zFZG7h0arrrm@gNFKC|^lIU0zftBCH}PW~dpmpG?kO=KHy;J}Q(qC}3w>S%Etx>?2# zW4^PmA~s=^8@XhJ9Y}cJ`}vF@@;AVG+;H&F8f%cfThJ$bEt`;jC23ZnGBwtjo*UQa)bQeE$PJJLvXAZQV>YBViZB9J?32NC>zb=f%jTW2E?Op^HNUUFN z3qz*0F$`vDX{Nq`DqjfS;d=42-4k)*V5g0}?OgmqR@6<8vdud$tRn|YtSFq$8me0ft{gX>77;9^&$%zB43>s6vy{oF^^f5*DbA+z8N4|wbMhY_B zOSx}T*#jg4i(MW#VPp&eBkE2S!yRr^+?Bj6t0fzS)*ppFB$hD8pFcHO(cv;$^7Lb; zc*eg>VcD|K0lAfz)+tJmor6aYQgVXECralMeDnR@mfrUeKX~>S~ zsRQR~Pm`m&-nUgp@+_J-;Ukz zX(1k9y3q>MP+1pn^E;w5*jUDBNkt?W$RT4IG8Rr+f76&?j+G5RGC!?0);}0pP8ddscJ1aS}GoX-6HgiKb0s|lAO^7e6uIF^4c}0VN)FPzwGjj@w)r`U> z^3{g%fhix&6Ay|6+=&xwXRn!66<~|swz!5ssW{AG|758!Fa}mC}r`51uO)@c&u(c_;{^BI3_Ek&UkpyH)LLf3E`a zX=GarIG`WjdVcYWh?H7uwku5S3f$`{DB|%$WKt;HF%5@a1HYTeD)Mtco|1Mdhkm&Ils%)FkGHIpml zqU6Rg?_8{~7IJ2Eva>n+8E~7D6z{hVQ+Xy)xFNxz?s5BTD)!x4X8Dhj_9w8a-TuMy z)~*6hW`rw2v1S00$o66)xYpsU_L(|6PJ$YY;ds5HoG$NFhPA50T(-^dO1&Yg&_LRB zG@XLR-h^xwgCP!eb`AQwQLiN7LmTQS!uGQ#T^rg=vypj18DXN<6|Qe&5VNJfzKS3z z!>i`AD~+r7nYW=FBW?~Npe|8F%9W+E1uMh^(xiQpqr1>8*9Ak0QrTrbDmvfWgGT2E z)$nskx_`t>mKxQ;HQec(OeuR@dir~!$Fss|S(N!~Y_*KnT|#kizpZj;1f5V-h&DS7 zso>BxN`sP6vpuX3^j*u#PG|FC;TvyZwJQ!H3kKwZ#aQ1=m{$+@JFx#*K)r(8t8xin zmD~TvJ^tDFwaG?H6IesM|nb2=5FTJ|rDC9F^517pkk@Db5Fys7VKcZ%^X5?+y z)%2#m+T0q|9bmi}hxJ=NN@$ygVHpbqT^|xHR}q}0hl6;~%(12pq}EV^k3-Fw1(yvK zQbZS+R6?Fj8haIUswypy#(!a68I}LF?bsQ&g=3BTc*BOSTquNaq$A@W!jP1=Y8p#| z+R}lxzF$f5T(Y8U^i|p{w9pd;y7VV%H{D4-Ye!m;lUTVbLc3EuVzpWG@p1FI0>kuuulMEZkP7+P+$cBwvBP2|OR_b~HR zpOfoP*M5!LXII@|e76L7vyF9Rn|6cG5rBT)@*B`1ZWyt)NLpWc`poznz@qyu zg@(g!Wd9@`wa)Gn&uY5>+lEFHb&=h*g7@=OR_V(^!adOJ*Q*ekW4RObWo(STJ&Aq+L;%774Vr{`@ z&S9=o@qbwt;7GeMCW@!Jo$aTmhNDs~eesfHdNj(t9%s@qeiIHO9lf#zoki zo?hE;`HNK~{Hgao701@C7t9+YBQ;YpcBaXT*BoiZmIrBh@UzZ`I^JCO8p}5n3`m;n z4B+IIr9-K1S7HcBw9rw0%CX7Vq&~6+zrC}O?vw-9Oit{!}KcVB*tpNWj$UFF}@s`CF=53I$ zpbnTT@Qa+0;tlrKR2uEtt#?I~_gjq32#=u>wEMWe{RLSV{VMM;`tYLZ(-;)PQFIsX zW96RR$0J4_xl*C&HXL^B8|JrDf-Tsg<>F9f8xv9~8S$p-S;b=p?93v z8tD`2;L5{2zCKVPP1Ny+_lWk)aPqr59Gn_OkHU+#TG=Td8#lT!3d)pm&8+J@yAwej zv2>!ix1w?&UKh0W7#XYqH%C26N;oQxpx0}wC+DT?uCFV^dR+jc0 z>ST(k6FIO{x%0}=+=P>fX47P0Ez4T2X`ISHH|Rp#Roz~rJ?f_HsfQii3R#tt9(`-N zHQPl^^d#3y<3w3Ktb~B}GARrwe~?bId1lm>(VE?w8pTYUAihi5<%!lr-P^Y3kF&eX zO~3_SQ@bqs4FCm{d;j>P-Qd4vy2h@js*6`fzBN_wVuUgz8x?NDR5vd2(v3k}BAU|; zS9}O@hxJ}Bd}+Gx6-A@Nl*<)#c2x%XrHba*)Y?P4+%t?W@L%W=z~V1f*|kZkeYVnr ze0M|{^*N=_DGU#Fd&VDYaC(3DM4Y-<0WF5$8~0Fi691W_kf;1v&RI@{nq1(bf<=?| zcKXNXO-xiXGQ?LqL3LOYbbz@kvXA3)BSsn1l`9}mTQYmFJ-!W(Q@xx7OQnZTh$;;8 z4|CoA3&ySgVPT{(bWot8)wR8b3GLMI+>BoQVFe%{UcNBUlkDd@aAFPA9N)Tj9oZ}{u9`P+7p9L5ip?>+ z5i-u)(0Tr1{c&NU+L1|BOA+&9TulJOWBTVQ{FZfsPFCmpg~=iB{uy7UbAXFN~|@*886iIetQ$ z5~o&gPRQQY)=-@35J#>UHvE$Ln#%H%M@ZE!^Wrm?Jvc4X2{ua{V}eJYz4@Cc&}r2Pt+HzAh7x zl1_9e@cZ_gF9jxb70qlYD{jhxoHR}SQMh^Bk~=Urbi*-ocnW(3T9L<9ye6`$JZlI$ zrQH$=$@P)>zb8e+@~}&tX@qO|5?(BkvPzpLn$|B2>zA>YK&ZQZ5>rRXC7or)ub5KJ z_gh8`Wyj!xFqHhz#~Pbx<>T}ib(V`Ny2E1pC*#wlrH6IDP1eiJgwZsEJvgbf37Q!s zyDBzrtVqeoUFkOfGR)#dmhGB*`OjT@2vh+nC0MrSs0kK6a31ePeJN^w(H<9vEd6rR zt!Mo4cAn)ihR1fZnNhC;30j%5iC7-_PUGpB0S{TtsS5HNV7K{6a;#2zw_8Q-kdl?M z91c?s0YoC-8^%P%samjUNIQ`Fwy%13hdg4q?gq0xiC?KP6$zvj)78qDYa?%OaHXfk zu`3bHwu>SM@(Y7fhr#ch_dTt6P9fO&Y1yIsy5^&K;eUJpwwV7Fe}nzH(tGb-*R(*kt31?Cn4NbM>x#;x;9Y%U!D_0Ck0huj1L`O<1z=Wj?VW} zD*hA!N4e$_aZzby9$vzXEE?zzE8H{r2lf9XDGA0A|F4m}Tt{Lm-!*+%%Yf3q(zVPWD6O9_MpMpK@pm_&d(P@ipW$d%FC^H=| zH}BSQF^~%fY7tR~qVzTSsQXZs#wTCnZ!v6_$)9^?p{$=WP5EJJ);vzQf3_{o4Hhf3 zL*hyPm$Ji8bnDVewL>E2Y*U2w(^>k`pjlqV5NKp7RVyXb+Uavw8L$KN%!i_IPatZP{Qw8 zoW>YFSG=5Nq^X?0UssFUcU0w9Vp?!?dB$OB%77?bbC;=KVxfu$DwGKMY>no-IQhEY zfU9yrUz9i=6Fr@esrrTXxq^swKA4%>aOksu%hZs{p4$3}Q2b|g$x|O{x zagvkLaUUbF*DvOGkGSd4Do z?~^^iW(>g#KIbkuKoL5nf~ANNW+@KrIL2`g&fQ;{5c)4ZWuDdc;S^o1d#|S*vg@}I zOQd|1;^Xx?^yt4hM2f5o)|hb)e$u+0JyKR!|M9^6fi>ymyI1)42B#=!t3A7@ZK0pP z=+e7fuH-*^rg4_?0u_G)uKh?ZBe3F`)pYnKKU-N@Qf1pISFj>7Q&H8b4e2frqWWp8vUw%SRQy{YQyVv_l>ysO^KD8$qmc5=+YkdQk8dO(8 z%b<4z#f^D)p(PU(1&Y5``fw370H5GHU(^Z0{l#^-{x$RVM91X^njz+en4EVgBcI6z zv^ZW>yM#>NEg%UBLV{IoZ-zS35o99n;cYV%-(2j+PSg!5hinq0_K)FXo9=L1J3VDs zjeZ?O#RS_ay;bU_GsCL~lUj73GK#0aFwz1N$oQ4!8sde38t}W{>;sJL`--k!Ex3gQdpbWy^sQHFN@?UbdxW>FNKJ5}t53|bsS>};W< z5fX^(qmSwWK8RyJ#$i~O?n?xzqmOja@iBmQ)_|kJPM+C~AS`ww$D==O+VD z3mp)SUcF*#R#UJVz`9 znq}=uzTMnZ5e?0%P!FgZY#fZb95B_M-B;OZa?_44bg!TXN(kt78cj6WsIU;+XTcv^ z*+5Zzf1}kfOTP(*MRr7s)A(+QH%b_WgJ2%Ce(K2NvDaBST*|wta(K+P4yL209qJLzRBarX$N}X;5Pp@0YZ!3B zl#cnkHRCi$Zt|Qs?}+8#C^g4(RN|eA(`MONHy%&R&uWmoCdEIVJwLPCWXjF2AuIP( zz*Lm$OiYh~xw-H4 z8I`5^7fEMC{2_`U8j*~~0Of;qQz>H|EaKzRD#sg)NJ>v00hONxeqIA1s{z!XHi>+6 zo{8QP%l(K+3!A!im#)T9zxLO0ys`8XzSdIdjXV=wL{2MwCs2K_kuGxr99L=LrentG zv^c-NZSGF*eCWszm)3njt>_bDpw@&)xZXq}f~4$;vgOQqrv=F@S@ zN6)-t?%8`YF>nmW44ear)H{tKYvYb|7fUM<=cx=eE(K>Tku4?NiGa2rgIF>CMC@`P;60MfftaRB%Wk@*dV#t>MMW25rCW?;+ z6Vt5z+ZCGw5prThj;^&!XvL>hos5xcpV!v|J+|Ag;FnGmKPHiy?{$G&e5?!OZATn( zwX)`d_AOMZRac8#27xB8ARp-rE&SudbJ@G^&od6wqyXj!w;-IkT?ejK`qcYJJUoVn zs-MWT$rb5w4bC_b*v<|~Y$Ng`siBwC+ry%6uN{A#r>xPdD`;?+XeLoPW{e z7|^$^sup1j7!b3E#(n;l+sdAy?Erv}&zFg%s*2yDFORgR69PTVp;{E6`#w>?W`s3Wz(a#Q{UkmOeu5*6V?hg=cf_X*UQs7+F?}L1qT} zlSuUz23J?T`U>8aCMi-M?A;BXnL}eu1)BMis4U0FLPtjIChQs+G76dGKB`;LYtW03 zrD2yuqs6uvUnmO&Hj{Xm=!IG7w%QvRQ~=Mu^@@|3tZK1pGlA5@WKxcIB~9!!2EbX# zs0aB9vZnVqj+;vEfh5f1x(Qa({_p!VG}I);KSd}kZg%$AGrCbEbGA4pM-6<4Ce2OB zFCWblnaCe6ki_Tn?hQZxnQTp~t8qx*U=X*zJig7s0P%+mrAKGsu5sE8o3dS6kw;pW-#nEF zO3#f8LmCKmmugs}PcbfgJ>Ai?Rq9()$y`Ta8k@Vs`y3}b!#3`j3Z~+bX4`EX=F$-= z%n}&RvtmWnD6K8w@AlUgf^mbxn_;_&9fipVAZfCi;$6mNdpFm_vp3nyK#6`}^57U* ziic8SRn*Uv^Yl+2^l0g`naLtGN_@r2J_xZ;@4%45nMrAi*I8L1=%wuuc&mnN^3n#M zvBNTAC)W4uDy+99T1!!M0c8t9nmwth9T?^0KlT(0l-S|YG&w%?D64jsgpP_0i1Rcf zRE3d>!$bjX)@DTn9dOc0SnWti#Zuw4?D-RdRu(QcQmFWM+_i2mYIMEZYs6M?=E!IS zp7zrun*_rdX~gTJoypiw4Aez0yYSqV_K#&oJ%F>(6`Qk%CW2qxDwnwKlW5+DKy9ht zi~lNz2Mi|}&bgqf2{VJ5MX&pPnYiH-)+T-q$+ytSMQwps()Q1;gi$8Bk2 zZ9x?KG0=*WX@Uy&3~De)JyQ=RQ9(9XA|A44nGTb?6@=od+=xNF;UaIQ*|CxH?F*8l zWyZqGm40_oKfRkg%wm6aV^`PwR8HzMcH;(6oY z?mW?AJF{k7en*iU7jVN6y(uTge~wjo9*AGZ1U$b8>R8|N(kROQ^;5c`@{v7nq@Ee6 z_;#eCZ)#H`rMS#?K=CTQ&r9WyWRH{W$iY-15D1pQ#*|saOjL}!^Xv7wYta_e4BG*@ zsd6n?KUoaP)S-dE`}1l6r@tom8DV+*wp4@TR}A%RKJI`S>Z@VAc~V5OIJ;ea-p_B^rngOuit{|{$b~eqqJ{)EZI`xk0Vy&1Kiqs$jv$GMaBvw?6EMH@ z%#(%lL0W3oD6+`Xwb=E@WLaj%i88;B7_?rdNHzc9aaTgc(wwlMjeyQNX z#mVTA_1PJ^7z#GQdXiwcav{3eim_&Fee~EV^y4|tIa&k7Ch~XIB}RSjxX zqP^wsWYUiFcmqKbKK{XArFSHNl!kkv?UORm*j2w@m5Ql>xtZ^!NunLZI*QN`p=@kP zU~T&gPTPc`_dw3Q$U+LSx#bDwSI*eO*e?mA`9Eo;WlnR)`H_5>fDbQ*VxZS(S~eC}j7#3A*Cgxi5sUoWFY* zSRi6}eSy5g7TX9x3M19aiU$-tiBZOtIhK@^o(9+NIF6}iekOr|)PhNZFTG}PxSn%% zd_bVA$+PY4HSN|yaOU}w{y>+BU@+zPaKL8qwG|%H)-H65$_Y*{)T26NVtavyK+6*y6R&Q#)<_UG5R{P+&HW213@HkK8iK87 z)2MXZibPiANkBt5s0dT#4UeQsBlDSxh8G#^19kJkt-HCH5h68vVTHX=bcu6y=zLxb zkQX$weDE+~+%=EaD1jEcN!w|$N0L%9EP&sJVnLL5>nn-M=m(zP2r3pl!HVxki* zTMna0S+`+>A$XK+YTiID`OSvRsB;7j>wB=`McG()b14*^S4%>tidn4lLR4vyu@5fr zyc_~W?{N&-FW)=bY4>6+>o&$Zi-(&gz;58qQEPs`N!FzgXx39Jt}9cK{KN*wte-^V zpIeh%D1jeZJp;h@a#{V%lz8{J4SLj7=+dFCM|cH*GLjm98kd7Q!P6b_Oc9V-U)xE0o0Q z8SOhWwZZn2pCny7u7kWL(PvjkX(C|nDEcZtERFb%aHPj^Jr*7&DOD8 z2fHk3xco4$4i;&hhb|Q@kzc=v=(biawqPu?-nwK59qMSlpo zj%2{a)zgI7R6L-)Jt(0`$74|pe3yLO_{iF;ClI{m-dqfk2L~rpmq~WXVh@9}$J;`tf*%73n-2f8P204lgPW4UHn z>Wu3UD3IuV8=!Qb9p|+wg@$-5n>nErmzp>c89vD8n*7x1Wm`Ma5NA46w~o5`gx(ez zHB=~msoo99^@Le^fqIw0{krd;|3bGDgO+w@K>bX7QI3lijoj1iAX>P3R-PewHaeZB zwdqvAzQhKm(^)JSew02T?Fz2I6G~qGV#K499G%}tI9ske7-_JaB!~ums*!2{j=^E+ zNI*zP)X;I^xC~K}_F5OCz{|}JpC>bhk+f?WlWzdm>5F9?x|r+x3Klt>o~Z-U-8C!e zlt$a*;~PW4oN1Xu9TUxqp!am*Qa8r27OE+OajDB8`?^V^o9z{YDUm9l%*|T3bbw=0%{E6B?wDwk@n1e-~2>l;YcMfL~%D*Ow!I?kOjo(k~dab!Vs4JdOhstox0n${Tst2@idAaXT~) zvZ<;u+6imcG4#SWi=q-u@5C%5-R-IKJH1T1EjwWJZukBOXh^t*Ep7P>j?i!v!O zsY$z7w*rOjLvvj449KD(4U~Y@HkU5pmCpN;6SG+C;)T06z)D=V#f>ud8ug>M zg}dc7h7(%tw`A>krVzR>`-mmekx#lVzJegYnl#h8xZ@r4#z)}UcCt*W+hdD<|n zwvw^I?9bihag+%qBP`TOXfPD@F)=cfVPlEbK#x7T$W%D=ch zu9_g{biwl_54MvQyw3qKT@`ofB16+=zFoAQ911I;19nl-Y;*aBU^}okyu64ugPa8(r-T9c9lk}tVH6OV zM(?IXE%rEURz)U9(Ox7ZhliTzsy8or2x*DSU^XH$n2Fg+BO=s0-idbC2iXSVhAB9E zU>4s4%bGuAxgUUONhQ&ookfH*%fI$*BQc@e+j<4M?VjNyU1msB!5G$%$bmRdZ5k6# zH3KV{kheYf2p@Owy`VkGakw z6?cg)4#%SdNoNgt}5M_7|9Gm>13R(`9dfkfACoVoWh{O zCsU#5f$@R%dm?M@6Y#Wo&!bWcrPiKerjFjugIhiC`bHbE%~{Z=7~fo!AoC#uzMZ-w zbs*p-qIr2d?If(_`FGuL}*Xc5=A}K9oemuoCi}rBd{`vkg@}O47pgD}OGO!e}V9D7= zdA{Q@*bxsgA?ZfI{!l63+>Un@=p%XjibBLL!NAvzi4kl-2hi|RtihzEx)o>G4l|r^ zg3gI`I!_h9qp`$1&(;uD?Rm3^!KEdaY_seXwUx(tJle!Y7f??AlJ;&;zTo`i!?=p` zJOKd~7LX;CN!lH+V!eCH)@NOqUMxw6ni+l8Ao>|l?Wk-Sl`6HYBoo$VLW26T2*aTg|n!L~)Li07TjMMc%25`R)5_HjF5E7Dh90yL^uVe%RebEd?Fua;1{H)i0z?o2xndVVaBZPdo}qHOHBW(Qd=Lv$E(chf zHAXYJYcT~WCPk`Nf?BN5pM<3>K>~U~kw;k8x@=kPTyN*y_0KWAPIZ?SlQLVe;dd}% zCJw<>xx0#FJ$3E->+f#oX8^anqcS(=xwSD5T|YWXzAv>#fz~rw!;y zzX9bGl8|Jf>dmT{4vz<9e9daH3|^f?xuHmCx5S;oVN9UY!u#|k5`MH0tBm?odoJPW zK4MKfnz0GyU5?c4_+w=vT;ko6*#lNP$y%P6Wdp3q%C(>u-EZM#$F7OB+d;G-QX$f^CAeaVYiqpe#b^Ny|~Ev zrU|#%B(2;GT0pX;yJl=S+g2nS1x$#SzFHy(O1S{Ii&`*Hs1FU!1lzMwni5)>r8?5s zd(45s^0i$LL~hd@cbq@aA^vF%7y=B&u4z|XhGf>qC6B#|L;$3i&>sol78$|e`xqN4 zGH+#F`H^=gPYQwdWa8m1*@n1iuxSw<#~D>?Y|Yk{DoJ*l^;Nyj2$l6sbaXVW{>rL9 zd;|noqjY5&CAr||ByvAtG$(-ym0QTI<-7x!Oz_hRaczL-qKS*NlFCujm3pTAP>^3m1qW5RVC{>a5%L|M17v~Nof8Y{d)`YiYLyMA?p z(h!y;bdS9iuNjBa{OUV0G?dTkdX=yL(0S}szH(H6Uma)D)&ekyA(#d2!gE*cs=vO} zr%+dO;)zt6Fk!~#Qj{k_KT2^oJj7ajhm!%Upj4k)+Alo?Q|a+;FIJmZ*Ulm3&^H+6 zysqbXT7*WbemqTIY)*(;sI0@HA;OE=L|;a_2qPM#&ld4&;nS8epkhj{ZMhTNXoe-s zgB*-;r)q~)`StTEg7|DYV)1KLApk7CJ3h|9M7&R$DaHEj&Vi`MC)}+`b~J1$nldem z>&V!=bv@d*P2ZPg-6Bqp1rv38$b-t6E1kf|^*k(>O{GF(c zk)hFKU9x7*_!s%UT!T_iy4}DLvIAhT2`__qv)g+ICcsY?Bk~bR%d=_XOIo!{_ERwD z$zTNAU`_D{C`vjoCyhbl87@Qp-IRA3Bc5u$X65g|- zst5pTg@}^&7%8Oay*_5vm_h-kvppq--sfjh&-S~mM`bubNvdKN2z|U=sIY578B5Ow zFuWlmi@>DzWQ)Uu(AZGru-gYG5APNl_tWOY-tFWJY!OkWECn#8#R+6@sxft2xvsdF zJjZojO0_oc&%XXeod*RCSh2k`M+C`@_ zQI~Ui5j%NYrhMlk_4VYmHDpY4F6si#V6|1h_bHhbv}f_LR`RiEP^3mm7#zkIr_S66R^@`CjCQz=4-~y)x|()1}k0lI`l0 zul%LyFzR&5y$z+7M1K!Fd{EF*N`!0}_k! znM^*}u%5#RYnCiKE2}&pYozIL8Ms(4F~gbng*SjPSINaA%tNP88`3T8(czbY`pwMG%vd>bNPqAje5JEd>D7Mt7M5De?-yV?Yl`-z##CCSz@qX#AN0u>y0WEcS4hlcSo7XFg6aR@?3*9 zLrT(}66D)nBurOFVx?}E9_KLq)pau|f06z?$krh@P*tE%C$JfP3Vce!sF8600Hk`< z`KdM4%FA2tAPUK^VK1t1PczPxacjv7cCwc)W_hY9!cHJ0k~eIL>1%wWzvHcz^(DUe zq}#WF!+VVKxcR=9oU^^iW1E4?_?c2}AtP}#^LiR73ft;xULbC-GOQ_7CIF`on}f>C zAG%BJhvKv3cT-=guqyE*+G>6ptXIbXPnsQa2<0tCl6#$Y#5#4}|nhN#Hl5QjzG9yz550w~v`nRo-iVmbuZ+1!q3}y>zjRGLI~tAro@1J|TSx?x20@YW4kB@aca3y? zF}1RHen(X}D{^;R{sv0T#F~+W9kVR5o2o)ql=p4bQvjHPVoOoVTC$;prb>)zer|Ft z3rN7D1cFLI26k?ir6jVkGqiHGt-OY#&9J<|1V?lZpP6;OoEaRwDpVwN8FR1UPFDyK zQA5D-biZv^EZT!ngM)~^%tmV7n7oH_%L%h|kU_6(1ng~u8#WEl53&{B5ga$mcq8@s zUv#2K?Vg>EZQEf$54rb*)PN`f?Tw-4+U@cqOL*pUCi<45sP?EhcDttJDIyIuKXs^^ zNVjeP3DQpq6H~#~;uC`cJ(X%}5IDV)Pb6-X9kmio(m9&dQYppA=cA@7>QJSqAS@sO z_nxOJ0*Y~`5_bA1-|Ed>6)e{=vvvqDaP0bBg49NzdUpFiH)1`cm15F=tFgSX2G~U^ zp6=g1_VO=d5tXh`paBv=6nLA!kp&=tI5r@^~)J zqJv(Y9D{5lx-Mb_N3#*aXU^}0h1&?(J z?+tMQwtF}`tRn)6H8BTmI5^=+fI+SyKmP!2&7!txEH^MJ&?98Q>~aoZ2(Gm)tGG$; zDAf0gcbFSnP=u#aBpC-r?1*%{Ss>q$Gg^>4HS@ZK+8~lP>C@|bh#XKvcSr|XZsDe_ zgeKh1B~INacMjI<8!!HurAW9aD+7km2-`dN$Z-&b3yR&@PL8)M6cPmwd#BlREm=q) zDh4gJAg6)X%+>4`!_MMtt9O~$75k^Dt0u};Ey{hFE9l?a19R~9#_FV*)(nW_y{uuh zNQafWt9R{~SF|^W+grJQWTNC9xK;9enrx>EK@dc#Mzu-PwfC7rHZ6%J5|T-%k~SV1 zaN6e(v0Uf#<1nM$vvpc%g&$L{Syycy`WA|P1BF7GqhQRtUsm(4Eq!6~!`RTD;wn`dRv)%!qbT-6-L(qcWw)c?yFtJusUIWD*J_w86xE})@GoB-{J_~~%1BQStUlLLyA{%4i- zy|Ox0vTEGoz-J9rASQ=^(XwywyTzwDI6qM&fZr40BJH#`ZA?U z+XGFm{LQEAEZ-(OjC;_`W8MxKw|JR> z<){45Q)4i*g%--r-j#aj?g^GJ44gVT}p7fv5$n|V8%8xE9Ka@y~!3u$I`y;jN{{Z85S=}8twocK`>zIRj-Q}4W zG>EMF8un3GZTGdEnl$%5*n|$Yr7NouDw2}3Ci4>x8nZ+wfRV1Q=SG}8q1xpcpFlgXJYzp1zcIEMMU0MO8q*W%a;Rzc(Euf$jxgcr6Xxs}zj0Fkc*bI;n zbrDGl9Wt125F{(ylje!)Y+fK-v)Wy*uexy5R#|I|g`U2yt*Glor#Eo!lhI$0(l3 ziFXHN`JEJ}g-iMp&oy8w3P9U3-A*o7FbWy()A_sIQc9Flq^L*T!V8WrDM?rwCuhm- ztj@I4({U%37J#Izg%CJu+r(;8YBuq8vXsG}WSWh?$>P9CgwYAubv-?-vM(}Xh4PfD zhCrm9uD0#0EFh68olK^ViM(yVIvqO;xhh zN&(_b%ngo^IiRUW4$Fg6O4D6v-nx{LKoPvBQqan!H%*|VtQ85uDt7tZZQ4?94*F{{ zq3qM*7%r^{7Y2&ZZi;Edw+QDcWj)L|WGlKW?uLKZfg;&e#4e{{s2#K|gM^@fVhkk4 z?R9rZPCQj9*}wfx%Q%!2p7GEDzi$Rz228cf7`40HI`?%}?PjBEZ`rnzOpvOhfI80J z#^Ea2VSUOUlHvwuUupsR9oy}JH zQmpcTB4UHtp$8Y4ri&K|f_`q3F`SZlvkn3z4LIq=;JH{$0TEw6(+$tUl6|54xBA23_NKp-fMJ1ERZTUeE}tZPA8 z75LuNlLTADM4*ATqDz3ADM6V#do^-pAbfA_b1cl|E4+z7kW?m%38?xh!HP6U2Wq!y z3803@FgkNZPT@NAyzrRjKBF;7@1-)GLSt^ab~Vhtd3J(POe9dxdU)4%l-l{uU^8CY zk2{WU6p<1@)XtiHF05K|KOL?b zXxkIEVy*1TmIx<>S~7D5B-Vk;0}dpeM-65Ux^+2dx&R-eX5=f{S`wv2XsHAd#wUw$ zowYOC5dQ!Z#mUTOrpCW_(B-D?SvZZ%t%U_g-AX{>A|Po=c3teY6oHelV0WrOI_X~S z!t~llaPh6@KW{W>)N^^PR<+#h6_tcWSt2n+5+o7b1R1JLXKOz}V5L^j2H*;hZlG}j zHkP)Fa}A)GYcl$z=R1PBhqlLq;UQTmfGdqasO=&YJ9Tz9{3_XjlUxY@0I088rPG?+ zUM1jD@rQe-;?DOuIiClK{3`s)^xSgo=DZ?jk%zX}pD`@B(_R8E;>)Mw3DNZy^IjHB z5P*ODe@3p$@lOx4s67_izkpJGq#X$P>p-e%FL(z0ieBfU$9%w@Qurl^=12Rd{K5TN;L^4y8$+OLJZzjvS;v2_XhQ}5pnT_XVN7TRPx5oK5>|H5INll(i+4DxI)`r; zJk-mn=O#I}RK~uu z^L1tSs^tTc*=B5l!pfxjXGds6EFK)q+E@&E7Em?WKTZr4J^aZh4(``h+6t|yXkg-oWVB0s0;Y%1!AP6Q|&k*61;%;>`# z6#YUac!uK|9y^Zy4eS6?RiMKLiU(gdj)6r$N{VL^fi;ua()BA`;R+IHXNO~Av{xim zC|Ns&U=>XKx?Y?q6B8=aQK$3jZHz+{#)R#s0i|7y3UGx&$bx18?fBZj$q(C9DI@?i z2WGo#!K{-SQ%^I6YQ`ch+(5*EAxRWfXfzWPYGTk@Yp&PNCa!VSmDw8~wTDItuAtpOa-G0mOXGq%6vez=#|E zC6~T|a=~AzjtpocS<~lNFcUE_Z|cc&n?YeMAX~C%Qq_XM2oQjt2F0dfm(=VIsBR zxy{(_U-1_`o??-T0Rcn>M*7qaF9Lu{s~~IJzjo&qw{GE)g)&pr^;l{&niD&zPmRnG zFx*!tAjTYc#E)d*jYfcmTf1pkNvc5sNfgsjKGk;CBF;Pe_L8cWGLDs<$ z?cXX;?u}gaES8m*626B$(*sz>IRJeAdzicZ)DwgOF-`ur&@}aXMcguFMp#|V${ZF_ zlS4J>E_;b&O~i(=O~K5y_a(9_Q;5TVmYyxlAZQ-AxQby3(b=@6kXwA_SO}TZR1VVO zTB+=BU;SH~pa)PAf|I-C^cgx>3v3mvN)y&zVZToAp`af?7@W5i9n;?K(`_7?+~J5D zcYTKIYV8x!z!TDvmG7qNISE+;k+;&=Qt*-;L7>&>Fio*D8|r1uwyZNFsL1pN+igl9 zxzq&pP0Na=r0$c^b)}xD}t)!|(9h?17+5z8ZY4rS?UC}d6KSOXFQ$B|h?U2729Gwpce3hi$ajR01!SpORDcD2*jD4kl}+OHntUdP8jxfwvZ&rxX_92fHsYk zZ`-z$V&KOiT5Q8RHEGuuiAhXrMxm6%Q*#wSI`mGAdKxTBI=)_{m87qgI;A^zs|&?r zPzgy)pprq|uc6fnA#)`{5z)N#!`taLuN$c&ze1g}zo zkQ*255-wqiEmYEwG-_tyz$a0J`kNMlS9M2W5O+vCdRxWRQkz|bEY+)O%po?wOr|Ix zYGlO&6=4cOk|atQn|9>e?$QLuM7pRc0S0}l9zAU)Ml1HVG+^OE z(~S!fWWc3I9nF+?3hyNU01ieN)Nn6Bg`^&bi2*BFNYzBDTevA(r*BZ5bS~gk%Ik_Y zcaGTI>T&t7Ak1v-{%u)l0V|KcsMz5QyVXe>s;MFwC%Zkg-MxiUuI)WsT<^ntZEp;} z500i|LfWA&7QI1{(=~>_lLddJa@ZJANgjUI(rgLzx#AGAt`PH-?~#;ML(SayBTr5JdWizV~^A>dhT=YHS(Jg@zJLgKdP*u<<~trYo$##INoSvLxM-V+?;%vBMw417O&U1vEaEd7X-DtnDY$}_ zfS_nW5(wNud}oUORDhO65mEs2Z9!2oi7# z7N?oX=6pZ&@1UTb6g`AUJqzmst( zl0lu>VlOnO@(2b+gYV{fgya~MV*8tLYe&=*y)h0zN^PAxF)BC*A`2~gK( zsoKqvq3PpRdx2%(ClH|khR-ct(Wuu=G~V3AF@(f`>3KyeD3}B=1{a9z9Rz%wn`tD% zpwf2rYTr$-dZ%PdjIyAv;MD4LW%&UYDkx9I9M;XC5`d82m}~)!ynG!o9>!P)lxWk4 zsB(&l?DV?Bqyw2j-zRWtjwV?k1rfB5k2{v(>Lv_h9Dx;n(QUf(5Kf}K4`dw(E`~O0JJI;qtboIK`A{1-Hy1`vi3~VI)o5=wpt4KRW5>H%u+5xnrCwEGRsoV7CRfR`| zJs?j*zcD&NJufjM-3$OqsBi?w6CRTes!4++bQxlZBiSkF4!sKVeGCIo6TFVl?m|Z2 zH`4F`bx|HmFx&Uzurvc{?t?SYN$YlnEAH;4523u2qggBKV(~|G1dhK;VQ`DB?#w5lR!P^z|yHYALwY|<;60qU;)w(W_kOdq0gs#z? zAcLZuK$28Sl0fr4)tS|0B(3g93sQsyS5vXylvE;WDy}aDi;*ZA=wxZq+)7kQ8%C_i z(lN1Ir?x5glPzk>Qh>0MqlniR9n^5L>wrXR<)jpjmyFCSWntk6KpIdEp$)9aRm9^? zvNWw8kZ^!boh{2m=@fa`8 z5<%;ubuy9+fhi0DtAHpbQxTyXSZu;ouq#(4=!k*6)R3!S37T+dL?D1j;&-_hEd)h= zI@>a-5IoLB#T7t_=yU@SJAEPC>+$Pva^}?OLq@>w9Zx`*^A?5?8vJ^gMky8#8AE9ndzR9arjP&t4yNt1SGPT?I==DPbg8<7 zAUMxC=|JCB8eu2528{uEi;LmdfmvYiiwSO3#J6=T;O1;$iyFO`vzoT6cde%4 zPN`6yp54S$KRoe2%4Ha?WECTArI<~+CUMsFhglb$B55sW{X;r6B8Yu5Y$v8 zIK(F0998P-Xa^IBcXaseY}`VHaZ_`UZUlgLu}e-9@#$a*z=?NLYFp``b}^GyJh-&s zp|8f!;p0)G0G9Q~LX0j2d%UO0xLt;M%sy&IQ9Y1)7R}6vKgtV9j#R$Q@VrN zkWX1@w!y|bTAc4KFe1=k?C>nDPX{baSQ^ukOU54=>gDVc85z#F-HWr(75E0* zSE5?FO(RavpYmwoJf#T$Nu?TsR0nA7{!V6&YOeP7CCHPYpN(qXG!i>LD%na3fH0l^ z02Fr^x9p~a&7;V`VE0Jk-RB7cI-b>9!K{2kxwM@?4n$BV8j6O>e0iHVtAmSBLJ~rD zRk3o~WSQw`2cbY3K&f#qB;wZ(PsyoVoD1wBrd$e1pMfJrASdDGNmQ6@-R@0v(b~e3 z-tSJE{cW7Wrs#^weDYb$2P`D1Clu@EVmqc;XsIGP^kvTKF$$fm0~AV@Hcst#H^Krv zs?SS>VL?FmNjx<4Ig4D4t$^59ElMB&6MiY_h<>@@YbwZPiT$LGgwT0|@!Vr!#V57;cY@|pp9{-QxK@}P7N94fKB^y+ zDQ0v_an*GxaTR%24(8kLH&M2wm2j(9KwW4W4sXXY86r}voW=W=yXJV1aE_BKD(M{y zdfn7Xw~aW(aAFvs?Tvb0kY~R#XLCKQgk2~^z4gA`);kTVmF{e2I1rPTp?{lp`#mmD zcLauO*r+bsw;@!DHQj;*|;u5Z_ zy#VOg*H$aMN#9qGprB(I!H*|LVW@WHsBAu;LlP>$A2r(Ev+eX2%axb=&n8JMq#Z!% zY1zn1Qvd=>QfWEmchWZr>;6qg@&rJ{^ z+S7CoL(h}(5T_9j32=cS-w}cdYT1EVCs!R}PqiE&x>P$^0M&p-O2{N6V&Xw1v@Iz_ zlSw4C05%6+NgmbIr)R}Ock}`{wh>S|Nvqdos3ZjI2VJf?QS5XCyAUvr zkEO>eE2*hJ8yT4FPDom+2m#pL!$?W#X=*LR8al98eR+%<8v^bvjf?dmmcm&yIVjgt}vP0 z;i(~FD?;SlAfaFFEjGwF#Ek)tqQERiCybbGz$SL@>7CBY0YV54h$^U7&CCK$G~1$_ zTd{UT(Cz+k+sqQB4bi)`%CeBFVAId)V9I5ILI{bhgT9U}>~Ld$5e-3Ck!!j^r%Q~r zB|y;Ev9KZ}DQPF#Gpvd8yK?SfJG(*jJA;5gr0{l1iID&&X~Qmnk7(=A{RhE} zj%V|-6uw#y+_fo8e^qfSu$bV`o(HqNmNPZkvx(*+*vH)Lrgt-uxy-R~R@Jn9)o>{^ zQ0;J7!jnB$!#_1xzZUa=SzqoZc}ndSDf5mY_UTe?Bq=HiNvI&mBY~JN=SH#N{x6&G z)qaxY6I*MQAoOfJL@>0^bp(yBQZH;Yd~b&SV#;UX3L?eWj0-)|f1ySvu)*9PO$`mj zNJ#v{V;)&I(^F{ozuK>+I^UK6LX2!tFWK$^cGb&Ds#cPd)3x~`?W)dYR+IoBkbl$0 zCGb~|?S20M(a$KjnR0HWJN|Ctx{*7Z$Uvw&{LtJ|Q-==jf)B;C$-w|v(ng_CFL@ito6fdEo9wLLAtK}n?Wc_kdjKT z+_ax*X#fIET7mE%7@z%={{Y&uKmPzJ+w%1D^W%SqUkChbyX1xY@RQEHQ9s3B`JT9A zhV47DlDXD@a(KV?FaH3_$e;fJ>Tqvm^I3bw&D^!{w526M0ACQPiB?$NNHoxpS`R+& z?_U1^6@TV^KjyU*c;(=QDFX?46CJ3126dPZF@^CmVP5{625D@Y#>5ZY39g zv*!N*<#QiY(M8LI{{R;6_>2DlGw3`mOHwb&l&@wnDN2sz-RI&OP^Twv+V9v{5fx~> zbjxP{0OhkER_u#+fd2s7=KlcYvmaK_QFif2o7bxc_>2DlGk2SN_pg75zw0ACjW0F13@xnw0uNFwjD zqM3`I(pnEP_wL^R01S!L`HMF#<}(XL!(%gR3<8B^vnE8$f}01k zW4X6;5sbcQ@%%Pp*J@kd5VEx1#HFj*j7n0!>D~8>Ko@Sr^KQui{{Rtx=4q%TpY3e- zaubdH4xuD~0n6s}6#On^{{Z%!!Fr#;&40>feyQ<-<=@=9{v!VX%-8%~OG4}fsYwJ~ zoGK}v`1eSc8#pxI?UlED6NFGV)Keda&FLTeT+h`!=PDFA8M@E@7c=!wlNT=j?eFmx zfAux*3{scRB}z~MF^NS{+`I1*(x*;ROZMsq6!(u~U1r8^NdC*2)<67BpcV-I9M;eL zE@$eWJd4*zNjGm30RI3He>V0jwf#%K;#~Q*<4t$^hk)rZk`L(SzvVMORN6xBB|x0M zYmfXcXX=|CETtyxkd-MSFGxz1nT7uVNgLWeJAi){@A#KL%*j)y7jN6BSWdT?gHBFs z{!=yeO*wqZ#jM%?0J7#|{^{4k)Kr_rN>Yf3#GZqOaU8ngt=}9EP4bp{yi9WA@=wo|46050E_ams5zJOMfi(ZNJ z%jf)4y0mSSm#q)|o`XVYh7|zfNq2sUW4O!7E}`oNUD?$wz?p1AsnOYpOV~S_B*rn0 zjSxPk!#^=*@-T(|y`lG3d&vXcRZl^Yx*sgY4EJ1zisd{>Gi80@t+i2U=-N1m518@m z0?ug{;DL8teWbwXksf#Es9KOhkfN0ef(#IQUrYV@dpDLDY|s}J2q?Rv(nUMef*|Wd zwyz4X#TF8GLV}b5)+BmAn_)^NGqg666>@~sXxp};i`Y)dqw?%?lu5d!;Fy^rpF^_L zw45qxWm1wxpcrV^uJb}oLWZo4e1u0G#SR7ZjR2&Y;WZnYmAi_y+9S3RVw+AdHcs#kQliY3Tajxl<20|JO|b9gxKMxoI}w31Ilb+nCo9uiy< zMY(Ss&1mXS5}h20bY70eKwiBJx2taFv|iixo$@H0&}FNg8bhdaHlxqbon=^2rqc5A z%nsLEe#q%<+}ZKDmdur%PgZpQ0G7z-=jZdHGV?9*o%t2kEHNv zP`{;c>;B8HmHzs>{hK5H0RI3pbv{3uMB%lSOzVc$8KD)zp*o#i#+{^{$|yaN*gX$T zUVnYw{>_p9004a6e|=v5&658B0DRo+p7O4AzBNm>J1_qL-ZTIyqoVwKGja;-;3h6`!-Ab0rPb_ca`oWbv_Y9U+CNY7au$*6HS4(JbyHa!)GX+_-$dD z5nL)0652jBduCfCJ1h^I9XEk1?+#?qx762sU--5^+k2z+A1nR$d;2y^`~mZBtH-l9 zBORQg;$W)QD3d4JQDkYg7oN5PNDY6&eT(D$zxgh{OZ)Es0QP*B_ygwLPmku2?Agku zb;D?;Xhm?SPNk?*YD+OZ)rl6zaCQdsO&&LRf8nw}^6B}r`|kJle3$qG=GMzSMs+?h zOV)cX{{Y?wh+u~mUO$?-80_U269rbW5hhGxQHkK`{r7wOK1=)o^LILTcet)~eil2o z`akkrXg2DU_6He0Kbl10wUtckhSnLRR|{mI5!lpyb1fCL2XJG z^O!{c0K_k|_;%kR{r7wOK1=)o^L-D6-d1YW>&t+vmy}h6Q`kn6FFDh}2c7gHdJo;A zv;uZ)Z&?Z>U=r(mR5$Y8$?bgHjEj?TI|7FA^8$@hac>$Iw|0jS3MpL}hDjh0Q-h_U zc)Bq1d!&0_gtg2>8Z>_vQRUn1K8LLz%>m7Z>%aq&{UcGJlI{~$>#B7Pfn;jkZKqb_ z25NP3Tx9CxhQ=~exwhpzTLoe>q`SH>Wu@qCy9NQ(W$%TdCq-@TZC){Z7Me>=%6;L) zIk%5_56Y6mHjH_Sww54*(iLptP>l%s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@bej8w1+h}zqB|NYcJ;V2q8ELZb`5ZSlnF_2oOkuy9U?AVR3@H6I_A?w-p=~3$Bao;=w(* z=JGu6z2|%HKXA{jIcKI%*L2mdtE#8FYfk^0`?mrhR+3kg2Vgt_05Bdez`q9o(I-<6 zb3txSK29EPZooejKo0Qi`E$JIcm)3?Vj|-IL!_jnWMpKIcVc2v(w8sE$S5c%9xEOn zR8&;-^t7~$j7;q8eE)g?PypZwCdMNH{vX1^eEI|%1Az1FKehj%|A+AiT8!c<>DE(ah_BT*pI|9cMq(q>QHQkTGV12Tk+z+MK`q<`V z5p6Deau)=SS}+jr%KJzM-qW>+2&2DJnlIs8N@qgu~wFYJVFGXy21OFeg zi$!=jH{oXk_VWGLg_YKB* zo?Osf(qg*u*?`%lK`-|q=522-@Nlw70!a6cq+;iu%<@xMTJwS>wm!-b{*V0+ivQyb zb@D4t!io01OI58kL5?iQf%G68ZI8bP#tms87&ECt1z z0xUZl*CEdyZAiK})c3Z zBb)j?*>yUPq1?^Zz;F5^4emBf`B4s>-i}vM7n56QM-erJOO(p(j2^v1oM8J^1HGj_ z`3pTTBM|>R6}N^%Mh!kag9ozbAUY=jUOY;Fp=Lmf^NpA=m9gFw3;e&~C`xg)|L(=( zM4(_>o6Lt)^^M%wRXFubyPg~zj2B+fp9Mc$xdRhS`vdd&ba;B72>!Pd|06=9qQ^I#-$rM19Wk7bBz)az>yq@Rk`LswCiSQDEZ{I5=w zvj|)`d8tIF?d#VH9$FKS?pN7dW`kA51_z+Qe#2rl2ek(Tgd%iqf;&U-#Qv}|jQ=sg zq38AIk|uur2O^hVh#A2^;~FJ!jqckL)OxdwQQA-%S1-NNz^-nT1;uBQ$XKUJ>|wQN;moxgUfjkn>=8@6;q1J zc@ge26Yb$B^zGKsdDH-pb_MT~d}ELXiQ)ei!YeE$tWS9m=Q1?ME|fVd@393c2tua@ zbl0ZU&N?P~$JN?r#k!+7T|>~A4ZKSZKcXOtGNk{n$i%s#3u`;z@^Toua4^kz6-+yR zDSQF`vOw^z&9$5R>3N!dn|j1t`_jTpzev+TO#4pa7uje2{}W69V^tyAVS$!IOIpLh z?oAjP$UJKEM*hnyjz9SSTaU7l^vM50{<7)+@9dnDmsvTr)_k`3cC*ii2E=~~J5j;S z=FAVG6Qx~cb_!A?@^E{WMi%OMk6d97Yh3po(G33mJ^uj72K9eWqlP^FIgK3gOF2cd zt@?`?$fDrU{@|>9jC`8pJE@*WbDwxjJhk4;p%tQ+_+SJ5d%?Eci-?0t{Xv=^0~;^( z{n-g;xw{g5&w^1*oVi-!*{%$ZktpAy+jNmJkv}thDO;X=>S8wqulpXXd(0|DXDWbo z+2ru~=$w>2y0o6SCVagjKtC?!FsouS5l?-@eIgI#5mC*Z84)|#+SwXCD2T&T$+?ykX%G7nv2BZQ<~*J}P}lZ^*HzT~@7A5u zxj)aR>w)v>hjOS?!(242HUpBzV(zhr2G`d{JtBs{SKAxmPf0G<hFt-9a8&8mle7@np_Yp}M4<4y4G^hqL*n5Sa5 z&(u(b713%y)GOUSQ`$p<5%+2d!p7&5{{Z&u0r2oG_+PuJ_@ciXt>*qa;kX+J`UVE2 z4-_dC_YJWRhYV|v?B#=Or%UI50J5=Uy?=o8ysOXK-spg#_3fw{Htkz0ntQZVuj`-L z4wUfn;pzi>9!uU|*M9)qo31U;KLC@ax6=RNv^~uC{R4Osofus6Kl=Dl6p~0R{`yVD zpyvPK2+;Mf)~q<~=QV%|(%=(D5KiB0krVmK^-o|+n*O(cGW#6mYM5})e|1k7@>>Ze zCHn5Hwz(u*!7bKfbwj_+t^M~T8P6zKwvv`3?y+-pF0XTq{MzLU9oC%mpe`u7RHzF@ zAhw64z8aGZTEPeJKw;6U$3J*$A-?(10a!Ef0h!m{7uN#IYdZby>p31O{raw>S{Jc3 z&bN)R-56`-u01^3m?704-4|QJw2*(~ZG}7u#L`R_u7Owa<3?Gz%2qe=<9f64K9<6# z`wLc4>s9URy@xaXIHLLvBb-k#>(e%HS^*VSy*x<4(DbndF|mfH&4wEunI@N^u|ldr zc~m!ey|Xy~MU{DS*ly&PDa4Zzowm@A=Cq^Ti_IUVUuT{#RAgy-o3+Fq+k$J=q(V_>p%lSWA0ek5CZOkJqZ6k>>F0S<(gw;hH--%S@LQXlAtA*na zGe%BB9O6LmNWaKoMfTbjy8uhpKV+q3cpIASnG&a~BQTKiIAxzOgaq7o)Nd zobVW`7pccAsE+9-pnWzWIU8YC8T~q)(PT?>)}oXxpzVl%_;Q#d`|?etQ=fnEir)YZ z`y-f_v+-Wyt^WWqkUAsg=ss@2`_7ASOA<%ar)k^1#uEcc#%NC0*rln;eOea?q&3eo z-RL~V(cp;-bWKXedes?vqGjw}IEU|{>nneWc=b$r}H5kAant<+M_;%X7Y|_rP8- zPV`5l#QSbK%3-ptj<cD?<>j zp=`irTGNQ9jB#{twFS6o1 z=dU%ZDVLDpkHg`kmeyU0D^r1H4fcAnM(m8|9rXAy8|LFH$Ue2K!B7{Awg>ye z{$Ebg<~RqxX$bL8>82{v(?oQ9*^>)3I7&8vug*m~I*)woiuYb-scr=qe>N39df~x( zykE~?lVKU;X!zs$yGJBILpt`iwzx3^Xy3I)3)KLjaVU~Xk+v5;9~|?bKmCYfb<(m8 zl=qTkz7ZWBOZf-LuZF$TIRt^Tw=`f$;NJ#n7%P;MOwEV@zlRFz7pt&@k6BVL9#qVY zBnH!LSM`h=>SzvpRj>KeO=@CQY9g0q>>fD@cE7XO=Zp6D&U4O4hL87-#ur{Z7D-*> z)dkX`=SC{;7gu(O+knSvxvE8{r@oFiczP#?sqbpu z(1f+065va<9_2UcuUyrmM+Nm|P1<^<%A-`hE``ji83~_Xi7s5_oKeUVp7ku$AU(&` z_=0~@|DC3%h>-Fb>p4o+qJI(jxAH%#?BX_fDH3O$DGn%EChoUginqaDdld+zfd+o;@OA6)#$bGCc^aJ4ZbJx0V)4na-iKY)xxJ5ii>exn$A3-=$O zTm3)A5dAydvYHX?2;{8)cr$j*FVba9>)o zf1VflKfHSFwEq2Ug;c9G&vB#sbP+Cnuk-OIOazJ?#0)fwAAGV+CAJebV_C| z=J_)pk#TYH>%_U@y9W$df;`os{B|d!c6&-m>pK{m4qgCjO4TmVk6&D*zP#g}7pWM_M>FlL=rPatg{r26K`jZw%B zPgj~=wYdb(VP2%S1Xq}MUBAlvb=O@#Sqh3Tmv!XM`$Ak25L9a}54TXJ^*ild&Ueds z6$?a0UuXTzjyJ~~SC4+n&777HF&rcAybc8q)th74xo?rP_;FuSvhU3}jJ|K|$>zXbm8fR_q;P!>Pnyut)!}Cf zCMV$$LSw$Wah!Evf}SD-BN%s>)}8K4>1?Y4PB?JE$lP^Qy^NG3YKNwj2L2eLT|nY264yNdHi&->`->H0@s;MmG&rA-qM;!OWqO|5WSR;$pPE_n^ahA@iV zERWo0m;=)5WqEk;Q4Z2ukxq?&Ijgf$0oni)8oK=9r=T2n(t96T6XUV11Sv zh9pVZiQ2|A4%Lj)H+&Y|fYfF36Q1e78W^QWLaiM=4U*Q0pJt!y64sqxlUKhxmqv^4 zvCWbNfjM4)zS;6QM|D`5 zwX&6`1_z_z8NZRmSbIUlXqI;?O!`0)-ot^Q*&10~B-d=ipai>xfb$_n>ig&*3y98Z zk)We-%H8vFZgn}^n!KD>Iswk|LeG)A28X|VcV9A@M%o$le~Ju-d8C~`%|`FEGq?&A zE;Gd$n>GHXwr&9O>PNgSSz59XKvLf8iT5Wy->0fpNJQ`6y1pf(pwui!xcnaSS*l|F z)$XLX2YObeBOX=Ch9#0eTFCS=%=qZD-JDlp9mt3vtOs25ZT%0p4LCJi;MIyfMg*gB zu3!I{AiX-RJ+%Rk!20|y=4zuvA}vQ&E_-TfNS5fU7OMXj;eMeq>u8%rm>m`fw;MS; zz(n{DprnEd6sxOonJRF4n@ZAS*y7za%PF}Um99KcUi?)n(ELr9DAOxCQWMbWU(UPiyX=_? zHWUJ}(=ZnQl=kbz`ixdjpxf^E*|7NW#Qx1s3G0%n{k_m|%Y3)(aS zzRQw>zio<9veb%urrk!x_T`;gwy6PJ4fV6xrci?eHi_qkQ}A`Ft2HwH`HK6K7Cw0L zGi&Dc(&5UL^lx56u@xW7*^>{{XD1a63H z)p}LBn#C`LaD-)t%V{t5vyt zr~xbIJ*inf4mYoAWHT>A$wXfsA&lDb0tu|E%~c(?m9;qa;2CL&v_#r2?N8Kh^S&L{ zh{hQ<+cvp6Q0oZr)abcaVt4p#EvT!|{8pAOEPBpMTd_792Qr&^i9cKa0MqNOj9$=TW|n z{g&~JxjE-H`H*u}B=~ZtqcUD?w67j%854-GtK35-#NZehG`Jp!zR0mf9;)wY)mhGa zPfvNBc~&<}B%19=Y*(nj*5~cwSve!3wGN>2H+Or|qN29&3hIZaJ4CR(SZm@}_&t|& z09_&kH3HQkcUsQEyMk{A{w=xqPI9YrMhCMV^+T8X@`k*KL%7chW3YiwIWU*qN|>k3 zR)zq9qW-;8u&KZeS&FsRZ83V^7wkMWJQ$>O9Yx+cFOOF}X7p%TyMuvyVo5+C371hdDA!&&u|me(4Bq%94PlWsXmOCVcSz3d=k0h6gtz zMc3sm1;3@GQL8Vyd;jF1fmChgxgD(zx5uXK9f-t*%riFd=(*SA7$cGnk#^r0TFINm zBYS>akT>?V?)jEEF!+E$f}o~c<)vjW zXoNAy_l{XnCUw+cNdt}%HyPN^YgUo<gsI&+Avmm1@z|E=NzG(HZ#G++ey7{>FIGoSJlv>i{N>ZOV8Lv z+n~IL>89j+Mr4iIrd?3$yI}W<{zKAKD{EAe&IyA?V3Pz1XY*D4_9|w|+YMBLR#tSL z6}vmndy(UWr8Lpt_E!;--6 zX$3bQXB%f{i^t;_U2exJ2WAsQiL8qpP5LzI58LU9y~2P!iwhM1a!cX3$Bh1^!BqKN zl=^Ma#ycCKvMicnzL@s=riW0sls5tm!}`ffhNM-YWk2WyW=j%OUS@yRlwfC$PlwhF zL$D7uJ!u3(fVWlF7t6o+CGB`?vpm~g&JOkp3b*?Rd~MbI4Zc~=j5cX!8iUHl7=oCR zd#dDkl@D1>jZtqg9XG}{>PE#f@#5c1G=1;YwQpb2+cx{mQN?Hh#eb8r&CVS6YFa@F z2&L&5ZjiP&f0z`)jsy*9ZA3$j)=7+5By3}khvMlfVw}EkO8t$~^kIL4+Cac<@S@em zzMEFfjQJs&OA6Lk zz?Et{BT0+VL51a2AxGi1OT8{MUi@A2FQemOfh;wH%OuZ0Mn9Y?C*i^0eTLYb&X_3` zr0OqC4*0`WMnRwN)I$@nHYM2Mfkwm6Azy=f>yP~i>9sTCto997Q?GY5E z@sN#N=b9&a7cDJ{hf4Fy_7}J`y}pA4=5DgX9qv`w{W2Qq2Cg!MSni=Y{)Z;3XxX}F z&~;$)ym-WrXZG5gpSPZGQ(F?3nhp%g*h_N0N85pNPe1NW=*|KS9ygxVbZY^!90tRE zd2b27YLr5zV)a;?!;>gjsuH7)TSXwZX^N(UpD=;q<^k`%Ul`co?U%kCT7r0f`}>R`p?)7}g2n=@g4eaq zZRf0AtBF~j?*0-M31?#8FvgDrbkFNyztAzD^diFbe=$qLrvMq}6YV!rmN z&DvWTZg$fk3;H=>z%MGpT~iO;5C&ZoPHzy$i{Wo(&j%T#C|Ae)R8Yd>9ThDRGK@*> z{nQ2KHY8p0flV;?$nF_RM_5J=JC1MU>E$1Y3Qgruquxvm<8s&K>iO+WvscmR{YdSy zmuO&halagp(}&|T?)jwLmH?R!LNMh^&r0_XHtZX%Ll?56R<{^^(%3mSzJtQu3UT$A zrewOT_UkZ4#Aq@@4#mcs*VD>&CRpmoR#ktm37h%Cwrrk%b*5N?JS{%J%rS}KWXp(F zou?n^Yx&ywo2Btc|C(3n|Uapbu{;POGu(0)x(G9vX_&wMIz0nyQbCDYHn;% z%>ecFNHJ|8(XS(4i1D-#d*t}<7SlPNtXOAbJII5OnLKk0&MZ5D z)9%Th4pkAevyj`>LPO-Tc|&Aa=R#jWA#FQ^EeCLoxi++q8eLeA0{k(%?Z>V)fM)*P zG7hORxP39P_0?CRr82;wZjvaYs+6v&HKshlyg-pk+p)_KA$1^1Qi?Fz6d8j}^Z|zb zlto~Ek=dscrxmry3%=dcO=>UI9%qcY@6;K)M{4AT3GksWpQFU1fi%mT45HCA+D??_ z^oRMSq^QK#B`i!fGkdDp?FXVsyv08`$G)%At{6~uo#(LrXc&Z6Ug_&oEp0gk(dXU| z=~m;Q5_!Ls6lUnOcvOZp#a1Z_u&~i;Sy8|NdqJdALR z9r0urEBvK*UTl4B728}((1%4#+YFJ=qlqa|A@xf;7n`5W+bt5KebMfo90(cZRaS8r z7CTzgLDJ~??w}Gqo5rfUd32)2%0^_UU3e*EM;8cZ8-~?SX&464?S(FGvoyD%rm|N% z%cX-?pNqI(XYZc;(Sy=c_Pe`m*V1Z6kZ~CeC!DIX7tYLrAxlS#>=o=3mj%(cgZw6S zp8WL>8BW&`%J9{UzQ8SheyTvD`9I8pG%)?7fs5cw0~zM$)M#<|&*n_ZDnCWzNFq@M zrA~1p+~o*es$xl$D@;%}+Cgwx1r?g>Jd1_MqO%%LS=qGPJu4=krwE$<*_L6~eh#6( z3i*v~j@QL)`0k7O9pkedS3^)(voHxOyfb4*S&~%$(8vyI>1g!pqIDo)tC=_0vsWE^ z7Wzd&hh6(EZ!c6I(FA>!pc-Z)ezz?)%R8R%yu9PhFR3-6gjqYB(IS<^$S9@Puynvn z*~P2Zirm79UwmdJbFj>+@viKu0u^ndFuw>ZU2l$Vd*s}No>i0#1?>k-&|8qXSG-vz zD=5}x>+_Y;w&#n8{RfaY$Eym-u+XyTT$9Usbwpj|gjhW!X2>d7a4lS1i7jaIjoWIZ z|Dy22HJf$!h<0u{mhlK`(|Mun6E)%vY0%2?dPwC82shKe)b+giJQ*?^*kZXZ?J(r@ z&1Sw-hHT zz1z}P=Gb^QS1dAAXvx#4K3xmQPD2a3Lm3v;Ql*p13tT2}Yf|b04*DOU_YJsQB=f*` z_kbMn32V(ja&=S5#*Nxh>*&6;|HC|$^|%i+k#i0p?z=KxeU7{2j3&tYFD8IB>6y1 z1||0!mKjRPv~IFrwBz0c%?72-dX$?jKsfq6C^agsf z^!zySY84D9Iy;M++_P=QEzD^Dw0K6}Kdke|z6KYubzAysp@Te9knY{R_e zB9ypOjC)D+t3ZU(9?yY2Lp4O7Tk_G|kgGvVd#m#{z8jR?yTgpcb*;OY&DXd~f08pFZ}9AMRiL!Pi7Oy)z5 zES&Y%_5yOW%RJdbAu-k$Q#Fi^v{9aAJe*$om5~%7`5+tm zhAmBm05V6UB;xMF4o;h?;LZ{h`)4d6OinJ736%FmdQDu2lI};9d02`UokZEkrU1qT zi;n>$A6UQ1yBIZ^4l|uq)N1f>2a6U}Y_{ zZiVx|xBX0oCfsP%8MGdvw3}VnFD@h^^^-0UL!1D>7tORs`ElL4ekH9UBjrz%TUxOu zXGKaO3+hovl&w?0C5cRrubqpmk3WJ#*=2AdXs*fQ!{IT0orYO;#BLQqf0U2+KDBc4 zu4adGW6A;fw+BODtCq@dq5GiS<^E0op;w$~8BgkKds{zLR1Gf@fK^K0Eq0qlB2dXX zx)~KY(A?84H4K*3mdv@C1=nBb<8h?OET`hnr$pBg=Nw?6!c*3cVJMH#=2Wxzs-~5` zx&22Q&|b)9D2g1o#+k6Tjl`DtVAZ~pGR(XU^GQska7I#}0>(4SSIW@p2OnY$m&TgRmT37)+}AQmTTfL^5QVlBtMSyZY{~XI7BIN^4pfpY zP>E|1%yzTldMN1X{d6>|d9sS0*wT$}W>zP0drVLB%#szrzqc&7djhqWZdu*E2fo@1 zjA+F^rlx&f(z^F>PJY~bW@T`3{u3{!wk(}@pVO@RLLXT_@9Ja3vY!_ER%`uw+}>@; z%J>II+qe`TfSZ`k3k zyTT9iw(}2$Am)DWwfbri_wR{Szy zne*iNz&kT3mF`-@rlXbM_d;g(oMUNxc?JiW%>@uB1jZ4?EA{jlWF@vB8?pP2?9VIh zhGCbR>Iy( z$rlGIaV5Hi1Ld22w7|WvORC|wnVgo8sXlUxev-H22J+II6NTWgaupygwJ1&j7x6XS z$JZe1v?=ha6DDlp%jJlD#7GPc8xcT z5ggzj;c!r|mrjqga6}~i zpMQW+ej}9dHtrT%(Fx(?3}SlLhn%|EWKEp)7(y@q${>g*Nova_mKW8OI9LijzG^|{ z_RV`Uk?1_Ge!rG@2}L$)b>3Sf{{z6_8-!O@egPPv=@AuvX!I8<10xd%YsWa`aIU!< z*-m`{p=Y8TkHvSX>_E^Kqhu6UU_EVoL^Rp}G6%QW@dTi_xTwA?Zu!|89p!_x-;uM% zHrobsNrc*pWD!xH^%jvR9tN7MMkk-5L;mla1J|b7ucw4Bc9iA@?6Zn4J%a z<&bC!mx~Oov%@tf?E-E5^?HI+mP*n3m)U-$=00La>19i3o#PIWu$1&qRy>wM6&H!u z0d2#wyS0+@6+YdN>6j^qQTCGPZ$i+WBwd_`mco%#xPCHsdi#Jo7H*3=Qe*Lb7NtMO ze*!C|)U6ku4pUYFY_Cye{BCWzMKyU(Psan*j#(`vq)2a8qzY@|_5>3sLJHmp z(^8SuPob6Lvn!R;_UjIRJ5*8S5I0^dyuBZ4l0<9tGJEciD9bY9S@w1`Yw0jq8a zJk{QoFNmZ?uP{O1L5WpjJa-3l5HyYz$TLMz7QUNKALRa<`a)+o24FJ}-xtfBpkpx2KA^AJ?c4r3yvlp5jL%bO` z6NGkJI2Zn2dL;>5bDewf4?wE$@T%Iu!g}>+1h#mfk!teL+rH?udP6Yv573D7PPg-q zi0Yf=?tcK;r51NnV_yRar6)jBN(3wESkQKFG@VJsM^RjzPY07FP$i`!F zu?NbP>8)V_O-6xzzU0&|;p0+TK3v*O*W?i{aw8dUXts3(>t5J29d2p~-6Dk?Q5_N1 zmktX~nh+lfLZ0;_jK@_yHl5I9?3!BiN5dFz`r9bFxuVw8IKiRd}Jrnf#Qsmyov zFh0S9JL$P6A&fUw^WZ7S@Zg(y6K9|^7B#V(b_C!VrgOgf?8%vxalMrB(4WJ*(lss+ z^Tqq)AKS8Ge<6~K9%(*7lA#yOyX*{fQcvL*{vGMPId&vr){PfpeAXYaVsE;2+E)h0 zFVAZrH1?u_KBRQt5mHMH_UCoQ?lEiPw3f^#EfgE0)qhSS`EgKlHThSip>uE(Q6z$> zK_pvQk9rk8N`IvDTD}&hU<`L9sBLgKqB=FSqu1e0aPhFkuWgKaCN^cjnUg36wbWQA ziWqbFQ84{9pSs^?x@}d*lBdYrx)W1!BoG#)X49)sqa%n9RY5hUoxg9;K!Iy@@v2P^ zGXCmK?CQ#Ig&o&{IwtkRy9iv)79{B1qi)OWC&1zYhg0ONd#Lj|k%t$nfibfuPcn(8 zHf`e_z;czqipx^2vv?8x^Lp!skb2(w^L1oj-gRa;^n0Dps&$VzDA`(pc{132#uqr( zGM+yZ@O|DM$NUH4N}JSh73TF1a2Fz_?(Iq9V>nLkH&poR&I^YKw*fS-9}nfZ&nikO znqQf<7tDwz#EkSX79Bwt-WtWr&Ug4YU?7Bw729`pyBDn=ENTA%maHm6m79`N0&OY0V;hAJRfwAYe*UtOczI&m5>N@AbTg5ITtLEFzS574XX2e(@vm9UvU#q~}ML%}t5!e4xMV)XD;y zQ8O*Tk3~N2FMHkgIEG9{w#77~ENSHfwY-%w(-LV8U~=*!cm6wP99uZ?u%qucFSD=0 zSa0JN5#vdZ1G{(jKymYoJ%E9DkVWnk9JeVMTH){qifR;IA#|U+S9NPzoF>=8JtK5YtH}{9^?nqgHN0ZVg6a%~+!BlzY0t%U zA28_D6mUvfV-2>3uZ*w>kVPlTCE(H)if5SRp3MvQEoOn0h7+Kmbb6A?$CYX>3wPTkK5Rr8 zsrZ1evZI-CP7vIUs#@P>lPfjb$`juuyCFKNs#9d}B1cVI6_fmclE@4E?2ijuE<=oY zK*H%VVzkBnjB{4W!(m5`AcA@UKCB7RM%}UJ;Y$9T6J6h~JC%tYk{NEk2g1%|UtakG z4-;)>R5FV{435bfVtkHg`yItn> zx@(CQZJyha3>8pt^&tOtCou=AE^r_k(rLKUH_t9<_}ll2N@z|ri*@wnOjO-ryB7KD zJMip8_{R8=|A~Y&k$CCjyIidXhe;)sO6fx%Mqc!GwfQOLrTg*+5c(*wwd1WGOyMK$ zRdA*=Pu>{|*7@kjAL)@kc#dkp&T&to-C~mad zhv?GF=OQqC&=i5^s<9PDQQ)TGJMV_M(xKXRhIoz;)AiP%9lgq9d;O-trSf-PY;Dd{ zE~9WZQs~Ni@L_6|U3cyND*Km3h*ZSyhmspj4`c^yEV4Z{>$j#wwAPvR=O?7Gw*<(8 zv~#re`7c^6$^iP=zGbD+T{h?Z#n|rnb^UGE&#zJy%xmWr?GWpMDqRcJWe!Kf+Ca3c zzD0si(s$2I1DVN@g4v9lZe(PlgwYo?Yn)M=3um@O$@2`0$#wKx09(GkRMRnd@wjp) zL*KrFwd+=Am#{Sp0QVW7SZM zLm#{vW5hX_FjStq9cy0_I7{POKf#OSTMqP_TsLUbJ5lwFn`c0H3M+YT;swn6UZ%zA zuxlnQ$a((=?(8u&IBKp4zw2^5-_#d$!6C+jB-2shF?Ri!D*`%a>a;kH|xzAs;hc8}UMOWLlguy;^woe`r)gG-jJwJMe zpwk((@Eu?~Wxo`&DjFsx^txDU&psN02_@MXQN_4ozLSVKt|X_tFJX8&!S9au`C}U$ z^(A`^xosjlh2&I$-4Yz%NpHRK3XG3(eC6JG+BTp+6F59^Ug-V2xh{`5-C!lueXtsy zF62GwA!KQ4s8jqijnf9MP2yrW-mu?Vt3h1u{--t1fkX?u!CaFLtz-(v?-ZGrdG!LT zbIrsKGo0*fjNtf)2(cQ9j!?IimOnPft%*w@+U1h!}tO_mT55 z7tdiK0ehMXk+{}Qw2nbVr%vp#vuPleus$mS*GdWuP_=9P-#H{kgJbZ|vbNh4lRb6= z-F`uy6gL)D28tZ#t`zoaHmUDdZ2e)2FvAA7Fvy&1PKGdea=deHgh@&;MSqF?8cgIK zKgW|{RR>1|fgA=YA+BgUDo*-yE#9+gahLMQ%=`KG4)Gm#ZaV9k$B&1Ih6M)9!N?*m z)o2{{eHhC32_mJ7u&ksi!3Up_`U=A{=Q3QVRqIbQN5O$`9=p4Ug?u%4GCm@*-3y%^mISeBKGGp1*S8MK4X6U_0{-2z3()V= zN89R~Eb7w#ZvQi#d!Q;4Rh~)Jk5Mun$;p;RKE<+~Z9`5b_0V@M(>x`lTtrmd_cg7q z-r?nxThg+6i4Ik{n0dus7cr#rhwlt{L^v+1^f@sCRo+DNy>_o0D^bTxBuBRh{#Y+n z>Z?JBw!6)k7m&~PdREFCCHF4(V@UhQu@brz)^eC+qe7PDD>5^VZPm2Jmx-6Xjon>09OjoS z>&X8*;w5h?9TDeSk7%M_x=LZbWx7|T*0XM% zau4rTB(KM9S_?+C)4zo1(CrYK$6NFFiW!ivQ(7s1xr$@%=x?DLl}rWuv&EJRx614_ zUnSoT31sNcQf@u{_(E-k|3$%Td);oxDZru5?kycWY1i?mla^bT=KU>$-f>#9QxKScZFI8sp|N0+!i{2oo`^m zNT4It%OZ7lStI_wnPp+QL1?uB`-*j^&uKocOj9YGl< zgT(c}fi4aAeEi}mJKV=NQ7=lqSfwOX2TyL}er|D`JCKOod&dK&@LoDai?HT@wJ`Ah zCC~ixQrrsL8j*Y27kgIxu-JLbb)`L4r+WEWmaOI%+7#F6=fhI>2O99I)zqhZcxb8R z&OOp+sojMWtMgig7?PVu(@vA}tT8ZL|7T-^7dEj?BE zyD!XRYgM7mA1?s>e8Go_y5`sBFQ%_eHE+r3hR4rnz3M24u>R}Z(8 zgm4YcilLt)&D^i8pL=LAR$2tu%krC zW!#a{Gc+?n1pI<)_o8q{*05+4c9o=fl(TU1aUuDjR|mEcxa^pcv`!W~*dx31!t@a9 z((xP^?Gd87+k?Nma%LmnGeM`=y z;F~f@i=j{OQ4u+HMiuej2NreqNl)t(huvmT8hBh0nJUO#xK#sKtcl1Ogq`$BI#bZA>JIFD;XGes?9V&l4cTL! z$No){I%*=hW5XZPnHsoEa$J$Y?(vj1dZeDFah~52o`lyRFyEBCphHFF#I(wT_6@Lhd?EE(kaUonz$5#{CTV>g59CI5Y%f zv)cQ6hc?3^T#zSMY6w_5WBl{nV`HPch!eli@6ve6>uL|f?!=>x!y)-c7DqDa+iT~M zlJlo0H8jvQ?c6;tJCD_s=y{iAS4tZyPIkR>!`AkBHhSAgG-s$MeJeC;9}R6KWC2gDo5I%sbg>azzw}aJK`|E ztFgFqlCOsk%kM@qpREqG+0t%%F7Z&ak&O||wE$6{8&ypAT|YU66R`ct?`jk&p*9IA zyj^nS?fEM+Z+c*}ChI7&m@gj)sNRA(lT3@>lt6BENym2%OLIUouB01$-+}}w>6NXT z)af3Yz)pm_FaOMF9&&o&e4M1nmeZdn5k0%L(1qCRFJkhUzdfx+`FmBDYaP@L54mj0 z&_td-X~yH-uzAAIqgLr}MjiRE*e3}jeV^YU&nV`bNzlSBXIX*Y;K&VxO9#5MPI|P zIk`YQOzJ8s0Y1e8gMH1WQ!v((KE)q!I5^xUWZMmdT6KT5pR+tS;llNP| zb5&+1!{2Ukl=Qmp6CUQ=#WK?x1m$Ttc)P!<5%_!?m`$fYHN3Nwv<#;<@vPNd?$}{+ z%UDB0b9d=GW3 zy9{zPOT@tuuUB$bI|gn#&Rb!c6FTH{RCH*h7#|75c$=TGBEv1R1B+#)<-@ZR!LoSA z9aB_g7|-{z>Y1+S3!k4yN4Bty!Z?OO-hnR}NQTMK@m*`2*NGqwE7cJXM7M*&cG3#V z7Xi?#=apRLr?o^|x>`Y50m-80uu_Dv*JL#7=ULY-6DcSGAX!BQ@@6J+v!(e%C>3$knD>$}_}4!D>_yDX zV7CA?mocv}8%^Ybde=Wc*tH1jJQEW6Zqn3`cnKtKno$>P^7ZV}>!ZFJOsRGhn{8uq zvi0Xf?zy=^$W0z-CIQiHkQMVSY7pl5J*J4uJ^WDv!mar#P1gyaCaEeBE1nH$?JOXP z;x*U2EZ_F+zbf(RZ_o;MO0etmCb=mEAD5VTKjKXvmu-)_ek17$Z8cow_Ul(%l!gz> z_O~9AeJRcSzhZsS&ZUMWbQtR7Q(TmYUhrH!rWYG+FeU&JdHpCq*_cA+v}8GfEiC|! z$%9p;;@27)=78IywR9KUXxTUbGU-uYb6UyKWw#k>-HQgUEp8xO%Tm(+0C4vV$5lMZ zWHY=v(U8`(J3E*O6qeM<>RImaJYN@Eju2dZOG3#x3pTmSTeUjVR=I{j%`Ml0Cm)Zw zhPvxGa=~q*D|<7RC`6N5jdWKYSwvm_5qwiKd5T!@tVOp6PH5adb)&9@jvx$X;`(Qe#wrT!I9 zHa-6U9^|sNfKd&7^KIaJRcJQ|AcLK^B zLH2rOk2Y0D{MU}-=MZ<=x@mjc%r ztju&o^DIhE7|HhUQbFIx+7UZ&;V0UwD`NRB1iVsXhg!2(@mMChaWGbN!_7Qi*3%v} z1kYIgD$XWOZada6+L$4(6g_h&){EPm?(^wfrx^A)PBkK+PMqGw6NfFf_Y)$9m4p~^ z(d~n{!R8>Qiff?P+k(be#!*grVHv~QXE~(h8>Af(csy0?714ItZfHL6njY#QXED%E zwz@7B1Mf+W6YKP@dLpJLe}CbOaGpK!nVI*}czkmSZqqLZ7hiE3OGU43u`MOVy^oU=EO<^_n`#=Kbx@nZgt2x|8(_N-_kC|~FyQk4IvEW!YwSx8AyKqisG7Y&7CO?*V zYua00Ba6Xc^0xL8-OTk33}&K}+7ZO4W+xqc7;U|yU|1kzag*LkCeA+qEg#2=*qf!< zWRmQyff-!qv;k8!26!)%$;HVtj(nMApI@^N3+*uW`MeRfHHS~TJ>q_m4qd|7#C|gy zhBoZmV;bXZTO&r$2&9u<5g|lNE)a84r%wL>M+G{&<9Hk`kT4dZ&v?$ujF!Yi9@*}I z3@GK-j9tQAX3epd3vCBAq|UHh<0nl5WI=FJ^yNQo&boDZ_ z$N(Im87F*GmZ{Ej+iZ;4q?*%->gHX%*0kjS=sf8L>1oo7*VV-PR(6*ARjnqYm23VE z&){Eq*P`Pd0JElMK(9Xzrz0u0F$LzV?Ks?PH}ZkIj> z<1v4< z$eV3sRyt2|=RDIYEsgNz{rufceDk;jTW)xJJidnhg^17`_~5K{%R}qs3!j-XH`|Wz z=KlZ+@%`5Pr>R_crVMF^xO!SWDmLI+V4jLE4L2#ORVDqkXc!n?skkSYGhUR&_{Gbw z46}p|Bzw7uBfP*A?;%EmLkP?!D;xLlT6Xcr4{WyE7)g6yHKA{ua4Kq&(idZ~HZHNZ zuy%vLejhvsl; zaRIcZnrX(ocqub7meSi@wCpw^$1`W&8D}=p#o`5KvL}5ImY(a^T=#>CCov6sjj_eq zp`-!6DK%O5voVkzQh5581=M}VlOgYfh%GA~Al;ASEtc&qS+{Fqe)%*ccBivDj*D`~jA1S_ zJD;KFf<3v*0!&w{?+jIYk9TY*Pd3iq#qo>GJ@efO?Vu|p?uqQ zC^yhYw&mH42<9W8a0z4R5Aoa(AH;X`v;)fIqxpjl2W6L!aSf+mX!(>9UAmnddV;Av z{IkoVwu!;~!)Q=hc|BlHAxJ-&DcQqU{{Y+S+1H&z!I4T4#d>6M_VXfIS^nX@6tQon zb?J^mg}LLnr?t#B>;Uju;}7B@Kbb?{RUHu(ozv);P5X|T`?_)c8mVbwOS83rhQmoQ z8tQvmPO&?nBnM|}pz7f<07klLP1-8Yxk5WIt!<5^LHDG5-AgjXuGhJbYua4X2VKZ# zKWOQx0KnFhUpjlkoCabf=^7L#x=e3YIxW2tPq()+xB&I{!l-W^k7!*GA=CGkkk)}Y zPOH}>2KiG};@1*f*fMGD3Wf7>vr6 z&FdQX(QUbun4RWCavxd{+n)L9UGd4RbC?3sGa9=>n9-aWq24v!3$C@?QZl)|==s*} zCFzbx$)RrJnAmBE6&)zH-DPB*+Mru$Nz3Sz;bSK-S`O>MTopd1`WCGbDYNVk6_G5umw`^?z zuClD+03>NtJ-g4$0cynK8_h!#gk>=`A4Y?i048)2P%R9&_Rx(AE$__I2-le*gD*y% z%TUngfEt2v6IrD>lOFdGQ{Ir~mzeT;QV0=h9P!;$VH_4)S&+?4)7zO8HkHab&!rk7 z;2Gdn5e#r^VkKK<+LP+^B;5@lWHTly&WK}P;az9D+83S)23tJI7R!TxCY?x^ykwzV z=NoaHb$xt4y_LyqK@{qW^eS4o0MQ6s@MLJZAe~2l1=6M0e(GE|9`Zq-u^nUoA8z&Csse z9s}FF%9@_oQ!7WL>&}=YRw=wht7O`m7(MIGVYcmfba~b;(oJJCt})pi9j)AB(2;G| zCQF2FOec?}L|NxEpe}5(HbK@BgPdGxF+4&clI)<;nsp&+8;xkneSUJB&d?x=k8M!+ zi(%B_JjGaCcz3!0(>D^7TrAeXc9fwsr&T2>?k&-|yYx)TTuF^}JJ)tkBveIBMERDS z<2~#Bx?JKaIzLD$4RFpH?RhjsTsavL>RD}hFAgJ4SW-RBUBhF7&Vjf3zTUK6(|`zf zgCi?-?n+~hT`kXA&kpWoOxXl+il>{~HjKc}LIjfHVYMY`aJiY8@DS^QKmw;o;D-&D zk*w7mE;8E%1b_ho0hB9R!P*noIwPcItHQfD*MK>}k2I>xRP&1Rt~ln2fz8Na79__&(Dae2=YOlxAyQ}W z0nz9S*`wYAf4n60P^h-Bn(!0eB~*?y1Fz6!aooW51J7SN=*tFax+a|R6-#tiT6OE{ zJd31|H;(U|deZn$uBzJ9bT>{}2#y2SItuXSM3U{DHO(m+KaYJsO1;4Bq6pNH$c($0 z%r;2}wKLsK6cn2PImlr+S>e$!4FbNc_b4O9C<6A>J_oFuB zU7F@o)SCATPAs0a%&i0(O%Jx^SB%dlU>`t6MRpEABTq7;$<3Ly8b{8nR!3)ngV3vK z!#tBne6cA7 zo!>Gk*+aOGZ90;i=TI5$?;Q(qE7T@@hLB)!M>YAx2Q{2X^wDEnC9Zij&CEEiZ97lv zXbY_4ZB7#q*_ijM+#}Slm_Jb7?Ptt{w|ozKFgw8moEb!b9BC8X^Da}fo7IyUPfw^S zjK96z3gsE7e-m(O?_(2B+)1i9`fjRg*~v-NId?vXNVMD!;mw`?&d*hF40$L8eo_z zxQmTx>`>FI%z!R0GEyLFOoC^t+pZBdSq(F(B%m$ZFwxvy9Nxb&W_O7)N10K3j_uEo zIen#i&D^ckG}^atNj2jIWM=IUr41Xdb-+P@xrVV0E}7x;@d;)C1KXKtTXR4Hh>D$? zNCpBE5cV~rEKKkD(;VAw8sZ_>FwODe1QOUqBss< z=Wb31D+6LRaq9^gvbC+$nLdQpk(2_UB$=R~?=)2gwe21n`J>RQCM1FVQS3Prz{|&7 z^10L7YPG~TIMB^?^m^BC{iJ0vzuH%v=+o8fNDmDe>k~TgPHARCF{ufG zplA@cPEhJhka+Hn3msX`l&JCPwi6lhtf`N zzfhFhWrn6CcY1V4yz^!Vf_dYSD#Nq6&N}i{T{|u9LP*VU4jk*HtuE}D=JOv?sdHG< zr!{>(b$eZwoWvCyg(ZzJE7d;5D~($xUBjkV330hXCEC_tbgr46J!kYHyHF=WPrbv; zxdXIJdiv_B=+jZ$W6lgKjUaL0WeZY=-aS+vpHlIpMMt$xW5XQttL$8m9<;VxOIv+Yyp~o-)#WbfO`IX3E z5u_6ouR7*d#RYn(w6@L;8J`m1;YsR38*T-rq!=a^;Am*~W2rTS>+jEU=^599yz<)(_FOnqS7vFB3N6|3(qF~K^&CiSed^KoFt)gA;*;Z)tng_n&T&j z&kK!$&7kh%1zN5kQCz8Dgq|-P&??T)mHN>a799A|1QE1B0QicXWFFXmIB` zhk8@pQCdf?Q>=+8L5_KANi?Dk%X7KAwOURfQlqI3TpEAo(4LB@47$rMYn=jkYB(fI z!)LO27YA;WxbrJqYn<3>fE}$O^+@kU2*$sZhJMSO>a_goeg40H9o_UkaJc9BUZfq{j^Jp7 z66&oaw_A0j0xQ&!dBx2j4SD^&D0>nbLEux}5VKV-T_sjN8(K&d_UK*?lcz~u$-M`0 z&ufc8flVnq%T;Glwu@XYch{LWZP*YbtlJKANg$#eHbtIGke$Y~=7%@}4K&cCVoP0Y z#{(f+^~|koXyk*Kterz)iEYqK@t$R=ao~c~3tHLDt#|3>mF)Fbs`N7BO~%HYRd!;? zcWbmLcmTTR(JkGnlBs1q*|grYX_dOT(_a3hIN6BGYXWsORWCE5b#5k=11-X0h>Y3b znWpNGQcI>K!NabSmaj=~N(PndkHZhGoN)H&UM5q;|>f zg=P}|ZU%td#p1jG(!A(qmlsIpPH=hm+`u~58rk6^e4^@h)wY2#%3^v7V;o*#uumUv zLQ~uu0;9N6R{?BxXq1|qcCG;TA8A|+B=$@z1C-{-3gK~kqyXOFqUw=39R_2kiXPL1 zmu71}tIQX&$d?1xqVcvK*+y)`qu*d5FI@V^1%R?3EYB( z*{FLxTCRbdz=#{gS08PlNC&Q080hXn($dFHoyw)sQ?_F3Ne(7Aq=PgP3L}u;D!uhg zEi{2nBp*>#3@1_2nU$b%MR>flz=-emg+TDr0*HoWc&lKI=`RvJ!RzQ$Rpsu9(~&aW zz2HZ5g=<6>ku>j0mkk_asX5=WY?U0 z$wutfNLFLB45h>#MOQZF;qB6c%{fkBM?qGWy!)J~ZS8s!AnwYvIO{}Ij)hWfk?uRJ zCKoxhccgNzFf@ZP&mS^^w(Wp(L%jZl+m8|>g#6xh+h%x?=I33z2Zen8#t;c~I93m^ zSHl_^Pt~ypzeUZPW8QZvx3Ep?hczgwGdM|<@nMbxgD01mrdxAB19ti#8LrBl4rG@} zaWJ_hs2WR`n&m+A`W4z52qs86LCQzBS`LiB$T8JItXx`TJ2zgc?(UP<3(c2sKHcjf zZ_etRjvdiFimjFqTd#Z~o*LhMq<4=06gQ+g2v;WSjc$~EGhzB?|G$o zyc1SX=^Qx%>mxux=_8Rk4`ptS?32W4>QpVuyrN4^`3i)FgFT*#D*^j1!Kl+!mdB9s zCyFhvE)co{T7K(z=~bSN)+Di@Np$rgTz>xm0UeWCFBg-fuE`=Ih|k*6LtnJ(fMuZt zGTD|6<=cZZ93~_YIxc-B&Nh~1#|p53BVS;xZZc3b4JoNX3uNm=@Y2@6G@0wW5&H1#STrS{n0O z@b4coW#?vR^pv)O(hTq*_pT1vaNnlCI>0!3$wU%qMt8xW?=3l%rACk9W%fMz}7r= zoJk`54#I*ps1!M+h(D>rQC7Zl*}#C)Fuqv)PQtdTqv-)TkEr z8>ck_@foPfaVnvY;Sib_Y}SsXKtR@uftf=A*R*v@w#?I;&X((rCrIG9(B_hNs2cl~ z*_sIz^Dh9#z|49D5^|TB)mQ^Ppg-gbbSI98e9Z$+sa@^5j3ACOsA0~7NILLA za@(16M2}BOvz~4+HSnuL{89k>v-7UEKA=;21po%+cRAN%u+Rq)ubptPxX`W}Sw!tC zyH8pMZ8Ee64BbH}mN<{Q%9(7r!zhE-`clgb*O3ux%*Z29c&awrqGdNMh|H7=txy?2 zQ%K8F0ZVfq8N*C()gVjFU@tdqY{q90cTc5SQ8t5|B;}513i%{XQ_;aeW?2VT zo`ODPvT_)#cZr2|7^_G<$5mp{b9m)cS34uJ;nV0bLVjJDX;jr@5^O~JD;TpsQ*CKX$4 z?i#t>H04%f1!qyHk?K@=yU6I8uWgg2#t%DdgL-Bvhfpi`^$qL=7Nlt7Vx99LY~EE|l$|%BX$ThUp~k&?*ek zSP2esnwg!*hPAE|_kedum5`$Dgyytd66fU}q?2qo&=~Jte71-X=Jw%}mw`H>?b|Nm z031n3%~gGy9Ca=`fuasQu;FsN=7`<8p?;;+biknLJk<8XZK-cN+2;px&>zdCg-+XcCi}dz0?lWB|fl zzeJsl$eIDm>r_y>JJ&&d?F=d#j18k#EsKu^7T<;?RYO?PMrN5iy+}vragqQ;hMQxs zdDEIeWLy>Vpb>{Hc<)`>I&_rAT9Qa1dBC4_OBHKDGT`RPHvr?$WunD-*)6%H3p4ZkOt6F1TVnFK%V$gId%yASdcniI+ z4$GE%#*5h^u4!D7+Km^Van8}jS`DL{0OeH>+7gi4rL-bSdvabCIb0A?!G!NlZOcpq z7D@^WOdfRd3tO913ED%BT)=KAh6x0M6RfqDxaD^N6+J3f;r{@ZaX5oGgRr�I2Sv z_eXoH+_e{!)JEbSS{2RVnbjkkL>439FNptmw=(LGcumfc@}Dcnb;dG3Qp4v)~1ULEg_ zl|jxH-S2aRk{CX+6i*6-)}DkO5Dg7CYwJ(S4y%nJ+qo6ah!$<`G3H9hT4rOv^d~=Z zS0cz7({0yip;>FZNeik1916*<89@vj;;KvfrIZEJGU;ox_W{O zq3;b3LcV_LsuD`D206ITPOABfKmbJ6c$ECz%a3dBRy|&IpgvMjU3f0j$vl8 zUK~|+nP?}fVHC&OBCaE=Q_{Q~8KXq+H7J{H1WV3M!fVQ`MV9eV0-d9kn$q4Y+4hcs zK#9&6AP#d?K(1@M+bbysca_bkZ6bWh z1;*pOs1P;Qt+z+!JszayTYE(*=t}|zQoi5Ha&9PdIO?w3eek)LY9t(~ah8EadgDf# zsxi)O8$|r2$!}?B*1c;`j=hDg%t{@tAf0^3n@={$BxauU$IZeC5U)TkYrl3mJ!Nuo0 zfN4a=ny4DnB<7-3#>=}pbSFC4Auf#2s1=mWS4|O2eYmKd^hki?8U<`8Gx!Nc2aJ&V zZ&nsXs^;WWIUaZ@5*fv%dUB!*n$ei}=Den)o`$fI81p;=CKJQZltAD~))66~YJEvS7K(*O89FCb5^lRS91k*MzR*ZZZL4`?6Vni@mHB%#PjFRg zC3(+uKmf07E*(!Q;bCVLVu37UNTAA4?qLO4t~?hDF1oT+^@W!ZA|-Z^yc%1o>&b;h zGc$6y0Fhk12~88dYG6D+7mR*OHnZ<`jc;3x2d{!pv#1*7yrQ)WZ^OJJ5s!mxY}v-+ zqz;ZQ=%)BnJRjwpHV`)D^Z1APG4@S8N8amoJCvP(0w6?E2AvDjhYa=0=&ZjF@Jttn z_Jekpf4bl&_e4W_E4--+p)ju6uVh4H(4~bH=vNaFq;JGOFJSS_E@1pU%J$kjEMZt*;`=wRDfucbL#)uyKt$*DEr6`8) zL^16G9MIA`mmDp>(u0q<4M=7<#X zOQx%$;p^*6nbjrw(NN6Z-=Q&}lf-f%WTLH_YM|w~9!p1EPb%qqO&e9$VeMKH=M&Wd z4A(in;DkXg1}c_SnP&a!lL1)T7U)Pl!9mu~2G3lS)1P;lF|MY8qk`;>;XoU$n#@X8 zMBAB(?1&rZRD(QLm3FiPyso$nN?TnN0=vp#X?Fru6m4{-{mGeo)nU!1ioIwUErY2| z{4=#{N$;OYRB3}TF}*tF>#i9<72F8(tnLsfxc0?KqT_A=PGeocb;mJ4yJs_?#CM|~ zak*d-?bN6(h1XkaW-5BnC|k`?BfT-Uz*S~rs&RmWup$neP+fxRA_&ekfSYQ~mWgw@ z$O+I7T@x7G-3fQx-0^N84MCadmByHF8P#6^SPX8Y4zh&2OWZQ)3yx%xYt*Yy5(Dkx zZQ5vVQs+tCzj|}Y*HxU>=6LHzR@Z6CAwkZ$N4K(d{VSr#ck;kK-RqYxZ3IMBcGbr) z9I5(y0xwy62nAK3#4Qo_ou}035bhuyIg-VjVT4UkO)ks_dLZA5M5?fM+dIMrJ+EtI zr2E`a_j0*~pn_>h??LQexCJ0Jjcin{1gw_zS!*^V!B@=NlSItKi(;1!Xw9MbZO`b zS%5W1*u()Xf>77m9PGlA!9C)=)lJ$tjnEXP05wPLY-NXK2SWn9i8JiAA&)N_SGghW zeARZ@hYY>p7(gIO4B$%&D&5gAXcul;Y*6XmlGZvVDb%zdh-he*xDDY5!=9W{t~PXC zwkxW50vlz))Nl*xKa{&RF*E(f{{WSI)sCLE^pDD>EMhm6&*)bp73RQ@hyT3FEalCl#>?BK4an^ zl`moQT){ZpX1f=A<9=6Y?y4TLpjX=FIJN9^94O|HK?l4V&qDneV}gQyZ@~+gHey=Q zp3}f_;^sb;jn2yxzKhGB3{}U_r^yL%5hZzzx^xlVycl><5*qeb%)CfznsUi#cl4i` zXXhedBZbFS5bgxjpzWvT`?k*fSQpL`3993S1d!#b50vJ(LS3fkHPI65AZW5HZrON~*&HWQta7x|wdUkq zEx2`{-b$kgZw$wW0H7mlem4%F5MkXV66a>timdxHE%d2ChT-WhFo!0zUORiVuR3Sp z%LZoPGh`msS{D|Nt#$EfvU&#e5xI5TNC%-_V^U9DR%lU^ zas8(TjwHb5m81=b>;|~$+_CbThNWoQ0*wGdv+)Sy+%4+7-?ax&H>W<`mWgl)Y&UXOQ8|R9a1tr_z4j5c@S{Ins8Y|df)QuIofSY)F z)`4>4a@>f4-_*284HC1jyE%bHI-{*Q&jo9MWuhIYEWng=25~M6Ueya|Wpt~&XVp^h zIr|hH%dBaNs1)YzMOzf$mRbTqyx?F(6$hFi9!_J{njGg;p-W4Qs3;k5nl;D15m}=` zwIoD$EWMb5mMYhr!&Gp%96(ueXGG=)$Te1A9e8#RSVLj%Fg)giMCV&`n+S1awzHyu zo7Y@y(qI)fwsw2pjSk5Zydv~fE zbV_sXF|fiw;|n5&-p%Qqr+1|^yvcm^r?Kr?Qf3ug(D)My*NhnhR5o(A0Or~Zm?Y|` zbB!mVP@9=Solhb&HdeGdSXclY+LUFBp!QD<5bki)XqM2gQ2WfvNfiZF(^%@ocU4@t zxIzpE+jt^X2GyoQ`cLIslZV7``$qo&m3-LVVQpr2Zc_gM%Y^K2kFGz;yKw&aYZ$qG z66O1P3Z6t0t~tn76zXwbLFtTQ6jo}DV}CJtb`pZ--QG>8aOr%8^AdZM7a94ZABoJ{ zQHE*qD*l&|PPT@WS(trK7mvbVo0}1JmiIKC+3INRJD?%FKl2ZVW1ej?;kNK$qUa6J z{T-q@3_H1eG4FlH7rDjmH7+E$p1G=^BxtLiz2B!HagTko**=BwPYe9fjKJk(-};+E zkn{G9A5rIb0O_R!FR5TWR|Vp2&4;)N)z-+1jsBo`PZr4UM@XqkKN8!mhX_1{R%K@^ zwdJZg&mU%paS1rPO~zra*l~@c9MQNEJ13ykcsx6}tEnzKzdyX?Fla%Jl(BS<^}qRp z5BP5qzBCt3)xW*3{$a{EMOVzqCMKuBkxX+YtpyQrmPpJT*7s@1f@`v(GuydG$G4ec zVqguBI8v!Cx0K6FcPE=pVa(G{ZB<<2QUNV{9wFnW+eC|&i#tKV!@pmh2us>#Wmp`; zLKOrCyon--MT|TlSUvXp!m+s5=pd!Y?WTrMaYSYaDR%VsU2}D)x6&(AGC} z^Ps@?DS#1Bs+cUAF;KJl23Z>bf-2dk(tVatXaik?(FdPP3HzW;Tua9!&uibq?3N_ru)xmfJv%( z@NX6${Sn?{6@Wf$T6cpNZLyvH9fy11wbLTMYVlu*cZ_k<6EVEs_S{9L zqr!YrpY%s{F*pzZ0Jea>%l0n1&CE6-uVY%`8qtV=>5Ac9d7#rmK~b%(&+lISZO^2n z{{Wht=6K!k!sd*vqg>|bOnaw-)MLk$%}iIvjjok;_iGq@SNL2{$tvc5_HlpVaX%!Z z`%9nQyZXodCAHyhdxiYZbG_Ki=R59Q(n@289$8ds{XYhXTed%ei~j%%i1{Q=)!*sy zeHdI%$tas{T`eDLcA`((KX>&jXS#ojd-ad{N@?T&0R8@-gG41q5dQ%5xW1e=BjlI1 zuRp-W*Z%!EQYzw1YMl2EuXadrD1I@AL8Aw{73yH%M3A^onN7%J*jSX z58z_I!eYKjXZ@tk!xz{1Tu;d_cw3z2-puD8bPUdOqwMbaMPXaHbhI6<>VQw$K0nN_ zm&Xwk_iuhN_+1c^OaB0~i~j%%iuom7t?n3XU-(>4$td>H=l5>@vENB;_`{ydmP1_T zf-cTi7SC_$!g+X1O#RzejD8bDds$eZ_ZvL_Q z58hk86yo80xy~R&W^@vHt)`XHOh|Kdc%dBP5+{V!y)Teo1s?{{T;lHRFZE zpZ%mSi@oi+WHrucBJ_s2&6t1HJ?k9Yx&6z(tbfv4%MLi2onERmLQkAT8#ur4n4gkE z+45_{rmf3 z_VpSg+LOopJYV=sPst@)b6uJ<0>hQT?QsG8&@Jv;V78>*K z+-0Ghd(MgM@EVic6kd)xEME}=XwCyE^RGt-?~hw}O-gx+{NHBTEy7wib?w~jv#d0B zi0jIsSl<=LW9feLi+{G>&A)_dd(5mxr%L)w<(0a+shSmbh{4N%%0^FMEBdRL73y9Uejp2Wrp2)Y4ZgKtQ4$%IliaV@9?kni{d`+BN1=|-JagNZm z>DRdTaN@r>qwrY#Rs!I;hq&u!h}EtiS+xXpqpW284^W)j#A{Uo@XyUyTvk?M!*JJ4 zGaux;sG76)sr8i5Lxh3|Izf=LElfse$L72)~3XWJSu?={KG<$AKU}2_->Jn}2IlT)V_a5y@D}0z%Q@JpEtHYyMUdl%V>r`OxUV6Z_ zw=&q`PZR2@McanuGy{h9;-9f+f45R}0kgBB(6+-8?q%C-Z0N4-7G^;A+HZjp%{oJK>5(&*q)$kht17E|CI4xO6k3>KfrQ z3z{>UGSzsyFC9{@9??UU(iNqcS};K*iU6IMkx?BosL&)kj1MyNv{@BBsvQ*zKpF*m zRhd=2!=*fEY4}MSY-Ilc{=X1E^|0yy!Psf;3UT3=&L0zX&1)SvOQ460%_DDynH*<` zG=;19t8*WK@Za!uev0%&UJ>hW8K=!_DpUEczBzt;pIVVK7j8%W4~G8$gR%5f8^^p} z`0*FrHt_F#)-}#AwjEICibIbD$_T~Qo^!P8- z4@&3dPjM~rVfNN%b?^Q~g+PLTk78~?eVi-eV>PVNCQrQLx%+be0Q!6v>IbE7K0BMS zD)6(}zdu*}msEX(hK`};Ngh9nXS-_(-|4lMXhm|kZ|X|@w|HrX!4LZb(yZ~`5f{?I zhO2YNPuf}jo8SC_Mt^Q!{{T;d{Xq1qzsIpJ%bLy=+SS&xYGmr-nDN#*MfIRV4+SFd z&vN+Rm;9F$L-yVO0PJA@08l+CqsQ?~_ibTYI_0&NXhm|kPNgv8tgh7N`;F=zV7sra z2m5aS0Q!6v>Ko}?{kMPpJ`42&(zE4HaV+s~=Gp$6-~54LpKf#;yn7XLGupzwCNo;i z5@g0_81pKA+r9pu1^R*MTF;L1>WOcK`?vZ($#q6s?zgCUl?RXFneN)cxB6{mnh{*C z8cBq|+jsu})8M~QJt|Ke&Jh>V!JGoz1pXr%{{ZAW{ESh5+jsu}v4j0U^sV0wy!=(G z_um2UzT$G}?lXL@dEw>ZmiN(|`VEqFS`D}ZJ5)XM&Z%mTPr5q}=UzE-1m}Qgu{=TY z_7l6-w2H;?AAOxgdiehUd@OYZo~woAq=j|vbX|2+D9O zExNphc*K3IftYgiFhyU5avd6Xn literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000033_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000033_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..30d6bf11d5057d4f3d362a569c9a483120d798c5 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTf=;xHN}597n=ktz=JH;6_L z3z*!fI3y%S!vQlKM#BU%Oknv2M308gX!uZ)9w^aEIPFaV)}TGCx%s8VsaUIFCSWxz j)#;SM1aZdMGiPi;tWkR4!eOX~J)!giN|L3CKtc}yoY<7- literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000034.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000034.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d675402c5b1cfb30d8b2e4d8c75978a794e71581 GIT binary patch literal 30287 zcmd3NWmH^C)9&C99D=*MyIXK~cXtS`!Civ8I}@D2U4sXQ!686!hhX7y&MV*hUHf_0 z?OAJjt=+r3s-AwTy7ungZ!2%x05mx%St$SnBme;M{sX-I0iY_FdYW^yGIKDqv9bc* z)&LR!_zxctKOiFiUC>a`{;ObMU|?Zkz5k=3VPJgxh=qlNgY$mjy~4x8BPJ#!q@bW= zVBmNg0<;4FkWdis1@P|z1p^HU0SgBY0r;O9-qnGEhJi%`KtMr4LBYeq!N5Vl!azeo z03gw!$k?DU#7tF#i`bJ~G09=nE@zV)`qi;NitoVA&s|YqnQ2I*aD)_pZrr8hEO8GT zut;??ANuhjRk=P~qN0=S;2J!wY zb~B1(ERMH606IK`KV%RT1o^vo02Dgl4Gci~hc4&<5x`>#9Vk-6I|@gD++`<$e$+>O z&Z{c`h8f_#p)7V_ORQ?Ygr7$RlCj`g#Zy4C0u=LhYlv!A^bu)fcFRgs``Ey-S ze&Jp6w|#Urh7~T;|LuY((#7BU35@6Gbnm8z0sh}L+b{bN@nd~{JXk`Ge9ccW5Tz=w z@`G^L*(C$`7+qJCUU=6&q`}1IqT1fqIV)4H*JV5~+u#N3Ge0FjbYyL6&HW&h>|)B? zOwy-HSsW>V8Gc(1t>RRq^Lxj8n+u4AHDtt(hvKQSt+UZ_)aPPKp*Z2wPc{0-AB7rQP! zANQgcm_)5iZlMaItkpis*3E#@A$^*D$$q@ioqeVTE8C{yYvrhB@nL%%Dc;9BFQ@L( zREYh5K%DS5{B^}kcyc}FTH$icem46CdRwp&zxDhn6yrYOv1;d7*-yY_ES9b3|855( zW*h9x&B=!bU4}pm{Pb2JMHTu_t9+FwE{fg6V;gUe`JSTvB5IhBXynB%qj~c`AkzN1 zV3R$FszU*>Bk`VM!#cxq7efC7jDN7o2sml4R?J?)CKc{5L3pTXkB`69oCw2{9@`%x z4MAY3elwB?Qe!gL1+vHv*Epret8>Fc!u>ZqvX*CS_BrX*iqrf%WF+T(5Yq z=*jbC@7T}lwk*N;gzo&6j_MVm6i(B$kQqMe^Il!r9=&q`4egBu*S@KAO{{j3G-&E&MUl%)| zHjTnSb7J1nP{~amY53a@&s~uIc%#OjWto7&#qs^;fkW&VCs&8HLADMbhakZ(f2*PT zHot0O5QAEgQoPiIa`_>$WnaTI zpn1sL21k@w`#-U`m!^?skPuowVXR3m(XDZ}cWNcTB}fK7YN@5`)u=JBEg)m6)&eD` zUHpPP%heY*GCM6&A%1rL?Ro#Cg~=O$qHwvp>;k#UD4wyY6ip(V_>crOhQKs@R(+fu z7KE){ykuKekH8yUd+Kb@TZ+$YPSQb9zTB#?WJNGG`R8;jl zMBd+_MEY;b)qVuw{U?u@;Qv2wmfgIPt=fYf)FK@bs&5X1Sp3Dl>7ftaP`%gY0MbaIVBfQRh3#3K;*ahrxGC-8H5YhN(3 zO@GN}uMD5~FHCitbDM24ZgxRy^7cydbRO1TE#-q zUXvhD%SJwb_l#KH%Y5dPiU+-he#R1_!I|WAXhH#^nU8jyn#e>Gp{?KaE9V#HN!PLq zf%}h^>J(I5>1mcTz4tgRR^~a}FTKU`42Nlzb03e@vD~BVLD-pFI~;>%k<(VkhjtH% z!I1c~q2jLB>ofR9?+TDiGzloO_-xhZF{H=~Xi{#qPdD1j3(%Kt^-jY7q+{7{Pv8tV z3>*+iKj58x$a_Xq^w_n9w)^j{OnJ~;h!9)3yoLG0OgriZvD-wCz-z7t2eqX8{-vH$k zev2ZqB?eU17qfS?{y%TJlmzVOqY^244ShIhGCy;VLJ<>Sxq5(sjqfseZc<5z+uFB7~E z3vSh8mB&s#b6e@@ zWT|_9&af6vS+!^`1Ny6+I^?tr*TQmp`j5+31=j?Zg%t0I8N91``R2e*wk~KyrL8go zeTba?^Fk83mToW7_MYXSxeNKBc{p#iUagy6*^GyRCz+?#yeETRL*;q>fZ8vrPhLsg z6UJk{4``YkJ~64_(FJCE!sGb*#hMY>%mrDVb}f6KS}zX8h3ch+<)Z~ob^PXaJbQ>7 zbDqD3%(1h4JuWD3zjNlh&HD1!%$&JBCH~-94X}`S1F%g8RGbMNUC7KCmbE%%M$K11 z^sD$yNw=SO=I$QiLdv$+qc3!eEOs5v?LTe3YX-`FQt0h^1I){xxxF@SRd_o$E?-OM z7i9s{s_ADvBRSECAZ9DYnu2j@=5CM{(Lb@5dZD;gdF#1n+;%8(2poLe`fD(icQft& zXkR;h;atXE9*<)4(s)#tTkrAs2##l7+~ms3u>LS{7<|BCWSKL~%H1t`A8CiKl%ue2 zHqv}%AZ(t)>=ti~S^0p%ukUM=C|y8wvWEIS(EPQ8?n3jGGeF`7L*xoisiUC0ao&lz zJh`ndA1;V5rnVRivo&ZRJ80sbuo`_B7Q9Aq{!in&-!V!A>TZ=yyfZ`5@qTqf`j>Uj z#FsI&c7ta9$zy}YjS4>&E_zJ7kE?j6o7hX_0mmC4dhGd~*;0{%hgqN7N`^IV826<$-GCVO zTs2?9TW$JKQ6ogknX|N~+(1ViX?ozySF$}tR>RWoZBA7!91JS~;x{B`e9FFcN6fDh zH!Np-N*%qW>9H=vr9OfWS)CmBo~o#&lTD%(%nPsLf}etzNu{By^PLMS@A($%6v1oA z<+F8jq}+slmdXcTXb&4}kv;r@(HN%h6Qlmr`Y!_NYo||lD5|!vUd#G4>j+T#M0U^k zRJ|J^t?&%)-nTiF-L_uBf0N&8>BmH#Ed<}U^1i2n**B-WFG zIX8woJ>w_-*Mf~WRsYeXbyFZKvH9hwn-4@3 zpS&WTS-$mn6YC-_yy{H{Froe(te4!5Y~F*eB8lz0&bRM&8SDCA{;l)BwOIUiIQQc{ z+GY#SXTI_rZ4#XJG(K+s4dTBg0Ci`|ir#}PXF$a}&hzhP1z6tnU4J*KIR3YUe;fF$ z^p4?Q0p#i3)Bncs-x6y6rTl&F=uxlI<(B1L#thU1nQy^=N%*(&XMFDv=I#6q5A0+i zpOd(J%}Q5jeIubIYnH}f`oy=Oqy)Ksh?fB0A9Ap)0=^DAgCUh4e$YY=)r{9gNf=S<3lnDzjg#MMKZN{#kq0w zO7B+NDWK@^KW9^aSKWV1Kt9nU*3S|kSn39Rw(;G#;RCMvptskJ_!vwpm1}UeHHX?O z2^Ogu_3}?YFu#@}OQc*HiMd}tCfxb9j3qd_*NdZdD2VxrP$SdjO#5>u^cgMu=CIuV=#uz+{z_M`>W}#w;ODOsU1OC7=>%5~cm)(X zyFgpL94I}y-=dJytD{37mMvVY845dR3DBX86o6H}>pHP_ zq%rseHc@@aKJaeZfd6BgeVcP0juH02XDIgwd;My~Ut_)OfH5^}kfuUjbocr0;M-9e^KdFyM!cg*9;Jl*~)s0DGgaFUQ0s zBuZ9$rnoGjDZe7eN6JP7)VdWobDBLUp?xYLO}u}q-&W0$Y&2~U4_EI*WI9D`xvcOd zQ3$?F4VdfQ1RWwK6ctYRD;|Crrhre%u8?58=yZPM0cUB24#CB9vz}lktO&`y^2ljg z+qtfj`d(L|>!moNBzw`p&RK044dY8hM$xF%RBn;PKwtyebMPo^4>TTUbE@#Ac75Lf zpKC|Uxpl#{`%ToL4VSsn4XTLsbu8?wT3WzibUPreb$e0fJWPeUa-Ckrh3%8$3F2Xv z!(b&pGFWkRl0~5hm@!qsqbQHaa&JhJk|mwIXLnB{a2%N8JMjMhRV6mI?sHqFK%-kv z#SygE!YgUvkNEN3L0xY9ITF9UQ-fzo+9NkCq%bGo9RBJrm6FEAdefh34B3kwR#o50 z;MG_S(}qaRB~qZ$EPrV5r)GUoyVlg*Sa9Es^!tcdQSclYmiueQ98qewIW3Z zZxlS~p*x}dETq+|RGDMSF7V-7Shk|X15EI*H$d*_gUDmSJq~JaK_S%ZmwU*YD1%=6 zfP1?ZztG>%GfG}3<)AUFgq4DjwRkpIUDQRbg+ z_0QWN_T9M1f*U^jq?!tvgx0?--Dp~s%C^oNRO7G&NzXw{v|{UAD&Xe;YCi@D#>1Jjp?OLY;4lf z8Jj@$STx0tVUU4*QDp6RN9c;vA!-DXSaBB7KFztWj+Xy}P6uWb8E%V?wr zD(vnh%{8kGv_86$*gSXJp4i~pq3g%tC>w`oC!EN4U)xD1Em2f&9^0@AaxCSQ(M!WW z?M>@iRVipVM9WfLS!X`J8)Qlj;k1iRAo78eXmJmS%TW_njm1F5x-vI@z+NjEHK8?;drVAY`P z5iQBS(|UjYX$Fj>ffH3ofgU(9YLubFgh7KjGXS~~Equ=GJ*c9=i$~XOs(Z=KHXeejL5mmAtFZe{HlZ*p5lb#o=VlNh{s5yqY+Kg~9Klg(3Z~lY}TEil--z@$R#k?>=4n zKJifhUK!s*{Y35(f-5qiEYxmEl~2A@x4C$p)I5t>72RIfkq%oVB{zG{wA=}{w@N4aQ?b> zV!`h;TdT(7nm2&X+3Cx4_RsO5=Lxq*noK8a=Dj}6a%GORh(+^5MX3%U3?#R@nWEv+ zay6h@npK;CeD?N1XD|-z{fX}KmA>D89o!>+K<5dG_RHWwRw7fGjLLje0G6$jR)v0ljuF*65ID1Zlmb()y2F$3_3 z2<11CH82?E@PF1VU*vC!be+ve(__wO4G`6&aQYr@%%KtE=y03ZPb%PBlv%|R+Bh*G zkU4z|BKz5Pj3dXA!yF2sO6`HSM!VeYvB{|Uc@&8`7w{^^RyK_deNVf)Pm>~yVU5LT z1A3A9)pm_Fh3YZP^IWU7a#oR~hecthriyaRCHdSs`Xcx3aG;=0C&uc*E$Fe*np4pnnok}f@L?Vo@m?J@0k^Pa)N*b1?*IRhNf&74o7XL@zMDN#kV<9 zo(I%*cpUF|u;n41I$|;L-syI1m*UEf1#OZ>Ys;@O0X2t{P^wo%+%_z`#^c=g+hPga z=sGlQmu&n{3cY=ASTZ|mLsshJSk;X*^=kOj$Tl$`;y!`j09Yp$Za!I$X*ECtmnZA) zT%RniG6Kb&o8WV_;=p;8OTyR!+o~ZJYY+2KJ*v6&k>1jeE|WdZA<5>UL!p-3z#`dl zn$z2EVz)Ednhc1>*@&AP`lp|A*p?}4$2=N7w+viIHtwhNZP6X4T8gsM3mmlS$VD|>=R>jG=Q+y3$isEnUXnRiv;i2A_JBs&Z@2{MPKYW2y8m%KN z;;&qY3|_|MJu1SBZ+X4}BsZj2f>=A@LOl4DeFh-Y?V%)tbk^7rHIp&ryD=BEoXUVg zyHhAM4N?yt+@Q@PO?_nu{=R1F2TdW2`UC7;GJ|5%tiN5Z6CR2(h}n#d1_sM`>NB1f z;AN2V)Sa?(k9CY|mPZ$nSc)ZUQk_#Cv9$m_ZSWB<{ zO;*x#aCJu!EOKfgL#bMoH3T{ck{nrb^I9`n_kNTw_k{Wcd9HJnyFA%a5Fuzyx z#>YZd101cz!Ugs(&h-xYuZ0>54QACoq?u=fX&U**m$DM+#}~m}vLMX`jzzBiDQE+% zGJ^e^b>le^D$1S-@(UjN7H{s-OE-F;XGT54M|vBNOTo28kKvl+#t_WVAI$N`P^!0y zdI)!E5r-9rS+$L>Q}zuV#;)?SA4pPx3-+*4K60y`@~~7E6T5>h2K)vqngv?1*`IDm z%woaC?T)wJ$Y7DN3V2GQSX%K@Dblu%u)bX!l_1{m1!}a<=!lj7OgUu!d#-_)YAhQf zd*w-D*2clvO|cLVLyp$yV>j)}q+4v$2|N3f@b1n zk7NJIcHVt{xn?qAbHLxplqu85d28SDfPZv;IqP;1$}7d5t^n$&$Nb>YnehjGOoE33 zejh>$^?os*CQ{@9Qjf=sqsn@1CKDHZv)yQvP2Pa^m>XUUDF1M=@}X;Y*CNpf!Ljn0 zN0uf5F=i{;>Li=01Q;PKGWsF{LQe&>-+ex7|X#UVkf$!f~vTBfoIF zIp29bg;EK`)qr*Cxjg14)2f3XrT9k z0S9{|suSOHIb&Mga+Lsxei(W3Vs827vnwI)w+#ih<$d~!5hfwZ)KhIvr<(5V%dRI9 zTv7lf%H2*T+&oDQF}W(Ut1}@J)`r?wAdDrubmYLw=L(w7Cghm#%v=*rq`O?m4;mmB z{bdVL(HWEp1|&gEzQlg~mQp@yh;Z;s3{61Cg1WubqD9-X1*eJg35XVZE_KyfujyW| z6Uuj4N~#z3t`G)p+_TETyq&GbI5Q6Kf=Z)EE6gr|nW3lq=`S9)%VgskNzP~olbjq9vw>&45TdACyA$q0@Q zxn{1^Dy8kg_mJXPr9&!FO}5M}S?N^Dk-b7q1aXSk8hllURBS$i#kJ|nE5EEp+oecj ztL6WoLq&%1=)WP@jUDRBsQOTm8 z{v!U!pqO43kvSlQqHwNjjCI?~u|U-g*03_>Q9JdZ=?hv$kJ=JYKhcb5<#u^JnVmWO z_2_wh4$O)mKEa})HfC}Bsn@HFUNgzL_%SNvQ=$>bQOwTTm;M*o0J!dQ&<=FJW6$iQ zf5oLY5?H?fDodLP_ex;(xu&-mof=1zTZXb_T;xhuh$A=A;aWzV6!*Nt#8Xa5b* z$Q?%Jf%`u_kYZ~7$CQ4HIGcB5X&S9u zu>uwZr6gY+GKnGjlFK<@^R0@!%W*j)?i?E{=^7G(C$0^&KD&Qwd4>i({W%H+=_0X> zdav*##+>L@l@0W{OWhu%FO+>8jc%wkFr?jfG-$)tSytzd;FY2w%E8iB#WbT^bH<{n z%1Xk}*;YX4?)<2zYwO~2#XTP_WS5LcUsF6x-5Q(M?4VS(%L_r!LuWlS%N{7tgt`QC zL7$SOwRle%{|t-Z;$WtXq!ms2@)|SDuVjM)V2t<7P&#gRKV%D;GU0K4OwJ2 z)W}@eHJwhmdAm(;u?-sOdFQxO9qhd?Q?NcxEyi+X)DYP2tM`GgK!d1ng-=jm5Rds zY&J4hvdB{%0}2DBW=)d^4(SsesVTR# z4SjSnVb26?ak8gj8mm)EC#qSMI*n#GW?fDeBm+*BK^DHBTu5NK?tp-UM!C-5r)t`Q zij$LJU0W!SBzq~0Wo=ptqzu1jJ_skLSAsYRA*Cjk&>{}|=H+8}KQ?Lrslz4yDyGI`2#5mEK2n&gqVETFDo zU&YCjvNUW$%abMp4AfXuCD_Gm(o_mTR2fpPs-*1B7Sr;-9iY)PS5x68j!&!aD4 z@MCqr$D%kTp%5zk_c!5i-^3(zLT%2TVEpDQ&^%5Oz22uUm_OTJloh-F48!D3_Lb{t zR%@<^7G#q)YX9y@;j>;z-_dwI@F*~w@N4xowCzuA!|X~fm5k`Fv4OpW&dYSrw+DNt z_8erCuys9a`&FCu;hhODl5(PHeJiT6lBm|ewRI)T!FpJ(knPAnBVSi$#RwwM308d0 z-s^=qXw3H=|JYeE@Av)J@HZc<)6vZR0y!Y=wS49_N{SB}TgTN4A4F#l!CszoN}hj? z9WZj|wXj9eO?$x)^_xauL}A{J{7=Dw{D4F^eBC0jykP=hss{E|{t!b6(>@^sVeNzT zc+=GP1AG3WKSXkluq=DFzr{S3YbO3srux*YT!kKhjZ%!e%!|BcDx=1HI!mE1+#Vj?Fx^gw(ua8ABjlQW>ezmHB`ve=uM3f)#!u{XQtQd zwv0fetG{-zu=N36QVE;p49w7A-Yi+UP!mXvd0s)il0wF<{tdu`mzfes1M@H|uq*QQx8-C$28RcP^EXP*Orl@~ zuh7By7Uoq8jGZj;xPnm1<+b-!FdQ zII!E_QkXS2xW_&*&)vXmMqEc?Nm$(fb$GIGbjaQ+zio2z2oJ4Lz^-(1zR5IVek%CEEOSVG!xGZ_wW{GNTr% zCKhM!7>O%i^8$Eic%r@nJZn0G<}>Y4ZX!X_R@En!3du}}3N2Bfl!r@XnDHXf1Qu~Rs^aK;IH;>ZWkWk{ux%tKQDMH z(xFPlgZzo;s$-rd6JhqeWv7*FWT-Yf+jEP0Xu=)sXQ;YNgF;5VpM-YmRgt~_n+OiV z{?3p^VtuDtiYIj}C9G|j-2H+G=zIyG7-Y9MF>IP`ta8l$=_U{{!cK!yk8dxDmG{6) z)LcO8uK7&1Y0Q;IyWgYvcbgoXAZjU$jcK5z0`8fTdAbGsROi6#uvCVNmYWA`+s^YM zo>sIuM5;a1&!lRMayb*3W5Xpg2o0zNgu+5ZD@2oKVs_v!s~J14Z1OhVye84BYdr3N4)_g1w2Gp)$xm)1XX`Vi7>wRE-pS+ubjx`jjf=I>XvJu z)f`?CYp>g7TT0z1xfn=zSFx*?83SK_~(M$3;!;pO) z$f?qR#v^VECkGcg^-E>Px))u94yZ%gg32`NRQM}Q^ewqvy?7?6Uz1}us21U9PkxoE z8l?x<6Dxa#iUtnqx;ZXE=!L(#J^Ik?$=sM_x2p& z{ra+|8fbThWR>>m?qutOfX}(~U$e1PM;CM(t*pD>+4^$$EzlBI&2T1H(xHQQK`tp(>)w zOt-MTX5v!ZRr8x?^H8#S3gnX+p+Qv-flb1B&%Wlo(o2dAyKJsZd8H!CzOx>jGu3$8 z02z1A_S$&NZMbOcd{{;$I7v^$I^KN?6e(|)nHn6+PV&*CU2LD zL_NsIn!o}4J>$vmduB(&Q(Pe#IrS83Iaq2H&|rd7MjOZq+l%q_4l(p((;G^vpWA@s zO>6STG1II+`28$gpsWNAEXgP-^LKisyKmw^(Ojed{(mBkLQ(FAc`w zj)Gv}+yfG&Owck_FUlFhLfI;SWjbn?ntA(_E(nN7(`_~U$|AfJTt5F~hK2hU5$|L0`B)@-At@kL&l{r{sD`od|rdP`Jp6AU$^91;5 zme1SsGa;Xj*5VKn5|J8P+?-1A)arc_GDqRvLiES*$Urx@q{?QVL!hQvHY*&Dhu+Gf z$Ui0sb@bCE8(}$V;7~_CBUD3_&#P3nptjAyMZQ}PA)WUS`mr)UU9X_lrC7ozBiONX|l-EHw-4?Jy7%21z zI?xxIM-4BD;1Dw9Mf=?cUyrZfwP(w=#tp%1pcKG>B>nTaiAi6-o;22f{^l|L@EG(8 z2_X31PLx@lH*XlczY9`ezeD(SI;>_bUfCcyG?ZUv(cBQm)jFW|yuuO1D>c2wK>1;6 z4prp^pqrj#6l$PY4~;u#@Hwrpr3 zZxl}!KeqSV*)}!^o|{C!9BM-`&S3%PHu^;i_i%ZEdFhmQdQz=y zHKVUntY`2>yWf&}XcPrL=^iVdO*|_Eg-0kRN5UOvCKjBd(g70flOZ(eZ)B)fn~&)8 zCPSY@7*n8~d5P8t_shIpOmfp>^mgfV@#KDx{F z?F{$>fVA-rLb=qh3ZZC4?4|vJnmsp4Fy>yZ39}en#DELZ3?x7=`#yg6$sIaMV!x`z z$%4VjX^yhlN%Pbe^r_CX_5q~}<3c6j%^CudUsxo{M73}3N!ur)z1b)P$V|kqpa%L0`ti$Vj%ia!4-`0X>1d|otb?y{2tu+Sk9M(u&-Z&; zyVXc_Qzq&!L`NPR0gq_pOo_yja^Eyy8evM$4@G|-%^U!lw)~|?;G3@*7;hB%(7<&9 zsT|*TM1(7q9CU1L3eZ;VsbH#WDPsl4&T5wzGL#9uF~|D-PHVKZ?2VRXJo=4kYtKF* zv<2~x-xC}C9-qa;{+%4DBGi0gxUa%g)9qCA883mE1hN=XmFGjcJ4usPv@ldijWYWo z62gg=-yFFGHC%A%ctYY%Zz_oyp;8GuIkdlJ03k0??4d`jVJ0x_SCo+#XpA(i_M`oz7>f zO_0tO-a#W~L+#gj<4>D9Y*Chx0)>=cvD6!^~2p}nPTmRN`uRr47FKrHn-ZC9@8uZbn z>7?X6E>T^Mvs$*X=@i33<`^)w%JOmLCm^fuJ1fjj1kflnx*|D}W3BldUs@8B>bGLJa2M2`u5hy=GfV8k38<$0J0AjOfzCCkKqt~7 zG^(z|7)0blcI0da>cJO(@)6A5Hww&B^Q}+Q`atYo=FExd#llo8N^dV%m-XGhmI;jR zm=nx2P&R4&)(gmesgkz>l`YYkZ%d|-w!xtx9FOjOE|BZeoeUlsJ99yvgL@8i6^hX6 z>_D;lQ+XPU5WU7_{)&`^f%J4w*Hhaz^2HI5wS6FMi!NkW^Gt znn)-{r@Fvg3y!(Kpx)(0`w&Zsuu|$q+QWIw%DuKywCbvrHY_QS5 z2DMdde5YHJhK@_3pm2rr-68J6l5;UQ!feok1BOs;7|fRSRiG*1`o)lh2K7-0Fdv;|qp*o;!bkBuA3pCF&z*6K_ z!DZ=q881&(JR?AkI#V`rt56B9e@a~~6^XOtFB>p~{nM#szZjUKw|jzQ;|JpY`5DE1 zgDdw%kf`QGBaygZU8drN*TEQMttbwKG)goscZP;^?T78}Tu%liztYcGI4Kt2+pbZ* zc?T}AdR<8~ZFUdc{N3$U*4~j*R%@^S2B76F`58a317&9=!4#8FeTdV%yNQ^&D@cLS z8I=+m;8{)8Eg?Us<4@KIfJG3&GQfoOJi9|I#K}q`fOx^|Y#9BiTCega(Bg{zTxT#Q zmRbAr;UJBjix`)`C0|0TT_P4%yoq>nY0n&YIUMbeYs|sudpbOy(#i4g%2y|k3;LU5 z8(V>x@I~TbnRGi(tkwAK(nv@x`GE!^H(m8od@SjG+g$|Ah%FFgTNSr~iJ_=7Xu0OU?v4_PLSRqIV7*SpU)i2%x%$B9F zp!ra?yXN^s?`1YO=B35VEa%~C0ehUV{?Vu++B9oK)6q3=E(#VbaW??%mU>j2C%{D@Oz{X_1zX{Gj<5 zcBjqZp;MKXtCrFYpN0_5n!ml&7EVF$_vQ+U5hC+=dX{q(BJT+1xbM<)cD&;l&H(jj z_;2ygSsYrHsT=Q|-}ebObdaH=n+Zl^Haj)7yschyUZa!eHbnVFdz&v2@B^r{iKsWb zs!rce3np$>w~u~(N|@A)amMcQ_Whxh%5RHdSxmXLU5B(3(G2MxUr4*5K)I5mcdx~S zjWXYAmx&CYRCIuET=wFik>{|%)9f8qN2YUJ5m*^nzNkttVv z=0krU?aE?0;T)7qW*ZLXhJ?lP$3T_chkGFiQ0Y=BRyb-DoM1bb-VZ5l;nc1>61#e~K$O82h~agELWR2O}sB=52IJxBjjik+tW;O0kk? z67G3dly}UTl($YSFk9rQM0{?VX!=+=EzfMkoj3y9N`nNWGp%+tJsbHfM$KUN!D%LR zT)Us(P-&+K8X3n(R2h9wjBF&yidRw8lCZ~Xr zz@c+XLjj+g)L8zSE#r{yTG;At^5z99A#v)qhrp7CSF|_8_vck#jx`e+PqyA`y&tq$ zZ9pzMUzOHcIwlO}3m(}YT~0D^QZ3Z!3gLy03ayBsh|&4jIOR;f?p+&^YsFIws$D@K za10TILZHhQ|HZD~w-pnBwv-~6o^x@3!^BinroEP%Isp6XpT%A~4_6eJr~W?vZrl*ma`$N^KV) zQ!y*3ULd>B&*YH6v$*IS(xA8+(d>{*@xB8X^>uNut@ipDU~8HKmDr$XQ@O+6&|8zC z$X|;gBiJf(wl#r3xlk6p74=v~q5TFxNMHGT9R~L2!J=&*w6!>37)a z)-*R+c&%%stsuT^BOAwPG_gq+sPPB)N3Oh``rwIEHOfV3P_feUiyBnZN9F?w>+l}- z8j)m;msCwh;kzLuX_yL5&L9Lh$Jeg!T$={+64Fo3Gdx={O05DI0eLDw;`cGA5Yp%r zjsR}7S_Z4KtkvkGgsqin%4&Yhg^2N97s~dF6c$|HX`3=>#CilYlq!`c7lg4ZFZX36 z?mm?s>oC86?TSog(g7)voBr@y4g#yg1HmG%b+hGh)CulVV~uJm1P-fldD6}qZ*?iu zw1W;7PJV2F=Z*o@l6xITnbIH2PIL@%cJycVC~G_Tp15uM>nJ4ta;_MwVhyA>-+u-mA}9h0Z!ryYOrUN;+#ZHtK=Aig1JhlFbs zlBJq7ebQlXyknV1b+I)(t(>RZyc+(};Z$WfgnQ{LjGUWVtgpqO^I*cPOX>B(QBP4d ze8^^kAn!f1N?=zhfNC`24bacRNVxg?tBF4gYlC;W$=>luD%_JU*~o0|ms*{iy)E)-d9Pd+Xj} za|_*JoGlxl+zw90I~j10bS+nwhRP3ERly?)^4O4r`{Mok7dMgt!SP88g2jXT`k4_6 zV(=RvuOm1$D~p{^oWCTidG`<&x=chhPnImj(KSRPW=RL?;mMp+R;Sx&z%=UHO zq-Bea+WP1@nNgHB!zerWOXrZ9BU-zf z<*Fr(Ihqa1EY3UOlwXO>)wPU?if_7D$c%FAI2Dq~$?w;i;vcn}Cih3ob!?HGngEwM zwC~5#XS^Q14F$|kSv6yr}Wi^e*Q2$4$C!-X(qWB8EZABJ73zrBZAs*Sf$!u*3uwQSMD=M znlE!qRW&9Xct~fNV`SyeE47-^cTa=L;-FM4+!)H9E$o{u zm<}dysvtZ}pB8G^O-aXXn=^()+tCl-u9SJt;xLH8canJ&%?!FRDDLaQxXLJ+(zJ*h z`)= z>y!oi-YIKU?a?%iCn*ms`nBlPAC3qUbm(y7TS9CL>Z9oztycsK8bG3r3neTW+ZK9@ zVvXzx_b+*T!)sqXjE;uNvlyeQGAa)PR>F1mPmn{u`DtYN z(rA-cp~D|@NpG;Nq)JM?&IyRA509w}QSJvH8hFpBRq2i@)8Bjgn|MHP zFD15Z7FOsJH*&C*gQaqvs=2PrDu_0^Ads>;G_~e%xzg7c?65B*SVGRHH6v1>kbGL{ zAV+-3Ig`^i8fST#xDa(mL~IFaE7-SwdmY($R+u{Wi(^CZ&{)@`t+#&!!!NQ2-gOj7 zHi+5FQB`70I-2k71kxpQ@^inR-*NOtO+UR`X$1G6@ot#RvktY*z!UuW)OpaZSq@}0 zJ4>Oy^rFg6CTL_z;662AX?M^k>1QGjS(Mn6ynOC*`OTdCKLRi^&&@OuzzO33%8!0z zHQgS0K+QLQs78^SbF|S9U?5ZxSF>fu;;2{d*(RCV#4u;)$H3+Fxjzq+VpA?G|_$+ z@VMWpdKNj`W>bP$I4d2B(<*$F8y=q8gcUGJrd7N4_>aX~ZCD{dHlfsePsLkf@9Sj1 zpJiDy!9}|zmwiXL>mB2sO~gxyotxH0LJG*OrOw<+m?(3+qbF4AajFjIGX1^}#_P@J z^6bgi?&-P3x>$TsGi{&+>KR#iPc?Ms4gT#j)PO5By148%UD1f~^>}*lN}zQMk_p@E zvg%p6cIlHkxOMN?rML%Vz3ubBsLCbBY;lg;80tk#sbe*2C$L`g*T|*?Ca`}Glw$;7 zIPE=3wue;(rM3__VLrSR>%_}mKJZ4-0+y&Qyl&WiLd|Nh2^TT9x7P(Ia?FffVHWV- zW3=_C>e)!Zm?GFa95%|d(>2zf3QP|ZLx?Ah{cg3SMS=H^$bJhH=@)ndiQz5k+O0b* z=8L-wd^k<+Cwz9M6HYeuDGIKqCAKlOHI?4>66aiB;k_i$)gxE?e;(D^bi+hzzYQOX z1qx;#sN(Yk_`rZ8OPc1<*gi_BV2Ca^5z!--5u$$a+xI^o!3E*d>RA+Zl1AI? z8|<0(LpD9Ojo#Ioby~m%JcM75$v5py5iL8hi`$0ap}QE_+HjvCCoV?Qw}kDSdnS7> zk)&Ux)DJAqD(kBUal$a!>{WZZ#FG)$cHwl>;yT&;w!U@?Edvtmy9ZPtuwSJdp1W6c z)!Hl!T%349*L7ObF(tMzuMQ@YUb8c#Z}&y^EeIC8{{Y7KO7YcDcr-++F zjLDoa_>QF^s=a1e6@UO~b;q}U@@HhFIQOFx4fyRR;!=EGke!DA03e7U;?Q>E%drBA z@qauHv7zJ*SoY(~g#1$*NXe3HC+~VPEI}TnW|pwvK?V%W#u)nWSM?1ff%ixk*`!2l zz_?EZEx{JFe=pcuPL_t5=*;1=ejhZqjatb)9Bql?ByFAvgdKW_k!&_gZ!^l6Ob3{m zwmcmZbF>DOq=fPR0B_i#r%SYxg^wNo0Cp9PJ=bA3?GWq0+6X}fwMlpdregDlsIs)z zf;>@a>kaJmX4{f%cD(z&nCuaz3x`JGx13Wc!(1@D>5u9q-}Np!qiPszYHP)#9v=kI z>5mLVW3p$D1A>zu++i(@B20@6N6lth(l}fW9uiD*l#l~}79JDKargzjMli?4J0wK% z7NkhWM36??f=%tP&jHS|DZvf`B<(Rg{{R%tQh33)Y;WMPsvH~ukO+vp4tx+8o6^@< zi7_XM*~f1c$Ae%1jsE~Cw1Ty+V)8-i>%-=&4-jmCZ1KEyTYoYJdU0$yQv;z>FTUhz zT?bKqZ!5Zq#ZznM#Xk6@09q~$V~pS>KnKaXyGbsBe++Ha`q0|$v(bvg}Zpyv`gz6s`)BFUFX6B{R~EC;0sb4y#S z7~{rzmMS#ENyG1ND!>vq@LA0s@y(8qi)=`6icV|Ai5?-$IQ*4;MZC(}<|N~St{G3m ziJW?`u~9IElI&H+2SayYCly-2+V@ugm&pAa?2m$yv>}bEUM zCAZD^oOt4Z(vC8=k#TuVFL4$d&i&~<-7x0(i?wkWLVN}o)=j-fVw?kZiuGRSc)08X zf|~M2Nw2f~zHLe;F&bC|02*dD{8jHMIz>Y#7Jz#J_N5pDNgly|&DKLX6GS4$M*(vpYuBrao?>^ED>BdlZ3jtfM!jOmEWPVz$;%`YL<;4fZJ3 z)WTDRWk5@4p>N?ce}T9t+SGFW>%WBpb`m}WCbp%qRBBc6zqi+wAr~7)j7ETe)bDP^ znx0wZ^-j0YCDt7}qu~9CBB&6oHmNCrYmY$I-N5CqfsF2%PnJwo8vaRdukE{G~7(_5%{RaQm0wX zXo)Q`+9cd?fn_;JQ)2^K+hch9!egkYvV$8@EF`bq7PL69={sh|-qeb&)oFvn<+jqV zp5}`hO!xtFZ6UHEE!v^;6q!ZwsFRDh3E_!saJfZ7iO$iqYBWvF%=JWbs~zaWQ4nTF zAHvq#wL&GwaJa6mFxvBQEvESEqWt3Xr2yq;F zvBMLBxF}O-xpCNJo{>9|DG>7-+XPT98yypm4oKp-T;mot*&j7M_IhmqK`}daV?Q;D znR;Yd(GC}!ZP6UcHKG=29hZ`J3}hIR2qHz4t7?zwi8H*FO|&Nm!NYJmu#zHE1qx-h zF7aV5!{@?O+8LeZ&SbD&4d7#ak++uVB((3?Pb_webEWrf;^@@`83a$=yc-RrH@a&< z@e>04M^^~OFqFJW@fD{Dar@ebrKmv%0b_y21TD<=_3U*KvocT|n<0yzHV zS^0P-9%3*XB=}^JhZNRgHLh>Zh~{0uv`xg3946Ky;bqsxCLoLTVabOisd`dA&UkCl^}Tv$$^O}eddY*W%! z?#W3Y8qoJUWI6^Iy5n>eUaPJnqo)L|Mdi(HyiWplOLy>aOA1-H^IaEG1#vZ>Z4Nf(0_+OB^Q>^dS3 z6&PJ}y6FHt)g~bFtO7tMhcan&iS>n9&avUOStiI9B6p|BI%8_4HghL}W>XPtinA8F z8XO)QwCvesmepwxO6Pc5d6Fo#_w+x6*=fuT@kx)LQl{cdw48aBm$jv&xRBu{#@54% zLnyd7gK>|^B;kG&&b6um9=@`ZH@c!rqU2r*VW#Qnh}&#RrrDoeiLI*=4gG#81fu02 zZ{pJ&Mk`d^P0b;~_uH``P9%`izWb3*0r6wZds1UFRt=hP5Tv}l{uhtx9>p1$BT#?^ zuN=3swUT&kG7XNY<_I&hN23q2u-7Q2)NiHE6_(X#+7#eKl5MD?vrme{{wa-k;uUI? z>Q<>NxB@OmQZOn;wf0t6>AJ4WrE`cL@cAp0@;Rq;S`T4Fp&M)Q_$c?jD!a8zv&!>_ z?sKgV%lW4rWPv*(LRzYdB3STK#X=4+rCa0~k4$@~+MeGbtYT{qbpFy>su0hg`ISNT zgCKN9$=SU}hq!6k2Hl(f3+OrT%UO@}Sm9WP!DzFia5Fz0%gJ{%7)W$T##-P;$TkSp zgHuCt+GUQ#+TC!>0c1ppwmoX|oHUWNz~tG;sxVP+MgiJDDWtZ?RtMN&V)14c#f~m( zo3ltbY*aHWv8Y^Te(w%mMPtA|_9l9krMT)5j6xHECeZ=oCe3tlB7RGwTtREMSg%7Y z*hf-Rs)o&{PB&U_9|KL};6xtu!0yh=!Lk6WCR+G0fH?Z|DN$Xs5Jk{y@QgN(#Vv1u zZ(jjaq%%oj9P`S43ge8m!wL9_zdESg)Xo;3KQy;ycst?jQ02XbPgR`9xgn_PkPu7& zAj4*CZ}_O+wGu36vO-zRUOZc;w|Pc}EjHjuYtub0IBnvy-U!E=aknyz?;mdt=~i;> zbyqk#A}5!ON64q7y%sH+y<7O(b(VY3Tqp5AJ*nCSOd1ZNTt^Zn4*Qo%@7Y-A11Gj` zBY_tN`0hpsm|T4dRdhcR0U%!-J4oRZ!EKd*1H~{R*28q2itlkug9A;$*|^9cZs#hq z=)rIdj+3APa!ELt9v-+#QyPbaneRaR*4T$uZy1|w8$WueC?5#ZcZT9@WE^d~kgJ`* za3HpI0$ba{Vs>3PTuRYZ{EF3kxFb|Z9Gk$A`{d%QcTGQkZ=)yvr7!k+ZX|}aw-fy$ zF&lklYq)5TFxdwUk&g+y0-mTIsQypM5bmTt^+I_!_S~(o1~~r!6C6Z9i|o{=4Hgnx z7M7bGphn%Wu;!6>2_WNmaCNY?xDF{hiQZP%>D*D)3}Quw7{&1$mjys)Rl{aa{DuZ> zdkVZ58UPI%NCXjg2Tb2?%PtW~uAFr^mvFQX#Z00w;zZSgXI`DeZ{ASNEFCasPp&FO z%xQCQ1+%JpF4_hFMYwzxd!~R$Ygr_d;kbehjU;d~2yo*H^(Zuy)JrOBX*Y1%cwgGO zDh-d@J(44xKBW-gS^z#ET21AVebyxp3~QWp-LTCgYqlBmO!KFJwAYIxySDee?xumm z)T=tg0jOGe(khPXq1B`x=_Ho!z+7&&^y);&@W{3X;8;uvj#y9HN_0^&NcWONz$BY+ zbS!s(x|r~dK!H2$@m2J`h+|sN0eLOozT1}K{wBD*5(VM`-SFmFLNSHMiFJk(iD^_r z`BOuzx{b%mb6hs6tuR+y=xIo0I;LJu^p%-dlBiN?&LLt~97(}q9*IJfY0g#4mdhS? ztanc-YC_Z{=X#DOU=pQ%&f|&(OpD-+Xr*L`$IC^ow%5361RL!tQ|wc)t^JT%MjfeZ zMQ;%urL@kunAz*>Sr&G#l$n5~{=^Q>b++XxjvR#i%39EG7fBv!0uAEWD9n2{ss8}6 z(>ICbKG2#n)Wx)qUeb+gq+7+(d4Cn-a$#eR4LjGU8FmeVI~Kd9{{Tuirf)N#@}iAF zeJSmt{6IRFqM9tkx8jm>G)4{8$@`S=8}=5qLX5NTZHmJwRcmy|%~|_jiW%+FX6+pQ)vor&*mk7vVl^^=$y*h_u?#+H=XyYr&^>!pWn9C|-a9)FDig4^TZrdn zsW#6ba{Cv1tgsMWJG@FT)Q1k;y{gA}JVe@hJJmzt%KExX&7qb6U$R4rV5dimdX#Ib z$9H{&GM!ePM4h%$)9DCgkf%ykz@<2gTWX=+fFv928%kKMXK_0=sfH+I$MDCgNATj_ zyy`ermjm}ex!#ugzr#9C)U+|_cyyB909NI@ltXJYy}They=e!;0OFRG5XlnY3`hlf zzc2Gzl(MX**06?Z5<|9`^oJGm-v0no<-#qr{tAD~o(&jxq*bc2>sKeZ9&cM$hd&v*xR|qhNp;jhz?k3c)gEtQcQN3gH!{ z4QZ)u_ye{_tSr2|9i9=S-E_PiCBT5ukKgNAsZyC3&k5Vx#u9?G)ZolH){Y35^kzf0 z+hSvCHc8P;H%=NxJYjQLX$MP$%s{s)Ae{)P(NkbBcR40+MsMgjv`Y(T#?@KrS!ZTE=T>%l)stl$_N_m~haNj4r3 zCU_1gCsZy29OxY(6KKZ{5}B4doZgU`o)}`i-;;h&t5`O#n12a?$f7^YcMWIm4a3FQ zzKvJPauu@UF6_N9#*YS;hPj*C;c!md-)hlixH7hCa*vEL+AkL~$hz1KiM;yHD)k;= z`DZhim*t#m)5l4@X7UIRp6m({_Oba5$?3xy}*G z!D53<_@hiNGORh#1+!ztW+PqLf=&cLI7Zp|u8mD#CJ){gjU?mBWc93<9U-T)6{0j_ zcG@=;AY?bf{I- zndB%D94YT9x?>GpWc(AH`asx@SRaa8t5F|%PT;CrG&2t1{?bq>lhS*+&$TJpwl>h` za)Iy?t9`A);h=M>`hdDyrvf>?Y0afc7k8pxo~iA=XboK`wJthR1UxRL&T)1oitAaWbm9q^eZt91>2>Rv=ss zPi(6l>%@49hA)yh1#CXy-*<0Xts$k79raLpMReD5U#4Pr4~nKLwa>Cys0~u?3N(VQ z(oep5F+9OARd}@a;G*f&@sdigDmL&^(hjV=DjCILl^$GV*lr4o>P#9&+fVOYWkCc@ z&g5wxhrqK|yMt_Fu~O7Y8i_rnDdo=a?fG&f>n=D&pet1k3}(DHxn=IhnN>2}b|JX@ z7LZb=CfMVsZuzT$#gbsVS29%=_=#x04a&tA#F8TWQfV4!7Wd6K+b9)IrtVfSbKZXT z3ciZ37G=2ll_?KK9%Z3eIF*-FC&VNU2`!>xz>*cF7lU@e+7YD!%EVcFp4%RdGqq5u zsQTii8Bf7g)8dH%N4SRuYV7JC1xall?I;yYYH+pvY_8LA1nUCF0;z}io3hRCh;vDaMB!0ZcMar>{vAF85hprw}NA7 zhty}>ji;P&2R;#V;@kaN=X(hDh8GSIU}|foEqoIXbgytyFH==rFW3& ze~e+dFw|#6Kpe;%B9521cxgMd0)9!2V}O`*+rkRlGXNl~dS8xkhl3AR2D6|nU~QrlfA!HOi<^DCL7{{VNbxA;&0 z0Hms#^rE}9W$OdBoBjn!Qh0*k(${Rp){WcyQyTXghAd1HIW5e%$|O2kZxEQ=v)}6q zo2d1pUMD+9@fUnM6G^_84K5f+jhjMcJiSZ1To#!UAnm|{t-C7PwA)eQUQA4%N-!3} z$~CS(JBfLyi4W;C4wU>87}~CB0@_9-Tn}oO&Coh%mYtZz@Cp6HYK*}hB3-219*B$$ z@3Ad)7WF~o{BFkE@dx0T!a>ys*W#v|mh`6>D@KGEZ-{K(t6E~-sUk%(`EdK&yi$>cSwAXlvAyM74QG!RV@3@foTRBLOl>gMFb1`-16m=V zZ~!ZW!s@PCuSuRPZo~u(7gmKq$WdAED9QPYd~ELOeIsB$mt(nl)VkAfNv&SOX!!S; z`8IRQ{6)R(FQi~5KjblQEw`v4QM}9V@v6q*d4Dndv6RkDN)eW;6X74jao*jXt>p!B zUnAb6pCk}&_1Q0BZuKFa^1RML`fIR(vBKT9TY2VIG)NDU4+H^#TKhL=6vN{>NrwSw{8H)EYhQViaZY*uAaKGyX#v)^ zySuO2l};*`07a8(xHw51oN!2{9Z2gD1pov=A$AE((q8RxZ&_S;ZY?MQJ=<@+Yr4=b zEwbo>FMIm%Ppsf+jimrClUQ8u5bIn3GPyTMU=yoM6<~uhm<3j~&fI%P!DB9atwTuw z-7M4(1W6HfXTbVEZH`+d?Eraf8nVMBCGU{~nI*0)#dYa7(Ai<{@j`37Emeyn#fjna zSc=tlAA+qT)4;OVLrIlEu7fG9zk{7+%ndMvi>-E<)i0H|RWg$3*nr7aR*{!Cp95Awj8TH+|>bs$(mT#z`;6^()kGuC@EHkJP*Br>{xl|@M$TPqNQ6KnUa*?8VhE2 zszPOM?m>BXU|N2V1Y=~aQ)ST(C*YYQX`48M+O1P_A_!C|^Cz*oo=n0_XtZ7&4(t~x z<_J1u$09?icz_PJ)3f4&-6Q~YPSgQ*nRPuxoNw!7vrw1FTzfmVmuj3naaJkR1)xB| zNkWZAnr_gep8aGF&u7J5sivD?UwWeJG)C^VNeI9qZ?aSAmbu!E#42(MYj0s-IHpN- zwUet%?n7Wy1mFe=vuhB0I~IdW#-_j<@knKZQ@g3DxJCg!r9wsoo+^oe>2F0{ZAcy* zH>5KTA~fT9Rt-9J1U0eRPqk4n!#{Z0Y0ot5eW@@S_O?HV;l9)R(`#rA@CZJ;(o4J7 zLvfD8w={;q>G0VJj!V7xsNWVIyVmH`)Y;yuQgcQA_YH`4x4J=%z?7y8?(il-ovF=q zC?VeLaqB6e)i#oeIrS6TKU(nyjc3(Sq1UZl`hCN6x!09tzS0Ke?dxC{ko>Aka`pcJ`-76p*1dXpU;h9) z_?9FEwSqv!R()?rukc$MDw18}SlgIY9hx%@u->EOo0Z-OG}25{stz{zD~*K+p326V zt5xROkV4DQF>O}M+d^{(5XS?m6-NC92_s}zXg`ROB~y6^}&4;^GyQmy(;Y& zrtL2O0E_US^#RttW6=^X*1YT2O`uId)`B`mnOH4!c)1eIR*YpTr9xR_!0=E>!u@t5 z4fnSqw#tCh42%9Mogx63?cTQc+($Yf22^vX*nlQ>rcy0)gzZ(TgHR2T^H!J|7%k!( zc6?F;bzJ87MWkDmBcpdJ^Pu#mQX>#|CbFq55+j*LYBb+F{c%F~1I$y}tZ8m1;;hjOYa4{B z^(he9oL6j~iHc@rTGl`?sW3b4gy6@6B`_A>0VIMaicX&K_f{X`x~N?Vh_-3brU~hC z`x0opT*6p%h&Urq0>}w9Rr;YPS?x|>Y)g{d1}TJ&WybX8RV^_pyAxQ}z2;H@=B@5h zp83IQr6!SScUi1oVdXr|_d-f^3ujA>*2 z7j+fJRjGSecKcPaT6ik&BAkF`LP;^nLZO2rX-j)vV02|_dOMcfL%ncROA-KUuds8Z z&y4R~CS3_L5}6Er#D=Cc>FwH-4TNn?YjlG*B>-9vY4Ds9%e0-_5*pV@G85^>pBR-p z2Pn~T(I*3fWhU^z;!+6{r|#?tpt+-RHkBr;(qjguy7RjT?A)WIy2|6otqpC@uN7{d zSTqx2PA2y3N_3HXq(?G|9wWgytT-th09rvDzQoZ=8;AWryeZUkYh-I_V!A0>0+STv zTw8bVU27mn_)48()X2W;O4o87i7HvNm_|P&cTutw>kVv=6w66C&#ut9-ReA4h^H%V zr#%t!=0>2`ZX|DY;^DDz)Gsong*WNI`u_l>VACTDd%rAF)*7D@fy-)cKF?}pIxS;h zl6gLH3hM;;PR*bad5QQYXq}_ib|%9^q0g-ndJ+pMi)0><2O#2ra00zgN=r|c2O&-N9fW{nR2Wg*`xf2KxxZ`DrfG7Q< z*Wk3*NH>p%E~f;ACs;s{2HV$MWHsBGK5M%t0T^EU?7&TPJap8U_5uPw{B?Cd{#8l8AIldWoAs6K(6^E{qKvtFnN zV+V=nun&oVzJH(m&zj6ja>MNG^o8mkTpP)K>&Lmyc`k7-ClXAP%D9=Hqucp!=B&@<9_^+FJXXd6pDd#|G!@gjapZPZy_O?dgz3VD6t3%l4y{6&K zAcNRODH>s7M^HOffrTEdHm{_E8&WnM&d?dmrD_KkFoB0iUA*h!w`jjdXoD;6Sg(_L zwp?ZN)tQ4y*51cnIShr#%q@?=&AA~AIAQSimE8b{gYD&*P9Pk5mI%Jq{1+UOCrAK{ zInu>2A`c-=BTgqBH-ds=vTzoKYRbu17j@4j|m~Wr^L@juQ;(yeD0Tm`fS$B40 zV7lvE;7K?2>x-sAcCpW4s6Zsv-d8C4W>C2hc-o`QL_Lz-BD(k z>-a3`oNRa$#-8sBOm80qn$?JNPVJCBML%#n)}q^lhds{3uIh|XZY6e}iMG~;fgNiV zDiRe|HP0%~bH@~H0;#L81&|Abke6o(v=tq!?RhntIQz(kU01__E8q78jL7KKLiK{o}Oy^(6ppwHI7< zAXDz0u4i|tOL!&5rypb{1&SjEaofnZT7ahAdZso<;)Ye0hDFBJiqoeRBPfZ3#HAOR zBXWBh;%6DKRvDHzB~+wj?FBFk+W=pRu*|%4mFCj05eePgV+%KP7kMMJH&!X#)~k=l z+OGF{wk@@xr-AGEDAY`p!*@br)RznCpdqDZ7Q>UrZV;;N`)J&B{Aa4h#drGn0 z>HvIftQv4;@w$FwEk>*AkT$tQhlyM>LFkT19by)nApOzU55o?nqUO2;0UVRcnWO>H zBYeo}kD61&1;cjRo8v290DLARvi|@C*1;M>ohNchYlDo9QL_}-4Qp{ChW5MRf?#Fj2SgWe`@a}+Z0h`BXQa|h>g11t!z3G# zJ$%v?YPQ7D^aj@yF@Dn z?Nz`pF!$;8?XqT~0TwpxK!K-EF9Hw1tQKEXT*;gKmRSH9Byh2V%&fP>-q{vN3BND% zJg=AJ-g6ZICZmX8yU&H9Lq^HkU~gW3Ir*bEURjnOXJNLiY7xBP-b?RZd(4DZZr=d^ z0H~0btZx2z0I&Bm%U6fz2`&mKd0r98$R- zGtK#Pri0*2x|^iacWZd>7g0z-Nc#_#{$a{z;gtx>);Kl9XmS3B@7N3;61_tF>b*yn z{$b7JA(aWtQuJNYX#SVu$YgMYG-#}SjjnTWURU!SF;G#g8++4s+hW)|yh=|Y=9y11 zY|*K4iydc<(YLfn-k;_ae$kYw(T{)JUC;{rbI>`4^P5&Z5?HBkS-6)=D6oM#vgtbAgu~} zTGDqCBbdXXzbGpSVm6&R zDKI1uP6&UB>t9o|21?r~l1qV4gDTr2(j8xdmu4+&pz1F+#c8>EjXo<%y>rLm@Lkc{ zJItxp8_N`3McX5`{{R%WW!z7R;x`#n%x|OzBqY6!)rbK-Y&Vu>8%2S>5Cngz#b7g0 zy}fuR={*)n<_oEYaTY%}bFGG#UKqMvHXx1QN#`j@j?7`EKjO5; z2mb(1uHb+g;h5(1vZz%i>H$PK&`p0*aKtNd0mec$Hg`D7zEQe z@!8f?h|M+_E+a@AO3tHrgE$j2@P(>uEdLX%p?xvi4w)hV~6_JAgDg0}A{^GW{z!8&hYn#hs=0QVF8so>Yn!2baJc4B|V z%lj9DZCcRZ;?}qa{Mws;)Vda(DYc6&CeLa7r_ATC1qlpvTWm^6tEbDM#;M2r=sM=u;c zd$GfnKH>qY96yDdiT=x({UpNTUH<@w&0&w(b3IYrbFVwzR6pL;{{RxE-}NcKD|_6^ z+MMS`nAGP!KZjS+B`VUEarI-^R+84YNBKFS#C?-BfBd8uIxR;uKlYig(OzY>YLL=u z)wl=z%AVDyKB4~hulSGK{-qjnzwgtyT2bg7t7RPA{{ZQ8Kcua)u9kt_XD$B#%I2@j z+^EktbK6pY4s$^jXG@E4eYzwla{7^awWI$4xch&(U8zLj>hOVS={*x%X7qDo{{U;5 z{UqH`=@hegZOUeUMN{&(IpVgbIo%j8bDbUvxN0RpvZYeMd&W!&H2RXK@ z(Q$0U{{WPbDTz))aN=p6(%(jyz?s6(g~|u0ex>9KjfRG z5>M*Jcy|VbdO3S=p_`BWE@s_yg6nEFp?phA4wnbD%y2Lb+V+Ei3(p7D`bxjzKlYT< z2pOE_?8~Cx(8U(5VYNQne<3vn-4SF5K#tUx;XBouEo<3vw2kXOVox`os!eEh;h=m} zexj{sf6hIDQVY?q`mVx}wWYwfq^$}7nG&?tgy5LWv88N?S{SAgC?*6hCKi=~ft*6( z%%(BSd#SiRjxBJ&w2((Ay7xX?`KF~JDd$_QxC*QuG5-J|Fn5uC#dr`_l8Gqw)hf|# zM~ZIL%|z31xGkG(9fsJZQmWS&)S$9V6H$`_TGm2A<@_LFKPP4A{S zP5vs(k%3hzG=n=gENUY48iEKjig7J)i5IetryL`#SVLHdK{`%2-W`V~A#wE!V?mB0 z0SV61B+lXv2dMm#0|TeSj?hdFOcNNbO?OBc+y~&K=~QWBZ@Ex3cpkLsY;#&>@OB}< z7mdNbDlN??13i}`=B-TaNFdG)JeNrc9zL}$?yK1AXgu3F2vwXwFc-J;9IYy>H3%WV zVm=GTcWt>|n^4a6ibr0+(Y;o4Nh;E_MS|)@X9aHTkT6bWTmgP{WG0*xI~?tm+Qx+4 z1mSh28z+SVD|cXYk04D9Y1_=M6QmF08k>hW~;s2fmeVXV3^{R*ShM^ z7Ae~m*EyuaS^(Ne36#ksO3hn5!q40eJG*Ves(d+v?UI@UCU61Rd(1P~ZbCw>aeH_LRKJ#kg0>CAuh zx2Z5C{I|Om^Kozg0N3UL#Qy+&mtQ_-MYs{9FL#m5J1t(L+S+a>D^L*j>S-PRSiuv1 z!)Q_iHA-Xb9$EhY(b4^t(3jAS6NBc%v3h^afByj7uRe#(dNyXM>L2$HEZP46?5J-3 zmEpYm&FlXFF?Cl`%T#@4HO?=j9eqy+Oq)*CrB>d;=33~Ih~4XTJLN#da3J(;KmIza z$o~L4ul=7T{s8u-x6iXo)uxtJvyPftX2f4hgyL0KE-bMqu*uTi!SGsf_5I41^UD7K z+45iD4{Fczy8i&#@?YRL+NXEY2dVRq^Jq^B+y4N(3LxO9KhLu@2rr|Rd`xPkswByc zPBDH9KhG=w0B6a6fIX{A<;$58J!h4Xr!oGY`YNwKPD5L5gBewapJrJ)XyqHnO)WEG zFQvl?Oa6CT`#wwj0qsfUwvwJ8UOl|(bItEoj?WEm{{WGBU+0zo0JG%3z#i1cdG=2k9}Z&+>D~R^RpkEwo!0)(lK%hzdsfGv=9$r@ zm2BgtmYJ~^(&0Fisg~82AU%?4ws;N4V%|HIALn)d0JG%3z;Cs8`Q2~q`7iJXwJp=) z2dVRj@2KO2Z~DT6w(;1ipFYi0jdZfFiH%gXM43G&6_4k2zq91Oz#i3C%bRTx>pZWa znEwDz{S{Z7XFmfIX=^x1lN7BQHnE zdIkaCXJomfIX^r%X1ISQmtO3YkhJ~nFFBAE~50&OJh3-zQR~I zHv>IuOxtk5jesEaLftB^sYtcZMhd$CTzpKAGoGe?(CE11f~?wOYLU+f%sc%*^o3EY z)u;Nd#A>z#0EN}Uv;=e}r3BQ? zgc#N|n*)yHz7+dtfe`Jx$IW>5pOW?s0OGvvVSP)m#@j~f3nj&-$LH}$O=i`~Nu6?T z0Nw&&aBd`ZtdImkTV2Eh+JF~Vr6Xt;OJj>jzSQ}hfOagvR0Nr(HWaFpO|t;awutIh xn}F;{%zu{dPm0M1pw`qT&VkxJisd%T2zz9XAyi-q)XCF|=*y<@#>6<2|JlOgQq=$e literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000034_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000034_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..394381d04232ec9c40eb65afb306edc01d98fa7b GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(@|jVz8X29g7bjf!IrGvu&? zv5{$P;Rcc)ZsNnuKd?9)^%o>0M#Ew7gadM1jIv=l6+t712||>TGIt8F2JKj{LW|h z{CC&wS+i%)ZmWK(>gnCRs{bth*#Kb3NJvWnpr8Q&sJ9Q`&np03&dAk-lbMl?k%gHV z@Mjqy3P61K4*4B2>fZ?i9pirz78Vu`4$j*Z0|N`|{d*i7JUqO&2X8+F1O%j{#Kcro z)So`F{pkZV0RYf2P;UwFuLKJN4S<4!M?n1R>0jwz8E>LM!@$D9V*sFF-frL#;Nalk zVG;f+0gVYmA;N+Mt4wJW;BcOc9jDS=12@eYe=+k#b@M$ul`(E0*byh;XP%lUHJeyn zZ4dhvhqFnr(`7y;02&$q3ypvP0}BQBcUMrD&=fE%SlG(BA~mo^)90#jDh>hfDZAk= zHjTfq22!y()#f?==>lLPLU}{`!$48K^$vi+1pI*jP+dPBwm?IO2VG<@+x-rIh~goDJzu?cRviS zl$?00lHQS56`?xgHmm8juEW90p>+#b$KX>H4{l(Z5g09gs;E(dH2j#O2MKino$Gsa z7}L8a*zYPmQm;pcJSq%-4(iuH96Q8Ivo3X1G8Dn)EAD>)xWz?hCFNUc*Q+-;cMPpY zOSh|;*JU-Gbl?NR6ypiY9=;2Mce<^`UV0~A%UW9MKj}G#21Nc&j-O}E7{J_9>?=KN zJ)*x)6yWj^_YpN+viJ?s9$^2*!OOVaRz-Gpl9Wg|yVgatgDcVEYi3|v zuHBokyL(rd>hq%lh^8p?4*|UY9%EA3ZPsSi2SxdHWzD~vr?bi-_`er``rnrgT=0s{ zeAg)16Vrb=!4d_O0LG8&=n)+uL@TMyUl9cC&EQ!pntuuvQsp+9XdXzvA{$ipk5v;8FJcMJbZt?C!_ zp0O+P#*ElTH^7b2|N4DnAE$omL5ob&= z;pK4lmg>E#y*etGVDI}UrgXvk=;_joXfVMTXqr^PJus8*f18B?)?YSOCH>U#NNe?8 zuWqJf|LDUhAqd2r$$$_nJ$9q1|!xgtLBaX@*08>-Av3g)du9KzFr)>Nr~7v;LLWm%`u2l4-X4si(( zVnMSEj|dsmhUx!@Yy1-fy?U>5Bl8pIQo^muN%?vgp7GCYxOF0f9Q12Pi>0p%Qod7G zzgvaEbhoDECm7Q0#~icL$@%^M6Nx+Z20|irJKVb5pRNKahqDmvh#M#-doWzG&~ufc z`AH(Gc4Dbgu|&Q%?Tk}M?S|GqK--uNgq-u2vHpjVNt)!>PA#{Qm?k#RbEGXuG#oqB z1Ps%6wlv7-aA+53n3NFG}tK{wwF;m4b~C{=Sx_hl*GWo1MZej}Z~4dNf9^WJu4 zJT23W%^b(My19}AS=uR0Dy{$#Im6%lKgHE9f&WiqDlGdyJo}9mRID6=jLMl!1e;XU zp)_-r+HW+1V2Ha_vhFH zfJYxlz@5YtidFvJA3&?}xs9YGT7{SdJV~96&8c1%U%`n@XFHL^oXBjcvJI+X{lr`; zM8VTt2OHdGd=leiLwhmxn0;p%w4$T@>?~OP0@+QJi%hFeN>uGXdU9YmNln=s@6?Ck zH1M6zl~Z+L(ST*2i0A!n>gYONTe+&Lvmu@~9ZQ_7`W#`^ef?McnWy&pM`z!+yE$NN zMe_wyGvW{KF(d;6Rxd^bIqD@vNdg=O#=A)n0^LX}=-yNR*2cG=O`IZ>&vTMJY zCxbnu$LyEB%>0#B^>?-v)l2Wrgv?*%uZyG}>PMYL4eR4euW#aX&gj_f{r&?`S8Ubq1Wn%B z6Ue7N*vLHPBBtKrpb@q=N8fT)q&F^VCl`Z)0}j>G#Amxn|}i z@`8@07`#M7u`j9q0QUEo^&AhampR0n7t}fv&~7TLa-!(V4V-CNmSy6 zYX*I~{6-N1Voarw5wZibb`;~Z>c-PQ!Ga|} z{=F@^_`gB7Ut+`Is-0JXXW=9QHAO-gOvH^D@5AJ7qb80q61TxKQP?w6(7E@JX67;W zUvZO|J7cJ_Hni~!I6<4G%;ZdM8jrHYCsmiU_XyJ?>YVhkhjTB7d`S@Eq~!?@U=lN{R42# zwprXVxniYO*bDaQamIKZC9MH=I7j?zeXNV-B?)j8i17Yqk(Sb5o2*xurBX9@pX{ z%swgTEm&Kb(leiy{Kb8Zb}pbbM-<@;Env~%IC18`4SJ9CPT=XL`++fKmTIoe`dV##6P8)`TrLsbyRjE`S{nU`ON=0HE2GX zTR*R6AC=x&y%hP?`Ch+qAr^mL`P?6X+#4QsFY1CP28Wq-SFV0##jC{-^Czw|(Kkk- z8li7DxSsfH7MW9hW8;7H^fp!a{6AXx%Rl~Zh59L^^&3h4w+6?%7s%-8(x~>}9GakJ zpGTplro|#ro%fUekis5s{`0je3%Hr_0R##?rBXr|&$X|GY>hLO`Gm|RwzD1o@{SYq zz+n*Hz~fDwI2cG8F!pjXV7@^k;X2clIzHB{pCcdCi$(ry^h}<4-rTBg2c0ju2}W^= zZ`o}j-(SYrd)e<*E(V+bDho0fNle=`;h$DJVnX3eo^}S^L>8PS0bro zQ?psg|Fk+`|Cn^22xGRTVK}80z4U$L?$>L-R?xxeb;!L$RJj?Yx0RCP7MT^c-HfV9 z1sWO#VjyZa9JFMh!!Iym0ynB^TOZ81xbj`doN%ZLwE{fKHHHmeyu0rFussXq!;OwF zZX$)d9o{Hq4^;X@xj_bl#zdvn=2Z&kWyy`CW!CF5T{I2LqF!k6u6>UTU3Oc;cEQZ- zCI7u{+f=d@G4&q+`?^n;;OI^sK0*!>i?MK4)a%?ymCqUb4ClI*-xH0;$;#!r@^w_u z8hiW3ATBDS(YEE7Eq@Qf`jh^2jWvAbgjDGkOHLH+9*d7~m;rxbgW6(4!(3syAvX*I zl@u)gm`^$6X$Dx>_>{3wut}I+W)d8clx4EmF=61nb!Xjgj#ohze4oF&w_|_xo@qdl zPV_E$4YV>p@A58*r+MK=7ZM6RY5nG)2T16%* z?irH5%4jQz*0lTmv`{WVw|&A5=({@ zle*aB`T~7O#z3ruD9UhK6~A14(DWaGsF5!b={jmRq-0Rn7dvi(JMgZv#ml#aX^GT! za!VCq%GD%y;T3#N3ggmfVX8DGiTTVg9NA8(z=DYA;eb-ZycE%EL*dES02~@y*Yzk4 zow?yzOP}?WJgDrpx@R1(KY4HULmmQ*MrDOW6N-=Nw6ezdouRjtAh4vXu0269m`XjS z*s}6n88~LQs2I<7@)JLha;;{o>V-n)tH@3byoGEaBws2d+U`BOfzIi|tU~F6O6`G< z9t1a&MUP{SRs_5CB*bhK#_{D-hCj-Uw1kbz)&KeZ`_F$|)Zppst)S^k>2v7I%KXd9 z(l=%!JLu@Fx(d}sb=fRQ-XXp#EBj|&8FX1$&=G6)#9o6BB|<$EM@06kX-yi(r-nK` zh2)97HJVhwal=7j9|Qk%AQ!SNzNt#V&3_TAck{VY7q#4+m39y-=7 z*Xer9o4a^PzKgM4S}mMIHH*l`4$D3v6}&m-R9ve_E98xa5N+T_h%9W8j9bfH4_x9A zSFNzJxugl5UQI1P{{gVgt3xc5v#}~Tg%e7<8`d2wCqXKZr7dN5uW#WUB!)QWQMIiN zX*cEZ3XRD~EAowt$f%=LZ@U&=HeN)cAJwdhEIcf~WtjF^?tO#XH)yV;<`s9&Gzg_? zTyC!u$`b_FJGu(FPeYI`?q}CQoaqMYp!>8L7&UGy(+pz{td`o2b|#%tMVO%(iL%fN zkn+-US%riu0+?J;ZGWms{$=ydE4x5onxzaa>rnSBJ28>yOgm9!QfyKtU!Yc>vhj>h z`|gKx&@wUHp!{76?Wn-6ZsRxZAOaYPIkv_`$O07jFQfu-?`w7YMYUPcR zSGL#FLn|A;+Q0i_fSF$H519JmBWlqZqOM84RMyLAg0bm<+URwd zMfk9{-04kr)}t_D~zimx45bx2_VI)SH4n`5h<--5kf6}5;bZ|~dt*443jl6k;l zV$vb?EF;H|>T-wJ`C;2}-C*_v$s)PZjDQBopEYCV!#F$&+7&pBOd*#yP2?~ueL#x~ z2OW0`pMAVBo>zY6z##XuL>&z^o;qN*g`pHqreR#o9>T!DgZ&53-3oZuLZnl;rSJjOwF020~htJTRcYUa= z)Gb!a`v!)*1Ra{xFkH8T)`G5)V>V4E3vfp+ujzl^^F>$BDH1mzUR(y+k?r};2if35w=u+^s3n+ z?tm1$4{dyVV5x)2Qb`TZ4{T=ZVMh^tO@5ywfgh6{h|joqi+cIQPloM~thtl+@g`&A zmOb+xwOqD?9bEY4bLc2{y8u*bQoN1kluXB(s?@sLd)%udZY639n{&_{7S0I9#+1JL zI>{;LWb)BoS*AV2OB*p_dp5(WiD#fQ%_6I|OkvZMFag++fwZ3o>Sjf09EAK5>ETAe z?lut&3K+%LqI7-8_c$u$;bJOsO!Cc7**aDxplWVVbFcEGRQd$`g0{j8Qb)bT5&vz& z&JBS^LKw%s6!RvFI~Mp<(!S$526Jwd9UXN*#1$6s4f&gsqzyx1-HhSFmd^KVW|t0& z1QXmE7}yWEnQ}nLUWxgY%`|uSI$~z-bxDgf zt@_Gz^cHh!O+wN;=QK-o52`)RqvPwl;3}lKh96Ngr8!7KAzQ-7e6w~H2gw35W)ny9 zrjqKw(v_sJO)S7>{n+9%wf`xVNn$AXt&{w;e1TFacVBu}=dIq#7ex$I-;C~=ru3e5 z-76SZtH7)xZrua|Vp^ZOQqIJkF0XH(ER3_6&js}qCq~80dP2~6oK|xA`msjT?&Exr z<%X^;YucYot=}KXAO@Q$#c-x%YTl<;74qupnk)R@)zy`7)YXO6)zzWY)z#qs-+$p$ zatyo!O&Ual;s?MN_d3f3#ZgsWirI@zg0z|k&N@iorR!9e5rzTS)pwj_-h+~mv`;fa zVP1I1pSXUQizD#XYx4JDa8V??YCVfw(%BX(P=^ev9P7?jhZybNXPBzFR^*%KfbTG+ zgSfaMFJ9*amA<2ABndWNOJ7P#w5VfIobmPC7GNRr2kHG88;A6XDiyrKcnjwWNw!fEDfoBhZBLmlntvKvDAxhL?ZZl9-8o<vFIK1imKj`yBJ$e08mJS$Ocw1STb9 z7O6sxxXepMs$Ow)I#PIIMc|oP;d=J*u-;~IQT;pkCN9+ow~&ONjwDvbiTr3zB^jZN zb%9r{3QYqDE|Mc?mPcx%I)&r*gSjrh2T|0rU;|-7g>0Hjo92gbYOL9G;6M82zb3>mhOSMDMaYi~vbk(DM z?*&i0(&i|yu~xB^AE~$4KT|9+;MF2ID_$kq`@v|KKM)-yj-Ul0`JjX*AuMS!76TelGP@7kfFr+1P_?dLmfSHqTpC|h`QIb?$1>X2;&Bk(j$@acI@AY4eza3?+mJvw@LU@Xv@a}UG{lnyiH^sY`w#)i+H4^gWaKYqfa^~lxTo8iMZO9c zAw>@AJ8;REg4$&k6CSu#DOSC;$|5nhvK%Q8vD`z+q>zv1kT+A_Fa;sT596(y6Y!KI zlyz@TzHLUR!8d&8?CdAjVn_Sf%)4lhH$<_lP_&@7qWXS_8fk9CzSozom8EL4B@U_u zEe3!D$YL7$=mo-!^8j3L1TxOeH^x69d`3f{eK) zR7bk0R$+)tCftQm#CcQ?uB8$RgAJe3G!Y?s0S1r^gBQ;0{AdoHplV^t8|lo=Sv(=; zAWZ0VtxD3W^6CLCh%AaKGoZgoBi_vu#bQB=PJNOsdVu}40Lnp#Ep__7ZUpPLM-R%_%SYMdX{Vx&jG+L9LPR70((g1F;?}(&;4b)4w`-e7-mzHDM;?f&tOUI`=NVr zS$fRni&=4jRKncB*a~?Y8RkR@MHMidl9p&}EaUXE%(2|4%KWmIs&u%@YY2&Yy&l3$ z`^?!%u~lhZZD+uG|DW2Kf|aA|smxk{DM8 zW=jIwbtjRG1zVI*Xc!W|lGk54+b?j+Q(aG3i&1!d$S?LyIifTtyAI0A%1=lsf2lxA zA#T?yD4f1%ki@emmgGk5aa%)$pyX2y7m9Ad&ggAfnlL zLA|SnmPft%7ej2nMo+c>KQCqe{pwC>%kh8_`Svm?l<%5+cFPA(4FMGz#?JNmT--qb zLsmAvCQ6m}bam&G z@hs<30M{NADcZ|eodRoKyLC=D{UM>AOBf^xK?k5a3jJ4?*d39js4|UTRKlTnFknPG3gr z#(`6u=wB~v;OCt0Db5eDw)V}85k?4EyAp@zjhTpwR)1mB*K`lChVSsk7S34A z)&TWSb!<93qe)lDSppZ|H>#o!d1!gPx2&!AuU@ySC&}UL6*$f?cB#~v)v;i&lhbR& zeQ324XNPZ~jbJmBY#8RbLAP?!DbSV8G@MiXB9d2>6fYIlxFt|6#V$=Py%pq+Oqerml1&u{y>A%1Yo6czTV_UZc`x8LalDyD3G+6Cw-I>S@!Msqcw z@8ovdVLgv1_ks2u2En^~k?a%!6Rr+iJ@W`H7;`YPK>-+C`VM0{Nmd=@D~!DUs&N#5 z07?Tb>4<I@e{@&mK{YpivCXYxmz?nZD8P0g z$OKcBLSesh-p*#J(w}#4`uocX5zY7}cZf(Ff@=G>U&G{=^|G3_Q4*e_jbFCsmoo3& z+sov<@dq(~%wS9!$mB~!Fy@m-_FT-Civ$w6%oVC}6*E>=H+L(FLJ3EWW_*H*v8~Uy z=%NE00@R$B-E4o7#c-FpuYdvao8ESIOUHa`MCy`Fb@HAN7;BTSTZ3C24Um~;@BH=9 z9te9*0(_XlY?Gj7E@f#g+kMfAAH= z(}NEv777awnuTuSJWsYU7i)X>8O+2`!B}$DHsKb60{OO7u=FLrjp^}+9n{uz(0L$( zYfvD&DjyEa;D$8SshjcmgcdmBcyR*yO@g$AM@mEH1u!gbOnsV$%*mem_y%{_COjSjR z#z-@$;e=t&I-?c#*0~z#_^TXpmV{{=l#9Y$Lt$;OyWE{KH?Tp#84j%tdiG4u8rx!% z-YYv8Y8x)>{1P+^*ZV|HGxd^l>(304(rYK%pQ9}*ex#O`-C<@6)tKIlxu=zfcvi%$ zr>n!O7U)RL*daLu-B@OBE;N2P4bf;GfJ+O|BE({JMRCQ&hJ`Ud(R2gT(AG zPMKrq%szB7>;qTRv^?c#d3E@{20_PW$<+;hu1Y?7f_Srx$$jFp9rB>^%ON8~)@r5* zLf!;~15?p%lgIP|u5_>X<`qVr zWkr{eBaz2E$$h)($@&N2I6j(a_n5_`--|m^?r!KK`|V|C@iwi}Bw(q_y8cF1*ny0B z1wG%1%JCxRW&7^C_8OHzZj`+s1~lb7twJ#je&U`BHzL_*vvU1K@@2SzmnXp+Roix8 z@I(S}M2~Fw`mD>#{u$dD>MsrZ^E5)j`9L{>>#Tc9Y4lnW9SG zLV-B1ylw0>ASLta2fs>Mnj1Uz^Qx6XF8g&P+z)h%vVk;Dk@38Apa?UHc&M+{;pR9g0&V(NTCx393hub&_X0Ff<%yxjF~xFm zB}iWF7xcn zT_uNXB@$35&-Gf((?nhUCYdG3S~{s8MV3V&ckmHm@p0T-42Pi2y3(Ac@Klh542(1) z%FM(uj1imEW#6^jXA5!@YAqU-5<0Dj+kRA;J;r=dS<8kcEH%-v<~7!%-xW#9elz$g_^BNlLwu)4D$Y_9dG|;6t!1m*> zCQpS1P)!28LzOlRxf58M`9F_tU=^b!c}7MhnPZj)_na+XWG0s~Q%criRlF_7ifWsG z!vDYts3Aev!kZL0lm^|FZGaHk7syzgx4lDIOs{KnKI)sSkse(fy)KA9T5%LuxQ&=+ znY8ncX+1m?Ynm}QgoK6vm^nowq(J9;Pv(siDv)U&sxWdJg(d_HJkXBTf&*yma{YRCthX47>5qd@n$CHXdd74Myi&q znkT{EbV18P@f!vt=aDv^MrSh^dq*B@i~&7U;voqp+(aLPq_4-Vz6U_eP_JWz*%ybK zgjgq^UFI+%ddr&YLov9bM7p4|GW^4mQPfB@5@}h9S2=7Ylu&POGJe1(tQT(Z*Jdmg z+@$MT!tS@@D0WwgjCr6qQO5-dM|1W!izOO!OdEw(=CIk+2?OHEo7nAZ*MIQnGtlkM zuGbwF5t0x9j9Q04&}UXdMH?e(kiFy}s6}?IrVA3C-nXUyyE=uw+oV*Q2}|nGYxE>G zyxI{}WC_s4L!fmYiiX|TGKO>;^3fiT@efq9WR0Sguq=L-vLl9B@0~b1o;k2Pb6y*`vx|s#xSheloQA07 zm!_-RV$8>0DasDio`BbrAWZ=VLKU?KZb<&NY=U|5M~Y!{Z4q&;bfmz!B35oR@7RbDHlf%*aU6)Ctn$O)ttU^ z0&46v+^;UKWCE9@J-nXOs^fNd2ANnP9%B!pfWZx{Vyr)Lk*Ic@tb#gLTDU@tv<1=Qa2Phbf ztv)Bqu!WoqeyAtvc#%m~n*+GsSB9*NnArT%X82X83ft7bD`BZ24K6OgAKMCnA63KL??ceEI`V=vd}Due9bAiBh4TXQGP{}E&Bx~cv1{1= zD*;(=d7)l&^gOqbkaq7r%Nl?#K2kQnR0x!BlQQozqjMm+%P;$o5aeXhL|8qxa*AXv zr0+Sw#aQ=!P?)Rqaz;Wd0b|cN6wYyp^A2{2ScntGM_+VpfUe_6)XR$ME809JQvjtJ zw$$GE1q9X9^>-qX9K7DwkxFO2v z?yN!Y*%s)QuQIA>r%Wm7G<|1cU7(Ap_38xbXU7x0l4Geb@vF92nK9Jla$x5~gDV`n@no)i!y zOS;PBvx>#7M~%(87&C|Wq*I>+Hpk4y(xvES75;z#QZCtxB9w;ZhBkZ&LoJF}>5^0R zEp`3XyaLx>)W+XiXP(Pp`8o+rW4mIk;og#oVnR{|8w+J?&kBO;EmukgEJGuRWIn~I z9pU1}%LVOc_shviuljGwy|Z!Z(8Pam!^ZtMFkK*M7AWBzSsaS8SpcJb9d19KOl%-zmwGIy1-4RKiM?&e zBihG>QTmj}zJ;LGSrn1TL-c&Hjk4fBw~u~;!GFSxEa^?LJcemm13H?a$ITY%n7bx( z$vu#kKVfW59H0DnG-%%(&tnG#>LP}<=g7INx~DDUClq)-DY7S^MeaV=QLldze}qzc zLr2lJMBaz1VIpf6AN2S-mxt+)5>Q-z-J4TEbMF#p-UJs6QR!4eGeJ zQRS!qj3e4%qxX{<6SR+HW4uu@ke56Tvl$snSt*Q0Ok11z@&%W_dv?pkj^IhAC^h8j zd-Vp}(Ah`M6!D7>L_-hmBQEVx-h?$QO2g`=p*A7 zbQU3e@#yTROFH}w|b0u2Z{8LfI%pxK(Rctf3KGf}f2 zYtDYZAnOCOrcWG>@F%6h)f7xJ3l#s}2@h>SGdzlR?L&4eR7h9Q4PM{=XsVHBDBD*- z017)@O|BXj6vEkO)T-!OqiXW<(|sS1koF7EqTP!smzm4v)5V9o4+WHM zZ#pzl98@I!{1M8z8l2U9r5l>X)~Q+HlHvR=l)po7(O}$RZIji@TO*nvR(hNRepgls z7y6h$J;ygY&%_D=<2^UU&%?t9PxA(##YbC14U? zo;1s!7(H;1c676*vY%sjyeyo3=|w03&k!VP?*qad=D)?N$hweXBTIqhZp=+-oZDQ8 zKDcvO?tvvgRmPIB3&;?_&WkzI9m5fP=Hzi&OTr)2rz!syCZ@Q(( z&M2YOa(@&F$z*e6YGV}ltXMhakrTPhO^1Cw(;M!r&gk2Y49&v7O&P%y<`EzBoflbyiDOoy1mkGFQv+gJ-H zJGGH%Y2Oh-9`ef@=P$l0Dq_yLx9nvaj&&VH5bs}~N>iNBpzGb8Vibr@*hbx{Vguc% zS*A6_*D-N>cy>HBp{}<-QI2aeL)1STHD6GAw(C?j=BA6zc|K%-!7Bp#{ej0#-3n}3 z$}S=%`mAA*fwc~hVuUGsSvSw=3Ja%5V-A}3Hph7^uD<1Aey*?vX3W)W8R=;i-W?p! z4qCvG^84SL!&tDym(wJy`8ly`+5uN|v=VV+46@+8+uK#=Lsj4^nvwl3-ve#ScbY+{ zKw}q0Weo*B-2;b*?a>WTI9ZNf_F{_T{NW+zj)d;~Yu`{^_9WGUebHvkC3kOmF^E=` zOAmdqG(EKG?O2GPx}OPH=QZyi+?i-?z+tpuIAt((UU0i zTOd94r-@=IhQsjn7!D{_YjkEp0mtg$XDY$PQsT(6f?eGy;&1kMWJbsmt&)w!JU1U# z$P%zat>2qGctsZk1AKD)^mO7VOK*ja=NoO?Xl#4B6#&R_# zR+Gp`kDGCBLj$B3LcIj4)26*wynIUxH`^X=h*r79?=uNb&l2;0am^VRh_p!Atl4lf zGtA4N&ZaB*g9)ABX}p_@Ie&#pZ1Kr$wySI=CGV2irNzolbA*23i8eW$WuPVw<8j_9 zSm?YU(XACzFBnV}q>?kjr*=n!zCrN|`U6N+9qes+S0Y+U!pRsZ=Dcd3oeFCTG`Q9) zu5M1#7Pg3F+y8-lD=Nh2uzpUV=3Hybd;n$CRV*gsR6^)B+dIeh8;LNL?_?a!e#+!C zg5bK98DGWRas@1M?c7Sl3cBto$$@j88Ba^QZj1z?s(fEx!&66}{TwSVt($fHO3!wu zhp3K6j>eC|DKew9!+YyOS+6RBWu!#TX%iiIC@KN(fp@Tj1Z~==FIG`)!A;4C4%F#q z@>Hr~rnC9{nnOk2J2go_TQy!AniKB|c5mJfI zTVfc;v?*tdLOGx zQ~JcV8K!u%-KhLC#r!dK!g8U=?2&0m_2!bauf0?67<7SaX z#l$C{AJ+-bvaYUsoGSiczpkz(=u06m@ot5CTeDJe0td&S>a(baqvmgdW05nYd2YL% zKmCw5E1E$=$P8t};xlWWE<*|Ev{#2iv2aZmL!d0@7cXOu^Cxcg_}CC3>%!i=ydg|n zb)rosQeau^T?Mn1aQwi=AHXl#@!sVv*rf=Pp#t&Qc;dsE2%|P%j&CAb{ebklP_IYK zPMRF~CwQ}nP*FG6K_K${^6M6b%U3a8bylKsncN}at>UuxP)?Lz=9}Mip*n|CP zmB!W`NsGqo=Dd@TQ7#W6f4YCU&hP`G&?_E-eGW3Lg&e!T_&!*FbXC>{NIJbWi1Nr!z+(JJt% zM-4#4hcs){0;o7n{H*p}hm>u1pVn^?I8qj!BMf`?x*HREdb;=UD3oS2(gY6#iC7#K z@$2UY2{4#*Cx{Lz4-KCg1a=w}(>hr2hqzj+l5`#eNmCIsqWo#y1jR@T)5P~s;_Q)R zaF|DtFiZrt=j#TVs%zG$T!N$DXuJ9=&E65LaQzt-MX-zE7fG~D80TfG$hYY0OqRJ; zp8A(?ivCHMPpSL-e*j`E0dmkbFuzQ!nUy!mA)fg~yBLRaP~%jYF9Bi^M2#d}kFk6E zoBDqM2lT>&FNb^&h7Dlb4d!%ot1g>AfX~tsY$}{o-$OhDv0ssMSR-Ikq^T#a;zq}{ zA4NcswIf_NS*Jfk1Qu|ygSXds9mQG_C7~+OB-xIJ!Ur7(zjhZ@L}?1cmu7u-v%R6w zjq6+5f29A~e~BG!=4N8sHeX+3v1YAkZQI~#aKBzv5-RNM_)NJbqr}!Un(bWL9zW&G zE$T$Q!RJOD#5Q<>iKu0;((WTId0)|a(ye>x(5dS=f8i;A#rFGaf16{cHTJ?Uc;x5o zs$37NgPRo`v6wzSm+y_cdjeUq+!b+;)+K)c$wpSFZxLrjFQNpi!;Vo1kzEHZM;AJb zXF73e#z-zP_9}+^=35!W5<$k|6H-Oc`x0oqZU%2N&oRJ6ZgInoq*IJE4{Ym*+i=>k zYx-j{v?@o|RpT&TzIx=ruHku#qckajg!o}?>+L|krl-PaJn9_r6^-0ZZRk#YUy7Hg z3vqw0LME-C3sfX_J572UKMj+gwSrf|K;!H@6!}t(&1c!Ccr*aB7zB)9%i zQ&hzNFs#kD6fs?63y;P@!ABAv3YxdmfJ1i?V`Ww@fKi(R%DME=lF7Cd8c=TUo0{X6X$u>q5GBR6*&@HYg=jZ{x9zHOEpE?^HANB3<+V+F?v1 zpw#Kkk51$-xAVu{``RWZMKmrH5!0(0f~~LwFot+s_{u&9m&_w*i?XC<74Usa;JMui zvjox(B-;|b{3xL0`HtaDv3U$Y`!dtYw|^mGMsB{G=5-W~W?M~3p8aZTNl{dct&q}N zBSzmYArP!1!}9e*5o}^!;YJ=qV6oKA3#@%7M~fD&J6B?1i;k)E_01Vy|A=Gg7V&J_ z!CvW@Yhf-plo0{WzYMw66{D)fnXtt44}f2(NhDI%wvH%-}?SYR8r;2 z^tfOoi9{Cg7J6hC=wwwXTzXr~F(OjnlCGi#oP8yBLZq3E_L2S6VHuw*g-}V`wxSqp z4n7TxoNrEd3RW2uQh|ug?s3+I>eEYrj#5&>Pj*7kY=3KTBkRfMTj%aq6uF^>ER6Aho!`jUkDq{Vv*>Yjj z`89ED;OP1P0=_^&zkPz4LC=^b(VKU=1*d!}>JFyR?LbZ7H@5q(yBQ5FoU*mA z4mev*RHXNfg-ZBW%FwXw~rsZM7 z4p{&@1-UxP`1=B0=^30xKpdhEUG171SaY=A!#LCrnuFT&_;8$aN?@oQcpZ5BCwcg$ z2KO#w_(?oOTjkTgsP9l9G)X`1^r>+PkmJ^(~l*F;LYaV~040sJ7O|Y=E^e6)pmp&NkRJ%~( z(r>(vS}l}=m?UC1oOkx-qr+1V@z$%L1|zh5KVF6&ThTwsyV^ z9ch;&3q&7`WtSzB4;cJmdhrK!LhRe+lGeIO8csy?3ACj-pgI5(jrw;Y{0MVlv4QHb z+=gD>Oo?-gL>VBB+(zfUXr><1Eo*IvkPI~X%9sw-eBcv&i6oCwV`Z&5&%!-s7Fo_< z=hUP@8g6Bc01Q7CBR9KxWkFLwuyKz>V~;EO5{x=a2M=kwYnl%JLffxn)~53BICM2t zY2j~6dwq*4P;p^_l=m*xIt`{Yv_PCY#`#su8cbt*&NGeg$`-ep;$nHm(=XFZO#X^> zXgRms1o^`2(%E!5#s$YIgAknhEvRp35PNrpt{~ru#%&X)RNTpnU?(LI()R|CS76X< z8U&m(`;u^>T^7oe7}ksX4pXAz6J+; zsWx46H#6tUcX&Od=A{g2V{dV$XLzRSF^vY=28Kw)1CpZy3JxH^r0fE`Rr6%!e5!hpo zTF&-O@0rVX%d@ejOPfE4&%>8`0j(P2#B6WQ-nHl-OH_y4d06#E2tsyRXM{8v>mtxh z<Wnf4P|?dV=;U(98N;2?RO`|s z=A5#aG5|S8IOdJ9Yrbcw+Ue14XE0J`ABCgvjz|MSkDxnsBX*-TPf$RXoYu~w{{UyB zH)-=f6;8`SKM)j@Vj175VN(Ojs-~Gt`p7!rH&-s%Xx_ z`F9~dXcw_l_oK<$NBBd!B!$OyK;Zs@t|m^QuGJG%E=a3;`YT;hLb#CVOx#@<%8lQG z<5R1c-x1c2rusQnc7itp>IVgxe2IzGZC1=rLThWZ2$jmH$LrR>T}}=8UP&LKBV(rB zp^h_ozb)B}RvoaBCr8F^1zMGwTIPtGAYf^&b2P{UuU4C)8$bCbx-iCe9twZ{6jNnH|ytz2q` z9U5KRtG4uY7C;%uBRI<47@s5y9Kt#ZXs!c*oDS>tBNuzLM?tRdG0Fm90BO@4CGvW~ zwtLqJyGagk9PJp&AnivI4-g&2b9HAHpChc!CIJy9Pgk!XlT?ROeyx2U?u$-G40NEI zv`Co(eM&=U1o?fxs%>RT$;=NF*gWGGnq$QQ9VtF-r{CD^xHMf636may8?Oh0z^q z=@_nov&Cwe=Z`N?S`PP|bHdYzx`TtgA-de{+meWBAP`&9NyDjJ4NhNx#+M0)NVpsP z;b}JcjPH@~C)$=6<9T4l0k@uhr7sQO5(^`lK2sb0seqin0#fW;Iw#lLlG69chqnyx z?4ABe8e9o%gOIU|xhZLRzYT_c-H&pZ6Y%gJ&lS%AI9$Z@JtA@FMz>_~erL(HoberZ z&d53gMco$D&h0CpE3|t@k?)1c&r!n990WgN(-8-x&z$$qLesG@gPt*S(Y-guZLV^RG{Eff#^sw9IlTA=ceD9%zABab7dNF*EBWVY3^7PyNkpBXFsL<&rJ=e~_|)}Iriiv2r`WXyhPIwkWQ^_C(vtUSan9KK{D+xR{i&D{herODGhLUR zI2^fMA&?YS89a#3H5Qh(3mX}d;d!{#ecup~UGyqFg|29eWOWmtJc#w8TUwhh5RhGmZfMFiSiodSyxRjF$nQb6m1C1na+wcc1ly#3t8N*u#s2`8Ve}2H z(^L`Yp^N3Pr?ePwJdl$sDH;g`D3?M*Ok|;{Qvo{GYJ@bVYJh;^`_0eR($V-ww60^J z&~0q{V!Atg@b28XhNDQZ1AH7jl}AOOF=azMJTr4-UeB^rt2%T5Aj^(()jN=y?nW0{ z2_s8M^I8Ga>3B69(gXCKK4bZEU0P<2&`W7BGB3nF5{}p;`@OO+6+iVkdDgzI8xd@> zTsLl&*03=%JlBq=rlqs^dl_WvIw}u2_@wLu!wxV6wS!2`TQYi7Hd>w~8;CODl3|K2 zd;KC#@jU`p7eANG;`-AUPL8Kl{{Zq-e9t4OBHwmYXUZP!KV6cT`K7B6T;tLdwwiCM z=2WEN47;bSeyOYVsWo7=OLiujWE!3z!vm7yvtVCA4Q})`9FJWxR z+&7e0z7D?zB*3Zv0JU~qU)?hPUI$gx&ePCwDTr&MOwXTJt#%nj&eKKo;J@)&dx`H! z*n@V%`#Z!njMNKk`w%Wn@7$ZQ)u~P8uD>3TPVBKN+3Vry8gGz&EsmS2xH+O3123CZ z=b;U;?9GO1Yg|L!C(;@$Jnp(Ls^;r)z9KQK5LHm?>O#$@D@>P z{71mkP5%HVcj~y?OMi&$0W#ZQy$xp5b$T?+fXyw>F#|K4t`^-<)hZFIhMSz$<~P7%7Qw7~+_K zZ5z`=Z=~sU##Zt})6xmzfoR{}n8Me#3vbFy^(f6iny@OqExO#(h#GQkR*3WKK|Rp< zK1q#&AfEjBDrW$OQ(=xi>kjnxczUlEox*!=LA11L7v=HhoYxmNU{p{{Om)5Xv9P_TzX0RWJ0Id{qZP_NOe zIO-nBj+Ev(#5u4mZ8#owloq(L8Da$bl}KO|(8#o$?GtVIu9(^BRBH$qIrI^}_X~_3 zWL_h6?7LJR(#8PnwfWz6<%?i(%y6*h`UV}AkaLEX!-cFYPC`?ev(ePx$}DuaMTTMq zkTmihYC46<$BHa6Y$CP?Q#ELZHO@XIG9lmqbqL|mN#}cw*{)TaINi&c<(JIk*QVCN z4Tjt_1_?3vtprX6ZWCyRhJ2(FVmTN#9P^H5%XXS{>H*<4qf1%K#ph_eh;p zB4F?|HQu;L@oVcX(Po?42P537OhYkKJrgL2e)*+6yxOr(VG3K#kTI70Lu*)Ty zSlbp|&2fG(U5Fq?`{&kHRH{HBqWX*ha%@j6#S2=bc#m%oY&4k8TPGrVrUE5rfiVpQ zpb`UYB7c+&V|ccEmol{-dRyVR-sEJ){v8+->CBX-?XqJ|e`S*$w20HDeEW5+rDSPV z%c%5wNyxbX2`HVcm}gI^0V}!aW3?~)R!q9s2T=r9T&^3oTi%Iz+ z-F6^fsdt@QOK>@2xm#KHqnp-6>`rOft|akap&{-saXA3oxym80FS$sj*(7pU>uk$& zBWcvtKtPR6l)_vNme@LD=oa&n(v?gKWVC#traG}r53-!dFJr1fs0jwmIUan<=2O|~ zQFsrxiD3qA7eSyI($L-U*d_;F>q+b5)NXNcqZ3M<9gd)=)2CBh*B2WTEh9T%Q=dld za1%OR&>qr!#G&6zR6IsUqaL*Raexc~%}h+m4WgkMz!v#{-3`m9G)o=wVFjizm`RQ) zg)j&QnVeKIKx%QU90a()Cv=#4Rls zb3SW)a%Di*nYf7EKGi11cUr^6Ds4HnF}fR*o7}jC=Zl2J+6-cHg@Uu2^9&(JR?D)@ z9pYOUd88W6Z1BX!JMze`cZlZ>5&HFYCBKYnCdU3*M@*`yRh4FSOr<%J>aG3UN)d(ayB5Y$YCeb1;FWjH|Id9^T zkC_%HeZ14@O*(oGM3E=?w7~Nv7jse^VgV#t=a+5Eo3WkAJrIFOg}ck6i_dmgc_q4u zAiMx1&mnk$YZ)7(dgy*wZs~CHDstBw+cE}YHJ#95S~`wHc7NhQBuG5@d4ey!ZAw&t zbc+%IJDn0^$okhot<*&3!c1E3bjY5WQD&DkW5hGPru?QA-HCT&p~Z7r%{H^HEhM)B zMpuw*lfp@}hUxn}$>?3jP!&UkuP|ff0OiUBrD56G!)al&w~$TpyyUuXM|GWEK(wd= z0n=k_hJ8EEL~hAH_Ev_T5oYN+blW>X9YMRHO`*Rm+AYhAD@Ja*o3jDtOJsqR&%r6fOv`~jM>$qmH#1I_%ny~> z^vRCAxuofl$=p~EmP=X6Mq~vN`8r7v!-(lq)D>lYd_mV96;>@+g*i2*_o=BU(#!nILm{5ge%;P5}uzsm)3{B_e zp4lD97x=%H?;^c4Ez-2Ec3H=c6zZ2phWnkz>|J^rF#(P(gRF-MHiMk=IpGVrq^t~+ z8_Z^pGD!0qF`>GmA`6Mg-25!3CE$`^<){!3u#V!lPlEAaLC4yqAZw4rdY3t8DPUd( zjU~4hIu1=?GI{FVsFj9+Y2bXfQE?W%<~|7RT4B;ic4FNsa@%Q(cJ0)z_E~2jYRqb} zE*#|!DH?68z-C_^8dq=)GU3(AHw6(*t?P81RT@ATA3k@EK30L0}-?^;2MV zXJ=D)j@@Z*b6be+T=_7PYX>-IZrjw@XKOm9qgo`D%dfUM(iy@dJby(B^#((7|tO+BE6@);8q2?MF@*u4Mda&^My>fsg39 zL;&Idaf4+xwxQq5LeA7{g~&10hFh`0BD3{5vBCy9;Ftgs3nP{xLG95_k=1=is9DI8 zG_u!b0@~e7)C@qgdaz4yf-deZA_AKp#>|W7_AK$C(%+jEwr^@RrlvZh7|bUIYts@X zz!|XVRh3IgbeS=2(5DY<8yPdyq8m{HG?LQbVZ;D1DO@~2iFDgI_$3nEZ`GM4UKU5k zKDD42O9o9jab8p^z0Lzkuo&AF22?&F(jXYOM_-y%U19PIPgKH?S%PhmHx{_(JHJ%X zrO-p2dvCe;A%h<8!b`7=3***^rrNE62hRT6sNm-BgT)<|Gla>Qj%dr5Yu*o>AZZBB zPnsk&jU;(=uPZ!E*18?502^+#Eq)uG?7%kP+EiT4E;E>H$4L08)oZC8MA)p5OoulS zCVEpTkkM`9qmsZe8E)LL%t8U6%|nJF@nt%9Bw!kDP}2U0T3$gLq6&}#Db)s8=yXNK zD$j^X%S&~6eW^j=v!rFqB)vcpi>5tFf%?P18M)XTHzho@6O34lEHLTKFh-{n^IADT z!7no6;eTo(3pen~uHHGk9_YmZTvX$Jp{PaUh0_AYY(xoNI1o5RvQTv@VM!6_y z3=socrKb$Hd{rp`_qC*9yn*Ua2T-4yI(qw)!&{($ZX{UZqenmv7M`U6UBhtQK;fLu zFtRn~4=&r5L3=>cnUj85{)w%qUtyn`01>2R2cz{#52j6@exuvHBvcK|fQ4jg&2`#$ zgQ=~fGJAC`mS=D|XY5@1j}42dx#Ppn1t5?W<5S3M?ojcB#B`J`2M5R-mXZVGny^PG zE!7&{YJMg$3!Swd7RL1!R2~-lJJz+fQKeiY=>E`6h@Ra`d{ekCZqHUpM7ueaHo=yd zQLOB(+_lxAk^^U_;)9)e*$VpZlGw1tMNq2Pb1Jypj^wJ9ofa^MWm{ae4C;=&<q4)mLRJe$2OkQ~*$Rj?r*;MGNIqnw$LWp~&B5_L+4*5n>0GA1apXj!Y z?Lz}k{JtC=+UmKkqrl^LZtN5(OI+yUlvHZFq!E&k=c};0P#L7=+furV1OiV=YruPk zsTpOudQ)9agBV`jM30YBkZG?mEG5B z*P2f;VULmuqeiI~0>E_2>Vu}6?=lUSA?_Nti+fc@o&)vHnaB(g0n7s5i+|dH)x#Ub zlh`Tgh`(-X=wxS$q1-9Ws&yC6VD==9fFQ|2ZN<~+fQf!Ea zVbZphS2u?bY22fEM&Z^O7|36x3Pu5RvS9c9 ziX>kywDLSQ%dB}Qw6)a97c(=80eWpJS~Xcnf7&yRp+&B9UCV!Bk_8%i3>coOy(tVz^w~ZzT+JHe4bM4-x9j>-Ihi=#``1ADzLARV#h*=Sz#bnQK(o_ca$wN%` z`&8qudtk4S?cAskTLi_WX=Jt&;C3_iEVgmmxe6v68K!aOw@GeS37%J5<=yH)EN3|s zMM`kM>&Z}bX}!ADckuuRAcQR!mb&d3SmLW44sps!Dop@b*aS1(uZ5T%?zI3|Z-`?# zB>8-ON-zP?eo^L<(&jc~jEA(QRxLE9IIWg9R1$Sv4;0!YJAQXs_JHv((I1C@dW{Bf zHTJQQzc{+J%YoKf3=Z7T8ES$}@KR`o9Zi=6JcodFHIs(^X!^QCZHWWT03MGUOR|j9 zz;Ql76)mPCBpV#~wtAvrsN`uiMQ& zp)}>Wz|Al{5J{(SV90a# zpEa{Cf%YvDLUo!~-k55W1q|m*4B-|k0wOsmSqF7s`v6qw$g#g7oF$(p7XC>(o2ypk z3J~WOx8>41lbcPBXgaTAhN)LuV5n1jfw_@~B!Z357}si?VplovI%C!BT{@KB=FYKT zOmAG1#A9|{x{?gC$QSZWlF_Srn`q_9JHnHZGLnuv*N8ueTp9>HDbN`vbxospG(e+a zAPB(_+CUtH^;u>|5ght_HylHncp|aWy5)3RNOt=Zbcr3kt2>N5$pP@PoVTWmFpDh* zCiVS_T1!u=Bf3iEyx}67NL%4Pw8qoDyZdCA*2u+Vf_hl}un+j$h0HLa!({+r6_b~8?F{eo?W-7<#_269mYMV3zW}&r=sduNuJ#*;l6j;JHoghkEu{$Am!em zH8(EaOKjnuIoU86v;)=lLV#30Ze%64o0?CxK1svb`YUWbM0Y9#IGEjk(Gb?~1sEpr zZrxz1jvE^&4Afg=igwc;2pQZYKZ+nvZlxi66aqkPTj{@AdE$t|_fh}?R4Z11AO%59 zPfE#@h-7nM=tybtw6wqu^R@j+@28_lZXR*^de9p8fM+E-WVyB-eQ6G^)>zHS?R5GT zDd{n5`j%jXS*fohkUnIhJUJbTRYz!Hc)2{c4`RkuIz^BP3?=5ojZq^gvLdZO&{|AH zUQqrc^~kW)XL6>QsOJ}7a(v2p`pu@&GzjWi3t}-y1@$r$(4=Y90(6z^KA24~=9Ed= z14fbwT$Vbl+@CZ^G4p1HlG6gKiJ?9BA9(7Df=V7u6|om&_RgQWKdjVL$6N$$(PDr%Bir!_jid(XS=>%4 z0(DUY6LOnMhLI-a(2s2nVYtTEn3_?J**GSa*Q7?ZkOE+Y38^f}MezclY1D zG%=Z(>%X!+ibh07aRpc#gmVIXz*&|-ZUSs>rO&?)UmL=QG`b|mGYUk-gtM5u_b3QdccUF(gWT8wQEN{#CI^z%%bj9vyymdX z*f>u)R13r{h{q8M1e9K0O|Ii7WoWjCRCi^1Ad<+T<#m0&iwb@ zQiDl3{vMwtK@v+q>Jo@-8bxN2HxeQss21?)76kM5stL9DW;*3k8XY3h*Y+riaNYdI zUd5RRa6YvF$2qW0Cl#%0nA`hR)Y}GuZT|qO!b7cKfLnooJ!>FMIB1Oe6wEk-Y3j7|xKSW$#yry| z8*H~<{{ZrUrOq}BWQoX6_`P%OSR^+ukG57FQFm({PBGk|B^$%XboqjyG@K^(&V7)W z=7z(C`HLy$Ft<4Gx}aIHjJg2!ESbl@G~n&_NqBVHM&qCAfX7T)9m@7r!Z?7(nD#BP z7dQU^4t*l&+Czpt1f+r%$^o`}y#6U;SMu9&zqLr|ZLB`2^{o=nZ#<;5`lqH)6uGS; z!=2B>`xY3RX*1Ow>b7%|I;MM+o&?Q=;!V%{p!uL7hb~4(wMQ}{;BP*uOpY0EXq;$t;2jCYWZ6 zi>d`V&M_G!nvuHJiCSu~IJQ6-Os;$WbIkdx0O+($I){0;N;#wufTYJpiL;fY^R6Vu zuUgB*9%pB*1lt6Ej85gpB66~IH)s)^*R`YR$W!#N7c}3kBskQ*aGl^W!-%E1t)XTa zo$>HVAt$(LgxU^cT5a4HJZmnK@~0O8@~F{d=sCgTOeM+L>@PUwmzo#EvUNJb`KTm~ zNPBOO+O~^@CXfnKz>;U2A)TuRlh9U);u}@|BVd#t5cy05_cgv4BjUa9F6l#I%XPs19Ky^rXop$cC}S*25%yEt^jff_EIdbS-X3 z;cEd3Mb4LbB8@#DSR;cs{)&ook9X?1yB>FDz}rZkT_F(Gavf-)P6lA93Sj5#Sau|W zmKd--{{UiTFwvZzuhgfxrvqg0%}ucaIY94OY=Q+c>zumOv;m#_lUU~g4RAeo9>n&w zt{Gnwhn2}$0FZIE=){`yi5`4vn zdYcG2`+Z6}fD>-4JCxc9bcQ#oW8$C}4%Bp=U&;ZI+o?}TIg&{3DQ?v+zMg^Y%`h3X zlh$m2i$D{R>sm57&qQ~p2RIz0e=K`SLyLsW{YnEQ!6N6DN+H&QV5Tq!M2$mmf%d4l z8HgjU>p+}0P8zxVZlEEfL3eJDmn7TUl8|Y{4zvWE(WY;w6Z)&DI1StFK5JrI!WIDc zS*-wd+q|hDk(dNG`%|BEE5z|14_ecut+<|3ZsQ6qd4e?C*nrF6^K1@YNzUN{&|QuG z#FqGiK=`Sb9Wd$k6#-x$nV?4~_9#?vju;_hu##Im&CM5e*=l`NElpm~4)Dk7}TiXe0ul!zBqgOEYhm$h`5(V$pg z6mG@tU8`@0f#Fm9$K-#)B37Cs{nocpO8J===i&WXk$pax*LX`lj>31z;O<7)Lcjy zl)%S711q8_f^)YeHik6jM{VS+?TZW;Q$YYU2J^*23Jq@Jd3Gp}En){KznMs2 z0OaTO6|W5@bm`?XfNZ=E!`4b05papt$67s~YD?JKWKGHLL1`xK1wlB#5w&2-lZpi;3j7^ z2{@Y!l>sdcv;ax!NPec02=)6FYXjXaaco9v1Nwj+2|cNS3f^CR^)DAk_J|M=%ev>2TmJaHt&e(lXQ2GL{2$ zoiEaRZcP<6Q_nZ zng#l+L|9`E&)Sb}&$Ml`N{uH}agUT)x5StLb&$PQ#rv;MX46w@b85amM#y06IlHRE zt#r%FwZY@|KXmO|U0^9Pz{z{8!PeoAASHb#VfPNrvO}tW9-zg;gdRTGm5a z13(rHu<{m0NM73HrQ*s|`dOp{Wa?*#F69l0n@GfF3i2NC{npvGtyMNy+3I>f@=59! zXZCIUtJuPmb6K?&wFkjA?|1D!?tYr|ix1&3Eq7CekGFc|3XYtY==#(tZ7ekE(`81w z#jX+x`RBa<0CY2{usA#a0Krh)a%Dn$pY<;t@*Q-?w8``T00&SG5Dzsfa>>^n6AATM zTW5KvS%#a%i(E9G+m1*&t|X2V(7cUf1Pb7G;(@8C8?`-f9Oql2?pTuS) za?B-{PDNAsKF8#Zaljb(Pr-*Dr^Dr2nYK>c}i26pup5R z?0u>rNv&xm!#L~qCIT%8c)aTK$E9kZ^ZFw+zzoUL+$sU#FYIC|9l7!pKvg!*({I)MY6N_)h7#JNcTf^vk{0j43NpWBja zThJ0tnYH<>kYi~OCB_s8Y5ZP6IJ;2JQa($MibyfcoVrP}o|dpf2BK8FI@JcPJtTlV zOBx8>DI_qo(z}?^JmpJSEWy6jEi;+*sETcQ%%C!u)C@PIsO#{}fGOl@5Yiwh$Ulc# z12vc4J_)P_I2=I6EEGw8o3*V2a>{2>GhyVQ7|=%PODz+}QaTp9Ov6bSTQZjBX&}1( z)d4zUOb2)i48hjL%YdG@o{A8}%PzfKIA}#-Il=B`{K|?~|rXN1BllE-ZoR zd^*xg+ku|s1D~cM4Dv&1i-F&DE;zZjQebItv56W&|O{|e~+&Y4K3L=8mKZ&cQ}gn$g7)XPP9K%Ka*iHyn~3A@K`9AR2(lo8dB+^DeVp z3jtMi}MJLO^74OT9loEL(AJ_)D-7w9SWR{=dCuSQJ`4P`Latm ztY+&#WixqN>YA)adShvTGbm>Z`ig+Yv}%)aC^pDq$p;l zu|dxbTQ-ne7MC@bBn8X;q1aoR9Fk4Up{<9nW7|=ngE=P**8+oxYe-!tS09N3q0v(c zErKN~sV%s{4kC8lgJs*PmTeT!b9uCjj?%tU??2s*t7R^$vN_e*KM(xiI)I1& z0A>#*^`=$_KQ*q5*E{9b`m9R_=T`WpsMiuC90-gEh0)CBzM1>O?%Fl>TANF0H^!yM ztPUqNwjt4+HcXtMe7D3Km(Ioi9@cO=OHH3@(-W08-C5$e1qHML&(*RFA zkQ2fPvYTt^5ZX=aJ%X8a5ZuW8Dgalnq z1SV5xYj0#lHrt4VE5l38@jz~)_{c5Xa!S%3K^@73p~3=yX%?73Pxy!D0y!wj{3l2l z1VfvFl%wH}v;uV!VmTrwNP}e!4Upl!Qz{r}jL-@!CE2DD>Mgb>5VekWnaHJ7Cs1iK zJJjnT9d#;ZL@i^EKuUOSo@(itjPOUPM3(3}jDm}xOR3*EKt8Lc9ezl_MZlR#hN}ynA%oQk+a-jNeYi>(%1 z&|F9WT$5qazInBN_GaCdiRAi+{{Ulqr1gd4_OEztduf{WoZ;x?!TpBAdyvbq_br3o zn+DM5QKhufeL?s`vj@>O4(Fy}cUR^%m@*6zc@~B4ebxKfwCs^jYSpyU%T>H)^q-u6 zivIx2s8&A6+OMW~O6fCcWmdzidzYBmJ>jzZ#hUhdjehpz=KlbRxrB0sbWG`$p~N5> z^4@5$@Zg6hR)TnHwaj(Hfdp>{1@KL(leSH!tq0uGdz<8K<}SYLaK2;v)3Y{~)lR+Q zvtXUiWDfSnkzQ>2%AQ9v!8zJW*%xW+OVrl7#ySeX8KtJ-(0q!9s)zBWhY~I&R)Ie#K7OKr zfrksS52V{H5;Kw$DjHa7Y2AlcCeFl zL)im~$Ehu~Ny)2bT`#{cQlO1GZ+1cgW2$MeK1j*b&>O;sbrv)d;(d!U1>N6@6!bSw z2_&dfre9_blC{+8vp0^R`YZJUhTn@e!3Ku3XC+{w_lXcX z$wToKAHs{0rWvVR0EUyQ zQ5u|aSHX4=I#!B-pfrol{{YcIJ7=L^X{q6C^j%&09Z|qkp-cuf%xNXn>e6q__J9E< z`6r)rd(dUEx=jB7;l*08#${-d>mYq?UW`yI5S#7TK;X7A7({Dyx>$aPakn$}Ze~W$|B*&$a&m zvh6=~mix=F~R7_Fbdymz-@{#59_m}DfpFtHm~}R z_^G!q`?@@fu^yqX8bCC5xxfAwY5S!IxX=BatlRzn8*%;tY(+C6lZ zQgfOi)Hc8NU8nAsoG+`8t5^L;{{VS*z3k_y4W!iq}W~!f&wf_LJ?LTyv z?{_#pkPdTy?|iN;$@s)hYlp6&{{Wd+{YU&(txUS%`Sf`cVm$zJobElFYLEWcX!TN4 zbKm%OeXRcg`nylvCHt|(&8$nD;xxvmInlvaEkv(A%C(>qR%y2pXHCp zY)7J(P4;cC{g-L`q^7=`i-8UWhfA8~IgSp1V26NmBL&12Dj)fkf7E~Xmt=q5+cF>Z z4v(Um%cWto?SyON@*_|l%FH|j%%v6@PPNBu)M-*Pc26`zn|m1KyygWLI|x|Hbu)q7 zi46n+B?j90T%->Xq=AE!}9r9sd|;vsRqho6{g`9g&Rh_ZJwa1Mee5YlbUp%uC7bz9_#(? zYoeaqG@QJ43m@^b4z|xwUolvQ(?3Ykt4_NcQk&ZLJ0~=P4@ziXpLP#=ZGP@~e$P@T zHXE_sZ8qu6eMjA&y!#&ArIn*+tVjS8{5Kv?fd`TEDnZ#Z+%~qH*CXHCJ9f`@)FKP1 zMN!dbf4WLJVYnkO;6WR~7Yi0rnmN3+s=)X_WV<4IkxXHtOoMTVL~UC??AuLRW_TXp z4=J(f5Kzyk=m42tDXdR8{X=@LrpJpIJBkxnRI?VDQa*$VYw5Fv>>X`NcA4S2tvERJ zeyEKUn||K}&;xjp^Rrm;DaEuRTJ1cX45*OI96GaA)U^u`P!+F{gNHQK6aZw~xf-iL zwGNjSi$W3!F_P%l0#FiVZuL~O7eP{4kCJEvi>Xs+DOHKdWP_U4VnI$d8qkAifSIHi zP#QJYF_2kSgIX@)u{qW&X4|Fo}DP^tcvK5T^NbI?cx2+tJ$S{ucLq zP?`eJ0LfMfR;xAvNsrRryQQcLm_R($ClsRaE&`sAm>L3rJKXLDw%9NK0Mqu5{Oz51 z76GBZ;U2~Bwi@o)_KJ^dSmJhCts&!6Nb&Y;#KD|q$nmvO{{VpZZ~Gk|?yl)g7u4Nc zA2)-BOVob#fA!k)i$}G3Htkg8f6)84{f>wCSB3ZYy{nITKB=X%RDD)4&M%}LaOaCj zwBozcm^D2{=2_1z0NXJOz3!PUtr3U*imS>00G-zV0EgJW&>s}We(-HV4MfWq<7d z0O+fV;j+ygK?>i0?M(S;Y_0zQ4J|WbFAf<^l7G(YVYBQ*s(|>Y^4p?Qq=5GP*{$5BK)BgaXs`4M_b^ie2_Am4Y#W9}$0NR=I(%Dfj;n&ROrE=PHAR?LIXyiSL znL>Y^R{sEp*uT&p6-U17{{X}6U+52t!^@TrQ}3Vqv}1K|{Kdl_NN0Qh0BU?{qq4dt zHBy|+oZMrfFZ}Me{65A0fcUPL%a<|_S?;fvzeoOxtIskZ4nZJ${{U)d%S&Z%_-SdI z5q&NhNrb=iy5I2o7y1L@qswlIPLcy7)V&=x7i}A7{{V9D{S00|{O-T}KF9um_^$7} zcB-30D_3*}KD&vF50RZc>z(Pcmd125eGbQn`b3XvNvl!E;1Nx`uA@k>z znr3IO*p(Jd)acD*94~pgYBF~pYU9WYq3=>lnI{6cz^|o~{{U?rDnD6N6r~`$8YZL^ zLYP#AMvRbXwtT2UN-9b#MMYIrP}%8y+55$|`@R@md`^5!6(JNx}5*-d&$Y(@wY+s@Ny_oAGRU zFDIlN(UiUCwEM$huSaQcsrxym>Te_XmnE>{Fhp{mYmyvd7tYPD+cS;zCj$d1^x83T z=&~6++97dgFq2AhC^rknq{g`KMgS@^h)T+E4trR4qXt%So@qgX_bG3&g-S8vq=Mp4 YcuoWylGx*xX{-aeYc8Be8U%m;*)U|LX8-^I literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000035_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000035_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..721fcde88fd1b1b23e235cc0465292f359c8e162 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUXw8V;0)19BLSvN6MR zG)!=Z33AwgRKfT#8lrRLio^U5qtU};D$Vf=2 zsew#PoWBPFt*;8;prBq={ByyOgrkWXxu2Wl5DS;)`lW;*pfx)qY(@A-S63}PJtOCf>m01_ZSoBCr4WazvvCD| z489YVkEi#M0{XcET~x~-ZJ8Y=7nlKbMaYBk;tUaw(~0W8m=o$UCf>EVQ+|S04`#$p z*H}WY3!S{1m=1X-IcY0Z-FjEukaSCf`YmCt@$@%9+gCF7#NjWY3V(#Yj0?w)AGw@0 zxg&j7#uFP1#04vl-mN_|++OD}do|G(?CmX9fBd07Gwh-8#a3VW4D`S{XYgb31m7)H z#NAu^DMXveiphdrLM9Nqpj>c#|JjG>{p6Xg7}5!J9ngkKaI@j-rBS$UVsF8+fXBzq z9^3{xf9Yzs^P{J~^iL%mP1VRXue_ABx)!zFRm{Ep+hzMG1cCq7q4)p8=7%i=hoOz` z54-^;zAcT(m`L*NT7K#Ro?*r=T{P3_@8W&t3~(7$g{`)J`g{N$d=$+Gi%p=cx^Ji= z4aOuq;`$XZY$}f651pr|;G6!k&|dQOnSo9Q;+e{E_;f1A_3pR-i)-COA}EQCj^!`& z!Td47I3Am&gc5bwb;ufTfKmgYzt-c&DYaQy9`0+{?DTk$KNDu@F{<2$AaMWdEZp;Q z=rZwhZE9ZCulVNEv0_24HgjBUS(M>`7(+tlm{(tp4$m!7E%ZF`4mHr$T-WHyxmhis~uEFYhKROmQlu6J)4^DQcmji(l1XGs4Oz=p=LH4N_ zBiN$u)(|9_w_Oli2p^bTafS`k`(GoPDA2*%Njo&|!Ms8$kN_=JfeijkLv|Z;JIDMr z)s>~J)`xz6rrPM-8bY|?Z0IBNvX#w?zX4A^e zQxItWH7OoZt_6|w=Co=vQ_t7N8O324DBr7He#_^2qnRp1P2s33-Ld$ZxgJlp+PFo_ z-E7aa(&DJ-qW@||W{!;B<^weg%dR+B!vQw|0*t3d#C+ol^>_I<{MDDanT_OMBd~7v z|MTWoFmHD7?8_S}Wv?+1WcRPh8~0jV=jq4y_e2b;z{gpaWKmhIJlb2iul}SjGHE4> zKLT+*-7ffyAmFXQ&{I;z(dn?Kba1$9uReEm8bq??aOiNz-RqAeH-k2Lg?>Z)i`W;^ z1TTX}V}u;CPbOXYDP!t0b1dj+vEh8l^~TOyTcj%2e=3PhC^J%HDS64{d(uJlp+RP+ z-YS#ea|wG!ZG~2432PK?nNCD9f`RbD;A1!YgQ#UgZlu{<+k@Rt?yP5hvgGLsud&~N z%=Cne@h{FQQ&(q{qIe`DF4@1jFsHOfSalnjt5-ULBs9Z{MjkvBe{g*~bhz}ml554J zWOxJQzG~2&9;!PrcxwK~x(@OmcR^VxO1vW~>XQV!`E)e$-Sa7Cr+4pxJpTM|w96TVTj6&^Xs+_&*(>m= zZ}D`)T+i_$BG^KvF((a>k=rP>z1>k6sz#$UlrJ)zY>oag@W1?eR{RZkot(b{OG`cE zx^!uv)3w!_2LY)TB_qSYv0-4sRORAM>kO-X{AF5KfekLxp4^fT9OO3opB=}nw!9<2 zOk&vut3nJE{ll1LKy^|HO79Iax0tdg)-@|y>S(;2Q4tAi7l=R-#TkhoV=_te~j@!J{9P=jOsa{3QC%girq?u?I(VxBE)1 zd+eKg)(|Ar{xKD@7lBya^=-D;X&?UU}ORrtI668{unTMApbX0MrGmJ7L~3o(Mjoqn~+q- zc!rEjwzT)G!0YU(Q8AcVdc4e_i2E_mO$anD2F@pQ$riE5>1$r!bKMf(R==lbu`q2cVTUFC|#G)JmpDA zNpxaGNqx~@RR~z7qYjy)=8_H++#5b-7fx&1F_@ zm#jzPC(0hoZ$Rsq>&8=wQSFbL*Q&AjaQR7p`10LWjk;IL{B%zD{ciyB^e++ZS3FZ{ zc;*!my|U}y?{eXbGrgf0x$6AOHdkwUz-lCm{12f2jA4##`$?Yr{ITUPpcNJU z{=@MVXxrK*AG>oK^hfRaEJcpaKl|R?ZvDaPmFM4p{exFywAQn>ZBKp@zjm;yscU;y z7qr%T)PFVDLG%x^=-j71IMkqJv#5R&HUCg+v~r;HRMYTbda&-53sxT;`M;+KztY3< zp4Cqx-qp`vT_*k}jX&f5ixgk$?~%QG{1p=#$kiWh`M);{ziOWTLwA2c`YURz4F6<( z<(xmmv;KwV3I1Y*XQAH!vscRT)IC=8zWJ5)eP@{6k3Gs5)r`>ZpF0@BVZboEf0Xyl ze>DC5;{WBtCCBlx9VM|?jc(b;7_BYAV-+G{iJA%~U(un{eX`|~=bXx%x6UBvh2~j( z>#`rzI1TYG#j{@fTShVgo-?9SrY?5M6RdCcjlUXbS7ICwnaHMhn5A4~M#Olu!Xl!J zyxW3Rp07xFPETm|h=6C0jn8vytquC(!DT~mc^~1QcfjftLu=7as5dr|+cq2Z`D+=O zlNsTOc$68yjIft*nC}xZ>>C!&^^a;rXoo%pzOTY&|o22jRd(9 ztZdy>W>=P6LeDe=VOblr)?wVQz7wS8VcsIZb5RMkdJ@7OPwXs=-^4zmT|-I_ab&d)ABtSKa($bOeu1J4T1cd5LivzX9#K+p{8^Ww#M4STNZGK7 zVu5Gv2a?W;0d-QMB$=byltG%)3T<6^36tGNP5X_pXtjxNU9mqogTp7h#ovT^J*B_K zPVDcrLwF70u&@~n;X~AG)%{7tKXLshEB}{=GZ>QR1k3YFa>;;&TQ_83$@JUFpG2*9 zcoC7&&R*CGovr+Zn@Df6jY8O!ioAh?#k|sxZ0rv?ENk&NO)q#fqqmJb!|cbe#qBT3 zf*JD^w%hfEKU31~0X}AAvQK`UCmUIoj|Z5u@~t_>-UmSDfJ5v_8i)fV8Jd5JO*Bq$ zr)%0LO9Fq5T49g!28;H6&r!+ZPg>!*z$4BXQ4u)|BM&}gPMb!|a!cfXRNB~wB1 zCAL=|J4HE+QJ|qJeb9XPg|xR_tK0HHHeXwpXYxHDgM5+Lh|*&U_E@4>0JtF+Ah$V? zTJ|uWRbO0!Fb+XFI8o6T%*Fbp+QguB`)NT8`ZRGHxu{o-AvzhYf_$u@`8u+>a#L<>^j&+dr3EZ- z%yCn1sV47}QICnAcbcvRF8e;hV zH=u7yeFfEw+$@d#j0B$jjPNyy5hAJCIk~)>)>_$;_OkJ?Rs}kW*EHhkpuatfHw4iW z5f;|h{CJ_h*H89&D*y-sYwE1sC+%XJU!0xW-)keY6YY4gJBO5SDhm-&(YGEy>{$ccg%$RfeQFVOw={T)74l)hSRWO;}z+C?E{T%7O`T z%*CInX@oN;$v@Y82@v)>;4QgO!*M~4Iv%Erd)*?7?vVSGo% z4V&nA=%|0I;6JQ7H1Wv3pF`%d&*rcruZ0NK3}XZaBn8_Egp(L0TYZ$}%hN=;igp37kG1eczb4FQhH#Y-~9yjWJMdOStJgRI>(G z8+Q6ptb-T8(l)wbD`GjrW$C{(Ghe2J%II5y&|~)0D>e31yy=whcE4!l%2rK{nO_|S z_Xor7;B2jzd4+s-GYhs9@tpp#`qf?6`6GPIA*-cB`N0t5UNSI2eqjPBjofB=f3IGMZwKWC^dMQ5+by!Z6q%~F_D)C-$S9~z!Z26~%8tk6YxjL@2X z0Ww5`NNMp(h#!~qe~7rr>Dj-#@`Z>Cb~K&><?UK-iMX$z{X%N z+y4f<9Zwz_N0I#;A&(3t%04rV@U7nW*nA3W_*z6sxFXi%AVs4_vc(-46b=Y{(>Gt^ zy7R2^gE6j?h%DTOzG^OcOp_5au97N=L!1O<7ybRi_-T7nY56W+5;_rWv4m+B9J$cd zlkNwX0mA)idx0kr8eNVm6)*4S9@ghXK`Ug3;>{hz@*+k+X?&UBbml#i?s@i^LVM)n z7}iY%CX}!%nd4&ozKiRxWfk)>&aC;8VmC=Z}~z7bSdUnJ`-t zkR+@77c4<=E0g=KC{X9O+uraL=}=-zc)%Q%H_ZOG##c!2K@EXtUyZ`RH_XG27dT&E zIdfTU(OC9>BE_3CQyOJgQFmFK@I*bq`0=ac_y)wJIiS{^MuTatJiOa9N4Q=Zo8MXC zu`D4Fb7bs?v&sIZv`q6ifVFK(mN#vErbb!IG}@I}-c= z#biPQXC?P4<)}%Wh7UGkSkjEL+Otg)np>ve!Q{cc;vI^?7CcwNFly}?at!ft`muKL z>tjLG$JU<4{-DevefjV5mD%2#gE3UbTDWui`M{XE@rAX{q&#q*{cc}tW|=0}*|(0~ z(YeZaAfuZb$N?b7c8CL?>B)1QGx@G^i+-81(I{JrfC9RZb2EHCv}+G1M80~;X&V_j z<41EyUfy=;wiJ5ZvaU;&StFu!e_WN*oR6WUmxoaP{<5|TC}MT~{=HuyO+B+drGBKB zc>dkWfQ2p;e`s_LbNpr z_d!o*00Xwia3=f+4sJ@FC$eAxrKEcE8~1ULCOH3Bc6F8&S|Cnsj(+BuRB=W$G4$dp zwe_6~^L=E#qRdux|DE3I2y8Q|TcCtNhxR23N^@}O>!ynTQ|$ztorMEVzda*6wb<4*qcq972pvA8OIgZv>Yh}k|@;Bh~4ZAD(r6w>O7FfVX zjSjUGhe1H<5!{~%0qn?$<4+OwDN5~fC@sk?JuXIXR*-(OIwK;tOVy#Gro84|od`hnIYwMlqWI)EpX_CU=*(mRknJ<4>kNu4F z4=)k^SzeIU{Y8X-FVkSi>i(C7Sgg+9mDr2=XWY54v1ZJKvH2@){(1c$Y41 zE1el*8Y1RD^+7G(G$68>^JGsDaunKHL{D#S+3X$czSvUcdayQXoU>!J!1{43c3dC| ziBvPTcyVG8{}V0lcUP9oTO3Kia$u&zbLB2YZ_V63)tplrwtINtxMWc=wF5o1XVXE^ zlnRX#ElO^hBWVHKubg2Dv*zxkZ>sW00$7!31ID39Ht(>5#%j~)ry&tIQ*OI;U{|}{W>#BI<|0d3$>0fDy(?v?cJmqFUvDxNS#jkclAFP zu!%P_QCF=P!b^?l>#+F_Dn^Wjm2?d%8?ibe!r}-?3%p!uH%>t#D-AIzpGUl)o2rL` z9(e_bQ+{Ex4Ue^nlc!`haya2GyK8b*lz5rMI;wEch+QFCxyv)oz3EpCoGB$wOD2_D z;d$VB!p#S@%K)pS8N`1i=cR8Qa{5dbXic~l@t>$s5W;(E4zz~JO899zC5F+qQk;l6 zt2^6-3-3X3tu32xRYPC=Y_2Lq{#g_O*5f|^(jHWZET$Z)>|q36!f8P^?fQJ|)X#z! z1iNs)_JR^0N;@(KAzzSNH6-=p9B`N~v@Lk2;qMH`SI%#JHRnIH2a0-cvVcbIyugh+ zqdE{|Vr;VHW6%oCwv_$31w)F)nqS@i#U}*>L=t-itviY_zpMl>vLgS3;wB0S9|R`ttdNJwDQx% z&+VDfo3=I-^l0DHO^^2VLl?2kXM__t?ObC*kDS(Vi{NwXDs_!$2#< z+xfu6l$-NQ12e*V}(8Ep>FTUA)Z&c1RORnC*}F_+&GXd^Q?%s%Vn-=aKy~wL5UME%s%snX{Ye)dY!mnKTpB2yX2N_~@ z2uG@qn=3;6$C=T(Tz>;>q07(wRy7mr7^4pnHm-}?+^$&gWy}a4%?TgQa4SgDc3#Ky z-ngg*;ox~C!J>+xKX&NzsiNYK&V>VE?TVDmdTxDB+YyP9Ymrdp3uc$%lYG_AwtOB6apPaw`8KL}E-feZ0+ zGpelDrVKy~HuQWx9T@nt^~n_UhV!a5CLTO9T7_FAIQQqK%vsV{O1Ak=h7AS%-@hf} zSL*}5UB^psUF@7~$pOqwHD&=)H+%`s>>P^#O0CTjo*)ue*M60nCGn#advjGe&yg6!4Mn( zzhysJpDfO4xQ=-4&En|~1P-BnI=7KzzP+R=J`O4|21)astZjoKjKvhxh->ICpAVDr z8RKy)6RSRyp5r+oUsUH}9)@Ni5Lt-8Rj;>VSICvf<+~F3Ip@yaH&V~(sxiv388`K8 z!UhDJB6#hOyUI3eXcb<#+gQEjwz;F6L$h%1aZ*~97By06k!CU%2y7$wf)(bUFs!W7JV|Bl84N_ZCeXepBvKMI}+ zi!R(fs5hKZYdXo^Eu>6mN!ia`07pQ@wvHMJGYuA9*kzQ>w384mae7?&Ew+@O@drVy zmC@>q_q>j8E%IQ?XptbMo<41Ai)=Ka_?u$T;#D5e)0$hDwDQplD~5g&_z7~VFOo4a zS;PIv6KHb?tm=RP!*%q~=_^N+GC4jeoL;fhwS9RmI=fm@I225Ac=w~?X!&V?Cf%X9e?3W_~rReSxxG>i|Lz^Yq4^puLy+Fj+5EoFoy_WTv4M`FAt&K+!0(@ zd%@luG?dT^hS+FIB&zyI{ut=#>%Plq4nfPS@;&Gc-@M+=fzWPbS`Uc($4@{4J|t#a&2chca*3BuSCui`ieA2#Wk@;u{+&Eo7Tk$W%DLK^#GKyYb%>M<_{m5<<*5^+_`k-H{zVk+ds&*}tKO6~G&mF3UZWKgKCF+akjK)XXBJF&oqae3<`?c z_ikLMO0ur*!`bE}*!wsnCH@{w;?Zu2u?H^YTuFO|mcSw%+8qNEjWP-)WgOh4D7gwB ze$RNc3fE0GOCR*Q!~0;ghTFOL)xx1D1r3w$J#t|=c%J=WIr*?{Hkn3mVM12|V$93U z=bh(WyG+?Lojl`iF zM%pwYt`+zx+S6(AdizrOez(-z)(wME>#5t*>Iy*QkdPT+rq@1XL1!z(<<#^OgjQG& z=Aonql+MV^6R%B}QenBU6GhuLgd!w>*}+=Yp8QmE)zsZeio=)a*wOM>FzpEV4G^_j z>{fWshYs-!vgRdH_78#;sL<*ABbi^0LvXsY4EsBZ$h=)oMC@x7XYAH4H7Qip^$-w^ zX&7dptrsOvn!}YDDK51wFW);F=W__OPX(^^%dLrTHk~%YYc^@F6Gu&&x2m-cNFMaD9IvJOm1dWq zzX^>f5F@AYoibZQ&qHJvFuYYFEr|W4fCD=1z(e+eX*!4;e9=8RYMWpS_@K-C<_WI(O62Pd=i3x)O~~; z39h(K2Sjndkg8Ljk)L4XnDfhb`Lkj3wNAf#msh`}W!^4qRv~6KBR3PjFwDl4;0|o> zdi`34toA>U4q&##_8+$)QD#egy|4c!hC*oxN>ci33D)`Vq%OxCX&}%Z+yu!cjyGQf zftnz;6=AB;;`3#{uw!RB8|QBEm?h#+YN@T!C=99mT4q1U>ZP)AhFm=2)KbbImLPj) zf|~vEurV^ttM`t5GMns+cR}SURcn^lQ}iG8z`j~67-Bq&TVmAN^AJWILEaYVn+>Fx zs`ZuD1_E7(O+p;obk&H8gggP%pvS}~blsRZWP#7R9if2pCzATg?dPcC#iqS}5J<%n zbGd3SghCqq%Qt>$!XjwvlbKSoy->A^#mue2cCn@)4cre|0(Xw`kRG z;_BeU;yhJ_K={Bf{n@n1dvJP{gN5PkwqXx5{?`!6@(-@f2@p^ojQlFab*@hu9s(M+%qgCxyAs3t!hRX0z#QMivQdG-b2;i* zFH$9$fOn>eaW<`BoJVxoU8u9uiu7CNYEu62V;Fpj>I;K2^{oMJ4fjw04xM~ibd_aRGD~iu1_WXQ4bbdYLJf-%Q0rsBAFDQqk}0us2uBxZ zRQ12Xtdd!!`+PXLV9L{f3XVgTiMR|2r&z4n79dg8JkyA$6gGQZ`lRGFJLpKk%6feZ zX{TWqERhw(Wawp}S6=g!Q)&59=j<^9sm@+Ep*X{asHZ&Eeyzz>xuu=LmMnrQ^5}%E z+!eGvVCh4?EJpZ@&OATT+}<62>^3M6CQy?`uEWC5$rx-4Q{J&vJ(n1QND-*EyeQD1 z;g@9tuvVh;dte(h5k9@(+PWVT%MF(?^LrE|28o9g%6)Hh=b6h4no**bF!ivhLTP5F z>^DVxbJkt`GM8TSWc(AxtUC-AYrHHd=~tzp#<__JpVZ)GjUiXXgfX6pd|-pIT0VQH z11l7=rcGln?)mEyp%S`2J&#?eJrc9sLA!%z&eXJywEjHjDcX_|2nye$Q~LR+(Ru^m zsG}=eqrq6GQAm&W%4T}gGF0EF3Csdj4-OYd4gP4nMMXY6((Bmthw_Fl2gW#v9~30z zDfPQEf?k)X3WK}ITIYP88KYYlpY7kO!oeo{rt2$D)72trjUtlPV`$aiN8>f8tIFcZ z{RSw?>IZ&c9F~w{QDf6EpHz7^>F`)ln)6)hRH^ITH|L_UA;&L3GyV9SX(Qc9P=`Y! zJf=z_xw2p!W8@l9G@bITXrdJhgLUC^C|iXQY%m=7M#Vkary% zByc(xs3CAVSE!+HalNllLl6QCU}t4ATGUg=H{_B_Nozmtw1T?A^_;wenHkf4!wT=q zwxOlRdCudJhd3*FLo}UmqbM>$iR6{WO8Z;{3xA^QJLM@pmD=oJj8YM`?OS5NU z6dTmSh1m+&gQv&>^Qo1QY=Pf8$^%ZgD(WwT>4T=S?Gxc>#j--~9|P1e^jDeK`NeaSf7#7r$d@lZD4F`q)OLhK;=ekl@8bY0%D2k9VYd1-X) zGz#(65zEI4XK_Z9CHfsV4q(4r75xU2+IT(^m3xgN!sH#l0%}GTAI7Gp-Gb*A_5qB! zi*$w-dogQc*KA^o_q@zN65*}Mkuj5zu1JU~tu%axWTt>KN1^-Inof_?giQQ8fKLNy zuEMy?Dm=h1!}bPakxs>|>&yi^eFcN8N8e#Gt-+nz3HE9^bDib$lP;cXm+xxoJcBqc zy!RgO*}cGeraUh&F%o^Ub8>z3QZE?&7-@^nqk4>#u*L4wK|AQ>N=x$*B7b*5#UPD8 z2T@Hw^b*`B6h&a)^9=cAB8KS)OKUsI_qde6&`Jqvg!m%CcQ&!o zoFA62Nx(uHkfze}WwJRwVpcAH_>)75w27qMPld@eJY)^)!!`v>3w;Lp+sw;H|gOsd% zjb<*R4EMt+(4KPguR`^1B#o@G!FCBT}SD4V2!d?y{E^@Yp;s>8J zs;t9+#eM@~K1Hi=WiiRe&R;koxIx$?>S!%<@uaCfp0RwkGwpyqu@}yM@f5xj3j|(Y zc+objcas)A`53COkX`R_&qLz<%9KxT(fa81@L`kUNlgn+yz+GqeWE}rEV8yOGI|ZQ z*G6R1TQ^Xax9x6*H0($!vTECxC30A}2^LR5qLkN)zbuD67V~<>JDibP5vAQE-b;mH zUH6!VKB>T@9i&_;YkYq_p)4XB7J}r&fEsG;gK@liz<^bs0h%SaBGjT<7A(Floau8OP;C4f!e@GN|_^w01MZ#1%8Y)CX?j3bFyJvvXtXjeQB}MqGi4ML#=zf#80qL zNL*DGwT(1%q)fGZOEX0=$Hhxem3o@a8N$|qfHF2Nf#yCVc_cK5K?D$g4_DRF(x4!g zWp5sgr`$ra0gi?ub+CME*2V3vH+oqfX#VL^4c;GF8ISPvt3gasygx@d-(ymc>CDtO zX~RBGY_}P}B>o8U^+z4QQ`f~{T>~%Kp&)D~+}Xi4H(by5kxbRbnB2Gi!1u=P!^^1< zkE{k@>Ox8q4Zpd6W#F=b!LQ+|cQXx6^vAGB`CQwF{e{3c07a}SdZ8xQFr^6q(m>f9 zs=A>O{OR-MG?QoXI%QJ%!{Qzvb2e>SV<}5Z44$rpx*$ET7|yb`<_0eX8Upj=&vATh z#A3=Vkt&N+E@f&aaP&Mtje2V@QrcFRah*+A#yhLB$NMy#_-ulN$^6@IK1|70cu$c^j?lz)9?l&7iAV_ zBv5hZ6d!VLOeCFNwYA_jMRW6r(C8j zBV1jsD$e0Ch?WIuZYpxqDcXBoRNfD7da5c<2i$6MzH)5>aEedJBNF0$i1bNV995-BJRKM(m7L#FKWC7uvzUQ%4u zqfJXaoL0$`AfHc;jV}Vtu#3ne6-*<~sZ4&dx7sSe_4ZUtacYKZ=Ka@XoU+9LF3c?XUG6N;~!9i0WzlDKbA)*{` zG@>sbtNW8hjKI2(nv>P*LMr?($8-s_S@oxrs*BCmKrx z<_&2EFK(qAjd5}>VUdUW>vpekoT=0Una9z2L^&!nEp?{DGBc4qQXa_6k(9W928@ox z1Yc;5p?iQKKpdBhu~Z0Sw`_Tpe+nU{m>_dhUtBpf#f_JPwb6aVG-O1LAkh#2S55ib zAVHA3V_MXJC5^YKs>&k?Wnc$V=*PqwLu;YTR4ud#5C^!x_+vof(r(@&;Fw0X!7%!3 z8ZsmKUNUU<+6_9V^v?ln>UW;yB!syk+zEMn{$%=@V__-;O{9niQZ$I7vZSO4q-GW4 zp=L=9krpHzY~?Dvi2JNGxE!h8D38@*;41pJF!H^Yozt6DWC@#1&!fmk*v~MC-;22x zetPzp|6-|5?6;ejHXu1;eC(;&ab;8%)XX8rBGz)_(+wW&pI|-T6T;;TPBVyl+kZ$k zGmLNI!Y15%NHt52vcdd};Uu(2gDOec<9~_Zj30JA>f$8ae@H2%;}J?hd5MrET_D6k zl7-rT$uUbtgszNlVrLpQ^GFiK>J-Y3Ol{&5JQ^aI7Af_SWksK$VfAerLq}SjT``2l z1{vptGxAIE#_OR&F?x9_@tES*%VEBFaa%J?OIinco<= zKw@_~LcNVYb0?T%uWySc7PZ9!1q1NTJ{m=QG1{(rc+>kxdP%TDG;YP3IdL$EN`Ma= zk#jS!B1LLs==meY_FiwiAXF7X{uHW}(JjiGNi;Dz{;lxMPC|ST2?Ib?TZQ+oQ#)*Y z5>(arQc^<+LOUsxqA>^CM#T(Y%lT!f%4BB6Ie(Mca_Z|9^H3XELWz_Zt}8k z#<1*pXKX1MF(a0@xT!JRk$nrRb1F)PMju5}Ga%yNO&R>smC{F!_BDQ^ZQ|3^O0u(# zQ~EyF?fYyBXrwAb%%Y&uE;Ea*9_!56dn@cI%2pTj494P|;p*fP)X^h%LahXAgJHvO zzvRB?Kw~oum9ya5pKZtlWVF`$+khwtcL+0kQX?frmP75182AfZP|XtfWLa35;i%?L zRg3`pw(#x__d%ZTY)ecgR}K<+w5ZDC_|$#%W0DxbnIY}+93Bls$xgE;2L6LBx8^+_ z8-DV+`j9xd86J%vJd+kOY45)f287I^$YZSTE2P*@6mnCctHLsN80ESJ4AeeriK;RZ8yWS^>EfTh|RJE!(k2tFUE zge#Fe%$F>&`3KxB#0o@9nyV_{EokL61@haILC`U+g=w7}g!15hF`P*8+JO@JUzC6o zvaXoKZC!Gzk{m;cU5bx=VmH9~FJX#u@x|dJk#=HoS}$~!9TcN-#OgS|;x3Ub=niJv za4nponaE{En#XGW0W-IIn_?g}EI6@AVn#)P)sFa)sHEQnHdls;Ki-Z_Ut0)Pi4~ci zJ706EGj4DBNOhLfPm**^o*Xf4r6(tM|9o@2fWS~RsWZXHd09!z8MSsuwt3jRWfZb* zH6ac$FHWsZjg9qcu&Ek5j8gkumo(4h@MGj{<}sr@vD3J}Et3f%;U%j6oXlkl*^)aH zC7<&C!8y7xB%c5jHl&AE+Yt2A8|9HfO|RRp9D69E1a-?u#N4^pn#%w^Fd0q=uH9KFt$g~@$M8m;5F0Xe zTo~M=iN_A#(fYI2q+KXr5gv}XBwYs=JX53c(PG}ZZcrJce{EouOzh!v#vHygD9PF% zc)eYHG^gK_)?911-DZA4uay_Z&A|iq>@g5>+xg|2`JI29G1iFz%1@M`XMefJER~co zc-&Ikt&fo+i%z_|u{v|h%k$Q8J=iT(KwOa``}7K@I|Jt(>->)ymvZ;Z7ni~uc<+wo zBiSE%5glRmu*(mGB@e|tlA2x5x%642m?uczn_+*d@MKNb9GW+7(G@RFq!F#K7n^rV zEIG0`$|;jalyrO*HdIRXnv&)dYsZTa3&%r{XM-K$?Os1`Z=o+RF&!tT&iJL2TF;8& zk@`WauOAM_DKJkJdFM0E*B`;YJniLl%noA-a$qfpfgmH|ambUn*K=jA3!#dp*-b3T zFC&*9jY1+9<2vHQd2)lAxczER6n%*t`v<-mM~PMiJ@y1g?G7KA2C!=VG`JHRyB&uP zwr>sIK9F2R9oz=dw!~}z#V%?%bK|GirYXud0x0N7v-P*rtpu1p7;Vv>I;^~&Q<7#> z8Lj@NW0S7(b##=OsmWg3zu;n@rw@YBU-xzW$rfk6xG(7OQG{v0H|4AAgQE``3)m2zQ<{2(o6@9iJmC^ z_-_DxOGX#Lc?Dwv5{hPjV+=xTTJiGiOYwDQIrTS&cFj8M(zDqEtG?d=bB62=^7v03 zoveLh`kMJJ$O#(wYTOj`y;`KO13Ep=X9E|TeGZfB$3+I1apoE(I>?wdYW2}Sh~Di6 z=R{006?urV-jE-x=6@isl@4B`PbOZvG?nmIRRtQ#aY2do=2slm_yTY2Ch%AbC}fC* zXCyin;T8M*4aw>DIp(WBNuuD3rmT}+zMlWoT)~Dj4yIR8!#!sW4%Wa!tkl9_?wg)C z(#%roNJSbBq5)no0F%DlXSmdUVaS-|wW(46!C$4oq2yvUKNj^)uc`DHq{JOxIbvkQ z$TNqD>oZ~x$eCxXuqEkZ?P0w z1H?N|krxoU{Ns6aI>;UM{g~SK&d|1T`iXgDmDHwo4#M}wBZeO)G)sxg0Xb3gPl1z= zarIQ5+DgrNH7pNp0nf+fxK}3$WV|N0VVt#bA;=1lpmip_B={|t2Ztuve97b@S;W_` z{MH3BK3(iY&ZyhTosrFv9b!3CCCPK0dPr{oY{g((xMWlD9|*Phc=Al&#BAQ7^N&CQ z3DLx2qLQ<<1>WzMuzK&%VUoIDdW7ZUjoT-^^cC31>?J|Inc~!5xhospu@TbgzTw5Q z(F%ie6>xGXu&@Bm^fDnp#WT;&A+lMBBK`<@yan*md^&x9hD>1X9JaFdW|Ojsad$8w z0k@7i%%1k-HNCh9r41J4BZIVe!%CgoVRr^A8t?|l6_UunOFexBH}3*l4))F>zV1?t z^0oi^W@67ErfMLEO-0x2!5bgiMQ4FW(w?1HpE5-QodUZ{+Zz7E%frzgrCg$8z(!jz z-kz}!VOLPObIMN-oKykPsD{cfOj6qTIKc}FbyCUZRU7?2r)|gRhqt%d9KWb_dV$)i zw7>{A34apyWKhIR3U@3{=&!_`wmMwH5`(Q~WGK=pw=|=jdIG0-vThj5vCCVRfC>b8 z{op+htH(@`wnVQttTw5&FTsU-&l`c({{Ci`V$~c1TL(YO>hWq>vTTFZW2lHp;D^$8HpG1qY1uj;&zM-LWSJ~Kyi>CXi2) z73>5DJco7Vfo_kr>UvU9?vPbYo{-DFE#c?aKf;wN(aUaVs2lx)`ktE{3Z|0Y_XNoO z>}7*HR0WK-%D&DqlO<9rj6Y^&&_MXb;*BOe`EH3x4p21bY(s2*is2Tp7(FB7hs_UZ z2m>_+c-jd(f&Z@nkS=f0YtHkH$mNZ(2Q^~hIEJ%PUtlw#uJBzIl)DMSLHGcv_>~eX zrFXuGScS57z!V&LOD+Hs1{s>qKbWL-Hu0lan4T{rN&*+~ihNP)*jgF4b8&)cAzrxL z`jftt5#&OX%X6{_?$A=bP%!9zZzjgk6zD+9BJG=!q|9~FDi4!Y7geaWau4*=;nF8? zY5{%i2i^C)HwVNh50_Un%a!v+Td@}QYTAO(q?r>#zgmgksNco6mCRbSJL9}gpij~( zT^+Ob)@}=lK{KpW4Jq0}j+IaVHHvA?H!E*9G4eatHt zE79DOuXKtL9JaZF<$q4@&K<7}(WGzd6U)i*{T^~UZX0QUk`hS-3P~U7tk#?6jkRYI zfB-_1vqL2AuWOVSLC@~aTM3ZR?gH$UKK^jyA3e%m&{H&xQnCWT)$leC-A>W}w zyiU<}Tt&r71!uP?UrXDG5%UfN7&>Tm0ok@C+n1lz3HPpBT2h;~)wCrz z6$5@6ah)7%IcpfCTT-Nm#FdoH4ghoZwQiHEt>W(m0TkK@)bif$CieYELm6DUNmoRX zB0z&N08no{s_)N6w_bMTzY-Qq+qe@=XmqF}5Y!Q?>GLiOQ#s4o8-n771qe|D5d z0uM(cYH2Yw+SiV81!YBbGbGPGj5*6^^^Y;bcdhRfwgLx85fYLTU~tkj+9~aMj1qga zr!qhT%ACx&RaC$@R7GQs!^?B7I0r#NN=};Y*R|@8V#voR231W1j7H2dDIOBDlRwv8lLSD31j5inx4 zo&2?SQ9wi)&`H@FB94p^+RW+kNQ(J!U|RTqB6xl6v@Hg>5Cn0loppBvfdo-QZ^C&? z6L%7SLpM@*g-4urXy+?1tVSbuD!0;<;wp&T+Q5@o)%ouc<&>{hWvj%hAtP?zdysCC zusAmsjIm10;W99dN!Dvf0i{{Y-M^SZe@R@C-uPx?UDS6C>J%K6-h)Q`ElZFImb;Og zwsEM*B#vh+*~@LWYtc`+^S73NaI{%CLUc+@3J$>a_vnuJw~GqUUCe-+M23ZO^o%5z zWZTVMk={@h)H_(PyLSSwAf+=>MKyI)IIA@kyG>ebgcL;U4a8nTv;=I~?|mrE*fhu4 z3im#5FuA#Niizwc>NfuXts0D+pWBD0VM~Q{3Q;>;v3}f0KpfuZ2$6wRiP^Q<)udpm zw(xs8Yw9?wz@*d;vf`%U3ZSELRuPX8oEzq*j<*sc^7qrLwbB z-t}D&7+Jv1;m*+>#?~>fx&_l`f!&0-cXV-ibm#S5u+}* z?S-;brCpkZQkzS)${7Yp5nYT+98Nqch=6zQZgeY0MzW;J{D!r0Q+EiSQr0gh;I@?8 zE2U2A4i7BnDOD142~EpM{)Cz48@ihrbB8Qe42)k0+n{B#P=y7nqz6D)YizrnVNQk1PHQPm#Drr=$*s_F!!LRC13>SBAKAL)8RS7CFZKIajO8pJZ_P2`y? zF&iVVuv3d+7K%Z+yVm8R5S$=)MH4!@iy$Wg3Bx+*7itMIZh~~vw~M$H zD%cD+4U$OjQ>*EF(If-j)!RAQLkdmQ{KDLF?0I$-gzu+RQQrc7W7+hFUgBd@9b4mb zmQtlbN>>t|U?J#Gp5I^$HxM8~TU|11K`zybY_T|kMyF1|-$dd;+7u|^s`95pQMJ4g zaFo`xJc#u(MzFQyD^4PyHJ_o?cyQrRrXfA1U2Uj$RsAq{lOE>Pz-gh3C#=3yV2rD_ z(Q#TxkwL}f%+0ieHQOAgbBBJ~QdYcED=0EFokdQqi=in8Zl+m^Un9sTvO{{U$` z;D70Fe)R&F2RQdMLm*6^)?8XcM(x}(m;gmI|&@7j-x!Spm3qDo68wmPiBazjzoc++;YM6yhlXKcGq@9`qA`2ks`9X{M1oMO zfXDQd`^2*`Z=z5}xQf+Hi0F!T@l->g3QfW!VIt_~IERzwu>SxjKa`3ycOqa8&Rku8 zeREB{P7Wd!6CH42s$0r}iBh5zL5<{T8wlILzO@u}!vu(jIe2Jeu|ClBXKskZMG%pYEZ+|uf$M|ya7xj=K( zmb#RwU!-6l*SwYKPkmjeqPW2h{_gnt06xtfRhqtbksY4KUCUgytSeNiF&g)9?~)_E zPyyQGp(g6Er{X$RF>tLX$*Kbk2ZmN_-}rNm~E4k#cd zsu~IXF%vJM1jz)%ohcA@=<9I)VMswe(YZR)p4J7**Or zbP_d@S5AvB`cn{f96{{ZoA{^=kS(fRw5^@;51^)B{X2!k3%k|rSf`P=DQfYag?90R}ZQ&G|R z{SSX}Yvfa3kGefhhi(FP+Hx4BLO&WFk zX|SjfQBe9Os*dY1C}Gr*(= z0}-jz`rAshESz|;5=e-TX?HgQssskvg>BQR`Z|wpgJ6{`;t(6DxfIygi>uQ?tiyKT zOm{>;=XXLkM1gTCp{Q|Ji)DBZQ=$+`RAjg9C05e_ZI&tVHGUhh6Bxibr(TpmUqjV5Z z6soUY*`rZTSZxi@+l}10-f^Q3I7q6FAv3+$CgRDF8be$b0B#Fom(X5!rgyAF=p+Qi z2A!=z7=VmQSC9^PNF2VGy@Q?M%Fb?WkwE;iB1!G@d4*U*mOA|7+V*zD*L%T|I?f?L zXg*bN;N!^y3@ff7%qsxv%hu{|3P#>rWz+^#xd?$o>&Ihk+{01B{-|!7xPXDI`8q1( zCtaOSMHx4{vohKj2@(=dQ1d&T*Hsk*@WM>;U|PNr!OlK7z~1K?y)4_Q@KTQnFJ4HKCo<-@+X`C`B5SC?5C4!9$^{c$bfh{J zQ17RhpIgZQ({86-XP0A9s){Dt0_uuDs7%aEVcA`GYUf*Da7ttK7~Wi4G7)7-*Srli z6Tj5nx@9Skib`K6spQOhlCg_%^5L%e`WtBhN*^qOCvefiS%DwYXZ>7*c?79Q19Aua zA<8%r)Kk!15kHtzCw>ij+G3?}1i^`crxJ$}J&&pFCZhfnN64itPi)dK*s0VpFHPdT%5CV3@NBTt@NFhQG>VkRY z;mGHS-HExZg02#?7}P~);Rz80Z8{m36nJXs-yGjdUy@9oB0a2k9H?V^N@eYf7O;|4 z9qrOcpo!Q$?a-ib1@4+fC{yiqJRLBSW^1G8t24|mER>zNiR?;g%^;T)FJ5kNLzxro zdjn{0L=)4Ku((kK%*{+Ws!Ab7H3&>j>I3JizK>|u-xArBGGGuy5O$FQctTuJm9wcr zck{V)W4eO2>{&?oM+o08>sNLNn!u*8cXm9I>5Pymr>+{?;w`b`gph&+D@m-RHhMH9 z%1{Jthk->RV_5~=G&JQd0oY1P5R%34$y&{34NR3z*0GigXLkcN` z$NJALJqSSIDIp=*TzL`e-$<*Wpr;Lz`B zS!7|vN#V*I#hC;RtlJH~4CeY=#(_m6Ua?>gjqloV{^|`;cBSfc{P<7~4q>f#W zrXx;G>_jzLH-;3CR*s`0yJ>Aoi48!L3fVlsy2zc}U=Le{ZY-b`d%aH-uY^pyg@RbRoO=45dU}!ujwg2< zIGD%go#vizSWb#IDfG%v29<59;f0CV;tRkf18g?dP2E39rx$RRFl>|30Rw}ex{-sZ zmmLXF?-TItrh{Xca=@dc9}=MJ`b=)#^@St0>bQ_odk!7251X+nyIq=&y}nNrg)K)4 zdw|!m&sJLnprUv{<2-oBW38n{sK!gohMHl=DaF}5?vRpVPJ~alo?Y&P(xD63LVBVD zlb_2X#tw`qV#P9ch}qh7cKLaoF(l}MRR#ym%}9q_L`+n2``pWR4Y*z43S*p|7_=p7 zB`JV(6x|yf#Ewrj5Nc_!(W$F+TbTNGxv8`?Z>HRAdBk#gjgioWMyJiY@LfPPgD17j z!XuZhv(TuLacjW5oUCC>3o7EX*zT2JS5$cx;INUj`Foxl1nK6$jtv%*{X})>=cD0I zQ1x{-k#0UQQzx(9-yGXGJTest1lE4{m;mOE#_7@Kx_$mm0-UAm(FcprLjZ<#dtS@U zx=)j~Q(5ND&RuUS!NLRAn%{g-QI1P@cuQl81dMnHBbfz)^b|4O4OVA~N-5CT2 za1Nh)x<(=rz+yG^!=?$|csJu>sOM!e8I1C<*|w!e#Ir=Ftw+=6r6 za$XCb&ZiZ$0F6S{N!o9z+7-u0;90?yiXynT9D+*(yST^wywbk( z=j?uZYnUY9SV2i94IYFGNy4tk(e5L4Fgv7j+~HzTRui-Ca2CL`IBN2vW16A@0ZzMl zLyW*^lG`MYy~w8^%pkoc1Z%dGrIin+Tv$=~=>yW$^d{Ke+BSpeq<4;GDiBT{kBHNE z(%_{BQh)D-exV9+m8_0`<%avqMxOjt~YvCMUjh;cn zp>ePzX};uCF7=znLjtFwh3-afU1{PB+X0A zWFG1N0D(TUCDD0+g9i~li1xY(0!;uue&;x>Bz z=e;eUDlr}u_BfYJL!cemcL*el>Qan%d383Id04`69;t2GB1?o-If8YA>JLgL;;#IV zp>bpY3Dw~kVZps}RGrP`-%T|x(8BcdBut2LS9&{;=x#2UMN6+mj*}ZndPwOEyRcgE zAOxDtL*I|{2k6*#x>!PvExdPi!+k4HN)VAqdeHPNw5V?t(fNxI4F=Dpw7XHM*1U^R zx%r72LMznc3wG|@m8t}3qn)O(B>sdvIJDSYw3Pv-8dIx#c?Q}qgrPLU>)TfTYS}>P zF;0R8?zv|r%V=&etLMOJHu2bmjrwhJtft{9wz43CkeX-*xHzcHTtZ00hI9lS8l{ef zHvy7W2DLiAQj_#KPI_v;{zy59Mbz1d3QW?P$o4f%%6}mV5Gs!%*CO&&n!->437I`{?sC;^BEc1+ zONnJ-(n@MO9&Y)5j#p;hQWtF~8i=0HrJuLMZyD6buX~kcBNcn!OQNHsR_&D0!VH82 ziXAy}(dZI&K0}vKdmriNKK}q?F>G`J0CF!R$11wcB^?Isp0HiI!3R=n>^{e!XzO@o zQ(5-;ya4i!ZFTc@BA&{6oUjo}&pWUb_CujG6d2>5P83coiafSs3>PHv6 zhWA}aq3t_5j?UL*8tjgwunum09u5)D<)b*%Cvf*tk9KYMcFO}IJ>0cm=H=G(3D#%J z)a%~RbNv#)cl6=_z~A({p)#ScMvrYq1;mcR51V4CIPnq=r10GiZ3MI<3iQ@}W3krV zyQ{x~-L)*qj3AgB>9MQ$MpEC3<^{DWX$e8NNCOc8NlXdTS(@pi^$S@%zE06in`uQW zy#D~HTAtGV8F@}z`FE7#vVHlaIL<&c ze(1qRk*Xin*|LLH^Vl%3Jz#W{C+EB`DdI_s$c1dUj_t(u#YY1n`M-#N9?j&ItEeBi zw?QZpw1M*=d66S+AFl0}q4AznXXMPEJS9Y`H@l%B3o6u*{fopr10|kOxD(J`HR%vF zKh~WU*RL1mUMJx>-w#nMMmu^7sH$~}{HLxfwfLu%DEgL!o2rmXT@Xr!{tOFJXl3bE z;mr9Ea=T-fYuG8}^gN^k%bu*M_(sH_%u(HsgQ#DcL)aqAhI!CM6;o>p9#wxE_dcs;2I&?<< zr29#1^wXkYu#|!(u7txo2wI;^YHXL!ZClrS7zD=I3jcIf2awvl%Yu?eK|mnCYI+C*xq^=)&JwG23f z%mp<%y7!e=)O1#&LJ)ULOGZxMAS+FzN8jY;HU3Oui@h-RkP@|8kZM&GmpvA5C{R%G z1L<$9S+)!TG0M8NgjxX!1nl-XDd>h|w_2`0z>RP6!jN{DJk@c9pu5VzL;=vX3&!Xz43ad~dy-xHjrjy6{?6s+Wg$U#6n!24d<#D&7>a8FfQpte?}t4WU_ z_>Nd{HtwvFpKaS2qJD2{fWaID_AfP0dVo&p5%sg|w^ zMW81{$CJx+b8ozCTm&H}^>e`7<1Hz-c6MRbr@cPsh)F86DHi zT7uyvWlJCpXHRREugs*HIEhI@rg!i6GAUw}1H4Yi9cFKE`}Roy-dSRWLXLK|H%16Vy8O_7>@s*GU@b>SdGP z8+ET|SGCY-8x&7j+s+$8rZ!DEQO&%gNE0)`#=-}9I0~x}6D3R)tn4OOs>gDcW5KUwxZ<_lAHz6Zz&fY|@)WOq$gXRHk0zowzupEvhUB63iaTzmBE9-B9 zW&zipPw6HS~0H$fiJ>l6nrl9GzKK}qFC0N_YySyy1(nT{T|_?^FQO$`^1UX+WT~Vkf-!=KF`}U zi0;mcQd6mMRebED?siC?6cedz+|1Sy&RS_vQLEN25>jt2buMx*g;3Mf(vk5urHwQ$ zB!_a*QGIsWk}smi+;S8VW9_eKAPveoW}4 zsA3oP7N#81ajQ!W;o|NCRnmabpZiVJEyfXy-CUDaP2hI(0bZNu5KAyNYopwX2FIV+SPs zg)Cwy+BubI02>=Z<{}32<={d^0GG7`QH^s&0%<`~*j18*Vld?-!JW)nxVopwMC-}6n7-J8El$u+zKlsf^~p2Y8j@L3Bv%GD&!ltY+B1(Rm2|heiBUAX9oi( zeF;cGFrrVZds13cc?CNUu**wFT!|-!RJ2#GJy9ejONyEb%WCb^Wlkm3Dw=H$O{xxe z4?K{tfT1Ez{n4@WTD?w7&$ydz;JSpYP$&XBLyDjU+ltztfa`9E0B9&Wof0K+y!xTd zrgoGz$umakPBjE?a*=Fd9YMec3dLfDxs;l1gv#c-Jx(&QT`(l_=6I{4e77>DS8)FT zq>-JU90~iu$HcU&@g-61ZJKc0Tg-2hgho(h92Y{ncDky$F%KMx3j_*{u z07`q*JEc6R-dbuuBI)i1AO|O&_KtZFvMB~#cvwQ?{U2qag7VN1udPV6JsEC5qg0mJoqi1Ty;zX*!fEBCM? zhhdoYJb<5ep80eZBncJe*xZp4A>Q4zTpK|LR?l%K;vBnJ4kUKY#@f58f{F8v$1MOE zjhZ$Tr3#IAxnv%}Jh(VO;ZTv!_q8+}XirdSD@fLM?jZY|iw9reH-a-rCu&wXr`-E{ znWZX;?vFR-ehnu#XNEue6#MNT(D45NYwc&FEp?Klmr@!&cgAuV07cufvHt*V9p&{z z!2bX-4~EWvE6m@N<&Z2V=2R-@pLBIW;9scQX&#Y(F=H%@cdjDZcdUUuTS@ACj(7Q8 zgi3O@&Exa0x{k!2`Tqcg`K{~sa|V-8W z{P&m4-^$zs^uG%_WK^;e7>@|Gn98qZFD71Lix)|YCQbxjOar?dOCjyj%0ZA}+)N4jinVR(o ze=Oz^J8<^Lxz;vg6W8siby3819$~)$LZu^(3FZ}GHg^&yWC7;{dY<8d7(wC;uoR@! z&tyDS6Hy0-)&*r~{U$o2+TFqhN~454Nl7|`!AIWSGLtAGHwZpQ+`x=vY(2E{apgmY zTbF`oyuM~vYNT-co`8_Q4+|uP9$d|n;g{pOQhb{COSleZ!jx62k5jm$ZAx5@eOX(` z_>;^*^-F0;HtKP9htT4-pVK{Vs=TAP4yV%_*mPO9z~-v0<+v>!v~NMir4T?Q#E6$F z#j$Xim>Zv4JxPCp89=rq8jT*IH$Ei=b(!*OVHVL?9lE&Lg@X=abZ)q^GDafqq{KsJ zcQdC55mI?=VsIu~1u2HouCCZGD!pUT12>3cZ4*i;7ars%h(TFJdTxtcIVHB5oe0rSZNp zSY`A)jbN6X1TkRck4j+SKM0!7813wHZDy5zK}Z9GdoY25IJPY&v>FDq`I>W1?P^a> z5wtlME%deu1XpR(UJWs1*cg;JK!RG;vo_Lr4^!suI-KpMj&%5X>?@@!NdV9eF5=>a z1saBNUD)E-$%KU{t6=MTw$-rY-!bbZ;Xo!^cSa&3&B|KR38QKzyWAmAstDf>$W$RK zq?FG&4g!{-Q~`>fQspPS5S{>k#XhKKA0p7JX&3Vg=(6^W^^0n_KwXPrC`yo-D!+Ex zekG&`B-OOP2=P*>{h9b<{{WiEAO5Y&@WcNAh|U2Y{{YUu4xfvb_dWn#zEYFZUZ@}U zZV&xW(HO@Hy<0Su?ERg9@Q)QI{{WIP{{Z(n6aN6#HWZt=OZP1=Eto~X5TX!~m;fqb zRh3obgF`|=e8O(t)Ze}R?a%q2P5h}v9yfd;N zO~n#LjM@JH{9Mnr)7)1!*v;wvlQZ>Co9Ql3__uieV*dc%bQE)?yjQYMZYp*pw4z>b01Xn8-xDFZlC_HXX>9mo0m#c`ImGP`HTMmnVlum z{{VLH_ly4kndPW|>$tRZ_=pfJ=LGBj0EEo_Q=)Nx7HDw&re=BiCdZ2vrsa^8DIlio z;ZaP&{{W=*v^;lA{%zs^0Q(nD_??zsTr_yURp?A8=}pCLvz-3`;d38U-M4V2ofd8m z{H|pG0Qj6Q3{gggB%05h0!xqSWJg919`-XS6^X4Rw*+Gc&WoKm$bOBu1i z<#Rv!Z8+X8wW8&al_?@F$V!x*7CuwOp$m)rTfg2f{%2*EmlsbTcc8$Iq+C)eEaxX^ z{jO#|@}3))H5PMIyY`u%s&V`+MM=C$l%gUrDN2sz&zP2gi??usZr%ieUXfz4{-$|( zRG-aG_Hg|S2MMF z-Y@=UtcCP>oaF=@$2V~g5`-|ev0B?-z?5kdElsu(Twjr#_yd6FD_Ob!Yj z#O#wTmf|+LQP+=NFhZtB#WS89sk3l^ek*G4Azf+T#6jZyFT=l*Wizmg-}rl`kfZr( zKhmmm7~NOo3}dQveH8U7DA4<#jCgO4=KMYrJw2gLjbzY&LaI~dI6qAA{{R=s_=t;3 z!*gw=z+Y#~KhxUtM!w^h@tn6uo>GDYUHrkiPsQh2Z+!aIg!h7ZjQJPn9yw&(%HvH< zwNLp?6g@|=@C6h4SJo;#pUusUO`d5-42r6-QU_-s#1f+honjscN~64b>CYkDluY0p z=id1kHFnTHUo~CQ02(1)N7oIaN6X8Z*(*sEfyyv4DK$U*dK=zr4h>o9$w1ZYY^r zi5dx(qn043>!&|2Ye~#(YB?1gDh(>4HkmInjA;PxYUu$f18`{93B(-fFlza8If+XO zQq|U7^6Dwa*Y9tZ#9^R&Y-lN_(p6G@jy1!9L;@pcV|Xax71+{Osq1n-ainY+Q&m71 z?T__;>1nFhmiI9#Jt+SGH~mgB6Z{=tot>RO+w&TK?`7WvkA*+!547;i!m|0B%1ThF z%G^l{7?MH1mL^7rsL`v8-MIty9vS}t!P)-+mHK(b8&anK07skcm(>3NE3b~G2iaHuCFT)FnzQw~C=l6s;#~+#}Dh-P>QWIPyBx zuXsd}e@c|?{)?}a{{VZv{huZM!1uQYk7kiLZDmtB;kAZnMR2H2rS2JY)3ZaV{{ReE zVD&rdd7NMO%l`n`@?YE!d#C%|@9fzx?gzc+a`n{^>bzWjb(xJXf93{}5F7>g?q=o{ zF3weCOcmNHi85mnj7i+<{`r6VK1=(7?|YtI%b5J6m$ z{%Ic#tg3(0YYfne!iJMgjj#8+-`VnC+z)$?Ex4t8tP`cJc+dE@c6V=o=002f^7r;^ zm-hqS-99~;xftx_7ZVt*Vu><^ss@zp;5fWs^B^Yi00KQ-T&y)V(d*4ar+_{kJ zd@ta?{$KhozIh)4dV^$m{%I42)>S+3+9{e5`cyZT-}~L~?D;S52ffFZ+!DT436G#k zF<#CqXYQ0bzby}v{{VZv{huZM!1unJ!mqD2m%n*&iFuMt2p{32e4jVYajdy(u#)6G zbx=SGnpaU*FiAk4m)PFGlwY9GA5$0lyI+y)IFQt$fbl|L@~frqhy2Z}dmUUrr+kmu z+}JV&aLBDK*3di`M5zM~W zJWFnRRN_x~Y$-M7HSBzYSIMbe5w4RD#o{BSoaRmvt+Ng3Qgs1ct~yjp)fv_J&SSV# zqaaA-*XnJo1n7Ckwc9V=Fn!*qONuGDr0dtKR-k^}1k?@(?`s?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@be#6H`s6bUeDvlfi5H@mnBFm06 z@saFzV&fX74o1Vm4917iBUK#T-!MLmMh~BnB#!L&Q8pyzk;4Hbg3hO2`j`T&L3>zp z^Gl0Uu~x%Oz-m~k(UTf|3Jgawi|5^OA2_S&VLF51^r~m-U;}7uX0f4V$ z=4m0o#mURb&BX=yvjTVxz{0_Kiu3f@-wOdg!GA44AdrNF+kv$%G z{G*_tprxgzW@H4hv-AGx2edqPfPsSY*u}pVGzq76CBrcrNs9|v7+3LZNetk*8o8%f~{-+y2h=t;h8i0nv06--Kpb-N8ECX;KuSX?( z+?N1cfefH{tmg!5o-&7RMuu^&`xVu!zt*0YSh+UV&L@R{Y`NLck?>BDR;2Qamy5S& z*~RO-Ab3$xYHL&67>6gV%Y@<0+%Dj`IJ)n8+~9$ItwJK%A508W`Fy(k3;b9|sxhsj@W9Idai-gUpKnziX~%gBx>3@dXgQwp&uaiLM==P%=TPb!gum6iXn5gu{w{l%iyZ)7%4 zmb7uEZujt95#~dzzoIhnA*baSIl^(3F6QAQ8{2d$)+7A4K^wwu3py#fE2c#MQ-em7 z|9hu5j6Qpw7>({Hdoz=L_k*YdQnE{IF?h?o!=A7|dY!sU*1p>#U4qvCYs1+YlvB-e z#il?J=*QU>g(8$VVceTQEGvr0I5h1@8r~i=8T)wuWy2=`1`n!$V5)k-4{`BC|I@Ww z@K?j!;&(b{Ip8rj*6hYp$zGC()CgF?6azE0#L;S;9!`soS9OU`u=&&{jf4K}DcFY) zF5#;G9EHi%VatlIGae@1Mx*I3_3JT-=h*0(AZm%V8X_%vRnXzZQO;{)NG(!&o5XV@ z4PFsoX)rMUu|7$cy6o?G8|-uX9({|xHs}L;r8q6NSSc#!e2mgq?an(=bvG-F2xfk| zH)08d_wu3F0NAVu>U*~GmuSI#|lgJ zU{!#^l0?(yVeFXLR3VP(_~}6L8`~0WxfdNJ#3e#7Zly^s0-yg#9ur8eQTA%G`EDEO z3En#h3WHWCSUf4-N8ZPd^qXQ!y4HgqXF@`8ppLXtj%Xs4r?aW_Kipq+xJg&Gw_UiS z-Ho;#qe`@sp)(j`8}4wL8c>r9_!t13d#T%;YLlW|Z$8YtDeSG>)}5~q3hEz_wunZH z{s)Ht?iV9t^>>~QNwus$*N}2Q+R_`dn^WK^6jADQr*q5SnJg=RpH2SkX-gfw3Wb76 zXNxk+JS~d+ZM%2w{?iE$KSm3Fzufk6*_=Gxt^RD?s7n-fwO$O$4>!hA&;6U7Ry*tm z+60NlIhRwoQ&B@%QIiQP>9hhz{z0kEzw@K>^VgHzH#RLfR%Er>&P;4Ec7w&Q(F&vf z0GO;etg6-+zLA(N2?CLgnk5+J80NC=hD%YU5K~vfzrAq&J5OJJGNE6D&L=g5tae$? z>%p+-Ax~lPULDv4)0L}VV*)#@NY6F*gi}}cD-B`E#y|QFGD*<6gdYAgK?PsMG_)Pb z85;El!OK^{Oi&1e9048Q-v32b|No#Uwe3ZHZk^9CIizB}`|93c-s3iL7}eR*+mgMF zfYz9C&%EYT-OAaY-u;El8}I59{!%h}rz4hqt?D|L2b^{u{hDQPPTy39Oq&-nx+{Dt znB9`y68n};8pfw!{q;<}Yq;rdMa|L-rgc#6#J#BB^}oRq8`4&P>#IW(Yi8nx?+eTG zLb!VG?c6f$zpvlp)h4l@OWC~)N@XX66RX5^C%bRsV%+zxpjw7yth{k7-%*WP8x zgMcr%QjCns^Rn&A+;4i;3%nuUi;=oJ+UWSyp^_s_jse$TW}uuJvMWUGCDdhuKO$;vOmoO=UW*zfGTWRt@#=fsJbOU4w( zn-5`E`HB`JCUwpi>u!f{d=zgT5BVRmZ||s2Ujl{YHgtYC3tulPJZ&hY z%Xo7yIlcI;%1_`r zduZmp!&kTZUy9XIqXPH&eR_X6X?W}SK>3&7Z2NW2koI?id;bZ;|Cf*d4=}tdW9to0 zr*L*0rxcQaCw?d?vylW|1ozMX0seBS>4(EV-Tk}&4aVG(?VpiPcVBETxIo*Ow<{4g zIDUf*1@9>3wg9K@Eh7}jKq(y%+qrmQD1QrKRODKnN?60cK^|jf^RJjaj?BuVJMAVe z+|wt0R${0W7{MV2t&vGmwl~aP zMdpsJXd#s{N5w`-vaBX4d@E zGx5nx7&9Mg#((dhzdGDkz-H^sm&cVc>nm8@c&Aw6h)rhU+6rAwS3sRO8AL?;DecW~Z&iAhrjC0>}X6tp`H5q|MO=l<*dyc8T5jscw;?KP|Y)W-o68rU- zR!Uuud!mOP5821lDxUd~SU!elHq2UM)p%T}Nw8EVL+PO8OG^{54JX@Cs&}#n_3|z_ zC|(nnEAjr*Uj|71J~J6BT-RKWs@Ym$N-H*;~o;!Jd7o9X@EpnLj z>^qDbB#9bXnoZY$Y;Pt@HA*SEu3F%eK1*FuScc)a06#>^(R}z36Izp+j!0qayhl@b*F6!#{}tdBq{ zBEq%ne?eTCc7Cq>(?8BLJ^Bn}xcg#vzz9|ibY=pZcP zBnx?A)`;^dlLUK!Z{7;=50?I|PJa{kkz4R}@qY*l*=lg!F2D8rvuJwH#54ucUETBu zdG7=a8LZU-x74cz(onV=MC?(1$p2dA{{sYH)^Vxn-fbti;s^vBT8j6GS_n`q+!ZfH=t* zw7gg`&B}umFTcDK(+YG1A^gF2b?kfsEDb&Svvq9#NwaCIghrefkG`LoIyT+}#&$Rc z#m?Ny2gVv}p9VjEZBRE~|^&h*D`a2iimP%Asa79acl^w+_`M>`(ZgZ~WG zKO$=?=+0UQqoHy^N*YS{_J?VQMMpK-&Ye~m0^-TGawx?J&8%h-jJB69ZdmHTB=0wN z&MrYo%MeH@2i5J+GOj#dY(QGd<&D0&@V;eR(_Ig_pm^nb;rs+s1V`YA^ zRN&@c31BK%P1%41W0Qw^cDm)kO3XC|8{xfvkwkTpo4=PV=V>j<^SvMy@wd{BiFUo_ zR`sCr?spvOS*GiEM{~((GMb7t$|-LY4*ZQjo{LzX1O2Iby*&9}`%b$vb_=CD?G#MS zfhBH=iPM3Mj*UMlUR>_f>2mMXIiXe?Hy-fN{+)DA`hR63rXXphX8*}bHjbXaY$bXl z94r!_+^sN~F!r*--o?dbZ%bBF&T)Lm>U|Q|bvV61B`b_%=UaF!ryw4zoe z<-5?(%eOQrmw1&5os3|Trk7_678Y@8@FOf=4W&%zV+7zQDHe{6hq&Q!K=#m zrV&%%G4U^t-{3kPI|V-uE1Kgme?7f*>8tPUO_FHYPQ9cogjdH_!kSxWQ_&q)%$8v(Jk!q&f7MZ>JGv@i{pxk|$m#>BhU?`uOmcJQ z1Id;T6263U*o!u zLzRR}W{5tH3gQ*xYVXPAjhvEW<9dXJ(|C&&+sXpN@@XKBB&VPZLld6tgzY`gD{>>1 z1yi=g*s3iy5mbMgJ7gZhp^8p~=TOU~mb#-rjPcd$Ky~@}T*ZE8d~Cl6Y(BcXGjmVJ zYok5rBkSR8D~nF#xXB$m(>Ht~|!b8O`Kl$6Y zPFQjd{Q|Eq)@r!U`vY)0CSzU!x(T~g3`}-lyu-H7r2ZXqPmE+A3RN?g!sMe1kdOGl zw~0oH9Sd@%LU$BZx%k`8E@&1PD{}ak^yEgz7NUR{w?BnJU1I-2vn{81892~ zq|J(Iv#Hq-1f-)*FTpX*c!ce>xqWIJ|2ZU7Ci&+ zH0o#`X}w1GA6YBt4*wX`QKekbtlJN9b6k?fjlBFFpVC0AT;9lhMJ8tA*0q*CH9u$E z`D%aF{d2^;qnqhCWUVEq{8?fD9zCnbeVHT>~cWTW`SDE&947K_!g>)~xfeL<+ zmiS}{Z;_|0wiz}z`UUR%nFSF6%_o*QRgJp$Vxj%c298k{&NxW|?fYut%f5X&F4-?` z!tQKuDsY!-pkbE?2EL52t>*%gU^oFmUnYT1Zf8`_8>G<`1Ni-bc~a; zi!rqfG(`>T4Wzw*(~&GN)o31PFhd~y;h<{7Sg1`b*B)nXh5zXn-A;dm7a0%9GdLSu zHE-{(&EkXw@%Yf{2y*G+tMA;;ixUl=?X--1pb7>AzT!#AX7m zCMnUFf#mi48$8^}rehk0h_SQ+wOK%>3xl45-Z@lu6B$DpjEBwZc zNeICam-U->A0pJ4Y3oE&*!&Jho62`3O}et}l>@Pbv%%xWv2p(O~duWkqVt$!cnhZ*kC#l8lDUg6fmeaJRH?VB%&?UxMaSl#8^>LX0@L z#NEqeAJ7lp0@>X%l}f{YYoM)_m`M13laCKh42sq305k{#gjp(rvD8rGCI*xksD=z9 zi2$aI%aK>DRrhiH0eCvq9Q^dS66N4f{V9fXi8EzpDD&J6#ZFjUttLmk#{)gHH$o^t zWk#Gzh+md9kWN)H@|4cAtGG-^g`$|<$PbUFt+e$z>#~$x^w(raPk$0TxEX_5^N>I$ z_T14XsQ^T@)~}G!uUY`snBrA_E}>QIk=@kT>|d%>q2V|Sl*tn9g4vyLJqE>{d=MES zUs_*kjjido^gQ#O1NVrq2iu3OUg<_(FDy(um_y_9bl-*Rdlh@(Mz-I%d^>rXUOK`W z^!6^+wcEJzLP>}z1gNH=AF*;R+RfDY z&{4u$PqFn=UC=0FzK2DMipcVpj-&mWxa=JDfe4ku&(St$nD7L!)QS{{BiUlnu~@MttU?9{p}?$Idt(CO8i1Uk)UM}<$k!ESNRw$lNBa<3bc8xtP+yKpj2k3aiz;i9@QsDz zWpetsQL5FfIlsX0PsqVd@-};QWua|z2m&3<226Tv8T1EW)aJ~!PH&&m+<(`a@YZUW zMJL|i{Z8WFCF1cmEuMbq@4AgWe}yFtq&uAyeRuE%@|MbfJx8yBIh1b?_>NUI{_E-Y zXbWdG4scVRrkp4E_wMt7*&{db;!_ph)$*8=-0zjcryjzoXLwYOqw%I=!z|ox1>W4* z3;4@oF^(Np@#ajmtJ}ZLk2TsGSL)MI-aHcE5cBa4iu_;e&G8`MrLkHya&g zGd1F8ihK2@-pQ2R)#3wl=M{qHSQW}yw2OYR^5(nU;lLr(*}hY(KW1Pny~g$EBSBI* z<;8n$7w`EJ-~Nmponz`x(mtX*s^y{zXbpED35h$sSNASk%6DA{eiB0^NzNTF8B1hm zY(S~~8Q&363#@xkCBt6aNhZF_AU}QSIx>MT^x3{tcY>3oKWd$S>02o9zbcSCQXO3^cIElL(HR)tsK>j9&*Me)L z_4SB7c%xak&n178aGCH{+Q~6}v!~;4-fpe0qOTpT*q`++eA;5I5b0}kieg=uf?^Yz7r@6s8eBA zIKbew{gZD8R|l36AP_Z3L4_Y(bl+hmzFo(}TUvn2@5gbixiXs3L4CR~TYm9|55^n?TSnYosB-sWL^!7;bEG)+k1_?v_t zua7IyI~Jb6V+BKZcIQJ>lXoe*amrr+9EPlVe*nsFNw##w(YvT1XA=beHeVcsKv5u; zhXZDSx|~?WXt$|n3Bl8dR|$OGpI*^B`}L`T=R5s*36fHj7vpUMbYOFJ7)T>vOYva0^%2ZI9Wv}(!arQY6<`$zuJ%PdJW zKf=-{#g0}#of+c)0QRgTF5`XP)8TQf_}}L@yT1Gbh#C2?L;L+!V&D(pEd39_bk+ppDVYHL~5k<@We@MqRlDi-xgnySWhVey?!`GihX(#>2HO^ z`T%B{wv$#huPX}MAjq%0lQo1YXCtH-@1mlfmEpM9HA6%xZy%FilwXLKbVd!JA6ko8 zPN>B`nh)I;ort;6Pjs%fYbxP?C>nLTxcpw1C2Qbrw4(i6G(z>56_YV%6rfo#5P$*E z>IaA#3n!@5$&c%~cGUN;YZTN9md_wYYqWwV@8WcJJ5`_76L}l64lpcyuyTP!bt7fUX3IwB|xh$`qy6lLbXOiDwO;o=|mjP@LSZ-E?4j+fv~{C@Lu;H zfXmB~=MFi$tKs0U0~Jv_hGr@=V;i+JC$$wT$^(S!Tu*{y-{zoHx(4*wPy1Tb6t7*&10I}aHpqS? z*4m|dbRv`QBwg*A>)x!gq0pfMHZ)A!_P&{578OR4dn$Zs1U06J{`#Y$U?WW(_X|0vcqP4_qpp;1oCU`7IkWyxQ zwa!o#Vt}5xE-K;hW4Udg`*RHA+e%9f)f}D1+_>cB+-be5 zk7%UPg{=)_(%#hGb85WKnX&+(|Kq`lwZ4JPVMg=tr(v`mj2OE&nS`Nkf8Z+B>kTM} zJ?zWIFh7Ki^s|J>mzec|3*D&rL6y|AJT z9wtWrteMBh$g(O?AE)JBFyL;<@`qC7cE0|hhm_1fjzDHJWjJ?1#IUTyT}cpWo+ZU^oozodxaAb`M( zqlkq?Otw8Vs34GAl*RqfIGqg|K!Dprf+<5&9G!1JNeqy#2!-Dlw<@fZ_PSHHH_0TA zHk8Wr+D!LJpxnumvUf}~SiTWC((l+Uq5B8|YI~oczfWq-k~1~e58%qwuALxl4|G>)R@RnO@b=?b$HgYn=R7mVvVMj*l~L{+g!5% z?IUN=p^(y6VR+8S(OeYgO4|@p%RH*X8EDVVoY?d}ZY;|@s`2wGue+MFp-9qh2$#)tzlcs7}LhAE&b z&tvB6AAn%Hb5?d=-=$*E%Je%iG4jKzdFt&JCI=?@Qy?TExgv8_lnMs4QE&5PJ8)lX zdq-_(Z{7Fy^!=;4$uWYk|Bcc5fwD%0cDNh777(ya`p& z%*u+6pWO~Y%L@x3v}u(VM}Gk9)(U;8&3W0y*{EQ{w|+GG8p%C^lZ&qLq*2I}t9`>W zd-jKh9%3%Pk9k>P$l+Htc?}tL4!N+(kNZ9#CM%h7=qH3E?JLUhctp-x!%M-;9I$$y zEi5vt(&Aw-@PnvBa%4@~=+Y1Ya`e2qgE_f$A})Mgechd>2`t-@pME1elr9$OwMlQ( z;U`erHmWs8D7Njfh(97A-4;ys)Q65^NFE&? z20tft(WFaYVhfP>RcUUkh0Ks?Btd(?q?fDMMRSrR?9+joACo|ol*h=EZ+GmT7a!rb ze)$yX%;U3Q*POQsI4CRuiO}KPb`xlxPF<1u+pUcdRFoiDrSN3CkX_!N)zd%t-tl}e= zGDf*p*=`jGv~`YMIHSU;Ble4>avQonvw~~>N$iX&objo*$w}Swl_TC#S#=f^-%{+SNq4et}{Pw$_2Lf?4pDAGniki^;sOBiBaZ~=N@SJRUFG>8| zH~(<;VK{x045L<~ObON@L7RT%HU3VS1vjqh9Q|Rx%xg+N-JkWE>D?P;3%19JS^giu z_<)3glNm^_UI-!oNt}*2NVM#uof-`hNl50l++)%vO@GlK_`v@_)y5Z5$ zpEMR_B~L%H0_mVZ$co&s)`-m$X{e|DyGw3fsdIM}Yamgb*0bW?*l2qZ_}IB9Sohd} zBAsN>i5Tt>r7~sYk-5B8j;b|O=k5^IY2= zdJ&@HSx($2v8k+RVyR|)uQfUMQ0+4HA(Qa0TcO&}#xt9NHFSlgsNl21+vk9u&u7lqSa{76f_%XQ@)bpX!ssx@lG)kQ|U1bb1DrcYqQW8e+izK32h?nT5l>r5`23W zJNU{qp#-K`_3>^lqtNrrEQx*6TEvi_yj`x+~UDqO|fuB zZAsDG?g|7!bX$ZMCSmj14n2k^kNaYAo3U$yrx+ewZGo8YQ)X#I{V1#|_6CE6->nVZ zYhHt)ND59~zQ&<*$-;WxpssRoY>Qn>Cbl#{%*LyKk~gSso+2&dIIw3t-w5Z5+3QmJ zdTss(u=DdNnfNE`s$z`I+6KA>jq8k9o@aS)$^7;0`+ zazzEigBIpv1g7)b%;|g$zrOuRdA|Nbu`bG9X!3ZHwt%SK+gyC#{iY0l#$(qt^RraG z82&@Ttx0FzuOAE^h0}y})ircecV#!ibLiDjWwzB;OjPxcyv422vxWj$nks7(0R2TG zZ}YPiz30=v<>VWGC5?+(%uoJ7x{y6)P2##o2-89ky_PZx%I@|Yv$?XtaUzghs%7qI zu)94+hocy=mq89p+uV)krSdAz$6Xz_4^JOTeO$UWkKx*%zZEukprc}Pb6LEGfP5L0 z&e2D2p1vwf9((z$?RLJ>lySF!BiW^?42&t z*m(}bCO2v%I+}J__Kqs%^%y&RM?AIcNG@LvC1%K=Gn+zaItUvnh{DQ@JSyJ%ZInG0 z&zGg`e*lHMU%m21sUkeIvK?Y#MO6gyl@nTnnKE zNB)!?3hug5WghxPs`25B!{KX-pzv#YJ6SSbJxf-#5}nEt$OT)%q+5GO3Kn>Dos6Wx z?xU>xK-np}vhhB$u!MA7b)Hm~VzgiKe!+YoPZ>iXA(ibL5C@i616}I5o6uv-Z}Cs9+eMBiS+$vM(;uOe_HeH_xlgXFH$WI?<~oz@*M~UN_pyO@v=Gd@onk z;M3UjyWZTP6w9N2Ghx22h2nq{BrQpZ*unpjI@$z9_y;zVFWb#6p9$Mf)mOdW7lPp= z;Ue3`XtFXHOkdxQ{>myX#x`*%j`6bbuAvgVDdzS)9nh0QPXrh;>B>=d7g-+Hi%b6= z&CDJ=ox_EK`=#ajy8{w0*1PSW;nj;j3xFmQV8PFF>o@p5Fu0|`bKqx}Vr*=pB;@7T z;3wDv{ej$TyeyUpc9T1k8RZG6HpM45eTgnYQ*_(>${qk5Ce7KC(hklSkix{*%EkL# zC9}4$-)nTz0Z;{mj6FjV{<8S9qW~0Xsenx8U|6U*HsR~45l4@+;v7s^u@sjSD)yq^ z^1MO!TL7>@3eCR^2G@+#%(_;7&SkF^`#u6%Ph94uS;~f*{As~$E(tq`G&O-CDJjc| zE0a>QNpWtq+d-s{2ehU0I$rhj7&*E@2Rw9 ziP&s103ZoYUel3L^w@+sB0`lW*zShEr_*7uONj&T`CK_dtS*_qb>K=%_KMc&CQ~yF z)}$L1sR{)%TEC2X9X_K!r3SxLG&^j3sM&RT!K8gg_q^WzNszs-r;4x2Lomi0px_|ORE*7#^D6qfspa3W*(hYvA3!nR zi*6@s=XQ)0SP}_}qD+26pe!?gDcx&`wug6tf|m&LQ)=XQ@rG~jN)VXymMKa3(rXIb zs(a0&_iIU*MCO;!9~Qk-pc=LyPva@C95C|8=)eUwkoij}Co=43Wu{ggv8atoN(r3Xo(jgUmRi z=g>%7Tu$@%EX@LzwyL_JeY~7QwE)I1!B((TFNhco-xktL*1|MdjJOc>$?FthmlLa> zdoq#J0`u7!vb#yL68iP&Z|n3S5nEKPTiZ^CLAa&hsz8%Ei+aPv6BI9kUdXcK&c$56 zp4ti=KRVS%*HneRnwH;%w1hn5V)aJORi;cxVd`@ox-St8n(Akv#WK^1#9}dIHkSBv zGJR%EMcHmv7(1cVKF=6&wWh;9wV@>kk6&HIXMViL$u!m@8BR9@!xF0u71_3Apmz0d z3D3hMonB1R=5BkoQoy>EYc1uy?L;VJ^#jiJU=ie#1zD5bfOZjiVY#Q(&D?Jg3X9nx zTiT^C>rag8R7M0i=^l)2Ly6B^j2Nj=CTavOff^nZsa07)Xo@b>nzZNxj3 z6a~&ebmf83Q>$JS;WQ4-$$X~>WtQOVcQ)9t@RbzM81Z1uHy0AuZKXl-`8K@I=~J{3 zgB)+T5SA*5HGM6SOh)K&S^nVpy;P#LEI1Zs-JEi)`1Vm{!*iDUgtrQ0!raQ0uPM9p zzE3>UcVeXL-q@eq4UOT(^d0c0K=LN?=(`b7j?<92m_L(IpTmLB2FY|^Rx4J{^<%)e z?To%q=$P&n*f(>^j2-UtEguk&$>eT2dvl#Ur)>G$7!d4++Z);tyNx}dAFLd2fLp)Q zACY2A5=>N~GLeV7b`6GT3#%@n(vjz(K|4dGp2FY&Z1w{>gWj1lcFI|jwL+uukWG33 z%UiWzHDlf-#`3=9zS|Hun|kTgRB5n_6*O#l+UtzZHsj2kn|;RLiT04%C=HpQAmm3& zO_Sm4>K)D2TU5f{u99`a7g)G$zhnJ^$(I!s)e90kGo#T^8Z+1PS}(5agJN3Dy-L0K zJKU5+QriHXZ%7Tw*e5EfxvbkvboPbDL3`FI#7WYDlk)P<&O|uY)NuN^h~1yO)~m+x zf3EVx{o+9;WtE6l1ctIk%c-k7P+KRZoP8svEVA2n(43&w!2rm}m3J4D))w!Ll<4(U znCC~*Xr|jmHfnrKtcd&8sl3Y4icTad=zX-?n}PyUpaJuCI!taG>5?LJNQ7TD0++KP{jVg@*rVGK}jJ#*@6(=xf@`mGAmo# z>^$>g%KsUposBIG-PWPmK@B#_MKdxXCqbWsP?hFOxT!Tb5P&MB&2zQ#iC*S$`=pd# zP0%#dCugb{pY}G4rJ9>uCP6?M8#7{YfqL#`IH)l8eP&OG?V14Q+5%9fIJn;Nb^D?X1 zmc2PfW(aHTh5b`0;=Vd*zNu>c2Q@9oaj;5WHR;nHLHEo}i@_?ExV~Jh5U_nlwPdWI z3`)-yhJDqrJ-D{bhM09SODzMaaB(~p$iPYQ#JLS|-O!yWwWWtpFHOzJ-5}))wdiK% zD`B0(aS-xYsQxvEcpn ztuQ&qDmn?WJHXCpf;vtkr&lXn1NQjl>d-?H$H+9}dM@DE7F~t{k;r_#DTQmu+F%1U zYmNqa0Zp=wi%oMK6ErZ~G>88*eF$yPUC>pyqU2Mp91aFC8>(hu;UtM>lp>Uk4X736 zYprY(u5@YlvB3?;6sXB2rovFSxXs~EU%^ILj~QcSl2Z3R)z3gg$a)aWneqql30|1q zD%o0;hyW{qWMXmAcmXPFuhZ>({3{c``!#`B%U57f4UJEQMu1EfU0X9j&bUKpE3;D8 z4S;;^i-8c?8vIM-75>L)U6zPlv0U5HBlZ^8?e0{BVqyo(^U~##T>Eo572j2wmVTF; zqyjNFf9&+W?x)ZtXu6#uvlMs$v&@BxHV0uaM4MckveWpfWPy+yniUf3%Xf_Q9v3*h zx>`K*kq~@?7Ql)l$%{-)8Z{VO5emeCMsWEXjYA*$GN!wRsv+b?1m+~-ACkSHaTy^# zVFOV}EKnKu_(KePu|9uAYVyV)jsoAp)Fo;?ax|CZsB?>edHenF(#q^{&{kkbUhD?8 zga3w@i#4LhMPVmAx*-}z`VSzpt5xnSZ9>K2M4CnLE$bHK`zsjdY`NJw_~K>37%wfZJLZYW;qVkhmL#wBG6`jSYzo}#h4I#(z$XNs6?D(e2n!cG5s>ga{^jL@g1lELA({7)geDzC7b zaU%xwa{Juv?_2l`(#_DHM*U=)uIJB3+ddD~6(jMWtrltM-%SZEvVW!;U36NpL5H^q zUaER%8Wl^<_!&W(R$s}Js`18l7G}PkqD;cWhv_x30M_PSJ&OSoc=t?&tQZ}GVL;T8 zm>@4ARk8_KqcyA%D*=`$GdszH3JS6dqw#5xt&bSQzrwu-XC~LF1njJlaSUT6uuLT> zWhODiG4lQVfKcppWTFGYOjQoV@u-ciEdpKRU%^8Ip_h48gHHAr3WjKU-|PtPnLSh> zZ;4vkG6Lz4U&<`vUFTLUOE8%bMf0ZA08|Sl=>+fV)?m;5f+sj0bH{=Q^LAcX)@wZq zFFwO&YGv-y5S~+7$Y?a|dTA}aeh6A$fc2P_G&&Pc1?la5@1{sbVesZn&V*+hdWPPj zyv%co;?~|6;t~y`@GpqU)lYGrdpJzk107v zX?{^|!Ze+-h$|pO8dSdY{W_v5hoLx*)19vj-HR^trR1oMCX2bn0nL}ZFV=dOcuLr8 zWjs+XJoH^s&q0KfGDlv|gQ3C>+spGBDA;UbbWUX&*=eI|MhITebeS}>MTNMBp^8wU zz)|u9IdeGl4`9v6s}*5c=B3o|r4>D-JaC=a>p)>?3ZKl>yntM`-O_Pkw|L#sOFuR6 zOXzhI&shnC4rQmpe>JDmo#)%%$(v*Y9)@f}1(}Ix#D92%#NASMXRhN;cNjT> z*!jcCwZVZ0GpgisV~sIm!D|&z&*Qn}yN3HKp$3^GD|Yya!r|e;NpWymEip6=rwNs1 zqEZn>d6Tgf2H|&kV=lX>Hi?WY+uLq+AK&Y4!J5H>lrTo-qxvptq}Gy(B4U_ju&)sn zZ>+OSn$c5;I#uRTTrAx(ph&_(RYz|zcJl8DDFmS}yyR-=@WVCAF;i88-kTwdUkHCw z$*{1l`?frkUYlarOtyhtz(%>Ci=5691^3oyONFX2DMVCvnO@-yphu{0R892E&uUni z@vs}8vB`}m4I;gyFu9VUvf@sAa@*IrCv=;Myj7FIVxr>8DTkO1ht*1+ss?JL5u%4g z~;X1hrDr&jfP8dJ417M`@KG z*s{wdr9`qKw2Pf(Zkvbb+CePSKV(G^=d?>pp5JTT;; zep{bwOqUyNy7aD1GvP8P-ma06imJDu=CuJY^dUk`*}dapVZIVts4kB6?=e@;wirXV%Kmjmu8Ds2z8WLm z#cptk7N8BGNc>x4Iu-M~9uvji`1flw3E=tjYl&mm z&MOC;*-$i8Q_nZcy*b;&q3qa7Fl+?;Y(t*cE2ok=={M2-OBk;6As*eJ!aQ@q2{JEV zK^*ro9`|V4jbXDjg6c^O#aeBAQK4OC2r|F1u#`mMyC~+#DpTrRE2%``L9m_Pi+-LOzYJF$0fqHEQu%KpWO2;B~I14mb^NGVS+fMH0SY4mgmX^!jfI+g+qyd*BCXN$q#&sUUritO6`a# zhl8k)t(rLQL?HyYnK-Yg+wG+Tc%%c%F%ya5$6|3RMvB@-81FNc2!Pk*Fmi-ONI5>}}=z?)3?wKy(a<+OmgRz2{ooQBnnZB5?#2 zIJzcOV5H(IalN<1JJKB%66K{(u%-hRlxhlwpnHriIhzA(M=5&Y>qH@MN-G5G$Q;+S zRi?=Ch?!;XO_lvUT#1rfl2b`S%}ba{Syz8<>x9Z+6eK@^5W(&%OuTBAB!+LbQPWIb zxE{frLqG6vm0(I2PY+C~o%>)nWt`z+_kqf6lw7RGkvUpPshe~eQJ=Zng0rz((%Bg2 zi7HXJ{AEo9S(=(K-6nqBrX-ceeSe&h)z!Qz6TCEa4W4$2R(Nq*s zseDX6Lm0CR*3PA=9!`XIw$z<48ITmHu^wvye;rWk2wG*IA&5Bm(W>9uLWqu~R!T3T ziqnVc>=h_sE#u;5C6r`iC+~FasJu^#6Oe9pc4)vPOlrQXa2UMOj135M zL7T>0ukAB~DpeEDg34@TJ38VvZuv78j4@%`oCf>c%o=Q{TnW`OU!M3qRSM;xGs6H1 z0`PkVq0;KUpdDjIQQV+2AibSpFapd6xU%6~O zQIG~BcEz&>u3a;jGg#JWjuBVYx5j8Jok0Ag7QLJRHO4#!vV8MS$3dX%d}-t73C}dn z=^E@eaoL^C57R>`1`17S4qdJ%{=jEuTqO>hTVI{ZOKf$PdUJU9sB3Dc5J5sC7TY^- zpepybL~nOgunG&eTBf z$~tJl(Nhr)5O{K^O!wJ>!lea{x=`eN;K#3=268r>6@tW_uFEjl{*br}RCxgiZxhj~ zdEE=&JB<3TXC#JW=%sY*7%ewabV?38u=x|!Z)Z?gNg1%*Uh2oephmiM^0+~B20HYN z`4Ak8DT`F^B^g*=z5ppjMQmL8CU+l$mzmrG6;TZR+3__yT?NpS@b0B(s%ROsRi@tt zZ|@mmJ?Q2LC7BLxNce5L;HKEBtcTOs!Jtq8k0yR#?Ad`fFhOYx7ljQrRzAqs4Ply8 z7>Kz(aR+2u=1RfsDn_3j;exf9G^XZX;penH&J56#p83*XOa*OEMU^l-p(&2xHPn=r zT*Nv(IBO41QFLn^&|;Dk|Kg_R}FCqmqd$E3_oh zae!lgp$AbgwAz0z*A3XzZRA9Q1XoM_jx(yw;RiXKAH1asl#R6}lo%kH=}CS3`t3u= z57n=St8tg)@C@$F1Hn8wY)Txms3;(-yM%yf6zRM~Scf@H0GX-~qySwm?*-NoTYD0T za^z|L=h_kmK^nNM(WSX`ig{REn&@0+kf>K<@1g~xzep%nalOhtNLm6S*Tq}OJ#*z+GWu51pOn0Rg_N(7Ao5z=Giz%jy_ zh%7;2YJ|rfDmB>QDBQ%6q-}PhXgJI|kP25>8Vz)>XM-qTd<2LXLGhg&3@V5k^vg1m zXlG|@0$t2Pv8+$Uj@{G+U=tg-b~@ZqS^`B6TOFzNNdh*wmH?P;#DPt_1FEj3b*T{* zjxI8aVQ^%CL%Tv633p7T+* zsX#7l1WQU#ND!gc=am7pJfllh*qR3xBotJ%@rIncoZ`hJ0ggDoW2zLZrZs!a-0wnt zQ|Be=ETs!>(&R#EhCFd`+~ekQnA8?hd$o{8#-c8z0}8vOY^mGO!FMxlNK$KFhb{04 zwXh1oSQ5yB5`vBL-NgWzaPgC)&@QV12uMJPauBRG@Nb^h9CGO;Dr;<}n`I?11Z}U) zn)=d^lWv>#``DFrQwbB`T`r`k3OTfe?c0eehD&!87O2znVpb>+-MI>`=RkWPAgHPp zJHr+1MhKXCag(6|*kU^S+y*IefB`*pW3M=)yfVS#@N7=^K~&B;BgNn<8~kCG9vwEr z018Qn8(ydyaP4M_J=4rM3MDlp#*a{FQ!)v9M~7Rc76zpWur3tpsrdZZtrIx5kO1R9 zlek1rOJM*A@pKxr$`UqJ*5Y~CGY~MlX~F0Y0;EV00#c^Wwk|PoJ21 z&5pXWM;@a_$Y4|?0Is^SBTAWpvs5ipzue4~Vn!YB6agVp zqNvmjj>|Y8?{~5=Em)ICaaA`iswO1#z@5&5W?te`G~C5;E3&SGx25gKN|O|Mm{C{& zZ#$g+;FE1>DoTkHf<7*nMBFO=5f6E*Xu_eb0qStBD4T^f+3sK*&C7URsHFe|?yS55 z5(eheJY5B?;!;fLb!A8iAZa8ksnHu)isbN?+5+bL;W&4eG?FcVbqkZhkaymY2D}^v z@2?LwpIAq^jwX(`O!u|I=(U4kCv5gr&<~x(_F^|}a;;v%ofz#S3Sh>sB_iOR2BTqS zR!k`8N>?I--qMzEtwVIR5p!{X($)%=qpeT9qClYs5K!FJtt~ao27Fw;)g(R)nR1Nj??gS@0l~RPdbJf z=pBEXyb1`UsSz;mXki3FJ|DH|1dXSm+YArRJ0Z{j+peJXvU7fV4%R}G(eHFHO?w1< z90pGBhso_Gv(@u<^lFeths-yxc;WGodmRO2%zRty_`4{LE93hE!9lS2AB9*InKbu3 znp6(3dk&8RhV#JB#o8;kto#9M1ycrm{vqj&9f_T$e^>GkBu27)_F+Eu1uE_-roKNn zqGE^L@eaup?-YZ?9>FbOrV^_BzwLT3#ZP;TeBNp*dp;rD`XU7JCBP5Qhl?f?6R7jr z$0X0XOgm!T1J7)Nz}+~WmzYR8{QBk6E22+V#vPS0_G82IU^?lipGj^G&v0!xRTHyt z1GfvYf$2nSCVor-l-K10++Ndp9YwG}s08%a{^lj#MYzOZodQ)Ss24zA4?h6h*U#Vx zaTS;+u(`dfrGM z8_YYQ#l&2Yx;sFEq$?4lmtu*o_hNiIJ`8w+0LBKcqM8xs#nl8IzI{xx0Ph3WPSWxR zCv({@kvf0d@%gqUDplbCFb~ZYR)M}@PFfFia31G(n;@ToV`s5Hb1Mv6j(SUKeyK>nA`9Lz#1xuzguQ0ZhsMl)7RcN7VSqQ^_I zv=gE0bQb4V>oeLVXS`-Kal&b^^oH$)C0f!RNGSw?ur!SAE)UiCmopLRa5FlN4HbwY zH58#rNT#4{Y2Y1#cs2Vf9MUF~9BB0;KzXt&uzBd|h;+G2T5uZt+#F&)*NJ#CQsw+K zj=;zFjetjE4cL(kBV*lQQ9vqb6UX*E5-B}Q;@EXjDbV&uxHt+SU{GT{1M_8BjiEgn zAMHAKBT#rJ=Gnopl=Tmb0+N%o^*zEik>|zOC?)8kLK(^6asogpOXN14ziB4ya`&p88Pb1coVnJsoe=6oGD1`_Z9_G z2>u@i%mqlkO@}r<2=`O-R?@)Ar;44ysOn9ojM7EjWKZgwai>y|LdcG}? zwIBMwy67&W27&?T#7|HSiBZs`Z$00HwgR*xsD4eXod;_Isxj#k;a_{qX$hS=PwZg1 zeFy*oO1BLfUCY-ltGsDG4{9cmYnij6M1L3L($uL~B<&vefm5#I?{Uc`L#xO3ze0^2 z6l_r|7{K7lOelDJ+)$E{2h!Mm&e$&w`KTJ18yTagEH%|d)cAYZE?lT^Rf^%nn!GrP z5*}z|w1s0xhjl76X~!|Klb+`GOU_nu$jO%uB#%9hL4<146)Ep;Wfw6g2qt&4iLP6* zNGRZRN|zK->n;#k!-9L;ne8u|Nf!(mr0dtCnM}Af*W5j&Dz3kHS2>1z0}Z8$)OENs z7s=^UD1{rp(5i2#D%F$kcO&-v+EVUH`n~RHG91cfswI8pKxjJ4X}*Oq(Bs63#B~Mnc4y}+=7mvq=`SrUf;uP0-pUv|j z99vTDX$x=T?^sXt{rQcw13#Bn6* zI;q=M1^dYnt53k7Y6xl=m2hYefcb}91vN9nXJM9D0}Y+U#&YM?Nb;R@7 z3J*+c0Qi~gF8s%bjQkiBioi45?sNxC4T?`$_&pM%=@{q=_^|^TpL@7Ibu0_|J7jc# z8465SU{cSu0+cX08mE%^C(tZv#wD-qDE@5lN zYf5>SDHyQ$c((u*(O!vct&MAxnjIV<4slNCStnKoOJHJjq2$$u=Klbf#mor~$rahx z4p|7uMZ}Cz0CaqVI8cp__fgWH7ZAYO4+l1Y_<^IrZ92g|E;vw-&x|`j1=IpXW3_}? zous)+?;RM(cEC(U*BRaiOfX_DO`@N)X~BV^XxrLd6jGx|j=zJluoF($`8%Ln8wueK z04rKRg!%TnB~+4oKPFYdt{4M({f;(2R1X0zfQD2ip@+A{+78gBRX`L3^$(i@p*XZ2 z_oWLPQQ&IsiAe8%XuQA=qXob)q$BL6ck0FI83hBzI~=ZJo#GD%RiFxBN_vx8EBMk8(I~d@N6qj`<#2h{Y_P7cNQSbM70LE67nk@K$^G)UI0#&Rh$IQ9{i%M9W zH0cau3Qm*O&xI+_lj75^W>S-dH7=!Knxc#{tAQQ_w9C1bDuM@0RJcV-Q6!m%MHqtm zsL`Z~&K-SbJYo;)mON4gcFZ@;;+5$Mo+r${sU1`ZWRN=<(F!RZa`4KR08l~b(b$Bc zKm+?RhY1c;%P9)+pr@hP^$*9@K;_}Xg5;?S}X{qdS0ddLe@9ySO%wgCxx1Y>aJmu$Ao5$QgU9Oo9)LI&%uQ z>4DLqdccl0VpWJhCS8J0+plZCG_WxxHS^f&4bQeC!X33na?>M_v^x&^WY!t360ru3 z_U!iwo$ec=5)aR@n%}rE5FoC)$GzIhs(3ol!ci8i<_c&{I&}?QAgD-Ch?z;3%xQe>nDa8Jz%(@V&|e2;2eu^4}2Km2jKa%LP$GC9;FY9 z4?4ZFI=?3%+(LA~Pf8&5fWyw*CS%XF=72P)sQOrV(a449aGw$IE}Z7{hWm%?`ye8M5k|^A%LwT&K|f;$#|j%zx$=|eDheiN{~`c zu;6n;VJ~sOsCj?^92qk30iFAq)om++b~dDe5Ck=X4f?CW$VJ8A0-u+Ik9Ssket>&m zsk)XkbRCRXOSTf!QtX!fFxqWgR)Li^RFVpcM_Y^%T^FOGmZ75wDps%_NLVndR$gpt zPd^qj(H;(zsAryn@gg^}U|P3^0Hx@wH#5@iFr)2tF$*Th0&q_e{pSMax}E|*?EpE8 zx)rZ!KH$d|_Ll(^PYy0hE~Ou9s0VCfp$MKVYP7Re6sLp-e_*L4W^3YI(wNx~60kQr zn}>6Np->xL!caycRC^vy@g1z;R>FSK>|jL2g!&!7lP5W|1U5p=LJWEDEO>4x>4kt! z?y&Ba5@QGhq_8Tk~IfD}S{5-MCP*`T8cPl<3=Ox5i`v=Zg@tm^}zflB^h9UnH&bgzb0-M+ZsI89ZAnUL_q;*lJ<@Op^1?>n;(DsA!yJ=Mp~wte``MDy7^- zBe1%ORdjO96OZi;2MV@SS0vEr;2IYKuEylNv4sHQt#>+RKF$Fe&dtvyB=a{m0q5et zo0meckmTFU&W2;pbE=9{-Udq~!y2L`-qv8-5gosi0#r(v?gDQ2dK!yl`?vJf$59b2 zn+xbJ1YS{%dYK?fP4bf%qoL*didGn|yEr)5NU0E60^X>E^xYu|n9w-4SB#DfYFvnI zHp;}&#D$iebkW#%L^$^maR55vUKXHGMuXO#)-`KJWWYp?bYQ?{b{Tpw`@@x!9S%}e zBye#8dzABsw&aj}9vV2xI1>Vtp7KlL~MwDp0fvcZ0;O zCu-7r7>brQkj62r&=Tk+U{15x4`VaP8ye9y_gArz!dkn<)C?qpr)$x2 zce>-~mvC!9JYrs`2gLUi8A*8M0L0=HV~1pG;LP9xyMo_OLyVh>Su)+-JYHBy*JHc^ zmlGsA0uqoaT%KbQojASb(4AH1+U=l1Dpb%%7cC23#3PI!iDd@bbfclTHU}@b42#Rv zN5P8$PuV^t*xXYRly}1d*cG&Z^9bKAg9^POOCiI#&}JkJ<>n%N6vM6{Ha!X_Q49hF zDK#WJbE;RO33f&e#IkdtA_X2;C~e+Ql0tZw3ks*w9dgXOn6;C}VT3&(5)wLD5DK(R zwuJN-BpYcJCxd_|cnC;H^WfpF7Y$FJTPm{>d!x^TA}m~8R%5jMgPQ>aB<-V?iP;X9 zY$PAvN!<(yKts?AaI4nval77sn>MY+_5)4Z`T|dYW(9hx9kIZmb_iX~2f#Q0^P$L^ zN>dzY;oU{cJ>vp4$o;MY^H)fKeE7SSut{CqCN(>nI`1y!%9sG?$PFA{x8nOGid>q9 zTdIIOSnn>asXXD}_LZGu7lrqQxWvPNmfdUCB|=GF-uMb3hKCds)yJh@S@>sth(UV(51fj4SwlE|iWFgagwhh#v7f z*{;yjO$Q7Q_ewCZtgDkUPK9nvwFH1p|UU5qlU zQ(DvWcLHOy#2Uv>!OOCh-@|DMDp61g2}vqE0DUJT{Z-ELnnl}_l(bBRtasKwffPO_ z2I+ps$()Okvv&6wmlgWtg}3VB2aE>Qa@P3nk`zlRoo=* z6!%x$=u^%8gW^7Jz2$C(+6ylAy@@e-Qu;cQN96Vib--8y#H|2xt!1t*3d{}PYw~cgE_8BIRzA|Y z%rGRa3AMPm3M*GbD-!M^-maixQxzn(zED-VAcA}vRm=b$3)s$tT?C zOa?i3HGK*X7)YscapE*~zC9lo9MX?@PO27z5S1C+^4D||y}Mc#Tgm_jt%s#0NSFvp z6gwO}5JvV=xPj2}>Uw~NMvW`}QAqVmj|3$tpE+Sl>g=Tvg!Hul*HeU3WDd6&m^>)T zxm^aHlaB^}r1WImK&!yLwu7s!q-aZ+F$WtsN=7LF^gRfsuJ}rwxw4Z}J`Wag0+b&I zW6)fbnFMydHcU*2b1V3L{EjybkY*lHmXB!dVb! zWP&bK(pNA89d;H7fK+9VvmFMzD4?%YX!Uc<@1<;Qf#6^ zMATzms;Aiy2J-Zkvu`}$gS}%4k{O$*pXXZ|Cw;_KNaIxO z#Qu~4**ewpaZE;SV0>_s-cLbl2BxpxIcMrWC-ROfxmd|xsqT;D+&{-}?9Z%-jc}LZ z+@;A}p~%-oyxpRZuR&?=TVLv^Q^3N8=fY8F=`YHNM@B>{j3Tb-G~oFhi)Hf)LBeXE zcMp^4ZP_bGQWTW~=emXQ=+)1)n7Lre6nJcSlzRwtE4EXpamX6W%URQKYA4k&*@YR= z@}{0Ui#kiRMxJ%^mM!P$=gRj17y1d1iLEs<|>y!W+kjArMTbkU{)R7 zu7c@OlxQWWr3#MoiEA(tKXyDnHz?l)<^F&|LR@7ob)aLrm_j;Fo~;)B<=oW3dW>5X zNxPcaZ_4=x5k#w37HZ@gzR6LlB9(4(ljx2Cz$P5Tp6Yl9rch^(U*1_eigCyw0v`ikCBHE!U9WQeW6amgAxEXHL ziB_KWH7G&oXE$bpb4)mtQjxX97{mzaha&u80Bm(yBMU0 zkyfz0+943(OvVZ*ZT zp&Mkhm0eP{$3v?fR7GnUl@*;Mb67>xt@9QSe0a4+E>V3tI|h?)GEqfTH8fmG>XHi{ zc}9&*z^*JAt2VP1)o$t$aSITnr~;J^kX&`qXN$i&bX9s-mU*8m{YgTxUF79pI{n>e zx_|Mk)NoY(uzoXrH!}H~w$=((g{cck*rb>ul+-`1l4bZ0*KCl>a=NahK+IcB+G&V) zR6nC9i{m)fDD50<;;or1%5g2~&ZTG{1CUf0sB@8cXrl*zo{c>kHoadqtcO65M$ks* zAQ1Wc8G~+`k9P*Fo1sEO=FyPr3m%D{ia4v4dPB!YGTMU79_j3I&7}>Xyj=k`8FXCq zZV-17UwgmY*Xsi0t}XgNC#EUaqt148SUw@G@B(Q`8zAoeuB}08#w!QUp@>^a`$Tck z>dFXa&}d~mMEjl1vzysp4WJqTTq>~s1>xwE<@W#eMPX0txt=Q z&Kw|g2N!8~kRF;7R*^}pvY~RJ=dpupE*KNF)NRJYW5?j^ks@6MwIBr@yBR{EATiL? zXu_dccT&Kh*!01G!pO-uj5C{f2qL6Zy4grcQ7rhTg@{B_ooKWVAIaHB#I0UnmBLGG zGT2*Iq~dBQP*2(O>T#2a6qkG?zHQm(ZrEccT@WythoK}p)Quc6T4JH#B)vd)hmsN! z2ox_;R&_LWwGBZs0Nh1|BH{er;$ zMX*G75<4HGMC%BrVv)M5n$v?BVJk(X6)j0X8mUJJ97&_)bKhC=b|p3e%q(pk+hg>i zOmq(y=sLQI)--JIE#z`t({r6z`xg7h#OR+IzcJ2o)_Xf4cS;a`#lGrecJty(Mq`_q z+UCBs=N7;DWTwJFq=2=aIJ=$zc)r7uzP9DFcx~pwa@TanUTIz=%0Eg+@jo1QdKr(V zrbLR)oHx3qN_DhuE30F0D=$1ey z^-hZLuJGa3(>l(4sQ0yI23m8nR8rxW1ojF#KR^2OmHWpg+spyp%Ur44i?EZkQmhZT zP0&qf!=1~gHgRg&zgL)99wqJh^Jc)U^PYZe?bA!ga5#MC;G4=li|Zt`wZLewqaUA;Pw?R>&a z8a10afT$NBe;3c@`N|{|e*36Zw4WcyxEDFK89>9l9Xj0OZ7SIF?zYd`SB!k0HD96$ z*j6e8@OSU@k8_+d${%m6I;{$~uAB7_R#Ta}vYV=nKPNWQv0DevR-J$MholEHFX(u_ z(*7%;?gM=;-l6g*!H_ zDkWYTH5o;;>_}*7(Z;;B1qz)WYWRU-=5Jz0i-q12Aa6aHDP3g ziaFNo?N?KWr8hM-FAI&E4QnNvbP{#?E_7w?pV&tIq${M(=2dch-ul&z)QbSV0@ z%PH3;37_*>FaH3v;9kgQvbU9H?pF6W5|t=h3CsW$F)GWd>OrBQAhhm!_PqW70QYWx zdG%jgmb-kpcp*wa!Y2ZhofS_+jT$XG?(5o2%`}~3XFK<7)SvlW-}rm}&D-wTzw)`S z@YlVYhHl>aYv1nN{`2Vh9cx-|;#$|}!BUi{{$1Z#mlKSgW9(_RBBv{y3nb+;Ky3d2 zm%r@U(U#07{{V@b>Hh$izwF!uUA|HYy?Ua5dvp8F{O#lZt?&0Pe|hD&cbJvS0ZZ=r zh@ZP=&;J0ef7!bWS)w6XeAwZZzwF%3Sz6M&0V`5L7pDq}W-bWyNN618-=@7E`EFDC zPRT|tzb|7sMO4|mjdIztKla!Cn?E;jX_n2O{{UMt?3px_Tr_%8VLs^7b=D0L*R6=Klb~<~^H`6?bl1H~tqf z?4LYtT|esH{{VG=-fP^Fr71V@Eoy*0B`HxTxuzmqjA18Vm$1sVhALR}a`~`-_L+)* z{7#ErVNp4J)j#^Vk7W7rcHsc?m#QcCi}}3gF8wXv_gDSrGQ@7QG*0J?oFwraZn z0Ltb)lQ_no^mBjVb3VzBmDHr%v;x$T{OQ7_CShVd5;wFgcSriSf84qK=Vcg0)6MPh zT1QNatG7S$`LjR$Y|qBiYFdtc*X~*QXDMI;Dl%g=kB`HzdyQU(1!{^g> z>;ul;oFM-3V`u9I<>{oKoCzm`LKi8cLRX}Cm+8^k`VGXJe><7Z99QnHK>DH4ZD2Z_#k2%g1Mql^wSw{cFu-V%}K zxB6A-exZ`j<+Jwr%)QjCZ5k;Z3Q4HDokYX1N^14V9f!n$7!tcNq3 zP7BIT=^d`NXAg_%c|$F3;!4Okk|<}f^ES4>bk!b*{{XDD#nfW71UBs@b#6j7s#6^jLC<6;{yaFVSZB z1SVfpEnHd2kl@_?;o|10m^x3ZfAJilG#}O1_>br1-yEMsJOk?a0{XJqtisZiD4e3w zLdGPJa~XM&wswO@(~QNL{{Zp)m;MUJ_gYw*hbMuKxg*$o~L`PtB1WznZr=EaekA;kAZnMR2H2rS3X(?#f~Tz&Hj#!~J)A z{I*B@I(}{c0A1ey04TtKTs{^RyiYGDpAK1G2)B}Y)2S&$Ka`tB2;j@%Z{5G)7 z2(A?k?uVAI4Kn#J_6N2e^&SYpnSje-S6_*AMok;c6eL&+z*e15O?E+X;`_?tUuZx&z8!zOSL+{p^k{9DK~bR5+0Z*n^_mGUL#rx70l5~$ z2T%i%Z)>G72*{xS0%9*}SUMm+0K{BKo+gB9)L>`uw^J@dtRm*ukT*zO_ zUW~o_md+Hc{{Ssl6W&G@(^UaO2Vj@yHo!K(BYTI^<=OOzetlK2?FLXqKr<9!3fNsF z;yx~~oNzTIXbDkD$r{TsSY1a&818{AAhlaL$~I27sB+v(T8<7&;?F3Wr4Y3qpNo@i zK}2ud;j7X-S#0`a$K3YT9S5pWPCWvPP_cZH5UD#Qr;U7C&RW`1oIq9PG0f=lpa0na Cw}^89 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000037_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000037_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..6e4d8056ef9588e9bcdf645df059dd0c9ae0ea96 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9nuNW8_>>(J&htZ?rkdPP+2hzg<6lU1K zs5l}lN5cnuxdRKA(X@g;t&D~fECevaiMDBH3a|$4Va?4iEl$N+4Ko3&VX0213?_)f h&Yn4A3u2AZ1jAu3G@M3hNO~bU96(93G!aPX0RThZvYG$@ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000038.jpg b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000038.jpg new file mode 100644 index 0000000000000000000000000000000000000000..38a2d7adc2d2955f10a27a55ed913241ea7ace38 GIT binary patch literal 27844 zcmdSAWmFv98a3FsLjysATOhc*yF-x1-Jx+O1c%@r+}+*X-GVzog1ZC@Nv6qr@11+s zeDizO6sx;Wo$5M|?EUO>sH*<6@@E?Wl#`N`0zg3n08o%Wz@Og$3EXlNiWePbL1v#kC>7 zUxLHb%`CX_9VaFL8X5otiwF+|`|1e97!)QnISiI4HY=>^n}E7JmH1f-CpcrZt3ELk zwjFVtddllS=RdswOhl+p(Ecz`6cFnG7)-#QH2^BiUso^z!hokl+NRJq9$|R=6t4T# z<-<26nU|$c>UeYYZ93XEilzh{_*u~UgVnrk6^sltz4ten1qbJ5mF4AMT2o?C-+7Vp zY^mRF{&?eH<8|>Iaq+mMlSNSJ4Y0rB$CeCkW}X*@n}4ZF$V;EpC?~_pbi9 zlh_nv$YpY-*g$*=3|q`~9dny>!>I6(u=Zuwo7xA1w5#l}7$&Mi)_c>L8kq!DB=W;=d99EBj}<+iiSO9Tkf_x~UuWHuFU=i0%{y5=J zbm~=darnCKt)Ejk`=!QgPpT|B>1o6no@cd-Bg>GZ+;o_{({G^Sm2uCP8x#C*GX85E zL&-|Eq)$>+$Y1pCXUAKcx+c*s8Xu$x@x$1RYPcqsKVx82$UA!sP50`}9b$MRF`>h> zBO=JhZ?gE`uK(9K28lO6k21gJpLOxY_)-|`M52fz^?&uwS1L)^F(>VqRQ1ZIAE;6Y zLg$AI$mokMg>|5NB{z4Hz=MzE#~REkxv=r5Ap)ER2A3>Qr4?9bj{nP+4l_IwQLtT- z@(6)&dZEs61Fk`0q|OE-ez7f(i1@z=DVya(u!SvF#P{Y1?PofSLk{uhOLuD z8L=G=<>vp+js8RFvSp?sm&q7cF0ZE@UvG|QD97N?fN2xd?!#a23IC>{%N^Wb&+ywm ziBbj}*QgvbDPhOOQa+>PqW=->W(0ni07~Wj>1is zQk+xQTl?!cuRz^Ecw-&jF0{c;Fbb33qk(A7v^uAaMv+(+LG-3jMkZ?t^av01JZ4p{ zKGeR)e>)OCHaq@x#@;DeV@#!TshzcjFggky3G_lwE6=D=hLd^%A6~i)%0J{0?B&O% z-Q7&>F#Cd#@Gr^vi^@I56(5@Sn!(jt($U-mpC~ME-i6b}LtZn1K8^q-8Cb=fZ<#lG z;`p*m-6g~Kao{2i50*^8_pde-{4HqE#x2{zXwyVK7Rib{`M_)YMf=OI*NoF7qJqCo`;tYW#xDWvUAtq2y^4{IFx;i2$?N6MQ~kyX)I1`W$u zAQiItvmx`RUeBW1V%gtl4I$Ak*3941TAC-Koeyl>f^HV<$Fcp`$H_nTba>4)<^2#| zEV$TmIk)mr{sFF0KM*GTVQ}bkZz$_~Pojdd_-lRIrwXI&cNMAoZ4Oxc#IHiBY|@we zq-*#mb77A+#`}o1KSiU!QYM z=F443qn~S>x+jjAw+6=ZKGwV@G!K$StDg;7Kjlc(e(7YX3FrR$@Xh=#_W#Yy%!|Xo zhC}_c%uT65$!i#pTtZG1I=6(l1j)kxI+85*L)g9HHH=pdfO+p5h03$5_)1tGzm-1_ zo^>buIuj_zEC1P&39%1>Bm>0trk*jD zYZ(BD&){{PsYJhv-TljgUxoB$<$s0x=pB8pDc|{Bc(5VXN1CrPJ*N0Dlkt+kVCTV( zC>W+mtR&2$905T+?xf=Zm9a}EO2dIM-BG`$L;K$#E&T(M^&)TVUpMHBwp`xsww<`q z320oZJGvBAy2ytF1g;>2K}p3%SeBYHRQrXU*$*zY>mPcg8(pJR{1bA?bK@&F{-XLf z_R!XE=B^#nI0{=P1q34tOkkpu=r~q{)-m_@39?<|4Z1@vZRz$!8qL%_=;Q%GxP(EB zR<7&X-HQu9n=*guydw2a!e2R%rIz{+CJ?KnFVfsEGq?QnU1S^`C_Wpkn4#!F3AqpL z1sf0g(@ri?5{WLx_C?1d`;biSk3d-Q)sN9)ILY}ih9)+P<0Nxr&zEG+Cu#0d;0NlI zUX_YQWG!#5{}FnW7+NOctWZjrMQH?i5~brIUy54^l3&*)$K-XYe%})W+BQE3X<2iU z8XW!rYLXG+nSWS4Dg5{YfYAGOL}?VxUG2_281Mpyl{@*Hzp7$GXRlJu(HVJ=hbOHhK^} zp{5wY=@(0_nCA+9;g)oK6=kaA(zT83zs<&Ezrzom&+72uEBiu_@4(30lEbUWQJcpFvLiQ~QA zJlCP4qqPZ>t_#l_F92#6Voxn*S8~ZbSqvdt%UZEG%P<}QiiU;h$F@D~`~m1P{#BOG z-skn-AOy*XTK*+Ozv004EOQGIkmprd)H@kWp8^|h+O2WGbZo{R2ak8tXVoY!lrIgA z>|o$q2cA(&4mc?`r5S`?`1(OcffQu!AHX>8JxA7s)(DO8V*cpVC%Rvnty|_bCp@c{ z*zT53?kX6ClMNyzOoK0C{Db~XWYSdS+05Cc_q>ZWiYx1Arc%PZy<9}z=B~g0O2j{^ z^{*1E`k`cc-GkxmP!Hj5s`ti2+ceuD)*&%fmaA6Rf2#Rk!U3tG|K?}=(24JVMr`!e znYQkHmDY;?DpoIbW|ME@-DKyaBCp&OzWD4byfywhgADB7Y7%&OCweE}*I*Cic|$Vz z2hcp>w(!Fm$B)j^I-C_f99a>?iXbklZ$li0_ze)l&1IG0 z=#a!&oM0nR7EUHh;@SDO-SbRknKec+7hT{d#^kdJf#-x{p?dXIzL4Y~*hO*R+oM3`uh;Ax_m z>mMzn_d8|a{{UJv{s8)^5{*k)ph)6co%PlymC{mln!S3qr|k_gReLYVZSYnEhQ)^7 z#rY4?GQ0u-#1AHaP5(bwymmkjRp7h=MIFw{((X5*#<-9;GdJ<2QkBz!@R3PVIi{2& zCq2PoLa^PV+s8ms1JS@=X`%ef%8s8>Qh(dD@vf_@G#b((!BX~dj*>O~Qf2`J))3LZ zxc_4ZH}5YCXjF|#kEpTY3bxVbt?DxP=9jsWO{F?&#@orJmIKGCX^J>(#$Z%a)JRCa z%kA-AI9+1TRe>u;CmM-0EofV8%atY<+_M%{P-gYoYkDRR&Q>X87sexi9h4CFKl~55 zUUy!tiY)@jbxK4Q2>P2hD&Pw{<^#Xqg4I609apr&>kvJAs_xJlKKhD{$DMgk>D`dN zk+&vtwrF@Uk4Z+QfkSLz-9+A|R^|a(A|3jz!HkYr>+)uJ^-PX8?^*X+rc7dzv+!DE zLb%S3p@*H_)T7DZXui%>XN6!K#7B|!gRXziQUE;?pu%XqAeS6&Q9)RSLg$Xt=<8gz z{hM4ViEk?RVZQY1LAu{y_&hGj?MJ&Iw;pCQm&O;l5SuoeN>hDMER!`<^(OyTJf%ui zHFU?5DeH&B7S9+M#kp11+Mgx1=QlN_H3+oh<_E`98Sm@+28P6U-DbKj zWYmIG9~p$+sH(7yqxFMiOx$euKbK{FEY%lQ-=Xk4-@*}pJy68>iqU0{@&>7WekNSo_ zTnj)=iXs{{7LEEnO)M?dY&>EDPy`ygyUgBWb)};|V_H!fNZ^L4gFjtN)<*pHO>aN! z3(l6!j%xb|9s(!IhYq62*CWKNT=y-J3x!bDghcC-|?Dqg}vW2>cE z9A`i}ZF%+NJb~=pZvJGiOEru%uil;Z8CSaGAl(S7f;G*2gy?l$F1|j#L9@Cj?mw2$TDQ;_k4s|8MWPgmCCvW%pPv(Og2N!O%m|d1YkcD)_pKp zAO}!NFn&P8DMu?)a`G7;e4ii^@g}Z`4DuFC5%Mk!Rcw*Ek*Cj*xQr}t$ZY1W4?!6^2X4VRW2>douxAN1iOKR}2et@Do*|IO?XJJV>w@e`dJ0fBe1bct9P zHL$I`X?Q57c4^6*gO0jj!(J{Bw{bz9-m+8g(OzrGTVDeaz{QH?Ekj(AaP>BJns>R= z-nuEDHJoQsGW6JXYAOnyK537HCUa_Oa-ZnLu5fLp7(fEfYmMF<%}Q`Ju=L`aP{2a$ zhl!M&x`QhbrHOc%;hxq$;M*6ZHM-~ex&2_iUpmtHJVZMEBraN9#m!b71D0sa(8JOJ zus6=w9L7%nkaYS3VBi+*t4U;De^+R16##$Br?hx;5Jg6MGC=6$boPQR+XhJO@lbU* zUTPI`;iJ$jNtrxtufZ2q+`r{$P^VokJ}yzoo1(&9xi5EnaABbE;lso0JAv8Z<&ub( zG4QNT_(dY-USzw)NJ8If^cCr=KDuc}DY=$~QC%%S-bb^?P}%0LO6~VF529EC?x-20 z*r4KG#gXxKifsC;jALOvCPSs#!D_%@M&JUY*qFk<>D+|)TE9Zj1if=)K+&TQd7uRw zt^$vFKo@7F@beB*zT>k9F|dGr%6*iPd%L=r+gL$n=D+g5U-XMzK30hy!%5 zmS1aX5HdNN{HqC}NBwYZ>;oblb8%7jA?TggpCC zqqHXNv~zwPsyuHl_tS8lw7kOcWGBUrqH4Hx10{8|IM`>;755Ehi<07+Q1TG-Y;ef& z#k^^GpD~y64|tR5`TR>&!J^zO{zzZt+z$#BDE&O?o^T}z7>Ez$s??l)`baWWImE^* zcF(>!nfpWqi!D~MZ8FHw4#N zB)B_9?s&#dKAkP8J*kM*6_VH>40;s6dgM#U&VvrYzUJ3-!?ZiOWZ9=i;kZ~Y5*eYo zNMyDKF02O4Mr=hkGZmkeXn0zOX<_1mJi}s=uDo(e=h>Qz4chz``&pl+1u!fO} zzSsnA!kMQT8zT`SU+d0I63}=GJlPm%Joi{^k74KEZn}j5+Wsx5NhXOruNvYnT>j;$ zbA)xFyG*J{V2Ep>@L<7oFcgOD*$xG=sGCVC!8eHPeGz9(npPIA~PY1 z{&8w(n0Q!~yrln|u9L>*4?sGHn^qTRr1K{MZk})E)9~wh0vG~#U1x-piXiHQAe%t= z9{B;X6l4^EG$BWD&1xZnr46f5My?tUDGAXl(Zw+e{sB}w1@J)Sd9YYWNny=kPG0cP zGVyx*W4nF3FUGF@)#SX5;17K%5>mwyZFusczJ>^%le|tl_E-<4(m)g&kG>4W+6{%4 zh^`&K?fk@ytcis(wYTKw%xGuzH)rZOb%l?X9vzLswc z5xAuz+DUP46C3wS^wC~bm_10-i@Wf5t;6xA^#cF~&o*MUEGeZ~mbF#iw<3vlM@~@ZHbDkgGGlQ|t8~fZ63h`kmiW>y8mo~Bl>zVWk zfJD2(92rA)%W!ogd>f-n(~Gm@co9j$q_ zyN3(z#Y0CM=fv!n#OaJb3WnTFm+;TbZXx058%hQ!{5x7*ntyG4%*5QmqL6< z;NZ*;@9Gs&9~7J(M2#j?;va}3LZ!I(V% zlFNM=;j;?w064}z=S%vuzab$wX~k~Ql9q}k)gk9!x`bQT%hCF@j!p^JsSrsbX-}rl z*gq$54o`^7g_!ZtxqYKShl=3y{c6qo*0fpKg}0v<{Ogu14Q#Tw9mX{sG)|Ol$V>3SEVi8ny~SmicwB z)n03YTO|tnp$U;}kd|B)LVhuFH7Rl%awGhvS1Bmhp=Ckre-pUhF=*BFl>@H+7}GsP z*)Cs59&fT(&&FETIE#u&ANAA)-71PZiVexFn88R~pQ?i9y3nv&1Pj(3X(rDuhtw8^T(J3iJrwhzAZh!Mtqy!IzBp~>Z)2N7e57?i zhlbg7MzWtud4zjxxD;-l6WrzOnZlPMFHPI;lR=E;dWJoBR2DYUDNkrA$}zqYj~sA* z!IzfT+d0#3H4wkK3{xvpej=M&Rx!VF_Vo)Ivd~)xbw$&TRs;zm#1<31j~6D{=64UAFtwcXp(2*qTOp?}8mp)? zXE--(l5FPHWW(KJ-J}Z*X9?r7k_mQu2i{M5ja$vdMAF7Q&=)*0{ZN#aCh8KIJPtfj zzlpHlU0Ba{NK5z8<+{T}FGqj2Ks7XZq*-`u22mIv3<6;#OB%g#Yz{lQzFmhSbB(9VlG2yGoZvu}zNVEaXn=6Nz9OTX5L=k-9bz$+r3t@mO%EO<2Qr zh_TJ*dRey7*+zD=5fBA%9aDcawI66?dNT`7jjksMvjdGc#QLvd*LRlx6cGdM@S1< z*SHNW<;LsMx4J)&fgA}|g+G$aq4)T&mshU2Joi3*`k+G~c^D-!eE9pchfDObS-co@&^F-YtDP} z7s8lbZqSep32i15IH<7iE5@Q}W!V1s>9M(swlOYHv-ye+G#0{zllRv z+~mCn2*alP<{JzwHY@fiM+Bsq4urirp+$>NV+R_6nrfU+5y>u0OA*SDwFF0>rD&X8 zDUi7`k~HrRz%<3SgRhEfC3|a;Ch2B1$=9;nA-rh@%?dq3YcNB?B-S&IiLn{ax{wEE zX5QxU58!rX)GhN&t;C{(AI~l))y38NGiQ{59!e&S%;ni}$lm_%+P+0w6H$?7C5O(^wN)9sn1lJU-j@^pqFJ>gH^B??H&7@JD zO}ET%Rx6BOPfY)HIE~PBQKC>UMeQ$2XgdrjL*O(**P0Qv@1A@i0(f;`G)6mCEw;jL?`69nwRn}p7%9JPi& z0OS$G#l7B>Hhw#y49y{(e#g&oNHF%StP~t1zwR0<{{Y_L(7^|Oh1-zr%B0bS->kpn z#zfh#_C6LxcXZ-fA`8k37=7<>iv9a=@cj5LY5#O%U|6wRCC3$BB8V*7Oj<|0LEFZ( z#~_hcQ7ZhDez#+}bs}2<3g+RA5NpHxE79@XKz5K(XT}R&*)PT6x8UY)N8{hW$vmGs z;EkzTQ*?1>0c<;pmtwMq2K)|i5drrPmUqVp=&Oki?rSwUnG`gb(PmqIesD2lksSEH z;&0gJ7dqQA?|rt=)b@YUfX12GzkyyxL{wa0Y!bxt2K~kJMH_MiPtT&sy+SR~qy>9< zQm_u=d1SN2ElSo}cP3#`pq0j0uh-h|dQDlaH;-{^4HV8^CO7%Mq2dncrkmHWF5IRS zhkkBAjYI0(LueNrBpbAD0I4>+cS(=&w5|~$h_pRsoMbKs_IYO{PCK;3AGG}#$aJ#n z>|>qvqu8$AQ)20YvzBM`o6(brb|AP(Yzb_VA9CUN7TtBDA3^LC@$2Y-*_S?sc5zU# z^&YML1sg@rLS~Sff6)4Ci(x=wiRA=yvIu~}S=dd`-+`+Ob zfb~V1EA1=kf@e*P{mMGtkK-E^&s1nw1Ab+(=g1T7E&|TAp2X=Vn_=8vckgitbr`&X zgy4++yO=!LN2XaoF99jHeb#(fHeL5_)Mh0$r3$twc6D>MYPZJi!ACdBR2_~%<`(^r z)wy#!lOyr2+!#zQI$I;y>$s7!3H;=`MM-GJ33K*v=Y)J34-rUZm{;Gh{6Y^r&cMWB z$PVzz?xH;216`DMDI0m8xQap1ShG&c?1ITTvW5m85&JL`PI%Jv-;h|~J17<9o;zb% zLAvxhtSYyIHf1s87QQn(+=CkJ3w$%v@{rrDeBN`sGv=K8vCJhv4Xf*-X}RLNdH zC8)x%5h1%sQ~|&+<^C{5F<7xtrC_o4aA;#-7795hBINLj1fxvAhcLxpF&iK?uGlAr z5Kcr)6k{%2-YKIL-l}ufTSZwXYg|OJYXjchw#sPtxdH%zofCQ9bPDgl59N%pk9^s* zJ9kZ!bHcTd>kNJLhX$OP77iRXDm-fh!$If>cK#G@v!AxS1?>x%*>|&Xn)yG)DoGhs zm}+ZfKChnL8)2$eZ2qE(u{KQ#hHr%OzC9>|Ph^f*$$A=hp@X=w+WeZj7{(Wr^u=NEea4H^T=Vo;-2Q9M0Vt%4XYEb`)g0UnR$uTy%#tx_I)>Pkac+eT7gAZtP zY+PHXw5CiTIuN()nzX(qDw2u2IgBJ!ZHu|I5(*%1TZTb@G90_odJsqSXM{B@a?umz z!zof@rX~J-kbXKmx@>f$Wy4l9wY`hBoT8<7VsG_$wp+;(qjO6$$MgOqtbRE)y`v+J z<@`8_&Xzg$G3xW=q{8rGQdlb4img6s696Vv`RAA8u1;nuHr9e*;Hp+Q*b*Hoi1mqaI%Z`<;uN=Fr zyH3eJk%cFr=ZG1HxYwxpIfE+)%CfK8n5fPJ-Td8hOlm;Et(`}^@(zMT6k<}Fnr1PJ z6@4*q^|JN}UJRwKM9qN-bMm<{Hj{BaV_{#tO;dIG^b#svgEgO>YnJS(s8aRVctM#!HXmo*W1+qDi(~3 zv>`Pet$V}86=*7oRXoW~g`Vi+*4*<}ch^<*51{O=_RR*xMcUB|*4kQVY7FB&MICf$ zd_z(rC9UvwH1IZ{4)5iA9&sdviwNFvePxd$&)ZH8XV0Y$M=DCN9^qMktG!*#8?v#KDuf+mg!t>>^G1;orlt@|&bmZAlw2xkLP)k#@R+B<00U^tEJ^Zm^ zZ=w?U9oTmBKr1#ZSB)ck>-kIZ5i<7kWvdNitIk2Y$Jl5&3pBP|Bioteg|b0?bw{5f zwDNh084-$agpfxcibaVC3$2^Ky-UWa$^ZZ+U zbqKZp*6f_*yKyFP~JFjE!dE-Mf%ne6;2&s8IG~)@<4(FbqBDJpS<@LlxvS>`bR;Jta$>vQBZmvILI6uc@ z=j~Mwlqd0>*`pr^IB{RZdX&Zv(G^Jv@IW7gdN`4YJ&I!4yPcsRL>md%BQ>PKrtVjT z5t_JW^dEp~KrszJ>rE8a8?`mSh$+U7P%9Qj3JUJU={WW*-I9|?0W;z^`Z%g`%L&=L$gMM-UVmCtO&>Hb6lhvkT#y;0(_924H@9dH# zNN=(&9bA>BnhmjCvh}}x`lerU(4Cm`GLuk~R3nwe*SAQkZp4HNFtNto^zMp0NMH`; zIOi8a*>y>4oNRJ!Q>j<>V^&Y*OyR+7%t!$rm&dg_V)0=J2@T~<`%&Ll{-#bx_e0-| z?2R>9#0OE}>-i~dIpvQuHkhGr=n8#O(<81M1lJ6obZJNx-+I5qBXs^GxpkRF}hB?<@!uQ;l*)MiW8bf2YQ#Dq0(7Urfld3;dYFs^B97N|@Z8>B?+H zeA&3qCWZTI=h-iPY>VE9l0tSQ$W;;TEv}H&aJ8sWI30YSl95W&axL#>x$+R~zV4t- zRXe{=72}aZkt8hw)f?u~&}1pIsV;vK;cpUO?T_ZR+Gk`%)psXMRV&){+jq!4iz`|1 z3d!OvCVAht_mOm4iHz23WabsSR7f$?euVulr3|67S96fth`oqb|&G5hjRjb3z z#Fi$yI(M!{BA%YlL=pqU?A8acI1{=oH73dWBZxa6#`xXc4HZ2re)gH>5(<$l4wagS zGeEWQ&OL)`;8<)hz1_S(Hu4=!F~ir@@suf~GHN3hKUmyX(G{_h( z-bTxbqDz!4#=!E2dvN86ajxVDd(fe*E#1FqvT5t*J+C`)YnDur$`&0V^+bf8Wg~hw zpo=+I(WEl_jnVGtX*`Lnp^0=X=wJX2};LcOgK!*3T_LH zIu?Os)RFAtnrW<@O%3GMq#>J@q%<~KDnV0&@93heQ{?O3M$B?{ICD_^sv zVS=QbMPkVA_HpQE65`~bI!jek12w;2?|BquSZq1F!=vKuw;vfcyQ@?WkO(a+zS7%` ztHTC`be##)zj$k`OHn6~PlsKC3niP6cq(y};s;cIK5HDj2l2PI{nB@5GgU2LdW^94C2yoWkLu70g>VZ ziaA&xHTG;Lp=rcRSI1;0VYaqhqxtQ|A9V2hE_V5zU5<2f(vBB@)qKq`WMrYn$g)1* zT9^NVN+t_tpv;`!iV2A>#5i`$Vyf4fhihK8crT3Y69Qksd~E6jpn7YceK>9At){zW zEAJ&L7ylVXS@tGJFCF{SRPn>lQ_JXbZPC5{q+%uU#GdU~NJG zWOzJW@|7jrL4&#_GVa2{n8}m;@|uIaFAg1y zo8H%7FU!>7F^OhC?BDu0mRzVhUZ0ACm==u^(}94omc5Yal{(ap#dwJn(^!0H@Ceq@ zhnK-RdJ2ww7wYB2XIrG8?1~RgKRsi*N8EE8xyH4xO~yws%M#K!$EH{kcNyGuEE*cV zU$!7bRPi-Buu3i59u04^3X2?+X)e+!;YoV$nYp8yNAfU)3O5~=;6+GiyVooGep5g+ z1u=+6DXY_+`YS{6nZZM=Q~@ffNXOOoN1->h{Ao7c#R(LCHcOP^F!~}p$fx^(x?2Ic z;S17=Pq%uX4+nCMVSeeECbV_?QukUXUUCjL+cG&y-T^zOv~GFVbfZLgm)iKLh-gxf z&c6)vPMP{wsB419FC>OCJu3&; z5#*vZ_fH9;e2I8u*hCPtn;QrfbS00heJ!gLGwR{w;x7d<#g;Do5$M6|4MQrEYUPed z=xy8!$|(2rVi}|=)dT1$xH&6i?h@aM+>7lkG*LKYvS}cfvWTX6`_^|wTfvI4C3f?) zk6Z4Mkq-g4T#i&ObCQGixAf!nRouJyj@yhto=lpw=H_}kP0l7=8Ei+HeOK+(Q)0~@ z)QKDOFO!eI-?2CVSX&m9yR^IuFju)|UA3Q5=$=zOK}|2EcA-0)An)x%Ceh zlVMQ|G+?viN;_g;$8z?Ju30+fnc-7`z{XzhglbvRAWNDXU#kKVw`cQguzN_`H{!B1 zu&&HDdK$@T?wLToU zX>|Y5PNw_`Y-v|jelcI7{PelD7Kl8Y7u=9 z5uK~MJE+ce0E~xNEK^G%owXL_Ms&t#-WN!v(4x|y1eW_&@3!>PsVuO4`g zccTBXaUu=0BCjtGPd~ZKLYXqeSFN2}3$IY05&Myoir;>MzH+$byK@~rYfJ7bc&vYp z6-$s~rTwySR-+k;p%^NBNr#bYw0luTBLy>biwiI#s%Tgb);)Us0`TsU_wUr&^hlMW zuoX}J>Pofa(9cwsb+^M9cnVc^SX+n;l9Xy1K&MWm27C^bT~3I_#$C97x8|-}fkP$$b2hupxCYA^q7RC(_J7NU$>7Ny{Ylf91IB!*mHL-oRQ&noAWtQ}Fv%HHgF zL#JYT!iC>YHi?)xk|Q_Q`H}cEt0dMK^rZ0SI*2uRC&>Hv?72Udb@aAV%ibeX#mcW$ zD{MRLARY92^<_kdD$Fydjd70k3X9$&Wm`LP(aqUOg>ZYqvtK(Nwd_;G^4TXM*y~>V zeIS=2_GTSz#omrimNcb*(i~z7&xd8u1iP(jRtsC_*oT3+3Yv!87W3CbsM+bCxiOLVKoQFhvS3W z$r>dxi`0W$4$kxEJyZ!dvy^X~!p~Z~tK6p*3lkK=YdhV+$l~u(zIN<{j<B1 zak+i_k$JKKfQC=-2qXj?*|jve-c%XYsGLJpIs*Bt!j?Nsc#kSnTtf|UuKL1ld&7Qb zm9JYTr#-EY*Kc9o2W3^!2@PSqeXz##&WK^DR2gH-z9fe9_fTy^Kh8$slxPok#AV zx_f>ZKP#flyk`MT*7r-Nb4SIV)LUGk#RNJ5Rei zkJYoP`=(_0mSLVLl7YZMLEY--OP#`q-C<1Re;GORV%Vv*nbdCvvs6+k-(88k>s>cBBSBwdjVeAgo0-C=l74f+NLkY^`DJ66O6nYypP%KY2U^v86-3B0v{Ky*Drw?X||m}KLsk@ z>i*erGxw@$r1Yu+S!aK+4R7udTr=$Cfasg=gX-3Y)8k3ht=MLnldZF;ZCjkPzZe%@ z#Abdt(O}t=w@qrf89}{7ipJ03e4Ybjaxo>yths$T2%KX@D}Jdu!lqNd!0M+lkwyMe zfq;PDs$v>myemcgJ;M!ixdg7UeyWJESV1dOMVPGVtbCh*5a3R@o3@-F$LSx+Q6E$u z$%E1A`n{hJ;a)v zYF@6`31J@!e*TcF9oQwsRaYtQkIP7fx25)An}6SdWYs&Tv-R*%A(j0~&P}`Utm4{( zUV52&ezx479(*!E3`cxU82}H zO^6^94ZkY*`8WY!x{6L+9F({TOxTM^1Djm1H&5gxY&2JESHmW1`-=xj>=FbHv^)oH zsy8uPTL!)dwc&&b4|G*~IcDL0j)-y$cUiN6%7Ys=ZT80Px|0`&;|r*XyLt|o<1s3n zre(ZXDEoj#Hb{&}lY{=nhU=MNf!ZUwOx`QEget86n2Vnaz464n8lx5g$*Apn99#vD zvd>iZQdWebRjb=D+3B;R(hB8MB$)Vx00+RMCmT(^oH%<8id=i1L1(dUZBD9|NX6oa z)Yl{LZ_-mv?#hMY#wFs5uyB4WmrTS?a=iqrIJA^Fa$u($`t1q2|9xnu_(a%1A zu}}k)Hw7h1S||oTcn-QErPOABJ<@jlm!vj$0CFEz=81u8IXu)usHm3875gz~KR%(F8k+QD-M_3)j%FU2#OWOxvOenmyJwaW2_gf1Kya5ltc!9KoN%z*%DU zeh!^}+Idr53ZfgKe32fT^M=Ep+wP0OPC4J8hiD-`jQ6*{o&rlVMhuXKf9LuARc_?7 zQAUf*V^ar4q$dyRS*Qh%UN3cDM3Iy<~6K-QJ>9qx!GJ z*#RjA9N@%Kz2ysM0O}QqYUra`Je4JKW|MZ>WWC#agkDwo{1@=M*JJ9Fl0o{c#@ovZafq+1n#FzDpMUmBix6sh%M-)0J1YNI3oZow8SwDhWbb! z)UdX9ITWgIg3%N-tjlp``Kec?915cIZV3+;4PqYb^xX!+C!;TJy+sH+wFJgDx48ruT zQR^lR<4U@><~5|i|7l&d$?&%?ntR(W`~H z@FuDPWuj(cExwT$%%AX`hX>ElI%d&mfn$UD_!*mzd>`;KZ}g zS4B*bnznBHXx_xxn0WO8=6Erjv38e3e1j0H9faWYt=?BYDOiT;D3_sW%Co*Dx~L$} zd5vhe*YV^4-82oZK2PtHzSR&TAL3$ss$nj8v`2O3GCY1%qCe{ zL#Q}u`eNSNC4(KMH~MaXfp26vb9IX>>nJk(%1KmsRz49-jO1Euf`x>+3r$hwQ|OMY z$4e=S{kMG<$aZYvV-A$Jir&}#<03rmSc;ka8UeolR{+~5B-jIBsolv=IGhCP3Y*=9 ztyyRm!NWwJK>JrPJ(YDAvbMf)!QoTQv1_yQ( z#ij{(16i_&9mlvk3?J0qsZKWr7clP{;BdgT?4Dj@TPO+@UEU2AGM>jenbcn0eG0V= zu1s_B#4W%s!(Vrqa^_OT=Liy2x^o(>W)=`YgXLKu_B^xR5O}23eaVq{vMM?CB0Y+g zR+9Hw5lifbv;|aDAnLHxg9xlRw{E!-Z z5Cr-Kq)(wf)Hi7MLIORhl)u~^J2)ZM8haENd%(`#L*{_!zuY~AaJSCnx~1fZl}C2n zVMIbxYxx!OW{m#;NcJji0PO1h%GFQy9Uf{FKmgrD_mw(iupN`y70S(Z7K}Qc3WAVv zzu>Ggqnn|Qr3|}dxSk5b;sp5+4P%W%4!G+1BQnOei6c-)%_|d07wA!}aWH}DbvA;N z47e-JkVU$#s?lqOMw6h`b{zKMFm@GK5MF%cUqQ7FxVe+s2gojX)OMl=Lj9jViU8LA zNAOrfr9^@`9+U$hi|ak87uJ7eV@c8@vg;ytIQB=eP#TLs=@#p{2<7;D7MKI~{84GV z9R!~g)rVoCUX5x2`_t-bx{#yG+9s0kXh^g;tg{Q+7h(mpC(-Gl?jQ}IhU zN9}rYR2%89l%ULOJsw12!XP)V@krk>rwsOuW8h&>I(XmHnNtVS2bESe>B#RMqvcY@ z40oG9w9)NI1Mjeo!TY_c1h{OTiSnt~pVB(MV3m#sLJ8t8@*zM+TiARQ9>PY=Auti8 z=d=&_mb65UC}+B*EI z<`Q&{m_IpD4FLDB^j>;_KqZ0Oz(`Z9=z-FjTt{M75c58s)Zz?dt^%-F9jc@ZIZ5?g zG$U$&uucZylVJ;<#$!71RwR~^b%d%y36ura!)>apee>F&_c$ATmbA%K7F{K#+c_*_ z8rKI2k#yssRc_NOC>N@{xsDtaSKXv`{Fb%yVGd(-x~LrBRsG)&npWmE{1v=>5wIn) z(F2baq#nc*9C{DngWk{+9D|euMj>fTyzKZPvNZ&Dk10c>Z5;QX^?>_Pf*MmQ3i-I}j2$L}}Dd z&dW!8@E7@eR7N4gw4(c5Pe>j_1mF)ib%nT?G{rpTett!PvOjm>^Ghtl`lHi=fMlL0 zpsm7fAoR#eB>BgoNu-V_21g0(1bmd{)^TTF;*fEjkNCnOH7)28de9nN1o!8cq)L-5 ztn}GHM{g^60+EMn)d_F9 zJvvoWORm)fhxWyD?~CZ>-2~~VJ~fo z2dH}xNw>P$*{{s4Yev_6{efeI8N_;ja>yBIpYESxfGiprJxOKL%=ALsBn(&d=s-%^ zk~;qYa$^n|j4|40=2Qbg6DO!+E1qfrKH#nAgIfy zTz#|bAy7zw(KFSH{{WLh*LRu;KQAf+LDj7v>ORzWT4YG;kD7%DbE55m%H>7CM|a3k zRE}^uA#=`X07F_26Y1nyNF5MJ9^q_iPM;M*rfsb@@BF&_i=9#KdAZr3uvJ{>Kj>W* z3Y3=WW@=R1gmChzw9kL7K)7M(A0=LyIIVbHbl`MUlc-O9X|yV%?KX{zU0V?rAx$OF zU8w902eagvN~tWw3EHVvaB~N5m18PJI88VONvY6nN4se$&8pzJnP=R}wK7&6PIg7% zRF6WKDq`p9+Ney~8^KRg3pz~0E$^Bg2N%GPigq^n{{RE>i1@BfCb`qBm1X1sWYQqK z3tjq}IuDeoTbNWSI%Prbhbg2%aEr_!F@%rD%BnVFRG{M`>7*T;4Z>hs{a^f&bkpN0 z2BVrL-9yNf+V0pJ+i6WA3ug(`A5Ma_QenP0?T;!!s_0tG)*YUo1tEwolWR{^y27YS z&aH&c=4rcO2dmmviqze=tds0gKE#0{#Xl77L3K0whg^FUVW#ba(hxx+dlJL%r_C}P zKTDPX!h6kd=swU86R{gdV`W3SKUY}4%~&jSey^q9IQFH$>Fu8?fQ~by0zEmRLE*#F zRN(9f;ytjYDTD79bp8cELJyjH?2viTpQ}r>u zE{}wSxpBS`@sFCER0Fzj^a(vMv8}93$vwW7Pe4K{4WJHyJ}N*GdG>ivl~{4Xt6X;K zl3ZP>BmG{*0TUz8B};i3Q*dW6Iusfw_53IBS^~pN=&ZG`ks=lzA1}p8u8w5@y5c}s z+d`FFfe8a%#DG|-Qvl|xzU^u3`6(9f2mlib(WC=Gb|~yXQX!-Z`gA5XlMTzj>nExc z=8)Ox(6@Neqf^uhYgpTYdG0?lXx2roTge8-HB}=?SI6(cS5&`#{38!{M z13&3d5gJ%FJ5+`b)r+B4?hiR`@V8+=T{u9N{90vKus-y3BAbhjuK9e#C)W3L>m$g3 zS$&|Ep1_b)n$|yH2Z2p#q{hJ*xR(_`kxahgdWyAnHZZVp61kNtvY9v!LrFh1SNW>7 z3AqE)YTBzQeeY`wmvkz%Go*|mt&U^@e2bhNix&hp-053YgHSNI>#DBf7|#?q+IeWI zVkTR(u8AEhSuJ9_zhPCV>;C|x1klaoXfBpvtt9H#7W4(UHmL%pOdm*2qyTh*$t9G_ z!8%3Nicbu5#g#fG1F?fp0yg^<<~E52TG{MW2^AkL5k5rlD-E8V3L6nkrR0a{ezZD} z1c2paflN-as=2SI{_UpT#W)cx2*&Kppj0!dgvVv;`K0v9&C$?^Jzm5ni5tBjOLbX# zy4dKT-{72AI}?RK`@|$D6{1i16di!aB==Ot*czs}w98F|$1Sj_QU?}_5#`oPRGe8e zZj}m^9f-z>rk3+;M^Cjd4cl;Q9@L7yA?4#SC5m@rPxOURuI~hwQ>!xBxtkt+DPfhE z?(Xyo#l(X4-l?#0cbPBDII?|ihXEp`HxSPElX9cR0`eKw?1IGh1hT~oMXroTgeMrYc%IJl%q z21SV4nj}H%SZ(j%9Dpa!%!A;UcMK=6T#ZXo?8?u?{cd!JwF4w6C2-U>)DKel2(twj#c!i`> z8oZv!37g}vdlhH58@Wn=m%Q%lqT5u!4gF`KL9|J{sqVo2eNd@jB!5xuG4fAsNBOWh z-`cCo8>S*u37t!i1tLg2)iGKg?!jcJiZK19G6(EyhnGz*bi#0UL0N9`XZe9{@3 zs^{Ful z%Aq9eDgmN>-)e*>-QrZDb}7v&VooloP;Y$ut^?DYAT&}gDaDQFA2p7Kj4($^uY_y` zNd&iTZRf2tRI~*h(RT$0Qgbhh;{`xXC%8p8o_rE z7P_MyZ+oFua0S{Pw7~aM_e>xb_J-WC!>vt^9LVTHvri)CdlZ`5j~IhU;}EaR3VtKB z^sHm5Ghlb8T4`)*R9jFB2v7{ii{4;wN1;+Tupo6QhQ6S+ho?%Q^SM3tM*>xXmhhTl z?&?+ETM|f)Dz!}5A{xf_l|re+(r40E^n&N}EMe{dPi4HStE;}G%nkunVAX3~a=Ii~ z0e3BF@4{6#OXG*NZ1>Mk&1}>QVfWUypbS-{+yw(*wFhlK!C<(O5xXJdnUuCVHQS=J z$3xnAN~cPwG&l#C*b-k7F=$RC$5Z|xj#6bNse#z3pnLnyjUXh`J?rTRCd1-`cR*By zoLGCoTBlzVRTq1ZQz+7qV8zrlS!?l1TZ}q%Nqb*P>2mX^iQ1zFR-Ry;VOMou;_3R8 zW{d?{ZVA4sK~AYOpQ}ewp;$4l?tlWrP;5#ho0Sa~aCc@?vd;~9o?g(Y2SiD-^&S`C znV%Ae(*m6@wRWHB8oQ$0RE>gT6O~jKL13Q6$f;7Gahd5~p*%OjGM*aPV?*A3>W=`p z{{VDw9Wto+x8)qBcsj0I{#voW{{V&Oi+@jRAZ;M7o4X^H`mOmN9c#O-DYa^cQh3^0 zM_O)s!aPUAABMd$2kCoz)ZN(O@-XfTnA~KBxEMBEpzSbkC>I8PH23nN?i3~917Kt&UVW-6; z&WJ%r&nuD+}}>cUD@9^$@r~lW|BG%sN}hGxvp8wJm~}Cl%M0o7ikSTVN~^HeF2ai;?m$>g~igRW~di(jMnDK-2cE>m6I1bSUw zr(M6ObnIN3gC&Cd*h-jlyA<)ZEeR#1+hJjX<7u(lnNutc762+tU}>3>tU0bCXRhl# zq!s)}9bqlh-LXOVhPFoe!^*V`5@36Fkf!HUcGC$WrXzBsNgt_50R>@WiiJ9DsO!Ey zYo~#X(Ov~Pp|5F>-k6nm<|;rC2N?3Ld$5gzwQ{Ov>9T6cQoR&(jnH!7Q} zz1$Tdc&>_|=(|5wA%_U-KNU>4y5y30l>#6rg0}$DQ15kV6Lmnn#nH)aBciO9*yvT- z?(vlzU<)Z06A%`l>sn#x9qM4bZL)|i67$*GtTbW(9f{e1(@Cusx~$%rMPkOb{PwEy z9_0G1E-Yw1mQH;p(WlJw@*%Sg8)7>&3L-V+6CUze90$o$z92R#QJBj~;;T59ItOV^ zBY0cdR=i<0(1`Y+E0g>#pOTAC)fbx~q!S9687?lmwAWx&OQCCJ!geTGIw0>XR__CQ zk*B96M;QvxaC12JAxKpQ1aVer(rcL0Bp*$RrZ){#+gE~vxSdw+fF|;~nGCbD#8anC z)o~|4ekWk*UZ2CiDQ0tRZEWs36vmX|D?79w_Qz-^fL}(+2{WI_;Y>C>vx%8g^%Y+B$5J)qfmFBSZsN~m>D4{)UOb4BO)XZ9Tgx3I|K z7tmoeXqrxXU=#FfI-X!KJtJ~$sNrBG9J>aL8# zAF%Zs7T~sKw z?0k{IXJ9X!u-N0(`;{}3c)Ww~R`-tPQN1`yAmFE=Qj!v;>EYpQf*;;DOru8>-D8g+ znMvicqbaYVS_zS&=bB~mM$L<-QRU1dm|W@|*_+(_l}1`UyfAV`CNwWPGc``oJ3kbn z;_X~Y^Jr(!gS}F{9UMnR{z{9plv(amB=Phsm{LhwRR=-Y+K$p2*gUG6sy+I+l$=Sy zp=uz|v?e5#K_xV$IZznll$)tqdlf|zUK*0>n3P{Gd4(9l7U1ZpOA=r_Rbo~WkTef@ z#bu1{I25KUEf|O#)jFktc7%fCu!#wv%_pkQC9P=XWhOppJS;W*9NlrBxoVgDJbx6? za0;hQ#4z$#D4C{LHoMMNP~XhB%7g^gz11u%RVJpe_m?(|9g0+H19CmSMH;6GjAOD_ zHl}8#EHfkGtW&3o^JSerV6(dfT~>ZxPj%gD$99S#ankU>5+jHbw(%#Tq?zBlgB`oJivwPjTc~>dVCgjS9 z)pXKQ^Q%6lrE|$v!sHvRDx0Nb>VqLv2I=9@(b%YBmk!mWf>S3=+9a%XNJtNqf+RT9 zYeXKUS;EpbtSv8fm6lk?CAd8_NTO`7Q#{h@w-bkAyHcFBp`&g802Dfu>AqfiQ{Xt^ z4@+5Ey1}qG(}o60I?ju}L`E#X#Qm)9l#;M8{LQ%mjpz zEW)eCkfn+3VxAc=oIwH-e9dbiPHO|DtiF%=8jC-g&6r8C~zrww}a$7S(tNFTJsRvg4G%OFBjtmTg)$v_`l@5 z_Cz|aQ~qv<6Ackrql*M%?PdP}q%XS)(jnbyO@gDUXKLNl{F5bz%BQ^>LiXRAJTRDKx4yxxJ|XC2GIi zBVt*k!|CNzuvZlsp-q+ ztxCg3lFJiaqO4OMY^fI&QY~u7=&nlUVqFOc#grOKTEgI~I;^OSq*+nKs^cauohrd& zNICdINvPwtI~2&pWg$#0Dw|M6w-Q#7h$10K7*y_v3Ov$YAjhVP*{KNxNt7uHZ6w0H z#ugAx#HucbDcr^3br(Ig10N4UtUuk^_OzrxF9(M}ve;_q~EyzpKRIr)z}o0hPQvpF;0>3+RfP_Eb?M$UMy zPar>ySG^Xd-t|M#v$fC{hR*(H%+{9WnlkM3j5+78y55WNUNFhlPfFEJdXJ#^XXfn1 z95nNze$TXT`>pNR@zi$7Ur)+svf1i;%)M6B8({aF!NA?8feZ6`wU^CT$Y^s&BD>X!Ft4WFY zq$S{$gvF4SQZ`ZwC1uJ$RRl_&5~LcmY7AlTT{P4KM5l-f0+ZdK@+%~aM+Zt4ntQkt z#>)o9B-9eZR2^j8Kq&jl6oYk68uX~xDB@71*E_$6aQP`Z92-2+*vw9Xtg@Zl4z^j_ z1^bbC6C4$HS!H5zG=!4uNd$P7sJiE&S$#n2P8&wfkf>936*iCJN!uzka?|m8K-M_z zT|hx+LghFHD|6hpnz3YZ%&sPql%Og~iE@x-p-*rGsZw;}=v1F~6#7pT!6%`0j1Ibf z665{VOBK@JS zlF;Xy2`F_jx~ufrjG$$M8eHp`>P07j^Vop818Iv0T;gt3hdxy^B-sEOxF$7I4j@U| zlJ*- zG_w9m5PGidR7=Xiz)GOR_UNSCy+;mRzC|ZD0gX_mC$@}7aOMpwumXR5Eq*KF9mc9gaQ{J9_*`4soe`E zj4Z`usSQe^aEMWwM>cSa|YUbSST3DQ|Tu!z}!?QjiRn+p- zcg_rTR}JMFL97mAONp;in|he1~Geukcv`DF)4*{xPhDOjFc`7GX_%I5z7?dy8AdvTcl%bWiIwy)~fH+@^}t55zb{{XzbAA~*6 zZG2C2qX-zB=SOm{GN-|37?xV{I7cU&bBX>}H?Q`3-mP?L(5k^~u5Nap<+C2GahY1V zYe}nG;2-xN{pBs~WB&kJxA~9$@;g_CJ_Vb7j=!xC{S|b?SDs~R#h~wNNdx|){{Xy?LP_J` zH8Y`8Etq?O{#!Qye`c@h)nSy)lYFLc{@%Zt`6_^7b;@m}CW@7Q*01@MKfI08@8Bhu*_X^W{{W@U{{Y)p z^-5pM<_@`R+5Z6e*^gGQ_`}@FltWzRf-1;soZTP&CyL?kDv$cLf6S@<<(n+D;p%>e zEhk9OtFtYe{{XhH>XvRQ{{Uvq{{Y)&JzBfr4lZqST;~ynF*(kTPXD$NT9QzTNUU0-sj@;rkW8;<;h&l_SvY25a}xGa0|;^3M)tVYr;#9LkNen&#>z&Yh%~UT2cHZW5uLi_LYV zwqd+Qz$y+2#el4^;;@-j+A3lw-i@kdiDsKmHKV4ZT;Ojqv7|~{m|76fLuFHG$=t5P zN%bj_p*>p{B1zSI-2PsjMi-IZ8%C?aIU0t-a)`oXD7pFtFa$1@FZRO8qO)?oaaNkf zH$e(=qWd99ST0?6jAdCQsuir{1To21MByEZgG7~dd!YzOl#=ArwO3St0V%?nI@wBy z5PB0_p-c@8NvI@+u5nRi)m%)z1Tesqf>Wvz8haHIw+1BY0W2$6Z3`NEgZ4sk9mXA| zeXE(+Ymam*6e>aBS2?7?98+?)E?H=$`T?(~;BO({HCDX`(R}I&*VAg;ktZ-m?R?i} zpqg%;MP*=VoceCUZmC!Q0Ce;_Kvh?FMMq~Ux!v6s5 z=>Gt9UMU`H+AnXp^7&r3^MC&U_bX4E{{YE)L7Sx`{{Zv+FaFAhKiOU<$NXBq_>1bg zmR_Ukv957_AnS)bT1}^Fw5Jt249vCKadxjo;Ic+yBlOt*#Z~410LH5S0A|U5kUXi( z@$A((!%HgJ*9|N)VlRclbt`qtizg2pjn@9nk^cZ7c~pPKs{a6H$$yYM zs~e~0B=o*F{_PIR{{Z@ddEJX&qVn8irDQzzbWj)fobs=wK?U*r!e-SXz# zM4p$ytlY=?f8eUTwKENGU{4-Bnq=Xnm2W&Wu+50R7Y=DZ<5hpNWWUHBRUTW=l<<%- zdLJGC0CtD$*8c$1UU&Rzul8)0`2)(D{{S7#RE%_TuhubAE{QT?UANe+a^kNZAVDKV z>HHSW%nfG`{1sQ3{{R`Q{hKBJK=P(H$Ma10Xyscv;iZOc7sBB>l_{3hmLf<~_gmN# zE|ffOxYI9^{{SJrRlnnP{{XY(zsMd{ozwF~Jui($`j2IA_{NKZmhT?TRE%`8uh-tC zi&H1wan@4*0LJToXUTt%JgZmBn{grPd?Vb{L?+!SyukdEHh#+ zg~ORj{{S1U{huZNK=P>a+oDs#K;_zWe>a@}0LORy3|@QuZnyS)m-z$AuH|b#HY(Mr z0o5wFyA6l9-g2>Ox>DH8!7ae=VqqQNp;D6?()PH4J*uNzQ5^4oyjhCjHMCgq-FzP3VylROWSNDKc_1ChJTsbe{o~qDzOmFWfV80wKt(mF zNv&EmNjjdXDb)2)g@ub&ZCWUz&PfCneJSBUQbfWQAqGl;xmcdhb<37#0zL&yF`|$s zYH3hjQw9*CKq-?d1O;TJViefZQR0~WL=P%x0NHv>jS6(zj0tpVZo1bKb#W*E*{z9Z Al>h($ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000038_action.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/humanplay/000038_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..f4d900f753ef5c6600e16259b4a86009cb2bb8d5 GIT binary patch literal 2866 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?smr+-z@SGguQa!yQZGI^zbI8Nw=}0DGqI>Bv2scePkd2o za(-TMNl|HX$&|@cdRW0Kr}VJrr9hNSp5o2WI>nhWX-fN)peY*OjG7tDJ*+7ul?ACF z^-TU2Q#v~!Qp`P!HB`)`UeZAS*retvUFr}Fwn`VP%8Ux5QCku9vryA@benTfm#UUX=OgJE`8D%3<5^`7| zL`TycEG$ODhj1Rl7DgbkQE^BZjD`dLaDe4CWExwTf#iu7p8~8wdsuVxON&#nR>Mrd qYFMh%DT4`O?b$PD_5!gjh&NPesN44rWg6u6;R--WvNRD$=m7vsu4$J5 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/launch_preprocess_slurm.sh b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/launch_preprocess_slurm.sh new file mode 100644 index 000000000..c1bc37b5d --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/launch_preprocess_slurm.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Create output directory if it doesn't exist +mkdir -p preprocess_output + +# Launch 8 jobs, one for each node (Total 64 GPUs) +# Each node processes 8 consecutive files (64 total files / 8 nodes = 8 files per node) +for node_id in {0..7}; do + # Calculate the starting file number for this node + start_file=$((node_id * 8)) + + echo "Launching node $node_id with files merge_${start_file}.txt to merge_$((start_file + 7)).txt" + + sbatch --job-name=mg-pre-${node_id} \ + --output=preprocess_output/mg-node-${node_id}.out \ + --error=preprocess_output/mg-node-${node_id}.err \ + $(pwd)/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm $start_file $node_id +done + +echo "All 8 nodes (64 GPUs) launched successfully!" diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_wangame_data_i2v.sh b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_wangame_data_i2v.sh new file mode 100644 index 000000000..85a4fd0d2 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_wangame_data_i2v.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +GPU_NUM=1 # 2,4,8 +MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" +DATA_MERGE_PATH="mc_wasd_10/merge.txt" +OUTPUT_DIR="mc_wasd_10/preprocessed/" + +# export CUDA_VISIBLE_DEVICES=0 +export MASTER_ADDR=localhost +export MASTER_PORT=29500 +export RANK=0 +export WORLD_SIZE=1 + +python fastvideo/pipelines/preprocess/v1_preprocess.py \ + --model_path $MODEL_PATH \ + --data_merge_path $DATA_MERGE_PATH \ + --preprocess_video_batch_size 10 \ + --seed 42 \ + --max_height 352 \ + --max_width 640 \ + --num_frames 77 \ + --dataloader_num_workers 0 \ + --output_dir=$OUTPUT_DIR \ + --samples_per_file 10 \ + --train_fps 25 \ + --flush_frequency 10 \ + --preprocess_task wangame \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm new file mode 100644 index 000000000..f60659a7b --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm @@ -0,0 +1,61 @@ +#!/bin/bash +#SBATCH --partition=main +#SBATCH --qos=hao +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=960G +#SBATCH --exclusive +#SBATCH --time=72:00:00 + +# conda init +source ~/conda/miniconda/bin/activate +conda activate fastvideo_kaiqin + +# Accept parameters from launch script +START_FILE=${1:-1} # Starting file number for this node +NODE_ID=${2:-0} # Node identifier (0-7) + +MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" +OUTPUT_BASE="traindata_0204_2130/preprocessed" + +# Port range calculation +base_port=$((29500 + NODE_ID * 100)) +gpu_ids=(0 1 2 3 4 5 6 7) + +for i in {1..8}; do + port=$((base_port + i)) + gpu=${gpu_ids[((i-1))]} + file_num=$((START_FILE + i - 1)) + + DATA_MERGE_PATH="traindata_0204_2130/merge_${file_num}.txt" + OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" + echo "DATA_MERGE_PATH: $DATA_MERGE_PATH" + echo "OUTPUT_DIR: $OUTPUT_DIR" + + # CPU binding (optional, kept from syn.slurm logic) + start_cpu=$(( (i-1)*2 )) + end_cpu=$(( start_cpu+1 )) + + echo "Starting GPU $gpu processing file merge_${file_num}.txt on port $port" + + CUDA_VISIBLE_DEVICES=$gpu taskset -c ${start_cpu}-${end_cpu} torchrun --nnodes=1 --nproc_per_node=1 --master_port $port \ + FastVideo/fastvideo/pipelines/preprocess/v1_preprocess.py \ + --model_path $MODEL_PATH \ + --data_merge_path $DATA_MERGE_PATH \ + --preprocess_video_batch_size 1 \ + --seed 42 \ + --max_height 352 \ + --max_width 640 \ + --num_frames 77 \ + --dataloader_num_workers 0 \ + --output_dir=$OUTPUT_DIR \ + --samples_per_file 8 \ + --train_fps 25 \ + --flush_frequency 8 \ + --preprocess_task wangame & +done + +wait +echo "Node $NODE_ID processing blocks completed!" diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation.json b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation.json new file mode 100644 index 000000000..3a4a47115 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation.json @@ -0,0 +1,14 @@ +{ + "data": [ + { + "caption": "00. Hold [W] + Static", + "image_path": "doom/000000.jpg", + "action_path": "action/000000_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 77 + } + ] +} \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation_vizdoom.json b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation_vizdoom.json new file mode 100644 index 000000000..1535db987 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation_vizdoom.json @@ -0,0 +1,84 @@ +{ + "data": [ + { + "caption": "00. Hold [W] + Static", + "image_path": "doom/000000.jpg", + "action_path": "action/000000_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "01. Hold [S] + Static", + "image_path": "doom/000001.jpg", + "action_path": "action/000001_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02. Hold [A] + Static", + "image_path": "doom/000002.jpg", + "action_path": "action/000002_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10. No Key + Hold [left]", + "image_path": "doom/000003.jpg", + "action_path": "action/000010_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12. No Key + Hold [up_right]", + "image_path": "doom/000004.jpg", + "action_path": "action/000012_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "21. Switch [W]->[D] + Static", + "image_path": "doom/000005.jpg", + "action_path": "action/000021_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "29. No Key + Switch [up_right]->[up_left]", + "image_path": "doom/000006.jpg", + "action_path": "action/000029_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "50. Hold [W] + Switch [up]->[down]", + "image_path": "doom/000007.jpg", + "action_path": "action/000050_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + } + ] +} \ No newline at end of file diff --git a/fastvideo/configs/models/dits/__init__.py b/fastvideo/configs/models/dits/__init__.py index 571d7f4f1..9032c6c79 100644 --- a/fastvideo/configs/models/dits/__init__.py +++ b/fastvideo/configs/models/dits/__init__.py @@ -7,9 +7,12 @@ from fastvideo.configs.models.dits.stepvideo import StepVideoConfig from fastvideo.configs.models.dits.wanvideo import WanVideoConfig from fastvideo.configs.models.dits.hyworld import HYWorldConfig +from fastvideo.configs.models.dits.wangamevideo import (WanGameVideoConfig, + WanLingBotVideoConfig) __all__ = [ "HunyuanVideoConfig", "HunyuanVideo15Config", "WanVideoConfig", "StepVideoConfig", "CosmosVideoConfig", "Cosmos25VideoConfig", - "LongCatVideoConfig", "LTX2VideoConfig", "HYWorldConfig" + "LongCatVideoConfig", "LTX2VideoConfig", "HYWorldConfig", + "WanGameVideoConfig", "WanLingBotVideoConfig" ] diff --git a/fastvideo/configs/models/dits/wangamevideo.py b/fastvideo/configs/models/dits/wangamevideo.py index b43442543..429d7407c 100644 --- a/fastvideo/configs/models/dits/wangamevideo.py +++ b/fastvideo/configs/models/dits/wangamevideo.py @@ -113,3 +113,10 @@ class WanGameVideoConfig(DiTConfig): arch_config: DiTArchConfig = field(default_factory=WanGameVideoArchConfig) prefix: str = "WanGame" + + +@dataclass +class WanLingBotVideoConfig(DiTConfig): + arch_config: DiTArchConfig = field(default_factory=WanGameVideoArchConfig) + + prefix: str = "WanLingBot" diff --git a/fastvideo/configs/pipelines/__init__.py b/fastvideo/configs/pipelines/__init__.py index 082e533e8..c22038d4d 100644 --- a/fastvideo/configs/pipelines/__init__.py +++ b/fastvideo/configs/pipelines/__init__.py @@ -10,7 +10,9 @@ get_pipeline_config_cls_from_name) from fastvideo.configs.pipelines.stepvideo import StepVideoT2VConfig from fastvideo.configs.pipelines.wan import (SelfForcingWanT2V480PConfig, + WanGameI2V480PConfig, WanI2V480PConfig, WanI2V720PConfig, + WanLingBotI2V480PConfig, WanT2V480PConfig, WanT2V720PConfig) __all__ = [ @@ -19,5 +21,6 @@ "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig", "StepVideoT2VConfig", "SelfForcingWanT2V480PConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig", "HYWorldConfig", + "WanGameI2V480PConfig", "WanLingBotI2V480PConfig", "get_pipeline_config_cls_from_name" ] diff --git a/fastvideo/configs/pipelines/registry.py b/fastvideo/configs/pipelines/registry.py index d4f9dfcba..db6e659ee 100644 --- a/fastvideo/configs/pipelines/registry.py +++ b/fastvideo/configs/pipelines/registry.py @@ -23,7 +23,7 @@ Wan2_2_I2V_A14B_Config, Wan2_2_T2V_A14B_Config, Wan2_2_TI2V_5B_Config, WanI2V480PConfig, WanI2V720PConfig, WanT2V480PConfig, WanT2V720PConfig, SelfForcingWanT2V480PConfig, WANV2VConfig, SelfForcingWan2_2_T2V480PConfig, - MatrixGameI2V480PConfig) + MatrixGameI2V480PConfig, WanGameI2V480PConfig, WanLingBotI2V480PConfig) # isort: on from fastvideo.logger import init_logger from fastvideo.utils import (maybe_download_model_index, @@ -98,6 +98,10 @@ lambda id: "wanpipeline" in id.lower(), "wanimagetovideo": lambda id: "wanimagetovideo" in id.lower(), + "wangameactionimagetovideo": + lambda id: "wangameactionimagetovideo" in id.lower(), + "wanlingbotimagetovideo": + lambda id: "wanlingbotimagetovideo" in id.lower(), "wandmdpipeline": lambda id: "wandmdpipeline" in id.lower(), "wancausaldmdpipeline": @@ -132,6 +136,8 @@ "wanpipeline": WanT2V480PConfig, # Base Wan config as fallback for any Wan variant "wanimagetovideo": WanI2V480PConfig, + "wangameactionimagetovideo": WanGameI2V480PConfig, + "wanlingbotimagetovideo": WanLingBotI2V480PConfig, "wandmdpipeline": FastWan2_1_T2V_480P_Config, "wancausaldmdpipeline": SelfForcingWanT2V480PConfig, "stepvideo": StepVideoT2VConfig, diff --git a/fastvideo/configs/pipelines/wan.py b/fastvideo/configs/pipelines/wan.py index 434839dcc..b4a76034f 100644 --- a/fastvideo/configs/pipelines/wan.py +++ b/fastvideo/configs/pipelines/wan.py @@ -7,6 +7,8 @@ from fastvideo.configs.models import DiTConfig, EncoderConfig, VAEConfig from fastvideo.configs.models.dits import WanVideoConfig from fastvideo.configs.models.dits.matrixgame import MatrixGameWanVideoConfig +from fastvideo.configs.models.dits.wangamevideo import (WanGameVideoConfig, + WanLingBotVideoConfig) from fastvideo.configs.models.encoders import (BaseEncoderOutput, CLIPVisionConfig, T5Config, WAN2_1ControlCLIPVisionConfig) @@ -112,6 +114,20 @@ class WANV2VConfig(WanI2V480PConfig): image_encoder_precision: str = 'bf16' +@dataclass +class WanLingBotI2V480PConfig(WanI2V480PConfig): + """Configuration for Wan LingBot image-to-video pipeline.""" + + dit_config: DiTConfig = field(default_factory=WanLingBotVideoConfig) + + +@dataclass +class WanGameI2V480PConfig(WanI2V480PConfig): + """Configuration for WanGame image-to-video pipeline.""" + + dit_config: DiTConfig = field(default_factory=WanGameVideoConfig) + + @dataclass class FastWan2_1_T2V_480P_Config(WanT2V480PConfig): """Base configuration for FastWan T2V 1.3B 480P pipeline architecture with DMD""" diff --git a/fastvideo/configs/sample/registry.py b/fastvideo/configs/sample/registry.py index f8253781b..a7986aa82 100644 --- a/fastvideo/configs/sample/registry.py +++ b/fastvideo/configs/sample/registry.py @@ -126,6 +126,10 @@ lambda id: "wanpipeline" in id.lower(), "wanimagetovideo": lambda id: "wanimagetovideo" in id.lower(), + "wangameactionimagetovideo": + lambda id: "wangameactionimagetovideo" in id.lower(), + "wanlingbotimagetovideo": + lambda id: "wanlingbotimagetovideo" in id.lower(), "stepvideo": lambda id: "stepvideo" in id.lower(), "wandmdpipeline": @@ -156,6 +160,8 @@ "wanpipeline": WanT2V_1_3B_SamplingParam, # Base Wan config as fallback for any Wan variant "wanimagetovideo": WanI2V_14B_480P_SamplingParam, + "wangameactionimagetovideo": Wan2_1_Fun_1_3B_InP_SamplingParam, + "wanlingbotimagetovideo": Wan2_1_Fun_1_3B_InP_SamplingParam, "wandmdpipeline": FastWanT2V480P_SamplingParam, "wancausaldmdpipeline": SelfForcingWan2_1_T2V_1_3B_480P_SamplingParam, "stepvideo": StepVideoT2VSamplingParam, diff --git a/fastvideo/dataset/dataloader/schema.py b/fastvideo/dataset/dataloader/schema.py index 50982b66d..422ba65d1 100644 --- a/fastvideo/dataset/dataloader/schema.py +++ b/fastvideo/dataset/dataloader/schema.py @@ -196,4 +196,6 @@ pa.field("num_frames", pa.int64()), pa.field("duration_sec", pa.float64()), pa.field("fps", pa.float64()), -]) \ No newline at end of file +]) + +pyarrow_schema_wangame_lingbot = pyarrow_schema_wangame \ No newline at end of file diff --git a/fastvideo/layers/visual_embedding.py b/fastvideo/layers/visual_embedding.py index 9d9f0e20e..fe8792dc9 100644 --- a/fastvideo/layers/visual_embedding.py +++ b/fastvideo/layers/visual_embedding.py @@ -60,6 +60,61 @@ def forward(self, x): return x +class WanCamControlPatchEmbedding(nn.Module): + """Lingbot World Patch embedding for Plucker features.""" + + def __init__( + self, + patch_size=(1, 2, 2), + in_chans=384, # 6 * 64 + embed_dim=2048, + bias=True, + dtype=None, + prefix: str = ""): + super().__init__() + # must be 3-tuple + if isinstance(patch_size, list | tuple): + if len(patch_size) != 3: + raise ValueError( + f"patch_size must have length 3, got {len(patch_size)}") + else: + raise ValueError(f"Unsupported patch_size type: {type(patch_size)}") + + self.patch_size = patch_size + pt, ph, pw = self.patch_size + self.in_features = in_chans * pt * ph * pw + self.proj = nn.Linear(self.in_features, + embed_dim, + bias=bias, + dtype=dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if x.dim() != 5: + raise ValueError( + f"Expected camera embedding shape [B, C, F, H, W], got {x.shape}" + ) + bsz, channels, frames, height, width = x.shape + pt, ph, pw = self.patch_size + if (frames % pt) != 0 or (height % ph) != 0 or (width % pw) != 0: + raise ValueError( + f"Input shape {x.shape} must be divisible by patch_size {self.patch_size}" + ) + + # '1 c (f c1) (h c2) (w c3) -> 1 (f h w) (c c1 c2 c3)', + x = x.view( + bsz, + channels, + frames // pt, + pt, + height // ph, + ph, + width // pw, + pw, + ) + x = x.permute(0, 2, 4, 6, 1, 3, 5, 7).reshape(bsz, -1, self.in_features) + return self.proj(x) + + class TimestepEmbedder(nn.Module): """ Embeds scalar timesteps into vector representations. @@ -252,4 +307,4 @@ def forward(self, timesteps: torch.Tensor) -> torch.Tensor: downscale_freq_shift=self.downscale_freq_shift, scale=self.scale, ) - return t_emb \ No newline at end of file + return t_emb diff --git a/fastvideo/models/dits/wangame_lingbot/__init__.py b/fastvideo/models/dits/wangame_lingbot/__init__.py new file mode 100644 index 000000000..ad549c889 --- /dev/null +++ b/fastvideo/models/dits/wangame_lingbot/__init__.py @@ -0,0 +1,5 @@ +from .model import WanLingBotTransformer3DModel + +__all__ = [ + "WanLingBotTransformer3DModel", +] diff --git a/fastvideo/models/loader/component_loader.py b/fastvideo/models/loader/component_loader.py index 18dc392a9..27c9e135c 100644 --- a/fastvideo/models/loader/component_loader.py +++ b/fastvideo/models/loader/component_loader.py @@ -810,6 +810,9 @@ def load(self, model_path: str, fastvideo_args: FastVideoArgs): cls_name.startswith("WanGame") or cls_name == "WanGameActionTransformer3DModel" or getattr(fastvideo_args.pipeline_config, "prefix", "") == "WanGame" + or cls_name.startswith("WanLingBot") + or cls_name == "WanLingBotTransformer3DModel" + or getattr(fastvideo_args.pipeline_config, "prefix", "") == "WanLingBot" ) model = maybe_load_fsdp_model( model_cls=model_cls, diff --git a/fastvideo/models/registry.py b/fastvideo/models/registry.py index 7861918a3..32648f22d 100644 --- a/fastvideo/models/registry.py +++ b/fastvideo/models/registry.py @@ -43,6 +43,7 @@ "WanTransformer3DModel": ("dits", "wanvideo", "WanTransformer3DModel"), "CausalWanTransformer3DModel": ("dits", "causal_wanvideo", "CausalWanTransformer3DModel"), "WanGameActionTransformer3DModel": ("dits", "wangame", "WanGameActionTransformer3DModel"), + "WanLingBotTransformer3DModel": ("dits", "wangame_lingbot", "WanLingBotTransformer3DModel"), "MatrixGameWanModel": ("dits", "matrixgame", "MatrixGameWanModel"), "CausalMatrixGameWanModel": ("dits", "matrixgame", "CausalMatrixGameWanModel"), } diff --git a/fastvideo/pipelines/pipeline_registry.py b/fastvideo/pipelines/pipeline_registry.py index 0540868b2..11222e431 100644 --- a/fastvideo/pipelines/pipeline_registry.py +++ b/fastvideo/pipelines/pipeline_registry.py @@ -22,6 +22,7 @@ "WanDMDPipeline": "wan", "WanImageToVideoPipeline": "wan", "WanGameActionImageToVideoPipeline": "wan", + "WanLingBotImageToVideoPipeline": "wan", "WanVideoToVideoPipeline": "wan", "WanCausalDMDPipeline": "wan", "TurboDiffusionPipeline": "turbodiffusion", From 2b48211db475c1babec9eb7b3a91b85ff785c145 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Mon, 9 Feb 2026 15:46:05 -0800 Subject: [PATCH 017/214] wangame ode init --- fastvideo/dataset/dataloader/record_schema.py | 93 +++ fastvideo/dataset/dataloader/schema.py | 49 +- fastvideo/models/dits/wangame/__init__.py | 4 + fastvideo/models/dits/wangame/causal_model.py | 98 ++++ fastvideo/models/registry.py | 2 + .../basic/wan/wangame_causal_dmd_pipeline.py | 121 ++++ .../basic/wan/wangame_i2v_pipeline.py | 6 +- fastvideo/pipelines/pipeline_registry.py | 1 + ...game_preprocess_pipeline_ode_trajectory.py | 471 ++++++++++++++++ .../training/wangame_ode_causal_pipeline.py | 528 ++++++++++++++++++ 10 files changed, 1371 insertions(+), 2 deletions(-) create mode 100644 fastvideo/models/dits/wangame/causal_model.py create mode 100644 fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py create mode 100644 fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py create mode 100644 fastvideo/training/wangame_ode_causal_pipeline.py diff --git a/fastvideo/dataset/dataloader/record_schema.py b/fastvideo/dataset/dataloader/record_schema.py index 1bc86dd7d..0eea39adf 100644 --- a/fastvideo/dataset/dataloader/record_schema.py +++ b/fastvideo/dataset/dataloader/record_schema.py @@ -188,3 +188,96 @@ def text_only_record_creator(text_name: str, text_embedding: np.ndarray, "caption": caption, } return record + + +def wangame_ode_record_creator( + video_name: str, + clip_feature: np.ndarray, + first_frame_latent: np.ndarray, + trajectory_latents: np.ndarray, + trajectory_timesteps: np.ndarray, + pil_image: np.ndarray | None = None, + keyboard_cond: np.ndarray | None = None, + mouse_cond: np.ndarray | None = None, + caption: str = "") -> dict[str, Any]: + """Create a ODE trajectory record matching pyarrow_schema_wangame + """ + assert trajectory_latents is not None, "trajectory_latents is required" + assert trajectory_timesteps is not None, "trajectory_timesteps is required" + assert clip_feature is not None, "clip_feature is required" + assert first_frame_latent is not None, "first_frame_latent is required" + + record = { + "id": video_name, + "file_name": video_name, + "caption": caption, + "media_type": "video", + } + + # I2V features + record.update({ + "clip_feature_bytes": clip_feature.tobytes(), + "clip_feature_shape": list(clip_feature.shape), + "clip_feature_dtype": str(clip_feature.dtype), + }) + + record.update({ + "first_frame_latent_bytes": first_frame_latent.tobytes(), + "first_frame_latent_shape": list(first_frame_latent.shape), + "first_frame_latent_dtype": str(first_frame_latent.dtype), + }) + + # Optional PIL Image + if pil_image is not None: + record.update({ + "pil_image_bytes": pil_image.tobytes(), + "pil_image_shape": list(pil_image.shape), + "pil_image_dtype": str(pil_image.dtype), + }) + else: + record.update({ + "pil_image_bytes": b"", + "pil_image_shape": [], + "pil_image_dtype": "", + }) + + # Actions + if keyboard_cond is not None: + record.update({ + "keyboard_cond_bytes": keyboard_cond.tobytes(), + "keyboard_cond_shape": list(keyboard_cond.shape), + "keyboard_cond_dtype": str(keyboard_cond.dtype), + }) + else: + record.update({ + "keyboard_cond_bytes": b"", + "keyboard_cond_shape": [], + "keyboard_cond_dtype": "", + }) + + if mouse_cond is not None: + record.update({ + "mouse_cond_bytes": mouse_cond.tobytes(), + "mouse_cond_shape": list(mouse_cond.shape), + "mouse_cond_dtype": str(mouse_cond.dtype), + }) + else: + record.update({ + "mouse_cond_bytes": b"", + "mouse_cond_shape": [], + "mouse_cond_dtype": "", + }) + + record.update({ + "trajectory_latents_bytes": trajectory_latents.tobytes(), + "trajectory_latents_shape": list(trajectory_latents.shape), + "trajectory_latents_dtype": str(trajectory_latents.dtype), + }) + + record.update({ + "trajectory_timesteps_bytes": trajectory_timesteps.tobytes(), + "trajectory_timesteps_shape": list(trajectory_timesteps.shape), + "trajectory_timesteps_dtype": str(trajectory_timesteps.dtype), + }) + + return record diff --git a/fastvideo/dataset/dataloader/schema.py b/fastvideo/dataset/dataloader/schema.py index 422ba65d1..4ff323392 100644 --- a/fastvideo/dataset/dataloader/schema.py +++ b/fastvideo/dataset/dataloader/schema.py @@ -198,4 +198,51 @@ pa.field("fps", pa.float64()), ]) -pyarrow_schema_wangame_lingbot = pyarrow_schema_wangame \ No newline at end of file +pyarrow_schema_wangame_lingbot = pyarrow_schema_wangame + +pyarrow_schema_ode_trajectory_wangame = pa.schema([ + pa.field("id", pa.string()), + # --- Image/Video VAE latents --- + # Tensors are stored as raw bytes with shape and dtype info for loading + pa.field("vae_latent_bytes", pa.binary()), + # e.g., [C, T, H, W] or [C, H, W] + pa.field("vae_latent_shape", pa.list_(pa.int64())), + # e.g., 'float32' + pa.field("vae_latent_dtype", pa.string()), + #I2V + pa.field("clip_feature_bytes", pa.binary()), + pa.field("clip_feature_shape", pa.list_(pa.int64())), + pa.field("clip_feature_dtype", pa.string()), + pa.field("first_frame_latent_bytes", pa.binary()), + pa.field("first_frame_latent_shape", pa.list_(pa.int64())), + pa.field("first_frame_latent_dtype", pa.string()), + # --- Action --- + pa.field("mouse_cond_bytes", pa.binary()), + pa.field("mouse_cond_shape", pa.list_(pa.int64())), # [T, 2] + pa.field("mouse_cond_dtype", pa.string()), + pa.field("keyboard_cond_bytes", pa.binary()), + pa.field("keyboard_cond_shape", pa.list_(pa.int64())), # [T, 4] + pa.field("keyboard_cond_dtype", pa.string()), + # I2V Validation + pa.field("pil_image_bytes", pa.binary()), + pa.field("pil_image_shape", pa.list_(pa.int64())), + pa.field("pil_image_dtype", pa.string()), + # --- ODE Trajectory --- + pa.field("trajectory_latents_bytes", pa.binary()), + pa.field("trajectory_latents_shape", pa.list_(pa.int64())), + pa.field("trajectory_latents_dtype", pa.string()), + pa.field("trajectory_timesteps_bytes", pa.binary()), + pa.field("trajectory_timesteps_shape", pa.list_(pa.int64())), + pa.field("trajectory_timesteps_dtype", pa.string()), + # --- Metadata --- + pa.field("file_name", pa.string()), + pa.field("caption", pa.string()), + pa.field("media_type", pa.string()), # 'image' or 'video' + pa.field("width", pa.int64()), + pa.field("height", pa.int64()), + # -- Video-specific (can be null/default for images) --- + # Number of frames processed (e.g., 1 for image, N for video) + pa.field("num_frames", pa.int64()), + pa.field("duration_sec", pa.float64()), + pa.field("fps", pa.float64()), +]) diff --git a/fastvideo/models/dits/wangame/__init__.py b/fastvideo/models/dits/wangame/__init__.py index 7dbe1cb64..ebe540626 100644 --- a/fastvideo/models/dits/wangame/__init__.py +++ b/fastvideo/models/dits/wangame/__init__.py @@ -1,8 +1,12 @@ from .model import WanGameActionTransformer3DModel +from .causal_model import (CausalWanGameTransformer3DModel, + CausalWanTransformer3DModel) from .hyworld_action_module import WanGameActionTimeImageEmbedding, WanGameActionSelfAttention __all__ = [ "WanGameActionTransformer3DModel", + "CausalWanTransformer3DModel", + "CausalWanGameTransformer3DModel", "WanGameActionTimeImageEmbedding", "WanGameActionSelfAttention", ] diff --git a/fastvideo/models/dits/wangame/causal_model.py b/fastvideo/models/dits/wangame/causal_model.py new file mode 100644 index 000000000..1034ec6c2 --- /dev/null +++ b/fastvideo/models/dits/wangame/causal_model.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +from typing import Any + +import torch + +from fastvideo.configs.models.dits.wangamevideo import WanGameVideoConfig +from fastvideo.models.dits.matrixgame.causal_model import CausalMatrixGameWanModel + +_DEFAULT_WANGAME_CAUSAL_CONFIG = WanGameVideoConfig() + + +class CausalWanTransformer3DModel(CausalMatrixGameWanModel): + supports_action_input = False + + _fsdp_shard_conditions = _DEFAULT_WANGAME_CAUSAL_CONFIG._fsdp_shard_conditions + _compile_conditions = _DEFAULT_WANGAME_CAUSAL_CONFIG._compile_conditions + _supported_attention_backends = _DEFAULT_WANGAME_CAUSAL_CONFIG._supported_attention_backends + param_names_mapping = _DEFAULT_WANGAME_CAUSAL_CONFIG.param_names_mapping + reverse_param_names_mapping = _DEFAULT_WANGAME_CAUSAL_CONFIG.reverse_param_names_mapping + lora_param_names_mapping = _DEFAULT_WANGAME_CAUSAL_CONFIG.lora_param_names_mapping + + def __init__(self, + config: WanGameVideoConfig, + hf_config: dict[str, Any], + **kwargs) -> None: + super().__init__(config=config, hf_config=hf_config, **kwargs) + + def _normalize_action_inputs( + self, + mouse_cond: torch.Tensor | None, + keyboard_cond: torch.Tensor | None, + ) -> tuple[torch.Tensor | None, torch.Tensor | None]: + # drop + if len(getattr(self, "action_config", {})) == 0: + return None, None + return mouse_cond, keyboard_cond + + def _forward_inference( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, + mouse_cond: torch.Tensor | None = None, + keyboard_cond: torch.Tensor | None = None, + kv_cache: dict | None = None, + kv_cache_mouse: dict | None = None, + kv_cache_keyboard: dict | None = None, + crossattn_cache: dict | None = None, + current_start: int = 0, + cache_start: int = 0, + start_frame: int = 0, + **kwargs, + ) -> torch.Tensor: + mouse_cond, keyboard_cond = self._normalize_action_inputs( + mouse_cond, keyboard_cond) + return super()._forward_inference( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + encoder_hidden_states_image=encoder_hidden_states_image, + mouse_cond=mouse_cond, + keyboard_cond=keyboard_cond, + kv_cache=kv_cache, + kv_cache_mouse=kv_cache_mouse, + kv_cache_keyboard=kv_cache_keyboard, + crossattn_cache=crossattn_cache, + current_start=current_start, + cache_start=cache_start, + start_frame=start_frame, + **kwargs, + ) + + def _forward_train( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, + mouse_cond: torch.Tensor | None = None, + keyboard_cond: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + mouse_cond, keyboard_cond = self._normalize_action_inputs( + mouse_cond, keyboard_cond) + return super()._forward_train( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + encoder_hidden_states_image=encoder_hidden_states_image, + mouse_cond=mouse_cond, + keyboard_cond=keyboard_cond, + **kwargs, + ) + + +class CausalWanGameTransformer3DModel(CausalWanTransformer3DModel): + pass diff --git a/fastvideo/models/registry.py b/fastvideo/models/registry.py index 32648f22d..5e3731217 100644 --- a/fastvideo/models/registry.py +++ b/fastvideo/models/registry.py @@ -42,6 +42,8 @@ # "HunyuanVideoTransformer3DModel": ("dits", "hunyuanvideo", "HunyuanVideoDiT"), "WanTransformer3DModel": ("dits", "wanvideo", "WanTransformer3DModel"), "CausalWanTransformer3DModel": ("dits", "causal_wanvideo", "CausalWanTransformer3DModel"), + "CausalWanGameTransformer3DModel": + ("dits", "wangame", "CausalWanGameTransformer3DModel"), "WanGameActionTransformer3DModel": ("dits", "wangame", "WanGameActionTransformer3DModel"), "WanLingBotTransformer3DModel": ("dits", "wangame_lingbot", "WanLingBotTransformer3DModel"), "MatrixGameWanModel": ("dits", "matrixgame", "MatrixGameWanModel"), diff --git a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py new file mode 100644 index 000000000..1f9b97909 --- /dev/null +++ b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +"""WanGame causal DMD pipeline implementation.""" + +from fastvideo.fastvideo_args import FastVideoArgs +import torch +from fastvideo.logger import init_logger +from fastvideo.pipelines import ComposedPipelineBase, ForwardBatch, LoRAPipeline + +from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage, ImageEncodingStage, + MatrixGameCausalDenoisingStage) +from fastvideo.pipelines.stages.image_encoding import ImageVAEEncodingStage + +logger = init_logger(__name__) + + +class WanGameCausalDMDPipeline(LoRAPipeline, ComposedPipelineBase): + _required_config_modules = [ + "vae", "transformer", "scheduler", "image_encoder", "image_processor" + ] + + def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: + self.add_stage(stage_name="input_validation_stage", + stage=InputValidationStage()) + + if (self.get_module("text_encoder", None) is not None + and self.get_module("tokenizer", None) is not None): + self.add_stage(stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + )) + + if (self.get_module("image_encoder", None) is not None + and self.get_module("image_processor", None) is not None): + self.add_stage( + stage_name="image_encoding_stage", + stage=ImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + )) + + self.add_stage(stage_name="conditioning_stage", + stage=ConditioningStage()) + + self.add_stage(stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer", None))) + + self.add_stage( + stage_name="image_latent_preparation_stage", + stage=ImageVAEEncodingStage(vae=self.get_module("vae"))) + + self.add_stage(stage_name="denoising_stage", + stage=MatrixGameCausalDenoisingStage( + transformer=self.get_module("transformer"), + transformer_2=self.get_module("transformer_2", None), + scheduler=self.get_module("scheduler"), + pipeline=self, + vae=self.get_module("vae"))) + + self.add_stage(stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"))) + + logger.info("WanGameCausalDMDPipeline initialized") + + @torch.no_grad() + def streaming_reset(self, batch: ForwardBatch, + fastvideo_args: FastVideoArgs): + if not self.post_init_called: + self.post_init() + + stages_to_run = [ + "input_validation_stage", "prompt_encoding_stage", + "image_encoding_stage", "conditioning_stage", + "latent_preparation_stage", "image_latent_preparation_stage" + ] + + for stage_name in stages_to_run: + if stage_name in self._stage_name_mapping: + batch = self._stage_name_mapping[stage_name].forward( + batch, fastvideo_args) + + denoiser = self._stage_name_mapping["denoising_stage"] + denoiser.streaming_reset(batch, fastvideo_args) + self._vae_cache = None + + def streaming_step(self, keyboard_action, mouse_action) -> ForwardBatch: + denoiser = self._stage_name_mapping["denoising_stage"] + ctx = denoiser._streaming_ctx + assert ctx is not None, "streaming_ctx must be set" + + start_idx = ctx.start_index + batch = denoiser.streaming_step(keyboard_action, mouse_action) + end_idx = ctx.start_index + + if end_idx > start_idx: + current_latents = batch.latents[:, :, start_idx:end_idx, :, :] + args = ctx.fastvideo_args + decoder = self._stage_name_mapping["decoding_stage"] + decoded_frames, self._vae_cache = decoder.streaming_decode( + current_latents, + args, + cache=self._vae_cache, + is_first_chunk=(start_idx == 0)) + batch.output = decoded_frames + else: + batch.output = None + + return batch + + def streaming_clear(self) -> None: + denoiser = self._stage_name_mapping.get("denoising_stage") + if denoiser is not None and hasattr(denoiser, "streaming_clear"): + denoiser.streaming_clear() + self._vae_cache = None + +EntryClass = [WanGameCausalDMDPipeline] diff --git a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py index e2ef763ee..cd7e53d4d 100644 --- a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py @@ -71,4 +71,8 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): stage=DecodingStage(vae=self.get_module("vae"))) -EntryClass = WanGameActionImageToVideoPipeline +class WanLingBotImageToVideoPipeline(WanGameActionImageToVideoPipeline): + pass + + +EntryClass = [WanGameActionImageToVideoPipeline, WanLingBotImageToVideoPipeline] diff --git a/fastvideo/pipelines/pipeline_registry.py b/fastvideo/pipelines/pipeline_registry.py index 11222e431..8bfb0a8e6 100644 --- a/fastvideo/pipelines/pipeline_registry.py +++ b/fastvideo/pipelines/pipeline_registry.py @@ -22,6 +22,7 @@ "WanDMDPipeline": "wan", "WanImageToVideoPipeline": "wan", "WanGameActionImageToVideoPipeline": "wan", + "WanGameCausalDMDPipeline": "wan", "WanLingBotImageToVideoPipeline": "wan", "WanVideoToVideoPipeline": "wan", "WanCausalDMDPipeline": "wan", diff --git a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py new file mode 100644 index 000000000..557ac89ce --- /dev/null +++ b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py @@ -0,0 +1,471 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +ODE Trajectory Data Preprocessing pipeline implementation. + +This module contains an implementation of the ODE Trajectory Data Preprocessing pipeline +using the modular pipeline architecture. + +Sec 4.3 of CausVid paper: https://arxiv.org/pdf/2412.07772 +""" + +import os +from collections.abc import Iterator +from typing import Any + +import numpy as np +import pyarrow as pa +import torch +from PIL import Image +from torch.utils.data import DataLoader +from torchdata.stateful_dataloader import StatefulDataLoader +from tqdm import tqdm + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset import getdataset +from fastvideo.dataset.dataloader.parquet_io import (ParquetDatasetWriter, + records_to_table) +from fastvideo.dataset.dataloader.record_schema import ( + WanGame_ode_record_creator) +from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_WanGame_ode_trajectory) +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.forward_context import set_forward_context +from fastvideo.logger import init_logger +from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( + SelfForcingFlowMatchScheduler) +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.pipelines.preprocess.preprocess_pipeline_base import ( + BasePreprocessPipeline) +from fastvideo.pipelines.stages import (DecodingStage, DenoisingStage, + InputValidationStage, + LatentPreparationStage, + WanGameImageEncodingStage, + TimestepPreparationStage) +from fastvideo.utils import save_decoded_latents_as_video, shallow_asdict + +logger = init_logger(__name__) + + +class PreprocessPipeline_WanGame_ODE_Trajectory(BasePreprocessPipeline): + """ODE Trajectory preprocessing pipeline implementation.""" + + _required_config_modules = [ + "vae", "image_encoder", "image_processor", "transformer", "scheduler" + ] + + preprocess_dataloader: StatefulDataLoader + preprocess_loader_iter: Iterator[dict[str, Any]] + pbar: Any + num_processed_samples: int + + def get_pyarrow_schema(self) -> pa.Schema: + """Return the PyArrow schema for ODE Trajectory pipeline.""" + return pyarrow_schema_WanGame_ode_trajectory + + def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): + """Set up pipeline stages with proper dependency injection.""" + assert fastvideo_args.pipeline_config.flow_shift == 5 + self.modules["scheduler"] = SelfForcingFlowMatchScheduler( + shift=fastvideo_args.pipeline_config.flow_shift, + sigma_min=0.0, + extra_one_step=True) + self.modules["scheduler"].set_timesteps(num_inference_steps=48, + denoising_strength=1.0) + + self.add_stage(stage_name="input_validation_stage", + stage=InputValidationStage()) + self.add_stage(stage_name="image_encoding_stage", + stage=WanGameImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + )) + self.add_stage(stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"))) + self.add_stage(stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer", None))) + self.add_stage(stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + pipeline=self, + )) + self.add_stage(stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"))) + + def get_extra_features(self, valid_data: dict[str, Any], + fastvideo_args: FastVideoArgs) -> dict[str, Any]: + + # TODO(will): move these to cpu at some point + self.get_module("image_encoder").to(get_local_torch_device()) + self.get_module("vae").to(get_local_torch_device()) + + features = {} + """Get CLIP features from the first frame of each video.""" + first_frame = valid_data["pixel_values"][:, :, 0, :, :].permute( + 0, 2, 3, 1) # (B, C, T, H, W) -> (B, H, W, C) + _, _, num_frames, height, width = valid_data["pixel_values"].shape + # latent_height = height // self.get_module( + # "vae").spatial_compression_ratio + # latent_width = width // self.get_module("vae").spatial_compression_ratio + + processed_images = [] + # Frame has values between -1 and 1 + for frame in first_frame: + frame = (frame + 1) * 127.5 + frame_pil = Image.fromarray(frame.cpu().numpy().astype(np.uint8)) + processed_img = self.get_module("image_processor")( + images=frame_pil, return_tensors="pt") + processed_images.append(processed_img) + + # Get CLIP features + pixel_values = torch.cat( + [img['pixel_values'] for img in processed_images], + dim=0).to(get_local_torch_device()) + with torch.no_grad(): + image_inputs = {'pixel_values': pixel_values} + with set_forward_context(current_timestep=0, attn_metadata=None): + clip_features = self.get_module("image_encoder")(**image_inputs) + clip_features = clip_features.last_hidden_state + + features["clip_feature"] = clip_features + features["pil_image"] = first_frame + """Get VAE features from the first frame of each video""" + video_conditions = [] + for frame in first_frame: + processed_img = frame.to(device="cpu", dtype=torch.float32) + processed_img = processed_img.unsqueeze(0).permute(0, 3, 1, + 2).unsqueeze(2) + # (B, H, W, C) -> (B, C, 1, H, W) + video_condition = torch.cat([ + processed_img, + processed_img.new_zeros(processed_img.shape[0], + processed_img.shape[1], num_frames - 1, + height, width) + ], + dim=2) + video_condition = video_condition.to( + device=get_local_torch_device(), dtype=torch.float32) + video_conditions.append(video_condition) + + video_conditions = torch.cat(video_conditions, dim=0) + + with torch.autocast(device_type="cuda", + dtype=torch.float32, + enabled=True): + encoder_outputs = self.get_module("vae").encode(video_conditions) + + # Use mode() instead of mean + latent_condition = encoder_outputs.mode() + + # Use latents_mean/latents_std normalization to match + vae = self.get_module("vae") + if (hasattr(vae.config, 'latents_mean') + and hasattr(vae.config, 'latents_std')): + latents_mean = torch.tensor(vae.config.latents_mean, + device=latent_condition.device, + dtype=latent_condition.dtype).view( + 1, -1, 1, 1, 1) + latents_std = torch.tensor(vae.config.latents_std, + device=latent_condition.device, + dtype=latent_condition.dtype).view( + 1, -1, 1, 1, 1) + latent_condition = (latent_condition - latents_mean) / latents_std + elif (hasattr(vae, "shift_factor") and vae.shift_factor is not None): + if isinstance(vae.shift_factor, torch.Tensor): + latent_condition -= vae.shift_factor.to( + latent_condition.device, latent_condition.dtype) + else: + latent_condition -= vae.shift_factor + + if isinstance(vae.scaling_factor, torch.Tensor): + latent_condition = latent_condition * vae.scaling_factor.to( + latent_condition.device, latent_condition.dtype) + else: + latent_condition = latent_condition * vae.scaling_factor + + # mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, + # latent_width) + # mask_lat_size[:, :, list(range(1, num_frames))] = 0 + # first_frame_mask = mask_lat_size[:, :, 0:1] + # first_frame_mask = torch.repeat_interleave( + # first_frame_mask, + # dim=2, + # repeats=self.get_module("vae").temporal_compression_ratio) + # mask_lat_size = torch.concat( + # [first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2) + # mask_lat_size = mask_lat_size.view( + # batch_size, -1, + # self.get_module("vae").temporal_compression_ratio, latent_height, + # latent_width) + # mask_lat_size = mask_lat_size.transpose(1, 2) + # mask_lat_size = mask_lat_size.to(latent_condition.device) + + # image_latent = torch.concat([mask_lat_size, latent_condition], dim=1) + + # Create mask_cond: ones for first frame, zeros for rest + # Shape: (B, 16, latent_frames, latent_height, latent_width) + mask_cond = torch.ones_like(latent_condition) + mask_cond[:, :, 1:] = 0 # Set all frames except first to 0 + # Create cond_concat: first 4 channels of mask + all 16 channels of img_cond + # Shape: (B, 20, latent_frames, latent_height, latent_width) + cond_concat = torch.cat([mask_cond[:, :4], latent_condition], dim=1) + features["first_frame_latent"] = cond_concat + + if "action_path" in valid_data and valid_data["action_path"]: + keyboard_cond_list = [] + mouse_cond_list = [] + keyboard_dim = self.get_module("transformer").config.arch_config.action_config["keyboard_dim_in"] + for action_path in valid_data["action_path"]: + if action_path: + action_data = np.load(action_path, allow_pickle=True) + if isinstance(action_data, + np.ndarray) and action_data.dtype == np.dtype('O'): + action_dict = action_data.item() + if "keyboard" in action_dict: + keyboard_cond_list.append( + action_dict["keyboard"][:keyboard_dim].astype(np.float32)) + if "mouse" in action_dict: + mouse_cond_list.append(action_dict["mouse"]) + else: + keyboard_cond_list.append(action_data[:keyboard_dim].astype(np.float32)) + if keyboard_cond_list: + features["keyboard_cond"] = keyboard_cond_list + if mouse_cond_list: + features["mouse_cond"] = mouse_cond_list + return features + + def preprocess_action_and_trajectory(self, fastvideo_args: FastVideoArgs, + args): + """Preprocess data and generate trajectory information.""" + + for batch_idx, data in enumerate(self.pbar): + if data is None: + continue + + with torch.inference_mode(): + # Filter out invalid samples (those with all zeros) + valid_indices = [] + for i, pixel_values in enumerate(data["pixel_values"]): + if not torch.all( + pixel_values == 0): # Check if all values are zero + valid_indices.append(i) + self.num_processed_samples += len(valid_indices) + + if not valid_indices: + continue + + # Create new batch with only valid samples + valid_data = { + "pixel_values": + torch.stack( + [data["pixel_values"][i] for i in valid_indices]), + "path": [data["path"][i] for i in valid_indices], + } + + if "fps" in data: + valid_data["fps"] = [data["fps"][i] for i in valid_indices] + if "duration" in data: + valid_data["duration"] = [ + data["duration"][i] for i in valid_indices + ] + if "action_path" in data: + valid_data["action_path"] = [ + data["action_path"][i] for i in valid_indices + ] + + pixel_values = valid_data["pixel_values"] + if pixel_values.shape[2] == 1 and args.num_frames is not None: + pixel_values = pixel_values.repeat( + 1, 1, args.num_frames, 1, 1) + valid_data["pixel_values"] = pixel_values + + # Get extra features if needed + extra_features = self.get_extra_features( + valid_data, fastvideo_args) + + clip_features = extra_features['clip_feature'] + image_latents = extra_features['first_frame_latent'] + image_latents = image_latents[:, :, :args.num_latent_t] + pil_image = extra_features['pil_image'] + if "keyboard_cond" in extra_features: + keyboard_cond = extra_features['keyboard_cond'] + else: + keyboard_cond = None + if "mouse_cond" in extra_features: + mouse_cond = extra_features['mouse_cond'] + else: + mouse_cond = None + + sampling_params = SamplingParam.from_pretrained(args.model_path) + + trajectory_latents = [] + trajectory_timesteps = [] + trajectory_decoded = [] + + device = get_local_torch_device() + for i in range(len(valid_indices)): + # Collect the trajectory data + batch = ForwardBatch(**shallow_asdict(sampling_params), ) + batch.image_embeds = [clip_features[i].unsqueeze(0)] + batch.image_latent = image_latents[i].unsqueeze(0) + batch.keyboard_cond = (torch.from_numpy( + keyboard_cond[i]).unsqueeze(0).to(device) if + keyboard_cond is not None else None) + batch.mouse_cond = (torch.from_numpy( + mouse_cond[i]).unsqueeze(0).to(device) + if mouse_cond is not None else None) + batch.num_inference_steps = 48 + batch.return_trajectory_latents = True + # Enabling this will save the decoded trajectory videos. + # Used for debugging. + batch.return_trajectory_decoded = False + batch.height = args.max_height + batch.width = args.max_width + batch.fps = args.train_fps + batch.num_frames = valid_data["pixel_values"].shape[2] + batch.guidance_scale = 6.0 + batch.do_classifier_free_guidance = False + batch.prompt = "" + + result_batch = self.input_validation_stage( + batch, fastvideo_args) + result_batch = self.timestep_preparation_stage( + batch, fastvideo_args) + result_batch.timesteps = result_batch.timesteps.to(device) + result_batch = self.latent_preparation_stage( + result_batch, fastvideo_args) + result_batch = self.denoising_stage(result_batch, + fastvideo_args) + result_batch = self.decoding_stage(result_batch, + fastvideo_args) + + trajectory_latents.append( + result_batch.trajectory_latents.cpu()) + trajectory_timesteps.append( + result_batch.trajectory_timesteps.cpu()) + trajectory_decoded.append(result_batch.trajectory_decoded) + + # Prepare extra features + extra_features = { + "trajectory_latents": trajectory_latents, + "trajectory_timesteps": trajectory_timesteps + } + + if batch.return_trajectory_decoded: + for i, decoded_frames in enumerate(trajectory_decoded): + for j, decoded_frame in enumerate(decoded_frames): + save_decoded_latents_as_video( + decoded_frame, + f"decoded_videos/trajectory_decoded_{i}_{j}.mp4", + args.train_fps) + + # Prepare batch data for Parquet dataset + batch_data: list[dict[str, Any]] = [] + + # Add progress bar for saving outputs + save_pbar = tqdm(enumerate(valid_data["path"]), + desc="Saving outputs", + unit="item", + leave=False) + + for idx, video_path in save_pbar: + video_name = os.path.basename(video_path).split(".")[0] + + clip_feature_np = clip_features[idx].cpu().numpy() + first_frame_latent_np = image_latents[idx].cpu().numpy() + pil_image_np = pil_image[idx].cpu().numpy() + keyboard_cond_np = keyboard_cond[ + idx] if keyboard_cond is not None else None + mouse_cond_np = mouse_cond[ + idx] if mouse_cond is not None else None + + # Get trajectory features for this sample + traj_latents = extra_features["trajectory_latents"][idx] + traj_timesteps = extra_features["trajectory_timesteps"][idx] + if isinstance(traj_latents, torch.Tensor): + traj_latents = traj_latents.cpu().float().numpy() + if isinstance(traj_timesteps, torch.Tensor): + traj_timesteps = traj_timesteps.cpu().float().numpy() + + # Create record for Parquet dataset + record: dict[str, Any] = WanGame_ode_record_creator( + video_name=video_name, + clip_feature=clip_feature_np, + first_frame_latent=first_frame_latent_np, + trajectory_latents=traj_latents, + trajectory_timesteps=traj_timesteps, + pil_image=pil_image_np, + keyboard_cond=keyboard_cond_np, + mouse_cond=mouse_cond_np, + caption="") + batch_data.append(record) + + if batch_data: + write_pbar = tqdm(total=1, + desc="Writing to Parquet dataset", + unit="batch") + table = records_to_table(batch_data, + self.get_pyarrow_schema()) + write_pbar.update(1) + write_pbar.close() + + if not hasattr(self, 'dataset_writer'): + self.dataset_writer = ParquetDatasetWriter( + out_dir=self.combined_parquet_dir, + samples_per_file=args.samples_per_file, + ) + self.dataset_writer.append_table(table) + + logger.info("Collected batch with %s samples", len(table)) + + if self.num_processed_samples >= args.flush_frequency: + written = self.dataset_writer.flush() + logger.info("Flushed %s samples to parquet", written) + self.num_processed_samples = 0 + + # Final flush for any remaining samples + if hasattr(self, 'dataset_writer'): + written = self.dataset_writer.flush(write_remainder=True) + if written: + logger.info("Final flush wrote %s samples", written) + + def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs, args): + if not self.post_init_called: + self.post_init() + + self.local_rank = int(os.getenv("RANK", 0)) + os.makedirs(args.output_dir, exist_ok=True) + # Create directory for combined data + self.combined_parquet_dir = os.path.join(args.output_dir, + "combined_parquet_dataset") + os.makedirs(self.combined_parquet_dir, exist_ok=True) + + # Loading dataset + train_dataset = getdataset(args) + + self.preprocess_dataloader = DataLoader( + train_dataset, + batch_size=args.preprocess_video_batch_size, + num_workers=args.dataloader_num_workers, + ) + + self.preprocess_loader_iter = iter(self.preprocess_dataloader) + + self.num_processed_samples = 0 + # Add progress bar for video preprocessing + self.pbar = tqdm(self.preprocess_loader_iter, + desc="Processing videos", + unit="batch", + disable=self.local_rank != 0) + + # Initialize class variables for data sharing + self.video_data: dict[str, Any] = {} # Store video metadata and paths + self.latent_data: dict[str, Any] = {} # Store latent tensors + self.preprocess_action_and_trajectory(fastvideo_args, args) + + +EntryClass = PreprocessPipeline_WanGame_ODE_Trajectory \ No newline at end of file diff --git a/fastvideo/training/wangame_ode_causal_pipeline.py b/fastvideo/training/wangame_ode_causal_pipeline.py new file mode 100644 index 000000000..cddbd5440 --- /dev/null +++ b/fastvideo/training/wangame_ode_causal_pipeline.py @@ -0,0 +1,528 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys +from copy import deepcopy +from typing import Any, cast + +import numpy as np +import torch +import torch.nn.functional as F + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_wangame_ode_trajectory) +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.forward_context import set_forward_context +from fastvideo.logger import init_logger +from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( + SelfForcingFlowMatchScheduler) +from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( + WanGameCausalDMDPipeline) +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch +from fastvideo.training.training_pipeline import TrainingPipeline +from fastvideo.training.training_utils import ( + clip_grad_norm_while_handling_failing_dtensor_cases) +from fastvideo.utils import shallow_asdict + +logger = init_logger(__name__) + + +class WanGameODEInitTrainingPipeline(TrainingPipeline): + """ + Training pipeline for ODE-init using precomputed denoising trajectories. + + Supervision: predict the next latent in the stored trajectory by + - feeding current latent at timestep t into the transformer to predict noise + - stepping the scheduler with the predicted noise + - minimizing MSE to the stored next latent at timestep t_next + """ + + _required_config_modules = ["scheduler", "transformer", "vae"] + + def initialize_pipeline(self, fastvideo_args: FastVideoArgs): + # Match the preprocess/generation scheduler for consistent stepping + self.modules["scheduler"] = SelfForcingFlowMatchScheduler( + shift=fastvideo_args.pipeline_config.flow_shift, + sigma_min=0.0, + extra_one_step=True) + self.modules["scheduler"].set_timesteps(num_inference_steps=1000, + training=True) + + def set_schemas(self): + self.train_dataset_schema = pyarrow_schema_wangame_ode_trajectory + + def initialize_training_pipeline(self, training_args: TrainingArgs): + super().initialize_training_pipeline(training_args) + + self.noise_scheduler = self.get_module("scheduler") + self.vae = self.get_module("vae") + self.vae.requires_grad_(False) + + self.timestep_shift = self.training_args.pipeline_config.flow_shift + assert self.timestep_shift == 5.0, "flow_shift must be 5.0" + self.noise_scheduler = SelfForcingFlowMatchScheduler( + shift=self.timestep_shift, sigma_min=0.0, extra_one_step=True) + self.noise_scheduler.set_timesteps(num_inference_steps=1000, + training=True) + + logger.info("dmd_denoising_steps: %s", + self.training_args.pipeline_config.dmd_denoising_steps) + self.dmd_denoising_steps = torch.tensor([1000, 750, 500, 250], + dtype=torch.long, + device=get_local_torch_device()) + if training_args.warp_denoising_step: # Warp the denoising step according to the scheduler time shift + timesteps = torch.cat((self.noise_scheduler.timesteps.cpu(), + torch.tensor([0], + dtype=torch.float32))).cuda() + logger.info("timesteps: %s", timesteps) + self.dmd_denoising_steps = timesteps[1000 - + self.dmd_denoising_steps] + logger.info("warped self.dmd_denoising_steps: %s", + self.dmd_denoising_steps) + else: + raise ValueError("warp_denoising_step must be true") + + self.dmd_denoising_steps = self.dmd_denoising_steps.to( + get_local_torch_device()) + + logger.info("denoising_step_list: %s", self.dmd_denoising_steps) + + logger.info( + "Initialized ODE-init training pipeline with %s denoising steps", + len(self.dmd_denoising_steps)) + # Cache for nearest trajectory index per DMD step (computed lazily on first batch) + self._cached_closest_idx_per_dmd = None + self.num_train_timestep = self.noise_scheduler.num_train_timesteps + self.manual_idx = 0 + + def initialize_validation_pipeline(self, training_args: TrainingArgs): + logger.info("Initializing validation pipeline...") + args_copy = deepcopy(training_args) + args_copy.inference_mode = True + # Use the same flow-matching scheduler as training for consistent validation. + validation_scheduler = SelfForcingFlowMatchScheduler( + shift=args_copy.pipeline_config.flow_shift, + sigma_min=0.0, + extra_one_step=True) + validation_scheduler.set_timesteps(num_inference_steps=1000, + training=True) + # Warm start validation with current transformer + self.validation_pipeline = WanGameCausalDMDPipeline.from_pretrained( + training_args.model_path, + args=args_copy, # type: ignore + inference_mode=True, + loaded_modules={ + "transformer": self.get_module("transformer"), + "vae": self.get_module("vae"), + "scheduler": validation_scheduler, + }, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + pin_cpu_memory=training_args.pin_cpu_memory, + dit_cpu_offload=True) + + def _get_next_batch( + self, + training_batch) -> tuple[TrainingBatch, torch.Tensor, torch.Tensor]: + batch = next(self.train_loader_iter, None) # type: ignore + if batch is None: + self.current_epoch += 1 + logger.info("Starting epoch %s", self.current_epoch) + self.train_loader_iter = iter(self.train_dataloader) + batch = next(self.train_loader_iter) + + # Required fields from parquet (ODE trajectory schema) + clip_feature = batch['clip_feature'] + first_frame_latent = batch['first_frame_latent'] + keyboard_cond = batch.get('keyboard_cond', None) + # keyboard_cond = keyboard_cond[:, :, :3] # TODO: remove hardcode + mouse_cond = batch.get('mouse_cond', None) + infos = batch['info_list'] + + # Trajectory tensors may include a leading singleton batch dim per row + trajectory_latents = batch['trajectory_latents'] + if trajectory_latents.dim() == 7: + # [B, 1, S, C, T, H, W] -> [B, S, C, T, H, W] + trajectory_latents = trajectory_latents[:, 0] + elif trajectory_latents.dim() == 6: + # already [B, S, C, T, H, W] + pass + else: + raise ValueError( + f"Unexpected trajectory_latents dim: {trajectory_latents.dim()}" + ) + + trajectory_timesteps = batch['trajectory_timesteps'] + if trajectory_timesteps.dim() == 3: + # [B, 1, S] -> [B, S] + trajectory_timesteps = trajectory_timesteps[:, 0] + elif trajectory_timesteps.dim() == 2: + # [B, S] + pass + else: + raise ValueError( + f"Unexpected trajectory_timesteps dim: {trajectory_timesteps.dim()}" + ) + # [B, S, C, T, H, W] -> [B, S, T, C, H, W] to match self-forcing + trajectory_latents = trajectory_latents.permute(0, 1, 3, 2, 4, 5) + + # Move to device + device = get_local_torch_device() + training_batch.image_embeds = clip_feature.to(device, dtype=torch.bfloat16) + training_batch.image_latents = first_frame_latent.to(device, dtype=torch.bfloat16) + if keyboard_cond is not None and keyboard_cond.numel() > 0: + training_batch.keyboard_cond = keyboard_cond.to(device, dtype=torch.bfloat16) + else: + training_batch.keyboard_cond = None + if mouse_cond is not None and mouse_cond.numel() > 0: + training_batch.mouse_cond = mouse_cond.to(device, dtype=torch.bfloat16) + else: + training_batch.mouse_cond = None + training_batch.infos = infos + + return training_batch, trajectory_latents.to( + device, dtype=torch.bfloat16), trajectory_timesteps.to(device) + + def _get_timestep(self, + min_timestep: int, + max_timestep: int, + batch_size: int, + num_frame: int, + num_frame_per_block: int, + uniform_timestep: bool = False) -> torch.Tensor: + if uniform_timestep: + timestep = torch.randint(min_timestep, + max_timestep, [batch_size, 1], + device=self.device, + dtype=torch.long).repeat(1, num_frame) + return timestep + else: + timestep = torch.randint(min_timestep, + max_timestep, [batch_size, num_frame], + device=self.device, + dtype=torch.long) + # logger.info(f"individual timestep: {timestep}") + # make the noise level the same within every block + timestep = timestep.reshape(timestep.shape[0], -1, + num_frame_per_block) + timestep[:, :, 1:] = timestep[:, :, 0:1] + timestep = timestep.reshape(timestep.shape[0], -1) + return timestep + + def _prepare_dit_inputs(self, + training_batch: TrainingBatch) -> TrainingBatch: + """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" + + # First, call parent method to prepare noise, timesteps, etc. for video latents + training_batch = super()._prepare_dit_inputs(training_batch) + + assert isinstance(training_batch.image_latents, torch.Tensor) + image_latents = training_batch.image_latents.to( + get_local_torch_device(), dtype=torch.bfloat16) + + temporal_compression_ratio = 4 + num_frames = (self.training_args.num_latent_t - + 1) * temporal_compression_ratio + 1 + batch_size, num_channels, _, latent_height, latent_width = image_latents.shape + mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, + latent_width) + mask_lat_size[:, :, 1:] = 0 + + first_frame_mask = mask_lat_size[:, :, :1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, dim=2, repeats=temporal_compression_ratio) + mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], + dim=2) + mask_lat_size = mask_lat_size.view(batch_size, -1, + temporal_compression_ratio, + latent_height, latent_width) + mask_lat_size = mask_lat_size.transpose(1, 2) + mask_lat_size = mask_lat_size.to( + image_latents.device).to(dtype=torch.bfloat16) + + training_batch.noisy_model_input = torch.cat( + [training_batch.noisy_model_input, mask_lat_size, image_latents], + dim=1) + + return training_batch + + def _step_predict_next_latent( + self, traj_latents: torch.Tensor, traj_timesteps: torch.Tensor, + image_embeds: torch.Tensor, + image_latents: torch.Tensor, + keyboard_cond: torch.Tensor | None, + mouse_cond: torch.Tensor | None + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, dict[str, + torch.Tensor]]: + latent_vis_dict: dict[str, torch.Tensor] = {} + device = get_local_torch_device() + target_latent = traj_latents[:, -1] + + # Shapes: traj_latents [B, S, C, T, H, W], traj_timesteps [B, S] + B, S, num_frames, num_channels, height, width = traj_latents.shape + + # Lazily cache nearest trajectory index per DMD step based on the (fixed) S timesteps + if self._cached_closest_idx_per_dmd is None: + traj_ts = traj_timesteps[0].float().cpu() + dmd_steps = self.dmd_denoising_steps.float().cpu() + closest_idx = torch.argmin( + torch.abs(traj_ts.unsqueeze(0) - dmd_steps.unsqueeze(1)), + dim=1) + self._cached_closest_idx_per_dmd = closest_idx.to(torch.long).cpu() + logger.info("self._cached_closest_idx_per_dmd: %s", + self._cached_closest_idx_per_dmd) + logger.info("corresponding timesteps: %s", + traj_ts[self._cached_closest_idx_per_dmd]) + + # Select the K indexes from traj_latents using self._cached_closest_idx_per_dmd + # traj_latents: [B, S, C, T, H, W], self._cached_closest_idx_per_dmd: [K] + # Output: [B, K, C, T, H, W] + assert self._cached_closest_idx_per_dmd is not None + relevant_traj_latents = torch.index_select( + traj_latents, + dim=1, + index=self._cached_closest_idx_per_dmd.to(traj_latents.device)) + logger.info("relevant_traj_latents: %s", relevant_traj_latents.shape) + # assert relevant_traj_latents.shape[0] == 1 + + indexes = self._get_timestep( # [B, num_frames] + 0, + len(self.dmd_denoising_steps), + B, + num_frames, + 3, + uniform_timestep=False) + logger.info("indexes: %s", indexes.shape) + logger.info("indexes: %s", indexes) + # noisy_input = relevant_traj_latents[indexes] + noisy_input = torch.gather( + relevant_traj_latents, + dim=1, + index=indexes.reshape(B, 1, num_frames, 1, 1, + 1).expand(-1, -1, -1, num_channels, height, + width).to(self.device)).squeeze(1) + latent_model_input = noisy_input.permute(0, 2, 1, 3, 4) + if image_latents is not None: + latent_model_input = torch.cat( + [ + latent_model_input, + image_latents.to(latent_model_input.device, + latent_model_input.dtype), + ], + dim=1) + timestep = self.dmd_denoising_steps[indexes] + logger.info("selected timestep for rank %s: %s", + self.global_rank, + timestep, + local_main_process_only=False) + + # Prepare inputs for transformer + latent_vis_dict["noisy_input"] = noisy_input.permute( + 0, 2, 1, 3, 4).detach().clone().cpu() + latent_vis_dict["x0"] = target_latent.permute(0, 2, 1, 3, + 4).detach().clone().cpu() + + latent_model_input = latent_model_input.to(device, dtype=torch.bfloat16) + timestep = timestep.to(device, dtype=torch.bfloat16) + + logger.info("========== Transformer Input ==========") + logger.info("hidden_states (latent_model_input) shape: %s, dtype: %s", latent_model_input.shape, latent_model_input.dtype) + logger.info("hidden_states min/max/mean: %.4f / %.4f / %.4f", + latent_model_input.min().item(), latent_model_input.max().item(), latent_model_input.mean().item()) + logger.info("encoder_hidden_states_image (image_embeds) shape: %s", image_embeds.shape if image_embeds is not None else None) + logger.info("timestep shape: %s, dtype: %s", timestep.shape, timestep.dtype) + logger.info("keyboard_cond: %s", keyboard_cond.shape if keyboard_cond is not None else None) + logger.info("mouse_cond: %s", mouse_cond.shape if mouse_cond is not None else None) + + input_kwargs = { + "hidden_states": latent_model_input, + "encoder_hidden_states": None, + "encoder_hidden_states_image": image_embeds, + "timestep": timestep, + "mouse_cond": mouse_cond, + "keyboard_cond": keyboard_cond, + "return_dict": False, + } + # Predict noise and step the scheduler to obtain next latent + with set_forward_context(current_timestep=timestep, + attn_metadata=None, + forward_batch=None): + noise_pred = self.transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + + logger.info("========== Transformer Output ==========") + logger.info("noise_pred shape: %s", noise_pred.shape) + logger.info("noise_pred min/max/mean: %.4f / %.4f / %.4f", + noise_pred.min().item(), noise_pred.max().item(), noise_pred.mean().item()) + + from fastvideo.models.utils import pred_noise_to_pred_video + pred_video = pred_noise_to_pred_video( + pred_noise=noise_pred.flatten(0, 1), + noise_input_latent=noisy_input.flatten(0, 1), + timestep=timestep.to(dtype=torch.bfloat16).flatten(0, 1), + scheduler=self.modules["scheduler"]).unflatten( + 0, noise_pred.shape[:2]) + latent_vis_dict["pred_video"] = pred_video.permute( + 0, 2, 1, 3, 4).detach().clone().cpu() + + return pred_video, target_latent, timestep, latent_vis_dict + + def train_one_step(self, training_batch): # type: ignore[override] + self.transformer.train() + self.optimizer.zero_grad() + training_batch.total_loss = 0.0 + args = cast(TrainingArgs, self.training_args) + + # Using cached nearest index per DMD step; computation happens in _step_predict_next_latent + + for _ in range(args.gradient_accumulation_steps): + training_batch, traj_latents, traj_timesteps = self._get_next_batch( + training_batch) + image_embeds = training_batch.image_embeds + image_latents = training_batch.image_latents + keyboard_cond = training_batch.keyboard_cond + mouse_cond = training_batch.mouse_cond + assert traj_latents.shape[0] == 1 + + # Shapes: traj_latents [B, S, C, T, H, W], traj_timesteps [B, S] + _, S = traj_latents.shape[0], traj_latents.shape[1] + if S < 2: + raise ValueError("Trajectory must contain at least 2 steps") + + # Forward to predict next latent by stepping scheduler with predicted noise + noise_pred, target_latent, t, latent_vis_dict = self._step_predict_next_latent( + traj_latents, traj_timesteps, image_embeds, image_latents, keyboard_cond, mouse_cond) + + training_batch.latent_vis_dict.update(latent_vis_dict) + + mask = t != 0 + + # Compute loss + loss = F.mse_loss(noise_pred[mask], + target_latent[mask], + reduction="mean") + loss = loss / args.gradient_accumulation_steps + + with set_forward_context(current_timestep=t, + attn_metadata=None, + forward_batch=None): + loss.backward() + avg_loss = loss.detach().clone() + training_batch.total_loss += avg_loss.item() + + # Clip grad and step optimizers + grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in self.transformer.parameters() if p.requires_grad], + args.max_grad_norm if args.max_grad_norm is not None else 0.0) + + self.optimizer.step() + self.lr_scheduler.step() + + if grad_norm is None: + grad_value = 0.0 + else: + try: + if isinstance(grad_norm, torch.Tensor): + grad_value = float(grad_norm.detach().float().item()) + else: + grad_value = float(grad_norm) + except Exception: + grad_value = 0.0 + training_batch.grad_norm = grad_value + B, S, T, C, H, W = traj_latents.shape + training_batch.raw_latent_shape = (B, C, T, H, W) + return training_batch + + def _prepare_validation_batch(self, sampling_param: SamplingParam, + training_args: TrainingArgs, + validation_batch: dict[str, Any], + num_inference_steps: int) -> ForwardBatch: + sampling_param.prompt = validation_batch['prompt'] + sampling_param.height = training_args.num_height + sampling_param.width = training_args.num_width + sampling_param.image_path = validation_batch.get( + 'image_path') or validation_batch.get('video_path') + sampling_param.num_inference_steps = num_inference_steps + sampling_param.data_type = "video" + assert self.seed is not None + sampling_param.seed = self.seed + + latents_size = [(sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, sampling_param.width // 8] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (training_args.num_latent_t - + 1) * temporal_compression_factor + 1 + sampling_param.num_frames = num_frames + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=torch.Generator(device="cpu").manual_seed(self.seed), + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=training_args.VSA_sparsity, + ) + if "image" in validation_batch and validation_batch["image"] is not None: + batch.pil_image = validation_batch["image"] + + if "keyboard_cond" in validation_batch and validation_batch[ + "keyboard_cond"] is not None: + keyboard_cond = validation_batch["keyboard_cond"] + keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) + keyboard_cond = keyboard_cond.unsqueeze(0) + batch.keyboard_cond = keyboard_cond + + if "mouse_cond" in validation_batch and validation_batch[ + "mouse_cond"] is not None: + mouse_cond = validation_batch["mouse_cond"] + mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) + mouse_cond = mouse_cond.unsqueeze(0) + batch.mouse_cond = mouse_cond + + return batch + + def visualize_intermediate_latents(self, training_batch: TrainingBatch, + training_args: TrainingArgs, step: int): + tracker_loss_dict: dict[str, Any] = {} + latents_vis_dict = training_batch.latent_vis_dict + latent_log_keys = ['noisy_input', 'x0', 'pred_video'] + for latent_key in latent_log_keys: + assert latent_key in latents_vis_dict and latents_vis_dict[ + latent_key] is not None + latent = latents_vis_dict[latent_key] + pixel_latent = self.validation_pipeline.decoding_stage.decode( + latent, training_args) + + video = pixel_latent.cpu().float() + video = video.permute(0, 2, 1, 3, 4) + video = (video * 255).numpy().astype(np.uint8) + video_artifact = self.tracker.video( + video, fps=16, format="mp4") # change to 16 for Wan2.1 + if video_artifact is not None: + tracker_loss_dict[latent_key] = video_artifact + # Clean up references + del video, pixel_latent, latent + + if self.global_rank == 0 and tracker_loss_dict: + self.tracker.log_artifacts(tracker_loss_dict, step) + + +def main(args) -> None: + logger.info("Starting ODE-init training pipeline...") + pipeline = WanGameODEInitTrainingPipeline.from_pretrained( + args.pretrained_model_name_or_path, args=args) + args = pipeline.training_args + pipeline.train() + logger.info("ODE-init training pipeline done") + + +if __name__ == "__main__": + argv = sys.argv + from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.utils import FlexibleArgumentParser + parser = FlexibleArgumentParser() + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args() + args.dit_cpu_offload = False + main(args) From b4c5420227e17524ee9297c2847463afb922a58b Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Mon, 9 Feb 2026 17:40:27 -0800 Subject: [PATCH 018/214] registry causal and ode init --- fastvideo/dataset/dataloader/record_schema.py | 93 ++++ fastvideo/dataset/dataloader/schema.py | 49 +- fastvideo/models/dits/wangame/__init__.py | 4 + .../models/dits/wangame_lingbot/cam_utils.py | 203 ++++++++ .../models/dits/wangame_lingbot/model.py | 451 ++++++++++++++++++ fastvideo/models/loader/fsdp_load.py | 9 +- fastvideo/models/registry.py | 2 + .../basic/wan/wangame_i2v_pipeline.py | 6 +- fastvideo/pipelines/pipeline_registry.py | 1 + fastvideo/pipelines/stages/denoising.py | 9 + .../wangame_lingbot_training_pipeline.py | 411 ++++++++++++++++ 11 files changed, 1235 insertions(+), 3 deletions(-) create mode 100644 fastvideo/models/dits/wangame_lingbot/cam_utils.py create mode 100644 fastvideo/models/dits/wangame_lingbot/model.py create mode 100644 fastvideo/training/wangame_lingbot_training_pipeline.py diff --git a/fastvideo/dataset/dataloader/record_schema.py b/fastvideo/dataset/dataloader/record_schema.py index 1bc86dd7d..0eea39adf 100644 --- a/fastvideo/dataset/dataloader/record_schema.py +++ b/fastvideo/dataset/dataloader/record_schema.py @@ -188,3 +188,96 @@ def text_only_record_creator(text_name: str, text_embedding: np.ndarray, "caption": caption, } return record + + +def wangame_ode_record_creator( + video_name: str, + clip_feature: np.ndarray, + first_frame_latent: np.ndarray, + trajectory_latents: np.ndarray, + trajectory_timesteps: np.ndarray, + pil_image: np.ndarray | None = None, + keyboard_cond: np.ndarray | None = None, + mouse_cond: np.ndarray | None = None, + caption: str = "") -> dict[str, Any]: + """Create a ODE trajectory record matching pyarrow_schema_wangame + """ + assert trajectory_latents is not None, "trajectory_latents is required" + assert trajectory_timesteps is not None, "trajectory_timesteps is required" + assert clip_feature is not None, "clip_feature is required" + assert first_frame_latent is not None, "first_frame_latent is required" + + record = { + "id": video_name, + "file_name": video_name, + "caption": caption, + "media_type": "video", + } + + # I2V features + record.update({ + "clip_feature_bytes": clip_feature.tobytes(), + "clip_feature_shape": list(clip_feature.shape), + "clip_feature_dtype": str(clip_feature.dtype), + }) + + record.update({ + "first_frame_latent_bytes": first_frame_latent.tobytes(), + "first_frame_latent_shape": list(first_frame_latent.shape), + "first_frame_latent_dtype": str(first_frame_latent.dtype), + }) + + # Optional PIL Image + if pil_image is not None: + record.update({ + "pil_image_bytes": pil_image.tobytes(), + "pil_image_shape": list(pil_image.shape), + "pil_image_dtype": str(pil_image.dtype), + }) + else: + record.update({ + "pil_image_bytes": b"", + "pil_image_shape": [], + "pil_image_dtype": "", + }) + + # Actions + if keyboard_cond is not None: + record.update({ + "keyboard_cond_bytes": keyboard_cond.tobytes(), + "keyboard_cond_shape": list(keyboard_cond.shape), + "keyboard_cond_dtype": str(keyboard_cond.dtype), + }) + else: + record.update({ + "keyboard_cond_bytes": b"", + "keyboard_cond_shape": [], + "keyboard_cond_dtype": "", + }) + + if mouse_cond is not None: + record.update({ + "mouse_cond_bytes": mouse_cond.tobytes(), + "mouse_cond_shape": list(mouse_cond.shape), + "mouse_cond_dtype": str(mouse_cond.dtype), + }) + else: + record.update({ + "mouse_cond_bytes": b"", + "mouse_cond_shape": [], + "mouse_cond_dtype": "", + }) + + record.update({ + "trajectory_latents_bytes": trajectory_latents.tobytes(), + "trajectory_latents_shape": list(trajectory_latents.shape), + "trajectory_latents_dtype": str(trajectory_latents.dtype), + }) + + record.update({ + "trajectory_timesteps_bytes": trajectory_timesteps.tobytes(), + "trajectory_timesteps_shape": list(trajectory_timesteps.shape), + "trajectory_timesteps_dtype": str(trajectory_timesteps.dtype), + }) + + return record diff --git a/fastvideo/dataset/dataloader/schema.py b/fastvideo/dataset/dataloader/schema.py index 422ba65d1..4ff323392 100644 --- a/fastvideo/dataset/dataloader/schema.py +++ b/fastvideo/dataset/dataloader/schema.py @@ -198,4 +198,51 @@ pa.field("fps", pa.float64()), ]) -pyarrow_schema_wangame_lingbot = pyarrow_schema_wangame \ No newline at end of file +pyarrow_schema_wangame_lingbot = pyarrow_schema_wangame + +pyarrow_schema_ode_trajectory_wangame = pa.schema([ + pa.field("id", pa.string()), + # --- Image/Video VAE latents --- + # Tensors are stored as raw bytes with shape and dtype info for loading + pa.field("vae_latent_bytes", pa.binary()), + # e.g., [C, T, H, W] or [C, H, W] + pa.field("vae_latent_shape", pa.list_(pa.int64())), + # e.g., 'float32' + pa.field("vae_latent_dtype", pa.string()), + #I2V + pa.field("clip_feature_bytes", pa.binary()), + pa.field("clip_feature_shape", pa.list_(pa.int64())), + pa.field("clip_feature_dtype", pa.string()), + pa.field("first_frame_latent_bytes", pa.binary()), + pa.field("first_frame_latent_shape", pa.list_(pa.int64())), + pa.field("first_frame_latent_dtype", pa.string()), + # --- Action --- + pa.field("mouse_cond_bytes", pa.binary()), + pa.field("mouse_cond_shape", pa.list_(pa.int64())), # [T, 2] + pa.field("mouse_cond_dtype", pa.string()), + pa.field("keyboard_cond_bytes", pa.binary()), + pa.field("keyboard_cond_shape", pa.list_(pa.int64())), # [T, 4] + pa.field("keyboard_cond_dtype", pa.string()), + # I2V Validation + pa.field("pil_image_bytes", pa.binary()), + pa.field("pil_image_shape", pa.list_(pa.int64())), + pa.field("pil_image_dtype", pa.string()), + # --- ODE Trajectory --- + pa.field("trajectory_latents_bytes", pa.binary()), + pa.field("trajectory_latents_shape", pa.list_(pa.int64())), + pa.field("trajectory_latents_dtype", pa.string()), + pa.field("trajectory_timesteps_bytes", pa.binary()), + pa.field("trajectory_timesteps_shape", pa.list_(pa.int64())), + pa.field("trajectory_timesteps_dtype", pa.string()), + # --- Metadata --- + pa.field("file_name", pa.string()), + pa.field("caption", pa.string()), + pa.field("media_type", pa.string()), # 'image' or 'video' + pa.field("width", pa.int64()), + pa.field("height", pa.int64()), + # -- Video-specific (can be null/default for images) --- + # Number of frames processed (e.g., 1 for image, N for video) + pa.field("num_frames", pa.int64()), + pa.field("duration_sec", pa.float64()), + pa.field("fps", pa.float64()), +]) diff --git a/fastvideo/models/dits/wangame/__init__.py b/fastvideo/models/dits/wangame/__init__.py index 7dbe1cb64..ebe540626 100644 --- a/fastvideo/models/dits/wangame/__init__.py +++ b/fastvideo/models/dits/wangame/__init__.py @@ -1,8 +1,12 @@ from .model import WanGameActionTransformer3DModel +from .causal_model import (CausalWanGameTransformer3DModel, + CausalWanTransformer3DModel) from .hyworld_action_module import WanGameActionTimeImageEmbedding, WanGameActionSelfAttention __all__ = [ "WanGameActionTransformer3DModel", + "CausalWanTransformer3DModel", + "CausalWanGameTransformer3DModel", "WanGameActionTimeImageEmbedding", "WanGameActionSelfAttention", ] diff --git a/fastvideo/models/dits/wangame_lingbot/cam_utils.py b/fastvideo/models/dits/wangame_lingbot/cam_utils.py new file mode 100644 index 000000000..fb72ec84a --- /dev/null +++ b/fastvideo/models/dits/wangame_lingbot/cam_utils.py @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from LingBot World: https://github.com/Robbyant/lingbot-world/blob/main/wan/utils/cam_utils.py + +import numpy as np +import os +import torch +from scipy.interpolate import interp1d +from scipy.spatial.transform import Rotation, Slerp + + +# --- Official Code (Leave Unchanged) --- + +def interpolate_camera_poses( + src_indices: np.ndarray, + src_rot_mat: np.ndarray, + src_trans_vec: np.ndarray, + tgt_indices: np.ndarray, +) -> torch.Tensor: + # interpolate translation + interp_func_trans = interp1d( + src_indices, + src_trans_vec, + axis=0, + kind='linear', + bounds_error=False, + fill_value="extrapolate", + ) + interpolated_trans_vec = interp_func_trans(tgt_indices) + + # interpolate rotation + src_quat_vec = Rotation.from_matrix(src_rot_mat) + # ensure there is no sudden change in qw + quats = src_quat_vec.as_quat().copy() # [N, 4] + for i in range(1, len(quats)): + if np.dot(quats[i], quats[i-1]) < 0: + quats[i] = -quats[i] + src_quat_vec = Rotation.from_quat(quats) + slerp_func_rot = Slerp(src_indices, src_quat_vec) + interpolated_rot_quat = slerp_func_rot(tgt_indices) + interpolated_rot_mat = interpolated_rot_quat.as_matrix() + + poses = np.zeros((len(tgt_indices), 4, 4)) + poses[:, :3, :3] = interpolated_rot_mat + poses[:, :3, 3] = interpolated_trans_vec + poses[:, 3, 3] = 1.0 + return torch.from_numpy(poses).float() + + +def SE3_inverse(T: torch.Tensor) -> torch.Tensor: + Rot = T[:, :3, :3] # [B,3,3] + trans = T[:, :3, 3:] # [B,3,1] + R_inv = Rot.transpose(-1, -2) + t_inv = -torch.bmm(R_inv, trans) + T_inv = torch.eye(4, device=T.device, dtype=T.dtype)[None, :, :].repeat(T.shape[0], 1, 1) + T_inv[:, :3, :3] = R_inv + T_inv[:, :3, 3:] = t_inv + return T_inv + + +def compute_relative_poses( + c2ws_mat: torch.Tensor, + framewise: bool = False, + normalize_trans: bool = True, +) -> torch.Tensor: + ref_w2cs = SE3_inverse(c2ws_mat[0:1]) + relative_poses = torch.matmul(ref_w2cs, c2ws_mat) + # ensure identity matrix for 1st frame + relative_poses[0] = torch.eye(4, device=c2ws_mat.device, dtype=c2ws_mat.dtype) + if framewise: + # compute pose between i and i+1 + relative_poses_framewise = torch.bmm(SE3_inverse(relative_poses[:-1]), relative_poses[1:]) + relative_poses[1:] = relative_poses_framewise + if normalize_trans: # note refer to camctrl2: "we scale the coordinate inputs to roughly 1 standard deviation to simplify model learning." + translations = relative_poses[:, :3, 3] # [f, 3] + max_norm = torch.norm(translations, dim=-1).max() + # only normlaize when moving + if max_norm > 0: + relative_poses[:, :3, 3] = translations / max_norm + return relative_poses + + +@torch.no_grad() +def create_meshgrid(n_frames: int, height: int, width: int, bias: float = 0.5, device='cuda', dtype=torch.float32) -> torch.Tensor: + x_range = torch.arange(width, device=device, dtype=dtype) + y_range = torch.arange(height, device=device, dtype=dtype) + grid_y, grid_x = torch.meshgrid(y_range, x_range, indexing='ij') + grid_xy = torch.stack([grid_x, grid_y], dim=-1).view([-1, 2]) + bias # [h*w, 2] + grid_xy = grid_xy[None, ...].repeat(n_frames, 1, 1) # [f, h*w, 2] + return grid_xy + + +def get_plucker_embeddings( + c2ws_mat: torch.Tensor, + Ks: torch.Tensor, + height: int, + width: int, +): + n_frames = c2ws_mat.shape[0] + grid_xy = create_meshgrid(n_frames, height, width, device=c2ws_mat.device, dtype=c2ws_mat.dtype) # [f, h*w, 2] + fx, fy, cx, cy = Ks.chunk(4, dim=-1) # [f, 1] + + i = grid_xy[..., 0] # [f, h*w] + j = grid_xy[..., 1] # [f, h*w] + zs = torch.ones_like(i) # [f, h*w] + xs = (i - cx) / fx * zs + ys = (j - cy) / fy * zs + + directions = torch.stack([xs, ys, zs], dim=-1) # [f, h*w, 3] + directions = directions / directions.norm(dim=-1, keepdim=True) # [f, h*w, 3] + + rays_d = directions @ c2ws_mat[:, :3, :3].transpose(-1, -2) # [f, h*w, 3] + rays_o = c2ws_mat[:, :3, 3] # [f, 3] + rays_o = rays_o[:, None, :].expand_as(rays_d) # [f, h*w, 3] + # rays_dxo = torch.cross(rays_o, rays_d, dim=-1) # [f, h*w, 3] + # note refer to: apt2 + plucker_embeddings = torch.cat([rays_o, rays_d], dim=-1) # [f, h*w, 6] + plucker_embeddings = plucker_embeddings.view([n_frames, height, width, 6]) # [f*h*w, 6] + return plucker_embeddings + + +def get_Ks_transformed( + Ks: torch.Tensor, + height_org: int, + width_org: int, + height_resize: int, + width_resize: int, + height_final: int, + width_final: int, +): + fx, fy, cx, cy = Ks.chunk(4, dim=-1) # [f, 1] + + scale_x = width_resize / width_org + scale_y = height_resize / height_org + + fx_resize = fx * scale_x + fy_resize = fy * scale_y + cx_resize = cx * scale_x + cy_resize = cy * scale_y + + crop_offset_x = (width_resize - width_final) / 2 + crop_offset_y = (height_resize - height_final) / 2 + + cx_final = cx_resize - crop_offset_x + cy_final = cy_resize - crop_offset_y + + Ks_transformed = torch.zeros_like(Ks) + Ks_transformed[:, 0:1] = fx_resize + Ks_transformed[:, 1:2] = fy_resize + Ks_transformed[:, 2:3] = cx_final + Ks_transformed[:, 3:4] = cy_final + + return Ks_transformed + + +# --- Custom --- + +def prepare_camera_embedding( + action_path: str, + num_frames: int, + height: int, + width: int, + spatial_scale: int = 8, +) -> tuple[torch.Tensor, int]: + c2ws = np.load(os.path.join(action_path, "poses.npy")) + len_c2ws = ((len(c2ws) - 1) // 4) * 4 + 1 + num_frames = min(num_frames, len_c2ws) + c2ws = c2ws[:num_frames] + + Ks = torch.from_numpy( + np.load(os.path.join(action_path, "intrinsics.npy")) + ).float() + Ks = get_Ks_transformed( + Ks, + height_org=480, + width_org=832, + height_resize=height, + width_resize=width, + height_final=height, + width_final=width, + ) + Ks = Ks[0] # use first frame + + len_c2ws = len(c2ws) + num_latent_frames = (len_c2ws - 1) // 4 + 1 + c2ws_infer = interpolate_camera_poses( + src_indices=np.linspace(0, len_c2ws - 1, len_c2ws), + src_rot_mat=c2ws[:, :3, :3], + src_trans_vec=c2ws[:, :3, 3], + tgt_indices=np.linspace(0, len_c2ws - 1, num_latent_frames), + ) + c2ws_infer = compute_relative_poses(c2ws_infer, framewise=True) + Ks = Ks.repeat(num_latent_frames, 1) + plucker = get_plucker_embeddings(c2ws_infer, Ks, height, width) # [F, H, W, 6] + + # reshpae + latent_height = height // spatial_scale + latent_width = width // spatial_scale + plucker = plucker.view(num_latent_frames, latent_height, spatial_scale, latent_width, spatial_scale, 6) + plucker = plucker.permute(0, 1, 3, 5, 2, 4).contiguous() + plucker = plucker.view(num_latent_frames, latent_height, latent_width, 6 * spatial_scale * spatial_scale) + c2ws_plucker_emb = plucker.permute(3, 0, 1, 2).contiguous().unsqueeze(0) + + return c2ws_plucker_emb, num_frames \ No newline at end of file diff --git a/fastvideo/models/dits/wangame_lingbot/model.py b/fastvideo/models/dits/wangame_lingbot/model.py new file mode 100644 index 000000000..a19c9b351 --- /dev/null +++ b/fastvideo/models/dits/wangame_lingbot/model.py @@ -0,0 +1,451 @@ +# SPDX-License-Identifier: Apache-2.0 + +import math +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from fastvideo.attention import DistributedAttention +from fastvideo.configs.models.dits.wangamevideo import WanGameVideoConfig +from fastvideo.distributed.parallel_state import get_sp_world_size +from fastvideo.layers.layernorm import (FP32LayerNorm, LayerNormScaleShift, + RMSNorm, ScaleResidual, + ScaleResidualLayerNormScaleShift) +from fastvideo.layers.linear import ReplicatedLinear +from fastvideo.layers.mlp import MLP +from fastvideo.layers.rotary_embedding import get_rotary_pos_embed +from fastvideo.layers.visual_embedding import (PatchEmbed, + WanCamControlPatchEmbedding) +from fastvideo.logger import init_logger +from fastvideo.models.dits.base import BaseDiT +from fastvideo.models.dits.wanvideo import (WanI2VCrossAttention, + WanTimeTextImageEmbedding) +from fastvideo.platforms import AttentionBackendEnum, current_platform + + +logger = init_logger(__name__) + + +class LingBotWorldCamConditioner(nn.Module): + + def __init__(self, dim: int) -> None: + super().__init__() + self.cam_injector = MLP(dim, dim, dim, bias=True, act_type="silu") + self.cam_scale_layer = nn.Linear(dim, dim) + self.cam_shift_layer = nn.Linear(dim, dim) + + def forward( + self, + hidden_states: torch.Tensor, + c2ws_plucker_emb: torch.Tensor | None, + ) -> torch.Tensor: + if c2ws_plucker_emb is None: + return hidden_states + assert c2ws_plucker_emb.shape == hidden_states.shape, ( + f"c2ws_plucker_emb shape must match hidden_states shape, got " + f"{tuple(c2ws_plucker_emb.shape)} vs {tuple(hidden_states.shape)}" + ) + c2ws_hidden_states = self.cam_injector(c2ws_plucker_emb) + c2ws_hidden_states = c2ws_hidden_states + c2ws_plucker_emb + cam_scale = self.cam_scale_layer(c2ws_hidden_states) + cam_shift = self.cam_shift_layer(c2ws_hidden_states) + return (1.0 + cam_scale) * hidden_states + cam_shift + + +class WanGameCrossAttention(WanI2VCrossAttention): + def forward(self, x, context, context_lens=None): + r""" + Args: + x(Tensor): Shape [B, L1, C] + context(Tensor): Shape [B, L2, C] + context_lens(Tensor): Shape [B] + """ + context_img = context + b, n, d = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.norm_q(self.to_q(x)[0]).view(b, -1, n, d) + k_img = self.norm_added_k(self.add_k_proj(context_img)[0]).view( + b, -1, n, d) + v_img = self.add_v_proj(context_img)[0].view(b, -1, n, d) + img_x = self.attn(q, k_img, v_img) + + # output + x = img_x.flatten(2) + x, _ = self.to_out(x) + return x + + +class WanGameActionTransformerBlock(nn.Module): + """ + Transformer block for WAN Action model with support for: + - Self-attention with RoPE and camera PRoPE + - Cross-attention with text/image context + - Feed-forward network with AdaLN modulation + """ + + def __init__(self, + dim: int, + ffn_dim: int, + num_heads: int, + local_attn_size: int = -1, + sink_size: int = 0, + qk_norm: str = "rms_norm_across_heads", + cross_attn_norm: bool = False, + eps: float = 1e-6, + added_kv_proj_dim: int | None = None, + supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None, + prefix: str = ""): + super().__init__() + + # 1. Self-attention + self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False) + self.to_q = ReplicatedLinear(dim, dim, bias=True) + self.to_k = ReplicatedLinear(dim, dim, bias=True) + self.to_v = ReplicatedLinear(dim, dim, bias=True) + + self.to_out = ReplicatedLinear(dim, dim, bias=True) + self.attn1 = DistributedAttention( + num_heads=num_heads, + head_size=dim // num_heads, + causal=False, + supported_attention_backends=supported_attention_backends, + prefix=f"{prefix}.attn1") + self.hidden_dim = dim + self.num_attention_heads = num_heads + self.local_attn_size = local_attn_size + dim_head = dim // num_heads + if qk_norm == "rms_norm": + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + elif qk_norm == "rms_norm_across_heads": + self.norm_q = RMSNorm(dim, eps=eps) + self.norm_k = RMSNorm(dim, eps=eps) + else: + print("QK Norm type not supported") + raise Exception + assert cross_attn_norm is True + self.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( + dim, + norm_type="layer", + eps=eps, + elementwise_affine=True, + dtype=torch.float32, + compute_dtype=torch.float32) + + # 2. Cross-attention (I2V only for now) + self.attn2 = WanGameCrossAttention(dim, + num_heads, + qk_norm=qk_norm, + eps=eps) + # norm3 for FFN input + self.norm3 = LayerNormScaleShift(dim, norm_type="layer", eps=eps, + elementwise_affine=False) + + # 3. Feed-forward + self.ffn = MLP(dim, ffn_dim, act_type="gelu_pytorch_tanh") + self.mlp_residual = ScaleResidual() + + self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + self.cam_conditioner = LingBotWorldCamConditioner(dim) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + kv_cache: dict | None = None, + crossattn_cache: dict | None = None, + current_start: int = 0, + cache_start: int | None = None, + viewmats: torch.Tensor | None = None, + Ks: torch.Tensor | None = None, + c2ws_plucker_emb: torch.Tensor | None = None, + is_cache: bool = False, + ) -> torch.Tensor: + if hidden_states.dim() == 4: + hidden_states = hidden_states.squeeze(1) + + num_frames = temb.shape[1] + frame_seqlen = hidden_states.shape[1] // num_frames + bs, seq_length, _ = hidden_states.shape + orig_dtype = hidden_states.dtype + + # Cast temb to float32 for scale/shift computation + e = self.scale_shift_table + temb.float() + assert e.shape == (bs, num_frames, 6, self.hidden_dim) + shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(6, dim=2) + + # 1. Self-attention + norm_hidden_states = (self.norm1(hidden_states.float()).unflatten( + dim=1, sizes=(num_frames, frame_seqlen)) * + (1 + scale_msa) + shift_msa).to(orig_dtype).flatten(1, 2) + query, _ = self.to_q(norm_hidden_states) + key, _ = self.to_k(norm_hidden_states) + value, _ = self.to_v(norm_hidden_states) + + if self.norm_q is not None: + query = self.norm_q(query) + if self.norm_k is not None: + key = self.norm_k(key) + + query = query.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + + attn_output, _ = self.attn1(query, key, value, freqs_cis=freqs_cis) + attn_output = attn_output.flatten(2) + attn_output, _ = self.to_out(attn_output) + attn_output = attn_output.squeeze(1) + + # Self-attention residual + norm in float32 + null_shift = null_scale = torch.tensor([0], device=hidden_states.device) + norm_hidden_states, hidden_states = self.self_attn_residual_norm( + hidden_states, attn_output, gate_msa, null_shift, null_scale) + norm_hidden_states, hidden_states = norm_hidden_states.to( + orig_dtype), hidden_states.to(orig_dtype) + # Inject camera condition + # must be applied after the self-attention residual update. + hidden_states = self.cam_conditioner(hidden_states, c2ws_plucker_emb) + norm_hidden_states = self.self_attn_residual_norm.norm(hidden_states) + norm_hidden_states = norm_hidden_states.to(orig_dtype) + + + # 2. Cross-attention + attn_output = self.attn2(norm_hidden_states.to(orig_dtype), + context=encoder_hidden_states, + context_lens=None) + # Cross-attention residual in bfloat16 + hidden_states = hidden_states + attn_output + + # norm3 for FFN input in float32 + norm_hidden_states = self.norm3( + hidden_states.float(), c_shift_msa, c_scale_msa + ).type_as(hidden_states) + + # 3. Feed-forward + ff_output = self.ffn(norm_hidden_states.to(orig_dtype)) + hidden_states = self.mlp_residual(hidden_states.float(), ff_output.float(), c_gate_msa) + hidden_states = hidden_states.to(orig_dtype) # Cast back to original dtype + + return hidden_states + +class WanLingBotTransformer3DModel(BaseDiT): + """ + WAN Action Transformer 3D Model for video generation with action conditioning. + + Extends the base WAN video model with: + - Action embedding support for controllable generation + - camera PRoPE attention for 3D-aware generation + - KV caching for autoregressive inference + """ + _fsdp_shard_conditions = WanGameVideoConfig()._fsdp_shard_conditions + _compile_conditions = WanGameVideoConfig()._compile_conditions + _supported_attention_backends = WanGameVideoConfig()._supported_attention_backends + param_names_mapping = WanGameVideoConfig().param_names_mapping + reverse_param_names_mapping = WanGameVideoConfig().reverse_param_names_mapping + lora_param_names_mapping = WanGameVideoConfig().lora_param_names_mapping + + def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> None: + super().__init__(config=config, hf_config=hf_config) + + inner_dim = config.num_attention_heads * config.attention_head_dim + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + self.attention_head_dim = config.attention_head_dim + self.in_channels = config.in_channels + self.out_channels = config.out_channels + self.num_channels_latents = config.num_channels_latents + self.patch_size = config.patch_size + self.local_attn_size = config.local_attn_size + self.inner_dim = inner_dim + + # 1. Patch & position embedding + self.patch_embedding = PatchEmbed(in_chans=config.in_channels, + embed_dim=inner_dim, + patch_size=config.patch_size, + flatten=False) + self.patch_embedding_wancamctrl = WanCamControlPatchEmbedding(in_chans=6 * 64, + embed_dim=inner_dim, + patch_size=config.patch_size) + self.c2ws_mlp = MLP(inner_dim, inner_dim, inner_dim, bias=True, act_type="silu") + + # 2. Condition embeddings (image-only) + self.condition_embedder = WanTimeTextImageEmbedding( + dim=inner_dim, + time_freq_dim=config.freq_dim, + text_embed_dim=0, + image_embed_dim=config.image_dim, + ) + + # 3. Transformer blocks + self.blocks = nn.ModuleList([ + WanGameActionTransformerBlock( + inner_dim, + config.ffn_dim, + config.num_attention_heads, + config.local_attn_size, + config.sink_size, + config.qk_norm, + config.cross_attn_norm, + config.eps, + config.added_kv_proj_dim, + supported_attention_backends=self._supported_attention_backends, + prefix=f"{config.prefix}.blocks.{i}") + for i in range(config.num_layers) + ]) + + # 4. Output norm & projection + self.norm_out = LayerNormScaleShift(inner_dim, + norm_type="layer", + eps=config.eps, + elementwise_affine=False, + dtype=torch.float32) + self.proj_out = nn.Linear( + inner_dim, config.out_channels * math.prod(config.patch_size)) + self.scale_shift_table = nn.Parameter(torch.randn(1, 2, inner_dim) / inner_dim**0.5) + + self.gradient_checkpointing = False + + # Causal-specific + self.num_frame_per_block = config.arch_config.num_frames_per_block + assert self.num_frame_per_block <= 3 + + self.__post_init__() + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor], + guidance=None, + action: torch.Tensor | None = None, + viewmats: torch.Tensor | None = None, + Ks: torch.Tensor | None = None, + c2ws_plucker_emb: torch.Tensor | None = None, + kv_cache: list[dict] | None = None, + crossattn_cache: list[dict] | None = None, + current_start: int = 0, + cache_start: int = 0, + start_frame: int = 0, + is_cache: bool = False, + **kwargs + ) -> torch.Tensor: + """ + Forward pass for both training and inference with KV caching. + + Args: + hidden_states: Video latents [B, C, T, H, W] + encoder_hidden_states: Text embeddings [B, L, D] + timestep: Timestep tensor + encoder_hidden_states_image: Optional image embeddings + action: Action tensor [B, T] for per-frame conditioning + viewmats: Camera view matrices for PRoPE [B, T, 4, 4] + Ks: Camera intrinsics for PRoPE [B, T, 3, 3] + c2ws_plucker_emb: Camera plucker embedding [B, C, T, H, W] + kv_cache: KV cache for autoregressive inference (list of dicts per layer) + crossattn_cache: Cross-attention cache for inference + current_start: Current position for KV cache + cache_start: Cache start position + start_frame: RoPE offset for new frames in autoregressive mode + is_cache: If True, populate KV cache and return early (cache-only mode) + """ + orig_dtype = hidden_states.dtype + if isinstance(encoder_hidden_states, list) and len(encoder_hidden_states) > 0: + encoder_hidden_states = encoder_hidden_states[0] + if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: + encoder_hidden_states_image = encoder_hidden_states_image[0] + # else: + # encoder_hidden_states_image = None + + batch_size, num_channels, num_frames, height, width = hidden_states.shape + p_t, p_h, p_w = self.patch_size + post_patch_num_frames = num_frames // p_t + post_patch_height = height // p_h + post_patch_width = width // p_w + + # Get rotary embeddings + d = self.hidden_size // self.num_attention_heads + rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] + freqs_cos, freqs_sin = get_rotary_pos_embed( + (post_patch_num_frames * get_sp_world_size(), post_patch_height, post_patch_width), + self.hidden_size, + self.num_attention_heads, + rope_dim_list, + dtype=torch.float32 if current_platform.is_mps() else torch.float64, + rope_theta=10000, + start_frame=start_frame + ) + freqs_cos = freqs_cos.to(hidden_states.device) + freqs_sin = freqs_sin.to(hidden_states.device) + freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None + + hidden_states = self.patch_embedding(hidden_states) + hidden_states = hidden_states.flatten(2).transpose(1, 2) + c2ws_hidden_states = None + if c2ws_plucker_emb is not None: + c2ws_plucker_emb = self.patch_embedding_wancamctrl( + c2ws_plucker_emb.to(device=hidden_states.device, dtype=hidden_states.dtype) + ) + c2ws_hidden_states = self.c2ws_mlp(c2ws_plucker_emb) + c2ws_plucker_emb = c2ws_plucker_emb + c2ws_hidden_states + + if timestep.dim() == 1: + timestep = timestep.unsqueeze(1).expand(-1, post_patch_num_frames) + if timestep.dim() == 2: + timestep = timestep.flatten() + + if encoder_hidden_states is None or ( + isinstance(encoder_hidden_states, torch.Tensor) + and encoder_hidden_states.numel() == 0): + encoder_hidden_states = hidden_states.new_zeros((batch_size, 0, self.hidden_size)) + temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( + timestep, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) + timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) + timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, + self.hidden_size) + + encoder_hidden_states = encoder_hidden_states_image + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states.new_zeros((batch_size, 0, self.hidden_size)) + + # Transformer blocks + for block_idx, block in enumerate(self.blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + block, hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, + kv_cache[block_idx] if kv_cache else None, + crossattn_cache[block_idx] if crossattn_cache else None, + current_start, cache_start, + viewmats, Ks, c2ws_hidden_states, is_cache) + else: + hidden_states = block( + hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, + kv_cache[block_idx] if kv_cache else None, + crossattn_cache[block_idx] if crossattn_cache else None, + current_start, cache_start, + viewmats, Ks, c2ws_hidden_states, is_cache) + + # If cache-only mode, return early + if is_cache: + return kv_cache + + # Output norm, projection & unpatchify + temb = temb.view(batch_size, post_patch_num_frames, -1).unsqueeze(2) + + shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) + hidden_states = self.norm_out(hidden_states, shift, scale) + hidden_states = self.proj_out(hidden_states) + + hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, + post_patch_height, + post_patch_width, p_t, p_h, p_w, + -1) + hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) + output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + return output diff --git a/fastvideo/models/loader/fsdp_load.py b/fastvideo/models/loader/fsdp_load.py index 05596dcd3..d9a3b6150 100644 --- a/fastvideo/models/loader/fsdp_load.py +++ b/fastvideo/models/loader/fsdp_load.py @@ -343,7 +343,14 @@ def load_model_from_full_model_state_dict( unused_keys) # List of allowed parameter name patterns (whitelist for new params not in checkpoint) - ALLOWED_NEW_PARAM_PATTERNS = ["gate_compress", "proj_l", "to_out_prope", "action_embedder"] + ALLOWED_NEW_PARAM_PATTERNS = [ + "gate_compress", + "proj_l", + "to_out_prope", + "action_embedder", + "patch_embedding_wancamctrl", + "cam_conditioner", + ] # Can be extended as needed # Patterns for params that need kaiming_uniform init (input projections need non-zero for gradient flow) KAIMING_INIT_PATTERNS = ["fc_in.weight"] diff --git a/fastvideo/models/registry.py b/fastvideo/models/registry.py index 32648f22d..5e3731217 100644 --- a/fastvideo/models/registry.py +++ b/fastvideo/models/registry.py @@ -42,6 +42,8 @@ # "HunyuanVideoTransformer3DModel": ("dits", "hunyuanvideo", "HunyuanVideoDiT"), "WanTransformer3DModel": ("dits", "wanvideo", "WanTransformer3DModel"), "CausalWanTransformer3DModel": ("dits", "causal_wanvideo", "CausalWanTransformer3DModel"), + "CausalWanGameTransformer3DModel": + ("dits", "wangame", "CausalWanGameTransformer3DModel"), "WanGameActionTransformer3DModel": ("dits", "wangame", "WanGameActionTransformer3DModel"), "WanLingBotTransformer3DModel": ("dits", "wangame_lingbot", "WanLingBotTransformer3DModel"), "MatrixGameWanModel": ("dits", "matrixgame", "MatrixGameWanModel"), diff --git a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py index e2ef763ee..cd7e53d4d 100644 --- a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py @@ -71,4 +71,8 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): stage=DecodingStage(vae=self.get_module("vae"))) -EntryClass = WanGameActionImageToVideoPipeline +class WanLingBotImageToVideoPipeline(WanGameActionImageToVideoPipeline): + pass + + +EntryClass = [WanGameActionImageToVideoPipeline, WanLingBotImageToVideoPipeline] diff --git a/fastvideo/pipelines/pipeline_registry.py b/fastvideo/pipelines/pipeline_registry.py index 11222e431..8bfb0a8e6 100644 --- a/fastvideo/pipelines/pipeline_registry.py +++ b/fastvideo/pipelines/pipeline_registry.py @@ -22,6 +22,7 @@ "WanDMDPipeline": "wan", "WanImageToVideoPipeline": "wan", "WanGameActionImageToVideoPipeline": "wan", + "WanGameCausalDMDPipeline": "wan", "WanLingBotImageToVideoPipeline": "wan", "WanVideoToVideoPipeline": "wan", "WanCausalDMDPipeline": "wan", diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index 3791e725c..44526b3b6 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -179,6 +179,14 @@ def forward( "action": action_labels.unsqueeze(0).to(get_local_torch_device(), dtype=target_dtype), }, ) + # from fastvideo.models.dits.wangame_lingbot.cam_utils import process_custom_actions as process_lingbot_actions + # num_frames = batch.num_frames + # latent_height = batch.height // 8 + # latent_width = batch.width // 8 + # c2ws_plucker_emb = process_lingbot_actions( + # num_frames, batch.keyboard_cond, batch.mouse_cond, + # latent_height=latent_height, latent_width=latent_width + # ).to(get_local_torch_device(), dtype=target_dtype) else: camera_action_kwargs = {} @@ -187,6 +195,7 @@ def forward( { "mouse_cond": batch.mouse_cond, "keyboard_cond": batch.keyboard_cond, + "c2ws_plucker_emb": batch.c2ws_plucker_emb, }, ) diff --git a/fastvideo/training/wangame_lingbot_training_pipeline.py b/fastvideo/training/wangame_lingbot_training_pipeline.py new file mode 100644 index 000000000..eaeb1ea47 --- /dev/null +++ b/fastvideo/training/wangame_lingbot_training_pipeline.py @@ -0,0 +1,411 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys +from copy import deepcopy +from typing import Any + +import numpy as np +import torch + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame_lingbot +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.forward_context import set_forward_context +from fastvideo.logger import init_logger +from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler) +from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import WanLingBotImageToVideoPipeline +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch +from fastvideo.training.training_pipeline import TrainingPipeline +from fastvideo.utils import is_vsa_available, shallow_asdict + +vsa_available = is_vsa_available() + +logger = init_logger(__name__) + + +class WanLingBotTrainingPipeline(TrainingPipeline): + """ + A training pipeline for WanGame-2.1-Fun-1.3B-InP. + """ + _required_config_modules = ["scheduler", "transformer", "vae"] + + def initialize_pipeline(self, fastvideo_args: FastVideoArgs): + self.modules["scheduler"] = FlowUniPCMultistepScheduler( + shift=fastvideo_args.pipeline_config.flow_shift) + + def create_training_stages(self, training_args: TrainingArgs): + """ + May be used in future refactors. + """ + pass + + def set_schemas(self): + self.train_dataset_schema = pyarrow_schema_wangame_lingbot + + def set_trainable(self) -> None: + """ + Override to only train newly added action-related parameters for Lingbot: + - patch_embedding_wancamctrl: embeds camera Plucker coordinates + - blocks.*.cam_conditioner: injects camera conditioning into transformer blocks + """ + train_action_only = getattr(self.fastvideo_args, "train_action_only", False) + + if not train_action_only: + # Default behavior: train all parameters + super().set_trainable() + return + + # Freeze all transformer parameters first + transformer = self.get_module("transformer") + transformer.train() + transformer.requires_grad_(False) + + # Define which parameter name patterns to train + action_param_patterns = [ + "patch_embedding_wancamctrl", + "cam_conditioner", + ] + + # Enable gradients for action-related parameters only + trainable_count = 0 + frozen_count = 0 + for name, param in transformer.named_parameters(): + should_train = any(pattern in name for pattern in action_param_patterns) + if should_train: + param.requires_grad_(True) + trainable_count += 1 + logger.info(f"Trainable: {name} ({param.numel()} params)") + else: + frozen_count += 1 + + logger.info(f"Action-only training: {trainable_count} trainable param groups, " + f"{frozen_count} frozen param groups") + + # ── Action module warmup ────────────────────────────────────────────── + # For the first `action_warmup_steps`, action modules (action_embedder, + # to_out_prope) have requires_grad=False so the base model stabilizes + # first. After warmup the gradients are re-enabled. + + _ACTION_PARAM_PATTERNS = [ + "patch_embedding_wancamctrl", + "cam_conditioner", + ] + + def _set_action_params_grad(self, requires_grad: bool) -> None: + """Toggle requires_grad for action-related parameters.""" + transformer = self.get_module("transformer") + count = 0 + for name, param in transformer.named_parameters(): + if any(p in name for p in self._ACTION_PARAM_PATTERNS): + param.requires_grad_(requires_grad) + count += 1 + state = "enabled" if requires_grad else "disabled" + logger.info("Gradients %s for %d action parameter groups", state, count) + + def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: + step = training_batch.current_timestep + warmup_steps = self.training_args.action_warmup_steps + + if warmup_steps > 0: + if step == 1: + # Freeze action params at the very first step + self._set_action_params_grad(False) + logger.info( + "Action warmup: freezing action modules for the first " + "%d steps to stabilize base model", warmup_steps) + elif step == warmup_steps + 1: + # Unfreeze action params once warmup is done + self._set_action_params_grad(True) + logger.info( + "Action warmup complete — action modules unfrozen at " + "step %d", step) + + return super().train_one_step(training_batch) + + def initialize_validation_pipeline(self, training_args: TrainingArgs): + logger.info("Initializing validation pipeline...") + # args_copy.pipeline_config.vae_config.load_encoder = False + # validation_pipeline = WanImageToVideoValidationPipeline.from_pretrained( + self.validation_pipeline = WanLingBotImageToVideoPipeline.from_pretrained( + training_args.model_path, + args=None, + inference_mode=True, + loaded_modules={ + "transformer": self.get_module("transformer"), + }, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + dit_cpu_offload=False) + + def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: + batch = next(self.train_loader_iter, None) # type: ignore + if batch is None: + self.current_epoch += 1 + logger.info("Starting epoch %s", self.current_epoch) + # Reset iterator for next epoch + self.train_loader_iter = iter(self.train_dataloader) + # Get first batch of new epoch + batch = next(self.train_loader_iter) + + latents = batch['vae_latent'] + latents = latents[:, :, :self.training_args.num_latent_t] + # encoder_hidden_states = batch['text_embedding'] + # encoder_attention_mask = batch['text_attention_mask'] + clip_features = batch['clip_feature'] + image_latents = batch['first_frame_latent'] + image_latents = image_latents[:, :, :self.training_args.num_latent_t] + pil_image = batch['pil_image'] + infos = batch['info_list'] + + training_batch.latents = latents.to(get_local_torch_device(), + dtype=torch.bfloat16) + training_batch.encoder_hidden_states = None + training_batch.encoder_attention_mask = None + training_batch.preprocessed_image = pil_image.to( + get_local_torch_device()) + training_batch.image_embeds = clip_features.to(get_local_torch_device()) + training_batch.image_latents = image_latents.to( + get_local_torch_device()) + training_batch.infos = infos + + # Action conditioning + if 'mouse_cond' in batch and batch['mouse_cond'].numel() > 0: + training_batch.mouse_cond = batch['mouse_cond'].to( + get_local_torch_device(), dtype=torch.bfloat16) + else: + training_batch.mouse_cond = None + + if 'keyboard_cond' in batch and batch['keyboard_cond'].numel() > 0: + training_batch.keyboard_cond = batch['keyboard_cond'].to( + get_local_torch_device(), dtype=torch.bfloat16) + else: + training_batch.keyboard_cond = None + + # Validate action temporal dimensions match video num_frames + expected_num_frames = (self.training_args.num_latent_t - 1) * 4 + 1 + if training_batch.keyboard_cond is not None: + assert training_batch.keyboard_cond.shape[1] == expected_num_frames, ( + f"keyboard_cond temporal dim {training_batch.keyboard_cond.shape[1]} " + f"!= expected {expected_num_frames} " + f"(num_latent_t={self.training_args.num_latent_t})") + if training_batch.mouse_cond is not None: + assert training_batch.mouse_cond.shape[1] == expected_num_frames, ( + f"mouse_cond temporal dim {training_batch.mouse_cond.shape[1]} " + f"!= expected {expected_num_frames} " + f"(num_latent_t={self.training_args.num_latent_t})") + + return training_batch + + def _prepare_dit_inputs(self, + training_batch: TrainingBatch) -> TrainingBatch: + """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" + + # First, call parent method to prepare noise, timesteps, etc. for video latents + training_batch = super()._prepare_dit_inputs(training_batch) + + assert isinstance(training_batch.image_latents, torch.Tensor) + image_latents = training_batch.image_latents.to( + get_local_torch_device(), dtype=torch.bfloat16) + + temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (self.training_args.num_latent_t - + 1) * temporal_compression_ratio + 1 + batch_size, num_channels, _, latent_height, latent_width = image_latents.shape + mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, + latent_width) + mask_lat_size[:, :, 1:] = 0 + + first_frame_mask = mask_lat_size[:, :, :1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, dim=2, repeats=temporal_compression_ratio) + mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], + dim=2) + mask_lat_size = mask_lat_size.view(batch_size, -1, + temporal_compression_ratio, + latent_height, latent_width) + mask_lat_size = mask_lat_size.transpose(1, 2) + mask_lat_size = mask_lat_size.to( + image_latents.device).to(dtype=torch.bfloat16) + + training_batch.noisy_model_input = torch.cat( + [training_batch.noisy_model_input, mask_lat_size, image_latents], + dim=1) + + return training_batch + + def _build_input_kwargs(self, + training_batch: TrainingBatch) -> TrainingBatch: + + # Image Embeds for conditioning + image_embeds = training_batch.image_embeds + assert torch.isnan(image_embeds).sum() == 0 + image_embeds = image_embeds.to(get_local_torch_device(), + dtype=torch.bfloat16) + encoder_hidden_states_image = image_embeds + + from fastvideo.models.dits.wangame_lingbot.cam_utils import process_custom_actions + + # Process actions for each batch sample + batch_size = training_batch.noisy_model_input.shape[0] + num_latent_t = training_batch.noisy_model_input.shape[2] + latent_height = training_batch.noisy_model_input.shape[3] + latent_width = training_batch.noisy_model_input.shape[4] + + temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (num_latent_t - 1) * temporal_compression_ratio + 1 + + c2ws_plucker_emb_list = [] + for b in range(batch_size): + # Lingbot's process_custom_actions returns [1, 6*spatial_scale^2, lat_f, H_lat, W_lat] + c2ws_plucker_emb = process_custom_actions( + num_frames=num_frames, + keyboard_cond=training_batch.keyboard_cond[b], + mouse_cond=training_batch.mouse_cond[b], + latent_height=latent_height, + latent_width=latent_width + ) + c2ws_plucker_emb_list.append(c2ws_plucker_emb) + + c2ws_plucker_emb = torch.cat(c2ws_plucker_emb_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) + + # c2ws_plucker_emb: [B, C, lat_f, H_lat, W_lat] + assert c2ws_plucker_emb.shape[2] == num_latent_t, ( + f"c2ws_plucker_emb temporal dim {c2ws_plucker_emb.shape[2]} != " + f"video latent temporal dim {num_latent_t}") + + training_batch.input_kwargs = { + "hidden_states": + training_batch.noisy_model_input, + "encoder_hidden_states": + training_batch.encoder_hidden_states, # None (no text conditioning) + "timestep": + training_batch.timesteps.to(get_local_torch_device(), + dtype=torch.bfloat16), + # "encoder_attention_mask": + # training_batch.encoder_attention_mask, + "encoder_hidden_states_image": + encoder_hidden_states_image, + # Action conditioning + "c2ws_plucker_emb": c2ws_plucker_emb, + "return_dict": + False, + } + return training_batch + + def _prepare_validation_batch(self, sampling_param: SamplingParam, + training_args: TrainingArgs, + validation_batch: dict[str, Any], + num_inference_steps: int) -> ForwardBatch: + sampling_param.prompt = validation_batch['prompt'] + sampling_param.height = training_args.num_height + sampling_param.width = training_args.num_width + sampling_param.image_path = validation_batch.get( + 'image_path') or validation_batch.get('video_path') + sampling_param.num_inference_steps = num_inference_steps + sampling_param.data_type = "video" + assert self.seed is not None + sampling_param.seed = self.seed + + latents_size = [(sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, sampling_param.width // 8] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (training_args.num_latent_t - + 1) * temporal_compression_factor + 1 + sampling_param.num_frames = num_frames + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=torch.Generator(device="cpu").manual_seed(self.seed), + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=training_args.VSA_sparsity, + ) + if "image" in validation_batch and validation_batch["image"] is not None: + batch.pil_image = validation_batch["image"] + + if "keyboard_cond" in validation_batch and validation_batch[ + "keyboard_cond"] is not None: + keyboard_cond = validation_batch["keyboard_cond"] + keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) + keyboard_cond = keyboard_cond.unsqueeze(0) + batch.keyboard_cond = keyboard_cond + + if "mouse_cond" in validation_batch and validation_batch[ + "mouse_cond"] is not None: + mouse_cond = validation_batch["mouse_cond"] + mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) + mouse_cond = mouse_cond.unsqueeze(0) + batch.mouse_cond = mouse_cond + + return batch + + def _post_process_validation_frames(self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: + """Apply action overlay to validation frames for WanGame. + + Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. + """ + # Check if action data is available + keyboard_cond = getattr(batch, 'keyboard_cond', None) + mouse_cond = getattr(batch, 'mouse_cond', None) + + if keyboard_cond is None and mouse_cond is None: + return frames + + # Import overlay functions + from fastvideo.models.dits.matrixgame.utils import ( + draw_keys_on_frame, draw_mouse_on_frame) + + # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) + if keyboard_cond is not None: + keyboard_cond = keyboard_cond.squeeze(0).cpu().float().numpy() # (T, 6) + if mouse_cond is not None: + mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) + + # MatrixGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] + key_names = ["W", "S", "A", "D", "left", "right"] + + processed_frames = [] + for frame_idx, frame in enumerate(frames): + frame = np.ascontiguousarray(frame.copy()) + + # Draw keyboard overlay + if keyboard_cond is not None and frame_idx < len(keyboard_cond): + keys = {key_names[i]: bool(keyboard_cond[frame_idx, i]) + for i in range(min(len(key_names), keyboard_cond.shape[1]))} + draw_keys_on_frame(frame, keys, mode='universal') + + # Draw mouse overlay + if mouse_cond is not None and frame_idx < len(mouse_cond): + pitch = float(mouse_cond[frame_idx, 0]) + yaw = float(mouse_cond[frame_idx, 1]) + draw_mouse_on_frame(frame, pitch, yaw) + + processed_frames.append(frame) + + return processed_frames + + +def main(args) -> None: + logger.info("Starting training pipeline...") + + pipeline = WanLingBotTrainingPipeline.from_pretrained( + args.pretrained_model_name_or_path, args=args) + args = pipeline.training_args + pipeline.train() + logger.info("Training pipeline done") + + +if __name__ == "__main__": + argv = sys.argv + from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.utils import FlexibleArgumentParser + parser = FlexibleArgumentParser() + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args() + args.dit_cpu_offload = False + main(args) \ No newline at end of file From 44418671e6a9fd2aaf1dbd3ec4e4b7f625329153 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Tue, 10 Feb 2026 01:42:50 +0000 Subject: [PATCH 019/214] update script --- .../finetune_ode_init.sh | 97 ++++++++++++++ .../finetune_ode_init.slurm | 123 ++++++++++++++++++ .../launch_preprocess_slurm.sh | 20 +++ .../ode_finetune_worker.slurm | 61 +++++++++ .../preprocess_worker.slurm | 62 +++++++++ .../causal_wangame_ode_init/validation.json | 84 ++++++++++++ 6 files changed, 447 insertions(+) create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/ode_finetune_worker.slurm create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/validation.json diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh new file mode 100644 index 000000000..897f88fdd --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" +export WANDB_BASE_URL="https://api.wandb.ai" +export WANDB_MODE=online +export TOKENIZERS_PARALLELISM=false + +# MODEL_PATH="FastVideo/Matrix-Game-2.0-Foundation-Diffusers" +# MODEL_PATH="Matrix-Game-2.0-Foundation-Diffusers" +MODEL_PATH="" +DATA_DIR="../vizdoom/preprocessed" +VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_matrixgame_ode_init/validation_vizdoom.json" +NUM_GPUS=8 +# export CUDA_VISIBLE_DEVICES=4,5,6,7 +# IP=[MASTER NODE IP] + +# Training arguments +training_args=( + --tracker_project_name "matrixgame_ode_init_vizdoom_8gpu" + --output_dir "checkpoints/matrixgame_ode_init_vizdoom_8gpu" + --wandb_run_name "0202_2115_steps1000_bs_32" + --max_train_steps 1000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 4 + --num_latent_t 21 + --num_height 352 + --num_width 640 + --num_frames 81 + --warp_denoising_step + --enable_gradient_checkpointing_type "full" +) + +# Parallel arguments +parallel_args=( + --num_gpus $NUM_GPUS + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim $NUM_GPUS +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 1 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 100 + --validation_sampling_steps "50" + --validation_guidance_scale "6.0" +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 6e-6 + --mixed_precision "bf16" + --weight_only_checkpointing_steps 100 + --training_state_checkpointing_steps 100 + --weight_decay 1e-4 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.1 + --multi_phased_distill_schedule "4000-1" + --not_apply_cfg_solver + --dit_precision "fp32" + --num_euler_timesteps 50 + --ema_start_step 0 +) + +# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t +torchrun \ + --nnodes 1 \ + --nproc_per_node $NUM_GPUS \ + fastvideo/training/matrixgame_ode_causal_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm new file mode 100644 index 000000000..c10314c0e --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm @@ -0,0 +1,123 @@ +#!/bin/bash +#SBATCH --job-name=mg-ode-64g +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=ode_train_output/ode_%j.out +#SBATCH --error=ode_train_output/ode_%j.err +#SBATCH --exclusive + +set -e -x + +# Environment Setup +source ~/conda/miniconda/bin/activate +conda activate mhuo-fv +export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" + +# Basic Info +export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" +export WANDB_MODE="online" +export NCCL_P2P_DISABLE=1 +export MASTER_PORT=29500 +export NODE_RANK=$SLURM_PROCID +nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) +export MASTER_ADDR=${nodes[0]} +export TOKENIZERS_PARALLELISM=false + +echo "MASTER_ADDR: $MASTER_ADDR" +echo "NODE_RANK: $NODE_RANK" + +# Configs +MODEL_PATH="" +DATA_DIR="../vizdoom/preprocessed" +VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_matrixgame_ode_init/validation_vizdoom.json" + +# Training arguments +training_args=( + --tracker_project_name "matrixgame_ode_init_vizdoom" + --output_dir "checkpoints/matrixgame_ode_init_vizdoom" + --wandb_run_name "0202_2300_steps2000_bs_32" + --max_train_steps 2000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 21 + --num_height 352 + --num_width 640 + --num_frames 81 + --warp_denoising_step + --enable_gradient_checkpointing_type "full" +) + +# Parallel arguments +parallel_args=( + --num_gpus 64 + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim 64 +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 200 + --validation_sampling_steps "50" + --validation_guidance_scale "6.0" +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 6e-6 + --mixed_precision "bf16" + --weight_only_checkpointing_steps 500 + --training_state_checkpointing_steps 500 + --weight_decay 1e-4 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.1 + --multi_phased_distill_schedule "4000-1" + --not_apply_cfg_solver + --dit_precision "fp32" + --num_euler_timesteps 50 + --ema_start_step 0 +) + +mkdir -p ode_train_output + +srun torchrun \ +--nnodes $SLURM_JOB_NUM_NODES \ +--nproc_per_node 8 \ +--node_rank $SLURM_PROCID \ +--rdzv_backend=c10d \ +--rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ + fastvideo/training/matrixgame_ode_causal_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh new file mode 100644 index 000000000..78fab2bb8 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Create output directory if it doesn't exist +mkdir -p preprocess_output + +# Launch 8 jobs, one for each node (Total 64 GPUs) +# Each node processes 8 consecutive files (64 total files / 8 nodes = 8 files per node) +for node_id in {0..3}; do + # Calculate the starting file number for this node + start_file=$((node_id * 8)) + + echo "Launching node $node_id with files merge_${start_file}.txt to merge_$((start_file + 7)).txt" + + sbatch --job-name=mg-pre-${node_id} \ + --output=preprocess_output/mg-node-${node_id}.out \ + --error=preprocess_output/mg-node-${node_id}.err \ + $(pwd)/FastVideo_kaiqin/examples/training/consistency_finetune/causal_matrixgame_ode_init/preprocess_worker.slurm $start_file $node_id +done + +echo "All 4 nodes (32 GPUs) launched successfully!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/ode_finetune_worker.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/ode_finetune_worker.slurm new file mode 100644 index 000000000..6311b44fa --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/ode_finetune_worker.slurm @@ -0,0 +1,61 @@ +#!/bin/bash +#SBATCH --partition=main +#SBATCH --qos=hao +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=960G +#SBATCH --exclusive +#SBATCH --time=72:00:00 + +# conda init +source ~/conda/miniconda/bin/activate +conda activate fastvideo_kaiqin + +# Accept parameters from launch script +START_FILE=${1:-0} # Starting file number for this node +NODE_ID=${2:-0} # Node identifier (0-7) + +MODEL_PATH="../Matrix-Game-2.0-Base-Diffusers" +OUTPUT_BASE="../FastvideoWorldModel-MC/preprocessed" + +# Port range calculation +base_port=$((29700 + NODE_ID * 100)) # Using a different port range to avoid collision with other tasks +gpu_ids=(0 1 2 3 4 5 6 7) + +for i in {1..8}; do + port=$((base_port + i)) + gpu=${gpu_ids[((i-1))]} + file_num=$((START_FILE + i - 1)) + + DATA_MERGE_PATH="../FastvideoWorldModel-MC/gen/merge_${file_num}.txt" + OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" + + # CPU binding + start_cpu=$(( (i-1)*2 )) + end_cpu=$(( start_cpu+1 )) + + echo "Starting GPU $gpu processing file merge_${file_num}.txt on port $port" + + CUDA_VISIBLE_DEVICES=$gpu taskset -c ${start_cpu}-${end_cpu} torchrun --nnodes=1 --nproc_per_node=1 --master_port $port \ + FastVideo/fastvideo/pipelines/preprocess/v1_preprocess.py \ + --model_path $MODEL_PATH \ + --data_merge_path $DATA_MERGE_PATH \ + --preprocess_video_batch_size 1 \ + --seed 42 \ + --max_height 352 \ + --max_width 640 \ + --num_frames 81 \ + --flow_shift 5.0 \ + --dataloader_num_workers 0 \ + --output_dir=$OUTPUT_DIR \ + --train_fps 25 \ + --samples_per_file 8 \ + --flush_frequency 8 \ + --video_length_tolerance_range 5 \ + --preprocess_task "matrixgame_ode_trajectory" & +done + +wait +echo "Node $NODE_ID ODE preprocessing blocks completed!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm new file mode 100644 index 000000000..617b135d9 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm @@ -0,0 +1,62 @@ +#!/bin/bash +#SBATCH --partition=main +#SBATCH --qos=hao +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=960G +#SBATCH --exclusive +#SBATCH --time=72:00:00 + +# conda init +source ~/conda/miniconda/bin/activate +conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv +export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin:$PYTHONPATH" + +# Accept parameters from launch script +START_FILE=${1:-1} # Starting file number for this node +NODE_ID=${2:-0} # Node identifier (0-7) + +MODEL_PATH="./Matrix-Game-2.0-Foundation-VizDoom1k-1000steps-Diffusers" +OUTPUT_BASE="traindata_0209_1500/mg_ode_init/preprocessed" + +# Port range calculation +base_port=$((29500 + NODE_ID * 100)) +gpu_ids=(0 1 2 3 4 5 6 7) + +for i in {1..8}; do + port=$((base_port + i)) + gpu=${gpu_ids[((i-1))]} + file_num=$((START_FILE + i - 1)) + + DATA_MERGE_PATH="traindata_0209_1500/mg_ode_init/merge_${file_num}.txt" + OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" + echo "DATA_MERGE_PATH: $DATA_MERGE_PATH" + echo "OUTPUT_DIR: $OUTPUT_DIR" + + # CPU binding (optional, kept from syn.slurm logic) + start_cpu=$(( (i-1)*2 )) + end_cpu=$(( start_cpu+1 )) + + echo "Starting GPU $gpu processing file merge_${file_num}.txt on port $port" + + CUDA_VISIBLE_DEVICES=$gpu taskset -c ${start_cpu}-${end_cpu} torchrun --nnodes=1 --nproc_per_node=1 --master_port $port \ + FastVideo/fastvideo/pipelines/preprocess/v1_preprocess.py \ + --model_path $MODEL_PATH \ + --data_merge_path $DATA_MERGE_PATH \ + --preprocess_video_batch_size 1 \ + --seed 42 \ + --max_height 352 \ + --max_width 640 \ + --num_frames 81 \ + --dataloader_num_workers 0 \ + --output_dir=$OUTPUT_DIR \ + --samples_per_file 8 \ + --train_fps 25 \ + --flush_frequency 8 \ + --preprocess_task matrixgame_ode_trajectory & +done + +wait +echo "Node $NODE_ID processing blocks completed!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json new file mode 100644 index 000000000..649826443 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json @@ -0,0 +1,84 @@ +{ + "data": [ + { + "caption": "00. Hold [W] + Static", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000000.jpg", + "action_path": "action/000000_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "01. Hold [S] + Static", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000001.jpg", + "action_path": "action/000001_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02. Hold [A] + Static", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000002.jpg", + "action_path": "action/000002_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10. No Key + Hold [left]", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000003.jpg", + "action_path": "action/000010_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12. No Key + Hold [up_right]", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000004.jpg", + "action_path": "action/000012_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "21. Switch [W]->[D] + Static", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000005.jpg", + "action_path": "action/000021_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "29. No Key + Switch [up_right]->[up_left]", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000006.jpg", + "action_path": "action/000029_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "50. Hold [W] + Switch [up]->[down]", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000007.jpg", + "action_path": "action/000050_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + } + ] +} \ No newline at end of file From 97c4fc66de52882b6c1c93d9734727951f245eb7 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Mon, 9 Feb 2026 21:30:34 -0800 Subject: [PATCH 020/214] draft code --- .../finetune_ode_init.sh | 95 +++ .../finetune_ode_init.slurm | 124 +++ .../launch_preprocess_slurm.sh | 20 + .../preprocess_worker.slurm | 61 ++ .../causal_wangame_ode_init/validation.json | 84 +++ fastvideo/models/dits/wangame/__init__.py | 6 +- fastvideo/models/dits/wangame/causal_model.py | 704 ++++++++++++++++++ fastvideo/models/loader/component_loader.py | 3 +- fastvideo/models/registry.py | 4 +- .../basic/wan/wangame_causal_dmd_pipeline.py | 71 ++ .../pipelines/preprocess/v1_preprocess.py | 9 +- ...game_preprocess_pipeline_ode_trajectory.py | 490 ++++++++++++ .../pipelines/stages/causal_denoising.py | 104 ++- .../training/wangame_ode_causal_pipeline.py | 556 ++++++++++++++ 14 files changed, 2315 insertions(+), 16 deletions(-) create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/validation.json create mode 100644 fastvideo/models/dits/wangame/causal_model.py create mode 100644 fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py create mode 100644 fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py create mode 100644 fastvideo/training/wangame_ode_causal_pipeline.py diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh new file mode 100644 index 000000000..d2ffb8a46 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +export WANDB_BASE_URL="https://api.wandb.ai" +export WANDB_MODE=online +export TOKENIZERS_PARALLELISM=false + +MODEL_PATH="" +DATA_DIR="../vizdoom/preprocessed" +VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" +NUM_GPUS=8 +# export CUDA_VISIBLE_DEVICES=4,5,6,7 +# IP=[MASTER NODE IP] + +# Training arguments +training_args=( + --tracker_project_name "wangame_ode_init" + --output_dir "checkpoints/wangame_ode_init" + --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" + --wandb_run_name "wangame_ode_init_8gpu" + --max_train_steps 2000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 4 + --num_latent_t 21 + --num_height 352 + --num_width 640 + --num_frames 81 + --warp_denoising_step + --enable_gradient_checkpointing_type "full" +) + +# Parallel arguments +parallel_args=( + --num_gpus $NUM_GPUS + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim $NUM_GPUS +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 1 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 100 + --validation_sampling_steps "50" + --validation_guidance_scale "6.0" +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 6e-6 + --mixed_precision "bf16" + --weight_only_checkpointing_steps 100 + --training_state_checkpointing_steps 100 + --weight_decay 1e-4 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.1 + --multi_phased_distill_schedule "4000-1" + --not_apply_cfg_solver + --dit_precision "fp32" + --num_euler_timesteps 50 + --ema_start_step 0 +) + +# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t +torchrun \ + --nnodes 1 \ + --nproc_per_node $NUM_GPUS \ + fastvideo/training/wangame_ode_causal_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm new file mode 100644 index 000000000..54097b411 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm @@ -0,0 +1,124 @@ +#!/bin/bash +#SBATCH --job-name=wg-ode-64g +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=ode_train_output/ode_%j.out +#SBATCH --error=ode_train_output/ode_%j.err +#SBATCH --exclusive + +set -e -x + +# Environment Setup +source ~/conda/miniconda/bin/activate +conda activate your-conda-env +export PYTHONPATH="/path/to/FastVideo:$PYTHONPATH" + +# Basic Info +export WANDB_API_KEY="your-wandb-api-key" +export WANDB_MODE="online" +export NCCL_P2P_DISABLE=1 +export MASTER_PORT=29500 +export NODE_RANK=$SLURM_PROCID +nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) +export MASTER_ADDR=${nodes[0]} +export TOKENIZERS_PARALLELISM=false + +echo "MASTER_ADDR: $MASTER_ADDR" +echo "NODE_RANK: $NODE_RANK" + +# Configs +MODEL_PATH="" +DATA_DIR="../vizdoom/preprocessed" +VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" + +# Training arguments +training_args=( + --tracker_project_name "wangame_ode_init" + --output_dir "checkpoints/wangame_ode_init" + --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" + --wandb_run_name "wangame_ode_init_64gpu" + --max_train_steps 2000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 21 + --num_height 352 + --num_width 640 + --num_frames 81 + --warp_denoising_step + --enable_gradient_checkpointing_type "full" +) + +# Parallel arguments +parallel_args=( + --num_gpus 64 + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim 64 +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 200 + --validation_sampling_steps "50" + --validation_guidance_scale "6.0" +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 6e-6 + --mixed_precision "bf16" + --weight_only_checkpointing_steps 500 + --training_state_checkpointing_steps 500 + --weight_decay 1e-4 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.1 + --multi_phased_distill_schedule "4000-1" + --not_apply_cfg_solver + --dit_precision "fp32" + --num_euler_timesteps 50 + --ema_start_step 0 +) + +mkdir -p ode_train_output + +srun torchrun \ +--nnodes $SLURM_JOB_NUM_NODES \ +--nproc_per_node 8 \ +--node_rank $SLURM_PROCID \ +--rdzv_backend=c10d \ +--rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ + fastvideo/training/wangame_ode_causal_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh new file mode 100644 index 000000000..7e236fe91 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Create output directory if it doesn't exist +mkdir -p preprocess_output + +# Launch 8 jobs, one for each node (Total 64 GPUs) +# Each node processes 8 consecutive files (64 total files / 8 nodes = 8 files per node) +for node_id in {0..3}; do + # Calculate the starting file number for this node + start_file=$((node_id * 8)) + + echo "Launching node $node_id with files merge_${start_file}.txt to merge_$((start_file + 7)).txt" + + sbatch --job-name=wg-pre-${node_id} \ + --output=preprocess_output/wg-node-${node_id}.out \ + --error=preprocess_output/wg-node-${node_id}.err \ + $(pwd)/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm $start_file $node_id +done + +echo "All 4 nodes (32 GPUs) launched successfully!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm new file mode 100644 index 000000000..c9f176323 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm @@ -0,0 +1,61 @@ +#!/bin/bash +#SBATCH --partition=main +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=8 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=960G +#SBATCH --exclusive +#SBATCH --time=72:00:00 + +# conda init +source ~/conda/miniconda/bin/activate +conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv +export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin:$PYTHONPATH" + +# Accept parameters from launch script +START_FILE=${1:-1} # Starting file number for this node +NODE_ID=${2:-0} # Node identifier (0-7) + +MODEL_PATH="./Matrix-Game-2.0-Foundation-VizDoom1k-1000steps-Diffusers" +OUTPUT_BASE="traindata_0209_1500/mg_ode_init/preprocessed" + +# Port range calculation +base_port=$((29500 + NODE_ID * 100)) +gpu_ids=(0 1 2 3 4 5 6 7) + +for i in {1..8}; do + port=$((base_port + i)) + gpu=${gpu_ids[((i-1))]} + file_num=$((START_FILE + i - 1)) + + DATA_MERGE_PATH="traindata_0209_1500/mg_ode_init/merge_${file_num}.txt" + OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" + echo "DATA_MERGE_PATH: $DATA_MERGE_PATH" + echo "OUTPUT_DIR: $OUTPUT_DIR" + + # CPU binding (optional, kept from syn.slurm logic) + start_cpu=$(( (i-1)*2 )) + end_cpu=$(( start_cpu+1 )) + + echo "Starting GPU $gpu processing file merge_${file_num}.txt on port $port" + + CUDA_VISIBLE_DEVICES=$gpu taskset -c ${start_cpu}-${end_cpu} torchrun --nnodes=1 --nproc_per_node=1 --master_port $port \ + FastVideo/fastvideo/pipelines/preprocess/v1_preprocess.py \ + --model_path $MODEL_PATH \ + --data_merge_path $DATA_MERGE_PATH \ + --preprocess_video_batch_size 1 \ + --seed 42 \ + --max_height 352 \ + --max_width 640 \ + --num_frames 81 \ + --dataloader_num_workers 0 \ + --output_dir=$OUTPUT_DIR \ + --samples_per_file 8 \ + --train_fps 25 \ + --flush_frequency 8 \ + --preprocess_task wangame_ode_trajectory & +done + +wait +echo "Node $NODE_ID processing blocks completed!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json new file mode 100644 index 000000000..649826443 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json @@ -0,0 +1,84 @@ +{ + "data": [ + { + "caption": "00. Hold [W] + Static", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000000.jpg", + "action_path": "action/000000_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "01. Hold [S] + Static", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000001.jpg", + "action_path": "action/000001_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02. Hold [A] + Static", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000002.jpg", + "action_path": "action/000002_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10. No Key + Hold [left]", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000003.jpg", + "action_path": "action/000010_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12. No Key + Hold [up_right]", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000004.jpg", + "action_path": "action/000012_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "21. Switch [W]->[D] + Static", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000005.jpg", + "action_path": "action/000021_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "29. No Key + Switch [up_right]->[up_left]", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000006.jpg", + "action_path": "action/000029_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "50. Hold [W] + Switch [up]->[down]", + "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000007.jpg", + "action_path": "action/000050_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + } + ] +} \ No newline at end of file diff --git a/fastvideo/models/dits/wangame/__init__.py b/fastvideo/models/dits/wangame/__init__.py index ebe540626..1d799e506 100644 --- a/fastvideo/models/dits/wangame/__init__.py +++ b/fastvideo/models/dits/wangame/__init__.py @@ -1,12 +1,10 @@ from .model import WanGameActionTransformer3DModel -from .causal_model import (CausalWanGameTransformer3DModel, - CausalWanTransformer3DModel) +from .causal_model import CausalWanGameActionTransformer3DModel from .hyworld_action_module import WanGameActionTimeImageEmbedding, WanGameActionSelfAttention __all__ = [ "WanGameActionTransformer3DModel", - "CausalWanTransformer3DModel", - "CausalWanGameTransformer3DModel", + "CausalWanGameActionTransformer3DModel", "WanGameActionTimeImageEmbedding", "WanGameActionSelfAttention", ] diff --git a/fastvideo/models/dits/wangame/causal_model.py b/fastvideo/models/dits/wangame/causal_model.py new file mode 100644 index 000000000..b7c33049a --- /dev/null +++ b/fastvideo/models/dits/wangame/causal_model.py @@ -0,0 +1,704 @@ +# SPDX-License-Identifier: Apache-2.0 + +import math +from typing import Any + +import torch +import torch.nn as nn + +from torch.nn.attention.flex_attention import create_block_mask, flex_attention +from torch.nn.attention.flex_attention import BlockMask +# wan 1.3B model has a weird channel / head configurations and require max-autotune to work with flexattention +# see https://github.com/pytorch/pytorch/issues/133254 +# change to default for other models +flex_attention = torch.compile( + flex_attention, dynamic=False, mode="max-autotune-no-cudagraphs") +import torch.distributed as dist + +from fastvideo.attention import LocalAttention +from fastvideo.configs.models.dits.wangamevideo import WanGameVideoConfig +from fastvideo.distributed.parallel_state import get_sp_world_size +from fastvideo.layers.layernorm import (FP32LayerNorm, LayerNormScaleShift, + RMSNorm, ScaleResidual, + ScaleResidualLayerNormScaleShift) +from fastvideo.layers.linear import ReplicatedLinear +from fastvideo.layers.mlp import MLP +from fastvideo.layers.rotary_embedding import (_apply_rotary_emb, + get_rotary_pos_embed) +from fastvideo.layers.visual_embedding import PatchEmbed +from fastvideo.logger import init_logger +from fastvideo.models.dits.base import BaseDiT +from fastvideo.models.dits.wanvideo import WanI2VCrossAttention +from fastvideo.platforms import AttentionBackendEnum, current_platform + +# Import ActionModule +from fastvideo.models.dits.wangame.hyworld_action_module import ( + WanGameActionTimeImageEmbedding, + WanGameActionSelfAttention +) + +logger = init_logger(__name__) + + +class CausalWanGameCrossAttention(WanI2VCrossAttention): + """Cross-attention for WanGame causal model""" + + def forward(self, x, context, context_lens=None, crossattn_cache=None): + r""" + Args: + x(Tensor): Shape [B, L1, C] + context(Tensor): Shape [B, L2, C] + context_lens(Tensor): Shape [B] + crossattn_cache: Optional cache dict for inference + """ + context_img = context + b, n, d = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.norm_q(self.to_q(x)[0]).view(b, -1, n, d) + + if crossattn_cache is not None: + if not crossattn_cache["is_init"]: + crossattn_cache["is_init"] = True + k_img = self.norm_added_k(self.add_k_proj(context_img)[0]).view( + b, -1, n, d) + v_img = self.add_v_proj(context_img)[0].view(b, -1, n, d) + crossattn_cache["k"] = k_img + crossattn_cache["v"] = v_img + else: + k_img = crossattn_cache["k"] + v_img = crossattn_cache["v"] + else: + k_img = self.norm_added_k(self.add_k_proj(context_img)[0]).view( + b, -1, n, d) + v_img = self.add_v_proj(context_img)[0].view(b, -1, n, d) + + img_x = self.attn(q, k_img, v_img) + + # output + x = img_x.flatten(2) + x, _ = self.to_out(x) + return x + + +class CausalWanGameActionSelfAttention(nn.Module): + """ + Causal self-attention module combining: + - WanGameActionSelfAttention (dual RoPE + PRoPE paths) for training with flex_attention + - KV caching for autoregressive inference + """ + + def __init__(self, + dim: int, + num_heads: int, + local_attn_size: int = -1, + sink_size: int = 0, + qk_norm=True, + eps=1e-6) -> None: + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.local_attn_size = local_attn_size + self.sink_size = sink_size + self.qk_norm = qk_norm + self.eps = eps + self.max_attention_size = 32760 if local_attn_size == -1 else local_attn_size * 1560 + + # Inner attention module with PRoPE support + self.attn = WanGameActionSelfAttention( + dim, + num_heads, + local_attn_size=local_attn_size, + sink_size=sink_size, + qk_norm=qk_norm, + eps=eps) + + # Separate local attention for KV cache inference + self.local_attn = LocalAttention( + num_heads=num_heads, + head_size=self.head_dim, + dropout_rate=0, + softmax_scale=None, + causal=False, + supported_attention_backends=(AttentionBackendEnum.FLASH_ATTN, + AttentionBackendEnum.TORCH_SDPA)) + + def forward(self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + block_mask: BlockMask | None = None, + kv_cache: dict | None = None, + current_start: int = 0, + cache_start: int | None = None, + viewmats: torch.Tensor | None = None, + Ks: torch.Tensor | None = None, + is_cache: bool = False): + """ + Forward pass with causal attention. + """ + if cache_start is None: + cache_start = current_start + + if kv_cache is None: + return self.attn( + q, k, v, freqs_cis, + kv_cache=None, + current_start=current_start, + cache_start=cache_start, + viewmats=viewmats, + Ks=Ks, + is_cache=is_cache + ) + else: + # Inference mode with KV cache + cos, sin = freqs_cis + roped_query = _apply_rotary_emb(q, cos, sin, is_neox_style=False).type_as(v) + roped_key = _apply_rotary_emb(k, cos, sin, is_neox_style=False).type_as(v) + + frame_seqlen = q.shape[1] + current_end = current_start + roped_query.shape[1] + sink_tokens = self.sink_size * frame_seqlen + # If we are using local attention and the current KV cache size is larger than the local attention size, we need to truncate the KV cache + kv_cache_size = kv_cache["k"].shape[1] + num_new_tokens = roped_query.shape[1] + + if self.local_attn_size != -1 and (current_end > kv_cache["global_end_index"].item()) and ( + num_new_tokens + kv_cache["local_end_index"].item() > kv_cache_size): + # Calculate the number of new tokens added in this step + # Shift existing cache content left to discard oldest tokens + # Clone the source slice to avoid overlapping memory error + num_evicted_tokens = num_new_tokens + kv_cache["local_end_index"].item() - kv_cache_size + num_rolled_tokens = kv_cache["local_end_index"].item() - num_evicted_tokens - sink_tokens + kv_cache["k"][:, sink_tokens:sink_tokens + num_rolled_tokens] = \ + kv_cache["k"][:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() + kv_cache["v"][:, sink_tokens:sink_tokens + num_rolled_tokens] = \ + kv_cache["v"][:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() + # Insert the new keys/values at the end + local_end_index = kv_cache["local_end_index"].item() + current_end - \ + kv_cache["global_end_index"].item() - num_evicted_tokens + local_start_index = local_end_index - num_new_tokens + kv_cache["k"][:, local_start_index:local_end_index] = roped_key + kv_cache["v"][:, local_start_index:local_end_index] = v + else: + # Assign new keys/values directly up to current_end + local_end_index = kv_cache["local_end_index"].item() + current_end - kv_cache["global_end_index"].item() + local_start_index = local_end_index - num_new_tokens + kv_cache["k"] = kv_cache["k"].detach() + kv_cache["v"] = kv_cache["v"].detach() + # logger.info("kv_cache['k'] is in comp graph: %s", kv_cache["k"].requires_grad or kv_cache["k"].grad_fn is not None) + kv_cache["k"][:, local_start_index:local_end_index] = roped_key + kv_cache["v"][:, local_start_index:local_end_index] = v + + x = self.local_attn( + roped_query, + kv_cache["k"][:, max(0, local_end_index - self.max_attention_size):local_end_index], + kv_cache["v"][:, max(0, local_end_index - self.max_attention_size):local_end_index] + ) + kv_cache["global_end_index"].fill_(current_end) + kv_cache["local_end_index"].fill_(local_end_index) + + # In inference mode, only return rope output; prope is zero + return x, torch.zeros_like(x) + + +class CausalWanGameActionTransformerBlock(nn.Module): + + def __init__(self, + dim: int, + ffn_dim: int, + num_heads: int, + local_attn_size: int = -1, + sink_size: int = 0, + qk_norm: str = "rms_norm_across_heads", + cross_attn_norm: bool = False, + eps: float = 1e-6, + added_kv_proj_dim: int | None = None, + supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None, + prefix: str = ""): + super().__init__() + + # 1. Self-attention + self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False) + self.to_q = ReplicatedLinear(dim, dim, bias=True) + self.to_k = ReplicatedLinear(dim, dim, bias=True) + self.to_v = ReplicatedLinear(dim, dim, bias=True) + self.to_out = ReplicatedLinear(dim, dim, bias=True) + + self.attn1 = CausalWanGameActionSelfAttention( + dim, + num_heads, + local_attn_size=local_attn_size, + sink_size=sink_size, + qk_norm=qk_norm, + eps=eps) + + self.hidden_dim = dim + self.num_attention_heads = num_heads + self.local_attn_size = local_attn_size + dim_head = dim // num_heads + + if qk_norm == "rms_norm": + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + elif qk_norm == "rms_norm_across_heads": + self.norm_q = RMSNorm(dim, eps=eps) + self.norm_k = RMSNorm(dim, eps=eps) + else: + raise ValueError(f"QK Norm type {qk_norm} not supported") + + assert cross_attn_norm is True + self.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( + dim, + norm_type="layer", + eps=eps, + elementwise_affine=True, + compute_dtype=torch.float32) + + # 2. Cross-attention (I2V only) + self.attn2 = CausalWanGameCrossAttention(dim, + num_heads, + qk_norm=qk_norm, + eps=eps) + # norm3 for FFN input + self.norm3 = LayerNormScaleShift(dim, norm_type="layer", eps=eps, + elementwise_affine=False) + + # 3. Feed-forward + self.ffn = MLP(dim, ffn_dim, act_type="gelu_pytorch_tanh") + self.mlp_residual = ScaleResidual() + + self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + + # PRoPE output projection (initialized via add_discrete_action_parameters on the model) + self.to_out_prope = ReplicatedLinear(dim, dim, bias=True) + nn.init.zeros_(self.to_out_prope.weight) + if self.to_out_prope.bias is not None: + nn.init.zeros_(self.to_out_prope.bias) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + freqs_cis: tuple[torch.Tensor, torch.Tensor], + block_mask: BlockMask | None = None, + kv_cache: dict | None = None, + crossattn_cache: dict | None = None, + current_start: int = 0, + cache_start: int | None = None, + viewmats: torch.Tensor | None = None, + Ks: torch.Tensor | None = None, + is_cache: bool = False, + ) -> torch.Tensor: + if hidden_states.dim() == 4: + hidden_states = hidden_states.squeeze(1) + + num_frames = temb.shape[1] + frame_seqlen = hidden_states.shape[1] // num_frames + bs, seq_length, _ = hidden_states.shape + orig_dtype = hidden_states.dtype + + # Cast temb to float32 for scale/shift computation + e = self.scale_shift_table + temb.float() + assert e.shape == (bs, num_frames, 6, self.hidden_dim) + shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(6, dim=2) + + # 1. Self-attention + norm_hidden_states = (self.norm1(hidden_states.float()).unflatten(dim=1, sizes=(num_frames, frame_seqlen)) * + (1 + scale_msa) + shift_msa).to(orig_dtype).flatten(1, 2) + + query, _ = self.to_q(norm_hidden_states) + key, _ = self.to_k(norm_hidden_states) + value, _ = self.to_v(norm_hidden_states) + + if self.norm_q is not None: + query = self.norm_q.forward_native(query) + if self.norm_k is not None: + key = self.norm_k.forward_native(key) + + query = query.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) + + # Self-attention with camera PRoPE + attn_output_rope, attn_output_prope = self.attn1( + query, key, value, freqs_cis, + block_mask, kv_cache, current_start, cache_start, + viewmats, Ks, is_cache=is_cache + ) + # Combine rope and prope outputs + attn_output_rope = attn_output_rope.flatten(2) + attn_output_rope, _ = self.to_out(attn_output_rope) + attn_output_prope = attn_output_prope.flatten(2) + attn_output_prope, _ = self.to_out_prope(attn_output_prope) + attn_output = attn_output_rope.squeeze(1) + attn_output_prope.squeeze(1) + + # Self-attention residual + norm in float32 + null_shift = null_scale = torch.zeros(1, device=hidden_states.device, dtype=torch.float32) + norm_hidden_states, hidden_states = self.self_attn_residual_norm( + hidden_states.float(), attn_output.float(), gate_msa, null_shift, null_scale) + hidden_states = hidden_states.type_as(attn_output) + norm_hidden_states = norm_hidden_states.type_as(attn_output) + + # 2. Cross-attention + attn_output = self.attn2(norm_hidden_states.to(orig_dtype), + context=encoder_hidden_states, + context_lens=None, + crossattn_cache=crossattn_cache) + # Cross-attention residual in bfloat16 + hidden_states = hidden_states + attn_output + + # norm3 for FFN input in float32 + norm_hidden_states = self.norm3( + hidden_states.float(), c_shift_msa, c_scale_msa + ).type_as(hidden_states) + + # 3. Feed-forward + ff_output = self.ffn(norm_hidden_states.to(orig_dtype)) + hidden_states = self.mlp_residual(hidden_states.float(), ff_output.float(), c_gate_msa) + hidden_states = hidden_states.to(orig_dtype) + + return hidden_states + + +class CausalWanGameActionTransformer3DModel(BaseDiT): + + _fsdp_shard_conditions = WanGameVideoConfig()._fsdp_shard_conditions + _compile_conditions = WanGameVideoConfig()._compile_conditions + _supported_attention_backends = WanGameVideoConfig()._supported_attention_backends + param_names_mapping = WanGameVideoConfig().param_names_mapping + reverse_param_names_mapping = WanGameVideoConfig().reverse_param_names_mapping + lora_param_names_mapping = WanGameVideoConfig().lora_param_names_mapping + + def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> None: + super().__init__(config=config, hf_config=hf_config) + + inner_dim = config.num_attention_heads * config.attention_head_dim + self.hidden_size = config.hidden_size + self.num_attention_heads = config.num_attention_heads + self.attention_head_dim = config.attention_head_dim + self.in_channels = config.in_channels + self.out_channels = config.out_channels + self.num_channels_latents = config.num_channels_latents + self.patch_size = config.patch_size + self.local_attn_size = config.local_attn_size + self.inner_dim = inner_dim + + # 1. Patch & position embedding + self.patch_embedding = PatchEmbed(in_chans=config.in_channels, + embed_dim=inner_dim, + patch_size=config.patch_size, + flatten=False) + + # 2. Condition embeddings + self.condition_embedder = WanGameActionTimeImageEmbedding( + dim=inner_dim, + time_freq_dim=config.freq_dim, + image_embed_dim=config.image_dim, + ) + + # 3. Transformer blocks + self.blocks = nn.ModuleList([ + CausalWanGameActionTransformerBlock( + inner_dim, + config.ffn_dim, + config.num_attention_heads, + config.local_attn_size, + config.sink_size, + config.qk_norm, + config.cross_attn_norm, + config.eps, + config.added_kv_proj_dim, + supported_attention_backends=self._supported_attention_backends, + prefix=f"{config.prefix}.blocks.{i}") + for i in range(config.num_layers) + ]) + + # 4. Output norm & projection + self.norm_out = LayerNormScaleShift(inner_dim, + norm_type="layer", + eps=config.eps, + elementwise_affine=False, + dtype=torch.float32) + self.proj_out = nn.Linear( + inner_dim, config.out_channels * math.prod(config.patch_size)) + self.scale_shift_table = nn.Parameter(torch.randn(1, 2, inner_dim) / inner_dim**0.5) + + self.gradient_checkpointing = False + + # Causal-specific + self.block_mask = None + self.num_frame_per_block = config.arch_config.num_frames_per_block + assert self.num_frame_per_block <= 3 + + self.__post_init__() + + @staticmethod + def _prepare_blockwise_causal_attn_mask( + device: torch.device | str, num_frames: int = 21, + frame_seqlen: int = 1560, num_frame_per_block=1, local_attn_size=-1 + ) -> BlockMask: + """ + we will divide the token sequence into the following format + [1 latent frame] [1 latent frame] ... [1 latent frame] + We use flexattention to construct the attention mask + """ + total_length = num_frames * frame_seqlen + + # we do right padding to get to a multiple of 128 + padded_length = math.ceil(total_length / 128) * 128 - total_length + + ends = torch.zeros(total_length + padded_length, + device=device, dtype=torch.long) + + # Block-wise causal mask will attend to all elements that are before the end of the current chunk + frame_indices = torch.arange( + start=0, + end=total_length, + step=frame_seqlen * num_frame_per_block, + device=device + ) + + for tmp in frame_indices: + ends[tmp:tmp + frame_seqlen * num_frame_per_block] = tmp + \ + frame_seqlen * num_frame_per_block + + def attention_mask(b, h, q_idx, kv_idx): + if local_attn_size == -1: + return (kv_idx < ends[q_idx]) | (q_idx == kv_idx) + else: + return ((kv_idx < ends[q_idx]) & (kv_idx >= (ends[q_idx] - local_attn_size * frame_seqlen))) | (q_idx == kv_idx) + + block_mask = create_block_mask(attention_mask, B=None, H=None, Q_LEN=total_length + padded_length, + KV_LEN=total_length + padded_length, _compile=False, device=device) + + if not dist.is_initialized() or dist.get_rank() == 0: + print( + f" cache a block wise causal mask with block size of {num_frame_per_block} frames") + print(block_mask) + + return block_mask + + def _forward_inference( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, + guidance=None, + action: torch.Tensor | None = None, + viewmats: torch.Tensor | None = None, + Ks: torch.Tensor | None = None, + kv_cache: list[dict] | None = None, + crossattn_cache: list[dict] | None = None, + current_start: int = 0, + cache_start: int = 0, + start_frame: int = 0, + is_cache: bool = False, + **kwargs + ) -> torch.Tensor: + r""" + Run the diffusion model with kv caching. + See Algorithm 2 of CausVid paper https://arxiv.org/abs/2412.07772 for details. + This function will be run for num_frame times. + Process the latent frames one by one (1560 tokens each) + """ + orig_dtype = hidden_states.dtype + if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: + encoder_hidden_states_image = encoder_hidden_states_image[0] + + batch_size, num_channels, num_frames, height, width = hidden_states.shape + p_t, p_h, p_w = self.patch_size + post_patch_num_frames = num_frames // p_t + post_patch_height = height // p_h + post_patch_width = width // p_w + + # Get rotary embeddings + d = self.hidden_size // self.num_attention_heads + rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] + freqs_cos, freqs_sin = get_rotary_pos_embed( + (post_patch_num_frames * get_sp_world_size(), post_patch_height, post_patch_width), + self.hidden_size, + self.num_attention_heads, + rope_dim_list, + dtype=torch.float32 if current_platform.is_mps() else torch.float64, + rope_theta=10000, + start_frame=start_frame + ) + freqs_cos = freqs_cos.to(hidden_states.device) + freqs_sin = freqs_sin.to(hidden_states.device) + freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None + + hidden_states = self.patch_embedding(hidden_states) + hidden_states = hidden_states.flatten(2).transpose(1, 2) + + if timestep.dim() == 2: + timestep = timestep.flatten() + + temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( + timestep, action, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) + + # condition_embedder returns: + # - temb: [B*T, dim] where T = post_patch_num_frames + # - timestep_proj: [B*T, 6*dim] + # Reshape to [B, T, 6, dim] for transformer blocks + timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) # [B*T, 6, dim] + timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, self.hidden_size) # [B, T, 6, dim] + + encoder_hidden_states = encoder_hidden_states_image + + # Transformer blocks + for block_idx, block in enumerate(self.blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func( + block, hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, + self.block_mask, + kv_cache[block_idx] if kv_cache else None, + crossattn_cache[block_idx] if crossattn_cache else None, + current_start, cache_start, + viewmats, Ks, is_cache) + else: + hidden_states = block( + hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, + block_mask=self.block_mask, + kv_cache=kv_cache[block_idx] if kv_cache else None, + crossattn_cache=crossattn_cache[block_idx] if crossattn_cache else None, + current_start=current_start, cache_start=cache_start, + viewmats=viewmats, Ks=Ks, is_cache=is_cache) + + # If cache-only mode, return early + if is_cache: + return kv_cache + + # Output norm, projection & unpatchify + # temb is [B*T, dim], reshape to [B, T, 1, dim] + temb = temb.view(batch_size, post_patch_num_frames, -1).unsqueeze(2) # [B, T, 1, dim] + + shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) + hidden_states = self.norm_out(hidden_states, shift, scale) + hidden_states = self.proj_out(hidden_states) + + hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, + post_patch_height, + post_patch_width, p_t, p_h, p_w, + -1) + hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) + output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + return output + + def _forward_train( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | list[torch.Tensor], + timestep: torch.LongTensor, + encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, + guidance=None, + action: torch.Tensor | None = None, + viewmats: torch.Tensor | None = None, + Ks: torch.Tensor | None = None, + start_frame: int = 0, + **kwargs + ) -> torch.Tensor: + + orig_dtype = hidden_states.dtype + if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: + encoder_hidden_states_image = encoder_hidden_states_image[0] + + batch_size, num_channels, num_frames, height, width = hidden_states.shape + p_t, p_h, p_w = self.patch_size + post_patch_num_frames = num_frames // p_t + post_patch_height = height // p_h + post_patch_width = width // p_w + + # Get rotary embeddings + d = self.hidden_size // self.num_attention_heads + rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] + freqs_cos, freqs_sin = get_rotary_pos_embed( + (post_patch_num_frames * get_sp_world_size(), post_patch_height, post_patch_width), + self.hidden_size, + self.num_attention_heads, + rope_dim_list, + dtype=torch.float32 if current_platform.is_mps() else torch.float64, + rope_theta=10000, + start_frame=start_frame + ) + freqs_cos = freqs_cos.to(hidden_states.device) + freqs_sin = freqs_sin.to(hidden_states.device) + freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None + + # Construct blockwise causal attn mask + if self.block_mask is None: + self.block_mask = self._prepare_blockwise_causal_attn_mask( + device=hidden_states.device, + num_frames=num_frames, + frame_seqlen=post_patch_height * post_patch_width, + num_frame_per_block=self.num_frame_per_block, + local_attn_size=self.local_attn_size + ) + + hidden_states = self.patch_embedding(hidden_states) + hidden_states = hidden_states.flatten(2).transpose(1, 2) + + if timestep.dim() == 2: + timestep = timestep.flatten() + + temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( + timestep, action, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) + + # condition_embedder returns: + # - temb: [B*T, dim] where T = post_patch_num_frames + # - timestep_proj: [B*T, 6*dim] + # Reshape to [B, T, 6, dim] for transformer blocks + timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) # [B*T, 6, dim] + timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, self.hidden_size) # [B, T, 6, dim] + + encoder_hidden_states = encoder_hidden_states_image + + # Transformer blocks + if torch.is_grad_enabled() and self.gradient_checkpointing: + for block in self.blocks: + hidden_states = self._gradient_checkpointing_func( + block, hidden_states, encoder_hidden_states, + timestep_proj, freqs_cis, + self.block_mask, + None, None, # kv_cache, crossattn_cache + 0, None, # current_start, cache_start + viewmats, Ks, False) # viewmats, Ks, is_cache + else: + for block in self.blocks: + hidden_states = block(hidden_states, encoder_hidden_states, + timestep_proj, freqs_cis, + block_mask=self.block_mask, + viewmats=viewmats, Ks=Ks) + + # Output norm, projection & unpatchify + # temb is [B*T, dim], reshape to [B, T, 1, dim] + temb = temb.view(batch_size, post_patch_num_frames, -1).unsqueeze(2) # [B, T, 1, dim] + + shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) + hidden_states = self.norm_out(hidden_states, shift, scale) + hidden_states = self.proj_out(hidden_states) + + hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, + post_patch_height, + post_patch_width, p_t, p_h, p_w, + -1) + hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) + output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + return output + + def forward( + self, + *args, + **kwargs + ): + if kwargs.get('kv_cache', None) is not None: + return self._forward_inference(*args, **kwargs) + else: + return self._forward_train(*args, **kwargs) diff --git a/fastvideo/models/loader/component_loader.py b/fastvideo/models/loader/component_loader.py index 27c9e135c..13f069504 100644 --- a/fastvideo/models/loader/component_loader.py +++ b/fastvideo/models/loader/component_loader.py @@ -813,6 +813,7 @@ def load(self, model_path: str, fastvideo_args: FastVideoArgs): or cls_name.startswith("WanLingBot") or cls_name == "WanLingBotTransformer3DModel" or getattr(fastvideo_args.pipeline_config, "prefix", "") == "WanLingBot" + or cls_name.startswith("CausalWanGameActionTransformer3DModel") ) model = maybe_load_fsdp_model( model_cls=model_cls, @@ -963,4 +964,4 @@ def load_module( ) # Load the module - return loader.load(component_model_path, fastvideo_args) \ No newline at end of file + return loader.load(component_model_path, fastvideo_args) diff --git a/fastvideo/models/registry.py b/fastvideo/models/registry.py index 5e3731217..79017652b 100644 --- a/fastvideo/models/registry.py +++ b/fastvideo/models/registry.py @@ -43,7 +43,9 @@ "WanTransformer3DModel": ("dits", "wanvideo", "WanTransformer3DModel"), "CausalWanTransformer3DModel": ("dits", "causal_wanvideo", "CausalWanTransformer3DModel"), "CausalWanGameTransformer3DModel": - ("dits", "wangame", "CausalWanGameTransformer3DModel"), + ("dits", "wangame", "CausalWanGameActionTransformer3DModel"), + "CausalWanGameActionTransformer3DModel": + ("dits", "wangame", "CausalWanGameActionTransformer3DModel"), "WanGameActionTransformer3DModel": ("dits", "wangame", "WanGameActionTransformer3DModel"), "WanLingBotTransformer3DModel": ("dits", "wangame_lingbot", "WanLingBotTransformer3DModel"), "MatrixGameWanModel": ("dits", "matrixgame", "MatrixGameWanModel"), diff --git a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py new file mode 100644 index 000000000..c99554cf2 --- /dev/null +++ b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +"""WanGame causal DMD pipeline implementation.""" + +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.logger import init_logger +from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline + +from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, + CausalDMDDenosingStage, + ImageEncodingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage) +from fastvideo.pipelines.stages.image_encoding import ( + ImageVAEEncodingStage) + +logger = init_logger(__name__) + + +class WanGameCausalDMDPipeline(LoRAPipeline, ComposedPipelineBase): + _required_config_modules = [ + "vae", "transformer", "scheduler", "image_encoder", "image_processor" + ] + + def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: + self.add_stage(stage_name="input_validation_stage", + stage=InputValidationStage()) + + if (self.get_module("text_encoder", None) is not None + and self.get_module("tokenizer", None) is not None): + self.add_stage(stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + )) + + if (self.get_module("image_encoder", None) is not None + and self.get_module("image_processor", None) is not None): + self.add_stage( + stage_name="image_encoding_stage", + stage=ImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + )) + + self.add_stage(stage_name="conditioning_stage", + stage=ConditioningStage()) + + self.add_stage(stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer", None))) + + self.add_stage( + stage_name="image_latent_preparation_stage", + stage=ImageVAEEncodingStage(vae=self.get_module("vae"))) + + self.add_stage(stage_name="denoising_stage", + stage=CausalDMDDenosingStage( + transformer=self.get_module("transformer"), + transformer_2=self.get_module("transformer_2", None), + scheduler=self.get_module("scheduler"), + vae=self.get_module("vae"))) + + self.add_stage(stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"))) + + logger.info( + "WanGameCausalDMDPipeline initialized with action support") + +EntryClass = WanGameCausalDMDPipeline diff --git a/fastvideo/pipelines/preprocess/v1_preprocess.py b/fastvideo/pipelines/preprocess/v1_preprocess.py index bcce47292..2a65c6a18 100644 --- a/fastvideo/pipelines/preprocess/v1_preprocess.py +++ b/fastvideo/pipelines/preprocess/v1_preprocess.py @@ -20,6 +20,8 @@ PreprocessPipeline_MatrixGame) from fastvideo.pipelines.preprocess.wangame.wangame_preprocess_pipeline import ( PreprocessPipeline_WanGame) +from fastvideo.pipelines.preprocess.wangame.wangame_preprocess_pipeline_ode_trajectory import ( + PreprocessPipeline_WanGame_ODE_Trajectory) from fastvideo.utils import maybe_download_model logger = init_logger(__name__) @@ -68,10 +70,13 @@ def main(args) -> None: PreprocessPipeline = PreprocessPipeline_MatrixGame elif args.preprocess_task == "wangame": PreprocessPipeline = PreprocessPipeline_WanGame + elif args.preprocess_task == "wangame_ode_trajectory": + fastvideo_args.pipeline_config.flow_shift = args.flow_shift if args.flow_shift is not None else 5.0 + PreprocessPipeline = PreprocessPipeline_WanGame_ODE_Trajectory else: raise ValueError( f"Invalid preprocess task: {args.preprocess_task}. " - f"Valid options: t2v, i2v, ode_trajectory, text_only, matrixgame, wangame") + f"Valid options: t2v, i2v, ode_trajectory, text_only, matrixgame, wangame, wangame_ode_trajectory") logger.info("Preprocess task: %s using %s", args.preprocess_task, PreprocessPipeline.__name__) @@ -119,7 +124,7 @@ def main(args) -> None: "--preprocess_task", type=str, default="t2v", - choices=["t2v", "i2v", "text_only", "ode_trajectory", "matrixgame", "wangame"], + choices=["t2v", "i2v", "text_only", "ode_trajectory", "matrixgame", "wangame", "wangame_ode_trajectory"], help="Type of preprocessing task to run") parser.add_argument("--train_fps", type=int, default=30) parser.add_argument("--use_image_num", type=int, default=0) diff --git a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py new file mode 100644 index 000000000..21082071c --- /dev/null +++ b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py @@ -0,0 +1,490 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +ODE Trajectory Data Preprocessing pipeline implementation. + +This module contains an implementation of the ODE Trajectory Data Preprocessing pipeline +using the modular pipeline architecture. + +Sec 4.3 of CausVid paper: https://arxiv.org/pdf/2412.07772 +""" + +import os +from collections.abc import Iterator +from typing import Any + +import numpy as np +import pyarrow as pa +import torch +from PIL import Image +from torch.utils.data import DataLoader +from torchdata.stateful_dataloader import StatefulDataLoader +from tqdm import tqdm + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset import getdataset +from fastvideo.dataset.dataloader.parquet_io import (ParquetDatasetWriter, + records_to_table) +from fastvideo.dataset.dataloader.record_schema import ( + wangame_ode_record_creator) +from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_ode_trajectory_wangame) +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.forward_context import set_forward_context +from fastvideo.logger import init_logger +from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( + SelfForcingFlowMatchScheduler) +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.pipelines.preprocess.preprocess_pipeline_base import ( + BasePreprocessPipeline) +from fastvideo.pipelines.stages import (CausalDMDDenosingStage, DecodingStage, + InputValidationStage, + LatentPreparationStage, + ImageEncodingStage, + TimestepPreparationStage) +from fastvideo.utils import save_decoded_latents_as_video, shallow_asdict + +logger = init_logger(__name__) + + +class PreprocessPipeline_WanGame_ODE_Trajectory(BasePreprocessPipeline): + """ODE Trajectory preprocessing pipeline implementation.""" + + _required_config_modules = [ + "vae", "image_encoder", "image_processor", "transformer", "scheduler" + ] + + preprocess_dataloader: StatefulDataLoader + preprocess_loader_iter: Iterator[dict[str, Any]] + pbar: Any + num_processed_samples: int + + def get_pyarrow_schema(self) -> pa.Schema: + """Return the PyArrow schema for ODE Trajectory pipeline.""" + return pyarrow_schema_ode_trajectory_wangame + + def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): + """Set up pipeline stages with proper dependency injection.""" + assert fastvideo_args.pipeline_config.flow_shift == 5 + self.modules["scheduler"] = SelfForcingFlowMatchScheduler( + shift=fastvideo_args.pipeline_config.flow_shift, + sigma_min=0.0, + extra_one_step=True) + self.modules["scheduler"].set_timesteps(num_inference_steps=48, + denoising_strength=1.0) + + self.add_stage(stage_name="input_validation_stage", + stage=InputValidationStage()) + self.add_stage(stage_name="image_encoding_stage", + stage=ImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + )) + self.add_stage(stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"))) + self.add_stage(stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer", None))) + self.add_stage(stage_name="denoising_stage", + stage=CausalDMDDenosingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + transformer_2=self.get_module("transformer_2", None), + vae=self.get_module("vae"), + )) + self.add_stage(stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"))) + + def get_extra_features(self, valid_data: dict[str, Any], + fastvideo_args: FastVideoArgs) -> dict[str, Any]: + + # TODO(will): move these to cpu at some point + self.get_module("image_encoder").to(get_local_torch_device()) + self.get_module("vae").to(get_local_torch_device()) + + features = {} + """Get CLIP features from the first frame of each video.""" + first_frame = valid_data["pixel_values"][:, :, 0, :, :].permute( + 0, 2, 3, 1) # (B, C, T, H, W) -> (B, H, W, C) + _, _, num_frames, height, width = valid_data["pixel_values"].shape + # latent_height = height // self.get_module( + # "vae").spatial_compression_ratio + # latent_width = width // self.get_module("vae").spatial_compression_ratio + + processed_images = [] + # Frame has values between -1 and 1 + for frame in first_frame: + frame = (frame + 1) * 127.5 + frame_pil = Image.fromarray(frame.cpu().numpy().astype(np.uint8)) + processed_img = self.get_module("image_processor")( + images=frame_pil, return_tensors="pt") + processed_images.append(processed_img) + + # Get CLIP features + pixel_values = torch.cat( + [img['pixel_values'] for img in processed_images], + dim=0).to(get_local_torch_device()) + with torch.no_grad(): + image_inputs = {'pixel_values': pixel_values} + with set_forward_context(current_timestep=0, attn_metadata=None): + clip_features = self.get_module("image_encoder")(**image_inputs) + clip_features = clip_features.last_hidden_state + + features["clip_feature"] = clip_features + features["pil_image"] = first_frame + """Get VAE features from the first frame of each video""" + video_conditions = [] + for frame in first_frame: + processed_img = frame.to(device="cpu", dtype=torch.float32) + processed_img = processed_img.unsqueeze(0).permute(0, 3, 1, + 2).unsqueeze(2) + # (B, H, W, C) -> (B, C, 1, H, W) + video_condition = torch.cat([ + processed_img, + processed_img.new_zeros(processed_img.shape[0], + processed_img.shape[1], num_frames - 1, + height, width) + ], + dim=2) + video_condition = video_condition.to( + device=get_local_torch_device(), dtype=torch.float32) + video_conditions.append(video_condition) + + video_conditions = torch.cat(video_conditions, dim=0) + + with torch.autocast(device_type="cuda", + dtype=torch.float32, + enabled=True): + encoder_outputs = self.get_module("vae").encode(video_conditions) + + # Use mode() instead of mean + latent_condition = encoder_outputs.mode() + + # Use latents_mean/latents_std normalization to match + vae = self.get_module("vae") + if (hasattr(vae.config, 'latents_mean') + and hasattr(vae.config, 'latents_std')): + latents_mean = torch.tensor(vae.config.latents_mean, + device=latent_condition.device, + dtype=latent_condition.dtype).view( + 1, -1, 1, 1, 1) + latents_std = torch.tensor(vae.config.latents_std, + device=latent_condition.device, + dtype=latent_condition.dtype).view( + 1, -1, 1, 1, 1) + latent_condition = (latent_condition - latents_mean) / latents_std + elif (hasattr(vae, "shift_factor") and vae.shift_factor is not None): + if isinstance(vae.shift_factor, torch.Tensor): + latent_condition -= vae.shift_factor.to( + latent_condition.device, latent_condition.dtype) + else: + latent_condition -= vae.shift_factor + + if isinstance(vae.scaling_factor, torch.Tensor): + latent_condition = latent_condition * vae.scaling_factor.to( + latent_condition.device, latent_condition.dtype) + else: + latent_condition = latent_condition * vae.scaling_factor + + # mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, + # latent_width) + # mask_lat_size[:, :, list(range(1, num_frames))] = 0 + # first_frame_mask = mask_lat_size[:, :, 0:1] + # first_frame_mask = torch.repeat_interleave( + # first_frame_mask, + # dim=2, + # repeats=self.get_module("vae").temporal_compression_ratio) + # mask_lat_size = torch.concat( + # [first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2) + # mask_lat_size = mask_lat_size.view( + # batch_size, -1, + # self.get_module("vae").temporal_compression_ratio, latent_height, + # latent_width) + # mask_lat_size = mask_lat_size.transpose(1, 2) + # mask_lat_size = mask_lat_size.to(latent_condition.device) + + # image_latent = torch.concat([mask_lat_size, latent_condition], dim=1) + + # Create mask_cond: ones for first frame, zeros for rest + # Shape: (B, 16, latent_frames, latent_height, latent_width) + mask_cond = torch.ones_like(latent_condition) + mask_cond[:, :, 1:] = 0 # Set all frames except first to 0 + # Create cond_concat: first 4 channels of mask + all 16 channels of img_cond + # Shape: (B, 20, latent_frames, latent_height, latent_width) + cond_concat = torch.cat([mask_cond[:, :4], latent_condition], dim=1) + features["first_frame_latent"] = cond_concat + + if "action_path" in valid_data and valid_data["action_path"]: + keyboard_cond_list = [] + mouse_cond_list = [] + arch_cfg = self.get_module("transformer").config.arch_config + action_cfg = getattr(arch_cfg, "action_config", {}) or {} + keyboard_dim = action_cfg.get("keyboard_dim_in", None) + for action_path in valid_data["action_path"]: + if action_path: + action_data = np.load(action_path, allow_pickle=True) + if isinstance(action_data, + np.ndarray) and action_data.dtype == np.dtype('O'): + action_dict = action_data.item() + if "keyboard" in action_dict: + keyboard = action_dict["keyboard"].astype(np.float32) + if keyboard_dim is not None: + if keyboard.ndim >= 2: + keyboard = keyboard[:, :keyboard_dim] + else: + keyboard = keyboard[:keyboard_dim] + keyboard_cond_list.append(keyboard) + if "mouse" in action_dict: + mouse_cond_list.append(action_dict["mouse"]) + else: + keyboard = action_data.astype(np.float32) + if keyboard_dim is not None: + if keyboard.ndim >= 2: + keyboard = keyboard[:, :keyboard_dim] + else: + keyboard = keyboard[:keyboard_dim] + keyboard_cond_list.append(keyboard) + if keyboard_cond_list: + features["keyboard_cond"] = keyboard_cond_list + if mouse_cond_list: + features["mouse_cond"] = mouse_cond_list + return features + + def preprocess_action_and_trajectory(self, fastvideo_args: FastVideoArgs, + args): + """Preprocess data and generate trajectory information.""" + + for batch_idx, data in enumerate(self.pbar): + if data is None: + continue + + with torch.inference_mode(): + # Filter out invalid samples (those with all zeros) + valid_indices = [] + for i, pixel_values in enumerate(data["pixel_values"]): + if not torch.all( + pixel_values == 0): # Check if all values are zero + valid_indices.append(i) + self.num_processed_samples += len(valid_indices) + + if not valid_indices: + continue + + # Create new batch with only valid samples + valid_data = { + "pixel_values": + torch.stack( + [data["pixel_values"][i] for i in valid_indices]), + "path": [data["path"][i] for i in valid_indices], + } + + if "fps" in data: + valid_data["fps"] = [data["fps"][i] for i in valid_indices] + if "duration" in data: + valid_data["duration"] = [ + data["duration"][i] for i in valid_indices + ] + if "action_path" in data: + valid_data["action_path"] = [ + data["action_path"][i] for i in valid_indices + ] + + pixel_values = valid_data["pixel_values"] + if pixel_values.shape[2] == 1 and args.num_frames is not None: + pixel_values = pixel_values.repeat( + 1, 1, args.num_frames, 1, 1) + valid_data["pixel_values"] = pixel_values + + # Get extra features if needed + extra_features = self.get_extra_features( + valid_data, fastvideo_args) + + clip_features = extra_features['clip_feature'] + image_latents = extra_features['first_frame_latent'] + image_latents = image_latents[:, :, :args.num_latent_t] + pil_image = extra_features['pil_image'] + if "keyboard_cond" in extra_features: + keyboard_cond = extra_features['keyboard_cond'] + else: + keyboard_cond = None + if "mouse_cond" in extra_features: + mouse_cond = extra_features['mouse_cond'] + else: + mouse_cond = None + + sampling_params = SamplingParam.from_pretrained(args.model_path) + + trajectory_latents = [] + trajectory_timesteps = [] + trajectory_decoded = [] + + device = get_local_torch_device() + for i in range(len(valid_indices)): + # Collect the trajectory data + batch = ForwardBatch(**shallow_asdict(sampling_params), ) + batch.image_embeds = [clip_features[i].unsqueeze(0)] + batch.image_latent = image_latents[i].unsqueeze(0) + batch.keyboard_cond = (torch.from_numpy( + keyboard_cond[i]).unsqueeze(0).to(device) if + keyboard_cond is not None else None) + batch.mouse_cond = (torch.from_numpy( + mouse_cond[i]).unsqueeze(0).to(device) + if mouse_cond is not None else None) + batch.num_inference_steps = 48 + batch.return_trajectory_latents = True + # Enabling this will save the decoded trajectory videos. + # Used for debugging. + batch.return_trajectory_decoded = False + batch.height = args.max_height + batch.width = args.max_width + batch.fps = args.train_fps + batch.num_frames = valid_data["pixel_values"].shape[2] + batch.guidance_scale = 6.0 + batch.do_classifier_free_guidance = False + batch.prompt = "" + batch.prompt_embeds = [ + torch.zeros((1, 0, self.get_module("transformer").hidden_size), + dtype=torch.bfloat16, + device=device) + ] + + result_batch = self.input_validation_stage( + batch, fastvideo_args) + result_batch = self.timestep_preparation_stage( + result_batch, fastvideo_args) + result_batch.timesteps = result_batch.timesteps.to(device) + result_batch = self.latent_preparation_stage( + result_batch, fastvideo_args) + result_batch = self.denoising_stage(result_batch, + fastvideo_args) + result_batch = self.decoding_stage(result_batch, + fastvideo_args) + + trajectory_latents.append( + result_batch.trajectory_latents.cpu()) + trajectory_timesteps.append( + result_batch.trajectory_timesteps.cpu()) + trajectory_decoded.append(result_batch.trajectory_decoded) + + # Prepare extra features + extra_features = { + "trajectory_latents": trajectory_latents, + "trajectory_timesteps": trajectory_timesteps + } + + if batch.return_trajectory_decoded: + for i, decoded_frames in enumerate(trajectory_decoded): + for j, decoded_frame in enumerate(decoded_frames): + save_decoded_latents_as_video( + decoded_frame, + f"decoded_videos/trajectory_decoded_{i}_{j}.mp4", + args.train_fps) + + # Prepare batch data for Parquet dataset + batch_data: list[dict[str, Any]] = [] + + # Add progress bar for saving outputs + save_pbar = tqdm(enumerate(valid_data["path"]), + desc="Saving outputs", + unit="item", + leave=False) + + for idx, video_path in save_pbar: + video_name = os.path.basename(video_path).split(".")[0] + + clip_feature_np = clip_features[idx].cpu().numpy() + first_frame_latent_np = image_latents[idx].cpu().numpy() + pil_image_np = pil_image[idx].cpu().numpy() + keyboard_cond_np = keyboard_cond[ + idx] if keyboard_cond is not None else None + mouse_cond_np = mouse_cond[ + idx] if mouse_cond is not None else None + + # Get trajectory features for this sample + traj_latents = extra_features["trajectory_latents"][idx] + traj_timesteps = extra_features["trajectory_timesteps"][idx] + if isinstance(traj_latents, torch.Tensor): + traj_latents = traj_latents.cpu().float().numpy() + if isinstance(traj_timesteps, torch.Tensor): + traj_timesteps = traj_timesteps.cpu().float().numpy() + + # Create record for Parquet dataset + record: dict[str, Any] = wangame_ode_record_creator( + video_name=video_name, + clip_feature=clip_feature_np, + first_frame_latent=first_frame_latent_np, + trajectory_latents=traj_latents, + trajectory_timesteps=traj_timesteps, + pil_image=pil_image_np, + keyboard_cond=keyboard_cond_np, + mouse_cond=mouse_cond_np, + caption="") + batch_data.append(record) + + if batch_data: + write_pbar = tqdm(total=1, + desc="Writing to Parquet dataset", + unit="batch") + table = records_to_table(batch_data, + self.get_pyarrow_schema()) + write_pbar.update(1) + write_pbar.close() + + if not hasattr(self, 'dataset_writer'): + self.dataset_writer = ParquetDatasetWriter( + out_dir=self.combined_parquet_dir, + samples_per_file=args.samples_per_file, + ) + self.dataset_writer.append_table(table) + + logger.info("Collected batch with %s samples", len(table)) + + if self.num_processed_samples >= args.flush_frequency: + written = self.dataset_writer.flush() + logger.info("Flushed %s samples to parquet", written) + self.num_processed_samples = 0 + + # Final flush for any remaining samples + if hasattr(self, 'dataset_writer'): + written = self.dataset_writer.flush(write_remainder=True) + if written: + logger.info("Final flush wrote %s samples", written) + + def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs, args): + if not self.post_init_called: + self.post_init() + + self.local_rank = int(os.getenv("RANK", 0)) + os.makedirs(args.output_dir, exist_ok=True) + # Create directory for combined data + self.combined_parquet_dir = os.path.join(args.output_dir, + "combined_parquet_dataset") + os.makedirs(self.combined_parquet_dir, exist_ok=True) + + # Loading dataset + train_dataset = getdataset(args) + + self.preprocess_dataloader = DataLoader( + train_dataset, + batch_size=args.preprocess_video_batch_size, + num_workers=args.dataloader_num_workers, + ) + + self.preprocess_loader_iter = iter(self.preprocess_dataloader) + + self.num_processed_samples = 0 + # Add progress bar for video preprocessing + self.pbar = tqdm(self.preprocess_loader_iter, + desc="Processing videos", + unit="batch", + disable=self.local_rank != 0) + + # Initialize class variables for data sharing + self.video_data: dict[str, Any] = {} # Store video metadata and paths + self.latent_data: dict[str, Any] = {} # Store latent tensors + self.preprocess_action_and_trajectory(fastvideo_args, args) + + +EntryClass = PreprocessPipeline_WanGame_ODE_Trajectory diff --git a/fastvideo/pipelines/stages/causal_denoising.py b/fastvideo/pipelines/stages/causal_denoising.py index 777314547..56999aeec 100644 --- a/fastvideo/pipelines/stages/causal_denoising.py +++ b/fastvideo/pipelines/stages/causal_denoising.py @@ -90,8 +90,17 @@ def forward( boundary_timestep = None high_noise_timesteps = None - # Image kwargs (kept empty unless caller provides compatible args) - image_kwargs: dict = {} + image_embeds = batch.image_embeds + if len(image_embeds) > 0: + assert not torch.isnan(image_embeds[0]).any() + image_embeds = [ + image_embed.to(target_dtype) for image_embed in image_embeds + ] + image_kwargs: dict[str, torch.Tensor | list[torch.Tensor]] = { + "encoder_hidden_states_image": image_embeds + } + else: + image_kwargs = {} pos_cond_kwargs = self.prepare_extra_func_kwargs( self.transformer.forward, @@ -110,7 +119,36 @@ def forward( latents = batch.latents # [B, C, T, H, W] b, c, t, h, w = latents.shape prompt_embeds = batch.prompt_embeds - assert torch.isnan(prompt_embeds[0]).sum() == 0 + if len(prompt_embeds) == 0: + prompt_embeds = [ + torch.zeros((b, 0, self.transformer.hidden_size), + device=latents.device, + dtype=target_dtype) + ] + else: + assert not torch.isnan(prompt_embeds[0]).any() + + viewmats_full = None + intrinsics_full = None + action_full = None + if batch.mouse_cond is not None and batch.keyboard_cond is not None: + from fastvideo.models.dits.hyworld.pose import process_custom_actions + + viewmats_list = [] + intrinsics_list = [] + action_list = [] + for bi in range(b): + vm, ks, action = process_custom_actions( + batch.keyboard_cond[bi], batch.mouse_cond[bi]) + viewmats_list.append(vm) + intrinsics_list.append(ks) + action_list.append(action) + viewmats_full = torch.stack(viewmats_list, dim=0).to( + device=latents.device, dtype=target_dtype) + intrinsics_full = torch.stack(intrinsics_list, dim=0).to( + device=latents.device, dtype=target_dtype) + action_full = torch.stack(action_list, dim=0).to( + device=latents.device, dtype=target_dtype) # Initialize or reset caches kv_cache1 = self._initialize_kv_cache(batch_size=latents.shape[0], @@ -188,6 +226,13 @@ def _get_kv_cache(timestep: float) -> list[dict]: set_forward_context(current_timestep=0, attn_metadata=None, forward_batch=batch): + first_frame_action_kwargs = {} + if action_full is not None: + first_frame_action_kwargs = { + "viewmats": viewmats_full[:, start_index:start_index + 1], + "Ks": intrinsics_full[:, start_index:start_index + 1], + "action": action_full[:, start_index:start_index + 1], + } self.transformer( first_frame_latent.to(target_dtype), prompt_embeds, @@ -197,6 +242,7 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, + **first_frame_action_kwargs, **image_kwargs, **pos_cond_kwargs, ) @@ -210,6 +256,7 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, + **first_frame_action_kwargs, **image_kwargs, **pos_cond_kwargs, ) @@ -237,13 +284,33 @@ def _get_kv_cache(timestep: float) -> list[dict]: noise_latents = noise_latents_btchw.clone() latent_model_input = current_latents.to(target_dtype) - if batch.image_latent is not None and independent_first_frame and start_index == 0: + if (batch.image_latent is not None + and not independent_first_frame): + image_latent_chunk = batch.image_latent[:, :, start_index: + start_index + + current_num_frames, :, :] + latent_model_input = torch.cat([ + latent_model_input, + image_latent_chunk.to(target_dtype) + ], + dim=1) + elif (batch.image_latent is not None + and independent_first_frame and start_index == 0): latent_model_input = torch.cat([ latent_model_input, batch.image_latent.to(target_dtype) ], dim=2) + camera_action_kwargs = {} + if action_full is not None: + end_index = start_index + current_num_frames + camera_action_kwargs = { + "viewmats": viewmats_full[:, start_index:end_index], + "Ks": intrinsics_full[:, start_index:end_index], + "action": action_full[:, start_index:end_index], + } + # Prepare inputs t_expand = t_cur.repeat(latent_model_input.shape[0]) @@ -292,6 +359,7 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, + **camera_action_kwargs, **image_kwargs, **pos_cond_kwargs, ).permute(0, 2, 1, 3, 4) @@ -363,6 +431,24 @@ def _get_kv_cache(timestep: float) -> list[dict]: device=latents.device, dtype=torch.long) * int(context_noise) context_bcthw = current_latents.to(target_dtype) + context_input = context_bcthw + if batch.image_latent is not None and not independent_first_frame: + image_context_chunk = batch.image_latent[:, :, start_index: + start_index + + current_num_frames, :, :] + context_input = torch.cat([ + context_input, + image_context_chunk.to(target_dtype) + ], + dim=1) + context_action_kwargs = {} + if action_full is not None: + end_index = start_index + current_num_frames + context_action_kwargs = { + "viewmats": viewmats_full[:, start_index:end_index], + "Ks": intrinsics_full[:, start_index:end_index], + "action": action_full[:, start_index:end_index], + } with torch.autocast(device_type="cuda", dtype=target_dtype, enabled=autocast_enabled), \ @@ -373,7 +459,7 @@ def _get_kv_cache(timestep: float) -> list[dict]: if boundary_timestep is not None: self.transformer_2( - context_bcthw, + context_input, prompt_embeds, t_expanded_context, kv_cache=kv_cache2, @@ -381,12 +467,13 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, + **context_action_kwargs, **image_kwargs, **pos_cond_kwargs, ) self.transformer( - context_bcthw, + context_input, prompt_embeds, t_expanded_context, kv_cache=kv_cache1, @@ -394,6 +481,7 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, + **context_action_kwargs, **image_kwargs, **pos_cond_kwargs, ) @@ -478,7 +566,7 @@ def verify_input(self, batch: ForwardBatch, result = VerificationResult() result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) - result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty) + result.add_check("prompt_embeds", batch.prompt_embeds, V.is_list) result.add_check("image_embeds", batch.image_embeds, V.is_list) result.add_check("image_latent", batch.image_latent, V.none_or_tensor_with_dims(5)) @@ -494,4 +582,4 @@ def verify_input(self, batch: ForwardBatch, result.add_check( "negative_prompt_embeds", batch.negative_prompt_embeds, lambda x: not batch.do_classifier_free_guidance or V.list_not_empty(x)) - return result \ No newline at end of file + return result diff --git a/fastvideo/training/wangame_ode_causal_pipeline.py b/fastvideo/training/wangame_ode_causal_pipeline.py new file mode 100644 index 000000000..779634c0b --- /dev/null +++ b/fastvideo/training/wangame_ode_causal_pipeline.py @@ -0,0 +1,556 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys +from copy import deepcopy +from typing import Any, cast + +import numpy as np +import torch +import torch.nn.functional as F + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_ode_trajectory_wangame) +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.forward_context import set_forward_context +from fastvideo.logger import init_logger +from fastvideo.models.dits.hyworld.pose import process_custom_actions +from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( + SelfForcingFlowMatchScheduler) +from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( + WanGameCausalDMDPipeline) +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch +from fastvideo.training.training_pipeline import TrainingPipeline +from fastvideo.training.training_utils import ( + clip_grad_norm_while_handling_failing_dtensor_cases) +from fastvideo.utils import shallow_asdict + +logger = init_logger(__name__) + + +class WanGameODEInitTrainingPipeline(TrainingPipeline): + """ + Training pipeline for ODE-init using precomputed denoising trajectories. + + Supervision: predict the next latent in the stored trajectory by + - feeding current latent at timestep t into the transformer to predict noise + - stepping the scheduler with the predicted noise + - minimizing MSE to the stored next latent at timestep t_next + """ + + _required_config_modules = ["scheduler", "transformer", "vae"] + + def initialize_pipeline(self, fastvideo_args: FastVideoArgs): + # Match the preprocess/generation scheduler for consistent stepping + self.modules["scheduler"] = SelfForcingFlowMatchScheduler( + shift=fastvideo_args.pipeline_config.flow_shift, + sigma_min=0.0, + extra_one_step=True) + self.modules["scheduler"].set_timesteps(num_inference_steps=1000, + training=True) + + def set_schemas(self): + self.train_dataset_schema = pyarrow_schema_ode_trajectory_wangame + + def initialize_training_pipeline(self, training_args: TrainingArgs): + super().initialize_training_pipeline(training_args) + + self.noise_scheduler = self.get_module("scheduler") + self.vae = self.get_module("vae") + self.vae.requires_grad_(False) + + self.timestep_shift = self.training_args.pipeline_config.flow_shift + assert self.timestep_shift == 5.0, "flow_shift must be 5.0" + self.noise_scheduler = SelfForcingFlowMatchScheduler( + shift=self.timestep_shift, sigma_min=0.0, extra_one_step=True) + self.noise_scheduler.set_timesteps(num_inference_steps=1000, + training=True) + + logger.info("dmd_denoising_steps: %s", + self.training_args.pipeline_config.dmd_denoising_steps) + self.dmd_denoising_steps = torch.tensor([1000, 750, 500, 250], + dtype=torch.long, + device=get_local_torch_device()) + if training_args.warp_denoising_step: # Warp the denoising step according to the scheduler time shift + timesteps = torch.cat((self.noise_scheduler.timesteps.cpu(), + torch.tensor([0], + dtype=torch.float32))).cuda() + logger.info("timesteps: %s", timesteps) + self.dmd_denoising_steps = timesteps[1000 - + self.dmd_denoising_steps] + logger.info("warped self.dmd_denoising_steps: %s", + self.dmd_denoising_steps) + else: + raise ValueError("warp_denoising_step must be true") + + self.dmd_denoising_steps = self.dmd_denoising_steps.to( + get_local_torch_device()) + + logger.info("denoising_step_list: %s", self.dmd_denoising_steps) + + logger.info( + "Initialized ODE-init training pipeline with %s denoising steps", + len(self.dmd_denoising_steps)) + # Cache for nearest trajectory index per DMD step (computed lazily on first batch) + self._cached_closest_idx_per_dmd = None + self.num_train_timestep = self.noise_scheduler.num_train_timesteps + self.manual_idx = 0 + + def initialize_validation_pipeline(self, training_args: TrainingArgs): + logger.info("Initializing validation pipeline...") + args_copy = deepcopy(training_args) + args_copy.inference_mode = True + # Use the same flow-matching scheduler as training for consistent validation. + validation_scheduler = SelfForcingFlowMatchScheduler( + shift=args_copy.pipeline_config.flow_shift, + sigma_min=0.0, + extra_one_step=True) + validation_scheduler.set_timesteps(num_inference_steps=1000, + training=True) + # Warm start validation with current transformer + self.validation_pipeline = WanGameCausalDMDPipeline.from_pretrained( + training_args.model_path, + args=args_copy, # type: ignore + inference_mode=True, + loaded_modules={ + "transformer": self.get_module("transformer"), + "vae": self.get_module("vae"), + "scheduler": validation_scheduler, + }, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + pin_cpu_memory=training_args.pin_cpu_memory, + dit_cpu_offload=True) + + def _get_next_batch( + self, + training_batch) -> tuple[TrainingBatch, torch.Tensor, torch.Tensor]: + batch = next(self.train_loader_iter, None) # type: ignore + if batch is None: + self.current_epoch += 1 + logger.info("Starting epoch %s", self.current_epoch) + self.train_loader_iter = iter(self.train_dataloader) + batch = next(self.train_loader_iter) + + # Required fields from parquet (ODE trajectory schema) + clip_feature = batch['clip_feature'] + first_frame_latent = batch['first_frame_latent'] + keyboard_cond = batch.get('keyboard_cond', None) + # keyboard_cond = keyboard_cond[:, :, :3] # TODO: remove hardcode + mouse_cond = batch.get('mouse_cond', None) + infos = batch['info_list'] + + # Trajectory tensors may include a leading singleton batch dim per row + trajectory_latents = batch['trajectory_latents'] + if trajectory_latents.dim() == 7: + # [B, 1, S, C, T, H, W] -> [B, S, C, T, H, W] + trajectory_latents = trajectory_latents[:, 0] + elif trajectory_latents.dim() == 6: + # already [B, S, C, T, H, W] + pass + else: + raise ValueError( + f"Unexpected trajectory_latents dim: {trajectory_latents.dim()}" + ) + + trajectory_timesteps = batch['trajectory_timesteps'] + if trajectory_timesteps.dim() == 3: + # [B, 1, S] -> [B, S] + trajectory_timesteps = trajectory_timesteps[:, 0] + elif trajectory_timesteps.dim() == 2: + # [B, S] + pass + else: + raise ValueError( + f"Unexpected trajectory_timesteps dim: {trajectory_timesteps.dim()}" + ) + # [B, S, C, T, H, W] -> [B, S, T, C, H, W] to match self-forcing + trajectory_latents = trajectory_latents.permute(0, 1, 3, 2, 4, 5) + + # Move to device + device = get_local_torch_device() + training_batch.image_embeds = clip_feature.to(device, dtype=torch.bfloat16) + training_batch.image_latents = first_frame_latent.to(device, dtype=torch.bfloat16) + if keyboard_cond is not None and keyboard_cond.numel() > 0: + training_batch.keyboard_cond = keyboard_cond.to(device, dtype=torch.bfloat16) + else: + training_batch.keyboard_cond = None + if mouse_cond is not None and mouse_cond.numel() > 0: + training_batch.mouse_cond = mouse_cond.to(device, dtype=torch.bfloat16) + else: + training_batch.mouse_cond = None + training_batch.infos = infos + + return training_batch, trajectory_latents.to( + device, dtype=torch.bfloat16), trajectory_timesteps.to(device) + + def _get_timestep(self, + min_timestep: int, + max_timestep: int, + batch_size: int, + num_frame: int, + num_frame_per_block: int, + uniform_timestep: bool = False) -> torch.Tensor: + if uniform_timestep: + timestep = torch.randint(min_timestep, + max_timestep, [batch_size, 1], + device=self.device, + dtype=torch.long).repeat(1, num_frame) + return timestep + else: + timestep = torch.randint(min_timestep, + max_timestep, [batch_size, num_frame], + device=self.device, + dtype=torch.long) + # logger.info(f"individual timestep: {timestep}") + # make the noise level the same within every block + timestep = timestep.reshape(timestep.shape[0], -1, + num_frame_per_block) + timestep[:, :, 1:] = timestep[:, :, 0:1] + timestep = timestep.reshape(timestep.shape[0], -1) + return timestep + + def _prepare_dit_inputs(self, + training_batch: TrainingBatch) -> TrainingBatch: + """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" + + # First, call parent method to prepare noise, timesteps, etc. for video latents + training_batch = super()._prepare_dit_inputs(training_batch) + + assert isinstance(training_batch.image_latents, torch.Tensor) + image_latents = training_batch.image_latents.to( + get_local_torch_device(), dtype=torch.bfloat16) + + temporal_compression_ratio = 4 + num_frames = (self.training_args.num_latent_t - + 1) * temporal_compression_ratio + 1 + batch_size, num_channels, _, latent_height, latent_width = image_latents.shape + mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, + latent_width) + mask_lat_size[:, :, 1:] = 0 + + first_frame_mask = mask_lat_size[:, :, :1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, dim=2, repeats=temporal_compression_ratio) + mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], + dim=2) + mask_lat_size = mask_lat_size.view(batch_size, -1, + temporal_compression_ratio, + latent_height, latent_width) + mask_lat_size = mask_lat_size.transpose(1, 2) + mask_lat_size = mask_lat_size.to( + image_latents.device).to(dtype=torch.bfloat16) + + training_batch.noisy_model_input = torch.cat( + [training_batch.noisy_model_input, mask_lat_size, image_latents], + dim=1) + + return training_batch + + def _step_predict_next_latent( + self, traj_latents: torch.Tensor, traj_timesteps: torch.Tensor, + image_embeds: torch.Tensor, + image_latents: torch.Tensor, + keyboard_cond: torch.Tensor | None, + mouse_cond: torch.Tensor | None + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, dict[str, + torch.Tensor]]: + latent_vis_dict: dict[str, torch.Tensor] = {} + device = get_local_torch_device() + target_latent = traj_latents[:, -1] + + # Shapes: traj_latents [B, S, C, T, H, W], traj_timesteps [B, S] + B, S, num_frames, num_channels, height, width = traj_latents.shape + + # Lazily cache nearest trajectory index per DMD step based on the (fixed) S timesteps + if self._cached_closest_idx_per_dmd is None: + traj_ts = traj_timesteps[0].float().cpu() + dmd_steps = self.dmd_denoising_steps.float().cpu() + closest_idx = torch.argmin( + torch.abs(traj_ts.unsqueeze(0) - dmd_steps.unsqueeze(1)), + dim=1) + self._cached_closest_idx_per_dmd = closest_idx.to(torch.long).cpu() + logger.info("self._cached_closest_idx_per_dmd: %s", + self._cached_closest_idx_per_dmd) + logger.info("corresponding timesteps: %s", + traj_ts[self._cached_closest_idx_per_dmd]) + + # Select the K indexes from traj_latents using self._cached_closest_idx_per_dmd + # traj_latents: [B, S, C, T, H, W], self._cached_closest_idx_per_dmd: [K] + # Output: [B, K, C, T, H, W] + assert self._cached_closest_idx_per_dmd is not None + relevant_traj_latents = torch.index_select( + traj_latents, + dim=1, + index=self._cached_closest_idx_per_dmd.to(traj_latents.device)) + logger.info("relevant_traj_latents: %s", relevant_traj_latents.shape) + # assert relevant_traj_latents.shape[0] == 1 + + indexes = self._get_timestep( # [B, num_frames] + 0, + len(self.dmd_denoising_steps), + B, + num_frames, + 3, + uniform_timestep=False) + logger.info("indexes: %s", indexes.shape) + logger.info("indexes: %s", indexes) + # noisy_input = relevant_traj_latents[indexes] + noisy_input = torch.gather( + relevant_traj_latents, + dim=1, + index=indexes.reshape(B, 1, num_frames, 1, 1, + 1).expand(-1, -1, -1, num_channels, height, + width).to(self.device)).squeeze(1) + latent_model_input = noisy_input.permute(0, 2, 1, 3, 4) + if image_latents is not None: + latent_model_input = torch.cat( + [ + latent_model_input, + image_latents.to(latent_model_input.device, + latent_model_input.dtype), + ], + dim=1) + timestep = self.dmd_denoising_steps[indexes] + logger.info("selected timestep for rank %s: %s", + self.global_rank, + timestep, + local_main_process_only=False) + + # Prepare inputs for transformer + latent_vis_dict["noisy_input"] = noisy_input.permute( + 0, 2, 1, 3, 4).detach().clone().cpu() + latent_vis_dict["x0"] = target_latent.permute(0, 2, 1, 3, + 4).detach().clone().cpu() + + latent_model_input = latent_model_input.to(device, dtype=torch.bfloat16) + timestep = timestep.to(device, dtype=torch.bfloat16) + + logger.info("========== Transformer Input ==========") + logger.info("hidden_states (latent_model_input) shape: %s, dtype: %s", latent_model_input.shape, latent_model_input.dtype) + logger.info("hidden_states min/max/mean: %.4f / %.4f / %.4f", + latent_model_input.min().item(), latent_model_input.max().item(), latent_model_input.mean().item()) + logger.info("encoder_hidden_states_image (image_embeds) shape: %s", image_embeds.shape if image_embeds is not None else None) + logger.info("timestep shape: %s, dtype: %s", timestep.shape, timestep.dtype) + logger.info("keyboard_cond: %s", keyboard_cond.shape if keyboard_cond is not None else None) + logger.info("mouse_cond: %s", mouse_cond.shape if mouse_cond is not None else None) + + if keyboard_cond is not None and mouse_cond is not None: + viewmats_list = [] + intrinsics_list = [] + action_labels_list = [] + for b in range(latent_model_input.shape[0]): + viewmats, intrinsics, action_labels = process_custom_actions( + keyboard_cond[b], mouse_cond[b]) + viewmats_list.append(viewmats) + intrinsics_list.append(intrinsics) + action_labels_list.append(action_labels) + viewmats = torch.stack(viewmats_list, dim=0).to(device=device, + dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, dim=0).to( + device=device, dtype=torch.bfloat16) + action_labels = torch.stack(action_labels_list, dim=0).to( + device=device, dtype=torch.bfloat16) + else: + viewmats = None + intrinsics = None + action_labels = None + + empty_text = torch.zeros((latent_model_input.shape[0], 0, + self.transformer.hidden_size), + device=device, + dtype=torch.bfloat16) + + input_kwargs = { + "hidden_states": latent_model_input, + "encoder_hidden_states": empty_text, + "encoder_hidden_states_image": image_embeds, + "timestep": timestep, + "viewmats": viewmats, + "Ks": intrinsics, + "action": action_labels, + "return_dict": False, + } + # Predict noise and step the scheduler to obtain next latent + with set_forward_context(current_timestep=timestep, + attn_metadata=None, + forward_batch=None): + noise_pred = self.transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + + logger.info("========== Transformer Output ==========") + logger.info("noise_pred shape: %s", noise_pred.shape) + logger.info("noise_pred min/max/mean: %.4f / %.4f / %.4f", + noise_pred.min().item(), noise_pred.max().item(), noise_pred.mean().item()) + + from fastvideo.models.utils import pred_noise_to_pred_video + pred_video = pred_noise_to_pred_video( + pred_noise=noise_pred.flatten(0, 1), + noise_input_latent=noisy_input.flatten(0, 1), + timestep=timestep.to(dtype=torch.bfloat16).flatten(0, 1), + scheduler=self.modules["scheduler"]).unflatten( + 0, noise_pred.shape[:2]) + latent_vis_dict["pred_video"] = pred_video.permute( + 0, 2, 1, 3, 4).detach().clone().cpu() + + return pred_video, target_latent, timestep, latent_vis_dict + + def train_one_step(self, training_batch): # type: ignore[override] + self.transformer.train() + self.optimizer.zero_grad() + training_batch.total_loss = 0.0 + args = cast(TrainingArgs, self.training_args) + + # Using cached nearest index per DMD step; computation happens in _step_predict_next_latent + + for _ in range(args.gradient_accumulation_steps): + training_batch, traj_latents, traj_timesteps = self._get_next_batch( + training_batch) + image_embeds = training_batch.image_embeds + image_latents = training_batch.image_latents + keyboard_cond = training_batch.keyboard_cond + mouse_cond = training_batch.mouse_cond + assert traj_latents.shape[0] == 1 + + # Shapes: traj_latents [B, S, C, T, H, W], traj_timesteps [B, S] + _, S = traj_latents.shape[0], traj_latents.shape[1] + if S < 2: + raise ValueError("Trajectory must contain at least 2 steps") + + # Forward to predict next latent by stepping scheduler with predicted noise + noise_pred, target_latent, t, latent_vis_dict = self._step_predict_next_latent( + traj_latents, traj_timesteps, image_embeds, image_latents, keyboard_cond, mouse_cond) + + training_batch.latent_vis_dict.update(latent_vis_dict) + + mask = t != 0 + + # Compute loss + loss = F.mse_loss(noise_pred[mask], + target_latent[mask], + reduction="mean") + loss = loss / args.gradient_accumulation_steps + + with set_forward_context(current_timestep=t, + attn_metadata=None, + forward_batch=None): + loss.backward() + avg_loss = loss.detach().clone() + training_batch.total_loss += avg_loss.item() + + # Clip grad and step optimizers + grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in self.transformer.parameters() if p.requires_grad], + args.max_grad_norm if args.max_grad_norm is not None else 0.0) + + self.optimizer.step() + self.lr_scheduler.step() + + if grad_norm is None: + grad_value = 0.0 + else: + try: + if isinstance(grad_norm, torch.Tensor): + grad_value = float(grad_norm.detach().float().item()) + else: + grad_value = float(grad_norm) + except Exception: + grad_value = 0.0 + training_batch.grad_norm = grad_value + B, S, T, C, H, W = traj_latents.shape + training_batch.raw_latent_shape = (B, C, T, H, W) + return training_batch + + def _prepare_validation_batch(self, sampling_param: SamplingParam, + training_args: TrainingArgs, + validation_batch: dict[str, Any], + num_inference_steps: int) -> ForwardBatch: + sampling_param.prompt = validation_batch['prompt'] + sampling_param.height = training_args.num_height + sampling_param.width = training_args.num_width + sampling_param.image_path = validation_batch.get( + 'image_path') or validation_batch.get('video_path') + sampling_param.num_inference_steps = num_inference_steps + sampling_param.data_type = "video" + assert self.seed is not None + sampling_param.seed = self.seed + + latents_size = [(sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, sampling_param.width // 8] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (training_args.num_latent_t - + 1) * temporal_compression_factor + 1 + sampling_param.num_frames = num_frames + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=torch.Generator(device="cpu").manual_seed(self.seed), + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=training_args.VSA_sparsity, + ) + if "image" in validation_batch and validation_batch["image"] is not None: + batch.pil_image = validation_batch["image"] + + if "keyboard_cond" in validation_batch and validation_batch[ + "keyboard_cond"] is not None: + keyboard_cond = validation_batch["keyboard_cond"] + keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) + keyboard_cond = keyboard_cond.unsqueeze(0) + batch.keyboard_cond = keyboard_cond + + if "mouse_cond" in validation_batch and validation_batch[ + "mouse_cond"] is not None: + mouse_cond = validation_batch["mouse_cond"] + mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) + mouse_cond = mouse_cond.unsqueeze(0) + batch.mouse_cond = mouse_cond + + return batch + + def visualize_intermediate_latents(self, training_batch: TrainingBatch, + training_args: TrainingArgs, step: int): + tracker_loss_dict: dict[str, Any] = {} + latents_vis_dict = training_batch.latent_vis_dict + latent_log_keys = ['noisy_input', 'x0', 'pred_video'] + for latent_key in latent_log_keys: + assert latent_key in latents_vis_dict and latents_vis_dict[ + latent_key] is not None + latent = latents_vis_dict[latent_key] + pixel_latent = self.validation_pipeline.decoding_stage.decode( + latent, training_args) + + video = pixel_latent.cpu().float() + video = video.permute(0, 2, 1, 3, 4) + video = (video * 255).numpy().astype(np.uint8) + video_artifact = self.tracker.video( + video, fps=16, format="mp4") # change to 16 for Wan2.1 + if video_artifact is not None: + tracker_loss_dict[latent_key] = video_artifact + # Clean up references + del video, pixel_latent, latent + + if self.global_rank == 0 and tracker_loss_dict: + self.tracker.log_artifacts(tracker_loss_dict, step) + + +def main(args) -> None: + logger.info("Starting ODE-init training pipeline...") + pipeline = WanGameODEInitTrainingPipeline.from_pretrained( + args.pretrained_model_name_or_path, args=args) + args = pipeline.training_args + pipeline.train() + logger.info("ODE-init training pipeline done") + + +if __name__ == "__main__": + argv = sys.argv + from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.utils import FlexibleArgumentParser + parser = FlexibleArgumentParser() + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args() + args.dit_cpu_offload = False + main(args) From f113791e4095bfb6001964b26dbf2c7fd68ee318 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Tue, 10 Feb 2026 08:46:42 +0000 Subject: [PATCH 021/214] some fix --- .../action/000000_action.npy | Bin 0 -> 2994 bytes .../action/000001_action.npy | Bin 0 -> 2994 bytes .../action/000002_action.npy | Bin 0 -> 2994 bytes .../action/000003_action.npy | Bin 0 -> 2994 bytes .../action/000004_action.npy | Bin 0 -> 2994 bytes .../action/000005_action.npy | Bin 0 -> 2994 bytes .../action/000006_action.npy | Bin 0 -> 2994 bytes .../action/000007_action.npy | Bin 0 -> 2994 bytes .../action/000008_action.npy | Bin 0 -> 2994 bytes .../action/000009_action.npy | Bin 0 -> 2994 bytes .../action/000010_action.npy | Bin 0 -> 2994 bytes .../action/000011_action.npy | Bin 0 -> 2994 bytes .../action/000012_action.npy | Bin 0 -> 2994 bytes .../action/000013_action.npy | Bin 0 -> 2994 bytes .../action/000014_action.npy | Bin 0 -> 2994 bytes .../action/000015_action.npy | Bin 0 -> 2994 bytes .../action/000016_action.npy | Bin 0 -> 2994 bytes .../action/000017_action.npy | Bin 0 -> 2994 bytes .../action/000018_action.npy | Bin 0 -> 2994 bytes .../action/000019_action.npy | Bin 0 -> 2994 bytes .../action/000020_action.npy | Bin 0 -> 2994 bytes .../action/000021_action.npy | Bin 0 -> 2994 bytes .../action/000022_action.npy | Bin 0 -> 2994 bytes .../action/000023_action.npy | Bin 0 -> 2994 bytes .../action/000024_action.npy | Bin 0 -> 2994 bytes .../action/000025_action.npy | Bin 0 -> 2994 bytes .../action/000026_action.npy | Bin 0 -> 2994 bytes .../action/000027_action.npy | Bin 0 -> 2994 bytes .../action/000028_action.npy | Bin 0 -> 2994 bytes .../action/000029_action.npy | Bin 0 -> 2994 bytes .../action/000030_action.npy | Bin 0 -> 2994 bytes .../action/000031_action.npy | Bin 0 -> 2994 bytes .../action/000032_action.npy | Bin 0 -> 2994 bytes .../action/000033_action.npy | Bin 0 -> 2994 bytes .../action/000034_action.npy | Bin 0 -> 2994 bytes .../action/000035_action.npy | Bin 0 -> 2994 bytes .../action/000036_action.npy | Bin 0 -> 2994 bytes .../action/000037_action.npy | Bin 0 -> 2994 bytes .../action/000038_action.npy | Bin 0 -> 2994 bytes .../action/000039_action.npy | Bin 0 -> 2994 bytes .../action/000040_action.npy | Bin 0 -> 2994 bytes .../action/000041_action.npy | Bin 0 -> 2994 bytes .../action/000042_action.npy | Bin 0 -> 2994 bytes .../action/000043_action.npy | Bin 0 -> 2994 bytes .../action/000044_action.npy | Bin 0 -> 2994 bytes .../action/000045_action.npy | Bin 0 -> 2994 bytes .../action/000046_action.npy | Bin 0 -> 2994 bytes .../action/000047_action.npy | Bin 0 -> 2994 bytes .../action/000048_action.npy | Bin 0 -> 2994 bytes .../action/000049_action.npy | Bin 0 -> 2994 bytes .../action/000050_action.npy | Bin 0 -> 2994 bytes .../action/000051_action.npy | Bin 0 -> 2994 bytes .../action/000052_action.npy | Bin 0 -> 2994 bytes .../action/000053_action.npy | Bin 0 -> 2994 bytes .../action/000054_action.npy | Bin 0 -> 2994 bytes .../action/000055_action.npy | Bin 0 -> 2994 bytes .../action/000056_action.npy | Bin 0 -> 2994 bytes .../action/000057_action.npy | Bin 0 -> 2994 bytes .../action/000058_action.npy | Bin 0 -> 2994 bytes .../action/000059_action.npy | Bin 0 -> 2994 bytes .../action/000060_action.npy | Bin 0 -> 2994 bytes .../action/000061_action.npy | Bin 0 -> 2994 bytes .../action/000062_action.npy | Bin 0 -> 2994 bytes .../action/000063_action.npy | Bin 0 -> 2994 bytes .../causal_wangame_ode_init/action/README.md | 66 ++++++ .../finetune_ode_init.sh | 18 +- .../preprocess_data.sh | 24 +++ .../causal_wangame_ode_init/validation.json | 16 +- ...game_preprocess_pipeline_ode_trajectory.py | 7 +- fastvideo/pipelines/stages/denoising.py | 2 +- .../training/wangame_ode_causal_pipeline.py | 4 +- fastvideo/utils.py | 2 +- visualize_trajectory.py | 198 ++++++++++++++++++ 73 files changed, 313 insertions(+), 24 deletions(-) create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000000_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000001_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000002_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000003_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000004_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000005_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000006_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000007_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000008_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000009_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000010_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000011_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000012_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000013_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000014_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000015_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000016_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000017_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000018_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000019_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000020_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000021_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000022_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000023_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000024_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000025_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000026_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000027_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000028_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000029_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000030_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000031_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000032_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000033_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000034_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000035_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000036_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000037_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000038_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000039_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000040_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000041_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000042_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000043_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000044_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000045_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000046_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000047_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000048_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000049_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000050_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000051_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000052_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000053_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000054_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000055_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000056_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000057_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000058_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000059_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000060_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000061_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000062_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000063_action.npy create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/README.md create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh create mode 100644 visualize_trajectory.py diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000000_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000000_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..8a6ab2ab14b855f0691ea93076134c495f72926b GIT binary patch literal 2994 zcmeHIF-yZh6i!lO#X37UWlIVrlY@v*sNi5QRl&hQ!X>#9vB`zYMF_UgrP{!4d4m5| zbBRMJZVE22TBwCFv;Vzu)Tm#Y{&VWJt9g`0sO$DI3P*JXdW=A zfa^^q@O~1uB^5Zz8qQT8-ryE2M;%5zu7|8hv8dOkCs1zMqaIfm;JWVRXB`bVV+MZt z086m=-V-i-f@lbl7U_aC>VH!@H_q-WL<6qPj=OAUTWpG8n|$;|Q>$^?+cwI4tBBM= z>d<`~(ht%P(hpq)q#wF!{LTLW^@QenO!5r>IOer_%paALjn=(duB8J1r+~0*j%u*K E06JO3F8}}l literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000001_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000001_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..e649b6c7273d76092c69f063d53cc78ae717dfbf GIT binary patch literal 2994 zcmeHIze~eF6uzX!igk8y%62J~Ob#MKp@M_KR0RhI376zb#3mOmi4bg|i`u|#d4m6~ z=7&NQw+=4%hIjAYJ-)|%e0Q(3Z!Rux74(Rnuul_@<6|8!rg)^|8RID@k;Ax8>+~6! zC$!2Z0f}j~H-;m9sc&YxtF1RYK#ht0C=27QuJD_nzNDZ`*AG(XK_TGy-7BPiDGQBNofuxIzKR`91g%+|bp?{8fwHosW#bm8^+sarn@P7={lFdlv F%uh&;#4i8< literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000002_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000002_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..3ee9a9c38c0fc4b02f535415193cac4201c448a3 GIT binary patch literal 2994 zcmeH|ze~eF6vr>AF=Cw^oU&aCC6j}QP^jQwFjc|9LBb`ua$=K|OCkhY=%O}oTb|&5 ztN9@`h?|1Tz2R~1-Qx@Q<-@(wxw*K!RnQ}P!U0Wuj*ktznBbv-r!h}CiQJe6v~Hh~ zSwgFR!pNLfwK*6XOXGL3yV`ogebj{5kFs!{Tdp5-YK7S>y(gTL9K@c>sUJs4$}>NO z5`@-Fg3zO&9Vx*^&v2@E@Cw&p+RA|SgchXvoYwQ|$KVv{SEL#8no^cAl=zfIxNe`J zS&BtJC1{St+8zwerTM$qU2VNteWeM$pXBkpaQu)m?8NyjyGM+n0{EWKa7dFhV|kcC z1zc~Cz>OqoAr-jFGn}dcyuvlut~ww+u18rh$2DG?96`Bmw|ZP%faiJVA0nFMjxq?! zJ6M9vcOGyNP{bl28qsS~Vti4$SMK%;kUrO@hh4UlZ){4 DWcb7{ literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000004_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000004_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..9ce5230372d7f3a39ac9eb871b638708e37e2d55 GIT binary patch literal 2994 zcmeH|ze~eF6vva)7_rU{PT7(|$>bm+6e>6vOjU4jkZ?(^L~MR=xd_1)x>Q@ZEl=>j z)%@_lh?|3h_lA4C?>)ZoK0J7(b8~TdD@%{klM&#|C&saE%qPZBH>L@p1jTMb0$j^y zXqMqh&q9>qsx}8heWCv@c9&bPMqg@z?Z$#5$#x!a;U$O!5GtWsn}*t_rF~^@KSS7O%JlH7+sQXKsdgK`Y4uRcQ9B#YZ6$QU zffMNNhuD)Y4^D}5;aTXMK$n-`)8)Y_aV|UyofGKt`X79ty3kCGl03sdo_VD@^9P&p TTIoU(l_2m}1nQN|QT13Kr_=zg literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000005_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000005_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..02cd7ab2e0691d85f98d3d6140435496260cf2dd GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGacRJ{HmZH3|;+@f9~Tc6;6 ztNmf2h}#V=d62xkm&X_K@g=Xc?yj!yCG?D5utyV@;ZqGSXLzjPdBjpiLOWs}tVY?BdMvyJiN5UA%fgjk6x>1;*7ni3sz+mwlxM_0=Cp^TynL1AoZn`EeiZ>=zoU(Wakjdn<(2lXi4*=S99{ zLhvAX=-x!_g&u?+gdX|~2tD-K*#3WjszY-nh_ZzK+2-Y{%^wxhjod$0tOW!A$3QLF Ij8v2H4YmmYtpET3 literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000007_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000007_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..b0f8a4d3e79be73f379dc65f8aca9f50c5a87b48 GIT binary patch literal 2994 zcmeH|u}i~16vi*9F=Cw^oU$c_lF30tC{%DTn5y95Amy4|iP$FLk_f>Tx>Q@ZEl=>j z)ui!(5jO`B?hW_defRj_ejL2Qy}7u&RmdZGGJTeK+&nhS*}&`?<~ZUhr{OT-KCA3A zI!RdRCjpIFdA2%TV{ZJeb{AW(W}9qE?1$Mj&h4QWab{1mNqSE?r#XndA!l9`CMnOn z6bca9rW1r7`gKbI4q3vfGJ+Sl2Fp=8K}%?Un#Zi_R|iM1Ue#AEq0GQ_-ShXdHjC3s zVYI%3Iap%nf#su!avuUU(5lct|BQ65ob6`_+Cm*4eoZ_1mPV?y{+nLEijYOG-l0k# z2{gG#K207-3Avz39|<&hNj^;;NC~;1N*@U{c}YG^9!Lqfp#Fa!$_XvBX_O`G$1$&# YWBy>>U#a+1sgl6o5~#jxM$5(i0K`53tpET3 literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000008_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000008_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..8a754f020b29a79843f069690896426397c4eb96 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarJu literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000010_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000010_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..e97638cfc59b49de2c798709bf1dc5e988c22fd0 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarc literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000015_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000015_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..a8c7fba01479245739ac912e26c76249940eec91 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGadYGO8=YfuXjGsP-=`FMtPVNrsKtwm|?!i9xA&#(cH_R z0;V>Vz_iG3ODeFWS2&klc!ygsEO`_RnCfRmLU_G0JArc3ehrws1lzVRziMloHOas& zA7BLr+j}B~8)NE2pad!h4YVIf>&DuBgJ8&%`EeKRYzs}|Xwy%PH?^L$y={AuZxsQd_sMpnHt7 GXMO{}?!+$u literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000017_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000017_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..29db3beb5a0180f62dc11684c506f12e0f4d6823 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV=$$M z!Q>WD7>&ATG_MTSyfSps=M-QK-NTxjUs{}swVG!FR`XJwP903hK|RVI4S|6R0Z@`H IO#~8p0K)3TF8}}l literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000018_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000018_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..19c0eba669a3dd77bd29b276e8edc30fc4accbae GIT binary patch literal 2994 zcmeHIze~eF6uzX!h;?>w%62J~Ob#MKp@M_KR0RhI376zb#O4Q=LHQM-HZGY9~<)~Y&&!R8* zS-Sxt;fKxiL;WhD&1PyNbP>9=4E&Y{s5UfLf+$Pqk8578*8D*+UddfpaTW~x9|N^z IGg2kv6WmzDF8}}l literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000019_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000019_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..d93f0a1d14b4a16ea50a17b888fde5e361295bf7 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaD0l59Oa|z(GVE85CA3F(nKJk F2LRtx#4i8< literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000020_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000020_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..7ae951a8fe075b3a6fd4df8416b8ce72f5845f52 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gW3Z(M zklO~En?~I;nr8-co*CTfa|*DA?qSW%FD*{RTFo;7t9hwTrw%6MARc9phQPpu04T|p ICISgP0L~-CF8}}l literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000023_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000023_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..6b483a2bec1c0413132a0c85bf31608b3502db32 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gW3Z(M znA--MyGGqJnr~of6GV@u2V&F16krYA!s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaJBn^=j4xl7kng}HH E0GKp!5dZ)H literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000031_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000031_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..1194a0be934c4e5c1227135515c854c21ba7ec3b GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa58g`>JBn^=j4xl7kng}HH E0Pa0;kpKVy literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000032_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000032_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..e7e2d0927f525b43f0349ba6b67f2274b971ee76 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa#@QB literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000033_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000033_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..3fa255b1c2bc1622532016d512ed5663c470b398 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtV1ihC_RN{RAl4{78V&=M N4nRq^G!aPX0RZ{t!Ts?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtV1ihC_RJYu5Nnhk4TphB M2cRTdng}HH0GV2ufB*mh literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000036_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000036_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..7c8fcfc3e36467407f735033f3046053a3a46cd8 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^DytQ@6B!(kxP M0Vv6qCISgP0Q|(6fB*mh literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000037_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000037_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..b5931a32db1828ad94d4331a87407ef9b3a30d55 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaCtc)$aDZo KvZaYYLJt572$_HY literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000038_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000038_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..9ee6cd72d22f8d652297a3233aae843455b24416 GIT binary patch literal 2994 zcmeH|F-yZh6vr>AF=Cw^oU$c_lF30tC{%DTn5y95Amx%=iP$FPk_f>Tx>OsutxxdF z_zmi3(p=+#5jO`1?+y2O|9kx5{dn+}?$z1(wSw-^gBg&-r{z=-s zv3{IR3SMyoQ?S_99m%~I(*Qy>)VOJ=e_A>h&gLV8O{Na^zq%cLb3@#%`=axqW}Rv_ zk*iATl7k>n-w(MbbsmC};3Bfr1%Wy*$*0akP!e23mbxHN=k-7MKzX5=Hj1-^{CMWo d^33lQ-Gz$K+iLa75y>n${N*0-mCbPZ*dI`T{%-&P literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000039_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000039_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..0fcbcf9672085c68454284c531e01fb3e1d58ab4 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*+NS5bqtON5f$t N(*Y>SmL>uTJpiBfmr(!! literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000042_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000042_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..8c5c83c9621edd2cebc15b626566703dbc90d9e8 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaF9sn^lZqWb$ literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000043_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000043_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..2adbe6dc83d774b4be358ba663492ab2f1c0522d GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*+PoC>{-ifldRU KBwLyYB=i73YHra0 literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000044_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000044_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..9adb526a17cc55cf37cc428cfd9179f000fe64dc GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^;$#iL;`&}jga KWJ?o)gdPB}#=p@3 literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000045_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000045_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..71a0e0328d53d2e67d08525c56f3a522b7e6b025 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaboS6e>6vnyTR7AmNf+iP$FPk_f>Tx~L7@mM8e3 z>Sxki;(-x22M6yB@7?|H@rV2I;1%}O@cdeq?xhFa$FWEBLrtHL^q!`VBa#plIuY?P zlTXnk#-$zyXokzX(d}sq?RT-g+<4JDQWbPJOsBKVbi9aQbDB<)TSN%TfbKX1dr=rC zB=r&~fT~p|(01t8Ed^N8Bb>-CJi{dzmfQ_mRP~c=hFQNdI)HptKeebl2ivyK-pi8zJIDpad#24cecU)`hkC1VM)?s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaF9sub@t$+Xk literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000053_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000053_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..453f150131c01ef3ec735f0a5f57b0182bd615d8 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaF9soaGt^fc4 literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000056_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000056_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..be2aa4c1be1b7d2740993ce4a526f41f2fcd4fff GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV=$$M z!Q>WD7>&ATG_MTSyfSps=M-QK-NTxjUs{}swVG!FR`XJwP901P3}?@ru?2Z-lpYO- OflLRWBwLyYB=i7i`k8s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpggEKvV zTs0augEMYM<7G5nAR$0(I7|W7&^@fV`K86FSgUy^U^Oq*>D0l*z;O1=8C#ICqx5Jv P3}iY0CE3zMAfX2Uf@qn5 literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000059_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000059_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..40745ce3833ddf43899d8251217dda4ad6d65eb2 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaD0l*z;O1=8C#Iiqx5Jv3}iY0 LCE3zMAfX2Ug=Cq4 literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000060_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000060_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..8c808a87ef94fe7b6846fc69915e658deb33c4fd GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gW3Z(M zklO~En?~I;nr8-co*CTfa|*DA?qSW%FD*{RTFo;7t9hwTrw%3thO=kR*n$FjlpYO- OflLRWBwLyYB=i7uGns$@ literal 0 HcmV?d00001 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000063_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000063_action.npy new file mode 100644 index 0000000000000000000000000000000000000000..4cd0c40eab41a204a9d9943087d8de28bc893553 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gW3Z(M znA--MyGGqJnr~of6GV@u2V&F16krYA![S] + Static +17. Switch [S]->[W] + Static +18. Switch [A]->[D] + Static +19. Switch [D]->[A] + Static +20. Switch [W]->[A] + Static +21. Switch [W]->[D] + Static +22. Switch [S]->[A] + Static +23. Switch [S]->[D] + Static +24. No Key + Switch [left]->[right] +25. No Key + Switch [right]->[left] +26. No Key + Switch [up]->[down] +27. No Key + Switch [down]->[up] +28. No Key + Switch [up_left]->[up_right] +29. No Key + Switch [up_right]->[up_left] +30. No Key + Switch [left]->[up] +31. No Key + Switch [right]->[down] +32. Hold [W] + Hold [left] +33. Hold [S] + Hold [left] +34. Hold [W] + Hold [right] +35. Hold [S] + Hold [right] +36. Hold [A] + Hold [up] +37. Hold [D] + Hold [up] +38. Hold [WA] + Hold [down] +39. Hold [WD] + Hold [down] +40. Hold [W] + Hold [up_left] +41. Hold [S] + Hold [up_left] +42. Hold [W] + Hold [up_right] +43. Hold [S] + Hold [up_right] +44. Hold [A] + Hold [down_left] +45. Hold [D] + Hold [down_right] +46. Hold [WA] + Hold [right] +47. Hold [WD] + Hold [left] +48. Hold [W] + Switch [left]->[right] +49. Hold [W] + Switch [right]->[left] +50. Hold [W] + Switch [up]->[down] +51. Hold [W] + Switch [down]->[up] +52. Hold [W] + Switch [left]->[up] +53. Hold [W] + Switch [right]->[up] +54. Hold [W] + Switch [left]->[down] +55. Hold [W] + Switch [right]->[down] +56. Switch [W]->[S] + Hold [up] +57. Switch [S]->[W] + Hold [up] +58. Switch [A]->[D] + Hold [up] +59. Switch [D]->[A] + Hold [up] +60. Switch [W]->[A] + Hold [up] +61. Switch [W]->[D] + Hold [up] +62. Switch [S]->[A] + Hold [up] +63. Switch [S]->[D] + Hold [up] diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh index d2ffb8a46..55c333c49 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh @@ -1,12 +1,14 @@ #!/bin/bash +export PYTHONPATH="/mnt/fast-disks/hao_lab/kaiqin/FastVideo_wangame:$PYTHONPATH" +export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=online +export WANDB_MODE=offline export TOKENIZERS_PARALLELISM=false -MODEL_PATH="" -DATA_DIR="../vizdoom/preprocessed" -VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" +MODEL_PATH="Wan2.1-Fun-1.3B-InP-Diffusers" +DATA_DIR="mc_wasd_action/preprocessed" +VALIDATION_DATASET_FILE="$(dirname "$0")/validation.json" NUM_GPUS=8 # export CUDA_VISIBLE_DEVICES=4,5,6,7 # IP=[MASTER NODE IP] @@ -16,8 +18,8 @@ training_args=( --tracker_project_name "wangame_ode_init" --output_dir "checkpoints/wangame_ode_init" --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" - --wandb_run_name "wangame_ode_init_8gpu" - --max_train_steps 2000 + --wandb_run_name "wangame_ode_init" + --max_train_steps 10000000 --train_batch_size 1 --train_sp_batch_size 1 --gradient_accumulation_steps 4 @@ -63,8 +65,8 @@ validation_args=( optimizer_args=( --learning_rate 6e-6 --mixed_precision "bf16" - --weight_only_checkpointing_steps 100 - --training_state_checkpointing_steps 100 + --weight_only_checkpointing_steps 10000000 + --training_state_checkpointing_steps 10000000 --weight_decay 1e-4 --max_grad_norm 1.0 ) diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh new file mode 100644 index 000000000..63c8ec77b --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +export PYTHONPATH="/mnt/fast-disks/hao_lab/kaiqin/FastVideo_wangame:$PYTHONPATH" + +GPU_NUM=1 # 2,4,8 +MODEL_PATH="./Wan2.1-Fun-1.3B-InP-Diffusers" +DATA_MERGE_PATH="mc_wasd_action/merge.txt" +OUTPUT_DIR="mc_wasd_action/preprocessed" + +torchrun --nproc_per_node=$GPU_NUM \ + fastvideo/pipelines/preprocess/v1_preprocess.py \ + --model_path $MODEL_PATH \ + --data_merge_path $DATA_MERGE_PATH \ + --preprocess_video_batch_size 1 \ + --seed 42 \ + --max_height 352 \ + --max_width 640 \ + --num_frames 81 \ + --dataloader_num_workers 0 \ + --output_dir=$OUTPUT_DIR \ + --samples_per_file 8 \ + --train_fps 25 \ + --flush_frequency 8 \ + --preprocess_task wangame_ode_trajectory & diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json index 649826443..d7a6f0c3c 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json @@ -2,7 +2,7 @@ "data": [ { "caption": "00. Hold [W] + Static", - "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000000.jpg", + "image_path": "../../../../mc_wasd_action/validate/0.jpg", "action_path": "action/000000_action.npy", "video_path": null, "num_inference_steps": 4, @@ -12,7 +12,7 @@ }, { "caption": "01. Hold [S] + Static", - "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000001.jpg", + "image_path": "../../../../mc_wasd_action/validate/1.jpg", "action_path": "action/000001_action.npy", "video_path": null, "num_inference_steps": 4, @@ -22,7 +22,7 @@ }, { "caption": "02. Hold [A] + Static", - "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000002.jpg", + "image_path": "../../../../mc_wasd_action/validate/2.jpg", "action_path": "action/000002_action.npy", "video_path": null, "num_inference_steps": 4, @@ -32,7 +32,7 @@ }, { "caption": "10. No Key + Hold [left]", - "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000003.jpg", + "image_path": "../../../../mc_wasd_action/validate/3.jpg", "action_path": "action/000010_action.npy", "video_path": null, "num_inference_steps": 4, @@ -42,7 +42,7 @@ }, { "caption": "12. No Key + Hold [up_right]", - "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000004.jpg", + "image_path": "../../../../mc_wasd_action/validate/4.jpg", "action_path": "action/000012_action.npy", "video_path": null, "num_inference_steps": 4, @@ -52,7 +52,7 @@ }, { "caption": "21. Switch [W]->[D] + Static", - "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000005.jpg", + "image_path": "../../../../mc_wasd_action/validate/5.jpg", "action_path": "action/000021_action.npy", "video_path": null, "num_inference_steps": 4, @@ -62,7 +62,7 @@ }, { "caption": "29. No Key + Switch [up_right]->[up_left]", - "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000006.jpg", + "image_path": "../../../../mc_wasd_action/validate/6.jpg", "action_path": "action/000029_action.npy", "video_path": null, "num_inference_steps": 4, @@ -72,7 +72,7 @@ }, { "caption": "50. Hold [W] + Switch [up]->[down]", - "image_path": "../../../../../FastvideoWorldModel-MC/gen/validate/000007.jpg", + "image_path": "../../../../mc_wasd_action/validate/7.jpg", "action_path": "action/000050_action.npy", "video_path": null, "num_inference_steps": 4, diff --git a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py index 21082071c..fcae785d0 100644 --- a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py +++ b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py @@ -37,7 +37,7 @@ from fastvideo.pipelines.pipeline_batch_info import ForwardBatch from fastvideo.pipelines.preprocess.preprocess_pipeline_base import ( BasePreprocessPipeline) -from fastvideo.pipelines.stages import (CausalDMDDenosingStage, DecodingStage, +from fastvideo.pipelines.stages import (DecodingStage, DenoisingStage, InputValidationStage, LatentPreparationStage, ImageEncodingStage, @@ -88,11 +88,10 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): scheduler=self.get_module("scheduler"), transformer=self.get_module("transformer", None))) self.add_stage(stage_name="denoising_stage", - stage=CausalDMDDenosingStage( + stage=DenoisingStage( transformer=self.get_module("transformer"), scheduler=self.get_module("scheduler"), - transformer_2=self.get_module("transformer_2", None), - vae=self.get_module("vae"), + pipeline=self, )) self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae"))) diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index 44526b3b6..6bb86da48 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -195,7 +195,7 @@ def forward( { "mouse_cond": batch.mouse_cond, "keyboard_cond": batch.keyboard_cond, - "c2ws_plucker_emb": batch.c2ws_plucker_emb, + # "c2ws_plucker_emb": batch.c2ws_plucker_emb, }, ) diff --git a/fastvideo/training/wangame_ode_causal_pipeline.py b/fastvideo/training/wangame_ode_causal_pipeline.py index 779634c0b..e6dbdb1c6 100644 --- a/fastvideo/training/wangame_ode_causal_pipeline.py +++ b/fastvideo/training/wangame_ode_causal_pipeline.py @@ -60,7 +60,7 @@ def initialize_training_pipeline(self, training_args: TrainingArgs): self.vae.requires_grad_(False) self.timestep_shift = self.training_args.pipeline_config.flow_shift - assert self.timestep_shift == 5.0, "flow_shift must be 5.0" + assert self.timestep_shift == 3.0, f"flow_shift must be 3.0, but got {self.timestep_shift}" self.noise_scheduler = SelfForcingFlowMatchScheduler( shift=self.timestep_shift, sigma_min=0.0, extra_one_step=True) self.noise_scheduler.set_timesteps(num_inference_steps=1000, @@ -68,7 +68,7 @@ def initialize_training_pipeline(self, training_args: TrainingArgs): logger.info("dmd_denoising_steps: %s", self.training_args.pipeline_config.dmd_denoising_steps) - self.dmd_denoising_steps = torch.tensor([1000, 750, 500, 250], + self.dmd_denoising_steps = torch.tensor([1000, 666, 333], dtype=torch.long, device=get_local_torch_device()) if training_args.warp_denoising_step: # Warp the denoising step according to the scheduler time shift diff --git a/fastvideo/utils.py b/fastvideo/utils.py index 9f1e364d1..b703743d4 100644 --- a/fastvideo/utils.py +++ b/fastvideo/utils.py @@ -936,7 +936,7 @@ def save_decoded_latents_as_video(decoded_latents: list[torch.Tensor], for x in videos: x = make_grid(x, nrow=6) x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) - frames.append((x * 255).numpy().astype(np.uint8)) + frames.append((x * 255).cpu().numpy().astype(np.uint8)) os.makedirs(os.path.dirname(output_path), exist_ok=True) imageio.mimsave(output_path, frames, fps=fps, format="mp4") diff --git a/visualize_trajectory.py b/visualize_trajectory.py new file mode 100644 index 000000000..505303190 --- /dev/null +++ b/visualize_trajectory.py @@ -0,0 +1,198 @@ +import argparse +import os +import numpy as np +import pyarrow.parquet as pq +import torch +from tqdm import tqdm + +from fastvideo import PipelineConfig +from fastvideo.configs.models.vaes import WanVAEConfig +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.models.loader.component_loader import VAELoader +from fastvideo.utils import maybe_download_model, save_decoded_latents_as_video + + +def _torch_dtype_from_precision(precision: str) -> torch.dtype: + precision = precision.lower() + if precision == "fp32": + return torch.float32 + if precision == "fp16": + return torch.float16 + if precision == "bf16": + return torch.bfloat16 + raise ValueError(f"Unsupported precision: {precision}") + + +def _denormalize_latents_for_vae(vae, latents: torch.Tensor) -> torch.Tensor: + if bool(getattr(vae, "handles_latent_denorm", False)): + return latents + + cfg = getattr(vae, "config", None) + + if cfg is not None and hasattr(cfg, "latents_mean") and hasattr( + cfg, "latents_std"): + latents_mean = torch.tensor(cfg.latents_mean, + device=latents.device, + dtype=latents.dtype).view(1, -1, 1, 1, 1) + latents_std = torch.tensor(cfg.latents_std, + device=latents.device, + dtype=latents.dtype).view(1, -1, 1, 1, 1) + return latents * latents_std + latents_mean + + if hasattr(vae, "scaling_factor"): + if isinstance(vae.scaling_factor, torch.Tensor): + latents = latents / vae.scaling_factor.to(latents.device, + latents.dtype) + else: + latents = latents / vae.scaling_factor + + if hasattr(vae, "shift_factor") and vae.shift_factor is not None: + if isinstance(vae.shift_factor, torch.Tensor): + latents = latents + vae.shift_factor.to(latents.device, + latents.dtype) + else: + latents = latents + vae.shift_factor + + return latents + + +@torch.inference_mode() +def _decode_with_vae(vae, latents: torch.Tensor, *, device: torch.device, + precision: str) -> torch.Tensor: + latents = latents.to(device=device) + target_dtype = _torch_dtype_from_precision(precision) + latents = latents.to(dtype=target_dtype) + + latents = _denormalize_latents_for_vae(vae, latents) + + use_autocast = (device.type == "cuda" and target_dtype != torch.float32) + with torch.autocast(device_type=device.type, + dtype=target_dtype, + enabled=use_autocast): + decoded = vae.decode(latents) + + return (decoded / 2 + 0.5).clamp(0, 1) + +def main(): + parser = argparse.ArgumentParser(description="Visualize Trajectory from Parquet file") + parser.add_argument("--parquet_path", type=str, required=True, help="Path to the input parquet file") + parser.add_argument("--model_path", type=str, required=True, help="Path to the model directory") + parser.add_argument("--output_dir", type=str, default="visualizations", help="Directory to save output videos") + parser.add_argument("--num_samples", type=int, default=1, help="Number of samples to visualize") + parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu") + parser.add_argument("--vae_precision", + type=str, + default="fp32", + choices=["fp32", "fp16", "bf16"], + help="Precision for VAE decoding") + parser.add_argument("--vae_subfolder", + type=str, + default="vae", + help="Subfolder name containing VAE weights/config") + parser.add_argument("--fps", type=int, default=25, help="Output video FPS") + parser.add_argument( + "--decode_steps", + type=str, + default="last", + help= + "Which trajectory steps to decode: 'last', 'all', or comma-separated indices (e.g. '0,10,20')", + ) + + args = parser.parse_args() + + device = torch.device(args.device) + print(f"Using device: {device}, vae_precision: {args.vae_precision}") + + os.makedirs(args.output_dir, exist_ok=True) + + # Load VAE (must load weights; creating AutoencoderKLWan(config) alone leaves random weights) + print(f"Loading model from {args.model_path}...") + model_root = maybe_download_model(args.model_path) + pipeline_config = PipelineConfig.from_pretrained(model_root) + pipeline_config.update_config_from_dict({ + "vae_precision": args.vae_precision, + "vae_config": WanVAEConfig(load_encoder=False, load_decoder=True), + }) + fastvideo_args = FastVideoArgs( + model_path=model_root, + num_gpus=1, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=True, + pipeline_config=pipeline_config, + ) + + vae_path = os.path.join(model_root, args.vae_subfolder) + vae = VAELoader().load(vae_path, fastvideo_args) + vae.to(device) + + # Read Parquet + print(f"Reading parquet file: {args.parquet_path}") + table = pq.read_table(args.parquet_path) + + # Iterate over rows + num_visualized = 0 + + pbar = tqdm(total=min(args.num_samples, table.num_rows)) + + for i in range(table.num_rows): + if num_visualized >= args.num_samples: + break + + row = table.slice(i, length=1) + record = row.to_pydict() + + video_id = record["id"][0] + + # Parse Latents + shape = record["trajectory_latents_shape"][0] + dtype = record["trajectory_latents_dtype"][0] + dtype = np.dtype(dtype) + + latents_bytes = record["trajectory_latents_bytes"][0] + # Copy to avoid read-only warning + latents_np = np.copy(np.frombuffer(latents_bytes, dtype=dtype).reshape(shape)) + + latents_tensor = torch.from_numpy(latents_np) + if latents_tensor.ndim == 6 and latents_tensor.shape[0] == 1: + latents_tensor = latents_tensor.squeeze(0) + + print(f"Decoding video {video_id} with shape {latents_tensor.shape}...") + + # create subfolder + vid_output_dir = os.path.join(args.output_dir, str(video_id)) + os.makedirs(vid_output_dir, exist_ok=True) + + # Pick steps to decode + steps = latents_tensor.shape[0] + if args.decode_steps == "last": + indices_to_decode = [steps - 1] + elif args.decode_steps == "all": + indices_to_decode = list(range(steps)) + else: + indices_to_decode = [int(x) for x in args.decode_steps.split(",") if x.strip() != ""] + indices_to_decode = [i for i in indices_to_decode if 0 <= i < steps] + if not indices_to_decode: + raise ValueError(f"No valid indices selected for decode_steps='{args.decode_steps}' with steps={steps}") + + for step in tqdm(indices_to_decode, desc=f"Decoding {video_id}", leave=False): + latent_step = latents_tensor[step].unsqueeze(0) # [1, C, T, H, W] + + decoded_video = _decode_with_vae(vae, + latent_step, + device=device, + precision=args.vae_precision) + + save_path = os.path.join(vid_output_dir, f"step_{step:03d}.mp4") + save_decoded_latents_as_video(decoded_video.float(), + save_path, + fps=args.fps) + + print(f"Saved {len(indices_to_decode)} step(s) to {vid_output_dir}") + num_visualized += 1 + pbar.update(1) + + pbar.close() + +if __name__ == "__main__": + main() From 5a767aadde79a7a2ac61a6a35abb412e40f9914a Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Tue, 10 Feb 2026 10:00:19 +0000 Subject: [PATCH 022/214] update --- fastvideo/configs/pipelines/wan.py | 3 + fastvideo/dataset/dataloader/schema.py | 7 --- .../dits/wangame/hyworld_action_module.py | 16 +++-- .../pipelines/stages/causal_denoising.py | 61 ++++++++++++++++--- .../training/wangame_ode_causal_pipeline.py | 1 + 5 files changed, 67 insertions(+), 21 deletions(-) diff --git a/fastvideo/configs/pipelines/wan.py b/fastvideo/configs/pipelines/wan.py index b4a76034f..32aad49ff 100644 --- a/fastvideo/configs/pipelines/wan.py +++ b/fastvideo/configs/pipelines/wan.py @@ -126,6 +126,9 @@ class WanGameI2V480PConfig(WanI2V480PConfig): """Configuration for WanGame image-to-video pipeline.""" dit_config: DiTConfig = field(default_factory=WanGameVideoConfig) + flow_shift: float | None = 3.0 + dmd_denoising_steps: list[int] | None = field( + default_factory=lambda: [1000, 666, 333]) @dataclass diff --git a/fastvideo/dataset/dataloader/schema.py b/fastvideo/dataset/dataloader/schema.py index 4ff323392..cdf9b42fb 100644 --- a/fastvideo/dataset/dataloader/schema.py +++ b/fastvideo/dataset/dataloader/schema.py @@ -202,13 +202,6 @@ pyarrow_schema_ode_trajectory_wangame = pa.schema([ pa.field("id", pa.string()), - # --- Image/Video VAE latents --- - # Tensors are stored as raw bytes with shape and dtype info for loading - pa.field("vae_latent_bytes", pa.binary()), - # e.g., [C, T, H, W] or [C, H, W] - pa.field("vae_latent_shape", pa.list_(pa.int64())), - # e.g., 'float32' - pa.field("vae_latent_dtype", pa.string()), #I2V pa.field("clip_feature_bytes", pa.binary()), pa.field("clip_feature_shape", pa.list_(pa.int64())), diff --git a/fastvideo/models/dits/wangame/hyworld_action_module.py b/fastvideo/models/dits/wangame/hyworld_action_module.py index 8bf26b67c..0e482c6c2 100644 --- a/fastvideo/models/dits/wangame/hyworld_action_module.py +++ b/fastvideo/models/dits/wangame/hyworld_action_module.py @@ -64,7 +64,7 @@ def forward( temb: [B*T, dim] combined timestep + action embedding timestep_proj: [B*T, 6*dim] modulation projection """ - # timestep: [B] -> temb: [B, dim] + # timestep may be [B] (one per sample) or [B*T] (one per frame, from causal training) temb = self.time_embedder(timestep, timestep_seq_len) # Handle action embedding for batch > 1 @@ -83,10 +83,16 @@ def forward( action_emb = action_emb.to(action_embedder_dtype) action_emb = self.action_embedder(action_emb).type_as(temb) # [B*T, dim] - # Expand temb to match action_emb: [B, dim] -> [B, T, dim] -> [B*T, dim] - # Each batch's temb is repeated for all its frames - temb_expanded = temb.unsqueeze(1).expand(-1, num_frames, -1) # [B, T, dim] - temb_expanded = temb_expanded.reshape(batch_size * num_frames, -1) # [B*T, dim] + # temb is [B*T, dim] when timestep was already per-frame (causal training), + # or [B, dim] when timestep is per-sample (inference). + # Only expand if temb is per-sample [B, dim]. + if temb.shape[0] == batch_size and num_frames > 1: + # Expand temb: [B, dim] -> [B, T, dim] -> [B*T, dim] + temb_expanded = temb.unsqueeze(1).expand(-1, num_frames, -1) # [B, T, dim] + temb_expanded = temb_expanded.reshape(batch_size * num_frames, -1) # [B*T, dim] + else: + # temb is already [B*T, dim] (per-frame timesteps) + temb_expanded = temb # Add action embedding to expanded temb temb = temb_expanded + action_emb # [B*T, dim] diff --git a/fastvideo/pipelines/stages/causal_denoising.py b/fastvideo/pipelines/stages/causal_denoising.py index 56999aeec..5d0343ba1 100644 --- a/fastvideo/pipelines/stages/causal_denoising.py +++ b/fastvideo/pipelines/stages/causal_denoising.py @@ -1,4 +1,6 @@ +import PIL.Image import torch # type: ignore +import torchvision.transforms.functional as TF from fastvideo.distributed import get_local_torch_device from fastvideo.fastvideo_args import FastVideoArgs @@ -198,14 +200,28 @@ def _get_kv_cache(timestep: float) -> list[dict]: # the image latent instead of appending along the channel dim assert self.vae is not None, "VAE is not provided for causal video gen task" self.vae = self.vae.to(get_local_torch_device()) - first_frame_latent = self.vae.encode(batch.pil_image).mean.float() - if (hasattr(self.vae, "shift_factor") - and self.vae.shift_factor is not None): - if isinstance(self.vae.shift_factor, torch.Tensor): - first_frame_latent -= self.vae.shift_factor.to( - first_frame_latent.device, first_frame_latent.dtype) + image_for_vae = batch.pil_image + if isinstance(image_for_vae, PIL.Image.Image): + # Fallback path when causal preprocessing did not convert PIL image to tensor. + image_for_vae = TF.to_tensor(image_for_vae).sub_(0.5).div_(0.5) + image_for_vae = image_for_vae.unsqueeze(0).unsqueeze(2) + elif isinstance(image_for_vae, torch.Tensor): + if image_for_vae.dim() == 4: + # [B, C, H, W] -> [B, C, 1, H, W] + image_for_vae = image_for_vae.unsqueeze(2) + elif image_for_vae.dim() == 5: + # Keep only first frame for first-frame latent initialization. + image_for_vae = image_for_vae[:, :, :1] else: - first_frame_latent -= self.vae.shift_factor + raise ValueError( + f"Unsupported image tensor shape for causal VAE encode: {tuple(image_for_vae.shape)}") + else: + raise TypeError( + f"Unsupported batch.pil_image type for causal VAE encode: {type(image_for_vae)}") + + image_for_vae = image_for_vae.to(get_local_torch_device(), + dtype=torch.float32) + first_frame_latent = self.vae.encode(image_for_vae).mean.float() if isinstance(self.vae.scaling_factor, torch.Tensor): first_frame_latent = first_frame_latent * self.vae.scaling_factor.to( @@ -226,6 +242,33 @@ def _get_kv_cache(timestep: float) -> list[dict]: set_forward_context(current_timestep=0, attn_metadata=None, forward_batch=batch): + first_frame_input = first_frame_latent.to(target_dtype) + if (batch.image_latent is not None + and not independent_first_frame): + # Keep channel layout consistent with the main denoising loop. + first_frame_image_latent = batch.image_latent[:, :, start_index:start_index + 1, :, :] + first_frame_input = torch.cat([ + first_frame_input, + first_frame_image_latent.to(target_dtype) + ], + dim=1) + elif (batch.image_latent is not None + and independent_first_frame and start_index == 0): + first_frame_input = torch.cat([ + first_frame_input, + batch.image_latent.to(target_dtype) + ], + dim=2) + + expected_in_channels = getattr(self.transformer, "in_channels", + None) + if (expected_in_channels is not None + and first_frame_input.shape[1] != expected_in_channels): + raise ValueError( + "Causal first-frame cache init channel mismatch: " + f"input channels={first_frame_input.shape[1]}, " + f"expected={expected_in_channels}.") + first_frame_action_kwargs = {} if action_full is not None: first_frame_action_kwargs = { @@ -234,7 +277,7 @@ def _get_kv_cache(timestep: float) -> list[dict]: "action": action_full[:, start_index:start_index + 1], } self.transformer( - first_frame_latent.to(target_dtype), + first_frame_input, prompt_embeds, t_zero, kv_cache=kv_cache1, @@ -248,7 +291,7 @@ def _get_kv_cache(timestep: float) -> list[dict]: ) if boundary_timestep is not None: self.transformer_2( - first_frame_latent.to(target_dtype), + first_frame_input, prompt_embeds, t_zero, kv_cache=kv_cache2, diff --git a/fastvideo/training/wangame_ode_causal_pipeline.py b/fastvideo/training/wangame_ode_causal_pipeline.py index e6dbdb1c6..ffd06ea00 100644 --- a/fastvideo/training/wangame_ode_causal_pipeline.py +++ b/fastvideo/training/wangame_ode_causal_pipeline.py @@ -66,6 +66,7 @@ def initialize_training_pipeline(self, training_args: TrainingArgs): self.noise_scheduler.set_timesteps(num_inference_steps=1000, training=True) + self.training_args.pipeline_config.dmd_denoising_steps = [1000, 666, 333] logger.info("dmd_denoising_steps: %s", self.training_args.pipeline_config.dmd_denoising_steps) self.dmd_denoising_steps = torch.tensor([1000, 666, 333], From 0683e09bf965c9e749433c94c678ad12de9e25a9 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Tue, 10 Feb 2026 10:25:23 +0000 Subject: [PATCH 023/214] precommit --- fastvideo/fastvideo_args.py | 40 ++++---- .../basic/wan/wangame_causal_dmd_pipeline.py | 78 ++++++++-------- .../basic/wan/wangame_i2v_pipeline.py | 21 +++-- .../pipelines/preprocess/v1_preprocess.py | 17 ++-- ...game_preprocess_pipeline_ode_trajectory.py | 23 +++-- .../pipelines/stages/causal_denoising.py | 49 ++++++---- fastvideo/pipelines/stages/denoising.py | 15 ++- fastvideo/training/training_pipeline.py | 7 +- .../wangame_lingbot_training_pipeline.py | 85 +++++++++-------- .../training/wangame_ode_causal_pipeline.py | 87 ++++++++++------- .../training/wangame_training_pipeline.py | 93 +++++++++++-------- 11 files changed, 289 insertions(+), 226 deletions(-) diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index 896c40e77..23cb79816 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -1003,18 +1003,22 @@ def from_cli_args(cls, args: argparse.Namespace) -> "TrainingArgs": @staticmethod def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: - parser.add_argument("--data-path", - type=str, - required=True, - help="Path to parquet files (comma-separated for multiple; path:N for repeat count)") + parser.add_argument( + "--data-path", + type=str, + required=True, + help= + "Path to parquet files (comma-separated for multiple; path:N for repeat count)" + ) parser.add_argument("--dataloader-num-workers", type=int, required=True, help="Number of workers for dataloader") - parser.add_argument("--reshuffle-each-epoch", - action=StoreBoolean, - default=TrainingArgs.reshuffle_each_epoch, - help="Whether to reshuffle dataset order each epoch") + parser.add_argument( + "--reshuffle-each-epoch", + action=StoreBoolean, + default=TrainingArgs.reshuffle_each_epoch, + help="Whether to reshuffle dataset order each epoch") parser.add_argument("--num-height", type=int, required=True, @@ -1104,9 +1108,10 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: parser.add_argument("--validation-steps", type=float, help="Number of validation steps") - parser.add_argument("--validation-num-samples", - type=int, - help="Limit number of validation samples (default: use all)") + parser.add_argument( + "--validation-num-samples", + type=int, + help="Limit number of validation samples (default: use all)") parser.add_argument("--log-validation", action=StoreBoolean, help="Whether to log validation results") @@ -1286,18 +1291,19 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: parser.add_argument("--lora-alpha", type=int, help="LoRA alpha") # Action-only training (freeze base model, only train action params) - parser.add_argument("--train-action-only", - action=StoreBoolean, - help="Whether to only train action-related parameters " - "(action_embedder and to_out_prope) while freezing base model") + parser.add_argument( + "--train-action-only", + action=StoreBoolean, + help="Whether to only train action-related parameters " + "(action_embedder and to_out_prope) while freezing base model") # Action warmup: keep action modules frozen for N steps parser.add_argument("--action-warmup-steps", type=int, default=0, help="Number of steps to keep action modules " - "(action_embedder, to_out_prope) frozen to let " - "the base model stabilize first") + "(action_embedder, to_out_prope) frozen to let " + "the base model stabilize first") # V-MoBA parameters parser.add_argument( diff --git a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py index c99554cf2..be5087fec 100644 --- a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py @@ -1,6 +1,6 @@ -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: Apache-2.0 """WanGame causal DMD pipeline implementation.""" - + from fastvideo.fastvideo_args import FastVideoArgs from fastvideo.logger import init_logger from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline @@ -11,29 +11,28 @@ InputValidationStage, LatentPreparationStage, TextEncodingStage) -from fastvideo.pipelines.stages.image_encoding import ( - ImageVAEEncodingStage) - -logger = init_logger(__name__) - - +from fastvideo.pipelines.stages.image_encoding import (ImageVAEEncodingStage) + +logger = init_logger(__name__) + + class WanGameCausalDMDPipeline(LoRAPipeline, ComposedPipelineBase): _required_config_modules = [ "vae", "transformer", "scheduler", "image_encoder", "image_processor" ] - - def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: - self.add_stage(stage_name="input_validation_stage", - stage=InputValidationStage()) - - if (self.get_module("text_encoder", None) is not None - and self.get_module("tokenizer", None) is not None): - self.add_stage(stage_name="prompt_encoding_stage", - stage=TextEncodingStage( - text_encoders=[self.get_module("text_encoder")], - tokenizers=[self.get_module("tokenizer")], - )) - + + def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: + self.add_stage(stage_name="input_validation_stage", + stage=InputValidationStage()) + + if (self.get_module("text_encoder", None) is not None + and self.get_module("tokenizer", None) is not None): + self.add_stage(stage_name="prompt_encoding_stage", + stage=TextEncodingStage( + text_encoders=[self.get_module("text_encoder")], + tokenizers=[self.get_module("tokenizer")], + )) + if (self.get_module("image_encoder", None) is not None and self.get_module("image_processor", None) is not None): self.add_stage( @@ -42,18 +41,17 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: image_encoder=self.get_module("image_encoder"), image_processor=self.get_module("image_processor"), )) - - self.add_stage(stage_name="conditioning_stage", - stage=ConditioningStage()) - - self.add_stage(stage_name="latent_preparation_stage", - stage=LatentPreparationStage( - scheduler=self.get_module("scheduler"), - transformer=self.get_module("transformer", None))) - - self.add_stage( - stage_name="image_latent_preparation_stage", - stage=ImageVAEEncodingStage(vae=self.get_module("vae"))) + + self.add_stage(stage_name="conditioning_stage", + stage=ConditioningStage()) + + self.add_stage(stage_name="latent_preparation_stage", + stage=LatentPreparationStage( + scheduler=self.get_module("scheduler"), + transformer=self.get_module("transformer", None))) + + self.add_stage(stage_name="image_latent_preparation_stage", + stage=ImageVAEEncodingStage(vae=self.get_module("vae"))) self.add_stage(stage_name="denoising_stage", stage=CausalDMDDenosingStage( @@ -61,11 +59,11 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: transformer_2=self.get_module("transformer_2", None), scheduler=self.get_module("scheduler"), vae=self.get_module("vae"))) - - self.add_stage(stage_name="decoding_stage", - stage=DecodingStage(vae=self.get_module("vae"))) - - logger.info( - "WanGameCausalDMDPipeline initialized with action support") - + + self.add_stage(stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"))) + + logger.info("WanGameCausalDMDPipeline initialized with action support") + + EntryClass = WanGameCausalDMDPipeline diff --git a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py index cd7e53d4d..300d41f19 100644 --- a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py @@ -12,10 +12,12 @@ from fastvideo.pipelines.lora_pipeline import LoRAPipeline # isort: off -from fastvideo.pipelines.stages import ( - ImageEncodingStage, ConditioningStage, DecodingStage, DenoisingStage, - ImageVAEEncodingStage, InputValidationStage, LatentPreparationStage, - TimestepPreparationStage) +from fastvideo.pipelines.stages import (ImageEncodingStage, ConditioningStage, + DecodingStage, DenoisingStage, + ImageVAEEncodingStage, + InputValidationStage, + LatentPreparationStage, + TimestepPreparationStage) # isort: on from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( FlowUniPCMultistepScheduler) @@ -40,12 +42,11 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) - self.add_stage( - stage_name="image_encoding_stage", - stage=ImageEncodingStage( - image_encoder=self.get_module("image_encoder"), - image_processor=self.get_module("image_processor"), - )) + self.add_stage(stage_name="image_encoding_stage", + stage=ImageEncodingStage( + image_encoder=self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + )) self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) diff --git a/fastvideo/pipelines/preprocess/v1_preprocess.py b/fastvideo/pipelines/preprocess/v1_preprocess.py index 2a65c6a18..e15e2239c 100644 --- a/fastvideo/pipelines/preprocess/v1_preprocess.py +++ b/fastvideo/pipelines/preprocess/v1_preprocess.py @@ -76,7 +76,8 @@ def main(args) -> None: else: raise ValueError( f"Invalid preprocess task: {args.preprocess_task}. " - f"Valid options: t2v, i2v, ode_trajectory, text_only, matrixgame, wangame, wangame_ode_trajectory") + f"Valid options: t2v, i2v, ode_trajectory, text_only, matrixgame, wangame, wangame_ode_trajectory" + ) logger.info("Preprocess task: %s using %s", args.preprocess_task, PreprocessPipeline.__name__) @@ -120,12 +121,14 @@ def main(args) -> None: parser.add_argument("--group_frame", action="store_true") # TODO parser.add_argument("--group_resolution", action="store_true") # TODO parser.add_argument("--flow_shift", type=float, default=None) - parser.add_argument( - "--preprocess_task", - type=str, - default="t2v", - choices=["t2v", "i2v", "text_only", "ode_trajectory", "matrixgame", "wangame", "wangame_ode_trajectory"], - help="Type of preprocessing task to run") + parser.add_argument("--preprocess_task", + type=str, + default="t2v", + choices=[ + "t2v", "i2v", "text_only", "ode_trajectory", + "matrixgame", "wangame", "wangame_ode_trajectory" + ], + help="Type of preprocessing task to run") parser.add_argument("--train_fps", type=int, default=30) parser.add_argument("--use_image_num", type=int, default=0) parser.add_argument("--text_max_length", type=int, default=256) diff --git a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py index fcae785d0..69f45453c 100644 --- a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py +++ b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py @@ -176,8 +176,8 @@ def get_extra_features(self, valid_data: dict[str, Any], latent_condition = (latent_condition - latents_mean) / latents_std elif (hasattr(vae, "shift_factor") and vae.shift_factor is not None): if isinstance(vae.shift_factor, torch.Tensor): - latent_condition -= vae.shift_factor.to( - latent_condition.device, latent_condition.dtype) + latent_condition -= vae.shift_factor.to(latent_condition.device, + latent_condition.dtype) else: latent_condition -= vae.shift_factor @@ -224,11 +224,13 @@ def get_extra_features(self, valid_data: dict[str, Any], for action_path in valid_data["action_path"]: if action_path: action_data = np.load(action_path, allow_pickle=True) - if isinstance(action_data, - np.ndarray) and action_data.dtype == np.dtype('O'): + if isinstance( + action_data, + np.ndarray) and action_data.dtype == np.dtype('O'): action_dict = action_data.item() if "keyboard" in action_dict: - keyboard = action_dict["keyboard"].astype(np.float32) + keyboard = action_dict["keyboard"].astype( + np.float32) if keyboard_dim is not None: if keyboard.ndim >= 2: keyboard = keyboard[:, :keyboard_dim] @@ -292,8 +294,8 @@ def preprocess_action_and_trajectory(self, fastvideo_args: FastVideoArgs, pixel_values = valid_data["pixel_values"] if pixel_values.shape[2] == 1 and args.num_frames is not None: - pixel_values = pixel_values.repeat( - 1, 1, args.num_frames, 1, 1) + pixel_values = pixel_values.repeat(1, 1, args.num_frames, 1, + 1) valid_data["pixel_values"] = pixel_values # Get extra features if needed @@ -344,9 +346,10 @@ def preprocess_action_and_trajectory(self, fastvideo_args: FastVideoArgs, batch.do_classifier_free_guidance = False batch.prompt = "" batch.prompt_embeds = [ - torch.zeros((1, 0, self.get_module("transformer").hidden_size), - dtype=torch.bfloat16, - device=device) + torch.zeros( + (1, 0, self.get_module("transformer").hidden_size), + dtype=torch.bfloat16, + device=device) ] result_batch = self.input_validation_stage( diff --git a/fastvideo/pipelines/stages/causal_denoising.py b/fastvideo/pipelines/stages/causal_denoising.py index 5d0343ba1..3dbfb38a2 100644 --- a/fastvideo/pipelines/stages/causal_denoising.py +++ b/fastvideo/pipelines/stages/causal_denoising.py @@ -140,17 +140,20 @@ def forward( intrinsics_list = [] action_list = [] for bi in range(b): - vm, ks, action = process_custom_actions( - batch.keyboard_cond[bi], batch.mouse_cond[bi]) + vm, ks, action = process_custom_actions(batch.keyboard_cond[bi], + batch.mouse_cond[bi]) viewmats_list.append(vm) intrinsics_list.append(ks) action_list.append(action) - viewmats_full = torch.stack(viewmats_list, dim=0).to( - device=latents.device, dtype=target_dtype) - intrinsics_full = torch.stack(intrinsics_list, dim=0).to( - device=latents.device, dtype=target_dtype) - action_full = torch.stack(action_list, dim=0).to( - device=latents.device, dtype=target_dtype) + viewmats_full = torch.stack(viewmats_list, + dim=0).to(device=latents.device, + dtype=target_dtype) + intrinsics_full = torch.stack(intrinsics_list, + dim=0).to(device=latents.device, + dtype=target_dtype) + action_full = torch.stack(action_list, + dim=0).to(device=latents.device, + dtype=target_dtype) # Initialize or reset caches kv_cache1 = self._initialize_kv_cache(batch_size=latents.shape[0], @@ -214,10 +217,12 @@ def _get_kv_cache(timestep: float) -> list[dict]: image_for_vae = image_for_vae[:, :, :1] else: raise ValueError( - f"Unsupported image tensor shape for causal VAE encode: {tuple(image_for_vae.shape)}") + f"Unsupported image tensor shape for causal VAE encode: {tuple(image_for_vae.shape)}" + ) else: raise TypeError( - f"Unsupported batch.pil_image type for causal VAE encode: {type(image_for_vae)}") + f"Unsupported batch.pil_image type for causal VAE encode: {type(image_for_vae)}" + ) image_for_vae = image_for_vae.to(get_local_torch_device(), dtype=torch.float32) @@ -246,14 +251,17 @@ def _get_kv_cache(timestep: float) -> list[dict]: if (batch.image_latent is not None and not independent_first_frame): # Keep channel layout consistent with the main denoising loop. - first_frame_image_latent = batch.image_latent[:, :, start_index:start_index + 1, :, :] + first_frame_image_latent = batch.image_latent[:, :, + start_index: + start_index + + 1, :, :] first_frame_input = torch.cat([ first_frame_input, first_frame_image_latent.to(target_dtype) ], dim=1) - elif (batch.image_latent is not None - and independent_first_frame and start_index == 0): + elif (batch.image_latent is not None and independent_first_frame + and start_index == 0): first_frame_input = torch.cat([ first_frame_input, batch.image_latent.to(target_dtype) @@ -272,7 +280,8 @@ def _get_kv_cache(timestep: float) -> list[dict]: first_frame_action_kwargs = {} if action_full is not None: first_frame_action_kwargs = { - "viewmats": viewmats_full[:, start_index:start_index + 1], + "viewmats": viewmats_full[:, + start_index:start_index + 1], "Ks": intrinsics_full[:, start_index:start_index + 1], "action": action_full[:, start_index:start_index + 1], } @@ -329,7 +338,8 @@ def _get_kv_cache(timestep: float) -> list[dict]: if (batch.image_latent is not None and not independent_first_frame): - image_latent_chunk = batch.image_latent[:, :, start_index: + image_latent_chunk = batch.image_latent[:, :, + start_index: start_index + current_num_frames, :, :] latent_model_input = torch.cat([ @@ -479,11 +489,10 @@ def _get_kv_cache(timestep: float) -> list[dict]: image_context_chunk = batch.image_latent[:, :, start_index: start_index + current_num_frames, :, :] - context_input = torch.cat([ - context_input, - image_context_chunk.to(target_dtype) - ], - dim=1) + context_input = torch.cat( + [context_input, + image_context_chunk.to(target_dtype)], + dim=1) context_action_kwargs = {} if action_full is not None: end_index = start_index + current_num_frames diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index 6bb86da48..0ab09f4eb 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -170,13 +170,20 @@ def forward( if batch.mouse_cond is not None and batch.keyboard_cond is not None: from fastvideo.models.dits.hyworld.pose import process_custom_actions - viewmats, intrinsics, action_labels = process_custom_actions(batch.keyboard_cond, batch.mouse_cond) + viewmats, intrinsics, action_labels = process_custom_actions( + batch.keyboard_cond, batch.mouse_cond) camera_action_kwargs = self.prepare_extra_func_kwargs( self.transformer.forward, { - "viewmats": viewmats.unsqueeze(0).to(get_local_torch_device(), dtype=target_dtype), - "Ks": intrinsics.unsqueeze(0).to(get_local_torch_device(), dtype=target_dtype), - "action": action_labels.unsqueeze(0).to(get_local_torch_device(), dtype=target_dtype), + "viewmats": + viewmats.unsqueeze(0).to(get_local_torch_device(), + dtype=target_dtype), + "Ks": + intrinsics.unsqueeze(0).to(get_local_torch_device(), + dtype=target_dtype), + "action": + action_labels.unsqueeze(0).to(get_local_torch_device(), + dtype=target_dtype), }, ) # from fastvideo.models.dits.wangame_lingbot.cam_utils import process_custom_actions as process_lingbot_actions diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index e51eab670..ae6e1e5b9 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -811,8 +811,9 @@ def _prepare_validation_batch(self, sampling_param: SamplingParam, return batch - def _post_process_validation_frames(self, frames: list[np.ndarray], - batch: ForwardBatch) -> list[np.ndarray]: + def _post_process_validation_frames( + self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: """Post-process validation frames before saving. Override this method in subclasses to add custom processing, @@ -906,7 +907,7 @@ def _log_validation(self, transformer, training_args, global_step) -> None: x = torchvision.utils.make_grid(x, nrow=6) x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) frames.append((x * 255).numpy().astype(np.uint8)) - + # Apply optional post-processing (e.g., overlay for action-conditioned models) frames = self._post_process_validation_frames(frames, batch) step_videos.append(frames) diff --git a/fastvideo/training/wangame_lingbot_training_pipeline.py b/fastvideo/training/wangame_lingbot_training_pipeline.py index eaeb1ea47..cb3eb8478 100644 --- a/fastvideo/training/wangame_lingbot_training_pipeline.py +++ b/fastvideo/training/wangame_lingbot_training_pipeline.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 import sys -from copy import deepcopy from typing import Any import numpy as np @@ -10,7 +9,6 @@ from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame_lingbot from fastvideo.distributed import get_local_torch_device from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.forward_context import set_forward_context from fastvideo.logger import init_logger from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( FlowUniPCMultistepScheduler) @@ -33,7 +31,7 @@ class WanLingBotTrainingPipeline(TrainingPipeline): def initialize_pipeline(self, fastvideo_args: FastVideoArgs): self.modules["scheduler"] = FlowUniPCMultistepScheduler( shift=fastvideo_args.pipeline_config.flow_shift) - + def create_training_stages(self, training_args: TrainingArgs): """ May be used in future refactors. @@ -49,44 +47,47 @@ def set_trainable(self) -> None: - patch_embedding_wancamctrl: embeds camera Plucker coordinates - blocks.*.cam_conditioner: injects camera conditioning into transformer blocks """ - train_action_only = getattr(self.fastvideo_args, "train_action_only", False) - + train_action_only = getattr(self.fastvideo_args, "train_action_only", + False) + if not train_action_only: # Default behavior: train all parameters super().set_trainable() return - + # Freeze all transformer parameters first transformer = self.get_module("transformer") transformer.train() transformer.requires_grad_(False) - + # Define which parameter name patterns to train action_param_patterns = [ "patch_embedding_wancamctrl", "cam_conditioner", ] - + # Enable gradients for action-related parameters only trainable_count = 0 frozen_count = 0 for name, param in transformer.named_parameters(): - should_train = any(pattern in name for pattern in action_param_patterns) + should_train = any(pattern in name + for pattern in action_param_patterns) if should_train: param.requires_grad_(True) trainable_count += 1 logger.info(f"Trainable: {name} ({param.numel()} params)") else: frozen_count += 1 - - logger.info(f"Action-only training: {trainable_count} trainable param groups, " - f"{frozen_count} frozen param groups") + + logger.info( + f"Action-only training: {trainable_count} trainable param groups, " + f"{frozen_count} frozen param groups") # ── Action module warmup ────────────────────────────────────────────── # For the first `action_warmup_steps`, action modules (action_embedder, # to_out_prope) have requires_grad=False so the base model stabilizes # first. After warmup the gradients are re-enabled. - + _ACTION_PARAM_PATTERNS = [ "patch_embedding_wancamctrl", "cam_conditioner", @@ -244,18 +245,18 @@ def _build_input_kwargs(self, image_embeds = image_embeds.to(get_local_torch_device(), dtype=torch.bfloat16) encoder_hidden_states_image = image_embeds - + from fastvideo.models.dits.wangame_lingbot.cam_utils import process_custom_actions - + # Process actions for each batch sample batch_size = training_batch.noisy_model_input.shape[0] num_latent_t = training_batch.noisy_model_input.shape[2] latent_height = training_batch.noisy_model_input.shape[3] latent_width = training_batch.noisy_model_input.shape[4] - + temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio num_frames = (num_latent_t - 1) * temporal_compression_ratio + 1 - + c2ws_plucker_emb_list = [] for b in range(batch_size): # Lingbot's process_custom_actions returns [1, 6*spatial_scale^2, lat_f, H_lat, W_lat] @@ -264,11 +265,12 @@ def _build_input_kwargs(self, keyboard_cond=training_batch.keyboard_cond[b], mouse_cond=training_batch.mouse_cond[b], latent_height=latent_height, - latent_width=latent_width - ) + latent_width=latent_width) c2ws_plucker_emb_list.append(c2ws_plucker_emb) - - c2ws_plucker_emb = torch.cat(c2ws_plucker_emb_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) + + c2ws_plucker_emb = torch.cat(c2ws_plucker_emb_list, + dim=0).to(get_local_torch_device(), + dtype=torch.bfloat16) # c2ws_plucker_emb: [B, C, lat_f, H_lat, W_lat] assert c2ws_plucker_emb.shape[2] == num_latent_t, ( @@ -288,7 +290,8 @@ def _build_input_kwargs(self, "encoder_hidden_states_image": encoder_hidden_states_image, # Action conditioning - "c2ws_plucker_emb": c2ws_plucker_emb, + "c2ws_plucker_emb": + c2ws_plucker_emb, "return_dict": False, } @@ -342,8 +345,9 @@ def _prepare_validation_batch(self, sampling_param: SamplingParam, return batch - def _post_process_validation_frames(self, frames: list[np.ndarray], - batch: ForwardBatch) -> list[np.ndarray]: + def _post_process_validation_frames( + self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: """Apply action overlay to validation frames for WanGame. Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. @@ -351,41 +355,44 @@ def _post_process_validation_frames(self, frames: list[np.ndarray], # Check if action data is available keyboard_cond = getattr(batch, 'keyboard_cond', None) mouse_cond = getattr(batch, 'mouse_cond', None) - + if keyboard_cond is None and mouse_cond is None: return frames - + # Import overlay functions - from fastvideo.models.dits.matrixgame.utils import ( - draw_keys_on_frame, draw_mouse_on_frame) - + from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, + draw_mouse_on_frame) + # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) if keyboard_cond is not None: - keyboard_cond = keyboard_cond.squeeze(0).cpu().float().numpy() # (T, 6) + keyboard_cond = keyboard_cond.squeeze( + 0).cpu().float().numpy() # (T, 6) if mouse_cond is not None: mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) - + # MatrixGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] key_names = ["W", "S", "A", "D", "left", "right"] - + processed_frames = [] for frame_idx, frame in enumerate(frames): frame = np.ascontiguousarray(frame.copy()) - + # Draw keyboard overlay if keyboard_cond is not None and frame_idx < len(keyboard_cond): - keys = {key_names[i]: bool(keyboard_cond[frame_idx, i]) - for i in range(min(len(key_names), keyboard_cond.shape[1]))} + keys = { + key_names[i]: bool(keyboard_cond[frame_idx, i]) + for i in range(min(len(key_names), keyboard_cond.shape[1])) + } draw_keys_on_frame(frame, keys, mode='universal') - + # Draw mouse overlay if mouse_cond is not None and frame_idx < len(mouse_cond): pitch = float(mouse_cond[frame_idx, 0]) yaw = float(mouse_cond[frame_idx, 1]) draw_mouse_on_frame(frame, pitch, yaw) - + processed_frames.append(frame) - + return processed_frames @@ -408,4 +415,4 @@ def main(args) -> None: parser = FastVideoArgs.add_cli_args(parser) args = parser.parse_args() args.dit_cpu_offload = False - main(args) \ No newline at end of file + main(args) diff --git a/fastvideo/training/wangame_ode_causal_pipeline.py b/fastvideo/training/wangame_ode_causal_pipeline.py index ffd06ea00..62038cc51 100644 --- a/fastvideo/training/wangame_ode_causal_pipeline.py +++ b/fastvideo/training/wangame_ode_causal_pipeline.py @@ -66,7 +66,9 @@ def initialize_training_pipeline(self, training_args: TrainingArgs): self.noise_scheduler.set_timesteps(num_inference_steps=1000, training=True) - self.training_args.pipeline_config.dmd_denoising_steps = [1000, 666, 333] + self.training_args.pipeline_config.dmd_denoising_steps = [ + 1000, 666, 333 + ] logger.info("dmd_denoising_steps: %s", self.training_args.pipeline_config.dmd_denoising_steps) self.dmd_denoising_steps = torch.tensor([1000, 666, 333], @@ -171,14 +173,18 @@ def _get_next_batch( # Move to device device = get_local_torch_device() - training_batch.image_embeds = clip_feature.to(device, dtype=torch.bfloat16) - training_batch.image_latents = first_frame_latent.to(device, dtype=torch.bfloat16) + training_batch.image_embeds = clip_feature.to(device, + dtype=torch.bfloat16) + training_batch.image_latents = first_frame_latent.to( + device, dtype=torch.bfloat16) if keyboard_cond is not None and keyboard_cond.numel() > 0: - training_batch.keyboard_cond = keyboard_cond.to(device, dtype=torch.bfloat16) + training_batch.keyboard_cond = keyboard_cond.to( + device, dtype=torch.bfloat16) else: training_batch.keyboard_cond = None if mouse_cond is not None and mouse_cond.numel() > 0: - training_batch.mouse_cond = mouse_cond.to(device, dtype=torch.bfloat16) + training_batch.mouse_cond = mouse_cond.to(device, + dtype=torch.bfloat16) else: training_batch.mouse_cond = None training_batch.infos = infos @@ -251,10 +257,8 @@ def _prepare_dit_inputs(self, def _step_predict_next_latent( self, traj_latents: torch.Tensor, traj_timesteps: torch.Tensor, - image_embeds: torch.Tensor, - image_latents: torch.Tensor, - keyboard_cond: torch.Tensor | None, - mouse_cond: torch.Tensor | None + image_embeds: torch.Tensor, image_latents: torch.Tensor, + keyboard_cond: torch.Tensor | None, mouse_cond: torch.Tensor | None ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, dict[str, torch.Tensor]]: latent_vis_dict: dict[str, torch.Tensor] = {} @@ -269,8 +273,7 @@ def _step_predict_next_latent( traj_ts = traj_timesteps[0].float().cpu() dmd_steps = self.dmd_denoising_steps.float().cpu() closest_idx = torch.argmin( - torch.abs(traj_ts.unsqueeze(0) - dmd_steps.unsqueeze(1)), - dim=1) + torch.abs(traj_ts.unsqueeze(0) - dmd_steps.unsqueeze(1)), dim=1) self._cached_closest_idx_per_dmd = closest_idx.to(torch.long).cpu() logger.info("self._cached_closest_idx_per_dmd: %s", self._cached_closest_idx_per_dmd) @@ -306,13 +309,12 @@ def _step_predict_next_latent( width).to(self.device)).squeeze(1) latent_model_input = noisy_input.permute(0, 2, 1, 3, 4) if image_latents is not None: - latent_model_input = torch.cat( - [ - latent_model_input, - image_latents.to(latent_model_input.device, - latent_model_input.dtype), - ], - dim=1) + latent_model_input = torch.cat([ + latent_model_input, + image_latents.to(latent_model_input.device, + latent_model_input.dtype), + ], + dim=1) timestep = self.dmd_denoising_steps[indexes] logger.info("selected timestep for rank %s: %s", self.global_rank, @@ -329,13 +331,20 @@ def _step_predict_next_latent( timestep = timestep.to(device, dtype=torch.bfloat16) logger.info("========== Transformer Input ==========") - logger.info("hidden_states (latent_model_input) shape: %s, dtype: %s", latent_model_input.shape, latent_model_input.dtype) + logger.info("hidden_states (latent_model_input) shape: %s, dtype: %s", + latent_model_input.shape, latent_model_input.dtype) logger.info("hidden_states min/max/mean: %.4f / %.4f / %.4f", - latent_model_input.min().item(), latent_model_input.max().item(), latent_model_input.mean().item()) - logger.info("encoder_hidden_states_image (image_embeds) shape: %s", image_embeds.shape if image_embeds is not None else None) - logger.info("timestep shape: %s, dtype: %s", timestep.shape, timestep.dtype) - logger.info("keyboard_cond: %s", keyboard_cond.shape if keyboard_cond is not None else None) - logger.info("mouse_cond: %s", mouse_cond.shape if mouse_cond is not None else None) + latent_model_input.min().item(), + latent_model_input.max().item(), + latent_model_input.mean().item()) + logger.info("encoder_hidden_states_image (image_embeds) shape: %s", + image_embeds.shape if image_embeds is not None else None) + logger.info("timestep shape: %s, dtype: %s", timestep.shape, + timestep.dtype) + logger.info("keyboard_cond: %s", + keyboard_cond.shape if keyboard_cond is not None else None) + logger.info("mouse_cond: %s", + mouse_cond.shape if mouse_cond is not None else None) if keyboard_cond is not None and mouse_cond is not None: viewmats_list = [] @@ -347,21 +356,24 @@ def _step_predict_next_latent( viewmats_list.append(viewmats) intrinsics_list.append(intrinsics) action_labels_list.append(action_labels) - viewmats = torch.stack(viewmats_list, dim=0).to(device=device, - dtype=torch.bfloat16) - intrinsics = torch.stack(intrinsics_list, dim=0).to( - device=device, dtype=torch.bfloat16) - action_labels = torch.stack(action_labels_list, dim=0).to( - device=device, dtype=torch.bfloat16) + viewmats = torch.stack(viewmats_list, + dim=0).to(device=device, + dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, + dim=0).to(device=device, + dtype=torch.bfloat16) + action_labels = torch.stack(action_labels_list, + dim=0).to(device=device, + dtype=torch.bfloat16) else: viewmats = None intrinsics = None action_labels = None - empty_text = torch.zeros((latent_model_input.shape[0], 0, - self.transformer.hidden_size), - device=device, - dtype=torch.bfloat16) + empty_text = torch.zeros( + (latent_model_input.shape[0], 0, self.transformer.hidden_size), + device=device, + dtype=torch.bfloat16) input_kwargs = { "hidden_states": latent_model_input, @@ -382,7 +394,9 @@ def _step_predict_next_latent( logger.info("========== Transformer Output ==========") logger.info("noise_pred shape: %s", noise_pred.shape) logger.info("noise_pred min/max/mean: %.4f / %.4f / %.4f", - noise_pred.min().item(), noise_pred.max().item(), noise_pred.mean().item()) + noise_pred.min().item(), + noise_pred.max().item(), + noise_pred.mean().item()) from fastvideo.models.utils import pred_noise_to_pred_video pred_video = pred_noise_to_pred_video( @@ -420,7 +434,8 @@ def train_one_step(self, training_batch): # type: ignore[override] # Forward to predict next latent by stepping scheduler with predicted noise noise_pred, target_latent, t, latent_vis_dict = self._step_predict_next_latent( - traj_latents, traj_timesteps, image_embeds, image_latents, keyboard_cond, mouse_cond) + traj_latents, traj_timesteps, image_embeds, image_latents, + keyboard_cond, mouse_cond) training_batch.latent_vis_dict.update(latent_vis_dict) diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index 8cb6f491f..1fbaed943 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 import sys -from copy import deepcopy from typing import Any import numpy as np @@ -10,7 +9,6 @@ from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame from fastvideo.distributed import get_local_torch_device from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.forward_context import set_forward_context from fastvideo.logger import init_logger from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( FlowUniPCMultistepScheduler) @@ -34,7 +32,7 @@ class WanGameTrainingPipeline(TrainingPipeline): def initialize_pipeline(self, fastvideo_args: FastVideoArgs): self.modules["scheduler"] = FlowUniPCMultistepScheduler( shift=fastvideo_args.pipeline_config.flow_shift) - + def create_training_stages(self, training_args: TrainingArgs): """ May be used in future refactors. @@ -53,38 +51,41 @@ def set_trainable(self) -> None: This freezes the base model (q/k/v projections, FFN, etc.) while allowing the action-conditioning path to be trained. """ - train_action_only = getattr(self.fastvideo_args, "train_action_only", False) - + train_action_only = getattr(self.fastvideo_args, "train_action_only", + False) + if not train_action_only: # Default behavior: train all parameters super().set_trainable() return - + # Freeze all transformer parameters first transformer = self.get_module("transformer") transformer.train() transformer.requires_grad_(False) - + # Define which parameter name patterns to train action_param_patterns = [ "condition_embedder.action_embedder", # Action embedding MLP "to_out_prope", # PRoPE output projections in each block ] - + # Enable gradients for action-related parameters only trainable_count = 0 frozen_count = 0 for name, param in transformer.named_parameters(): - should_train = any(pattern in name for pattern in action_param_patterns) + should_train = any(pattern in name + for pattern in action_param_patterns) if should_train: param.requires_grad_(True) trainable_count += 1 logger.info(f"Trainable: {name} ({param.numel()} params)") else: frozen_count += 1 - - logger.info(f"Action-only training: {trainable_count} trainable param groups, " - f"{frozen_count} frozen param groups") + + logger.info( + f"Action-only training: {trainable_count} trainable param groups, " + f"{frozen_count} frozen param groups") # ── Action module warmup ────────────────────────────────────────────── # For the first `action_warmup_steps`, action modules (action_embedder, @@ -117,8 +118,7 @@ def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: self._set_action_params_grad(False) local_trainable = count_trainable(self.transformer) total_trainable = count_trainable_total( - self.transformer, get_local_torch_device() - ) + self.transformer, get_local_torch_device()) logger.info( "Action warmup: freezing action modules for the first " "%d steps to stabilize base model", warmup_steps) @@ -261,19 +261,25 @@ def _build_input_kwargs(self, encoder_hidden_states_image = image_embeds from fastvideo.models.dits.hyworld.pose import process_custom_actions - + # Process actions for each batch sample batch_size = training_batch.noisy_model_input.shape[0] viewmats_list, intrinsics_list, action_labels_list = [], [], [] for b in range(batch_size): - v, i, a = process_custom_actions( - training_batch.keyboard_cond[b], training_batch.mouse_cond[b]) + v, i, a = process_custom_actions(training_batch.keyboard_cond[b], + training_batch.mouse_cond[b]) viewmats_list.append(v) intrinsics_list.append(i) action_labels_list.append(a) - viewmats = torch.stack(viewmats_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) - intrinsics = torch.stack(intrinsics_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) - action_labels = torch.stack(action_labels_list, dim=0).to(get_local_torch_device(), dtype=torch.bfloat16) + viewmats = torch.stack(viewmats_list, + dim=0).to(get_local_torch_device(), + dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, + dim=0).to(get_local_torch_device(), + dtype=torch.bfloat16) + action_labels = torch.stack(action_labels_list, + dim=0).to(get_local_torch_device(), + dtype=torch.bfloat16) # Validate processed action latent dim matches video latent dim num_latent_t = training_batch.noisy_model_input.shape[2] @@ -298,9 +304,12 @@ def _build_input_kwargs(self, "encoder_hidden_states_image": encoder_hidden_states_image, # Action conditioning - "viewmats": viewmats, - "Ks": intrinsics, - "action": action_labels, + "viewmats": + viewmats, + "Ks": + intrinsics, + "action": + action_labels, "return_dict": False, } @@ -354,8 +363,9 @@ def _prepare_validation_batch(self, sampling_param: SamplingParam, return batch - def _post_process_validation_frames(self, frames: list[np.ndarray], - batch: ForwardBatch) -> list[np.ndarray]: + def _post_process_validation_frames( + self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: """Apply action overlay to validation frames for WanGame. Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. @@ -363,41 +373,44 @@ def _post_process_validation_frames(self, frames: list[np.ndarray], # Check if action data is available keyboard_cond = getattr(batch, 'keyboard_cond', None) mouse_cond = getattr(batch, 'mouse_cond', None) - + if keyboard_cond is None and mouse_cond is None: return frames - + # Import overlay functions - from fastvideo.models.dits.matrixgame.utils import ( - draw_keys_on_frame, draw_mouse_on_frame) - + from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, + draw_mouse_on_frame) + # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) if keyboard_cond is not None: - keyboard_cond = keyboard_cond.squeeze(0).cpu().float().numpy() # (T, 6) + keyboard_cond = keyboard_cond.squeeze( + 0).cpu().float().numpy() # (T, 6) if mouse_cond is not None: mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) - + # MatrixGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] key_names = ["W", "S", "A", "D", "left", "right"] - + processed_frames = [] for frame_idx, frame in enumerate(frames): frame = np.ascontiguousarray(frame.copy()) - + # Draw keyboard overlay if keyboard_cond is not None and frame_idx < len(keyboard_cond): - keys = {key_names[i]: bool(keyboard_cond[frame_idx, i]) - for i in range(min(len(key_names), keyboard_cond.shape[1]))} + keys = { + key_names[i]: bool(keyboard_cond[frame_idx, i]) + for i in range(min(len(key_names), keyboard_cond.shape[1])) + } draw_keys_on_frame(frame, keys, mode='universal') - + # Draw mouse overlay if mouse_cond is not None and frame_idx < len(mouse_cond): pitch = float(mouse_cond[frame_idx, 0]) yaw = float(mouse_cond[frame_idx, 1]) draw_mouse_on_frame(frame, pitch, yaw) - + processed_frames.append(frame) - + return processed_frames @@ -420,4 +433,4 @@ def main(args) -> None: parser = FastVideoArgs.add_cli_args(parser) args = parser.parse_args() args.dit_cpu_offload = False - main(args) \ No newline at end of file + main(args) From fbddfba3cf162618bcac3bf1e56ac6f35513b950 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 11 Feb 2026 02:53:04 +0000 Subject: [PATCH 024/214] update --- .../finetune_ode_init.sh | 2 +- .../finetune_ode_init.slurm | 25 +- .../launch_preprocess_slurm.sh | 6 +- .../preprocess_worker.slurm | 9 +- .../causal_wangame_ode_init/validation.json | 610 +++++++++++++++++- .../training/wangame_ode_causal_pipeline.py | 35 +- 6 files changed, 625 insertions(+), 62 deletions(-) diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh index 55c333c49..c8df2b9da 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh @@ -3,7 +3,7 @@ export PYTHONPATH="/mnt/fast-disks/hao_lab/kaiqin/FastVideo_wangame:$PYTHONPATH" export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=offline +export WANDB_MODE=online export TOKENIZERS_PARALLELISM=false MODEL_PATH="Wan2.1-Fun-1.3B-InP-Diffusers" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm index 54097b411..c226012b1 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm @@ -1,8 +1,8 @@ #!/bin/bash -#SBATCH --job-name=wg-ode-64g +#SBATCH --job-name=wg-ode #SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 +#SBATCH --nodes=8 +#SBATCH --ntasks=8 #SBATCH --ntasks-per-node=1 #SBATCH --gres=gpu:8 #SBATCH --cpus-per-task=128 @@ -14,13 +14,12 @@ set -e -x # Environment Setup -source ~/conda/miniconda/bin/activate -conda activate your-conda-env -export PYTHONPATH="/path/to/FastVideo:$PYTHONPATH" +source ~/conda/miniconda/bin/activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv +export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" # Basic Info -export WANDB_API_KEY="your-wandb-api-key" -export WANDB_MODE="online" +export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" +export WANDB_MODE=online export NCCL_P2P_DISABLE=1 export MASTER_PORT=29500 export NODE_RANK=$SLURM_PROCID @@ -32,9 +31,10 @@ echo "MASTER_ADDR: $MASTER_ADDR" echo "NODE_RANK: $NODE_RANK" # Configs -MODEL_PATH="" -DATA_DIR="../vizdoom/preprocessed" +MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" +DATA_DIR="../traindata_0209_1500/ode_init/preprocessed" VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" +CKPT_SAFETENSOR="/mnt/weka/home/hao.zhang/mhuo/FastVideo/wangame_1.3b_with_warmup_lr_1e-5/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" # Training arguments training_args=( @@ -42,7 +42,7 @@ training_args=( --output_dir "checkpoints/wangame_ode_init" --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" --wandb_run_name "wangame_ode_init_64gpu" - --max_train_steps 2000 + --max_train_steps 5000 --train_batch_size 1 --train_sp_batch_size 1 --gradient_accumulation_steps 1 @@ -52,6 +52,7 @@ training_args=( --num_frames 81 --warp_denoising_step --enable_gradient_checkpointing_type "full" + --init_weights_from_safetensors $CKPT_SAFETENSOR ) # Parallel arguments @@ -79,7 +80,7 @@ dataset_args=( validation_args=( --log_validation --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 200 + --validation_steps 100 --validation_sampling_steps "50" --validation_guidance_scale "6.0" ) diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh index 7e236fe91..30d8f13f8 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh @@ -5,7 +5,7 @@ mkdir -p preprocess_output # Launch 8 jobs, one for each node (Total 64 GPUs) # Each node processes 8 consecutive files (64 total files / 8 nodes = 8 files per node) -for node_id in {0..3}; do +for node_id in {0..2}; do # Calculate the starting file number for this node start_file=$((node_id * 8)) @@ -14,7 +14,7 @@ for node_id in {0..3}; do sbatch --job-name=wg-pre-${node_id} \ --output=preprocess_output/wg-node-${node_id}.out \ --error=preprocess_output/wg-node-${node_id}.err \ - $(pwd)/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm $start_file $node_id + $(pwd)/FastVideo/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm $start_file $node_id done -echo "All 4 nodes (32 GPUs) launched successfully!" +echo "All 3 nodes (24 GPUs) launched successfully!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm index c9f176323..50e3ef126 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm @@ -9,16 +9,15 @@ #SBATCH --time=72:00:00 # conda init -source ~/conda/miniconda/bin/activate -conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv -export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin:$PYTHONPATH" +source ~/conda/miniconda/bin/activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv +export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" # Accept parameters from launch script START_FILE=${1:-1} # Starting file number for this node NODE_ID=${2:-0} # Node identifier (0-7) -MODEL_PATH="./Matrix-Game-2.0-Foundation-VizDoom1k-1000steps-Diffusers" -OUTPUT_BASE="traindata_0209_1500/mg_ode_init/preprocessed" +MODEL_PATH="./Wan2.1-Fun-1.3B-InP-Diffusers" +OUTPUT_BASE="traindata_0209_1500/ode_init/preprocessed" # Port range calculation base_port=$((29500 + NODE_ID * 100)) diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json index d7a6f0c3c..94be9d122 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json @@ -1,9 +1,9 @@ { "data": [ { - "caption": "00. Hold [W] + Static", - "image_path": "../../../../mc_wasd_action/validate/0.jpg", - "action_path": "action/000000_action.npy", + "caption": "00: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -11,9 +11,9 @@ "num_frames": 81 }, { - "caption": "01. Hold [S] + Static", - "image_path": "../../../../mc_wasd_action/validate/1.jpg", - "action_path": "action/000001_action.npy", + "caption": "00: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -21,9 +21,9 @@ "num_frames": 81 }, { - "caption": "02. Hold [A] + Static", - "image_path": "../../../../mc_wasd_action/validate/2.jpg", - "action_path": "action/000002_action.npy", + "caption": "00: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -31,9 +31,9 @@ "num_frames": 81 }, { - "caption": "10. No Key + Hold [left]", - "image_path": "../../../../mc_wasd_action/validate/3.jpg", - "action_path": "action/000010_action.npy", + "caption": "00: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -41,9 +41,9 @@ "num_frames": 81 }, { - "caption": "12. No Key + Hold [up_right]", - "image_path": "../../../../mc_wasd_action/validate/4.jpg", - "action_path": "action/000012_action.npy", + "caption": "01: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -51,9 +51,9 @@ "num_frames": 81 }, { - "caption": "21. Switch [W]->[D] + Static", - "image_path": "../../../../mc_wasd_action/validate/5.jpg", - "action_path": "action/000021_action.npy", + "caption": "01: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -61,9 +61,9 @@ "num_frames": 81 }, { - "caption": "29. No Key + Switch [up_right]->[up_left]", - "image_path": "../../../../mc_wasd_action/validate/6.jpg", - "action_path": "action/000029_action.npy", + "caption": "01: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -71,9 +71,569 @@ "num_frames": 81 }, { - "caption": "50. Hold [W] + Switch [up]->[down]", - "image_path": "../../../../mc_wasd_action/validate/7.jpg", - "action_path": "action/000050_action.npy", + "caption": "01: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "03: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "03: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "03: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "03: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "04: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "04: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "04: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "04: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "05: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "05: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "05: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "05: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "06: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "06: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "06: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "06: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "07: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "07: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "07: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "07: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "08: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000008.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "08: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000008.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "08: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000008.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "08: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000008.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "09: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000009.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "09: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000009.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "09: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000009.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "09: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000009.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000010.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000010.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000010.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000010.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "11: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000011.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "11: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000011.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "11: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000011.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "11: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000011.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000012.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000012.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000012.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000012.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "13: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000013.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "13: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000013.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "13: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000013.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "13: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000013.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "14: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000014.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "14: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000014.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "14: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000014.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "14: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000014.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "15: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000015.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "15: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000015.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "15: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000015.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "15: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000015.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -81,4 +641,4 @@ "num_frames": 81 } ] -} \ No newline at end of file +} diff --git a/fastvideo/training/wangame_ode_causal_pipeline.py b/fastvideo/training/wangame_ode_causal_pipeline.py index 62038cc51..2547f3445 100644 --- a/fastvideo/training/wangame_ode_causal_pipeline.py +++ b/fastvideo/training/wangame_ode_causal_pipeline.py @@ -19,6 +19,7 @@ SelfForcingFlowMatchScheduler) from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( WanGameCausalDMDPipeline) +from fastvideo.pipelines.stages.decoding import DecodingStage from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch from fastvideo.training.training_pipeline import TrainingPipeline from fastvideo.training.training_utils import ( @@ -60,18 +61,18 @@ def initialize_training_pipeline(self, training_args: TrainingArgs): self.vae.requires_grad_(False) self.timestep_shift = self.training_args.pipeline_config.flow_shift - assert self.timestep_shift == 3.0, f"flow_shift must be 3.0, but got {self.timestep_shift}" self.noise_scheduler = SelfForcingFlowMatchScheduler( shift=self.timestep_shift, sigma_min=0.0, extra_one_step=True) self.noise_scheduler.set_timesteps(num_inference_steps=1000, training=True) - self.training_args.pipeline_config.dmd_denoising_steps = [ - 1000, 666, 333 - ] + self.training_args.pipeline_config.dmd_denoising_steps = [1000, 666, 333, 0] + self.add_stage(stage_name="decoding_stage", + stage=DecodingStage(vae=self.get_module("vae"))) + logger.info("dmd_denoising_steps: %s", self.training_args.pipeline_config.dmd_denoising_steps) - self.dmd_denoising_steps = torch.tensor([1000, 666, 333], + self.dmd_denoising_steps = torch.tensor([1000, 666, 333, 0], dtype=torch.long, device=get_local_torch_device()) if training_args.warp_denoising_step: # Warp the denoising step according to the scheduler time shift @@ -189,8 +190,12 @@ def _get_next_batch( training_batch.mouse_cond = None training_batch.infos = infos - return training_batch, trajectory_latents.to( - device, dtype=torch.bfloat16), trajectory_timesteps.to(device) + return training_batch, trajectory_latents[:, :, :self.training_args. + num_latent_t].to( + device, + dtype=torch.bfloat16 + ), trajectory_timesteps.to( + device) def _get_timestep(self, min_timestep: int, @@ -270,15 +275,14 @@ def _step_predict_next_latent( # Lazily cache nearest trajectory index per DMD step based on the (fixed) S timesteps if self._cached_closest_idx_per_dmd is None: - traj_ts = traj_timesteps[0].float().cpu() - dmd_steps = self.dmd_denoising_steps.float().cpu() - closest_idx = torch.argmin( - torch.abs(traj_ts.unsqueeze(0) - dmd_steps.unsqueeze(1)), dim=1) - self._cached_closest_idx_per_dmd = closest_idx.to(torch.long).cpu() + self._cached_closest_idx_per_dmd = torch.tensor( + [0, 12, 24, 36, S - 1], dtype=torch.long).cpu() + # [0, 1, 2, 3], dtype=torch.long).cpu() logger.info("self._cached_closest_idx_per_dmd: %s", self._cached_closest_idx_per_dmd) - logger.info("corresponding timesteps: %s", - traj_ts[self._cached_closest_idx_per_dmd]) + logger.info( + "corresponding timesteps: %s", self.noise_scheduler.timesteps[ + self._cached_closest_idx_per_dmd]) # Select the K indexes from traj_latents using self._cached_closest_idx_per_dmd # traj_latents: [B, S, C, T, H, W], self._cached_closest_idx_per_dmd: [K] @@ -534,8 +538,7 @@ def visualize_intermediate_latents(self, training_batch: TrainingBatch, assert latent_key in latents_vis_dict and latents_vis_dict[ latent_key] is not None latent = latents_vis_dict[latent_key] - pixel_latent = self.validation_pipeline.decoding_stage.decode( - latent, training_args) + pixel_latent = self.decoding_stage.decode(latent, training_args) video = pixel_latent.cpu().float() video = video.permute(0, 2, 1, 3, 4) From 8f9458d06c41c6a7122587bd7f2d8c890149e271 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Wed, 11 Feb 2026 19:15:32 +0000 Subject: [PATCH 025/214] validation --- .../scripts/collect_samples_to_shao.py | 118 ++++ .../scripts/generate_validation_to_shao.py | 87 +++ .../validation_to_shao.json | 604 ++++++++++++++++++ 3 files changed, 809 insertions(+) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/collect_samples_to_shao.py create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_to_shao.py create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_to_shao.json diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/collect_samples_to_shao.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/collect_samples_to_shao.py new file mode 100644 index 000000000..1b2e886ef --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/collect_samples_to_shao.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +""" +For each data dir (from finetune_wangame.slurm), randomly pick 10 samples (mp4 + action.npy), +copy to to_shao// as 01.mp4, 01_action.npy, ..., 10.mp4, 10_action.npy, +and extract first frame as 01.jpg, ..., 10.jpg. +""" +import os +import random +import shutil + +import cv2 + +# Data dirs from finetune_wangame.slurm (paths with "preprocessed"; we use "video" or "videos" for mp4/npy) +DATA_DIRS = [ + "/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed", + "/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed", + "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed", + "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed", + "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed", + "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed", +] + +OUT_ROOT = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao" +NUM_SAMPLES = 10 + + +def get_video_dir(preprocessed_path: str) -> str | None: + """Replace 'preprocessed' with 'video' (or 'videos') to get the dir containing mp4/npy.""" + video_path = preprocessed_path.replace("preprocessed", "video") + if os.path.isdir(video_path): + return video_path + videos_path = preprocessed_path.replace("preprocessed", "videos") + if os.path.isdir(videos_path): + return videos_path + return None + + +# Override short name for specific data dirs (e.g. traindata_0204_2130 -> fully_random) +SHORT_NAME_OVERRIDES: dict[str, str] = { + "traindata_0204_2130": "fully_random", +} + + +def get_short_name(preprocessed_path: str) -> str: + """Short name = parent folder of the preprocessed dir, e.g. 1_wasd_only.""" + name = os.path.basename(os.path.normpath(os.path.dirname(preprocessed_path))) + return SHORT_NAME_OVERRIDES.get(name, name) + + +def find_samples(video_dir: str) -> list[str]: + """Return list of base names (no extension) that have both xxxxxx.mp4 and xxxxxx_action.npy.""" + samples = [] + for f in os.listdir(video_dir): + if f.endswith(".mp4"): + base = f[:-4] + action_path = os.path.join(video_dir, f"{base}_action.npy") + if os.path.isfile(action_path): + samples.append(base) + return samples + + +def extract_first_frame(mp4_path: str, jpg_path: str) -> None: + cap = cv2.VideoCapture(mp4_path) + ret, frame = cap.read() + cap.release() + if ret: + cv2.imwrite(jpg_path, frame) + + +def main() -> None: + random.seed(42) + os.makedirs(OUT_ROOT, exist_ok=True) + total_dir = os.path.join(OUT_ROOT, "total") + os.makedirs(total_dir, exist_ok=True) + total_idx = 0 + + for preprocessed_path in DATA_DIRS: + video_dir = get_video_dir(preprocessed_path) + if video_dir is None: + print(f"Skip (video dir not found): {preprocessed_path}") + continue + + short_name = get_short_name(preprocessed_path) + samples = find_samples(video_dir) + if len(samples) < NUM_SAMPLES: + print(f"Skip {short_name}: only {len(samples)} samples (need {NUM_SAMPLES})") + continue + + chosen = random.sample(samples, NUM_SAMPLES) + out_dir = os.path.join(OUT_ROOT, short_name) + os.makedirs(out_dir, exist_ok=True) + + for i, base in enumerate(chosen, start=1): + num_str = f"{i:02d}" + src_mp4 = os.path.join(video_dir, f"{base}.mp4") + src_npy = os.path.join(video_dir, f"{base}_action.npy") + dst_mp4 = os.path.join(out_dir, f"{num_str}.mp4") + dst_npy = os.path.join(out_dir, f"{num_str}_action.npy") + dst_jpg = os.path.join(out_dir, f"{num_str}.jpg") + + shutil.copy2(src_mp4, dst_mp4) + shutil.copy2(src_npy, dst_npy) + extract_first_frame(dst_mp4, dst_jpg) + + # Copy into total/ with global numbering + total_idx += 1 + t_str = f"{total_idx:02d}" + shutil.copy2(dst_mp4, os.path.join(total_dir, f"{t_str}.mp4")) + shutil.copy2(dst_npy, os.path.join(total_dir, f"{t_str}_action.npy")) + shutil.copy2(dst_jpg, os.path.join(total_dir, f"{t_str}.jpg")) + + print(f"Done: {short_name} -> {out_dir} ({NUM_SAMPLES} samples)") + + print(f"Done: total -> {total_dir} ({total_idx} samples)") + + +if __name__ == "__main__": + main() diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_to_shao.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_to_shao.py new file mode 100644 index 000000000..9074c5e25 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_to_shao.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +""" +Generate a validation JSON where all image_path and action_path entries come from +examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao. + +Scans each subfolder of to_shao for pairs (NN.jpg, NN_action.npy) and emits one +validation entry per pair. Uses the same fixed_fields as generate_validation.py. +""" +import json +import os + +# Paths relative to this script / repo +FINETUNE_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +) +TO_SHAO_DIR = os.path.join(FINETUNE_DIR, "to_shao") +OUTPUT_PATH = os.path.join(FINETUNE_DIR, "validation_to_shao.json") + +# Same fixed fields as generate_validation.py +FIXED_FIELDS = { + "video_path": None, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77, +} + + +def collect_samples_from_to_shao(): + """Yield (caption, image_path, action_path) for each sample under to_shao.""" + if not os.path.isdir(TO_SHAO_DIR): + raise FileNotFoundError(f"to_shao directory not found: {TO_SHAO_DIR}") + + for subdir_name in sorted(os.listdir(TO_SHAO_DIR)): + subdir = os.path.join(TO_SHAO_DIR, subdir_name) + if not os.path.isdir(subdir): + continue + # Find all NN.jpg (or NN.jpeg) and matching NN_action.npy + for f in sorted(os.listdir(subdir)): + if f.endswith(".jpg") or f.endswith(".jpeg"): + base = f[: -4] if f.endswith(".jpg") else f[:-5] + action_name = f"{base}_action.npy" + action_path = os.path.join(subdir, action_name) + if not os.path.isfile(action_path): + continue + image_path = os.path.join(subdir, f) + caption = f"to_shao/{subdir_name}/{base}" + yield caption, image_path, action_path + + +def main(): + data = [] + for caption, image_path, action_path in collect_samples_from_to_shao(): + data.append( + { + "caption": caption, + "image_path": image_path, + "action_path": action_path, + **FIXED_FIELDS, + } + ) + + output = {"data": data} + with open(OUTPUT_PATH, "w") as f: + json.dump(output, f, indent=4) + + print(f"Generated {len(data)} entries to {OUTPUT_PATH}") + + # Check all paths exist + missing = [] + with open(OUTPUT_PATH) as f: + loaded = json.load(f) + for i, item in enumerate(loaded["data"]): + for key in ("image_path", "action_path"): + path = item.get(key) + if path and not os.path.isfile(path): + missing.append((i, key, path)) + if missing: + print("Missing paths:") + for idx, key, path in missing: + print(f" [{idx}] {key}: {path}") + else: + print("All paths exist.") + + +if __name__ == "__main__": + main() diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_to_shao.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_to_shao.json new file mode 100644 index 000000000..9921efeba --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_to_shao.json @@ -0,0 +1,604 @@ +{ + "data": [ + { + "caption": "to_shao/1_wasd_only/01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/01.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/01_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/1_wasd_only/02", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/02.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/02_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/1_wasd_only/03", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/03.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/03_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/1_wasd_only/04", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/04.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/04_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/1_wasd_only/05", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/05.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/05_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/1_wasd_only/06", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/06.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/06_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/1_wasd_only/07", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/07.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/07_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/1_wasd_only/08", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/08.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/08_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/1_wasd_only/09", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/09.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/09_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/1_wasd_only/10", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/10.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/1_wasd_only/10_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/01.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/01_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/02", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/02.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/02_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/03", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/03.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/03_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/04", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/04.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/04_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/05", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/05.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/05_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/06", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/06.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/06_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/07", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/07.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/07_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/08", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/08.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/08_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/09", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/09.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/09_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera/10", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/10.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera/10_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/01.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/01_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/02", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/02.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/02_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/03", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/03.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/03_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/04", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/04.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/04_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/05", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/05.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/05_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/06", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/06.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/06_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/07", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/07.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/07_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/08", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/08.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/08_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/09", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/09.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/09_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/camera4hold_alpha1/10", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/10.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/camera4hold_alpha1/10_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/01.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/01_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/02", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/02.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/02_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/03", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/03.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/03_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/04", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/04.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/04_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/05", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/05.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/05_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/06", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/06.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/06_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/07", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/07.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/07_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/08", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/08.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/08_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/09", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/09.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/09_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/fully_random/10", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/10.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/fully_random/10_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/01.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/01_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/02", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/02.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/02_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/03", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/03.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/03_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/04", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/04.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/04_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/05", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/05.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/05_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/06", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/06.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/06_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/07", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/07.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/07_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/08", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/08.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/08_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/09", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/09.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/09_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasd4holdrandview_simple_1key1mouse1/10", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/10.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasd4holdrandview_simple_1key1mouse1/10_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/01", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/01.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/01_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/02", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/02.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/02_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/03", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/03.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/03_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/04", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/04.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/04_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/05", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/05.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/05_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/06", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/06.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/06_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/07", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/07.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/07_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/08", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/08.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/08_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/09", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/09.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/09_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "to_shao/wasdonly_alpha1/10", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/10.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao/wasdonly_alpha1/10_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + } + ] +} \ No newline at end of file From 2e94ceaf60fa1f550c8b93d7b9951ccdf9cb4357 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Wed, 11 Feb 2026 19:53:56 +0000 Subject: [PATCH 026/214] registry; Lingbot need to validate --- fastvideo/registry.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fastvideo/registry.py b/fastvideo/registry.py index c520be15f..6950e89cc 100644 --- a/fastvideo/registry.py +++ b/fastvideo/registry.py @@ -45,6 +45,8 @@ WanI2V720PConfig, WanT2V480PConfig, WanT2V720PConfig, + WanGameI2V480PConfig, + WanLingBotI2V480PConfig ) from fastvideo.configs.pipelines.sd35 import SD35Config from fastvideo.configs.sample.base import SamplingParam @@ -555,6 +557,14 @@ def _register_configs() -> None: "FastVideo/SFWan2.2-I2V-A14B-Preview-Diffusers", ], ) + register_configs( + sampling_param_cls=Wan2_1_Fun_1_3B_InP_SamplingParam, + pipeline_config_cls=WanGameI2V480PConfig, + hf_model_paths=[ + "weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers", + ], + ) + # TODO: Need to add Lingbot # SD3.5 register_configs( From 7fc6c894df18e751e9bab0da14400c25d59dfb37 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Wed, 11 Feb 2026 23:01:19 +0000 Subject: [PATCH 027/214] revert back seed and scheduler --- fastvideo/training/training_pipeline.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index f9f56c2b8..61cc038e1 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -15,6 +15,7 @@ import torch import torch.distributed as dist import torchvision +from diffusers import FlowMatchEulerDiscreteScheduler from einops import rearrange from torch.utils.data import DataLoader from torchdata.stateful_dataloader import StatefulDataLoader @@ -116,7 +117,7 @@ def initialize_training_pipeline(self, training_args: TrainingArgs): # Set random seeds for deterministic training assert self.seed is not None, "seed must be set" - set_random_seed(self.seed + self.global_rank) + set_random_seed(self.seed) self.transformer.train() if training_args.enable_gradient_checkpointing_type is not None: self.transformer = apply_activation_checkpointing( @@ -614,15 +615,15 @@ def train(self) -> None: ) # Set random seeds for deterministic training - self.noise_random_generator = torch.Generator( - device="cpu").manual_seed(self.seed + self.global_rank) + self.noise_random_generator = torch.Generator(device="cpu").manual_seed( + self.seed) self.noise_gen_cuda = torch.Generator( - device=current_platform.device_name).manual_seed(self.seed + - self.global_rank) + device=current_platform.device_name).manual_seed(self.seed) self.validation_random_generator = torch.Generator( - device="cpu").manual_seed(self.seed + self.global_rank) - logger.info("Initialized random seeds with seed: %s", - self.seed + self.global_rank) + device="cpu").manual_seed(self.seed) + logger.info("Initialized random seeds with seed: %s", self.seed) + + self.noise_scheduler = FlowMatchEulerDiscreteScheduler() if self.training_args.resume_from_checkpoint: self._resume_from_checkpoint() From 6d7683215effc81f99e759c53d78d923653eec26 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 11 Feb 2026 17:12:47 -0800 Subject: [PATCH 028/214] some fix --- fastvideo/models/dits/wangame/causal_model.py | 198 ++++++++++++++---- ...game_preprocess_pipeline_ode_trajectory.py | 36 ++-- .../training/wangame_ode_causal_pipeline.py | 68 +++++- 3 files changed, 242 insertions(+), 60 deletions(-) diff --git a/fastvideo/models/dits/wangame/causal_model.py b/fastvideo/models/dits/wangame/causal_model.py index b7c33049a..9791c67bf 100644 --- a/fastvideo/models/dits/wangame/causal_model.py +++ b/fastvideo/models/dits/wangame/causal_model.py @@ -36,6 +36,7 @@ WanGameActionTimeImageEmbedding, WanGameActionSelfAttention ) +from fastvideo.models.dits.hyworld.camera_rope import prope_qkv logger = init_logger(__name__) @@ -81,12 +82,7 @@ def forward(self, x, context, context_lens=None, crossattn_cache=None): return x -class CausalWanGameActionSelfAttention(nn.Module): - """ - Causal self-attention module combining: - - WanGameActionSelfAttention (dual RoPE + PRoPE paths) for training with flex_attention - - KV caching for autoregressive inference - """ +class CausalWanGameActionSelfAttention(WanGameActionSelfAttention): def __init__(self, dim: int, @@ -95,27 +91,17 @@ def __init__(self, sink_size: int = 0, qk_norm=True, eps=1e-6) -> None: - assert dim % num_heads == 0 - super().__init__() - self.dim = dim - self.num_heads = num_heads - self.head_dim = dim // num_heads - self.local_attn_size = local_attn_size - self.sink_size = sink_size - self.qk_norm = qk_norm - self.eps = eps - self.max_attention_size = 32760 if local_attn_size == -1 else local_attn_size * 1560 - - # Inner attention module with PRoPE support - self.attn = WanGameActionSelfAttention( - dim, - num_heads, + super().__init__( + dim=dim, + num_heads=num_heads, local_attn_size=local_attn_size, sink_size=sink_size, qk_norm=qk_norm, - eps=eps) + eps=eps, + ) + self.max_attention_size = 32760 if local_attn_size == -1 else local_attn_size * 1560 - # Separate local attention for KV cache inference + # Local attention for KV-cache inference self.local_attn = LocalAttention( num_heads=num_heads, head_size=self.head_dim, @@ -125,6 +111,60 @@ def __init__(self, supported_attention_backends=(AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.TORCH_SDPA)) + @staticmethod + def _masked_flex_attn( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + block_mask: BlockMask, + ) -> torch.Tensor: + padded_length = math.ceil(query.shape[1] / 128) * 128 - query.shape[1] + if padded_length > 0: + query = torch.cat( + [ + query, + torch.zeros( + [query.shape[0], padded_length, query.shape[2], query.shape[3]], + device=query.device, + dtype=value.dtype, + ), + ], + dim=1, + ) + key = torch.cat( + [ + key, + torch.zeros( + [key.shape[0], padded_length, key.shape[2], key.shape[3]], + device=key.device, + dtype=value.dtype, + ), + ], + dim=1, + ) + value = torch.cat( + [ + value, + torch.zeros( + [value.shape[0], padded_length, value.shape[2], value.shape[3]], + device=value.device, + dtype=value.dtype, + ), + ], + dim=1, + ) + + out = flex_attention( + query=query.transpose(2, 1), + key=key.transpose(2, 1), + value=value.transpose(2, 1), + block_mask=block_mask, + ).transpose(2, 1) + + if padded_length > 0: + out = out[:, :-padded_length] + return out + def forward(self, q: torch.Tensor, k: torch.Tensor, @@ -144,20 +184,61 @@ def forward(self, cache_start = current_start if kv_cache is None: - return self.attn( - q, k, v, freqs_cis, - kv_cache=None, - current_start=current_start, - cache_start=cache_start, + if block_mask is None: + raise ValueError( + "block_mask must be provided for causal training attention") + if viewmats is None or Ks is None: + raise ValueError( + "viewmats and Ks must be provided for WanGame causal attention") + + cos, sin = freqs_cis + query_rope = _apply_rotary_emb( + q, cos, sin, is_neox_style=False).type_as(v) + key_rope = _apply_rotary_emb( + k, cos, sin, is_neox_style=False).type_as(v) + rope_output = self._masked_flex_attn( + query_rope, key_rope, v, block_mask) + + # PRoPE path with the same causal mask. + query_prope, key_prope, value_prope, apply_fn_o = prope_qkv( + q.transpose(1, 2), + k.transpose(1, 2), + v.transpose(1, 2), viewmats=viewmats, Ks=Ks, - is_cache=is_cache + patches_x=40, + patches_y=22, ) + query_prope = query_prope.transpose(1, 2) + key_prope = key_prope.transpose(1, 2) + value_prope = value_prope.transpose(1, 2) + prope_output = self._masked_flex_attn( + query_prope, key_prope, value_prope, block_mask) + prope_output = apply_fn_o( + prope_output.transpose(1, 2)).transpose(1, 2) + + return rope_output, prope_output else: # Inference mode with KV cache + if viewmats is None or Ks is None: + raise ValueError( + "viewmats and Ks must be provided for WanGame causal attention") + cos, sin = freqs_cis roped_query = _apply_rotary_emb(q, cos, sin, is_neox_style=False).type_as(v) roped_key = _apply_rotary_emb(k, cos, sin, is_neox_style=False).type_as(v) + query_prope, key_prope, value_prope, apply_fn_o = prope_qkv( + q.transpose(1, 2), + k.transpose(1, 2), + v.transpose(1, 2), + viewmats=viewmats, + Ks=Ks, + patches_x=40, + patches_y=22, + ) + query_prope = query_prope.transpose(1, 2).type_as(v) + key_prope = key_prope.transpose(1, 2).type_as(v) + value_prope = value_prope.transpose(1, 2).type_as(v) frame_seqlen = q.shape[1] current_end = current_start + roped_query.shape[1] @@ -166,6 +247,22 @@ def forward(self, kv_cache_size = kv_cache["k"].shape[1] num_new_tokens = roped_query.shape[1] + # rope+prope + if kv_cache["k"].shape[-1] == self.head_dim: + kv_cache["k"] = torch.cat( + [kv_cache["k"], torch.zeros_like(kv_cache["k"])], dim=-1) + kv_cache["v"] = torch.cat( + [kv_cache["v"], torch.zeros_like(kv_cache["v"])], dim=-1) + elif kv_cache["k"].shape[-1] != self.head_dim * 2: + raise ValueError( + f"Unexpected kv_cache head dim: {kv_cache['k'].shape[-1]}, " + f"expected {self.head_dim} or {self.head_dim * 2}") + + cache_k_rope = kv_cache["k"][..., :self.head_dim] + cache_k_prope = kv_cache["k"][..., self.head_dim:] + cache_v_rope = kv_cache["v"][..., :self.head_dim] + cache_v_prope = kv_cache["v"][..., self.head_dim:] + if self.local_attn_size != -1 and (current_end > kv_cache["global_end_index"].item()) and ( num_new_tokens + kv_cache["local_end_index"].item() > kv_cache_size): # Calculate the number of new tokens added in this step @@ -173,36 +270,53 @@ def forward(self, # Clone the source slice to avoid overlapping memory error num_evicted_tokens = num_new_tokens + kv_cache["local_end_index"].item() - kv_cache_size num_rolled_tokens = kv_cache["local_end_index"].item() - num_evicted_tokens - sink_tokens - kv_cache["k"][:, sink_tokens:sink_tokens + num_rolled_tokens] = \ - kv_cache["k"][:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() - kv_cache["v"][:, sink_tokens:sink_tokens + num_rolled_tokens] = \ - kv_cache["v"][:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() + cache_k_rope[:, sink_tokens:sink_tokens + num_rolled_tokens] = \ + cache_k_rope[:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() + cache_v_rope[:, sink_tokens:sink_tokens + num_rolled_tokens] = \ + cache_v_rope[:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() + cache_k_prope[:, sink_tokens:sink_tokens + num_rolled_tokens] = \ + cache_k_prope[:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() + cache_v_prope[:, sink_tokens:sink_tokens + num_rolled_tokens] = \ + cache_v_prope[:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() # Insert the new keys/values at the end local_end_index = kv_cache["local_end_index"].item() + current_end - \ kv_cache["global_end_index"].item() - num_evicted_tokens local_start_index = local_end_index - num_new_tokens - kv_cache["k"][:, local_start_index:local_end_index] = roped_key - kv_cache["v"][:, local_start_index:local_end_index] = v + cache_k_rope[:, local_start_index:local_end_index] = roped_key + cache_v_rope[:, local_start_index:local_end_index] = v + cache_k_prope[:, local_start_index:local_end_index] = key_prope + cache_v_prope[:, local_start_index:local_end_index] = value_prope else: # Assign new keys/values directly up to current_end local_end_index = kv_cache["local_end_index"].item() + current_end - kv_cache["global_end_index"].item() local_start_index = local_end_index - num_new_tokens kv_cache["k"] = kv_cache["k"].detach() kv_cache["v"] = kv_cache["v"].detach() + cache_k_rope = kv_cache["k"][..., :self.head_dim] + cache_k_prope = kv_cache["k"][..., self.head_dim:] + cache_v_rope = kv_cache["v"][..., :self.head_dim] + cache_v_prope = kv_cache["v"][..., self.head_dim:] # logger.info("kv_cache['k'] is in comp graph: %s", kv_cache["k"].requires_grad or kv_cache["k"].grad_fn is not None) - kv_cache["k"][:, local_start_index:local_end_index] = roped_key - kv_cache["v"][:, local_start_index:local_end_index] = v + cache_k_rope[:, local_start_index:local_end_index] = roped_key + cache_v_rope[:, local_start_index:local_end_index] = v + cache_k_prope[:, local_start_index:local_end_index] = key_prope + cache_v_prope[:, local_start_index:local_end_index] = value_prope - x = self.local_attn( + rope_x = self.local_attn( roped_query, - kv_cache["k"][:, max(0, local_end_index - self.max_attention_size):local_end_index], - kv_cache["v"][:, max(0, local_end_index - self.max_attention_size):local_end_index] + cache_k_rope[:, max(0, local_end_index - self.max_attention_size):local_end_index], + cache_v_rope[:, max(0, local_end_index - self.max_attention_size):local_end_index] + ) + prope_x = self.local_attn( + query_prope, + cache_k_prope[:, max(0, local_end_index - self.max_attention_size):local_end_index], + cache_v_prope[:, max(0, local_end_index - self.max_attention_size):local_end_index] ) + prope_x = apply_fn_o(prope_x.transpose(1, 2)).transpose(1, 2) kv_cache["global_end_index"].fill_(current_end) kv_cache["local_end_index"].fill_(local_end_index) - # In inference mode, only return rope output; prope is zero - return x, torch.zeros_like(x) + return rope_x, prope_x class CausalWanGameActionTransformerBlock(nn.Module): diff --git a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py index 69f45453c..def0dab3f 100644 --- a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py +++ b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py @@ -216,12 +216,12 @@ def get_extra_features(self, valid_data: dict[str, Any], features["first_frame_latent"] = cond_concat if "action_path" in valid_data and valid_data["action_path"]: - keyboard_cond_list = [] - mouse_cond_list = [] + keyboard_cond_list = [None] * len(valid_data["action_path"]) + mouse_cond_list = [None] * len(valid_data["action_path"]) arch_cfg = self.get_module("transformer").config.arch_config action_cfg = getattr(arch_cfg, "action_config", {}) or {} keyboard_dim = action_cfg.get("keyboard_dim_in", None) - for action_path in valid_data["action_path"]: + for idx, action_path in enumerate(valid_data["action_path"]): if action_path: action_data = np.load(action_path, allow_pickle=True) if isinstance( @@ -236,9 +236,10 @@ def get_extra_features(self, valid_data: dict[str, Any], keyboard = keyboard[:, :keyboard_dim] else: keyboard = keyboard[:keyboard_dim] - keyboard_cond_list.append(keyboard) + keyboard_cond_list[idx] = keyboard if "mouse" in action_dict: - mouse_cond_list.append(action_dict["mouse"]) + mouse_cond_list[idx] = action_dict["mouse"].astype( + np.float32) else: keyboard = action_data.astype(np.float32) if keyboard_dim is not None: @@ -246,11 +247,9 @@ def get_extra_features(self, valid_data: dict[str, Any], keyboard = keyboard[:, :keyboard_dim] else: keyboard = keyboard[:keyboard_dim] - keyboard_cond_list.append(keyboard) - if keyboard_cond_list: - features["keyboard_cond"] = keyboard_cond_list - if mouse_cond_list: - features["mouse_cond"] = mouse_cond_list + keyboard_cond_list[idx] = keyboard + features["keyboard_cond"] = keyboard_cond_list + features["mouse_cond"] = mouse_cond_list return features def preprocess_action_and_trajectory(self, fastvideo_args: FastVideoArgs, @@ -327,12 +326,17 @@ def preprocess_action_and_trajectory(self, fastvideo_args: FastVideoArgs, batch = ForwardBatch(**shallow_asdict(sampling_params), ) batch.image_embeds = [clip_features[i].unsqueeze(0)] batch.image_latent = image_latents[i].unsqueeze(0) - batch.keyboard_cond = (torch.from_numpy( - keyboard_cond[i]).unsqueeze(0).to(device) if - keyboard_cond is not None else None) - batch.mouse_cond = (torch.from_numpy( - mouse_cond[i]).unsqueeze(0).to(device) - if mouse_cond is not None else None) + sample_keyboard = keyboard_cond[ + i] if keyboard_cond is not None else None + sample_mouse = mouse_cond[i] if mouse_cond is not None else None + if sample_keyboard is not None and sample_mouse is not None: + batch.keyboard_cond = torch.from_numpy( + sample_keyboard).unsqueeze(0).to(device) + batch.mouse_cond = torch.from_numpy( + sample_mouse).unsqueeze(0).to(device) + else: + batch.keyboard_cond = None + batch.mouse_cond = None batch.num_inference_steps = 48 batch.return_trajectory_latents = True # Enabling this will save the decoded trajectory videos. diff --git a/fastvideo/training/wangame_ode_causal_pipeline.py b/fastvideo/training/wangame_ode_causal_pipeline.py index 2547f3445..512f5c6b7 100644 --- a/fastvideo/training/wangame_ode_causal_pipeline.py +++ b/fastvideo/training/wangame_ode_causal_pipeline.py @@ -66,13 +66,13 @@ def initialize_training_pipeline(self, training_args: TrainingArgs): self.noise_scheduler.set_timesteps(num_inference_steps=1000, training=True) - self.training_args.pipeline_config.dmd_denoising_steps = [1000, 666, 333, 0] + self.training_args.pipeline_config.dmd_denoising_steps = [1000, 750, 500, 250, 0] self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae"))) logger.info("dmd_denoising_steps: %s", self.training_args.pipeline_config.dmd_denoising_steps) - self.dmd_denoising_steps = torch.tensor([1000, 666, 333, 0], + self.dmd_denoising_steps = torch.tensor([1000, 750, 500, 250, 0], dtype=torch.long, device=get_local_torch_device()) if training_args.warp_denoising_step: # Warp the denoising step according to the scheduler time shift @@ -190,6 +190,19 @@ def _get_next_batch( training_batch.mouse_cond = None training_batch.infos = infos + # Validate action temporal dimensions match expected video frame count. + expected_num_frames = (self.training_args.num_latent_t - 1) * 4 + 1 + if training_batch.keyboard_cond is not None: + assert training_batch.keyboard_cond.shape[1] == expected_num_frames, ( + f"keyboard_cond temporal dim {training_batch.keyboard_cond.shape[1]} " + f"!= expected {expected_num_frames} " + f"(num_latent_t={self.training_args.num_latent_t})") + if training_batch.mouse_cond is not None: + assert training_batch.mouse_cond.shape[1] == expected_num_frames, ( + f"mouse_cond temporal dim {training_batch.mouse_cond.shape[1]} " + f"!= expected {expected_num_frames} " + f"(num_latent_t={self.training_args.num_latent_t})") + return training_batch, trajectory_latents[:, :, :self.training_args. num_latent_t].to( device, @@ -529,6 +542,57 @@ def _prepare_validation_batch(self, sampling_param: SamplingParam, return batch + def _post_process_validation_frames( + self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: + """Apply action overlay to validation frames for WanGame. + + Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. + """ + # Check if action data is available + keyboard_cond = getattr(batch, 'keyboard_cond', None) + mouse_cond = getattr(batch, 'mouse_cond', None) + + if keyboard_cond is None and mouse_cond is None: + return frames + + # Import overlay functions + from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, + draw_mouse_on_frame) + + # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) + if keyboard_cond is not None: + keyboard_cond = keyboard_cond.squeeze( + 0).cpu().float().numpy() # (T, 6) + if mouse_cond is not None: + mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) + + # MatrixGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] + key_names = ["W", "S", "A", "D", "left", "right"] + + processed_frames = [] + for frame_idx, frame in enumerate(frames): + frame = np.ascontiguousarray(frame.copy()) + + # Draw keyboard overlay + if keyboard_cond is not None and frame_idx < len(keyboard_cond): + keys = { + key_names[i]: bool(keyboard_cond[frame_idx, i]) + for i in range(min(len(key_names), keyboard_cond.shape[1])) + } + draw_keys_on_frame(frame, keys, mode='universal') + + # Draw mouse overlay + if mouse_cond is not None and frame_idx < len(mouse_cond): + pitch = float(mouse_cond[frame_idx, 0]) + yaw = float(mouse_cond[frame_idx, 1]) + draw_mouse_on_frame(frame, pitch, yaw) + + processed_frames.append(frame) + + return processed_frames + + def visualize_intermediate_latents(self, training_batch: TrainingBatch, training_args: TrainingArgs, step: int): tracker_loss_dict: dict[str, Any] = {} From 3a84f74f1e6c7c38eae7fed31326005f899c56c1 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 11 Feb 2026 18:02:05 -0800 Subject: [PATCH 029/214] fix causal denoising --- fastvideo/pipelines/stages/causal_denoising.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/fastvideo/pipelines/stages/causal_denoising.py b/fastvideo/pipelines/stages/causal_denoising.py index 3dbfb38a2..2ee07ab1a 100644 --- a/fastvideo/pipelines/stages/causal_denoising.py +++ b/fastvideo/pipelines/stages/causal_denoising.py @@ -314,7 +314,13 @@ def _get_kv_cache(timestep: float) -> list[dict]: ) start_index += 1 - block_sizes.pop(0) + if len(block_sizes) == 0: + raise ValueError( + "block_sizes is empty after first-frame initialization") + if block_sizes[0] > 1: + block_sizes[0] -= 1 + else: + block_sizes.pop(0) latents[:, :, :1, :, :] = first_frame_latent # DMD loop in causal blocks @@ -364,8 +370,9 @@ def _get_kv_cache(timestep: float) -> list[dict]: "action": action_full[:, start_index:end_index], } - # Prepare inputs - t_expand = t_cur.repeat(latent_model_input.shape[0]) + # Prepare inputs [B*T, C, H, W] + t_expand = t_cur.repeat(latent_model_input.shape[0] * + current_num_frames) # Attention metadata if needed if (vsa_available and self.attn_backend From 89a48e439f229234b9bd8dae690a729aff7347e8 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Thu, 12 Feb 2026 20:19:14 +0000 Subject: [PATCH 030/214] validation 81 frame --- .../WanGame2.1_1.3b_i2v/actions_81/A.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/A_d.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/A_dl.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/A_dr.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/A_l.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/A_r.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/A_u.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/A_ul.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/A_ur.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/D.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/D_d.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/D_dl.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/D_dr.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/D_l.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/D_r.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/D_u.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/D_ul.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/D_ur.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/README.md | 147 +++++ .../WanGame2.1_1.3b_i2v/actions_81/S.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SA.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SA_d.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SA_dl.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SA_dr.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SA_l.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SA_r.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SA_u.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SA_ul.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SA_ur.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SD.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SD_d.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SD_dl.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SD_dr.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SD_l.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SD_r.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SD_u.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SD_ul.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/SD_ur.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/S_d.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/S_dl.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/S_dr.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/S_l.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/S_r.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/S_u.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/S_ul.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/S_ur.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/W.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WA.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WA_d.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WA_dl.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WA_dr.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WA_l.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WA_r.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WA_u.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WA_ul.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WA_ur.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WD.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WD_d.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WD_dl.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WD_dr.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WD_l.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WD_r.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WD_u.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WD_ul.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/WD_ur.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/W_d.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/W_dl.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/W_dr.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/W_l.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/W_r.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/W_u.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/W_ul.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/W_ur.npy | Bin 0 -> 2994 bytes .../actions_81/camera_1_action_rand_1.npy | Bin 0 -> 2994 bytes .../actions_81/camera_1_action_rand_1_f4.npy | Bin 0 -> 2994 bytes .../actions_81/camera_1_action_rand_2.npy | Bin 0 -> 2994 bytes .../actions_81/camera_1_action_rand_2_f4.npy | Bin 0 -> 2994 bytes .../actions_81/camera_1_action_rand_3.npy | Bin 0 -> 2994 bytes .../actions_81/camera_1_action_rand_3_f4.npy | Bin 0 -> 2994 bytes .../actions_81/camera_1_action_rand_4.npy | Bin 0 -> 2994 bytes .../actions_81/camera_1_action_rand_4_f4.npy | Bin 0 -> 2994 bytes .../actions_81/camera_2_action_rand_1.npy | Bin 0 -> 2994 bytes .../actions_81/camera_2_action_rand_1_f4.npy | Bin 0 -> 2994 bytes .../actions_81/camera_2_action_rand_2.npy | Bin 0 -> 2994 bytes .../actions_81/camera_2_action_rand_2_f4.npy | Bin 0 -> 2994 bytes .../actions_81/camera_2_action_rand_3.npy | Bin 0 -> 2994 bytes .../actions_81/camera_2_action_rand_3_f4.npy | Bin 0 -> 2994 bytes .../actions_81/camera_2_action_rand_4.npy | Bin 0 -> 2994 bytes .../actions_81/camera_2_action_rand_4_f4.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/d.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/dl.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/dr.npy | Bin 0 -> 2994 bytes .../actions_81/key_1_action_rand_1.npy | Bin 0 -> 2994 bytes .../actions_81/key_1_action_rand_1_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_1_action_rand_2.npy | Bin 0 -> 2994 bytes .../actions_81/key_1_action_rand_2_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_1_action_rand_3.npy | Bin 0 -> 2994 bytes .../actions_81/key_1_action_rand_3_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_1_action_rand_4.npy | Bin 0 -> 2994 bytes .../actions_81/key_1_action_rand_4_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_2_action_rand_1.npy | Bin 0 -> 2994 bytes .../actions_81/key_2_action_rand_1_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_2_action_rand_2.npy | Bin 0 -> 2994 bytes .../actions_81/key_2_action_rand_2_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_2_action_rand_3.npy | Bin 0 -> 2994 bytes .../actions_81/key_2_action_rand_3_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_2_action_rand_4.npy | Bin 0 -> 2994 bytes .../actions_81/key_2_action_rand_4_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_camera_1_action_rand_1.npy | Bin 0 -> 2994 bytes .../key_camera_1_action_rand_1_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_camera_1_action_rand_2.npy | Bin 0 -> 2994 bytes .../key_camera_1_action_rand_2_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_camera_1_action_rand_3.npy | Bin 0 -> 2994 bytes .../key_camera_1_action_rand_3_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_camera_1_action_rand_4.npy | Bin 0 -> 2994 bytes .../key_camera_1_action_rand_4_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_camera_2_action_rand_1.npy | Bin 0 -> 2994 bytes .../key_camera_2_action_rand_1_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_camera_2_action_rand_2.npy | Bin 0 -> 2994 bytes .../key_camera_2_action_rand_2_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_camera_2_action_rand_3.npy | Bin 0 -> 2994 bytes .../key_camera_2_action_rand_3_f4.npy | Bin 0 -> 2994 bytes .../actions_81/key_camera_2_action_rand_4.npy | Bin 0 -> 2994 bytes .../key_camera_2_action_rand_4_f4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_1_action_rand_1.npy | Bin 0 -> 2994 bytes .../key_camera_excl_1_action_rand_1_f4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_1_action_rand_2.npy | Bin 0 -> 2994 bytes .../key_camera_excl_1_action_rand_2_f4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_1_action_rand_3.npy | Bin 0 -> 2994 bytes .../key_camera_excl_1_action_rand_3_f4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_1_action_rand_4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_1_action_rand_4_f4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_2_action_rand_1.npy | Bin 0 -> 2994 bytes .../key_camera_excl_2_action_rand_1_f4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_2_action_rand_2.npy | Bin 0 -> 2994 bytes .../key_camera_excl_2_action_rand_2_f4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_2_action_rand_3.npy | Bin 0 -> 2994 bytes .../key_camera_excl_2_action_rand_3_f4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_2_action_rand_4.npy | Bin 0 -> 2994 bytes .../key_camera_excl_2_action_rand_4_f4.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/l.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/r.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/still.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/u.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/ul.npy | Bin 0 -> 2994 bytes .../WanGame2.1_1.3b_i2v/actions_81/ur.npy | Bin 0 -> 2994 bytes .../scripts/generate_actions.py | 6 +- .../scripts/generate_validation.py | 157 +++-- .../validation_to_shao.json | 604 ------------------ .../WanGame2.1_1.3b_i2v/validation_zelda.json | 324 ++++++++++ 150 files changed, 582 insertions(+), 656 deletions(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/README.md create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_ur.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/d.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/dl.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/dr.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_1.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_1_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_2.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_2_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_3.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_3_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_4_f4.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/l.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/r.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/still.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/u.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/ul.npy create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/ur.npy delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_to_shao.json create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_zelda.json diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A.npy new file mode 100644 index 0000000000000000000000000000000000000000..3ee9a9c38c0fc4b02f535415193cac4201c448a3 GIT binary patch literal 2994 zcmeH|ze~eF6vr>AF=Cw^oU&aCC6j}QP^jQwFjc|9LBb`ua$=K|OCkhY=%O}oTb|&5 ztN9@`h?|1Tz2R~1-Qx@Q<-@(wxw*K!RnQ}P!U0Wuj*ktznBbv-r!h}CiQJe6v~Hh~ zSwgFR!pNLfwK*6XOXGL3yV`ogebj{5kFs!{Tdp5-YK7S>y(gTL9K@c>sUJs4$}>NO z5`@-Fg3zO&9Vx*^&v2@E@Cw&p+RA|SgchXvoYwQ|s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^;eSvg9NhQmOn M15lDJO#~8p0RLaY00000 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_dl.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_dl.npy new file mode 100644 index 0000000000000000000000000000000000000000..9adb526a17cc55cf37cc428cfd9179f000fe64dc GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^;$#iL;`&}jga KWJ?o)gdPB}#=p@3 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..f0a7f52f548aface7740628d2bc93da88e771ae7 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^;eV%t%AG#myp N9e|Q-X(Eu&0{~Dumr(!! literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_l.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_l.npy new file mode 100644 index 0000000000000000000000000000000000000000..dd66e6cd137081f66cb28ab8bbb9a17e2af437d0 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CWvKc&z#u{VvW+H;V@9? M0F-1)6M=*t00=6=00000 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_r.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_r.npy new file mode 100644 index 0000000000000000000000000000000000000000..d9098ddb90d83c58b86f1be6949697299dc1551e GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CWvKc&z!LZu}109a2Tj` M07|l@i9kXR0H=?cfB*mh literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_u.npy new file mode 100644 index 0000000000000000000000000000000000000000..7c8fcfc3e36467407f735033f3046053a3a46cd8 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^DytQ@6B!(kxP M0Vv6qCISgP0Q|(6fB*mh literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..ed03696970ebb04b3d2bdf2be822a23d0e8af99d GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^CH@!nB-G#myp N9e|Q-X(Eu&0|2fomr(!! literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_ur.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A_ur.npy new file mode 100644 index 0000000000000000000000000000000000000000..8db65e55f41ad766a1d31c9c77a01f2fe2316901 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^D;;?Xb|=rjOI KvZaYYLJt5$KVv{SEL#8no^cAl=zfIxNe`J zS&BtJC1{St+8zwerTM$qU2VNteWeM$pXBkpaQu)m?8NyjyGM+n0{EWKa7dFhV|kcC z1zc~Cz>OqoAr-jFGn}dcyuvlut~ww+u18rh$2DG?96`Bmw|ZP%faiJVA0nFMjxq?! zJ6M9vcOGyNP{bl28qsS~Vti4$SMK%;kUrO@hh4UlZ){4 DWcb7{ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_d.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_d.npy new file mode 100644 index 0000000000000000000000000000000000000000..774120b9ba59ff4bb77f99bdd599882359086d87 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaBq literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_u.npy new file mode 100644 index 0000000000000000000000000000000000000000..b5931a32db1828ad94d4331a87407ef9b3a30d55 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaCtc)$aDZo KvZaYYLJt572$_HY literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..88cc81cdd5feccc180feaca0c2adb43a0b237a18 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa+~6! zC$!2Z0f}j~H-;m9sc&YxtF1RYK#ht0C=27QuJD_nzNDZ`*AG(XK_TGy-7BPiDGQBNofuxIzKR`91g%+|bp?{8fwHosW#bm8^+sarn@P7={lFdlv F%uh&;#4i8< literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA.npy new file mode 100644 index 0000000000000000000000000000000000000000..f06f6883ba0e18f120b8c4883331874912288f66 GIT binary patch literal 2994 zcmeHIze~h06i$2Ais$U$)Ga+YIys042PZgK?5W`3ASG>cRJ{HmZH3|;+@f9~Tc6;6 ztNmf2h}#V=d62xkm&X_K@g=Xc?yj!yCG?D5utyV@;ZqGSXLzjPdBjpiLOWs}tVY?BdMvyJiN5UA%fgjk6x>1;*7ni3sz+mwlxM_0=Cp^TynL1AoZn`EeiZ>=zoU(Wakjdn<(2lXi4*=S99{ zLhvAX=-x!_g&u?+gdX|~2tD-K*#3WjszY-nh_ZzK+2-Y{%^wxhjod$0tOW!A$3QLF Ij8v2H4YmmYtpET3 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_d.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_d.npy new file mode 100644 index 0000000000000000000000000000000000000000..6e261ad0d4218595de2ed992ff50e4068e616805 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(J&htZ?rkdPP+htY6=mk(2bHFOVaZhmQTD%NVA30Tcbbvku0F)*Axb7n8d@=s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(J&htZ?rkdPP+htY6=mk(2bHFOVaZhmQTD%NVA30Tcbbvku0F)*Axb7t=-9u0$m NP6MDMTbc+Y^Z*bW{6GKz literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..86c84ac14e0c207499817131537e6224a6b0204a GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(J&htZ?rkdPP+htY6=mk(2bHFOVaZhmQTD%NVA30Tcbbvku0F)*Axb7n6P+m6zs Q;V_Wt0F-1)6M=*t0Iz`4zyJUM literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_l.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_l.npy new file mode 100644 index 0000000000000000000000000000000000000000..6b1737e2703f9f37d383688cd276fb2f6ac1bf21 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(J&htZ?rkdPP+htY6=mk(2bHFOVaZhmQTD%NVA30Tcbbvku0L99J{=FDCYYm^=h Phk;55pd?$G2qg3XXK?;+ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_r.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_r.npy new file mode 100644 index 0000000000000000000000000000000000000000..ab569ec1357669ed37d64d2d20455e7a7e95cb17 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(J&htZ?rkdPP+htY6=mk(2bHFOVaZhmQTD%NVA30Tcbbvku0L99J{=8P?fHA;_$ P!$742P?9Z81QL1x2IST6 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_u.npy new file mode 100644 index 0000000000000000000000000000000000000000..82166f7545ce8aa699fdeb073581fec45121825b GIT binary patch literal 2994 zcmeHIK}*9h6iz$q#Ci7Mskd%0dh#Hm3?_K6*s0*bgOsezh`2T*O@-nHyQmxFwkP)mOfmC5}ii$}Q|gp3n6&W7RvbB=S6djTgQOH#qJ zP{0DDwdw>}PolDA0UmmRGouf0a05-xXwjOqB9V`X>Nk5wFrSrMHEE2&_x+2n!dA}f z2s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(J&htZ?rkdPP+htY6=mk(2bHFOVaZhmQTD%NVA30Tcbbvku0F)*AxbH)~k_m0w| Q;V_Wt0F-1)6M=*t01bfCzyJUM literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_ur.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SA_ur.npy new file mode 100644 index 0000000000000000000000000000000000000000..51bfbd41e728d45724bc88d841f126d0dbc636ca GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(J&htZ?rkdPP+htY6=mk(2bHFOVaZhmQTD%NVA30Tcbbvku0F)*AxbH;WQkA}fO NrvXrsElmUxdH}5Dt3Ut% literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD.npy new file mode 100644 index 0000000000000000000000000000000000000000..b0f8a4d3e79be73f379dc65f8aca9f50c5a87b48 GIT binary patch literal 2994 zcmeH|u}i~16vi*9F=Cw^oU$c_lF30tC{%DTn5y95Amy4|iP$FLk_f>Tx>Q@ZEl=>j z)ui!(5jO`B?hW_defRj_ejL2Qy}7u&RmdZGGJTeK+&nhS*}&`?<~ZUhr{OT-KCA3A zI!RdRCjpIFdA2%TV{ZJeb{AW(W}9qE?1$Mj&h4QWab{1mNqSE?r#XndA!l9`CMnOn z6bca9rW1r7`gKbI4q3vfGJ+Sl2Fp=8K}%?Un#Zi_R|iM1Ue#AEq0GQ_-ShXdHjC3s zVYI%3Iap%nf#su!avuUU(5lct|BQ65ob6`_+Cm*4eoZ_1mPV?y{+nLEijYOG-l0k# z2{gG#K207-3Avz39|<&hNj^;;NC~;1N*@U{c}YG^9!Lqfp#Fa!$_XvBX_O`G$1$&# YWBy>>U#a+1sgl6o5~#jxM$5(i0K`53tpET3 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_d.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_d.npy new file mode 100644 index 0000000000000000000000000000000000000000..eee574647814daa82014d5fa32acd886e3fb9579 GIT binary patch literal 2994 zcmeH|KTE?v7{)KDv0|MaoU$c_(#b(YC{%DTn5y95AmN%^iP$FPk_f>Tx>Q@Zt*_vh z@f+07r1|3uM%)}kxOcec?zxu-?#IEqvain0uN87n9t@8~E;kN!W70EPx-kfO%xU0+ z++(?YLWdE{{K%(cmYvOJOP}h$tL@pwv(X?$iQOO>jnlT{hMcuW$uPd9oYNGDN7 z22sotH-;4mt>^?%54^Hv1s0jZvC@YpxCGNuntn}aUYw3u-mmr!V7Vx-YC@TSZQG~s zSrv=biJ`x|fhm|`>yD-Uka7=v)z|V+zy29%U09os;5URi*#DY#^ev6#Y2BB4`6@zY zopOgNeI!ujBKcH#ASL92Dt#nST<`0%CIsD~6$iHktv!(q3Z0Y`Q literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_dl.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_dl.npy new file mode 100644 index 0000000000000000000000000000000000000000..76aba27caab44c163cb9609a18f355dfddff19ca GIT binary patch literal 2994 zcmeH|&q~8U5XLvDv0^=Y@RVCpC_Q-)5egMN7)(|0;6cJR*%Gl$$|ezlE%Z`tVQ(G5 zm+=kiGim-fV8ojT5q4mAcE24y*dGJ4vain0uN87n9t@8~E;kN!W70EPx-kfO%xU0+ z++(?XLWdE{^vI`Umfg)}OP}h$i|yIQv(X?$h}|F=jnlT{hMcuW$uPd9oYNGDN7 z22sotH-;4mt*8W154^Hu1s0jZvC@YpxCGNuntn}aUYw3uUa$5JV7VxtYC@TSZQG~s zSrnVqiJ`x|fhm|`>yD-Uka7=v)z@-Uzy4`yU09os;5URi*#GKw^v#XrZrzu9`71(Z zo$`cP>PVotJ#>}TXq*JA#dyTt6Y-7U-v-XvI)($_6HtN{6GKz literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..64d2f170c56912bf9c881adf76fd5134d0f98707 GIT binary patch literal 2994 zcmeH|F-yZh6vr>AF=Cw^oU)~b(#b(YDO7MUn5y95Amy4|iP$FPk_f>Tx~MJO)+hL7 z{08+i>80_35jO`B?!nzZ_rJ#<^5ftw+{@FmD~;Tdd)sHR$L)Q~9(U}9W%nbVa2j?a z?z7xJrh}MCKMv@K$#bpVuqM{;YIC~wY*)#u#CDhtM_IG$MVvK<=^(kGoYM@%R+lp` z3gd*QUIGgc#;Oy98Tv)b0vz%HM_Lb_-~wunRu3w|@RMxB@_xOu2lH8RRT0`4T-QB$ zm$llQRsy~GHB6u;Hf~wgizxRY&;uh64b0C-=iFI;grF+){_fYbgKudhPiw!J#a|IJ zZ53ar%0~iaE{ac?2U0>VsLDqIWnPL;nFmrrE~v^!0%cx`Pnic&LN2KP-v>FNg)xlM jnEg2BbvfpD=IxnI$lG%HD%YV>#o;gCLH=Y@D!1kby|vT8 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_l.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_l.npy new file mode 100644 index 0000000000000000000000000000000000000000..b80c5ca9a2c3c9236dfa9eed30662f5b48338aef GIT binary patch literal 2994 zcmeH|KTE?v7{)KDv0|MaoU$c_(#b(YC{%DTn5y95AmN%^iP$FPk_f>Tx>Q@Zt*_vh z@f+07r1|3uM%)}kxOcec?zxu-?#IEqvain0uN87n9t@8~E;kN!W70EPx-kfO%xU0+ z++(?YLWdE{{K%(cmYvOJOP}h$tL@pwv(X?$iQOO>jnlT{hMcuW$uPd9oYNGDN7 z22sotH-;4mt>^?%54^Hv1s0jZvC@YpxCGNuntn}aUYw3u-mmr!V7Vx-YC@TSZQG~s zSrv=biJ`x|fhm|`>yD-Uka7=v)z|V+zy29%U09os;5URi*#DY#^ev6#Y2BB4`6@zY zopOgNeI!ujBKcH#ASL92Dt#nSFd5LH=bEnl0@Qb*27q literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_r.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_r.npy new file mode 100644 index 0000000000000000000000000000000000000000..aec7b815690d5fe7f64f6c62fafea8fe1bd4678f GIT binary patch literal 2994 zcmeH|ze~eF6vr>Av0|MaoU$c_(#b(YDO7MUG*!XDLBch;60uFnB@u!xbWvNltxxce zs(+K_hYyUnIf!s?xbNNf9$&Z*2XAFx_0F#qa!(!%k3}vw4s~PNH(I(e40+6H;Dp>` zxqV7U5zG9@rxTW)&1OrV>A$P(`NoUUAVrDYAQ?~6w&RAJwa3XQzNMVg6vU3hnHvUC z%o8_;6$q{91W^yXvSkGpd4v;X0MBpkT z4psU{pvpz^sq#Qd$OTpUNTA9~@~QGbO2`FO`beP4OY*7mKuX93_5b^jO=zKw!z5xq hj(Ig3^LxwgLM@;Awq76SJE@YxU;Tsp%O*5i+8-4e)$afR literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_u.npy new file mode 100644 index 0000000000000000000000000000000000000000..de16302a1b17182f16c9d01aad0ad7c0d3c5ba3f GIT binary patch literal 2994 zcmeH|ze~eF6vr>Av0|MaoU$c_(#b(YDO7MUG*!XDLBch;60uFnB@u!xbWvNltxxce zs(+K_hYyUnIf!s?xbNNf9$&Z*2XAFx_0F#qa!(!%k3}vw4s~PNH(I(e40+6H;Dp>` zxqV7U5zG9@rxTW)&1OrV>A$P(`NoUUAVrDYAQ?~6w&RAJwa3XQzNMVg6vU3hnHvUC z%o8_;6$q{91W^yXvSkGpd4v;X0MBpkT z4psU{pvpz^sq#Qd$OTpUNTA9~@~QGbO2`FO`beP4OY*7mKuX93_5b^jO=zKw!z5xq hj(Ig3^LxwgLM7yFy*|z#ELC#&%YTr6*@R|G`vYoD)$afR literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..14c3cd070efb374267f29800f5d5aba891f8d171 GIT binary patch literal 2994 zcmeH|F-yZh6vr>AF=Cw^oU)~b(#b(YDO7MUn5y95Amy4|iP$FPk_f>Tx~MJO)+hL7 z{08+i>80_35jO`B?!nzZ_rJ#<^5ftw+{@FmD~;Tdd)sHR$L)Q~9(U}9W%nbVa2j?a z?z7xJrh}MCKMv@K$#bpVuqM{;YIC~wY*)#u#CDhtM_IG$MVvK<=^(kGoYM@%R+lp` z3gd*QUIGgc#;Oy98Tv)b0vz%HM_Lb_-~wunRu3w|@RMxB@_xOu2lH8RRT0`4T-QB$ zm$llQRsy~GHB6u;Hf~wgizxRY&;uh64b0C-=iFI;grF+){_fYbgKudhPiw!J#a|IJ zZ53ar%0~iaE{ac?2U0>VsLDqIWnPL;nFmrrE~v^!0%cx`Pnic&LN2KP-v>FNg)xlM jnEg2BbvfpD=IxnI$lG#xDA%h}#o;gCLH=Y@D!1kb8@1EG literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_ur.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/SD_ur.npy new file mode 100644 index 0000000000000000000000000000000000000000..3133f21a60c1e2b051c0ddeefd973e37b068c6de GIT binary patch literal 2994 zcmeH|O-sW-5QaCYv0^=Y@RVCpC_Q-)Q3@427@Df!!GnZtvL#}hluaT8Tj)h?VQ(G5 zA60*o=EDIa-aLq~1G}^P?(o7s49v>D>YZOJ za`}{wB9`fqPbVz9o6VL!(|;G+^NkmyL5dK&K{B4CZO08cYmbvrd`mf}DTp11GdB#P zm?v%wD-c>y38Ef&WyuOG@(3r&0G{CzOiO9{HKBQNI$?Re+CPBhqI{|eWeT=!pS@>M zY*r_R!SV)XV2Z6fmJUM7J@8du%T4|Ir=@jaZ9akD5bALMtJ~2xHw|l24TfQbI16rH%xuyds?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*;kukmaNFXgCaH NIshfv(nKJk2LSOZ!Ts?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*;mEQ9K$31Dysy LNwzc*Naz6oqLIJR literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..862630cd50025af8b8b5d4658b1a3a443d30ee1e GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*;kuAhsQ)N5f$t N(*Y>SmL>uTJpe)lmr(!! literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_l.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_l.npy new file mode 100644 index 0000000000000000000000000000000000000000..3fa255b1c2bc1622532016d512ed5663c470b398 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtV1ihC_RN{RAl4{78V&=M N4nRq^G!aPX0RZ{t!Ts?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtV1ihC_RJYu5Nnhk4TphB M2cRTdng}HH0GV2ufB*mh literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_u.npy new file mode 100644 index 0000000000000000000000000000000000000000..7574bf377034e48743dc904241225661bbdccea2 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*+NS$nsHoG#myp N9e|Q-X(Eu&0|4%inScNQ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..f7ff51e321146a30f917e2ae65f1da3db670c1b3 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*+NS5bqtON5f$t N(*Y>SmL>uTJpiBfmr(!! literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_ur.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_ur.npy new file mode 100644 index 0000000000000000000000000000000000000000..2adbe6dc83d774b4be358ba663492ab2f1c0522d GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*+PoC>{-ifldRU KBwLyYB=i73YHra0 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W.npy new file mode 100644 index 0000000000000000000000000000000000000000..8a6ab2ab14b855f0691ea93076134c495f72926b GIT binary patch literal 2994 zcmeHIF-yZh6i!lO#X37UWlIVrlY@v*sNi5QRl&hQ!X>#9vB`zYMF_UgrP{!4d4m5| zbBRMJZVE22TBwCFv;Vzu)Tm#Y{&VWJt9g`0sO$DI3P*JXdW=A zfa^^q@O~1uB^5Zz8qQT8-ryE2M;%5zu7|8hv8dOkCs1zMqaIfm;JWVRXB`bVV+MZt z086m=-V-i-f@lbl7U_aC>VH!@H_q-WL<6qPj=OAUTWpG8n|$;|Q>$^?+cwI4tBBM= z>d<`~(ht%P(hpq)q#wF!{LTLW^@QenO!5r>IOer_%paALjn=(duB8J1r+~0*j%u*K E06JO3F8}}l literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA.npy new file mode 100644 index 0000000000000000000000000000000000000000..9ce5230372d7f3a39ac9eb871b638708e37e2d55 GIT binary patch literal 2994 zcmeH|ze~eF6vva)7_rU{PT7(|$>bm+6e>6vOjU4jkZ?(^L~MR=xd_1)x>Q@ZEl=>j z)%@_lh?|3h_lA4C?>)ZoK0J7(b8~TdD@%{klM&#|C&saE%qPZBH>L@p1jTMb0$j^y zXqMqh&q9>qsx}8heWCv@c9&bPMqg@z?Z$#5$#x!a;U$O!5GtWsn}*t_rF~^@KSS7O%JlH7+sQXKsdgK`Y4uRcQ9B#YZ6$QU zffMNNhuD)Y4^D}5;aTXMK$n-`)8)Y_aV|UyofGKt`X79ty3kCGl03sdo_VD@^9P&p TTIoU(l_2m}1nQN|QT13Kr_=zg literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_d.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_d.npy new file mode 100644 index 0000000000000000000000000000000000000000..9ee6cd72d22f8d652297a3233aae843455b24416 GIT binary patch literal 2994 zcmeH|F-yZh6vr>AF=Cw^oU$c_lF30tC{%DTn5y95Amx%=iP$FPk_f>Tx>OsutxxdF z_zmi3(p=+#5jO`1?+y2O|9kx5{dn+}?$z1(wSw-^gBg&-r{z=-s zv3{IR3SMyoQ?S_99m%~I(*Qy>)VOJ=e_A>h&gLV8O{Na^zq%cLb3@#%`=axqW}Rv_ zk*iATl7k>n-w(MbbsmC};3Bfr1%Wy*$*0akP!e23mbxHN=k-7MKzX5=Hj1-^{CMWo d^33lQ-Gz$K+iLa75y>n${N*0-mCbPZ*dI`T{%-&P literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_dl.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_dl.npy new file mode 100644 index 0000000000000000000000000000000000000000..e8440b1d75308dea7a050c1822d4130eb2391b07 GIT binary patch literal 2994 zcmeH|O-sW-5QaCYF=9P?@RVCpC_Q-)5egMN7)(|0;6cJR*%Gn&V3P>J7J8{Ru(yuj zpYb2m-=z6)!H72x9^8T5Vc#8In1_K`SyyN0*Alu%54uZYhw6u#J{{^^O&^Cep*XNZ z>XK4E!IPL2dhFqd6mO%`)n?l7Vtc;vthZ4WY&S^#DC^lyNJ-C6C&?|Q6lcJ8Y)YIk zh!dJR39NuARV6SraO;v4m}mjV(g>d55)4!7crB*5Nfwc^ULGDmzO27mOqzmaS*Pzs zRF~OLV3gm$3=Fn)N3v0fsSBR$DW$2Wep;Fr=H?@KZ6=TRzq%cLb3>)u;6*K)n)mD8 zc&=YU7aTZ&=6r}bY4YHdI2WFU&IvSm2|i68oD%24v(Pz#Ca?d&2Z{~N6hBO3^5dG9 ci#5NO50)}QZ|n7|ID)}n&pl$Q3@427)(|0;6chZ*%Gn&!zK}eE%c(cu(yuj z%lHQMnRL^*V8ojT5AMM3u-^_J%#VRtxRjA_oh!$kDB|2IqsQl!yLvmqd4lv z)F-8Uj7KTqdK%z_@VC`!8x!Mqu{m9PHk)V_Y&*)sq_F#5Oo<)lqwEG#iVI*{eM-DI zN;8^!87zQlt4d&cViO>m*i9DAt(thB1>HmsPpTx>Q@ZtxxdF z_zmi3(p=(!5jO`1?+x$W{qOOI`|;qdoU60*YgM^d9#B9MpQ1wpP5P)~pg~MiiX%6s z0paoq9wwyFlMs(daW~r?V`}^^wr3m9sHK#_cB5=G&MnuEDX~V`Fulc;;vCqHONk#x zNlG(6g%vQp>;%?~f~sW&wlarf)q^Lv1k+aAVT0*GnvV(Z*ZK#rTvSgDrcS_doYVIr z%4By_@Rm0)1(R*vk=%f{Un97X<3OB%eADK}m2CRqBF3o!9^11I31B`Y6s4^5dG< eiZ#Et>@Bnk&f9wZ%1^RN4u3TVd}K3RBboS6e>6vnyTR7AmNf+iP$FPk_f>Tx~L7@mM8e3 z>Sxki;(-x22M6yB@7?|H@rV2I;1%}O@cdeq?xhFa$FWEBLrtHL^q!`VBa#plIuY?P zlTXnk#-$zyXokzX(d}sq?RT-g+<4JDQWbPJOsBKVbi9aQbDB<)TSN%TfbKX1dr=rC zB=r&~fT~p|(01t8Ed^N8Bb>-CJi{dzmfQ_mRP~c=hFQNdI)HptKeebl2ivyK-pi8zJIDpad#24cecU)`hkC1VM)?AF=Cw^oU$c_(#b(YDO7MUG*!XDLCPh$60u3jB@u!xbWvNlEl==6 z)z754#sed64i4TM?(zQj_{010;1$kQ|NL4(_vpd&N#arS&@iV1vu&88n5Gm*!eKnXe1;=sD-pg1& zb~gogc>^=B*w!7%-566J0yWUMX`p{v+86fb69i4Bj`qL09er~{+^zSj^P%S5YBrIp zO6rn>AW+{AxhHiVf|B4OveX5EIxoqm&O=ZVTtt?-AW-M^Klng-p_vxOSwen1^J;nK c_m;hdiqP9?b<7dTEIIt;9`KdTaQSpT08Ql8?*IS* literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..dab3680aa670ffd9b6b4d100c541e6c785737b20 GIT binary patch literal 2994 zcmeH|&q~8U5XLvDF=9P?@RVCpC^>l$Q3@427)(|0;6chZ*%Gn&!zK}eE%c(cu(yuj z%lHQMnRL^*V8ojT5AMM3u-^_J%#VRtxRjA_oh!$kDB|2IqsQl!yLvmqd4lv z)F-8Uj7KTqdK%z_@VC`!8x!Mqu{m9PHk)V_Y&*)sq_F#5Oo<)lqwEG#iVI*{eM-DI zN;8^!87zQlt4d&cViO>m*i9DAt(thB1>HmsPpAF=Cw^oU$c_(#b(YDO7MUG*!XDLBch;60u3j;0ej!R;z8yjNirfeB;GzqB8hyl!lY6<9abAoiH7eKoi;1;=sD-t#Cg zyUSp>ynz{5eCv*6!x&Q^0yWSI(?I{Uv@h(?UNklDR=tT_ zy`(NV2mHmsPmG1>O2G`!9`@L3j%ds|APs?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa A?*IS* literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_ul.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WD_ul.npy new file mode 100644 index 0000000000000000000000000000000000000000..1c661eb0aaecc0cb4471d1315fd957c72653c8ca GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGauTJphzkztI2y literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_dr.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_dr.npy new file mode 100644 index 0000000000000000000000000000000000000000..11bf6f33db8827e19f0b3956a0c3848b28fb2373 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa#@QB literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_r.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W_r.npy new file mode 100644 index 0000000000000000000000000000000000000000..8426ecc9e2ea863d9224de035d37ae886c3573dd GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaF9sn^lZqWb$ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_1.npy new file mode 100644 index 0000000000000000000000000000000000000000..99e01d86d37113323adc20f08796dd91f9f1dc69 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa;LiU670@duLF^xBz i2)G}w+wkcn#!rOw6JrK3VT{inO7w%0Y-u8p&;tOn6)E8W literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..c8ba36b77589b7b7ff4a45e5d5427a0bbd807ab5 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGauTJph>qF+TtR literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_4_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..024d45a67b56877f8b764ac2308c82db4130cfc1 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGawlon)=m7xjqd@Th literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_1.npy new file mode 100644 index 0000000000000000000000000000000000000000..6d0b8f98c640cd07c7104e94c13f15bbabe3174d GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaAY<%AdhNDpJnl7X2Fc|m`24hADe6P!> z={laz0@H^%a4ALtuR2x&nS(}N;Y{ekJKR8B6B>4fOO_vwDB{J=5zMAZs=|da=(>LK z9a(&6r#|#%_b`Dv-+82A&o!6@wrER8YFATA?ONM@1G~n>{^2Uy$uBm7Z0*l#GF5q& z3X}?z3X}?z3X}@`4+S9F&|Dh2fk!jfycn(dgIRkj67scNw$}7Tipz{Rc$8n{Ma=!- z*heP!$~|H>@k0Kz_f}5KoelA5*k@iW%EiS=ecszRb7AkPF6&^BE6cIA$gu|VxiOtR OZ-&BJ{AB|pI@BLC?FOm< literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..0b899bb5ae596a0aa9090fce224eef4fd1e8469f GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaq}xktO*f>Z)=fI`qeH2a0^4g+ z;)X#K)5MKo0!+_3ffWO9-ZBA8d4+Se1@CYRWlOF21*Us(+9TMnHI86Bnr{`D+6UXV zFTZ(Y$vTUnHGY5rl-bS`Nn4>qJ@7SOM^e9-Q(8CH_8a&mrnL_j*-n13Dacm;EY5Ew zvvfc@ARUknNC%_?|HA>`FErD;VG@z0&%DN;`GawNq$$eRY<9}m^p6nF5-v0k_eHoy zC*DE0a?Ys;b6q%(`j(s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGacu`x}TSxF` z`Umx!bf-;0q)7!4JnX>kBRey1W*&hVTbHM2R|>f!_l8SDhZ+02@z6A?y3q+(#B9G6 zFqh)|hPsMk+Qq>3g?`-pN?bRrenelBu?)8&e%Ly2*_2O=HV>4zaz_(hA zI)NWXEOsIo1J`Dez>B^+4;h0=p5RDn!wXzM*;FcCfopD*^eM)x%{>?m=dJ=*9>B7! zlMi9D%RG#rJ-UVgl=;RjP1=FYT<}y+LsGApQkv)H`ZIVXu6B06vK@S5BgocxE6%5q zw^TqXAQg}bNCl(<|3d)~8=7mqAP(u$HLr>_zcXqKRYKmU)5#A{9fuRKS(wX{=1%eu zeR+m`qCW?-daGmKn3sbqxmV;bYGfhS&OCW@r!ylj<}bxX%wJ(%@90bGX5y+C{AFWX GT(wVKSij={ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..407908ed2535315430047cbcc2de5983e4dff8b8 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarJu literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/dl.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/dl.npy new file mode 100644 index 0000000000000000000000000000000000000000..a8c7fba01479245739ac912e26c76249940eec91 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarc literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_1.npy new file mode 100644 index 0000000000000000000000000000000000000000..e2b83ee2935807e2b21451f769edcfa7d92c1e88 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaX( zjHVMvc;E^LSZLvrAKcC)>ut|hq3%yhu*jq>N z-)hnrkkC}2yLjmwX4%=v^Ue%`Sr~U$*Y~3EEWD@|4osq+D(bkWwiI>fk&qy_?-2`U z;R`eha2gM61ebXbj?L?#S5fOr7pa*?|P0tNN z5}6?^fXd5Epmo>EQx>2LGdLFq@CLV_>0;BaQP~P(A7}Ye?*x*0{#2vl7!1R>{7h}> z^iBwaSe{M;;^1cnOCLDBF=)bZpK9~ymff<*Y&um z_NL$eHsj)*=gG4No@|~o;m?M2LsQxDq5!X4^HRFzkCN_O;$O;EVLO4pOd$JZBb2)K E7tL_7hyVZp literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..0c6449a88f52349c70c3d63a0be6fb888dee0ee8 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGazOKBHkX8aA+;GMW}} zrv;Ds?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaep=Z>D|4wG&cv8E u^!zdfSVQ-)=H{0cr(&(s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaD0jl^VTRm8Uh0w0-z*Y Ing}HH0H{u=lmGw# literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..05d408daefce9389e0a25240077997a459e9a742 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa`4=gRx+74Qq5A(;+p{D?A=pNSG j{LF^E#P?9Z81QL1xP{X)C literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..35f6986bd212cb10c8af667d58404e6c5bef35c4 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9=O5*B#TRKq>GPq z|HE<;u5chEk4xW35g#f3BsU$9t8b))>(KV+6krYA!s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarHpO$8W(!@|VpI+&j`r$MMSVQ-)=H{0c ir(&(aV)KK`MF_UgMQ!1>KEeN1 z?-~Ok1RRPuG;er!@4JtWdmj(p%D%h0zE_oJ<;C=H>{9d8FsDPaYnbDZGKvBxq#l;= z1)9WI;4wiF7InMRHD< z1}otDnhCraco}5{mhuYc>ImN97TT8DAq}p3ERC?_Yr_**F0!WvSEpdx_T`s|b+Y;l zM#~47L7VSA;dB%t>H*P+E=5W6N7A~ncHe-sxHdj6s-2b86sb1&Y-U~6c~b}G{c446 z$G5%zd@6qPq5PiL`B~Q42E5o~ea|MK=z~OLUB+s>+N%t$_AB=+=M>K>fB&8jVncJ? g50e;Iu6a$Y`J?4vp%uSc-)jf{&jC5I2@92?{1(-ZHN9Wg%bMQyXh1Eu;Zd7# z`poM1B<6j`>XA4umde^dTW#$Qx8C%EG+k`p4ZFRl(r9^^B4B$BN?M-l z2Q+L2Faf4cmB8|@J(Enpke=aGZo(^EL(z~+PL8Q|5cLSxEA=B7k7loOOzwken&%(! znr=olfadrP22f->4pVK{vhQ8 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..91384122c98fbbcac8e1cd1cdf56fd5d0aac5b13 GIT binary patch literal 2994 zcmeHIF-yZh6i!lO#X37UWxEziCI=CrP{F}qs)B=qgiCTIVw03hA_QCLQf=Y3KEZ#h zm&U_-)X727yy1QKeR(|cJ|4VyxD3SW8-_VJ=u6h4P_Z@H_F0cZaQ8}i5X^t^cGW!b6`6TC0-mQDb2hT z7QpmnC9rzrmn93ZlqnpmEX(Rdlv#XZl4|=o$1<*c58Voa=Eo#x9&RDM%}sydCZ-l8*-0H zdEdE?Sf)q5GhkWVSl_7M*Q>3i;rxrWtWB0!4U+yKZFb#|vt~cJjxQa~os`5%moqmE zqL?RcOk)z}B$Grf@bZ)~+1dl`={ zmaWOd-i@g@I;VSTh{Y?G_CkkyK#^35{{olV+c0&uZA0`p|^qM!aYkqCidNwfjKmWhL OY`=i~WfLcRG~WTQSmMS2 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..06b8593fa11094ed45c69dc313d004eff5e6c76a GIT binary patch literal 2994 zcmeH|ze~eF6vva)7_rU{PT7(|$>bm+6e>6vOjU4jka9_`L~MR=NrYevUDO6{%M<)> zH9rhZHBKG6+#BA@ee>>1?!%Fn8FyFL_dNH^y(l&gEux&t%50?cWo6=%h#+t56B`%u z1)7F9*Fy&dIA5#1zC4#Vi~YsUhtlQBpod=U28lMde1bJMo<@&|5R?Et7!z#yUKo+s ziXa0jmYqN)&#qcB(79JQ=S_HrTTper=X9uOM@fK-eqnS1>9Ts&q5KRC!?^s)qfGiB z0yBMpIjD5+2`8qHhz*Y5h=r*m{aETZ`tBPzT`EkDn{H>n+_=JR_$gJVVtEq+i$VKt zrdM1ThyOVgY})uReVt2_5qnN-27|K(A0t82}uUe|SdI^+|Yimo4rxOL16 Z`ItXShfAUUL{=Gr?Gh-iY>e`xeFNiq(p&%l literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..8d9b47f3d0a7184a15852aea46bc129e68e1061c GIT binary patch literal 2994 zcmeHI!AiqG5KU5J#d`MOskgLHdh#G56e@Tyn5y8xgOs${60uFnCJ}-adQn^0TSxG3 zHQPE6msCsBs)g=h-_F}NZze-vN5<9J`L!(FOAnewye8ES^IET_mGWB4r9L&Cx=Ss> z=@YZmjaL#u@uZ#2OeQ&dX!oGLiQv_Q~rll1IPz zcVJCKyu%oA7GBh1y+!fZLmZR5v2I+4=V|(%?x*F=!lMRJthFjU_Khg^ot78Tx~MJO)+hMi zYOe7?4Fv}WQS*lT?tS-o@O~V;g?)8)eyxyu@?iKZ_PBAV8{?kQ){TC|6HY@X;y#o1 z2_3{N_v3($Se`dqZGEDDFSe%}FGhnbL+pm>aFm&j7jb3|(?N1eIj0$j9fvb73gd*Q zUIGgc+OiTvJ@ku`1z6+}jujW4;S!pb(h6!q^OJ1EWWCxufcdO=stIKbwr!uj<*_`h zP6F=y1}4xHTX!sTBg%aU)IgI>gZf8H>%!W6f}kPP{{CvWqf$2_yLDgd#Z-h$J9zu7 zm0$4qibMGwb5x{?IqF~b%QKc|#T@m&)yuhYX4Eo^$}fmi94Zcy0R98ig35Y-wdVKT kA>YtK8%Al&)~ z`poS4B<6j`9FRD#RBP%;%@(`ktv9VK&4cZ`VSfs?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9=O5*HMB-~$RU6$ z%tpnLLur&vObCpo8AvFhhXXnv7M7#pu#mtWZZO%w8%|SzHFOVaZhmQTD%NVA30Tcb obvku0!Ms9y`s|rAwjf)EDoqc!(=rxl9qyEd4=Bl&CISgP0Jd>O#sB~S literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_1_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..1c56411ac202b6b5182c019e9d571759d88ecd46 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaa%E_`}>V)crvZ z_rt=9dh`@v4c)_s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa$R1q!h!ThR1~&=xA0cxH nxeup2p|FF73qJjL-Gs?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa9|R-1Y>& zjNf2>CT-S@mC|iRd4f44FE1~@_x?#BW9|C<;zkr6gh$0Bu0fT7k}5b2ODonL6|nSYx~_^P;D9xCDpFq@$Pg=^#IsT zlM=(RU61;P2V-FJG!j_eHlvU+sKPUxh%I=9D=4XA*~&55^nxDY@lxXeM#E?+$HXVl zH0|swY|~DydC(f&LLW+O>z)KHN2exOk|lGfm7gKiOLg-FtOAqT`*GS)f+lcU{XHM4 ziuyqex3tR>@2A#lk>;M1& literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..f111ef9c2a6c7d475c6030b5191a369d3037eacc GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaG#v1U z1NPK2DhdmMp__iD0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*+NS$iuXz>178j@oiQOo7PrOdv(aic_Yb2~Th$w&4Y?psa}%yFg^i5BrGbOU(nAjN?#&hy&1d z{p>Tc`J&Z*Xisio2xYQ$kHWTVU<+)?mMPROE|A)#w)qTpiAbIOH0@}KCQw@Atw@_1 z)#KilSKjA&XeT}8(MzJWE}v)3d1|d|K6!C?&4w2@J(oN*(t7`O&tpzJnCbcQ<|kUb zb$Ic4ag({E{ra^adZCHja{~{3`^-zxGru=!jHS4YkJ;>s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(J&htZ?rkdPP+2lB!JIUT{+ z$TTQiv4N2!K9c+jifdy1ORPFr`Wb2JVg4Sahhqp#0oKqxthxE6#i>}Uc_v^rFV*SP z!36RA*)wPMf>^Yqfo9u6?VuG+Yd_G+-7s_ag2JCx7!uR8BPmTgpd?$G2qg3X7=pRG literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_1_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..086812a937da6b744f3825aedab6955aba3cc410 GIT binary patch literal 2994 zcmds&&q~8U5XRGJt*B=Yo_b3Q(vt@fMNsfyuvNi>2PtW@645qglL$oxFKP>W>j=I~ z-$0*9Gi?UKnm^d27Z;K*`|a#+W|qK?>K7-cmj&fsdC*Mew7IrZ)*c($W?8$oxyy}i z)8-~a`^dO+nD0B5(PRF+Qr#>+m9veN=f!tzRhbO2-t{`Yey!QIIjeQNJNL%m+~|W? zYjW1MyN=7fwhJQ=#YrTHQrDb@j6hdj;IPnwH#moiUZ`3NqG-DP9>aLGu?@q)bZJ2p z9-&^ZAAkE}^3wNRXbrF62`Xawmi1eYL`IX+EvbQ{`GHR~|kwNM&|bA9=`-IeGh3cUYrxQr>VJ zeJ907t)Z8PKWj}?cUa@PF)xYL9h6e*xQNBJu z_xxv^rdTk>r06(U|D|bstRZeM<;0jI4#mZ3+WxflIBwwQQ5!Pio|(^{>JH9sFGds3 VpL&ebfSGg#iG2KJJ;Q(1egM1^I!yoo literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..0a6fed43743e49a535375b634129eda07b08e291 GIT binary patch literal 2994 zcmeHI!AiqG5Z$E4iuLTlQ*LRY^yEQADOB)aFjc{W2PxZROT;!Qn?wk<(2Lr_-a3Lm z;~&&-(%r^|5Q+%3MQ8_hU*_#&l6hH}v2}fZaigFI^k_IFu&HsV8@;+w){T}=LyEnI zPaPuYQ``;+=K~ja3D1kAvfkI{i|xV2t5HDHV7p$_>Bg0Y?Nd_eMD6enQ;K6?I}J)~ z-wQ$-*&&R9X;USzyyqm6F_`EHPLw9Rz!eltrR3(A=7e#V2)$ZAfYC5H$}yz}mSvrN z@|bRBHH7Bq7WzzbM#5JEd@skhIt27GztXMT(da!DJ|EAxxX8oMpl(Y|m5qXwd^NCl|hcwC)>6ye; zkJJ&?J?|D^{$p_$h4qkzmD^C}i$j%))d0PeX)FC1f3_7%=6?EtjvbrN1sVlR)fu!K2Qka`B=%4XF zu)nGHsRM(!o4+>b!P{@Z6h0pl( zRD|^M(-d=oVe+WGD7M9;_O`SZ>Bw9dW7)r~9|n(_a?I~1YOC_NZq+rb_6u+S4dru& zIhM8M-(yo=zVCT0QQL%<*Z#-xHts-ZAj)^JXfAr^`_|?EY7gNrOx)I;$U0&yQ1};f zf$Q<$z0f??uw0uieC8GJnLip7`U)Xm<8f}u9C>{~nM1RvwGPhKjQns3kLV50;)huC v&1F1ek6+-vSPQ@)Z^E;$zaa0k8fy6FFWx5J{mv%%fU|AvAOEth?tQX9fiE7U literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..63dccfbc69622d5bebe91844d11426aad0ee4e0d GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaw(FQbn%hye?sX1CO=AJhQMgrfrJUJZ~)2Tk{kTuQ-C#e4{L6IX>lsnYMu#L%}aGU xbud8!;p~|+dqJ$BOw+;bw$L!56}=be2V0OoXhB0_nsy|m;SWl(rHMd74*+f7zq0@U literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..ea4808195d11bd099b74588d303a1be020367085 GIT binary patch literal 2994 zcmchYOH0E*5XZOCTCtuzc3g{l!Af>gY^L(JV;5KEfL$MY!abp!HbW=-a3L` z#&4jXNi%H+!qQig)`iV~XLjZ{vut4d? z2wrnqt|$DO2Yry~K_tjb%NmCC!C)_NoT|bbTtVJQ?bs_aZF!wG$9V004|?yztreMi zf^xZh`WuYF$0&JF?cG8b@^by2cdCvlEU+~@jZix?LK>IG+AG*usnvJGv;&34kaqN$ z8J>!<=hARR`d@yv4%wt?KXLqvU|?O9quOzL8^HCJVjO&T?x z@|LC44=L3ln>4N;^Hhe)BAYa7Jmy8snKz!|NYx&b`c)p;q$)qEKWToHdf{4&T4Q}= zg?Ztg7Ud^qSjVw(KdL`YPShI9iVt%azlPw2mg%PByL{|3uLaNicJJt2W9;Yq`zx4- zNi}@IMFgNL#)*rH`?a$Hw#Iujsct^1(F=pt*9`I~4=B(bP M=HoBxo582}2PN(ZhyVZp literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..c40fbf0784b11b4d63bf2650e68251c7660bbf19 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGazPKCp0s(XbGM@nQ5x z6^HqIl*SzbpfrM>PC%+h#UWuf8V-<-2Bqw zRIJrJ6R?_>>U8R0VqiFX=FDD@r|3b0>=?q>78*{pqCsI}3k@Gy(6n+l%v_4x4@$D7 Ii9kXR0N=Av!vFvP literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_4_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..9c843803c85722e450cfc38ece981fd5390ca250 GIT binary patch literal 2994 zcmdUw&r1S96vx*!GqQ7sPTp2QJavdD1cMGOX$2iRgskq!M(WD!ZXhXmsTAhs3;JjL z5A1KMeYykV&qZAk2WH>Qci#KF_lAY(crhAlBswrzTP-&?|myVX*k$%&8gNK=|ah*BwckX_KQZT)&O=? zprmBhESowd8+yPZLr-AQnm%&rfhv8#Rj3GG@Bj%llr+LDqT7u+L4P@a2Hno+D9l1l z$mMc3f9@E5)Qk+4JD(2k8npY{ZxrTOKU5wx4T4alLud9o9vB~vhojg}i zI_K@84eAcI!QnZ%<@e9`+^(zhcy1@TtLm%9Et?DjM9^&j literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_1.npy new file mode 100644 index 0000000000000000000000000000000000000000..c27b384ce68d7c6c9640f22d76071bd2aefa8ffb GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGalsnYMu#L%}aGUbud9ZdG^d1TM&y5^j@evG@&8Bp&bbfADXxu TW-jeqKKM-rCE3zMAfX2UtOf)n literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_1_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..c287275459a83f833aa25324a29fdfa96967c629 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaUN zkBW|l0R9jdO`F3ze1>xT($4KufHiavYi@pNaVpkoo(WjZOLaPRFyT)e)RsMa=8P@K vOmb;L?!jj+J~^nH_Tr=Q$pOvY3&Qx+faE9zg#19M`2(pRlw?a2frK6al8E=C literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..2d25d4de0a440c5322d1462a2317901777066583 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa{2uuOi&^@fV`K86FSgUy^U^Oq*>D0l*z;O1=8C#H%w5Ex1Hl=E4?T4XmKBeJH QNIxav14^=`i9kXR0CAiq)c^nh literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_3_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_3_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..6efa219b6589e8b726fbdd79d6469fcc615fd41d GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaf+hh*B#IaG3)N z1zd8}5T}M2w00vt^FeY$9nd-sK;c6VJ7^UT^m04B?4eaS4Lx(G0Bh(T*4+Hk;#92F zJQJ{*m+ExtV1jts?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaVRUn T`~(&!WH%vsP?9Z81QL1xq-qkm literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_1.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_1.npy new file mode 100644 index 0000000000000000000000000000000000000000..ee854730d43b7de2861502164bdda14cdb961c33 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaa0% literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_1_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_1_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..c74ad424018948ebc2f3961a4b22a98d82ed8086 GIT binary patch literal 2994 zcmeHI!AiqG5KU8K#d`MOskgLHdh#Hm6e@Tyn5y8xgOoJc60uFnCJ}-y^rE(~w~pY? z^bhJcX*TUjC`qd(K`1WlzULLP7FN*8(iVyRr{7vhcW!N#kY=lnr-4X4v}D-G4esM2xT_ML*U;sV)e zV5FLcWn)LRVGKm@BZ1@$Z5A>HiF<;ROcP$<3W`#uq-Tks*=`rnc)orBqv7l*OENu> zW%=yG8~!3yZD@{ep$|o}b&uSpsbCFsUKgmSo|{Wbm(u1l=y}4o_Je9i3u+uytG(rB zZ?&H5pc?*ON%2%YDXqWK3GdK9%R0cm3&XT{rps`#-fwx9mlh`FGfId0jyg+%ry5~8 zN%3j*eI0azeNX#2*sox%I6Tc|S`3*U+cW$*58e$;gpTQ0DD;}=y=#7NR2%Xf_dcCY zym|ahq_LrsnM@1k9;wf$Nqy0?C|q_niR*hwUEfb!{kZundh_Lx#?e}f7`OhaeEQ2e Iinmlg0cDS7PXGV_ literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_2.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_2.npy new file mode 100644 index 0000000000000000000000000000000000000000..91ed78f839dc94cc532de8bd6088a4e0a2c5db61 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(@|Jt__h0n~6o4s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa>(J&htc#9huMWpgY3lyND(JR zEyylX)njwts2Kf106D&B!KQ~#kll@JAGO4h%^YfM-<-2BqwRIJrJ6R?_>>U8R0qM!H9o;kA@WE(D; zkV}C2Yzg4KAhU5{BFzWs2ib!`6VgwNJjh&-I$Sh9GqAY{CcKE3pk1Gxue7d9H| QR*-wKVo;JTO#~8p0Lyzv{Qv*} literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_3.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_3.npy new file mode 100644 index 0000000000000000000000000000000000000000..ee82de991e051bb1becad68c87401f7d901240cd GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcan%3f|whc|*Xc%FSH{@`k z1sl7qv=K#iKe7$f5=S<3luiE-pjJF0n*(FRXqc~Hd>BoxIJ!OlsnYMu#L%}aGUbuba{y|ZV|>;uT FJpjdgT~Yu5 literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_4.npy new file mode 100644 index 0000000000000000000000000000000000000000..11f8a2d674d1daf835fca2f76b156ac15777898f GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaJ@~|@0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*+NS u5bvcm4GSb%(I7v7%%&j?3MZOjT7?VDTpET8INV|OkV}J-Y-u8p&;tP240`(j literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_4_f4.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_4_f4.npy new file mode 100644 index 0000000000000000000000000000000000000000..a2e991d39be8903bdaa2d21bcf8a1bba52cfd5cd GIT binary patch literal 2994 zcmeH|%}&BV5XVaqG2+>SCvJs=aPnY`gai{0HWoGU;K7j6E~ybH*|sJ`NVuqw?2R$; zWqbpCChbxt!D^}HrF|oEkt#;&76S!LG zHR*Nrch_Fg>kWyWqK%x#kGaTsGs?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaTY!N8mNV?d8>-u03hbPJ)Y=ImfoMm^x$InNh|W%0kK#+_ulr zyuh+w5R_wic1L4-Y5z~VtF3ousC5$iS()aQ7sixfFD>W%0WpRu5PKoRG0h6j%9uj~ z!t9(NtSsp*4fxs%oaqs~!40^+J|Y8QCcMgV>o=xHP_O%|fzTHa1i{6pR9E~{4pDs% zOK`=`Bd#KfSOUZ#W*bVZ?@0gJ-+l!$6vpiEXWGfHG_6gWd|3UfR8<5N0YyL&Py`f# l%^(1BLklydWr6=(^M+jW2lZraY{s~13Pqs(vL%wm`vU1Xhrj>; literal 0 HcmV?d00001 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/u.npy b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/u.npy new file mode 100644 index 0000000000000000000000000000000000000000..8a754f020b29a79843f069690896426397c4eb96 GIT binary patch literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa Date: Thu, 12 Feb 2026 23:40:41 +0000 Subject: [PATCH 031/214] update --- .../inference/basic/basic_causal_wangame.py | 49 ++ examples/inference/basic/basic_wangame.py | 8 +- .../finetune_ode_init.sh | 18 +- .../preprocess_data.sh | 4 +- .../causal_wangame_ode_init/validation.json | 620 +----------------- fastvideo/configs/pipelines/__init__.py | 5 +- fastvideo/configs/pipelines/wan.py | 11 +- .../basic/wan/wangame_causal_dmd_pipeline.py | 8 +- 8 files changed, 92 insertions(+), 631 deletions(-) create mode 100644 examples/inference/basic/basic_causal_wangame.py diff --git a/examples/inference/basic/basic_causal_wangame.py b/examples/inference/basic/basic_causal_wangame.py new file mode 100644 index 000000000..f18056945 --- /dev/null +++ b/examples/inference/basic/basic_causal_wangame.py @@ -0,0 +1,49 @@ +from fastvideo import VideoGenerator +from fastvideo.configs.pipelines import SelfForcingWanGameI2V480PConfig +from fastvideo.models.dits.matrixgame.utils import create_action_presets +import torch + +BASE_MODEL_PATH = "Wan2.1-Fun-1.3B-InP-Diffusers" +WEIGHTS_PATH = "checkpoints/wangame_ode_init/checkpoint-1200/transformer" + +OUTPUT_PATH = "video_samples_wangame" +IMAGE_PATH = "https://raw.githubusercontent.com/SkyworkAI/Matrix-Game/main/Matrix-Game-2/demo_images/universal/0000.png" + + +def main(): + generator = VideoGenerator.from_pretrained( + BASE_MODEL_PATH, + pipeline_config=SelfForcingWanGameI2V480PConfig(), + num_gpus=1, + use_fsdp_inference=False, + dit_cpu_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=True, + pin_cpu_memory=True, + override_pipeline_cls_name="WanGameCausalDMDPipeline", + override_transformer_cls_name="CausalWanGameActionTransformer3DModel", + init_weights_from_safetensors=WEIGHTS_PATH, + ) + + num_frames = 81 + actions = create_action_presets(num_frames, keyboard_dim=4) + actions["keyboard"] = torch.tensor([[1.0, 0.0, 0.0, 0.0]] * num_frames) + actions["mouse"] = torch.tensor([[0.0, 0.0]] * num_frames) + + generator.generate_video( + prompt="", + image_path=IMAGE_PATH, + mouse_cond=actions["mouse"].unsqueeze(0), + keyboard_cond=actions["keyboard"].unsqueeze(0), + num_frames=num_frames, + height=352, + width=640, + num_inference_steps=40, + guidance_scale=1.0, + output_path=OUTPUT_PATH, + save_video=True, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/inference/basic/basic_wangame.py b/examples/inference/basic/basic_wangame.py index 8ce3fa36c..d5154f824 100644 --- a/examples/inference/basic/basic_wangame.py +++ b/examples/inference/basic/basic_wangame.py @@ -2,11 +2,11 @@ from fastvideo.configs.pipelines import WanGameI2V480PConfig from fastvideo.models.dits.matrixgame.utils import create_action_presets -BASE_MODEL_PATH = "weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -WEIGHTS_PATH = "wangame_1.3b_overfit/checkpoint-10000/transformer/diffusion_pytorch_model.safetensors" +BASE_MODEL_PATH = "Wan2.1-Fun-1.3B-InP-Diffusers" +# WEIGHTS_PATH = "wangame_1.3b_overfit/checkpoint-10000/transformer/diffusion_pytorch_model.safetensors" OUTPUT_PATH = "video_samples_wangame" -IMAGE_PATH = "https://raw.githubusercontent.com/SkyworkAI/Matrix-Game/main/Matrix-Game-2/demo_images/universal/0000.png" +IMAGE_PATH = "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000000.jpg" def main(): @@ -21,7 +21,7 @@ def main(): pin_cpu_memory=True, override_pipeline_cls_name="WanGameActionImageToVideoPipeline", override_transformer_cls_name="WanGameActionTransformer3DModel", - init_weights_from_safetensors=WEIGHTS_PATH, + # init_weights_from_safetensors=WEIGHTS_PATH, ) num_frames = 77 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh index c8df2b9da..2ed04c0b4 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh @@ -7,10 +7,10 @@ export WANDB_MODE=online export TOKENIZERS_PARALLELISM=false MODEL_PATH="Wan2.1-Fun-1.3B-InP-Diffusers" -DATA_DIR="mc_wasd_action/preprocessed" +DATA_DIR="../traindata_0209_1500/ode_init_mc/preprocessed/combined_parquet_dataset/worker_0" VALIDATION_DATASET_FILE="$(dirname "$0")/validation.json" -NUM_GPUS=8 -# export CUDA_VISIBLE_DEVICES=4,5,6,7 +NUM_GPUS=4 +export CUDA_VISIBLE_DEVICES=4,5,6,7 # IP=[MASTER NODE IP] # Training arguments @@ -18,11 +18,11 @@ training_args=( --tracker_project_name "wangame_ode_init" --output_dir "checkpoints/wangame_ode_init" --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" - --wandb_run_name "wangame_ode_init" - --max_train_steps 10000000 + --wandb_run_name "0211_1830_steps2000_bs_8" + --max_train_steps 2000 --train_batch_size 1 --train_sp_batch_size 1 - --gradient_accumulation_steps 4 + --gradient_accumulation_steps 1 --num_latent_t 21 --num_height 352 --num_width 640 @@ -55,8 +55,10 @@ dataset_args=( # Validation arguments validation_args=( --log_validation + --log-visualization --validation_dataset_file "$VALIDATION_DATASET_FILE" --validation_steps 100 + --visualization-steps 100 --validation_sampling_steps "50" --validation_guidance_scale "6.0" ) @@ -65,8 +67,8 @@ validation_args=( optimizer_args=( --learning_rate 6e-6 --mixed_precision "bf16" - --weight_only_checkpointing_steps 10000000 - --training_state_checkpointing_steps 10000000 + --weight_only_checkpointing_steps 200 + --training_state_checkpointing_steps 200 --weight_decay 1e-4 --max_grad_norm 1.0 ) diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh index 63c8ec77b..172de86bf 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh @@ -4,8 +4,8 @@ export PYTHONPATH="/mnt/fast-disks/hao_lab/kaiqin/FastVideo_wangame:$PYTHONPATH" GPU_NUM=1 # 2,4,8 MODEL_PATH="./Wan2.1-Fun-1.3B-InP-Diffusers" -DATA_MERGE_PATH="mc_wasd_action/merge.txt" -OUTPUT_DIR="mc_wasd_action/preprocessed" +DATA_MERGE_PATH="../traindata_0209_1500/ode_init_mc/merge.txt" +OUTPUT_DIR="../traindata_0209_1500/ode_init_mc/preprocessed" torchrun --nproc_per_node=$GPU_NUM \ fastvideo/pipelines/preprocess/v1_preprocess.py \ diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json index 94be9d122..1425f2fef 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json @@ -1,9 +1,9 @@ { "data": [ { - "caption": "00: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "caption": "", + "image_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000000.jpg", + "action_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000000_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -11,9 +11,9 @@ "num_frames": 81 }, { - "caption": "00: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "caption": "", + "image_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000001.jpg", + "action_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000001_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -22,8 +22,8 @@ }, { "caption": "00: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "image_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000002.jpg", + "action_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000002_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -32,608 +32,8 @@ }, { "caption": "00: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "01: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "01: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "01: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "01: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "02: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "02: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "02: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "02: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "03: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "03: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "03: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "03: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "04: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "04: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "04: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "04: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "05: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "05: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "05: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "05: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "06: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000006.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "06: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000006.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "06: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000006.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "06: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000006.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "07: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000007.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "07: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000007.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "07: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000007.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "07: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000007.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "08: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000008.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "08: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000008.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "08: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000008.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "08: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000008.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "09: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000009.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "09: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000009.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "09: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000009.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "09: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000009.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "10: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000010.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "10: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000010.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "10: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000010.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "10: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000010.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "11: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000011.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "11: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000011.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "11: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000011.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "11: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000011.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "12: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000012.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "12: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000012.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "12: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000012.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "12: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000012.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "13: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000013.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "13: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000013.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "13: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000013.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "13: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000013.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "14: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000014.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "14: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000014.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "14: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000014.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "14: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000014.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "15: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000015.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "15: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000015.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "15: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000015.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "15: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0209_1500/ode_init/images/000015.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "image_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000003.jpg", + "action_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000003_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, diff --git a/fastvideo/configs/pipelines/__init__.py b/fastvideo/configs/pipelines/__init__.py index 25971d1ce..9347174b5 100644 --- a/fastvideo/configs/pipelines/__init__.py +++ b/fastvideo/configs/pipelines/__init__.py @@ -14,7 +14,8 @@ WanGameI2V480PConfig, WanI2V480PConfig, WanI2V720PConfig, WanLingBotI2V480PConfig, - WanT2V480PConfig, WanT2V720PConfig) + WanT2V480PConfig, WanT2V720PConfig, + SelfForcingWanGameI2V480PConfig) __all__ = [ "HunyuanConfig", "FastHunyuanConfig", "PipelineConfig", @@ -23,6 +24,6 @@ "WanI2V720PConfig", "StepVideoT2VConfig", "SelfForcingWanT2V480PConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig", "HYWorldConfig", "SD35Config", "LingBotWorldI2V480PConfig", - "WanGameI2V480PConfig", "WanLingBotI2V480PConfig", + "WanGameI2V480PConfig", "WanLingBotI2V480PConfig", "SelfForcingWanGameI2V480PConfig", "get_pipeline_config_cls_from_name" ] diff --git a/fastvideo/configs/pipelines/wan.py b/fastvideo/configs/pipelines/wan.py index 32aad49ff..996a734aa 100644 --- a/fastvideo/configs/pipelines/wan.py +++ b/fastvideo/configs/pipelines/wan.py @@ -128,7 +128,7 @@ class WanGameI2V480PConfig(WanI2V480PConfig): dit_config: DiTConfig = field(default_factory=WanGameVideoConfig) flow_shift: float | None = 3.0 dmd_denoising_steps: list[int] | None = field( - default_factory=lambda: [1000, 666, 333]) + default_factory=lambda: [1000, 750, 500, 250, 0]) @dataclass @@ -212,6 +212,15 @@ def __post_init__(self) -> None: self.vae_config.load_decoder = True +@dataclass +class SelfForcingWanGameI2V480PConfig(WanGameI2V480PConfig): + is_causal: bool = True + flow_shift: float | None = 3.0 + dmd_denoising_steps: list[int] | None = field( + default_factory=lambda: [1000, 750, 500, 250, 0]) + warp_denoising_step: bool = True + + # ============================================= # ============= Matrix Game =================== # ============================================= diff --git a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py index be5087fec..47ebdffde 100644 --- a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py @@ -7,11 +7,11 @@ from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, CausalDMDDenosingStage, - ImageEncodingStage, + MatrixGameImageEncodingStage, InputValidationStage, LatentPreparationStage, TextEncodingStage) -from fastvideo.pipelines.stages.image_encoding import (ImageVAEEncodingStage) +from fastvideo.pipelines.stages.image_encoding import (MatrixGameImageVAEEncodingStage) logger = init_logger(__name__) @@ -37,7 +37,7 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: and self.get_module("image_processor", None) is not None): self.add_stage( stage_name="image_encoding_stage", - stage=ImageEncodingStage( + stage=MatrixGameImageEncodingStage( image_encoder=self.get_module("image_encoder"), image_processor=self.get_module("image_processor"), )) @@ -51,7 +51,7 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: transformer=self.get_module("transformer", None))) self.add_stage(stage_name="image_latent_preparation_stage", - stage=ImageVAEEncodingStage(vae=self.get_module("vae"))) + stage=MatrixGameImageVAEEncodingStage(vae=self.get_module("vae"))) self.add_stage(stage_name="denoising_stage", stage=CausalDMDDenosingStage( From 9a38c3dfd47a5e0af05dfd94addd0400f5e17225 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Fri, 13 Feb 2026 21:05:59 +0000 Subject: [PATCH 032/214] val --- .../WanGame2.1_1.3b_i2v/scripts/generate_validation.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py index 07fb50695..3444ba1eb 100644 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py @@ -10,10 +10,12 @@ height = 480 width = 832 num_frames = 81 + action_dir = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81" elif train == "mc": height = 352 width = 640 num_frames = 77 + action_dir = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions" else: raise ValueError(f"Invalid train type: {train}") @@ -32,10 +34,6 @@ "num_frames": num_frames, } -if num_frames == 81: - action_dir = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81" -else: - action_dir = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions" # WASDudlr: single key W.npy, single camera u.npy, key+camera w_u.npy still = os.path.join(action_dir, "still.npy") key_W = os.path.join(action_dir, "W.npy") From bc1be4cc69730b18f0a03a36e698dea981edd27e Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Sat, 14 Feb 2026 07:35:29 +0000 Subject: [PATCH 033/214] use mg causal denoising --- .../finetune_ode_init.sh | 6 +- .../basic/wan/wangame_causal_dmd_pipeline.py | 5 +- .../pipelines/stages/causal_denoising.py | 187 ++---------------- .../pipelines/stages/matrixgame_denoising.py | 98 ++++++++- 4 files changed, 116 insertions(+), 180 deletions(-) diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh index 2ed04c0b4..9a351607b 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh @@ -9,7 +9,7 @@ export TOKENIZERS_PARALLELISM=false MODEL_PATH="Wan2.1-Fun-1.3B-InP-Diffusers" DATA_DIR="../traindata_0209_1500/ode_init_mc/preprocessed/combined_parquet_dataset/worker_0" VALIDATION_DATASET_FILE="$(dirname "$0")/validation.json" -NUM_GPUS=4 +NUM_GPUS=1 export CUDA_VISIBLE_DEVICES=4,5,6,7 # IP=[MASTER NODE IP] @@ -18,8 +18,8 @@ training_args=( --tracker_project_name "wangame_ode_init" --output_dir "checkpoints/wangame_ode_init" --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" - --wandb_run_name "0211_1830_steps2000_bs_8" - --max_train_steps 2000 + --wandb_run_name "0213_2100_test" + --max_train_steps 1 --train_batch_size 1 --train_sp_batch_size 1 --gradient_accumulation_steps 1 diff --git a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py index 47ebdffde..4940b2fd6 100644 --- a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py @@ -6,7 +6,7 @@ from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, - CausalDMDDenosingStage, + MatrixGameCausalDenoisingStage, MatrixGameImageEncodingStage, InputValidationStage, LatentPreparationStage, @@ -54,10 +54,11 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: stage=MatrixGameImageVAEEncodingStage(vae=self.get_module("vae"))) self.add_stage(stage_name="denoising_stage", - stage=CausalDMDDenosingStage( + stage=MatrixGameCausalDenoisingStage( transformer=self.get_module("transformer"), transformer_2=self.get_module("transformer_2", None), scheduler=self.get_module("scheduler"), + pipeline=self, vae=self.get_module("vae"))) self.add_stage(stage_name="decoding_stage", diff --git a/fastvideo/pipelines/stages/causal_denoising.py b/fastvideo/pipelines/stages/causal_denoising.py index 2ee07ab1a..777314547 100644 --- a/fastvideo/pipelines/stages/causal_denoising.py +++ b/fastvideo/pipelines/stages/causal_denoising.py @@ -1,6 +1,4 @@ -import PIL.Image import torch # type: ignore -import torchvision.transforms.functional as TF from fastvideo.distributed import get_local_torch_device from fastvideo.fastvideo_args import FastVideoArgs @@ -92,17 +90,8 @@ def forward( boundary_timestep = None high_noise_timesteps = None - image_embeds = batch.image_embeds - if len(image_embeds) > 0: - assert not torch.isnan(image_embeds[0]).any() - image_embeds = [ - image_embed.to(target_dtype) for image_embed in image_embeds - ] - image_kwargs: dict[str, torch.Tensor | list[torch.Tensor]] = { - "encoder_hidden_states_image": image_embeds - } - else: - image_kwargs = {} + # Image kwargs (kept empty unless caller provides compatible args) + image_kwargs: dict = {} pos_cond_kwargs = self.prepare_extra_func_kwargs( self.transformer.forward, @@ -121,39 +110,7 @@ def forward( latents = batch.latents # [B, C, T, H, W] b, c, t, h, w = latents.shape prompt_embeds = batch.prompt_embeds - if len(prompt_embeds) == 0: - prompt_embeds = [ - torch.zeros((b, 0, self.transformer.hidden_size), - device=latents.device, - dtype=target_dtype) - ] - else: - assert not torch.isnan(prompt_embeds[0]).any() - - viewmats_full = None - intrinsics_full = None - action_full = None - if batch.mouse_cond is not None and batch.keyboard_cond is not None: - from fastvideo.models.dits.hyworld.pose import process_custom_actions - - viewmats_list = [] - intrinsics_list = [] - action_list = [] - for bi in range(b): - vm, ks, action = process_custom_actions(batch.keyboard_cond[bi], - batch.mouse_cond[bi]) - viewmats_list.append(vm) - intrinsics_list.append(ks) - action_list.append(action) - viewmats_full = torch.stack(viewmats_list, - dim=0).to(device=latents.device, - dtype=target_dtype) - intrinsics_full = torch.stack(intrinsics_list, - dim=0).to(device=latents.device, - dtype=target_dtype) - action_full = torch.stack(action_list, - dim=0).to(device=latents.device, - dtype=target_dtype) + assert torch.isnan(prompt_embeds[0]).sum() == 0 # Initialize or reset caches kv_cache1 = self._initialize_kv_cache(batch_size=latents.shape[0], @@ -203,30 +160,14 @@ def _get_kv_cache(timestep: float) -> list[dict]: # the image latent instead of appending along the channel dim assert self.vae is not None, "VAE is not provided for causal video gen task" self.vae = self.vae.to(get_local_torch_device()) - image_for_vae = batch.pil_image - if isinstance(image_for_vae, PIL.Image.Image): - # Fallback path when causal preprocessing did not convert PIL image to tensor. - image_for_vae = TF.to_tensor(image_for_vae).sub_(0.5).div_(0.5) - image_for_vae = image_for_vae.unsqueeze(0).unsqueeze(2) - elif isinstance(image_for_vae, torch.Tensor): - if image_for_vae.dim() == 4: - # [B, C, H, W] -> [B, C, 1, H, W] - image_for_vae = image_for_vae.unsqueeze(2) - elif image_for_vae.dim() == 5: - # Keep only first frame for first-frame latent initialization. - image_for_vae = image_for_vae[:, :, :1] + first_frame_latent = self.vae.encode(batch.pil_image).mean.float() + if (hasattr(self.vae, "shift_factor") + and self.vae.shift_factor is not None): + if isinstance(self.vae.shift_factor, torch.Tensor): + first_frame_latent -= self.vae.shift_factor.to( + first_frame_latent.device, first_frame_latent.dtype) else: - raise ValueError( - f"Unsupported image tensor shape for causal VAE encode: {tuple(image_for_vae.shape)}" - ) - else: - raise TypeError( - f"Unsupported batch.pil_image type for causal VAE encode: {type(image_for_vae)}" - ) - - image_for_vae = image_for_vae.to(get_local_torch_device(), - dtype=torch.float32) - first_frame_latent = self.vae.encode(image_for_vae).mean.float() + first_frame_latent -= self.vae.shift_factor if isinstance(self.vae.scaling_factor, torch.Tensor): first_frame_latent = first_frame_latent * self.vae.scaling_factor.to( @@ -247,46 +188,8 @@ def _get_kv_cache(timestep: float) -> list[dict]: set_forward_context(current_timestep=0, attn_metadata=None, forward_batch=batch): - first_frame_input = first_frame_latent.to(target_dtype) - if (batch.image_latent is not None - and not independent_first_frame): - # Keep channel layout consistent with the main denoising loop. - first_frame_image_latent = batch.image_latent[:, :, - start_index: - start_index + - 1, :, :] - first_frame_input = torch.cat([ - first_frame_input, - first_frame_image_latent.to(target_dtype) - ], - dim=1) - elif (batch.image_latent is not None and independent_first_frame - and start_index == 0): - first_frame_input = torch.cat([ - first_frame_input, - batch.image_latent.to(target_dtype) - ], - dim=2) - - expected_in_channels = getattr(self.transformer, "in_channels", - None) - if (expected_in_channels is not None - and first_frame_input.shape[1] != expected_in_channels): - raise ValueError( - "Causal first-frame cache init channel mismatch: " - f"input channels={first_frame_input.shape[1]}, " - f"expected={expected_in_channels}.") - - first_frame_action_kwargs = {} - if action_full is not None: - first_frame_action_kwargs = { - "viewmats": viewmats_full[:, - start_index:start_index + 1], - "Ks": intrinsics_full[:, start_index:start_index + 1], - "action": action_full[:, start_index:start_index + 1], - } self.transformer( - first_frame_input, + first_frame_latent.to(target_dtype), prompt_embeds, t_zero, kv_cache=kv_cache1, @@ -294,13 +197,12 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, - **first_frame_action_kwargs, **image_kwargs, **pos_cond_kwargs, ) if boundary_timestep is not None: self.transformer_2( - first_frame_input, + first_frame_latent.to(target_dtype), prompt_embeds, t_zero, kv_cache=kv_cache2, @@ -308,19 +210,12 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, - **first_frame_action_kwargs, **image_kwargs, **pos_cond_kwargs, ) start_index += 1 - if len(block_sizes) == 0: - raise ValueError( - "block_sizes is empty after first-frame initialization") - if block_sizes[0] > 1: - block_sizes[0] -= 1 - else: - block_sizes.pop(0) + block_sizes.pop(0) latents[:, :, :1, :, :] = first_frame_latent # DMD loop in causal blocks @@ -342,37 +237,15 @@ def _get_kv_cache(timestep: float) -> list[dict]: noise_latents = noise_latents_btchw.clone() latent_model_input = current_latents.to(target_dtype) - if (batch.image_latent is not None - and not independent_first_frame): - image_latent_chunk = batch.image_latent[:, :, - start_index: - start_index + - current_num_frames, :, :] - latent_model_input = torch.cat([ - latent_model_input, - image_latent_chunk.to(target_dtype) - ], - dim=1) - elif (batch.image_latent is not None - and independent_first_frame and start_index == 0): + if batch.image_latent is not None and independent_first_frame and start_index == 0: latent_model_input = torch.cat([ latent_model_input, batch.image_latent.to(target_dtype) ], dim=2) - camera_action_kwargs = {} - if action_full is not None: - end_index = start_index + current_num_frames - camera_action_kwargs = { - "viewmats": viewmats_full[:, start_index:end_index], - "Ks": intrinsics_full[:, start_index:end_index], - "action": action_full[:, start_index:end_index], - } - - # Prepare inputs [B*T, C, H, W] - t_expand = t_cur.repeat(latent_model_input.shape[0] * - current_num_frames) + # Prepare inputs + t_expand = t_cur.repeat(latent_model_input.shape[0]) # Attention metadata if needed if (vsa_available and self.attn_backend @@ -419,7 +292,6 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, - **camera_action_kwargs, **image_kwargs, **pos_cond_kwargs, ).permute(0, 2, 1, 3, 4) @@ -491,23 +363,6 @@ def _get_kv_cache(timestep: float) -> list[dict]: device=latents.device, dtype=torch.long) * int(context_noise) context_bcthw = current_latents.to(target_dtype) - context_input = context_bcthw - if batch.image_latent is not None and not independent_first_frame: - image_context_chunk = batch.image_latent[:, :, start_index: - start_index + - current_num_frames, :, :] - context_input = torch.cat( - [context_input, - image_context_chunk.to(target_dtype)], - dim=1) - context_action_kwargs = {} - if action_full is not None: - end_index = start_index + current_num_frames - context_action_kwargs = { - "viewmats": viewmats_full[:, start_index:end_index], - "Ks": intrinsics_full[:, start_index:end_index], - "action": action_full[:, start_index:end_index], - } with torch.autocast(device_type="cuda", dtype=target_dtype, enabled=autocast_enabled), \ @@ -518,7 +373,7 @@ def _get_kv_cache(timestep: float) -> list[dict]: if boundary_timestep is not None: self.transformer_2( - context_input, + context_bcthw, prompt_embeds, t_expanded_context, kv_cache=kv_cache2, @@ -526,13 +381,12 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, - **context_action_kwargs, **image_kwargs, **pos_cond_kwargs, ) self.transformer( - context_input, + context_bcthw, prompt_embeds, t_expanded_context, kv_cache=kv_cache1, @@ -540,7 +394,6 @@ def _get_kv_cache(timestep: float) -> list[dict]: current_start=(pos_start_base + start_index) * self.frame_seq_length, start_frame=start_index, - **context_action_kwargs, **image_kwargs, **pos_cond_kwargs, ) @@ -625,7 +478,7 @@ def verify_input(self, batch: ForwardBatch, result = VerificationResult() result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) - result.add_check("prompt_embeds", batch.prompt_embeds, V.is_list) + result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty) result.add_check("image_embeds", batch.image_embeds, V.is_list) result.add_check("image_latent", batch.image_latent, V.none_or_tensor_with_dims(5)) @@ -641,4 +494,4 @@ def verify_input(self, batch: ForwardBatch, result.add_check( "negative_prompt_embeds", batch.negative_prompt_embeds, lambda x: not batch.do_classifier_free_guidance or V.list_not_empty(x)) - return result + return result \ No newline at end of file diff --git a/fastvideo/pipelines/stages/matrixgame_denoising.py b/fastvideo/pipelines/stages/matrixgame_denoising.py index 3b442b068..22e892dd6 100644 --- a/fastvideo/pipelines/stages/matrixgame_denoising.py +++ b/fastvideo/pipelines/stages/matrixgame_denoising.py @@ -62,6 +62,9 @@ class BlockProcessingContext: image_kwargs: dict[str, Any] pos_cond_kwargs: dict[str, Any] + viewmats_full: torch.Tensor | None = None + intrinsics_full: torch.Tensor | None = None + action_full: torch.Tensor | None = None def get_kv_cache(self, timestep_val: float) -> list[dict[Any, Any]]: if self.boundary_timestep is not None: @@ -105,10 +108,12 @@ def __init__(self, -1) except Exception: self.local_attn_size = -1 + try: + self.local_attn_size = getattr(self.transformer.model, + "local_attn_size", -1) + except Exception: + self.local_attn_size = -1 - assert self.local_attn_size != -1, ( - f"local_attn_size must be set for Matrix-Game causal inference, " - f"got {self.local_attn_size}. Check MatrixGameWanVideoArchConfig.") assert self.num_frame_per_block > 0, ( f"num_frame_per_block must be positive, got {self.num_frame_per_block}" ) @@ -134,7 +139,10 @@ def forward( ) and not fastvideo_args.disable_autocast latent_seq_length = batch.latents.shape[-1] * batch.latents.shape[-2] - patch_size = self.transformer.patch_size + if hasattr(self.transformer, "patch_size"): + patch_size = self.transformer.patch_size + else: + patch_size = self.transformer.config.arch_config.patch_size patch_ratio = patch_size[-1] * patch_size[-2] self.frame_seq_length = latent_seq_length // patch_ratio @@ -177,6 +185,31 @@ def forward( prompt_embeds = batch.prompt_embeds assert torch.isnan(prompt_embeds[0]).sum() == 0 + viewmats_full = None + intrinsics_full = None + action_full = None + if batch.mouse_cond is not None and batch.keyboard_cond is not None: + from fastvideo.models.dits.hyworld.pose import process_custom_actions + + viewmats_list = [] + intrinsics_list = [] + action_list = [] + for bi in range(b): + vm, ks, action = process_custom_actions(batch.keyboard_cond[bi], + batch.mouse_cond[bi]) + viewmats_list.append(vm) + intrinsics_list.append(ks) + action_list.append(action) + viewmats_full = torch.stack(viewmats_list, + dim=0).to(device=latents.device, + dtype=target_dtype) + intrinsics_full = torch.stack(intrinsics_list, + dim=0).to(device=latents.device, + dtype=target_dtype) + action_full = torch.stack(action_list, + dim=0).to(device=latents.device, + dtype=target_dtype) + kv_cache1 = self._initialize_kv_cache(batch_size=latents.shape[0], dtype=target_dtype, device=latents.device) @@ -236,6 +269,9 @@ def forward( "context_noise", 0), image_kwargs=image_kwargs, pos_cond_kwargs=pos_cond_kwargs, + viewmats_full=viewmats_full, + intrinsics_full=intrinsics_full, + action_full=action_full, ) context_noise = getattr(fastvideo_args.pipeline_config, "context_noise", @@ -251,6 +287,8 @@ def forward( action_kwargs = self._prepare_action_kwargs( batch, start_index, current_num_frames) + camera_action_kwargs = self._prepare_camera_action_kwargs( + ctx, start_index, current_num_frames) current_latents = self._process_single_block( current_latents=current_latents, @@ -260,6 +298,7 @@ def forward( timesteps=timesteps, ctx=ctx, action_kwargs=action_kwargs, + camera_action_kwargs=camera_action_kwargs, progress_bar=progress_bar, ) @@ -274,6 +313,7 @@ def forward( current_num_frames=current_num_frames, ctx=ctx, action_kwargs=action_kwargs, + camera_action_kwargs=camera_action_kwargs, context_noise=context_noise, ) @@ -429,6 +469,18 @@ def _initialize_crossattn_cache(self, batch_size: int, max_text_len: int, }) return crossattn_cache + def _prepare_camera_action_kwargs( + self, ctx: BlockProcessingContext, start_index: int, + current_num_frames: int) -> dict[str, Any]: + if ctx.action_full is None or ctx.viewmats_full is None or ctx.intrinsics_full is None: + return {} + end_index = start_index + current_num_frames + return { + "viewmats": ctx.viewmats_full[:, start_index:end_index], + "Ks": ctx.intrinsics_full[:, start_index:end_index], + "action": ctx.action_full[:, start_index:end_index], + } + def _process_single_block( self, current_latents: torch.Tensor, @@ -438,6 +490,7 @@ def _process_single_block( timesteps: torch.Tensor, ctx: BlockProcessingContext, action_kwargs: dict[str, Any], + camera_action_kwargs: dict[str, Any], noise_generator: Callable[[tuple, torch.dtype, int], torch.Tensor] | None = None, progress_bar: Any | None = None, @@ -456,7 +509,16 @@ def _process_single_block( independent_first_frame = getattr(self.transformer, 'independent_first_frame', False) - if batch.image_latent is not None and independent_first_frame and start_index == 0: + if batch.image_latent is not None and not independent_first_frame: + image_latent_chunk = batch.image_latent[:, :, start_index: + start_index + + current_num_frames, :, :] + latent_model_input = torch.cat([ + latent_model_input, + image_latent_chunk.to(ctx.target_dtype) + ], + dim=1) + elif batch.image_latent is not None and independent_first_frame and start_index == 0: latent_model_input = torch.cat([ latent_model_input, batch.image_latent.to(ctx.target_dtype) @@ -522,6 +584,7 @@ def _process_single_block( latent_model_input, prompt_embeds, t_expanded_noise, + **camera_action_kwargs, **ctx.image_kwargs, **ctx.pos_cond_kwargs, **model_kwargs, @@ -594,6 +657,7 @@ def _update_context_cache( current_num_frames: int, ctx: BlockProcessingContext, action_kwargs: dict[str, Any], + camera_action_kwargs: dict[str, Any], context_noise: float, ) -> None: prompt_embeds = batch.prompt_embeds @@ -604,6 +668,17 @@ def _update_context_cache( device=latents_device, dtype=torch.long) * int(context_noise) context_bcthw = current_latents.to(ctx.target_dtype) + context_input = context_bcthw + independent_first_frame = getattr(self.transformer, + "independent_first_frame", False) + if batch.image_latent is not None and not independent_first_frame: + image_context_chunk = batch.image_latent[:, :, start_index: + start_index + + current_num_frames, :, :] + context_input = torch.cat( + [context_input, + image_context_chunk.to(ctx.target_dtype)], + dim=1) with torch.autocast(device_type="cuda", dtype=ctx.target_dtype, @@ -630,21 +705,23 @@ def _update_context_cache( if ctx.boundary_timestep is not None and self.transformer_2 is not None: self.transformer_2( - context_bcthw, + context_input, prompt_embeds, t_context, kv_cache=ctx.kv_cache2, crossattn_cache=ctx.crossattn_cache, current_start=start_index * self.frame_seq_length, start_frame=start_index, + **camera_action_kwargs, **ctx.image_kwargs, **ctx.pos_cond_kwargs, ) self.transformer( - context_bcthw, + context_input, prompt_embeds, t_context, + **camera_action_kwargs, **ctx.image_kwargs, **ctx.pos_cond_kwargs, **context_model_kwargs, @@ -657,7 +734,10 @@ def streaming_reset(self, batch: ForwardBatch, ) and not fastvideo_args.disable_autocast latent_seq_length = batch.latents.shape[-1] * batch.latents.shape[-2] - patch_size = self.transformer.patch_size + if hasattr(self.transformer, "patch_size"): + patch_size = self.transformer.patch_size + else: + patch_size = self.transformer.config.arch_config.patch_size patch_ratio = patch_size[-1] * patch_size[-2] self.frame_seq_length = latent_seq_length // patch_ratio @@ -836,6 +916,7 @@ def streaming_noise_generator(shape: tuple, dtype: torch.dtype, timesteps=ctx.timesteps, ctx=ctx, action_kwargs=action_kwargs, + camera_action_kwargs={}, noise_generator=streaming_noise_generator, ) @@ -850,6 +931,7 @@ def streaming_noise_generator(shape: tuple, dtype: torch.dtype, current_num_frames=current_num_frames, ctx=ctx, action_kwargs=action_kwargs, + camera_action_kwargs={}, context_noise=ctx.context_noise, ) From c486e6c0b753497d961239b371f6f327c284e9a1 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Sat, 14 Feb 2026 20:02:37 -0800 Subject: [PATCH 034/214] fix cache handling logic --- fastvideo/models/dits/wangame/causal_model.py | 44 +++++++++++++++++-- .../pipelines/stages/matrixgame_denoising.py | 11 ++++- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/fastvideo/models/dits/wangame/causal_model.py b/fastvideo/models/dits/wangame/causal_model.py index 9791c67bf..64bfefafc 100644 --- a/fastvideo/models/dits/wangame/causal_model.py +++ b/fastvideo/models/dits/wangame/causal_model.py @@ -248,14 +248,52 @@ def forward(self, num_new_tokens = roped_query.shape[1] # rope+prope - if kv_cache["k"].shape[-1] == self.head_dim: + cache_head_dim = kv_cache["k"].shape[-1] + local_end_index = kv_cache["local_end_index"].item() + + # read cache but never mutate it. + if not is_cache: + if cache_head_dim not in (self.head_dim, self.head_dim * 2): + raise ValueError( + f"Unexpected kv_cache head dim: {cache_head_dim}, " + f"expected {self.head_dim} or {self.head_dim * 2}") + + cache_k_rope = kv_cache["k"][..., :self.head_dim] + cache_v_rope = kv_cache["v"][..., :self.head_dim] + rope_k = torch.cat( + [cache_k_rope[:, :local_end_index], roped_key], dim=1) + rope_v = torch.cat( + [cache_v_rope[:, :local_end_index], v], dim=1) + rope_k = rope_k[:, -self.max_attention_size:] + rope_v = rope_v[:, -self.max_attention_size:] + rope_x = self.local_attn(roped_query, rope_k, rope_v) + + if cache_head_dim == self.head_dim * 2: + cache_k_prope = kv_cache["k"][..., self.head_dim:] + cache_v_prope = kv_cache["v"][..., self.head_dim:] + prope_k = torch.cat( + [cache_k_prope[:, :local_end_index], key_prope], dim=1) + prope_v = torch.cat( + [cache_v_prope[:, :local_end_index], value_prope], dim=1) + prope_k = prope_k[:, -self.max_attention_size:] + prope_v = prope_v[:, -self.max_attention_size:] + prope_x = self.local_attn(query_prope, prope_k, prope_v) + else: + prope_x = self.local_attn( + query_prope, key_prope, value_prope) + + prope_x = apply_fn_o(prope_x.transpose(1, 2)).transpose(1, 2) + return rope_x, prope_x + + # update cache. + if cache_head_dim == self.head_dim: kv_cache["k"] = torch.cat( [kv_cache["k"], torch.zeros_like(kv_cache["k"])], dim=-1) kv_cache["v"] = torch.cat( [kv_cache["v"], torch.zeros_like(kv_cache["v"])], dim=-1) - elif kv_cache["k"].shape[-1] != self.head_dim * 2: + elif cache_head_dim != self.head_dim * 2: raise ValueError( - f"Unexpected kv_cache head dim: {kv_cache['k'].shape[-1]}, " + f"Unexpected kv_cache head dim: {cache_head_dim}, " f"expected {self.head_dim} or {self.head_dim * 2}") cache_k_rope = kv_cache["k"][..., :self.head_dim] diff --git a/fastvideo/pipelines/stages/matrixgame_denoising.py b/fastvideo/pipelines/stages/matrixgame_denoising.py index 22e892dd6..b48f1b42d 100644 --- a/fastvideo/pipelines/stages/matrixgame_denoising.py +++ b/fastvideo/pipelines/stages/matrixgame_denoising.py @@ -569,6 +569,7 @@ def _process_single_block( "crossattn_cache": ctx.crossattn_cache, "current_start": start_index * self.frame_seq_length, "start_frame": start_index, + "is_cache": False, } if self.use_action_module and current_model == self.transformer: @@ -692,6 +693,7 @@ def _update_context_cache( "crossattn_cache": ctx.crossattn_cache, "current_start": start_index * self.frame_seq_length, "start_frame": start_index, + "is_cache": True, } if self.use_action_module: @@ -704,7 +706,7 @@ def _update_context_cache( context_model_kwargs.update(action_kwargs) if ctx.boundary_timestep is not None and self.transformer_2 is not None: - self.transformer_2( + cache_update_ret_2 = self.transformer_2( context_input, prompt_embeds, t_context, @@ -712,12 +714,15 @@ def _update_context_cache( crossattn_cache=ctx.crossattn_cache, current_start=start_index * self.frame_seq_length, start_frame=start_index, + is_cache=True, **camera_action_kwargs, **ctx.image_kwargs, **ctx.pos_cond_kwargs, ) + if isinstance(cache_update_ret_2, list) and len(cache_update_ret_2) > 0: + ctx.kv_cache2 = cache_update_ret_2 - self.transformer( + cache_update_ret = self.transformer( context_input, prompt_embeds, t_context, @@ -726,6 +731,8 @@ def _update_context_cache( **ctx.pos_cond_kwargs, **context_model_kwargs, ) + if isinstance(cache_update_ret, list) and len(cache_update_ret) > 0: + ctx.kv_cache1 = cache_update_ret def streaming_reset(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: From 059fd2f937eed8dfa5a02f5a2fab652a73eb55c7 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Sat, 14 Feb 2026 22:09:35 -0800 Subject: [PATCH 035/214] add visualization --- fastvideo/training/distillation_pipeline.py | 36 ++++++++++++++++--- .../training/wangame_ode_causal_pipeline.py | 20 +++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/fastvideo/training/distillation_pipeline.py b/fastvideo/training/distillation_pipeline.py index ff9537a2a..f1b95a446 100644 --- a/fastvideo/training/distillation_pipeline.py +++ b/fastvideo/training/distillation_pipeline.py @@ -88,7 +88,7 @@ def load_modules(self, if training_args.real_score_model_path: logger.info("Loading real score transformer from: %s", training_args.real_score_model_path) - training_args.override_transformer_cls_name = "WanTransformer3DModel" + # training_args.override_transformer_cls_name = "WanTransformer3DModel" # TODO(will): can use deepcopy instead if the model is the same self.real_score_transformer = self.load_module_from_path( training_args.real_score_model_path, "transformer", @@ -114,7 +114,7 @@ def load_modules(self, if training_args.fake_score_model_path: logger.info("Loading fake score transformer from: %s", training_args.fake_score_model_path) - training_args.override_transformer_cls_name = "WanTransformer3DModel" + # training_args.override_transformer_cls_name = "WanTransformer3DModel" self.fake_score_transformer = self.load_module_from_path( training_args.fake_score_model_path, "transformer", training_args) @@ -1282,6 +1282,7 @@ def run_validation_with_ema( x = torchvision.utils.make_grid(x, nrow=6) x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) frames.append((x * 255).numpy().astype(np.uint8)) + frames = self._post_process_validation_frames(frames, batch) videos.append(frames) return videos, captions @@ -1368,6 +1369,8 @@ def visualize_intermediate_latents(self, training_batch: TrainingBatch, fake_score_log_keys = ['generator_pred_video'] dmd_log_keys = ['faker_score_pred_video', 'real_score_pred_video'] + os.makedirs(training_args.output_dir, exist_ok=True) + for latent_key in fake_score_log_keys: latents = fake_score_latents_vis_dict[latent_key] latents = latents.permute(0, 2, 1, 3, 4) @@ -1392,8 +1395,20 @@ def visualize_intermediate_latents(self, training_batch: TrainingBatch, video = video.cpu().float() video = video.permute(0, 2, 1, 3, 4) video = (video * 255).numpy().astype(np.uint8) + + video_filename = os.path.join(training_args.output_dir, + f"{latent_key}_step_{step}.mp4") + # [B, T, C, H, W] to [H, W, C] + video_frames = [ + np.transpose(video[0, t], (1, 2, 0)) + for t in range(video.shape[1]) + ] + video_frames = self._post_process_validation_frames( + video_frames, training_batch) + imageio.mimsave(video_filename, video_frames, fps=24) + video_artifact = self.tracker.video( - video, fps=24, format="mp4") # change to 16 for Wan2.1 + video, fps=24, format="mp4", caption=latent_key) # change to 16 for Wan2.1 if video_artifact is not None: tracker_loss_dict[latent_key] = video_artifact # Clean up references @@ -1425,8 +1440,21 @@ def visualize_intermediate_latents(self, training_batch: TrainingBatch, video = video.cpu().float() video = video.permute(0, 2, 1, 3, 4) video = (video * 255).numpy().astype(np.uint8) + + video_filename = os.path.join(training_args.output_dir, + f"{latent_key}_step_{step}.mp4") + # [B, T, C, H, W] to [H, W, C] + video_frames = [ + np.transpose(video[0, t], (1, 2, 0)) + for t in range(video.shape[1]) + ] + video_frames = self._post_process_validation_frames( + video_frames, training_batch) + imageio.mimsave(video_filename, video_frames, fps=24) + video_artifact = self.tracker.video( - video, fps=24, format="mp4") # change to 16 for Wan2.1 + video, fps=24, format="mp4", + video_filename, caption=latent_key) # change to 16 for Wan2.1 if video_artifact is not None: tracker_loss_dict[latent_key] = video_artifact # Clean up references diff --git a/fastvideo/training/wangame_ode_causal_pipeline.py b/fastvideo/training/wangame_ode_causal_pipeline.py index 512f5c6b7..ae408282e 100644 --- a/fastvideo/training/wangame_ode_causal_pipeline.py +++ b/fastvideo/training/wangame_ode_causal_pipeline.py @@ -607,6 +607,26 @@ def visualize_intermediate_latents(self, training_batch: TrainingBatch, video = pixel_latent.cpu().float() video = video.permute(0, 2, 1, 3, 4) video = (video * 255).numpy().astype(np.uint8) + + keyboard_cond = getattr(training_batch, "keyboard_cond", None) + mouse_cond = getattr(training_batch, "mouse_cond", None) + for batch_idx in range(video.shape[0]): + sample_batch = type("ValidationBatch", (), {})() + if keyboard_cond is not None and batch_idx < keyboard_cond.shape[0]: + sample_batch.keyboard_cond = keyboard_cond[batch_idx:batch_idx + 1] + if mouse_cond is not None and batch_idx < mouse_cond.shape[0]: + sample_batch.mouse_cond = mouse_cond[batch_idx:batch_idx + 1] + + video_frames = [ + np.transpose(video[batch_idx, frame_idx], (1, 2, 0)) + for frame_idx in range(video.shape[1]) + ] + video_frames = self._post_process_validation_frames( + video_frames, cast(ForwardBatch, sample_batch)) + video[batch_idx] = np.stack([ + np.transpose(frame, (2, 0, 1)) for frame in video_frames + ], axis=0) + video_artifact = self.tracker.video( video, fps=16, format="mp4") # change to 16 for Wan2.1 if video_artifact is not None: From ffec3b106aff76b80609fe6a3f408f7e7be2806d Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 07:17:59 +0000 Subject: [PATCH 036/214] try to read and design --- dev/design.md | 325 +++++++++++++++++++++++++++++++++++++++ dev/distill_structure.md | 262 +++++++++++++++++++++++++++++++ 2 files changed, 587 insertions(+) create mode 100644 dev/design.md create mode 100644 dev/distill_structure.md diff --git a/dev/design.md b/dev/design.md new file mode 100644 index 000000000..cb31a9693 --- /dev/null +++ b/dev/design.md @@ -0,0 +1,325 @@ +# Distill 重构设计草案：`models={teacher, student, critic, ...}` + 算法/模型解耦 + +这份文档的目标是：在不牺牲当前训练基础设施（分布式、tracker、checkpoint、 +VSA/VMoBA 等）的前提下，把 distill 训练部分重构成： + +1. **模型输入是显式的 `models` 映射**（teacher/student/critic/...），critic 可选 +2. **distill 算法与具体模型解耦**：算法通过 “pipeline adapter/capability” 来 + 调用模型，而不是写死 Wan 的张量布局/归一化/输入 key +3. **易扩展**：新增算法或新增模型架构，只需要新增一个 strategy 或 adapter， + 不需要到处改训练 loop + +下面所有 “设计” 都会同时写 “原因”，方便取舍。 + +## 一句话总结（推荐的最终形态） + +- `DistillTrainer`（统一训练 loop） + `DistillAlgorithm`（DMD2/SelfForcing/…） + + `DistillAdapter`（把具体 pipeline 适配成统一接口） + `ModelBundle` + （按角色组织的模型对象/optimizer/EMA）。 + +## 现状痛点（为什么要重构） + +> 这里只列与架构强相关的点；代码细节见 `dev/distill_structure.md`。 + +- distill 逻辑 **Wan 耦合严重**（normalize / override cls name / input layout） +- DMD 与 Self-forcing 两套 pipeline 各自维护一份训练 loop，重复且容易 drift +- “需要 teacher/critic/CFG/uncond embedding”等约束不显式，依赖隐含流程 +- MoE/dual-transformer（`transformer_2`）的选择、更新在不同 pipeline 里不统一 +- 想支持更多模型（Hunyuan/LTX2/LongCat/…）或更多 distill 算法会越来越难 + +## 设计目标与非目标 + +### 目标 + +- 用 `models: dict[str, ...]` 显式描述参与训练的所有模型角色 +- 算法（DMD2/Self-forcing/未来更多）只依赖统一接口，不依赖 Wan 特有细节 +- 训练 loop 统一（分布式/日志/checkpoint/validation 只实现一次） +- 在 adapter 层支持多种输入来源： + - 已 preprocess 的 parquet（含 `vae_latent`/`text_embedding`） + - 或者（可选）原始数据在线 encode（由 pipeline 决定） + +### 非目标（第一阶段先不做） + +- 不强求“一套 adapter 自动适配所有模型”，允许为不同架构写显式 adapter +- 不强求把所有训练 pipeline（finetune、matrixgame、ltx2 等）一起重构掉 +- 不追求立刻改变 checkpoint 格式（可以先兼容旧格式再演进） + +## 总体架构（模块分层） + +建议把 distill 拆成四层，从下到上： + +1. **Adapter 层**（模型相关） + - 把某个 pipeline/transformer 的 forward 细节封装起来 +2. **Algorithm/Strategy 层**（算法相关） + - DMD2 / Self-forcing / 其它 distill 算法 +3. **Trainer/Engine 层**（基础设施） + - 分布式、seed、grad accumulation、优化器 step、日志、checkpoint、验证 +4. **CLI/Entrypoint 层** + - 解析参数、加载 models、选择算法、启动 trainer + +## 关键设计决策（每项含原因） + +### 设计 1：用 `ModelBundle` 统一承载多角色模型（核心） + +**设计** + +- 引入一个只做 “容器” 的数据结构（建议 dataclass）： + + - `models: dict[str, ModelHandle]` + - 推荐 canonical keys：`student`, `teacher`, `critic` + - 允许扩展：`reward`, `refiner`, `aux_teacher`, `student_ema` 等 + +- `ModelHandle` 不是裸 `nn.Module`，而是： + + - `module`: 主要网络（如 transformer） + - `extra_modules`: 可选模块（如 `transformer_2`、image encoder 等） + - `optimizers`, `schedulers`, `ema`（可选） + - `trainable: bool` + `param_groups`（用于决定哪些参数会更新） + - `capabilities`: 模型端能力声明（见设计 3） + +**原因** + +- 角色显式化后，算法只需要声明 “我需要哪些 role”，无需硬编码一堆参数名 +- `ModelHandle` 把 optimizer/EMA 等训练态绑定到 role，checkpoint/save/resume + 逻辑才能自然泛化 +- 对 MoE/dual-transformer 这类 “一个 role 内部有多个可训练模块” 的情况， + `extra_modules` 能承载而不污染最上层 `models` 命名空间 + +### 设计 2：把“加载模型”从“训练算法”中剥离成 ModelFactory/Loader + +**设计** + +- 单独实现 `ModelFactory`： + - 输入：`ModelSpec`（路径/是否冻结/precision/offload/fsdp 等） + - 输出：`ModelHandle` +- teacher/critic 只加载算法需要的最小组件（通常只要 transformer），其余组件 + （VAE/scheduler/text encoder）优先复用 student pipeline 的 shared 部分， + 由 adapter 决定是否允许共享 + +**原因** + +- 让算法代码完全不关心 “从哪里 load”、“如何 CPU offload”、“如何 FSDP 包装” +- 有利于做 memory/throughput 优化（teacher/critic 可以走不同策略） +- 避免现状里 `load_modules` 里混杂大量 Wan 特判/override 的情况 + +### 设计 3：用 `DistillAdapter` 做“pipeline 适配层”，并显式声明 capability + +**设计** + +- 定义一个 adapter 接口（Protocol/ABC 均可），把下列能力抽象出来： + + 1. **latent 规范化/布局** + - `normalize_latents(x)` / `denormalize_latents(x)`（如需） + - `to_model_layout(x)` / `from_model_layout(x)` + 2. **conditioning 规范化** + - 把 dataset/pipeline 产生的 conditioning 变成统一结构 + （例如 `Conditioning` = dict[str, Tensor]） + 3. **噪声与 parameterization** + - `add_noise(x0, noise, t)` + - `pred_to_x0(pred, x_t, t)` 或 `pred_to_video_latent(...)` + - 声明 `prediction_type`（eps/v/x0/flow），由 adapter 负责转换 + 4. **模型 forward** + - `forward(role, x_t, t, cond, *, caches=None, return_dict=False)` + - 允许 adapter 在内部处理 `set_forward_context` / attention metadata + 5. **CFG 支持（可选）** + - `supports_cfg: bool` + - `build_uncond(cond, negative_prompt=...)` 或直接提供 + `uncond_conditioning_cache` + +- adapter 要返回一个 `Capabilities` 对象（dataclass）： + - 是否支持 CFG/uncond + - 是否支持 KV cache（self-forcing 需要） + - 是否存在/如何选择 `transformer_2`（MoE） + - 输入 key 要求（`encoder_attention_mask` 是否必须等） + +**原因** + +- “算法与模型解耦”只能靠明确的接口边界实现；adapter 是最合适的边界 +- capability 显式化后，算法可以做： + - `if not supports_cfg: raise/降级` + - `if supports_kv_cache: enable self-forcing` 否则 fallback +- 避免用脆弱的反射去“猜 pipeline 有哪些 stage/属性”，可维护性更高 + +### 设计 4：用 Strategy 模式承载 distill 算法（DMD2/SF/未来更多） + +**设计** + +- `DistillAlgorithm` 负责： + - 声明需要的 roles：`required_roles`, `optional_roles` + - 声明每一步要更新哪些 role（以及 update ratio/交替策略） + - 定义 loss： + - `compute_losses(batch, ctx) -> LossBundle` + - 或者更细：`losses_for(role)` + `metrics` + - （可选）维护算法内部状态（例如 self-forcing 的 cache 管理/exit flag 采样） + +- 例子： + - `DMD2Algorithm`： + - `required_roles = {student, teacher, critic}` + - loss = 当前 `_dmd_forward` + critic flow-matching + - `SelfForcingAlgorithm`： + - 基于 DMD2，但 student forward 换成 causal/self-forcing 的 trajectory + - 需要 `supports_kv_cache` + `is_causal` 等 capability + - `TeacherOnlyAlgorithm`（未来可选）： + - `required_roles = {student, teacher}` + - 不依赖 critic（满足 “critic optional” 的场景） + +**原因** + +- 把 “训练 loop” 从 pipeline class 里抽出来后，DMD/SF 不需要两份 train() +- 新增算法不会影响 adapter/trainer，只需要加一个 strategy 类和少量配置 +- 更容易做 unit test：给一个 dummy adapter + dummy models 就能测 loss 逻辑 + +### 设计 5：`DistillTrainer` 统一训练基础设施，并用“更新计划”驱动 optimizer + +**设计** + +- trainer 只做基础设施： + - 数据迭代（dataloader / batch builder） + - grad accumulation + autocast + clip grad + - 按 strategy 给出的 “更新计划” 去 step 不同 optimizer + - all-reduce loss/metrics + - tracker log + checkpoint + validation hook +- trainer 不写任何 DMD/SF 专有逻辑（最多提供 hook 点） +- “更新计划”可以是： + - `UpdatePlan = list[Update(role, loss_key, optimizer_key, scheduler_key)]` + - 或者简单：`roles_to_update = {...}` + `losses` + +**原因** + +- 训练 loop 只实现一次，避免 DMD/SF drift +- update ratio（generator_update_interval / dfake_gen_update_ratio）变成算法参数， + 不再散落在不同 pipeline 里 +- 支持更多角色/更多 optimizer 组合时不会爆炸 + +### 设计 6：把 distill 专有参数从 `TrainingArgs` 里拆成 `DistillConfig` + +**设计** + +- `TrainingArgs` 保持偏 “训练基础设施”： + - 数据、输出目录、分布式、optimizer 基本参数、logging/checkpoint +- distill 算法专有参数放到 `DistillConfig`： + - DMD2：timestep ratio、guidance scale、denoising steps、update interval 等 + - Self-forcing：num_frame_per_block、gradient masking 等 + - Critic：fake_score lr/betas/scheduler 等 +- CLI 层把 config 做成 namespace： + - `--distill.algo dmd2` + - `--distill.dmd2.generator_update_interval 5` + - `--distill.sf.num_frame_per_block 3` + +**原因** + +- 现在 `TrainingArgs` 已经非常大；继续塞 distill 参数会让其它训练模式更难维护 +- 分离后能清晰表达：某些参数只对某个算法生效 +- 便于做默认值管理（不同 pipeline/算法可提供不同 defaults） + +### 设计 7：checkpoint 以 role 为单位泛化（并提供推理导出） + +**设计** + +- 引入 `CheckpointManager`： + - `save(role_states, shared_states, step)` + - `load(...)` +- role_states 来自 `ModelHandle`： + - trainable role 保存 optimizer/scheduler/ema + - frozen role（teacher）通常只保存 path 或权重 hash（可选） +- “推理导出”是一个独立通道： + - 例如只导出 `student`（和可选 `student.transformer_2`）到 diffusers 格式 + +**原因** + +- 现有 `save_distillation_checkpoint` 已经在 role 粒度上开始泛化 + （generator/critic/generator_2/critic_2/real_score_2），继续泛化会更自然 +- 未来支持更多角色时（reward/refiner）不需要再复制粘贴一套 save/load + +### 设计 8：把 “uncond conditioning” 做成显式的 ConditioningProvider + +**设计** + +- 对需要 CFG 的算法，提供一个 `ConditioningProvider`： + - 在训练开始时就构建/缓存 negative prompt embedding（或从 dataset 读取） + - 不依赖 “是否开启 validation logging” +- provider 与 adapter 配合： + - adapter 负责怎么 encode negative prompt（模型不同，编码方式不同） + - provider 负责生命周期与 cache（rank0 广播、避免重复算） + +**原因** + +- 现状里 uncond embedding 依赖 `_log_validation`，属于隐式耦合，容易踩坑 +- provider 显式后，算法与 trainer 的依赖更清晰，validation 也可以变成可选 + +## CLI 形态建议（与 `models={...}` 对齐） + +### 推荐参数形式 + +- `--models.student ` +- `--models.teacher ` +- `--models.critic `（可选） + +如果需要支持更复杂的配置（比如每个 role 的 precision/offload）： + +- `--models_json path/to/models.json` + +示例 JSON（建议）： + +```json +{ + "student": {"path": "wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers", "trainable": true}, + "teacher": {"path": "Wan-AI/Wan2.1-T2V-14B-Diffusers", "trainable": false}, + "critic": {"path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "trainable": true} +} +``` + +**原因** + +- 直接映射到 `models: dict[str, ModelSpec]`，减少 “某个 role 的 path 到底对应哪个 + CLI 参数” 的歧义 +- JSON 形式在复杂场景下更可扩展（多模块、MoE、不同 offload 策略等） + +## 迁移计划（推荐分阶段） + +### Phase 0：只加新框架，不动现有入口 + +- 新增 `fastvideo/distill/`（或 `fastvideo/training/distill/`）： + - `models.py`：`ModelSpec/ModelHandle/ModelBundle` + - `adapters/`：`WanDistillAdapter`（先只支持 wan） + - `algorithms/`：`DMD2Algorithm`、`SelfForcingAlgorithm` + - `trainer.py`：`DistillTrainer` +- 先用单元测试覆盖核心 loss（不用真模型，dummy adapter 即可） + +**原因** + +- 风险最小：不影响现有脚本/训练 +- 先把边界（adapter/strategy/trainer）跑通，后面迁移才不痛 + +### Phase 1：把现有 Wan DMD / Wan Self-forcing 迁移到新框架（行为对齐） + +- 新建一个新的入口脚本（或训练文件）： + - `fastvideo/training/distill.py`（仅示例） +- 让旧入口（`wan_distillation_pipeline.py` 等）可以选用新 trainer（通过 flag） +- 对齐： + - loss 数值 + - checkpoint 目录结构 + - validation 输出 + +**原因** + +- 迁移时可 A/B 对比，减少“重构引入质量回归”的概率 + +### Phase 2：清理旧实现 + 扩展更多模型/算法 + +- 删除或冻结旧 distill pipeline（保留兼容入口也行） +- 为其它模型实现 adapter（Hunyuan/LTX2/LongCat…） +- 引入更多算法（teacher-only、multi-teacher、RLHF-style…） + +**原因** + +- 把新增工作限制在 “写 adapter / 写 strategy”，让扩展成本线性增长 + +## 额外建议（踩坑预防） + +- 明确 role 的训练/冻结策略：teacher 永远 `no_grad + eval`，critic/trainable + role 的 `requires_grad` 与 optimizer param group 必须绑定在一起 +- MoE/dual-transformer：建议把 “按 timestep 选择哪个 expert 更新” 的逻辑放到 + adapter 或 strategy 的单一位置，避免像现状一样分散在多处 +- lr scheduler 的 step 粒度：建议按 optimizer step，而不是按 global step + （否则 update ratio 会改变 effective schedule） diff --git a/dev/distill_structure.md b/dev/distill_structure.md new file mode 100644 index 000000000..d5195fbb9 --- /dev/null +++ b/dev/distill_structure.md @@ -0,0 +1,262 @@ +# FastVideo Distill 训练结构梳理（现状） + +本文是对当前仓库 distill 训练代码的“结构/逻辑”阅读笔记，目标是把： + +- 入口在哪里 +- 训练时有哪些模型（student/teacher/critic） +- 每一步在算什么 loss、更新谁 +- Self-forcing 相对 DMD 的差异 +- 目前实现的耦合点/限制 + +写清楚，方便后续重构和对齐实现细节。 + +## 代码入口与文件分布 + +- 训练主逻辑 + - `fastvideo/training/distillation_pipeline.py`：`DistillationPipeline` + （DMD/DMD2 风格 distillation） + - `fastvideo/training/self_forcing_distillation_pipeline.py`： + `SelfForcingDistillationPipeline`（Self-forcing distillation） +- Wan 封装/可运行入口（torchrun 直接跑这些文件） + - `fastvideo/training/wan_distillation_pipeline.py`： + `WanDistillationPipeline` + - `fastvideo/training/wan_self_forcing_distillation_pipeline.py`： + `WanSelfForcingDistillationPipeline` + - `fastvideo/training/wan_i2v_distillation_pipeline.py`： + `WanI2VDistillationPipeline` +- 推理/验证用 pipeline（用于 `_log_validation`） + - `fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py`：`WanDMDPipeline` + - `fastvideo/pipelines/basic/wan/wan_causal_dmd_pipeline.py`： + `WanCausalDMDPipeline` + - I2V：`fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py` +- 断点续训/保存 + - `fastvideo/training/training_utils.py`： + `save_distillation_checkpoint` / `load_distillation_checkpoint` +- 测试 + - `fastvideo/tests/training/distill/test_distill_dmd.py` + - `fastvideo/tests/training/self-forcing/test_self_forcing.py` + +## 先说结论：当前 distill 覆盖范围 + +1. **模型侧目前明显偏 Wan** + - `TrainingPipeline._normalize_dit_input` 固定走 + `normalize_dit_input('wan', ...)` + - teacher/critic 加载时强制设置 + `training_args.override_transformer_cls_name = "WanTransformer3DModel"` + - 因此“想 distill 其它架构”目前不是简单换配置就能跑通的。 + +2. **算法侧两条主线** + - **DMD/DMD2**：`DistillationPipeline` + （student + teacher(real score) + critic(fake score)） + - **Self-forcing**：`SelfForcingDistillationPipeline` + （在 DMD2 框架里，把 student 前向改成 causal/self-forcing 的 + blockwise generation + KV cache） + +3. **默认/隐含假设** + - DMD 路径里多处 `unflatten(0, (1, T))`，等价于默认 `batch_size == 1` + （脚本里确实几乎都设 `--train_batch_size 1`）。 + - DMD loss 需要 teacher 的 cond/uncond 两次前向做 CFG；uncond embedding + 目前依赖 validation 阶段用 negative prompt 编码得到 + （见 “unconditional conditioning 的来源” 一节）。 + +## DistillationPipeline（DMD/DMD2）训练逻辑 + +### 1) 参与训练/冻结的模块（roles） + +`DistillationPipeline.load_modules()` + `initialize_training_pipeline()` 搭出 +“三模型”结构： + +- **student / generator**：`self.transformer` + - 来自 `--pretrained_model_name_or_path` + - 这是要训练的主模型 + - VAE：`self.vae` 同样来自 student pipeline，但在 distill 中 **冻结** +- **teacher / real score**：`self.real_score_transformer` + - 来自 `--real_score_model_path` + - `requires_grad_(False)` + `eval()` +- **critic / fake score**：`self.fake_score_transformer` + - 来自 `--fake_score_model_path` + - **要训练**，有自己的一套 optimizer/scheduler +- （可选）MoE/双 transformer 支持（teacher/critic 优先） + - teacher/critic 会尝试加载 `transformer_2` + - 用 `boundary_ratio`（→ `boundary_timestep`）决定高噪/低噪 expert 选择： + - `_get_real_score_transformer(t)` + - `_get_fake_score_transformer(t)`（并设置 `train_fake_score_transformer_2`） + +> 备注：student 侧也可能有 `transformer_2`（例如 Wan2.2），但 DMD 路径的 +> optimizer step 目前主要写死在 `self.transformer` 上；self-forcing 路径则会 +> 在 `optimizer` / `optimizer_2` 间二选一。重构时建议统一“哪个 timestep +> 更新哪个 expert”的决策来源与实现方式。 + +### 2) 关键超参/调度 + +- `generator_update_interval` + - `step % generator_update_interval == 0` 才更新 student（generator） + - critic 每 step 都更新 +- `pipeline_config.dmd_denoising_steps` → `self.denoising_step_list` + - 这是 “student 训练/模拟推理” 用到的一组离散 timestep + - 若 `--warp_denoising_step`，会根据 scheduler 的 time shift 做一次映射 +- timestep sampling 范围（用于 DMD loss / critic loss 的随机 timestep） + - `min_timestep = min_timestep_ratio * num_train_timesteps` + - `max_timestep = max_timestep_ratio * num_train_timesteps` + - 最终都会 clamp 到 `[min_timestep, max_timestep]` +- teacher CFG + - `real_score_guidance_scale` + +### 3) batch / tensor 形状约定（T2V） + +从 parquet dataloader 读到的典型字段： + +- `vae_latent`: `[B, C, T_lat, H_lat, W_lat]` +- `text_embedding`, `text_attention_mask` +- `info_list`（日志用） + +进入 distill 训练后，关键变换： + +- `_prepare_dit_inputs` 里把 `latents` permute 成 `[B, T_lat, C, H, W]` + 并把 `self.video_latent_shape` 记录为这个形状 +- `_build_distill_input_kwargs` 会再 permute 回 `[B, C, T_lat, H, W]` + 作为 transformer 的 `hidden_states` + +### 4) 一个 step 内发生什么（train_one_step） + +`train_one_step` 的大体结构： + +1. 收集 `gradient_accumulation_steps` 个 batch + - 每个 batch 会构建 attention metadata，并复制出一份 `attn_metadata_vsa` + - `attn_metadata_vsa`：保留当前 VSA sparsity（稀疏注意力） + - `attn_metadata`：强行把 `VSA_sparsity = 0`（等价 dense） + - 代码用法上：student forward 常用 `attn_metadata_vsa`， + teacher/critic forward 用 `attn_metadata`（dense） + +2. **可选更新 student（generator）** + - 条件：`current_step % generator_update_interval == 0` + - 对每个 accumulated batch： + - 先算 `generator_pred_video` + - `--simulate_generator_forward`：`_generator_multi_step_simulation_forward` + （从纯噪声开始按 `denoising_step_list` 模拟推理，最后一步保留梯度） + - 否则：`_generator_forward` + （对数据里的 `vae_latent` 加噪声做一次 denoise；这里把 batch 维度 + `unflatten` 写死成 `(1, T)`，隐含 `B=1`） + - 再算 `dmd_loss = _dmd_forward(generator_pred_video, batch)` 并反传 + - clip grad → `self.optimizer.step()` → EMA update（若开启） + +3. **更新 critic（fake score）**（每个 step 都做） + - 对每个 accumulated batch： + - 先 `generator_pred_video`（no_grad） + - 再随机采样 `fake_score_timestep`，把 `generator_pred_video` 加噪声得到 + `noisy_generator_pred_video` + - critic 预测 `fake_score_pred_noise` + - 目标是 `target = noise - generator_pred_video` + - `flow_matching_loss = mean((pred - target)^2)` + - clip grad → critic optimizer step → critic scheduler step + - 注意：代码里 `self.lr_scheduler.step()`（student 的 lr scheduler）也会在 + critic 更新后每 step 都跑一次，即使这个 step 没有更新 student。 + 如果 `generator_update_interval > 1`，这会让 student 的 lr schedule + “按 step 走”而不是“按 optimizer step 走”。 + +### 5) DMD loss 细节（_dmd_forward） + +简化写成： + +- 输入：`x = generator_pred_video`（student 产出的 clean latent） +- 随机采样 timestep `t` +- 加噪声：`x_t = add_noise(x, eps, t)` +- critic（fake score）得到 `x_hat_fake` +- teacher（real score）分别做 cond/uncond 得到 `x_hat_real_cond`, + `x_hat_real_uncond`，再做 CFG： + + `x_hat_real = x_hat_real_cond + (x_hat_real_cond - x_hat_real_uncond) * s` + +- 构造一个“梯度方向”（代码变量名为 `grad`）： + + `g = (x_hat_fake - x_hat_real) / mean(abs(x - x_hat_real))` + +- DMD2 风格的 stop-grad target： + + `L = 0.5 * mse(x, (x - g).detach())` + +并把中间 latent 存到 `training_batch.dmd_latent_vis_dict` 里用于可视化。 + +### 6) unconditional conditioning 的来源 + +DMD loss 里 teacher 需要 uncond forward： + +- `training_batch.unconditional_dict` 来自 `_prepare_dit_inputs`，但它只在 + `self.negative_prompt_embeds` 已经存在时才会被创建 +- `self.negative_prompt_embeds` 当前是在 `_log_validation` 里，通过 + `validation_pipeline.prompt_encoding_stage` 对 `SamplingParam.negative_prompt` + 编码得到（会同时设置 attention mask） +- 因此目前的“默认用法”是：训练脚本基本都会开 `--log_validation`，保证在 + train 开始前 `_log_validation` 跑一次，从而初始化 negative prompt embedding + +> 重构时建议显式化这条依赖：uncond embedding 不应依赖 +> “是否开启 validation logging”。 + +## SelfForcingDistillationPipeline（Self-forcing）训练逻辑 + +Self-forcing pipeline 继承自 `DistillationPipeline`，但有三个核心变化： + +### 1) scheduler 换成 SelfForcingFlowMatchScheduler + +`initialize_training_pipeline` 里把 `self.noise_scheduler` 替换为 +`SelfForcingFlowMatchScheduler(..., training=True, extra_one_step=True)`， +并引入 self-forcing 的一批超参： + +- `dfake_gen_update_ratio`：generator 每 N step 更新一次（其余 step 只更 critic） +- `num_frame_per_block`, `independent_first_frame`, `same_step_across_blocks`, + `last_step_only`, `context_noise` 等 + +### 2) generator forward 变成 causal/blockwise + KV cache + +`_generator_multi_step_simulation_forward` 在 self-forcing 里被彻底重写： + +- 按 block 生成视频（每 block `num_frame_per_block` 帧） +- 每个 block 内做 denoising loop（遍历 `denoising_step_list`），并用随机 + `exit_flag` 决定在哪个 timestep 停止并保留梯度 +- KV cache / cross-attn cache + - `_initialize_simulation_caches` 预分配 cache 张量 + - 每生成完一段会用 `context_noise`（通常 0）再跑一次 timestep=0 的 forward + 来更新 cache（no_grad） +- gradient masking + - 当生成帧数 > 最小帧数时，会把一部分早期帧 detach，确保只在后面某些帧上回传梯度 + - 额外还有 “last 21 frames” 的处理逻辑：把更早的帧 decode→encode 变成 + image latent，再拼回去 + +总体上，这是把“推理时的 autoregressive/cached 过程”搬进训练。 + +### 3) train_one_step 变成“交替训练 generator / critic” + +- `train_generator = (step % dfake_gen_update_ratio == 0)` +- generator 更新时 + - `generator_loss` 默认仍然走 `_dmd_forward` + （所以仍依赖 teacher + critic） + - optimizer step 会在 `transformer` 和 `transformer_2` 之间二选一 + （基于 `self.train_transformer_2`） +- critic 更新时 + - `critic_loss` 仍然是 `faker_score_forward`（flow matching） + +## Checkpoint / Resume（distill 专用） + +`save_distillation_checkpoint` / `load_distillation_checkpoint` 的要点： + +- 保存 generator + critic 的 distributed checkpoint（可用于 resume） +- 同时保存 “consolidated generator weights”（用于推理/部署） +- 支持（可选）MoE 的 `transformer_2` / `critic_2` / `real_score_2` +- 保存 random state（保证 resume 后 timestep/noise 采样一致） +- EMA 以单独文件的方式保存（避免不同 rank 形状不一致） + +## 目前实现中最重要的耦合点/限制（重构 checklist） + +- **Wan-only 逻辑散落多处** + - normalize 走 `'wan'` + - teacher/critic 强制 `WanTransformer3DModel` +- **batch_size 隐式假设为 1** + - `_generator_forward`, `_dmd_forward`, `faker_score_forward` 里对 + `add_noise(...).unflatten(0, (1, T))` 的写法 +- **uncond embedding 的初始化路径不显式** + - 依赖 `_log_validation` 运行过才能构造 `unconditional_dict` +- **student lr scheduler 的 step 粒度** + - 当前按 “训练 step” 走，而不是按 “generator optimizer step” 走 +- **MoE/dual-transformer 的选择与更新逻辑不统一** + - teacher/critic 有 boundary 选择逻辑 + - student 的 optimizer 选择逻辑在 DMD 与 self-forcing 两条路径不一致 From 98a1d531d06d6b81b0099e0120c09815950a552e Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 07:30:42 +0000 Subject: [PATCH 037/214] read fastgen --- dev/fastgen_structure.md | 268 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 dev/fastgen_structure.md diff --git a/dev/fastgen_structure.md b/dev/fastgen_structure.md new file mode 100644 index 000000000..0421e4a3a --- /dev/null +++ b/dev/fastgen_structure.md @@ -0,0 +1,268 @@ +# FastGen 的 distillation 设计梳理（可借鉴点） + +这份文档是我在本机 `~/alex/FastGen` 仓库里阅读 distillation 相关代码后的结构总结， +重点关注 “架构设计优点/可复用模式”，用于反哺 FastVideo 的 distill 重构。 + +> 结论先行：FastGen 把 distillation 的关键复杂度分摊到了 +> `Trainer(基础设施)` / `Method(算法+多网络训练)` / `Network(架构+采样)` / +> `Dataset(数据与conditioning供给)` 四层，并用 config + callback + checkpoint +> 把它们解耦起来。 + +## 1. FastGen 的核心分层（强推荐） + +FastGen 的高层调用链可以简化成： + +``` +train.py + -> load config (python config + Hydra overrides) + -> instantiate(config.model_class) # "method" = FastGenModel 子类 + -> Trainer(config).run(model) + -> (DDP/FSDP wrap) + callbacks + checkpointer + -> dataloader + preprocess_data + -> for iter: + loss_map, outputs = model_ddp.single_train_step(data, iter) + backward(total_loss) + model.optimizers_schedulers_step(iter) + model.optimizers_zero_grad(iter) +``` + +这套结构的关键点在于： + +- **Trainer 永远只认识一个接口：`single_train_step()` + `total_loss`** +- “distillation 算法/多网络/多优化器/交替更新”等复杂逻辑都封装进 **method 类** +- **network 类只负责 forward / noise schedule /（可选）sample**，并提供少量 hook +- dataset 负责把 `real/condition/neg_condition` 等输入准备齐全（甚至可以预存为常量） + +这种分层特别适合你想做的 “`models={teacher, student, critic,...}` + 算法解耦”。 + +## 2. 仓库组织方式（distill 相关） + +FastGen 的 repo 结构（与 distill 相关部分）： + +- `train.py`：入口，加载 config，实例化 `model_class`，调用 `Trainer.run` +- `fastgen/trainer.py`：通用训练循环（DDP/FSDP、grad accum、validate、save/load、callbacks） +- `fastgen/methods/`：**训练方法/算法**（distill 的主体逻辑在这里） + - `methods/model.py`：`FastGenModel` 基类（多网络训练接口、precision、EMA、采样 loop 等） + - `methods/distribution_matching/dmd2.py`：DMD2（student/teacher/fake_score + 可选 discriminator） + - `methods/distribution_matching/causvid.py`：CausVid（因果 video distill，复用 DMD2 框架） + - `methods/distribution_matching/self_forcing.py`：Self-Forcing（继承 CausVid/DMD2，仅改 student rollout） + - 其它：CM/sCM/TCM/MeanFlow、KD、SFT 等（同一接口体系） +- `fastgen/networks/`：架构实现（EDM/SD/Wan/CogVideoX/Cosmos…） + - `networks/network.py`：`FastGenNetwork` / `CausalFastGenNetwork` 抽象接口 +- `fastgen/datasets/`：数据加载（webdataset + 预计算 latent/embedding） + - `datasets/wds_dataloaders.py`：支持 `files_map/presets_map` 注入常量 conditioning（比如 neg prompt） +- `fastgen/configs/`：配置系统（方法 config、实验 config、网络 config、数据 config） +- `fastgen/utils/`：基础设施（instantiate、distributed ddp/fsdp、checkpointer、logging、autoresume） +- `fastgen/callbacks/`：回调系统（EMA、grad clip、wandb、profiler、param count…） + +## 3. 配置系统：python config + LazyCall + Hydra override（优秀） + +FastGen 采用： + +- `attrs` 定义结构化 config（如 `BaseConfig/BaseModelConfig/BaseTrainerConfig`） +- `OmegaConf/DictConfig` 存储 LazyCall 结构并支持 object +- Hydra 的 override 语法做命令行参数覆盖：`python train.py --config=... - key=value` +- 训练启动时把 resolve 后的 config 序列化为 `config.yaml`，保证可复现 + +关键模式： + +1) **三层 config 分离** + +- `configs/config.py`：BaseConfig（训练通用字段） +- `configs/methods/config_*.py`：方法级默认参数（比如 DMD2 要 fake_score_optimizer） +- `configs/experiments/**/config_*.py`：具体实验（比如 WanT2V 的 input_shape、t_list、lr） + +2) **LazyCall/instantiate（“延迟实例化”）** + +- config 内用 `LazyCall` 记录 `_target_` + kwargs +- 真正创建对象统一走 `instantiate(cfg)` + +价值： + +- algorithm/network/dataloader/callback 全部可插拔 +- method 和 trainer 都不需要硬编码具体类名 + +## 4. Trainer：完全算法无关的训练循环（非常干净） + +`fastgen/trainer.py` 的设计要点： + +- **只要求模型实现 `single_train_step(data, iteration)`** +- backward 统一对 `loss_map["total_loss"]` 做 +- grad accumulation 通过： + - DDP：`ddp_sync_grad(model_ddp, sync_grads)` + - FSDP：`fsdp_sync_grad(model, sync_grads)` +- optimizer step / scheduler step 通过调用 model 的接口完成： + - `model.optimizers_schedulers_step(iteration)` + - `model.optimizers_zero_grad(iteration)` +- validate 可以复用 `single_train_step`（no_grad + autocast），并且支持 + `global_vars_val` 在一次 validation 中跑多个设置（例如限制 `MAX_VAL_STEPS`） + +这个框架天然支持你想要的 “算法与模型解耦”：Trainer 永远不关心 roles。 + +## 5. Method（算法层）：用“对象”承载多网络与更新策略（关键借鉴点） + +### 5.1 FastGenModel（统一训练接口 + 多网络容器） + +`fastgen/methods/model.py::FastGenModel` 承担了大量“应放在 distill 框架层”的事： + +- precision / AMP / FSDP precision 管理（`precision`, `precision_amp`, `precision_fsdp`） +- teacher 构建与冻结（`build_teacher()`） +- student 初始化（可从 teacher 或单独 ckpt）+ EMA 初始化（`_setup_ema()`） +- inference 的统一入口（`generator_fn` / `_student_sample_loop` / `sample`） +- checkpoint 需要的统一映射： + - `model_dict` / `optimizer_dict` / `scheduler_dict` + - `fsdp_dict`（决定哪些 module 要被 FSDP sharding；可选择把 teacher 放进去） + +> 这几乎就是你想要的 `ModelBundle/ModelHandle`，只是 FastGen 的实现把它融合在 +> `FastGenModel` 类里（OOP 风格）。 + +### 5.2 DMD2Model：多优化器/交替更新，Trainer 完全不需要知道 + +`fastgen/methods/distribution_matching/dmd2.py::DMD2Model` 的设计非常值得抄： + +- 模型组成： + - `net`：student（训练的 generator） + - `teacher`：冻结 + - `fake_score`：critic（训练） + - `discriminator`：可选（GAN loss 时训练） +- **交替更新策略**由 config 控制：`student_update_freq` +- 关键：通过覆写 `get_optimizers()` / `get_lr_schedulers()` 实现 + “每个 iteration step 哪些 optimizer/scheduler 走一步” + +伪代码大概是： + +```python +if iter % student_update_freq == 0: + optimizers = [net_optimizer] +else: + optimizers = [fake_score_optimizer, (optional) discriminator_optimizer] +``` + +这种方式的价值： + +- 训练 loop 不会膨胀（Trainer 永远固定） +- method 可以自由扩展更多 role/更多 optimizer，而不用改 Trainer +- scheduler 的 step 粒度天然与 optimizer step 对齐（避免 update ratio 引入 lr schedule 偏差） + +### 5.3 SelfForcingModel：继承链复用算法框架，只替换 student rollout + +Self-forcing 在 FastGen 里是： + +``` +SelfForcingModel -> CausVidModel -> DMD2Model -> FastGenModel +``` + +它的主要改动非常克制： + +- 不改 Trainer，不改 DMD2 的 loss 框架 +- 只覆写 `gen_data_from_net()`：从普通 student forward 换成 + `rollout_with_gradient()`（blockwise causal rollout，只有 exit step 保留梯度） +- rollout 的随机 exit steps 用广播同步（rank0 采样，其他 rank broadcast） +- KV cache 用 **network 内部缓存**（`CausalFastGenNetwork.clear_caches()`）， + method 侧只调用 `store_kv=True/False` 的 forward + +这体现出非常强的“算法复用”能力：Self-forcing 只是 DMD2 的一个 student 采样策略。 + +## 6. Network 抽象：统一 forward contract + causal 扩展（非常实用） + +`fastgen/networks/network.py` 的接口设计对 distill 非常友好： + +- `FastGenNetwork.forward(x_t, t, condition=..., fwd_pred_type=..., feature_indices=...)` + - 允许 method 侧用统一的方式拿到： + - x0/eps/flow 等不同 pred_type + - 中间 features（给 discriminator） + - logvar（给某些 consistency/uncertainty 变体） +- noise schedule 在 network 内统一（`self.noise_scheduler`），method 层不用各写一遍 +- `CausalFastGenNetwork` 增加： + - `chunk_size/total_num_frames` + - `clear_caches()` 抽象，明确缓存生命周期 + +另一个小但很关键的点： + +- `FastGenModel._student_sample_loop` 是通用 multistep loop， + 但如果 network 实现了 `preserve_conditioning(x, condition)`， + loop 会自动调用它来保留 I2V / V2W 的 conditioning 帧/掩码（避免 loop 被各种模型特例污染） + +## 7. 数据/conditioning 供给：把 uncond/neg prompt 变成“数据常量”（非常推荐） + +FastGen 的 WebDataset loader（`datasets/wds_dataloaders.py`）提供了两个很棒的能力： + +- `files_map`：把某些 key 从外部文件加载为常量（每个 batch 都带上） + - 典型用法：`neg_condition`（negative prompt embedding）从 `.npy` 读取一次 +- `presets_map`：把某些 key 直接用预设常量填充（比如 WAN 的负面 prompt 字符串） + +这带来的直接收益： + +- CFG/uncond 的输入不依赖 “validation 是否跑过” 或 “训练时是否临时 encode” +- 能把 expensive 的 negative prompt embedding 预先算好并缓存 +- 对 offline / 大规模训练更友好（避免每 step 重新 encode） + +`Trainer.preprocess_data()` 进一步提供可选的在线 encode： + +- 若 batch 里是 raw video/image/text，则自动用 network 内的 + `vae/text_encoder/image_encoder` 编码成 latent/embeddings +- 同时保留 `*_raw` 字段，便于日志/可视化 + +这等价于把“pipeline stage”做成一个轻量的 preprocessing hook，集中在 Trainer。 + +## 8. 分布式与 checkpoint：围绕 `model_dict` 泛化（非常可维护） + +### 8.1 DDP training_step 的包装（很巧） + +`fastgen/utils/distributed/ddp.py::DDPWrapper` 做了一个小技巧： + +- 训练逻辑在 `single_train_step`，但 DDP 的 hook 是绑定在 `forward()` 上的 +- wrapper 临时把 `module.forward` 指到 `single_train_step`，然后调用 `self(...)` + 以触发 DDP 的 forward/backward hook + +这让 “训练 step 不是 forward” 的设计依然能吃到 DDP 的正确行为。 + +### 8.2 FSDP2：支持 meta-init 的内存友好加载 + +`FastGenModel._get_meta_init_context()` + `utils/distributed/fsdp.py::model_to_fsdp` 支持： + +- 非 rank0 在 `torch.device("meta")` 上构建大模型（几乎零内存） +- rank0 负责加载权重 +- FSDP wrap 后通过 `sync_module_states` 广播权重到所有 rank + +对于 10B+ 模型，这个设计非常关键：大幅降低启动时间与 I/O contention。 + +### 8.3 Checkpointer：同一套保存/加载适配 DDP 与 FSDP + +`utils/checkpointer.py`： + +- 非 FSDP：rank0 写单个 `.pth`（包含 model/optim/scheduler/grad_scaler/callbacks/iteration） +- FSDP：用 `torch.distributed.checkpoint` 分别保存每个 `model_dict` key 的 + sharded state（例如 `.net_model`、`.fake_score_model`），并且把 + scheduler/grad_scaler/callbacks/iteration 仍写到 `.pth` + +这种按 `model_dict` key 泛化的 checkpoint 方式，对多网络 distill 非常自然。 + +## 9. 对 FastVideo 的直接启发（建议落地清单） + +结合你要做的 “`models={teacher, student, critic, ...}` + 算法解耦”，FastGen +给出的可落地模式： + +1) **Trainer 只依赖统一接口** +- 固化为：`algorithm/model.single_train_step()` 返回 `loss_map["total_loss"]` +- 其余都放到算法层：update ratio、多优化器 step、cache 管理等 + +2) **把多网络/多 optimizer 的“调度”做成可覆盖函数** +- `get_optimizers(iter)` / `get_lr_schedulers(iter)` 的模式非常清晰 +- 这比在 Trainer 里写 if/else 或在 pipeline 里复制 train_loop 更可维护 + +3) **role → checkpoint 的映射要结构化** +- 借鉴 `model_dict/optimizer_dict/scheduler_dict` 的思路： + `models` 映射天然就是 checkpoint 的 namespace + +4) **uncond/negative conditioning 不要依赖 validation** +- 学 FastGen：把 `neg_condition` 做成 dataset 常量（files_map/presets_map） + 或者训练开始时一次性 encode 并缓存/广播 + +5) **network 抽象要显式支持 distill 需求** +- forward contract 支持 `fwd_pred_type`、features、可选采样 +- causal network 明确 cache 生命周期（`clear_caches()`） + +6) **大模型加载要考虑 meta-init + FSDP2** +- 如果 FastVideo 也要跑 14B/更大 teacher，多机多卡启动成本会很明显 +- meta-init + rank0 broadcast 是成熟方案 From ace0cac7e9c8e2d602e5f9fd996515341a302de7 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 07:41:27 +0000 Subject: [PATCH 038/214] designing --- dev/design.md | 552 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 331 insertions(+), 221 deletions(-) diff --git a/dev/design.md b/dev/design.md index cb31a9693..7d5750700 100644 --- a/dev/design.md +++ b/dev/design.md @@ -1,325 +1,435 @@ -# Distill 重构设计草案：`models={teacher, student, critic, ...}` + 算法/模型解耦 +# Distill 重构设计（吸收 FastGen 架构）：`models={...}` + Method/Trainer/Adapter 解耦 -这份文档的目标是：在不牺牲当前训练基础设施（分布式、tracker、checkpoint、 -VSA/VMoBA 等）的前提下，把 distill 训练部分重构成： +本文是基于： -1. **模型输入是显式的 `models` 映射**（teacher/student/critic/...），critic 可选 -2. **distill 算法与具体模型解耦**：算法通过 “pipeline adapter/capability” 来 - 调用模型，而不是写死 Wan 的张量布局/归一化/输入 key -3. **易扩展**：新增算法或新增模型架构，只需要新增一个 strategy 或 adapter， - 不需要到处改训练 loop +- FastVideo 当前 distill 实现：`dev/distill_structure.md` +- FastGen distillation pipeline 的优秀结构：`dev/fastgen_structure.md` -下面所有 “设计” 都会同时写 “原因”，方便取舍。 +做出的“面向落地”的重构设计草案。重点是把 **算法**（DMD2/Self-forcing/…） +与 **模型/管线**（Wan/其它架构）彻底解耦，并让训练循环（Trainer）保持长期稳定。 -## 一句话总结（推荐的最终形态） +--- -- `DistillTrainer`（统一训练 loop） + `DistillAlgorithm`（DMD2/SelfForcing/…） - + `DistillAdapter`（把具体 pipeline 适配成统一接口） + `ModelBundle` - （按角色组织的模型对象/optimizer/EMA）。 +## 0. TL;DR（推荐最终形态） -## 现状痛点（为什么要重构） +把 FastGen 的四层结构迁移到 FastVideo，并显式引入 `models={...}`： -> 这里只列与架构强相关的点；代码细节见 `dev/distill_structure.md`。 +- `DistillTrainer`：只做训练基础设施（循环、分布式、grad accum、logging、ckpt、validate） +- `DistillMethod`：一个“可训练对象”，封装 distill 算法 + 多角色模型 + 多优化器/交替更新 +- `DistillAdapter`：把具体 pipeline/network 适配成统一的 noise/forward/CFG/cache 接口 +- `ModelBundle`：`models={student, teacher, critic, ...}` 的统一容器（含 optim/ema/fsdp 策略） +- `ConditioningProvider`（或 dataset 常量注入）：显式提供 `neg_condition` 等 conditioning 常量 -- distill 逻辑 **Wan 耦合严重**（normalize / override cls name / input layout） -- DMD 与 Self-forcing 两套 pipeline 各自维护一份训练 loop，重复且容易 drift -- “需要 teacher/critic/CFG/uncond embedding”等约束不显式，依赖隐含流程 -- MoE/dual-transformer（`transformer_2`）的选择、更新在不同 pipeline 里不统一 -- 想支持更多模型（Hunyuan/LTX2/LongCat/…）或更多 distill 算法会越来越难 +关键原则：**Trainer 不认识 teacher/critic，也不写 DMD/SF 的 if/else。** -## 设计目标与非目标 +--- -### 目标 +## 1. 现状与痛点（FastVideo） -- 用 `models: dict[str, ...]` 显式描述参与训练的所有模型角色 -- 算法（DMD2/Self-forcing/未来更多）只依赖统一接口，不依赖 Wan 特有细节 -- 训练 loop 统一（分布式/日志/checkpoint/validation 只实现一次） -- 在 adapter 层支持多种输入来源： - - 已 preprocess 的 parquet（含 `vae_latent`/`text_embedding`） - - 或者（可选）原始数据在线 encode（由 pipeline 决定） +（细节见 `dev/distill_structure.md`，这里仅列架构层面的痛点） -### 非目标（第一阶段先不做） +- **Wan 耦合**：normalize/layout/transformer override 等散落在训练代码里，换模型不可能仅靠配置 +- **算法分叉**：DMD2 与 Self-forcing 各自维护训练 loop，扩展新算法/新模型成本高且容易 drift +- **conditioning 隐式依赖**：`neg_condition/uncond` 可能依赖 validation 初始化等副作用 +- **多网络调度不稳**：交替更新时，scheduler step 与 optimizer step 不严格对齐（会引入 lr 偏差） +- **专家/双 transformer 逻辑分散**：MoE/expert 选择与“哪个 timestep 更新哪个 expert”缺乏单点抽象 -- 不强求“一套 adapter 自动适配所有模型”，允许为不同架构写显式 adapter -- 不强求把所有训练 pipeline（finetune、matrixgame、ltx2 等）一起重构掉 -- 不追求立刻改变 checkpoint 格式（可以先兼容旧格式再演进） +--- -## 总体架构（模块分层） +## 2. FastGen 架构要点（我们要吸收什么） -建议把 distill 拆成四层，从下到上： +### 2.1 FastGen 的四层分离（核心） -1. **Adapter 层**（模型相关） - - 把某个 pipeline/transformer 的 forward 细节封装起来 -2. **Algorithm/Strategy 层**（算法相关） - - DMD2 / Self-forcing / 其它 distill 算法 -3. **Trainer/Engine 层**（基础设施） - - 分布式、seed、grad accumulation、优化器 step、日志、checkpoint、验证 -4. **CLI/Entrypoint 层** - - 解析参数、加载 models、选择算法、启动 trainer +FastGen 把 distillation 的复杂度拆成： -## 关键设计决策（每项含原因） +1. `Trainer`：训练循环与分布式/accum/validate/ckpt/callbacks +2. `Method(FastGenModel)`：算法 + 多网络容器 + 多优化器调度（交替更新在这里） +3. `Network`：统一 forward contract（pred_type/features/cache），并提供少量 hook +4. `Dataset/Preprocess`：提供 `real/condition/neg_condition`，并支持常量注入 -### 设计 1：用 `ModelBundle` 统一承载多角色模型（核心） +这种结构的长期价值是：**训练循环不随算法增长而膨胀**，算法复用与组合能力强。 + +### 2.2 FastGen → FastVideo 对照表（建议直接照搬） + +| FastGen 概念 | FastVideo 目标概念 | 说明 | +|---|---|---| +| `Trainer.run(model)` | `DistillTrainer.run(method)` | Trainer 只依赖统一接口 | +| `FastGenModel.single_train_step()` | `DistillMethod.single_train_step()` | 返回 `loss_map["total_loss"]` | +| `get_optimizers()/get_lr_schedulers()` | 同名接口 | 交替更新/多 optim 的唯一入口 | +| `model_dict/optimizer_dict/scheduler_dict` | 同名映射 | checkpoint 以 role 命名空间泛化 | +| `DDPWrapper.single_train_step()` | `DDPTrainStepWrapper`（可选） | 让训练 step 吃到 DDP hooks | +| dataset `files_map/presets_map` | dataset 常量注入（推荐） | `neg_condition` 不再隐式依赖 | +| callbacks + checkpointer + autoresume | 回调 + checkpoint + resume | 基础设施通用化，算法不介入 | +| meta-init + rank0 broadcast（FSDP2） | 可选的大模型加载策略 | teacher/critic ≥10B 时显著收益 | + +### 2.3 我们希望复制的“具体模式”（不止抽象名词） + +- **统一训练入口**：Trainer 每个（micro）step 都只做： + - `loss_map, outputs = method_ddp.single_train_step(batch, iter)` + - backward 只对 `loss_map["total_loss"]` + - 在 accum 最后一步调用： + - `method.optimizers_schedulers_step(iter)` + - `method.optimizers_zero_grad(iter)` +- **交替更新收敛到 method**：更新 student/critic 的比例完全由 + `get_optimizers(iter)` 决定，Trainer 永不写 role-aware 的分支。 +- **conditioning 显式化**：`neg_condition` 最好是 dataset 常量（或启动时一次性缓存/广播）， + 绝不依赖 validation 副作用。 +- **role 命名空间 checkpoint**：把保存/加载做成 “按 role key 的映射”，未来加模型不会改协议。 + +--- + +## 3. 总体架构（FastVideo 版本） + +### 3.1 一次训练的总数据流（推荐） + +```text +CLI/YAML config + -> build ModelBundle(models={student, teacher, critic?, ...}) + -> build DistillAdapter.from_pipelines(bundle) # pipeline/network 适配 + -> build DistillMethod(adapter, bundle, method_cfg) + -> DistillTrainer(trainer_cfg, callbacks, checkpointer).run(method) +``` + +### 3.2 分层职责（把边界画清楚） + +1. **Data/Conditioning 层** + - dataloader 输出：`real`、`condition`、`neg_condition`（可选）以及 I2V/V2V 的额外条件 + - `ConditioningProvider`：若 dataloader 不提供 `neg_condition`，则构建并缓存/广播 + +2. **Adapter/Network 层（模型相关）** + - `DistillAdapter`：layout/normalize/noise schedule/CFG/forward/(可选)cache + - 每个架构一个 adapter：`WanAdapter`、`HunyuanAdapter`、`LTX2Adapter`… + +3. **Method 层（算法相关 + 多网络训练）** + - `DistillMethod` 基类（FastGenModel analog） + - `DMD2Method` / `SelfForcingMethod` /（未来）`TeacherOnlyMethod` 等 + +4. **Trainer/Engine 层（基础设施）** + - `DistillTrainer.run(method)`：DDP/FSDP、grad accum、日志、验证、断点、回调 + - Trainer 永不写 DMD/SF 专有逻辑 + +--- + +## 4. 核心对象与接口（建议 API） + +### 4.1 `ModelBundle`：角色显式化（外部输入） + +目标：让入口层显式传入 `models={student, teacher, critic, ...}`，并把所有 +“训练态（optim/ema/fsdp 策略）”结构化地挂在 role 下。 + +```text +ModelBundle + roles: dict[str, RoleHandle] # key == "student"/"teacher"/"critic"/... + +RoleHandle + modules: dict[str, nn.Module] # e.g. {"transformer": ..., "transformer_2": ...} + frozen: bool + precision: optional # bf16/fp16/fp32 + fsdp_policy: optional # shard strategy / ignored modules + ema: optional + optimizers/schedulers: optional +``` + +约定： + +- canonical roles：`student`, `teacher`, `critic` +- optional roles：`discriminator`, `reward`, `refiner`, `aux_teacher`, ... + +### 4.2 `DistillAdapter`：把 pipeline/network 适配成算法可消费接口 + +adapter 的职责是“怎么调用模型”，而不是“什么时候更新谁”。建议接口包含： + +- noise & target： + - `add_noise(x0, noise, t) -> x_t` + - `pred_to_x0(pred, x_t, t)`（或统一为 `pred_to_target`） +- forward： + - `forward(role, x_t, t, cond, *, fwd_pred_type=..., neg_cond=None, cfg=None, caches=None)` +- layout & normalize（按模型需要）： + - `to_model_layout(x)` / `from_model_layout(x)` + - `normalize_latents` / `denormalize_latents` +- conditioning： + - `encode_condition(raw_cond) -> cond` + - `encode_neg_condition(raw_neg_cond) -> neg_cond`（或由 dataset 提供 embedding） +- cache（可选，Self-forcing 用）： + - `supports_kv_cache` + - `clear_caches(role=...)` + +此外建议 adapter 暴露 capabilities，避免 method 靠 if/else 猜： + +```text +adapter.capabilities = { + "supports_cfg": True/False, + "supports_kv_cache": True/False, + "supported_pred_types": {...}, + "supports_features": True/False, + "supports_expert_routing": ..., +} +``` + +### 4.3 `DistillMethod`：算法 + 多网络 + 多优化器调度（核心） + +这是 FastGen 最值得抄的点：把 distill 的关键复杂度集中在 method。 + +**最小接口（强制）** + +- `single_train_step(batch, iteration) -> (loss_map, outputs)` + - `loss_map` 必须包含 `total_loss` + - `outputs` 仅用于日志/验证/可视化 +- `get_optimizers(iteration)` / `get_lr_schedulers(iteration)` + - 返回本次 iteration 应该 step 的 optimizer/scheduler 列表（交替更新就在这里实现） +- `optimizers_schedulers_step(iteration)` / `optimizers_zero_grad(iteration)` + - Trainer 只调用它们，不关心内部有哪些 optimizer +- `model_dict/optimizer_dict/scheduler_dict` + - 给 CheckpointManager 使用（key == role 或 role 内模块） + +**建议能力（可选但推荐）** + +- `autocast()` / `grad_scaler`：统一 AMP 管理，Trainer 不关心精度细节 +- `sample_for_logging(...)`：返回可调用的采样函数或采样结果，Trainer 不写采样逻辑 +- `set_trainable(role, enabled)`：method 内部统一 `requires_grad_` 切换（Self-forcing/critic alternation） + +> 直接收益：scheduler step 的粒度天然与 optimizer step 对齐，避免 update ratio 引入 lr 偏差。 + +### 4.4 `DistillTrainer`：完全算法无关的训练循环 + +Trainer 只依赖 method 的统一接口，推荐对齐 FastGen 的关键形态： + +- grad accumulation：Trainer 计算 `sync_grads`，并在 DDP/FSDP 下用 context 禁止同步 +- forward/backward：只围绕 `loss_map["total_loss"]` +- step/zero_grad：只在 accum 最后一次调用 method 接口 +- validate：可复用 `single_train_step`（no_grad + autocast），并允许 method 扩展额外 eval +- callbacks：把 EMA / grad clip / logger / profiler 等都做成回调（可保存状态） + +**DDP 的一个关键实现点（强烈建议照 FastGen）** + +如果 `single_train_step` 不是 `forward()`，DDP 的隐式 hooks 可能不生效。 +FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step`， +然后通过 `ddp_model(*args)` 触发 hooks。 + +在 FastVideo 落地时建议二选一： + +1) `DistillMethod.forward = single_train_step`（简单，但 forward 被占用） +2) 实现一个 `DDPTrainStepWrapper.single_train_step()`（推荐，行为更明确） + +### 4.5 `CheckpointManager`：围绕 role 命名空间泛化 + +建议统一协议： + +- 输入：`model_dict/optimizer_dict/scheduler_dict/grad_scaler/callback_state/iteration` +- 输出：按 role key 保存（尤其是 FSDP sharded state） + +并显式支持： + +- **只导出 student** 的 inference 权重（teacher/critic 不随推理包分发） +- **兼容旧 distill ckpt**：至少提供一次性迁移脚本或兼容 loader + +### 4.6 `Callback` 系统：把“训练周边复杂度”解耦出去 + +把这些都做成 callbacks（并进入 checkpoint），Trainer/Method 都不硬编码： + +- EMA 更新 +- grad clipping +- logging（wandb/tensorboard/本地） +- profiler/step timer +- param count / debug dumps +- autoresume（从 ckpt 恢复 callback 状态） + +--- + +## 5. 关键设计决策（每条含原因） + +### 设计 1：引入 `DistillMethod`（Method 中心，而非 Algorithm/Trainer 中心） **设计** -- 引入一个只做 “容器” 的数据结构（建议 dataclass）： +- DMD2/Self-forcing/未来算法都实现为 `DistillMethod` 子类 +- Method 负责多角色模型、交替更新、optim/sched/EMA、缓存生命周期等 + +**原因** + +- distill 的“本质复杂度”就是多网络 + 多优化器调度；放在 Method 最自然 +- Trainer 只需要稳定地做基础设施，长期维护成本最低 - - `models: dict[str, ModelHandle]` - - 推荐 canonical keys：`student`, `teacher`, `critic` - - 允许扩展：`reward`, `refiner`, `aux_teacher`, `student_ema` 等 +### 设计 2：`models={...}` 显式输入 + `ModelBundle` 结构化承载训练态 -- `ModelHandle` 不是裸 `nn.Module`，而是： +**设计** - - `module`: 主要网络（如 transformer） - - `extra_modules`: 可选模块（如 `transformer_2`、image encoder 等） - - `optimizers`, `schedulers`, `ema`（可选） - - `trainable: bool` + `param_groups`（用于决定哪些参数会更新） - - `capabilities`: 模型端能力声明（见设计 3） +- 配置/CLI 显式给出 `student/teacher/critic?` +- `ModelBundle` 统一挂载冻结策略、precision、FSDP 策略、EMA、optim/sched **原因** -- 角色显式化后，算法只需要声明 “我需要哪些 role”，无需硬编码一堆参数名 -- `ModelHandle` 把 optimizer/EMA 等训练态绑定到 role，checkpoint/save/resume - 逻辑才能自然泛化 -- 对 MoE/dual-transformer 这类 “一个 role 内部有多个可训练模块” 的情况， - `extra_modules` 能承载而不污染最上层 `models` 命名空间 +- 角色显式化是解耦的前提，且天然支持未来扩展更多角色 +- checkpoint 也可以自然以 role 命名空间组织 -### 设计 2：把“加载模型”从“训练算法”中剥离成 ModelFactory/Loader +### 设计 3：Trainer 固化成“只认一个接口”的稳定循环 **设计** -- 单独实现 `ModelFactory`： - - 输入：`ModelSpec`（路径/是否冻结/precision/offload/fsdp 等） - - 输出：`ModelHandle` -- teacher/critic 只加载算法需要的最小组件（通常只要 transformer），其余组件 - （VAE/scheduler/text encoder）优先复用 student pipeline 的 shared 部分， - 由 adapter 决定是否允许共享 +- Trainer 仅调用： + - `loss_map, outputs = method_ddp.single_train_step(...)` + - backward `total_loss` + - `method.optimizers_schedulers_step()` / `optimizers_zero_grad()` +- validate 也尽可能复用 `single_train_step` **原因** -- 让算法代码完全不关心 “从哪里 load”、“如何 CPU offload”、“如何 FSDP 包装” -- 有利于做 memory/throughput 优化（teacher/critic 可以走不同策略） -- 避免现状里 `load_modules` 里混杂大量 Wan 特判/override 的情况 +- 彻底杜绝算法越写越多导致 trainer 分叉、难以测试和维护 +- validate 复用训练逻辑能减少“训练/验证 drift” -### 设计 3：用 `DistillAdapter` 做“pipeline 适配层”，并显式声明 capability +### 设计 4：交替更新/多 optimizer 调度统一走 `get_optimizers()`（FastGen 关键模式） **设计** -- 定义一个 adapter 接口（Protocol/ABC 均可），把下列能力抽象出来： - - 1. **latent 规范化/布局** - - `normalize_latents(x)` / `denormalize_latents(x)`（如需） - - `to_model_layout(x)` / `from_model_layout(x)` - 2. **conditioning 规范化** - - 把 dataset/pipeline 产生的 conditioning 变成统一结构 - （例如 `Conditioning` = dict[str, Tensor]） - 3. **噪声与 parameterization** - - `add_noise(x0, noise, t)` - - `pred_to_x0(pred, x_t, t)` 或 `pred_to_video_latent(...)` - - 声明 `prediction_type`（eps/v/x0/flow），由 adapter 负责转换 - 4. **模型 forward** - - `forward(role, x_t, t, cond, *, caches=None, return_dict=False)` - - 允许 adapter 在内部处理 `set_forward_context` / attention metadata - 5. **CFG 支持（可选）** - - `supports_cfg: bool` - - `build_uncond(cond, negative_prompt=...)` 或直接提供 - `uncond_conditioning_cache` - -- adapter 要返回一个 `Capabilities` 对象（dataclass）： - - 是否支持 CFG/uncond - - 是否支持 KV cache（self-forcing 需要） - - 是否存在/如何选择 `transformer_2`（MoE） - - 输入 key 要求（`encoder_attention_mask` 是否必须等） +- DMD2： + - `iter % student_update_freq == 0`：更新 student + - 否则更新 critic（+ 可选 discriminator） +- Self-forcing： + - 复用相同调度，只替换 student rollout **原因** -- “算法与模型解耦”只能靠明确的接口边界实现；adapter 是最合适的边界 -- capability 显式化后，算法可以做： - - `if not supports_cfg: raise/降级` - - `if supports_kv_cache: enable self-forcing` 否则 fallback -- 避免用脆弱的反射去“猜 pipeline 有哪些 stage/属性”，可维护性更高 +- update ratio 的复杂度从 trainer 中消失，且扩展更多 role 不改 trainer +- scheduler step 与 optimizer step 自动对齐（减少 lr 偏差） -### 设计 4：用 Strategy 模式承载 distill 算法（DMD2/SF/未来更多） +### 设计 5：adapter 明确 capability，而不是靠 if/else 猜 **设计** -- `DistillAlgorithm` 负责： - - 声明需要的 roles：`required_roles`, `optional_roles` - - 声明每一步要更新哪些 role（以及 update ratio/交替策略） - - 定义 loss： - - `compute_losses(batch, ctx) -> LossBundle` - - 或者更细：`losses_for(role)` + `metrics` - - （可选）维护算法内部状态（例如 self-forcing 的 cache 管理/exit flag 采样） - -- 例子： - - `DMD2Algorithm`： - - `required_roles = {student, teacher, critic}` - - loss = 当前 `_dmd_forward` + critic flow-matching - - `SelfForcingAlgorithm`： - - 基于 DMD2，但 student forward 换成 causal/self-forcing 的 trajectory - - 需要 `supports_kv_cache` + `is_causal` 等 capability - - `TeacherOnlyAlgorithm`（未来可选）： - - `required_roles = {student, teacher}` - - 不依赖 critic（满足 “critic optional” 的场景） +- adapter 暴露 `capabilities`，method 启动时检查依赖 +- 自适应训练：method 根据 capability 选择路径或报错/降级 **原因** -- 把 “训练 loop” 从 pipeline class 里抽出来后，DMD/SF 不需要两份 train() -- 新增算法不会影响 adapter/trainer，只需要加一个 strategy 类和少量配置 -- 更容易做 unit test：给一个 dummy adapter + dummy models 就能测 loss 逻辑 +- 新增模型架构时，差异集中在 adapter；method 保持稳定 +- 失败要早失败（init-time），避免跑到中途才出形状/feature 错 -### 设计 5：`DistillTrainer` 统一训练基础设施，并用“更新计划”驱动 optimizer +### 设计 6：`neg_condition` 变成“数据常量”或“启动一次性缓存/广播” **设计** -- trainer 只做基础设施： - - 数据迭代（dataloader / batch builder） - - grad accumulation + autocast + clip grad - - 按 strategy 给出的 “更新计划” 去 step 不同 optimizer - - all-reduce loss/metrics - - tracker log + checkpoint + validation hook -- trainer 不写任何 DMD/SF 专有逻辑（最多提供 hook 点） -- “更新计划”可以是： - - `UpdatePlan = list[Update(role, loss_key, optimizer_key, scheduler_key)]` - - 或者简单：`roles_to_update = {...}` + `losses` +两条路径（二选一或同时支持）： + +1) dataset 常量注入：dataloader 直接输出 `neg_condition` embedding +2) provider 缓存：训练开始时用 adapter 编码 negative prompt，并缓存/广播 **原因** -- 训练 loop 只实现一次，避免 DMD/SF drift -- update ratio（generator_update_interval / dfake_gen_update_ratio）变成算法参数， - 不再散落在不同 pipeline 里 -- 支持更多角色/更多 optimizer 组合时不会爆炸 +- 消除 “依赖 validation 初始化 uncond embedding” 这类隐式耦合 +- negative prompt embedding 通常是常量，适合缓存（性能更稳） -### 设计 6：把 distill 专有参数从 `TrainingArgs` 里拆成 `DistillConfig` +### 设计 7：用 role 命名空间做 checkpoint 协议（对齐 `model_dict`） **设计** -- `TrainingArgs` 保持偏 “训练基础设施”： - - 数据、输出目录、分布式、optimizer 基本参数、logging/checkpoint -- distill 算法专有参数放到 `DistillConfig`： - - DMD2：timestep ratio、guidance scale、denoising steps、update interval 等 - - Self-forcing：num_frame_per_block、gradient masking 等 - - Critic：fake_score lr/betas/scheduler 等 -- CLI 层把 config 做成 namespace： - - `--distill.algo dmd2` - - `--distill.dmd2.generator_update_interval 5` - - `--distill.sf.num_frame_per_block 3` +- `model_dict/optimizer_dict/scheduler_dict` 的 key 直接是 role +- FSDP 情况下按 role key 分 shard 保存；非 FSDP rank0 写单文件 **原因** -- 现在 `TrainingArgs` 已经非常大；继续塞 distill 参数会让其它训练模式更难维护 -- 分离后能清晰表达：某些参数只对某个算法生效 -- 便于做默认值管理（不同 pipeline/算法可提供不同 defaults） +- 多网络 distill 的保存/加载本质就是 “多个命名空间的 state” +- 未来新增/删减 role 不改变 checkpoint 顶层协议 -### 设计 7：checkpoint 以 role 为单位泛化（并提供推理导出） +### 设计 8：DDP 训练 step 触发 hooks（借鉴 FastGen 的 wrapper 技巧） **设计** -- 引入 `CheckpointManager`： - - `save(role_states, shared_states, step)` - - `load(...)` -- role_states 来自 `ModelHandle`： - - trainable role 保存 optimizer/scheduler/ema - - frozen role（teacher）通常只保存 path 或权重 hash（可选） -- “推理导出”是一个独立通道： - - 例如只导出 `student`（和可选 `student.transformer_2`）到 diffusers 格式 +- 为 DDP 场景提供 `DDPTrainStepWrapper.single_train_step()`： + 临时重定向 `forward` 到 `single_train_step`，再调用 `ddp_model(...)` **原因** -- 现有 `save_distillation_checkpoint` 已经在 role 粒度上开始泛化 - （generator/critic/generator_2/critic_2/real_score_2），继续泛化会更自然 -- 未来支持更多角色时（reward/refiner）不需要再复制粘贴一套 save/load +- 让 “训练 step != forward” 的结构仍能享受 DDP 的正确行为与 hooks +- 避免强行把算法逻辑写进 `forward` 导致语义混乱 -### 设计 8：把 “uncond conditioning” 做成显式的 ConditioningProvider +### 设计 9（可选）：meta-init + rank0 broadcast 的大模型加载 **设计** -- 对需要 CFG 的算法，提供一个 `ConditioningProvider`： - - 在训练开始时就构建/缓存 negative prompt embedding（或从 dataset 读取） - - 不依赖 “是否开启 validation logging” -- provider 与 adapter 配合： - - adapter 负责怎么 encode negative prompt（模型不同，编码方式不同） - - provider 负责生命周期与 cache（rank0 广播、避免重复算） +- teacher/critic ≥10B 时，非 rank0 以 `torch.device("meta")` 构建空权重 +- rank0 加载权重，FSDP wrap 后 broadcast **原因** -- 现状里 uncond embedding 依赖 `_log_validation`，属于隐式耦合，容易踩坑 -- provider 显式后，算法与 trainer 的依赖更清晰，validation 也可以变成可选 +- 显著减少多机启动 I/O contention 与峰值内存 -## CLI 形态建议（与 `models={...}` 对齐） +--- -### 推荐参数形式 +## 6. 配置与 CLI 形态（渐进式） + +### 6.1 最小可用（建议先落地） - `--models.student ` - `--models.teacher ` - `--models.critic `（可选） +- `--distill.method dmd2|self_forcing|teacher_only` + +distill 专有参数建议用 namespace： -如果需要支持更复杂的配置（比如每个 role 的 precision/offload）： +- `--distill.dmd2.student_update_freq 5` +- `--distill.dmd2.guidance_scale 3.5` +- `--distill.sf.num_frame_per_block 3` + +### 6.2 复杂配置（建议支持） - `--models_json path/to/models.json` + - per-role precision/offload/trainable/fsdp_policy/ckpt_path 等 -示例 JSON（建议）： +### 6.3 配置系统演进（可选吸收 FastGen 的优点） -```json -{ - "student": {"path": "wlsaidhi/SFWan2.1-T2V-1.3B-Diffusers", "trainable": true}, - "teacher": {"path": "Wan-AI/Wan2.1-T2V-14B-Diffusers", "trainable": false}, - "critic": {"path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "trainable": true} -} -``` +FastGen 的 python config + instantiate + override 很优秀，但 FastVideo 现阶段可以先： -**原因** +- 保留现有 YAML/argparse +- 内部把配置整理成结构化 dataclass： + - `TrainerConfig / DataConfig / MethodConfig / ModelsConfig` +- 后续再逐步引入“延迟实例化/override”能力（不阻塞 distill 重构） -- 直接映射到 `models: dict[str, ModelSpec]`，减少 “某个 role 的 path 到底对应哪个 - CLI 参数” 的歧义 -- JSON 形式在复杂场景下更可扩展（多模块、MoE、不同 offload 策略等） +--- -## 迁移计划（推荐分阶段） +## 7. 目录落地建议（FastVideo 内） -### Phase 0：只加新框架，不动现有入口 +建议新增 `fastvideo/distill/`（或 `fastvideo/training/distill/`），结构对齐 FastGen： -- 新增 `fastvideo/distill/`（或 `fastvideo/training/distill/`）： - - `models.py`：`ModelSpec/ModelHandle/ModelBundle` - - `adapters/`：`WanDistillAdapter`（先只支持 wan） - - `algorithms/`：`DMD2Algorithm`、`SelfForcingAlgorithm` - - `trainer.py`：`DistillTrainer` -- 先用单元测试覆盖核心 loss（不用真模型，dummy adapter 即可） +- `fastvideo/distill/bundle.py`：`ModelBundle/RoleHandle/RoleSpec` +- `fastvideo/distill/adapters/`：`WanAdapter`（先支持 Wan）、后续扩展更多模型 +- `fastvideo/distill/methods/`：`base.py`、`dmd2.py`、`self_forcing.py` +- `fastvideo/distill/trainer.py`：`DistillTrainer` +- `fastvideo/distill/checkpoint.py`：`CheckpointManager`（先兼容旧格式） +- `fastvideo/distill/callbacks/`：EMA/clip/log/profiler 等 -**原因** +旧入口（如 `fastvideo/training/*distillation_pipeline.py`）先保留， +通过 flag 切新旧框架做 A/B 对齐。 -- 风险最小：不影响现有脚本/训练 -- 先把边界（adapter/strategy/trainer）跑通，后面迁移才不痛 +--- -### Phase 1：把现有 Wan DMD / Wan Self-forcing 迁移到新框架（行为对齐） +## 8. 迁移计划（低风险） -- 新建一个新的入口脚本（或训练文件）： - - `fastvideo/training/distill.py`（仅示例） -- 让旧入口（`wan_distillation_pipeline.py` 等）可以选用新 trainer（通过 flag） -- 对齐： - - loss 数值 - - checkpoint 目录结构 - - validation 输出 +### Phase 0：框架落地 + Wan(DMD2) 跑通 -**原因** +- 新增 `DistillTrainer/DistillMethod/ModelBundle/WanAdapter` +- `DMD2Method` 覆盖现有 Wan distill 训练（student+teacher+critic） +- checkpoint 至少能： + - 保存/加载新格式 + - 从旧格式加载 student 初始化（兼容迁移） -- 迁移时可 A/B 对比，减少“重构引入质量回归”的概率 +### Phase 1：Self-forcing 迁移（复用 DMD2 框架） -### Phase 2：清理旧实现 + 扩展更多模型/算法 +- `SelfForcingMethod(DMD2Method)`：只覆写 student rollout / cache 生命周期 +- 对齐现有 self-forcing 输出与 loss（允许数值差异但要解释） -- 删除或冻结旧 distill pipeline（保留兼容入口也行） -- 为其它模型实现 adapter（Hunyuan/LTX2/LongCat…） -- 引入更多算法（teacher-only、multi-teacher、RLHF-style…） +### Phase 2：清理旧实现 + 扩展新模型/新算法 -**原因** +- 逐步冻结或移除旧 distill pipeline（保留兼容入口亦可） +- 新增更多 adapter（Hunyuan/LTX2/LongCat…） +- 新增更多 method（teacher-only、多 teacher、KD 轨迹蒸馏等） + +--- -- 把新增工作限制在 “写 adapter / 写 strategy”，让扩展成本线性增长 +## 9. Guardrails / 测试建议（避免重构“跑得通但不可维护”） -## 额外建议（踩坑预防） +- **scheduler step 对齐测试**：交替更新下，未 step 的 optimizer 对应 scheduler 不应 step +- **batch_size > 1**：消除所有隐式 `B==1` 的 reshape/unflatten 假设 +- **role 可选性**：critic 可选时应有清晰报错/降级路径（teacher-only） +- **conditioning 显式性**：训练开始前必须具备 `neg_condition`（来自数据或 provider） +- **checkpoint roundtrip**：save → load → loss 不发散（最小 smoke test） -- 明确 role 的训练/冻结策略：teacher 永远 `no_grad + eval`，critic/trainable - role 的 `requires_grad` 与 optimizer param group 必须绑定在一起 -- MoE/dual-transformer：建议把 “按 timestep 选择哪个 expert 更新” 的逻辑放到 - adapter 或 strategy 的单一位置，避免像现状一样分散在多处 -- lr scheduler 的 step 粒度：建议按 optimizer step，而不是按 global step - （否则 update ratio 会改变 effective schedule） From b3f9faa295f1be47489dd69119752dbc222e6c4e Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 07:58:20 +0000 Subject: [PATCH 039/214] phase 0 --- dev/design.md | 8 ++- dev/phases/phase_0.md | 132 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 dev/phases/phase_0.md diff --git a/dev/design.md b/dev/design.md index 7d5750700..b8b64553e 100644 --- a/dev/design.md +++ b/dev/design.md @@ -132,8 +132,11 @@ RoleHandle 约定： -- canonical roles：`student`, `teacher`, `critic` -- optional roles：`discriminator`, `reward`, `refiner`, `aux_teacher`, ... +- `role` 只是一个字符串 key；Trainer/Checkpoint 对所有 role **一视同仁**（不做“主次”区分）。 +- 为了可读性，推荐使用一些常见命名（非强制）： + `student`, `teacher`, `critic`, `discriminator`, `reward`, `refiner`, `aux_teacher`, ... +- 每个 `DistillMethod` 应显式声明并在初始化时校验自己需要的 roles + （例如 DMD2 需要 `student+teacher+critic`，teacher-only 只需要 `student+teacher`）。 ### 4.2 `DistillAdapter`：把 pipeline/network 适配成算法可消费接口 @@ -432,4 +435,3 @@ FastGen 的 python config + instantiate + override 很优秀，但 FastVideo 现 - **role 可选性**：critic 可选时应有清晰报错/降级路径（teacher-only） - **conditioning 显式性**：训练开始前必须具备 `neg_condition`（来自数据或 provider） - **checkpoint roundtrip**：save → load → loss 不发散（最小 smoke test） - diff --git a/dev/phases/phase_0.md b/dev/phases/phase_0.md new file mode 100644 index 000000000..98e281830 --- /dev/null +++ b/dev/phases/phase_0.md @@ -0,0 +1,132 @@ +# Phase 0：新 Distill 框架落地 + Wan(DMD2) 跑通（实践记录/执行清单） + +目标（Phase 0 的“可交付”）： + +1. 在 FastVideo 内落地一套 **Method/Trainer/Adapter/Bundle** 的 distill 框架骨架， + 代码可 import，可扩展，不影响现有 `fastvideo/training/*distillation_pipeline.py`。 +2. 用新框架跑通 **Wan 的 DMD2**（student+teacher+critic），并提供一个独立入口脚本， + 便于 A/B 对齐旧实现。 +3. 在 Phase 0 就消除一个当前硬耦合：**DMD2 所需的 uncond/neg_condition 不再依赖 validation 副作用**。 + +非目标（Phase 0 暂不强求）： + +- 完整复刻旧 pipeline 的所有日志/可视化/validation 产物（可以逐步补） +- role-based 通用 checkpoint 协议（Phase 0 先复用现有 distill ckpt utils，后续再迁移） +- 支持除 Wan 以外的模型（Phase 0 只做 `WanAdapter`） + +--- + +## 0. 关键风险与应对 + +### 风险 A：`negative_prompt_embeds` 目前只在 validation 中被初始化 + +现状：`fastvideo/training/distillation_pipeline.py` 里，`_prepare_dit_inputs()` 只有在 +`self.negative_prompt_embeds` 已经存在时才会构造 `unconditional_dict`； +而 `self.negative_prompt_embeds` 是在 `_log_validation()` 里通过 +`validation_pipeline.prompt_encoding_stage(...)` 赋值的。 + +如果不跑 validation，DMD2 会在 real_score 的 uncond forward 直接报错 +（`text_dict cannot be None`）。 + +Phase 0 方案：引入一个最小的 `ConditioningProvider`（Wan 版本），在训练开始前： + +- 从 `SamplingParam.from_pretrained(model_path).negative_prompt` 取 negative prompt 字符串 +- 用一个轻量的 prompt encoder（优先复用 `WanDMDPipeline` 的 `prompt_encoding_stage`） + 计算 `negative_prompt_embeds / negative_prompt_attention_mask` +- 缓存到 adapter/method，并确保每个 step 都能显式提供 uncond conditioning + +这一步是 Phase 0 必做，不然新 Trainer/Method 没法脱离 validation。 + +### 风险 B：Phase 0 会短期“仍然 Wan 耦合” + +为了尽快跑通 + 降低风险，Phase 0 允许 `WanAdapter` 通过 **wrap 现有 pipeline 的 helper +methods**（normalize/noise/timestep/attention metadata/build_input_kwargs 等）实现。 + +后续 Phase 1/2 再把这些 helper 从 pipeline 迁移/重写进 adapter，彻底摆脱旧实现。 + +--- + +## 1. 代码落地点（具体到文件） + +> 约定：Phase 0 把新框架放到 `fastvideo/distillation/`（目前该目录为空）。 + +### 1.1 新增 distill 框架骨架 + +- `fastvideo/distillation/__init__.py` + - 导出 Phase 0 需要的核心类（Trainer/Method/Bundle） +- `fastvideo/distillation/bundle.py` + - `RoleHandle` / `ModelBundle`：`roles: dict[str, RoleHandle]` +- `fastvideo/distillation/trainer.py` + - `DistillTrainer`：通用训练循环（grad accum + step/zero_grad），不认识 roles +- `fastvideo/distillation/methods/base.py` + - `DistillMethod` 抽象：`single_train_step()`、`get_optimizers()` 等 +- `fastvideo/distillation/adapters/base.py` + - `DistillAdapter` 抽象：`prepare_batch()`、`forward_*()`、conditioning provider hook + +### 1.2 Phase 0 的 Wan 实现（pipeline-backed，先跑通） + +- `fastvideo/distillation/adapters/wan.py` + - `WanPipelineAdapter`： + - 复用 `fastvideo/training/distillation_pipeline.py` 的 helper 方法做数据准备/forward + - 提供 `ensure_negative_conditioning()`，不依赖 validation +- `fastvideo/distillation/methods/wan_dmd2.py` + - `WanDMD2Method`： + - 实现 DMD2 的 loss 计算（generator_loss + fake_score_loss） + - 实现 update schedule（`generator_update_interval`）与 optimizer/scheduler step 对齐 + +### 1.3 独立入口（不影响旧脚本） + +- `fastvideo/training/wan_distillation_v2.py` + - 行为与 `wan_distillation_pipeline.py` 类似，但走新框架： + - 构建 `WanDistillationPipeline.from_pretrained(...)`（仅用于复用现有加载/优化器/dataloader） + - 构建 `WanPipelineAdapter` + `WanDMD2Method` + - 用 `DistillTrainer.run(...)` 启动训练 + +### 1.4 最小单测（CPU 可跑） + +- `fastvideo/tests/distillation/test_phase0_schedule.py` + - 只测 method 的 optimizer/scheduler 选择逻辑是否与 update ratio 对齐 + - 不依赖 GPU/模型权重 + +--- + +## 2. Phase 0 训练循环的行为约定（便于 A/B） + +为了尽量可对齐旧实现，Phase 0 的新 Trainer 约定： + +- global `step` 仍然按旧 pipeline 的语义：从 `init_steps+1` 开始，到 `max_train_steps` +- grad accumulation 由 Trainer 处理（每个 microbatch 调一次 `single_train_step`，最后 step） +- generator optimizer/scheduler 只在 `step % generator_update_interval == 0` 时 step + （这是对旧实现的一个显式修正：旧实现会每步 step generator scheduler） +- fake_score optimizer/scheduler 每步 step + +> 如果后续发现这个 scheduler 行为变化会影响 A/B 对齐，我们可以在 Phase 0 +> 加一个 “legacy 模式开关”；但默认先按“optimizer step 对齐 scheduler step”的正确语义实现。 + +--- + +## 3. 开始实践（本次提交会先做到什么程度） + +本次实现优先级： + +1. 新框架骨架文件可 import（`fastvideo.distillation.*`） +2. `WanPipelineAdapter.ensure_negative_conditioning()` 可在无 validation 的情况下生成 neg embeds +3. `WanDMD2Method.single_train_step()` 能产出 `loss_map["total_loss"]` +4. `DistillTrainer.run()` 能跑若干 step（最小 smoke）并 step optimizers +5. 加一个 schedule 单测，确保 `get_optimizers/get_lr_schedulers` 与 update ratio 对齐 + +后续增量（Phase 0 内可迭代）： + +- checkpoint/resume 接入（优先复用 `save_distillation_checkpoint/load_distillation_checkpoint`） +- validation/logging 接入（可先调用旧 pipeline 的 `_log_validation`） + +--- + +## 4. “大设计硬伤”停工汇报标准 + +如果在 Phase 0 实践过程中出现以下情况，我会暂停继续写代码并直接汇报你： + +- `models={...}` + adapter 的抽象无法覆盖 Wan 的关键差异（例如 conditioning/CFG 方式根本不一致） +- DMD2 的计算图要求导致 Method/Trainer 的边界必须反转（Trainer 不可算法无关） +- 现有 pipeline 的 helper 复用导致强耦合无法逐步迁移（必须一次性大重构才可跑通） + From b936be8ed7093ffe3339d4661b8c4587988dc7a1 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 08:02:29 +0000 Subject: [PATCH 040/214] progressing phase 0 --- fastvideo/distillation/__init__.py | 11 ++ fastvideo/distillation/adapters/__init__.py | 8 ++ fastvideo/distillation/adapters/base.py | 20 ++++ fastvideo/distillation/bundle.py | 39 +++++++ fastvideo/distillation/methods/__init__.py | 8 ++ fastvideo/distillation/methods/base.py | 53 +++++++++ fastvideo/distillation/trainer.py | 115 ++++++++++++++++++++ 7 files changed, 254 insertions(+) create mode 100644 fastvideo/distillation/__init__.py create mode 100644 fastvideo/distillation/adapters/__init__.py create mode 100644 fastvideo/distillation/adapters/base.py create mode 100644 fastvideo/distillation/bundle.py create mode 100644 fastvideo/distillation/methods/__init__.py create mode 100644 fastvideo/distillation/methods/base.py create mode 100644 fastvideo/distillation/trainer.py diff --git a/fastvideo/distillation/__init__.py b/fastvideo/distillation/__init__.py new file mode 100644 index 000000000..2b5d503e8 --- /dev/null +++ b/fastvideo/distillation/__init__.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 + +from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.trainer import DistillTrainer + +__all__ = [ + "DistillTrainer", + "ModelBundle", + "RoleHandle", +] + diff --git a/fastvideo/distillation/adapters/__init__.py b/fastvideo/distillation/adapters/__init__.py new file mode 100644 index 000000000..c4c72f746 --- /dev/null +++ b/fastvideo/distillation/adapters/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +from fastvideo.distillation.adapters.base import DistillAdapter + +__all__ = [ + "DistillAdapter", +] + diff --git a/fastvideo/distillation/adapters/base.py b/fastvideo/distillation/adapters/base.py new file mode 100644 index 000000000..bb6d1db0d --- /dev/null +++ b/fastvideo/distillation/adapters/base.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + +from fastvideo.pipelines import TrainingBatch + + +class DistillAdapter(ABC): + @abstractmethod + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + ) -> TrainingBatch: + raise NotImplementedError + diff --git a/fastvideo/distillation/bundle.py b/fastvideo/distillation/bundle.py new file mode 100644 index 000000000..47e6016bb --- /dev/null +++ b/fastvideo/distillation/bundle.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import torch + +RoleName = str + + +@dataclass(slots=True) +class RoleHandle: + modules: dict[str, torch.nn.Module] = field(default_factory=dict) + optimizers: dict[str, torch.optim.Optimizer] = field(default_factory=dict) + lr_schedulers: dict[str, Any] = field(default_factory=dict) + frozen: bool = False + + def require_module(self, name: str) -> torch.nn.Module: + if name not in self.modules: + raise KeyError(f"Missing module '{name}'") + return self.modules[name] + + +@dataclass(slots=True) +class ModelBundle: + roles: dict[RoleName, RoleHandle] + + def require_roles(self, roles: list[RoleName]) -> None: + missing = [role for role in roles if role not in self.roles] + if missing: + raise KeyError(f"Missing roles: {missing}") + + def role(self, role: RoleName) -> RoleHandle: + if role not in self.roles: + raise KeyError(f"Unknown role: {role}") + return self.roles[role] + diff --git a/fastvideo/distillation/methods/__init__.py b/fastvideo/distillation/methods/__init__.py new file mode 100644 index 000000000..1b9de0b13 --- /dev/null +++ b/fastvideo/distillation/methods/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +from fastvideo.distillation.methods.base import DistillMethod + +__all__ = [ + "DistillMethod", +] + diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py new file mode 100644 index 000000000..3af2e1e02 --- /dev/null +++ b/fastvideo/distillation/methods/base.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Sequence +from typing import Any + +import torch + +from fastvideo.distillation.bundle import ModelBundle + + +class DistillMethod(torch.nn.Module, ABC): + def __init__(self, bundle: ModelBundle) -> None: + super().__init__() + self.bundle = bundle + self.role_modules = torch.nn.ModuleDict() + for role, handle in bundle.roles.items(): + if handle.modules: + self.role_modules[role] = torch.nn.ModuleDict(handle.modules) + + @abstractmethod + def single_train_step( + self, + batch: dict[str, Any], + iteration: int, + *, + current_vsa_sparsity: float = 0.0, + ) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + raise NotImplementedError + + @abstractmethod + def get_optimizers(self, iteration: int) -> Sequence[torch.optim.Optimizer]: + raise NotImplementedError + + @abstractmethod + def get_lr_schedulers(self, iteration: int) -> Sequence[Any]: + raise NotImplementedError + + def optimizers_schedulers_step(self, iteration: int) -> None: + for optimizer in self.get_optimizers(iteration): + optimizer.step() + for scheduler in self.get_lr_schedulers(iteration): + scheduler.step() + + def optimizers_zero_grad(self, iteration: int) -> None: + for optimizer in self.get_optimizers(iteration): + try: + optimizer.zero_grad(set_to_none=True) + except TypeError: + optimizer.zero_grad() + diff --git a/fastvideo/distillation/trainer.py b/fastvideo/distillation/trainer.py new file mode 100644 index 000000000..e64a1d449 --- /dev/null +++ b/fastvideo/distillation/trainer.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import time +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any + +import torch +from tqdm.auto import tqdm + +from fastvideo.distributed import get_sp_group, get_world_group +from fastvideo.fastvideo_args import TrainingArgs +from fastvideo.training.trackers import BaseTracker, DummyTracker + + +@dataclass(slots=True) +class TrainLoopState: + step: int + accum_iter: int + current_vsa_sparsity: float + + +class DistillTrainer: + def __init__(self, training_args: TrainingArgs, *, tracker: BaseTracker + | None = None) -> None: + self.training_args = training_args + self.world_group = get_world_group() + self.sp_group = get_sp_group() + self.global_rank = self.world_group.rank + self.local_rank = self.world_group.local_rank + self.tracker = tracker or DummyTracker() + + def _iter_dataloader(self, dataloader: Any) -> Iterator[dict[str, Any]]: + data_iter = iter(dataloader) + while True: + batch = next(data_iter, None) + if batch is None: + data_iter = iter(dataloader) + batch = next(data_iter) + yield batch + + def _get_current_vsa_sparsity(self, step: int) -> float: + # Phase 0: keep behavior close to existing pipelines. + vsa_sparsity = self.training_args.VSA_sparsity + vsa_decay_rate = self.training_args.VSA_decay_rate + vsa_decay_interval_steps = self.training_args.VSA_decay_interval_steps + if vsa_decay_interval_steps > 1: + current_decay_times = min(step // vsa_decay_interval_steps, + int(vsa_sparsity // vsa_decay_rate)) + return current_decay_times * vsa_decay_rate + return vsa_sparsity + + def run( + self, + method: torch.nn.Module, + *, + dataloader: Any, + max_steps: int, + start_step: int = 0, + ) -> None: + grad_accum = max(1, int(self.training_args.gradient_accumulation_steps + or 1)) + + if hasattr(method, "on_train_start"): + method.on_train_start() # type: ignore[attr-defined] + + if hasattr(method, "optimizers_zero_grad"): + method.optimizers_zero_grad(start_step) # type: ignore[attr-defined] + + data_stream = self._iter_dataloader(dataloader) + progress = tqdm( + range(start_step + 1, max_steps + 1), + initial=start_step, + desc="Steps", + disable=self.local_rank > 0, + ) + for step in progress: + t0 = time.perf_counter() + current_vsa_sparsity = self._get_current_vsa_sparsity(step) + + loss_sums: dict[str, float] = {} + for accum_iter in range(grad_accum): + batch = next(data_stream) + if hasattr(method, "single_train_step"): + loss_map, _ = method.single_train_step( # type: ignore[attr-defined] + batch, + step, + current_vsa_sparsity=current_vsa_sparsity, + ) + else: + raise AttributeError( + "method must implement single_train_step()") + + total_loss = loss_map["total_loss"] / grad_accum + total_loss.backward() + + for k, v in loss_map.items(): + if isinstance(v, torch.Tensor): + loss_sums[k] = loss_sums.get(k, 0.0) + float( + v.detach().item()) + + if hasattr(method, "optimizers_schedulers_step"): + method.optimizers_schedulers_step(step) # type: ignore[attr-defined] + if hasattr(method, "optimizers_zero_grad"): + method.optimizers_zero_grad(step) # type: ignore[attr-defined] + + metrics = {k: v / grad_accum for k, v in loss_sums.items()} + metrics["step_time_sec"] = time.perf_counter() - t0 + if self.global_rank == 0 and metrics: + self.tracker.log(metrics, step) + + self.tracker.finish() + From a8bada41af29bb1ba5f8a754c001a9c2cff1117c Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 08:14:41 +0000 Subject: [PATCH 041/214] phase 0 should be done --- dev/phases/phase_0.md | 13 +- fastvideo/distillation/adapters/wan.py | 180 +++++++++++++++++ fastvideo/distillation/methods/base.py | 12 +- fastvideo/distillation/methods/wan_dmd2.py | 191 ++++++++++++++++++ fastvideo/distillation/trainer.py | 14 +- .../distillation/test_phase0_schedule.py | 82 ++++++++ fastvideo/training/wan_distillation_v2.py | 72 +++++++ 7 files changed, 558 insertions(+), 6 deletions(-) create mode 100644 fastvideo/distillation/adapters/wan.py create mode 100644 fastvideo/distillation/methods/wan_dmd2.py create mode 100644 fastvideo/tests/distillation/test_phase0_schedule.py create mode 100644 fastvideo/training/wan_distillation_v2.py diff --git a/dev/phases/phase_0.md b/dev/phases/phase_0.md index 98e281830..fd3e1cde0 100644 --- a/dev/phases/phase_0.md +++ b/dev/phases/phase_0.md @@ -100,6 +100,18 @@ methods**（normalize/noise/timestep/attention metadata/build_input_kwargs 等 （这是对旧实现的一个显式修正：旧实现会每步 step generator scheduler） - fake_score optimizer/scheduler 每步 step +关于 backward 的一个 Phase 0 现实约束： + +- 由于 FastVideo 的 attention/kernel 依赖 `set_forward_context(...)`，并且训练里常开 + activation checkpointing，**backward 可能触发 forward 重算**，重算时也必须处于正确的 + forward_context 里。 +- 旧实现通过在 backward 前重新 `set_forward_context` 来保证这一点（且 generator/critic + 的 context 可能不同）。 +- 因此 Phase 0 的接口在 `DistillMethod` 里增加 `backward(loss_map, outputs, grad_accum_rounds)` + 这个 hook：Trainer 调用它，但不关心里面怎么拆分 loss/怎么设置 context。 + 默认实现仍然是对 `total_loss` 做 backward；Wan(DMD2) method 会覆写为 + “generator_loss 在 vsa context 下 backward + fake_score_loss 在 normal context 下 backward”。 + > 如果后续发现这个 scheduler 行为变化会影响 A/B 对齐，我们可以在 Phase 0 > 加一个 “legacy 模式开关”；但默认先按“optimizer step 对齐 scheduler step”的正确语义实现。 @@ -129,4 +141,3 @@ methods**（normalize/noise/timestep/attention metadata/build_input_kwargs 等 - `models={...}` + adapter 的抽象无法覆盖 Wan 的关键差异（例如 conditioning/CFG 方式根本不一致） - DMD2 的计算图要求导致 Method/Trainer 的边界必须反转（Trainer 不可算法无关） - 现有 pipeline 的 helper 复用导致强耦合无法逐步迁移（必须一次性大重构才可跑通） - diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py new file mode 100644 index 000000000..5623d40e8 --- /dev/null +++ b/fastvideo/distillation/adapters/wan.py @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import copy +import gc +from typing import Any + +import torch + +from fastvideo.configs.sample import SamplingParam +from fastvideo.distributed import get_local_torch_device, get_world_group +from fastvideo.pipelines import TrainingBatch +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.pipelines.basic.wan.wan_dmd_pipeline import WanDMDPipeline +from fastvideo.training.distillation_pipeline import DistillationPipeline + +from fastvideo.distillation.adapters.base import DistillAdapter + + +class WanPipelineAdapter(DistillAdapter): + def __init__(self, pipeline: DistillationPipeline) -> None: + self.pipeline = pipeline + + def _get_training_dtype(self) -> torch.dtype: + # Phase 0: match existing training pipelines (autocast bf16). + return torch.bfloat16 + + def ensure_negative_conditioning(self) -> None: + if getattr(self.pipeline, "negative_prompt_embeds", None) is not None: + return + + training_args = self.pipeline.training_args + world_group = get_world_group() + device = get_local_torch_device() + dtype = self._get_training_dtype() + + neg_embeds: torch.Tensor | None = None + neg_mask: torch.Tensor | None = None + + if world_group.rank_in_group == 0: + sampling_param = SamplingParam.from_pretrained(training_args.model_path) + negative_prompt = sampling_param.negative_prompt + + prompt_pipeline = getattr(self.pipeline, "validation_pipeline", None) + created_pipeline = False + if prompt_pipeline is None: + args_copy = copy.deepcopy(training_args) + args_copy.inference_mode = True + prompt_pipeline = WanDMDPipeline.from_pretrained( + training_args.model_path, + args=args_copy, + inference_mode=True, + loaded_modules={"transformer": self.pipeline.get_module("transformer")}, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + pin_cpu_memory=training_args.pin_cpu_memory, + dit_cpu_offload=True, + ) + created_pipeline = True + + batch_negative = ForwardBatch( + data_type="video", + prompt=negative_prompt, + prompt_embeds=[], + prompt_attention_mask=[], + ) + result_batch = prompt_pipeline.prompt_encoding_stage( # type: ignore[attr-defined] + batch_negative, + training_args, + ) + + neg_embeds = result_batch.prompt_embeds[0].to(device=device, dtype=dtype) + neg_mask = result_batch.prompt_attention_mask[0].to(device=device, dtype=dtype) + + if created_pipeline: + del prompt_pipeline + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + meta = torch.zeros((2,), device=device, dtype=torch.int64) + if world_group.rank_in_group == 0: + assert neg_embeds is not None + assert neg_mask is not None + meta[0] = neg_embeds.ndim + meta[1] = neg_mask.ndim + world_group.broadcast(meta, src=0) + embed_ndim, mask_ndim = (int(meta[0].item()), int(meta[1].item())) + + max_ndim = 8 + embed_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) + mask_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) + if world_group.rank_in_group == 0: + assert neg_embeds is not None + assert neg_mask is not None + embed_shape[:embed_ndim] = torch.tensor( + list(neg_embeds.shape), device=device, dtype=torch.int64 + ) + mask_shape[:mask_ndim] = torch.tensor( + list(neg_mask.shape), device=device, dtype=torch.int64 + ) + world_group.broadcast(embed_shape, src=0) + world_group.broadcast(mask_shape, src=0) + + embed_sizes = tuple(int(x) for x in embed_shape[:embed_ndim].tolist()) + mask_sizes = tuple(int(x) for x in mask_shape[:mask_ndim].tolist()) + + if world_group.rank_in_group != 0: + neg_embeds = torch.empty(embed_sizes, device=device, dtype=dtype) + neg_mask = torch.empty(mask_sizes, device=device, dtype=dtype) + assert neg_embeds is not None + assert neg_mask is not None + + world_group.broadcast(neg_embeds, src=0) + world_group.broadcast(neg_mask, src=0) + + self.pipeline.negative_prompt_embeds = neg_embeds + self.pipeline.negative_prompt_attention_mask = neg_mask + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + ) -> TrainingBatch: + self.ensure_negative_conditioning() + + device = get_local_torch_device() + dtype = self._get_training_dtype() + + training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + encoder_hidden_states = raw_batch["text_embedding"] + encoder_attention_mask = raw_batch["text_attention_mask"] + infos = raw_batch.get("info_list") + + if self.pipeline.training_args.simulate_generator_forward: + batch_size = encoder_hidden_states.shape[0] + vae_config = self.pipeline.training_args.pipeline_config.vae_config.arch_config + num_channels = vae_config.z_dim + spatial_compression_ratio = vae_config.spatial_compression_ratio + latent_height = self.pipeline.training_args.num_height // spatial_compression_ratio + latent_width = self.pipeline.training_args.num_width // spatial_compression_ratio + latents = torch.zeros( + batch_size, + num_channels, + self.pipeline.training_args.num_latent_t, + latent_height, + latent_width, + device=device, + dtype=dtype, + ) + else: + if "vae_latent" not in raw_batch: + raise ValueError( + "vae_latent not found in batch and simulate_generator_forward is False" + ) + latents = raw_batch["vae_latent"] + latents = latents[:, :, : self.pipeline.training_args.num_latent_t] + latents = latents.to(device, dtype=dtype) + + training_batch.latents = latents + training_batch.encoder_hidden_states = encoder_hidden_states.to( + device, dtype=dtype + ) + training_batch.encoder_attention_mask = encoder_attention_mask.to( + device, dtype=dtype + ) + training_batch.infos = infos + + training_batch = self.pipeline._normalize_dit_input(training_batch) + training_batch = self.pipeline._prepare_dit_inputs(training_batch) + training_batch = self.pipeline._build_attention_metadata(training_batch) + + training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) + if training_batch.attn_metadata is not None: + training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] + + return training_batch diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py index 3af2e1e02..3c14aced4 100644 --- a/fastvideo/distillation/methods/base.py +++ b/fastvideo/distillation/methods/base.py @@ -38,6 +38,17 @@ def get_optimizers(self, iteration: int) -> Sequence[torch.optim.Optimizer]: def get_lr_schedulers(self, iteration: int) -> Sequence[Any]: raise NotImplementedError + def backward( + self, + loss_map: dict[str, torch.Tensor], + outputs: dict[str, Any], + *, + grad_accum_rounds: int = 1, + ) -> None: + del outputs + grad_accum_rounds = max(1, int(grad_accum_rounds)) + (loss_map["total_loss"] / grad_accum_rounds).backward() + def optimizers_schedulers_step(self, iteration: int) -> None: for optimizer in self.get_optimizers(iteration): optimizer.step() @@ -50,4 +61,3 @@ def optimizers_zero_grad(self, iteration: int) -> None: optimizer.zero_grad(set_to_none=True) except TypeError: optimizer.zero_grad() - diff --git a/fastvideo/distillation/methods/wan_dmd2.py b/fastvideo/distillation/methods/wan_dmd2.py new file mode 100644 index 000000000..81420bba9 --- /dev/null +++ b/fastvideo/distillation/methods/wan_dmd2.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import copy +from typing import Any + +import torch + +from fastvideo.distributed import get_world_group +from fastvideo.forward_context import set_forward_context +from fastvideo.training.training_utils import ( + clip_grad_norm_while_handling_failing_dtensor_cases, +) +from fastvideo.utils import set_random_seed + +from fastvideo.distillation.bundle import ModelBundle +from fastvideo.distillation.methods.base import DistillMethod +from fastvideo.distillation.adapters.wan import WanPipelineAdapter + + +class WanDMD2Method(DistillMethod): + def __init__( + self, + *, + bundle: ModelBundle, + adapter: WanPipelineAdapter, + ) -> None: + super().__init__(bundle) + self.adapter = adapter + self.training_args = adapter.pipeline.training_args + self.world_group = get_world_group() + + def on_train_start(self) -> None: + seed = self.training_args.seed + if seed is None: + raise ValueError("training_args.seed must be set for distillation") + + pipeline = self.adapter.pipeline + if pipeline.sp_world_size > 1: + sp_group_seed = seed + (pipeline.global_rank // pipeline.sp_world_size) + set_random_seed(sp_group_seed) + else: + set_random_seed(seed + pipeline.global_rank) + + pipeline.noise_random_generator = torch.Generator(device="cpu").manual_seed(seed) + if pipeline.device.type == "cuda": + pipeline.noise_gen_cuda = torch.Generator(device="cuda").manual_seed(seed) + else: + pipeline.noise_gen_cuda = torch.Generator(device=pipeline.device).manual_seed(seed) + + self.adapter.ensure_negative_conditioning() + + def _should_update_student(self, iteration: int) -> bool: + interval = int(self.training_args.generator_update_interval or 1) + if interval <= 0: + return True + return iteration % interval == 0 + + def _clip_grad_norm(self, module: torch.nn.Module) -> float: + max_grad_norm = self.training_args.max_grad_norm + if not max_grad_norm: + return 0.0 + grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in module.parameters()], + float(max_grad_norm), + foreach=None, + ) + return float(grad_norm.item()) if grad_norm is not None else 0.0 + + def single_train_step( + self, + batch: dict[str, Any], + iteration: int, + *, + current_vsa_sparsity: float = 0.0, + ) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + pipeline = self.adapter.pipeline + pipeline.current_trainstep = iteration + + training_batch = self.adapter.prepare_batch( + batch, + current_vsa_sparsity=current_vsa_sparsity, + ) + + update_student = self._should_update_student(iteration) + device = pipeline.device + device_type = device.type + + generator_loss = torch.zeros((), device=device, dtype=training_batch.latents.dtype) + batch_gen = None + student_backward_ctx = None + if update_student: + batch_gen = copy.deepcopy(training_batch) + with torch.autocast(device_type, dtype=batch_gen.latents.dtype): + with set_forward_context( + current_timestep=batch_gen.timesteps, + attn_metadata=batch_gen.attn_metadata_vsa, + ): + if self.training_args.simulate_generator_forward: + generator_pred_video = pipeline._generator_multi_step_simulation_forward( + batch_gen + ) + else: + generator_pred_video = pipeline._generator_forward(batch_gen) + + with set_forward_context( + current_timestep=batch_gen.timesteps, + attn_metadata=batch_gen.attn_metadata, + ): + generator_loss = pipeline._dmd_forward( + generator_pred_video=generator_pred_video, + training_batch=batch_gen, + ) + student_backward_ctx = (batch_gen.timesteps, batch_gen.attn_metadata_vsa) + + batch_fake = copy.deepcopy(training_batch) + with torch.autocast(device_type, dtype=batch_fake.latents.dtype): + _, fake_score_loss = pipeline.faker_score_forward(batch_fake) + + total_loss = generator_loss + fake_score_loss + loss_map = { + "total_loss": total_loss, + "generator_loss": generator_loss, + "fake_score_loss": fake_score_loss, + } + outputs = {} + if update_student and batch_gen is not None: + outputs["dmd_latent_vis_dict"] = batch_gen.dmd_latent_vis_dict + outputs["fake_score_latent_vis_dict"] = batch_fake.fake_score_latent_vis_dict + outputs["_fv_backward"] = { + "update_student": update_student, + "student_ctx": student_backward_ctx, + "critic_ctx": (batch_fake.timesteps, batch_fake.attn_metadata), + } + return loss_map, outputs + + def backward( + self, + loss_map: dict[str, torch.Tensor], + outputs: dict[str, Any], + *, + grad_accum_rounds: int = 1, + ) -> None: + grad_accum_rounds = max(1, int(grad_accum_rounds)) + backward_ctx = outputs.get("_fv_backward") + if not isinstance(backward_ctx, dict): + super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + return + + update_student = bool(backward_ctx.get("update_student", False)) + if update_student: + student_ctx = backward_ctx.get("student_ctx") + if student_ctx is None: + raise RuntimeError("Missing student backward context") + timesteps, attn_metadata = student_ctx + with set_forward_context( + current_timestep=timesteps, + attn_metadata=attn_metadata, + ): + (loss_map["generator_loss"] / grad_accum_rounds).backward() + + timesteps, attn_metadata = backward_ctx["critic_ctx"] + with set_forward_context( + current_timestep=timesteps, + attn_metadata=attn_metadata, + ): + (loss_map["fake_score_loss"] / grad_accum_rounds).backward() + + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + optimizers: list[torch.optim.Optimizer] = [] + optimizers.extend(self.bundle.role("critic").optimizers.values()) + if self._should_update_student(iteration): + optimizers.extend(self.bundle.role("student").optimizers.values()) + return optimizers + + def get_lr_schedulers(self, iteration: int) -> list[Any]: + schedulers: list[Any] = [] + schedulers.extend(self.bundle.role("critic").lr_schedulers.values()) + if self._should_update_student(iteration): + schedulers.extend(self.bundle.role("student").lr_schedulers.values()) + return schedulers + + def optimizers_schedulers_step(self, iteration: int) -> None: + if self._should_update_student(iteration): + for module in self.bundle.role("student").modules.values(): + self._clip_grad_norm(module) + for module in self.bundle.role("critic").modules.values(): + self._clip_grad_norm(module) + + super().optimizers_schedulers_step(iteration) diff --git a/fastvideo/distillation/trainer.py b/fastvideo/distillation/trainer.py index e64a1d449..7f198548f 100644 --- a/fastvideo/distillation/trainer.py +++ b/fastvideo/distillation/trainer.py @@ -84,7 +84,7 @@ def run( for accum_iter in range(grad_accum): batch = next(data_stream) if hasattr(method, "single_train_step"): - loss_map, _ = method.single_train_step( # type: ignore[attr-defined] + loss_map, outputs = method.single_train_step( # type: ignore[attr-defined] batch, step, current_vsa_sparsity=current_vsa_sparsity, @@ -93,8 +93,15 @@ def run( raise AttributeError( "method must implement single_train_step()") - total_loss = loss_map["total_loss"] / grad_accum - total_loss.backward() + if hasattr(method, "backward"): + method.backward( # type: ignore[attr-defined] + loss_map, + outputs, + grad_accum_rounds=grad_accum, + ) + else: + total_loss = loss_map["total_loss"] / grad_accum + total_loss.backward() for k, v in loss_map.items(): if isinstance(v, torch.Tensor): @@ -112,4 +119,3 @@ def run( self.tracker.log(metrics, step) self.tracker.finish() - diff --git a/fastvideo/tests/distillation/test_phase0_schedule.py b/fastvideo/tests/distillation/test_phase0_schedule.py new file mode 100644 index 000000000..7c9666725 --- /dev/null +++ b/fastvideo/tests/distillation/test_phase0_schedule.py @@ -0,0 +1,82 @@ +import torch + +from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.methods.base import DistillMethod + + +class _FakeScheduler: + def __init__(self) -> None: + self.step_calls = 0 + + def step(self) -> None: + self.step_calls += 1 + + +class _FakeOptimizer(torch.optim.Optimizer): + def __init__(self) -> None: + super().__init__([torch.zeros((), requires_grad=True)], {}) + self.step_calls = 0 + self.zero_grad_calls = 0 + + def step(self, closure=None): # noqa: ANN001, ANN201 + self.step_calls += 1 + if closure is not None: + closure() + + def zero_grad(self, *args, **kwargs): # noqa: ANN002, ANN003, ANN201 + self.zero_grad_calls += 1 + + +class _ScheduleMethod(DistillMethod): + def __init__(self, interval: int) -> None: + self.student_opt = _FakeOptimizer() + self.critic_opt = _FakeOptimizer() + self.student_sched = _FakeScheduler() + self.critic_sched = _FakeScheduler() + bundle = ModelBundle( + roles={ + "student": + RoleHandle( + optimizers={"main": self.student_opt}, + lr_schedulers={"main": self.student_sched}, + ), + "critic": + RoleHandle( + optimizers={"main": self.critic_opt}, + lr_schedulers={"main": self.critic_sched}, + ), + }) + super().__init__(bundle) + self.interval = interval + + def _update_student(self, iteration: int) -> bool: + return iteration % self.interval == 0 + + def single_train_step(self, batch, iteration, *, current_vsa_sparsity=0.0): # noqa: ANN001, ANN201 + loss = torch.zeros((), requires_grad=True) + return {"total_loss": loss}, {} + + def get_optimizers(self, iteration): # noqa: ANN001, ANN201 + optimizers = [self.critic_opt] + if self._update_student(iteration): + optimizers.append(self.student_opt) + return optimizers + + def get_lr_schedulers(self, iteration): # noqa: ANN001, ANN201 + schedulers = [self.critic_sched] + if self._update_student(iteration): + schedulers.append(self.student_sched) + return schedulers + + +def test_phase0_optimizer_scheduler_alignment() -> None: + method = _ScheduleMethod(interval=5) + + for step in range(1, 11): + method.optimizers_schedulers_step(step) + + assert method.critic_opt.step_calls == 10 + assert method.critic_sched.step_calls == 10 + assert method.student_opt.step_calls == 2 + assert method.student_sched.step_calls == 2 + diff --git a/fastvideo/training/wan_distillation_v2.py b/fastvideo/training/wan_distillation_v2.py new file mode 100644 index 000000000..e5c928886 --- /dev/null +++ b/fastvideo/training/wan_distillation_v2.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys + +from fastvideo.distillation import DistillTrainer, ModelBundle, RoleHandle +from fastvideo.distillation.adapters.wan import WanPipelineAdapter +from fastvideo.distillation.methods.wan_dmd2 import WanDMD2Method +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.logger import init_logger +from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline +from fastvideo.utils import FlexibleArgumentParser + +logger = init_logger(__name__) + + +def _build_bundle_from_wan_pipeline( + pipeline: WanDistillationPipeline, +) -> ModelBundle: + roles: dict[str, RoleHandle] = { + "student": + RoleHandle( + modules={"transformer": pipeline.transformer}, + optimizers={"main": pipeline.optimizer}, + lr_schedulers={"main": pipeline.lr_scheduler}, + ), + "teacher": + RoleHandle( + modules={"transformer": pipeline.real_score_transformer}, + frozen=True, + ), + "critic": + RoleHandle( + modules={"transformer": pipeline.fake_score_transformer}, + optimizers={"main": pipeline.fake_score_optimizer}, + lr_schedulers={"main": pipeline.fake_score_lr_scheduler}, + ), + } + return ModelBundle(roles=roles) + + +def main(args) -> None: + logger.info("Starting Wan distillation v2 (Method/Trainer)...") + + pipeline = WanDistillationPipeline.from_pretrained( + args.pretrained_model_name_or_path, args=args) + training_args = pipeline.training_args + + adapter = WanPipelineAdapter(pipeline) + bundle = _build_bundle_from_wan_pipeline(pipeline) + method = WanDMD2Method(bundle=bundle, adapter=adapter) + + trainer = DistillTrainer(training_args, tracker=pipeline.tracker) + trainer.run( + method, + dataloader=pipeline.train_dataloader, + max_steps=training_args.max_train_steps, + start_step=pipeline.init_steps, + ) + + logger.info("Wan distillation v2 completed") + + +if __name__ == "__main__": + argv = sys.argv + parser = FlexibleArgumentParser() + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args(argv[1:]) + main(args) + From c4c2d89ceda8d03dfa09b1edc45ae3140edb39fd Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 08:26:22 +0000 Subject: [PATCH 042/214] phase0 warning --- dev/phases/phase_0.md | 51 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/dev/phases/phase_0.md b/dev/phases/phase_0.md index fd3e1cde0..81bbc8377 100644 --- a/dev/phases/phase_0.md +++ b/dev/phases/phase_0.md @@ -141,3 +141,54 @@ methods**（normalize/noise/timestep/attention metadata/build_input_kwargs 等 - `models={...}` + adapter 的抽象无法覆盖 Wan 的关键差异（例如 conditioning/CFG 方式根本不一致） - DMD2 的计算图要求导致 Method/Trainer 的边界必须反转（Trainer 不可算法无关） - 现有 pipeline 的 helper 复用导致强耦合无法逐步迁移（必须一次性大重构才可跑通） + +--- + +## 5. Phase 0 的“耦合债务”与命名说明（非常重要，避免未来遗忘） + +### 5.1 为什么现在会有 `WanDMD2Method` 这种名字？ + +结论：这是 **Phase 0 的过渡实现**，名字里带 `Wan` 是“刻意暴露耦合”，防止误用。 + +原因：当前 `fastvideo/distillation/methods/wan_dmd2.py` 并不是一个纯算法层的 DMD2。 +它直接复用/依赖了旧实现 `fastvideo/training/distillation_pipeline.py` 的 Wan-only 私有逻辑： + +- DMD2 的关键计算来自旧 pipeline 的内部函数：`_dmd_forward(...)`、`faker_score_forward(...)`、 + `_generator_forward(...)` 等（它们隐含了 layout/normalize/CFG/uncond 等具体假设） +- `fastvideo/distillation/adapters/wan.py` 也在复用旧 pipeline 的 helper： + `_normalize_dit_input/_prepare_dit_inputs/_build_attention_metadata` + +因此它在语义上等价于：**“把旧 Wan distill pipeline 包了一层 Method/Trainer 外壳”**， +而不是一个可对接任意 adapter 的“通用 DMD2Method”。 + +### 5.2 FastGen 有没有类似的做法？ + +FastGen 的命名与分层更“干净”： + +- 算法层叫 `DMD2Model`（算法名），不会叫 `WanDMD2Model` +- 网络/架构差异在 `networks/*` + config 里选择（网络与算法解耦） + +所以我们现在的 `WanDMD2Method` 更像是 Phase 0 的迁移脚手架，而不是最终形态。 + +### 5.3 TODO（必须做）：把 `WanDMD2Method` 演进为 **算法名 method + 模型名 adapter** + +为了避免“又一次耦合到 Wan”，必须把 Phase 0 的耦合逐步清掉，目标对齐 FastGen： + +1) **把算法从旧 pipeline 里抠出来** + - 新增：`fastvideo/distillation/methods/dmd2.py`（`DMD2Method`，不依赖任何具体模型） + - `DMD2Method` 只依赖 adapter 提供的 primitives（noise/pred_to_x0/teacher_cfg/critic_loss 等） + +2) **把模型差异收敛到 adapter（WanAdapter）** + - 演进：`WanPipelineAdapter` -> `WanAdapter` + - `WanAdapter` 不再调用 `DistillationPipeline` 的私有 helper 方法， + 而是自己实现 normalize/layout/attention metadata/输入 kwargs 组装等 + +3) **最终命名与入口应变成** + - `DMD2Method + WanAdapter`（method 不带模型名） + - `fastvideo/training/wan_distillation_v2.py` 里只选择 adapter，不再选择“WanDMD2Method” + +4) **迁移后应删除/冻结 Phase 0 的 pipeline-backed 版本** + - 避免未来复制粘贴 `WanDMD2Method` 去做其它模型（那会把耦合扩散） + +> 备注：Phase 0 用 `WanDMD2Method` 的意义是“先把训练循环与多 optimizer 调度结构稳定下来”， +> 但我们必须把它当成临时脚手架，Phase 1/2 逐步替换为真正解耦的 method+adapter。 From 5720ae9113e4b5f8889fdf31954556317e97262d Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 09:01:21 +0000 Subject: [PATCH 043/214] validation in method --- dev/phases/phase_0.md | 3 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.sh | 112 ++++++++++++++++++ fastvideo/distillation/methods/wan_dmd2.py | 41 ++++++- fastvideo/distillation/trainer.py | 12 ++ 4 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh diff --git a/dev/phases/phase_0.md b/dev/phases/phase_0.md index 81bbc8377..1a3c780d3 100644 --- a/dev/phases/phase_0.md +++ b/dev/phases/phase_0.md @@ -130,7 +130,8 @@ methods**（normalize/noise/timestep/attention metadata/build_input_kwargs 等 后续增量（Phase 0 内可迭代）： - checkpoint/resume 接入（优先复用 `save_distillation_checkpoint/load_distillation_checkpoint`） -- validation/logging 接入（可先调用旧 pipeline 的 `_log_validation`） +- validation 接入：已通过 `DistillTrainer` -> `method.log_validation(step)` hook + 接入旧 pipeline 的 `_log_validation`（见 `WanDMD2Method.log_validation()`） --- diff --git a/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh new file mode 100644 index 000000000..c125fb55e --- /dev/null +++ b/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh @@ -0,0 +1,112 @@ +#!/bin/bash +set -e -x + +# Phase 0 example: run Wan DMD2 distillation via Method/Trainer entrypoint. +# Validation is supported via `--log_validation` (see `validation_args` below). + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_PORT=${MASTER_PORT:-29503} + +NUM_GPUS=${NUM_GPUS:-1} + +# Models +STUDENT_MODEL_PATH=${STUDENT_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} +# For best distillation, point TEACHER_MODEL_PATH to a stronger teacher (e.g. 14B). +# For a cheaper smoke run, set it to the same 1.3B model. +TEACHER_MODEL_PATH=${TEACHER_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-14B-Diffusers"} +CRITIC_MODEL_PATH=${CRITIC_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} + +# Data (parquet dataset folder) +DATA_DIR=${DATA_DIR:-"your_data_dir"} +VALIDATION_DATASET_FILE=${VALIDATION_DATASET_FILE:-"your_validation_dataset_file"} + +OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase0_wan2.1_t2v_1.3B_dmd2_8steps"} + +training_args=( + --tracker_project_name "phase0_wan_dmd2_8steps" + --output_dir "$OUTPUT_DIR" + --max_train_steps 4000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 21 + --num_height 480 + --num_width 832 + --num_frames 81 + --enable_gradient_checkpointing_type "full" + --simulate_generator_forward +) + +parallel_args=( + --num_gpus "$NUM_GPUS" + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim "$NUM_GPUS" + --hsdp_shard_dim 1 +) + +model_args=( + --model_path "$STUDENT_MODEL_PATH" + --pretrained_model_name_or_path "$STUDENT_MODEL_PATH" + --real_score_model_path "$TEACHER_MODEL_PATH" + --fake_score_model_path "$CRITIC_MODEL_PATH" +) + +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 50 + --validation_sampling_steps "8" + --validation_guidance_scale "6.0" # not used for dmd inference +) + +optimizer_args=( + --learning_rate 1e-5 + --mixed_precision "bf16" + --weight_decay 0.01 + --betas '0.0,0.999' + --max_grad_norm 1.0 + --fake_score_learning_rate 8e-6 + --fake_score_betas '0.0,0.999' +) + +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.0 + --dit_precision "fp32" + --flow_shift 8 + --seed 1000 +) + +dmd_args=( + # 8-step schedule (same as Wan2.2 self-forcing examples) + --dmd_denoising_steps '1000,850,700,550,350,275,200,125' + --min_timestep_ratio 0.02 + --max_timestep_ratio 0.98 + --generator_update_interval 5 + --real_score_guidance_scale 3.5 +) + +torchrun \ +--nnodes 1 \ +--master_port "$MASTER_PORT" \ +--nproc_per_node "$NUM_GPUS" \ + fastvideo/training/wan_distillation_v2.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ + "${dmd_args[@]}" diff --git a/fastvideo/distillation/methods/wan_dmd2.py b/fastvideo/distillation/methods/wan_dmd2.py index 81420bba9..ca5510b40 100644 --- a/fastvideo/distillation/methods/wan_dmd2.py +++ b/fastvideo/distillation/methods/wan_dmd2.py @@ -43,14 +43,39 @@ def on_train_start(self) -> None: else: set_random_seed(seed + pipeline.global_rank) - pipeline.noise_random_generator = torch.Generator(device="cpu").manual_seed(seed) + pipeline.noise_random_generator = torch.Generator( + device="cpu").manual_seed(seed) + pipeline.validation_random_generator = torch.Generator( + device="cpu").manual_seed(seed) if pipeline.device.type == "cuda": pipeline.noise_gen_cuda = torch.Generator(device="cuda").manual_seed(seed) else: - pipeline.noise_gen_cuda = torch.Generator(device=pipeline.device).manual_seed(seed) + pipeline.noise_gen_cuda = torch.Generator( + device=pipeline.device).manual_seed(seed) self.adapter.ensure_negative_conditioning() + def log_validation(self, iteration: int) -> None: + pipeline = self.adapter.pipeline + training_args = pipeline.training_args + if not getattr(training_args, "log_validation", False): + return + + if getattr(pipeline, "validation_pipeline", None) is None: + pipeline.initialize_validation_pipeline(training_args) + + old_inference_mode = training_args.inference_mode + old_dit_cpu_offload = training_args.dit_cpu_offload + try: + pipeline._log_validation( + pipeline.transformer, + training_args, + iteration, + ) + finally: + training_args.inference_mode = old_inference_mode + training_args.dit_cpu_offload = old_dit_cpu_offload + def _should_update_student(self, iteration: int) -> bool: interval = int(self.training_args.generator_update_interval or 1) if interval <= 0: @@ -87,7 +112,11 @@ def single_train_step( device = pipeline.device device_type = device.type - generator_loss = torch.zeros((), device=device, dtype=training_batch.latents.dtype) + generator_loss = torch.zeros( + (), + device=device, + dtype=training_batch.latents.dtype, + ) batch_gen = None student_backward_ctx = None if update_student: @@ -98,9 +127,9 @@ def single_train_step( attn_metadata=batch_gen.attn_metadata_vsa, ): if self.training_args.simulate_generator_forward: - generator_pred_video = pipeline._generator_multi_step_simulation_forward( - batch_gen - ) + generator_pred_video = ( + pipeline._generator_multi_step_simulation_forward( + batch_gen)) else: generator_pred_video = pipeline._generator_forward(batch_gen) diff --git a/fastvideo/distillation/trainer.py b/fastvideo/distillation/trainer.py index 7f198548f..ba5a599f7 100644 --- a/fastvideo/distillation/trainer.py +++ b/fastvideo/distillation/trainer.py @@ -66,6 +66,12 @@ def run( if hasattr(method, "on_train_start"): method.on_train_start() # type: ignore[attr-defined] + validation_interval = int(self.training_args.validation_steps or 0) + if (getattr(self.training_args, "log_validation", False) + and validation_interval > 0 and hasattr(method, + "log_validation")): + method.log_validation(start_step) # type: ignore[attr-defined] + if hasattr(method, "optimizers_zero_grad"): method.optimizers_zero_grad(start_step) # type: ignore[attr-defined] @@ -118,4 +124,10 @@ def run( if self.global_rank == 0 and metrics: self.tracker.log(metrics, step) + if (getattr(self.training_args, "log_validation", False) + and validation_interval > 0 + and step % validation_interval == 0 + and hasattr(method, "log_validation")): + method.log_validation(step) # type: ignore[attr-defined] + self.tracker.finish() From c8da42e1cddffdda1366dcb83df1d4269709b242 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 09:22:51 +0000 Subject: [PATCH 044/214] scripts for testing phase0 --- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.sh | 4 +- temp.sh | 128 ++++++++++++++++++ 2 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 temp.sh diff --git a/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh index c125fb55e..fd8b8166f 100644 --- a/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh +++ b/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh @@ -45,8 +45,8 @@ parallel_args=( --num_gpus "$NUM_GPUS" --sp_size 1 --tp_size 1 - --hsdp_replicate_dim "$NUM_GPUS" - --hsdp_shard_dim 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim "$NUM_GPUS" ) model_args=( diff --git a/temp.sh b/temp.sh new file mode 100644 index 000000000..febd4d4ca --- /dev/null +++ b/temp.sh @@ -0,0 +1,128 @@ +#!/bin/bash +set -e -x + +# One-shot launch script for Phase0 (Method/Trainer) Wan DMD2 few-step distillation. +# Uses local Wan-Syn parquet dataset + a small validation json already in this repo. +# +# Notes: +# - By default this runs W&B in offline mode (safer for overnight runs). +# If you want online logging: +# export WANDB_MODE=online +# export WANDB_API_KEY=... +# - Phase0 v2 currently focuses on "can it train + can it validate". +# Checkpoint save/resume is NOT wired yet in v2. + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29503} + +if [[ -z "${NUM_GPUS:-}" ]]; then + if command -v nvidia-smi >/dev/null 2>&1; then + NUM_GPUS=$(nvidia-smi -L | wc -l) + else + NUM_GPUS=1 + fi +fi + +# Models +STUDENT_MODEL_PATH=${STUDENT_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} +TEACHER_MODEL_PATH=${TEACHER_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-14B-Diffusers"} +CRITIC_MODEL_PATH=${CRITIC_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} + +# Data (parquet dataset folder) +DATA_DIR=${DATA_DIR:-"data/Wan-Syn_77x448x832_600k"} +DEFAULT_VALIDATION_DATASET_FILE=\ +"examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json" +VALIDATION_DATASET_FILE=${VALIDATION_DATASET_FILE:-"$DEFAULT_VALIDATION_DATASET_FILE"} + +OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase0_wan2.1_dmd2_8steps_wansyn"} + +training_args=( + --tracker_project_name "phase0_wan_dmd2_8steps_wansyn" + --output_dir "$OUTPUT_DIR" + --max_train_steps 4000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 20 + --num_height 448 + --num_width 832 + --num_frames 77 + --enable_gradient_checkpointing_type "full" + --simulate_generator_forward +) + +parallel_args=( + --num_gpus "$NUM_GPUS" + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim "$NUM_GPUS" +) + +model_args=( + --model_path "$STUDENT_MODEL_PATH" + --pretrained_model_name_or_path "$STUDENT_MODEL_PATH" + --real_score_model_path "$TEACHER_MODEL_PATH" + --fake_score_model_path "$CRITIC_MODEL_PATH" +) + +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 50 + --validation_sampling_steps "8" + --validation_guidance_scale "6.0" +) + +optimizer_args=( + --learning_rate 2e-6 + --mixed_precision "bf16" + --weight_decay 0.01 + --betas '0.0,0.999' + --max_grad_norm 1.0 + --fake_score_learning_rate 8e-6 + --fake_score_betas '0.0,0.999' +) + +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.0 + --dit_precision "fp32" + --flow_shift 8 + --seed 1000 +) + +dmd_args=( + --dmd_denoising_steps '1000,850,700,550,350,275,200,125' + --min_timestep_ratio 0.02 + --max_timestep_ratio 0.98 + --generator_update_interval 5 + --real_score_guidance_scale 3.5 +) + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/wan_distillation_v2.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ + "${dmd_args[@]}" From 0c1bd0de419874171a0602c041cd7f4b4866a1ca Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 19:25:05 +0000 Subject: [PATCH 045/214] temp launch --- temp.sh => examples/distillation/phase0/temp.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) rename temp.sh => examples/distillation/phase0/temp.sh (94%) diff --git a/temp.sh b/examples/distillation/phase0/temp.sh similarity index 94% rename from temp.sh rename to examples/distillation/phase0/temp.sh index febd4d4ca..63743f2c7 100644 --- a/temp.sh +++ b/examples/distillation/phase0/temp.sh @@ -21,6 +21,11 @@ export WANDB_MODE=${WANDB_MODE:-offline} export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} export MASTER_PORT=${MASTER_PORT:-29503} +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + if [[ -z "${NUM_GPUS:-}" ]]; then if command -v nvidia-smi >/dev/null 2>&1; then NUM_GPUS=$(nvidia-smi -L | wc -l) @@ -40,7 +45,8 @@ DEFAULT_VALIDATION_DATASET_FILE=\ "examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json" VALIDATION_DATASET_FILE=${VALIDATION_DATASET_FILE:-"$DEFAULT_VALIDATION_DATASET_FILE"} -OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase0_wan2.1_dmd2_8steps_wansyn"} +RUN_ID=${RUN_ID:-"$(date +%Y%m%d_%H%M%S)"} +OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase0_wan2.1_dmd2_8steps_wansyn_${RUN_ID}"} training_args=( --tracker_project_name "phase0_wan_dmd2_8steps_wansyn" From 97fa792db65531513a0f38df5f2f75f1ef4aaad5 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 22:09:33 +0000 Subject: [PATCH 046/214] phase 1 design --- dev/design.md | 67 ++++++++++++++++++++++++++++++++----------- dev/phases/phase_0.md | 34 +++++++++++++++++++++- 2 files changed, 83 insertions(+), 18 deletions(-) diff --git a/dev/design.md b/dev/design.md index b8b64553e..b0e4bd37d 100644 --- a/dev/design.md +++ b/dev/design.md @@ -391,14 +391,17 @@ FastGen 的 python config + instantiate + override 很优秀，但 FastVideo 现 ## 7. 目录落地建议（FastVideo 内） -建议新增 `fastvideo/distill/`（或 `fastvideo/training/distill/`），结构对齐 FastGen： +Phase 0 的实践表明：先把新框架以 **additive** 方式落地到一个独立目录最稳妥。 +目前已选择并落地在 `fastvideo/distillation/`（建议继续沿用该路径，避免再迁一次目录）。 -- `fastvideo/distill/bundle.py`：`ModelBundle/RoleHandle/RoleSpec` -- `fastvideo/distill/adapters/`：`WanAdapter`（先支持 Wan）、后续扩展更多模型 -- `fastvideo/distill/methods/`：`base.py`、`dmd2.py`、`self_forcing.py` -- `fastvideo/distill/trainer.py`：`DistillTrainer` -- `fastvideo/distill/checkpoint.py`：`CheckpointManager`（先兼容旧格式） -- `fastvideo/distill/callbacks/`：EMA/clip/log/profiler 等 +建议结构（已部分实现）： + +- `fastvideo/distillation/bundle.py`：`ModelBundle/RoleHandle` +- `fastvideo/distillation/adapters/`：`WanPipelineAdapter`（Phase 0 过渡版）→ `WanAdapter`（目标） +- `fastvideo/distillation/methods/`：`base.py`、（目标）`dmd2.py`、（目标）`self_forcing.py` +- `fastvideo/distillation/trainer.py`：`DistillTrainer` +- （后续）`fastvideo/distillation/checkpoint.py`：role-based `CheckpointManager`（先兼容旧格式） +- （后续）`fastvideo/distillation/callbacks/`：EMA/clip/log/profiler 等 旧入口（如 `fastvideo/training/*distillation_pipeline.py`）先保留， 通过 flag 切新旧框架做 A/B 对齐。 @@ -407,24 +410,54 @@ FastGen 的 python config + instantiate + override 很优秀，但 FastVideo 现 ## 8. 迁移计划（低风险） -### Phase 0：框架落地 + Wan(DMD2) 跑通 +### Phase 0（已完成）：框架落地 + Wan(DMD2) 跑通（过渡实现） + +Phase 0 的定位在实践中更明确了：它是“**把旧 Wan distill pipeline 包一层新框架壳**”， +先把训练循环/多 optimizer 调度/validation hook 等基础设施固定下来，再逐步解耦。 + +- ✅ 新增 `DistillTrainer/DistillMethod/ModelBundle/(pipeline-backed) WanAdapter` +- ✅ 新增一个 additive 入口：`fastvideo/training/wan_distillation_v2.py` + - 复用 legacy `WanDistillationPipeline` 完成模型加载/optimizer/dataloader/tracker + - 再把 student/teacher/critic 打包为 `ModelBundle(roles={...})` +- ✅ 跑通 Wan DMD2（student + teacher + critic） + - 过渡命名：`WanDMD2Method`（刻意暴露耦合，避免误解为通用 DMD2） +- ✅ 消除一个关键隐式耦合：训练前显式初始化 `neg/uncond conditioning` + - 不再依赖 validation 的副作用（见 `ensure_negative_conditioning()`） +- ✅ 修正并用单测锁定一个关键语义：scheduler step 与 optimizer step 对齐 + - `generator_update_interval > 1` 时不会“空 step scheduler” +- ✅ 提供 few-step distill 示例脚本 + 可直接跑的 temp 脚本 + +Phase 0 明确没有做（刻意延期）： -- 新增 `DistillTrainer/DistillMethod/ModelBundle/WanAdapter` -- `DMD2Method` 覆盖现有 Wan distill 训练（student+teacher+critic） -- checkpoint 至少能： - - 保存/加载新格式 - - 从旧格式加载 student 初始化（兼容迁移） +- ❌ v2 path 的 checkpoint/save/resume（role-based） +- ❌ `DMD2Method` 的真正算法解耦（目前仍调用旧 pipeline 内部函数） +- ❌ Self-forcing v2 迁移 -### Phase 1：Self-forcing 迁移（复用 DMD2 框架） +### Phase 1（建议开启）：算法与模型真正解耦（先把 DMD2 “抠出来”） -- `SelfForcingMethod(DMD2Method)`：只覆写 student rollout / cache 生命周期 -- 对齐现有 self-forcing 输出与 loss（允许数值差异但要解释） +Phase 1 的核心目标：把 Phase 0 的“脚手架耦合”逐步替换为 **Method(算法) + Adapter(模型)** +的稳定边界，让其它模型/其它方法可以复用 Trainer。 + +- 产出通用算法：`fastvideo/distillation/methods/dmd2.py::DMD2Method` + - 不再依赖 `fastvideo/training/distillation_pipeline.py` 的私有函数 + - 只依赖 adapter 提供的 primitives（noise/add_noise/pred_to_x0/teacher_cfg/critic_forward 等） +- 产出真正模型适配：`WanAdapter`（替换 `WanPipelineAdapter`） + - 逐步把 normalize/layout/attention metadata/输入 kwargs 组装等从 legacy pipeline 迁出 +- Builder 层雏形：从 `TrainingArgs/FastVideoArgs`（或 `--models_json`）直接构建 + `ModelBundle + Adapter + Method` + - 目标：最终不再依赖 legacy pipeline 才能启动 v2 +- Validation 进一步抽象（可选）：把“怎么验证”从 method 里抽走，变成通用 hook/组件 ### Phase 2：清理旧实现 + 扩展新模型/新算法 -- 逐步冻结或移除旧 distill pipeline（保留兼容入口亦可） +在 Phase 1 的稳定边界之上，Phase 2 再做“功能扩展 + 旧实现收敛”： + +- Self-forcing v2：`SelfForcingMethod(DMD2Method)`（只覆写 student rollout / cache 生命周期） + - 并把 ODE-init（若需要）归类为 **student 初始化策略**（builder/config 层），而不是 Trainer 特例 +- role-based checkpoint/save/resume（v2 path） - 新增更多 adapter（Hunyuan/LTX2/LongCat…） - 新增更多 method（teacher-only、多 teacher、KD 轨迹蒸馏等） +- 逐步冻结或移除旧 distill pipeline（保留兼容入口亦可） --- diff --git a/dev/phases/phase_0.md b/dev/phases/phase_0.md index 1a3c780d3..8f937a4f5 100644 --- a/dev/phases/phase_0.md +++ b/dev/phases/phase_0.md @@ -16,6 +16,38 @@ --- +## Phase 0 TODO List（Review Checklist） + +> 目的：把 Phase 0 的“可交付”拆成可审查的 checklist，方便你决定是否开启 Phase 1。 + +### Phase 0 可交付（应全部完成） + +- [x] 新 distill 框架骨架落地（Method/Trainer/Adapter/Bundle），且 **不影响旧 pipeline** + - `fastvideo/distillation/` +- [x] 新框架跑通 Wan DMD2（student + teacher + critic） + - 入口：`fastvideo/training/wan_distillation_v2.py` + - method：`fastvideo/distillation/methods/wan_dmd2.py` + - adapter：`fastvideo/distillation/adapters/wan.py` +- [x] 消除 DMD2 的隐式耦合：uncond/negative conditioning **不再依赖 validation 副作用** + - `fastvideo/distillation/adapters/wan.py`（`ensure_negative_conditioning()`） +- [x] 多优化器更新节奏下，optimizer step 与 lr_scheduler step 对齐（避免 lr schedule 漂移） + - `fastvideo/distillation/methods/base.py` + - `fastvideo/tests/distillation/test_phase0_schedule.py` +- [x] 提供 few-step distill 示例脚本（可直接改路径运行） + - `examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh` + - `examples/distillation/phase0/temp.sh` +- [x] validation 能在新 Trainer 中被统一触发（Phase 0 先复用旧 `_log_validation`） + - hook：`fastvideo/distillation/trainer.py` + - 实现：`fastvideo/distillation/methods/wan_dmd2.py::log_validation` + +### Phase 0 明确不做 / 延后到 Phase 1+ + +- [ ] 把 `WanDMD2Method` 演进为通用 `DMD2Method`（算法与 Wan 解耦） +- [ ] 把 `WanPipelineAdapter` 演进为真正 `WanAdapter`（不再调用旧 pipeline 私有 helper） +- [ ] v2 path 的 checkpoint/save/resume（role-based） +- [ ] Self-forcing v2（method + adapter） +- [ ] Builder 层：用 config 直接构建 `models={...}`（不再依赖 legacy pipeline 负责加载） + ## 0. 关键风险与应对 ### 风险 A：`negative_prompt_embeds` 目前只在 validation 中被初始化 @@ -48,7 +80,7 @@ methods**（normalize/noise/timestep/attention metadata/build_input_kwargs 等 ## 1. 代码落地点（具体到文件） -> 约定：Phase 0 把新框架放到 `fastvideo/distillation/`（目前该目录为空）。 +> 约定：Phase 0 把新框架放到 `fastvideo/distillation/`。 ### 1.1 新增 distill 框架骨架 From d6ecdad88b636892cddfbd4314ac6153a8d78439 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 22:46:06 +0000 Subject: [PATCH 047/214] progressing phase 1 --- dev/phases/phase_1.md | 169 +++++ .../distill_wan2.1_t2v_1.3B_dmd2_8steps.sh | 113 +++ fastvideo/distillation/adapters/base.py | 8 +- fastvideo/distillation/adapters/wan.py | 708 +++++++++++++++++- fastvideo/distillation/builder.py | 41 + .../methods/consistency_model/__init__.py | 4 + .../methods/distribution_matching/__init__.py | 8 + .../methods/distribution_matching/dmd2.py | 250 +++++++ .../methods/fine_tuning/__init__.py | 4 + .../knowledge_distillation/__init__.py | 4 + fastvideo/training/wan_distillation_v3.py | 46 ++ 11 files changed, 1350 insertions(+), 5 deletions(-) create mode 100644 dev/phases/phase_1.md create mode 100644 examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh create mode 100644 fastvideo/distillation/builder.py create mode 100644 fastvideo/distillation/methods/consistency_model/__init__.py create mode 100644 fastvideo/distillation/methods/distribution_matching/__init__.py create mode 100644 fastvideo/distillation/methods/distribution_matching/dmd2.py create mode 100644 fastvideo/distillation/methods/fine_tuning/__init__.py create mode 100644 fastvideo/distillation/methods/knowledge_distillation/__init__.py create mode 100644 fastvideo/training/wan_distillation_v3.py diff --git a/dev/phases/phase_1.md b/dev/phases/phase_1.md new file mode 100644 index 000000000..9073816d6 --- /dev/null +++ b/dev/phases/phase_1.md @@ -0,0 +1,169 @@ +# Phase 1：把 DMD2 “抠出来”（算法与 Wan 解耦）+ Builder 雏形 + +Phase 0 已经证明：`Trainer/Method/Adapter/Bundle` 这套骨架是能跑通的（Wan DMD2 也已实测 +能收敛/validation 变清晰）。但 Phase 0 的核心问题仍然存在： + +- `WanDMD2Method` 仍在调用 legacy pipeline 的私有函数（`_generator_forward/_dmd_forward/ + faker_score_forward/...`），本质还是 **Wan+方法耦合**。 +- `WanPipelineAdapter` 仍依赖 legacy pipeline 的 helper（normalize/prepare inputs/build metadata）。 +- 启动路径仍然是“先起 legacy pipeline，再手工拼 bundle+method+trainer”，builder 还只是散落在 + entrypoint 里的一段 glue code。 + +Phase 1 的定位：**在不破坏 Phase 0 可跑性的前提下**，把算法层从 Wan/pipeline 中抽离出来， +建立稳定边界：`Method(算法)` 只依赖 `Adapter(模型 primitives)` + `ModelBundle(roles)`。 + +--- + +## Phase 1 目标（可交付） + +1. 产出通用算法实现：`DMD2Method`（不再 import/调用 `fastvideo/training/distillation_pipeline.py` + 的任何私有函数）。 +2. 产出可复用的 Wan 适配层：`WanAdapter`（逐步替代 `WanPipelineAdapter`，目标是不再依赖 legacy + pipeline 的 helper；Phase 1 允许“过渡期双实现并存”）。 +3. 落地 Builder 雏形：从 FastVideo 现有 args/config（`TrainingArgs/FastVideoArgs`，或新增一个 + `--models_json`）构建： + - `ModelBundle(roles={...})` + - `Adapter` + - `Method` + 并让 entrypoint 变成“选择 + instantiate”，而不是手写胶水。 +4. 保持 Phase 0 路径可用：`wan_distillation_v2.py` + `WanDMD2Method` 暂不删除（仅标注过渡/弃用）。 + +--- + +## Phase 1 非目标（明确不做 / 延后） + +- Self-forcing v2（以及 ODE-init）—— Phase 2 再做 +- role-based checkpoint/save/resume 协议—— Phase 2 再统一 +- 多模型家族混搭（例如 student=Wan、teacher=SDXL）—— 先不承诺 +- 完整的 validation 抽象（先保留 `method.log_validation` hook） + +--- + +## Phase 1 TODO List（Review Checklist） + +> Phase 1 的 checklist 会在实施过程中持续更新、打钩（像 Phase 0 一样）。 + +### A. 方法目录结构（对齐 FastGen 的“分层 catalog”思路） + +- [x] 建立 methods 分层目录（至少有目录与 `__init__.py`） + - `fastvideo/distillation/methods/distribution_matching/` + - `fastvideo/distillation/methods/consistency_model/` + - `fastvideo/distillation/methods/knowledge_distillation/` + - `fastvideo/distillation/methods/fine_tuning/` +- [x] 把 Phase 1 的 `DMD2Method` 放到 + `fastvideo/distillation/methods/distribution_matching/dmd2.py` + +### B. 通用 `DMD2Method`（算法层） + +- [x] 新增 `DMD2Method`（算法实现），并显式 `bundle.require_roles([...])` + - 最小要求：`student/teacher/critic`（未来可扩展 role，但 Phase 1 先按 DMD2 固定需求） +- [x] `DMD2Method` 不持有 pipeline，不调用 legacy pipeline 私有函数 +- [x] 保留 Phase 0 的关键语义： + - generator update interval（`generator_update_interval`） + - backward 期 forward recompute 的 `forward_context` 约束（要么由 adapter 管，要么 method 管） + +### C. Adapter 接口升级（让算法真正可复用） + +- [x] 定义“DMD2 需要的 adapter primitives”（以 Protocol 的方式定义在 `DMD2Method` 内部） + - `prepare_batch(...)` + - `student_predict_x0(...)`（或“generator_pred_video”等等价语义） + - `teacher_predict_x0(cond/uncond, guidance_scale, ...)` + - `critic_flow_matching_loss(...)`（或拆成 critic forward + target 构造） + - `add_noise(...) / sample_timestep(...) / shift+clamp timestep(...)` + - `forward_context(...)`（如果决定由 adapter 托管 forward_context） +- [x] 让 `DMD2Method` 只依赖这些 primitives（而不是 Wan 细节） + +### D. `WanAdapter`（模型层，逐步摆脱 pipeline helper） + +- [x] 在 `fastvideo/distillation/adapters/wan.py` 中新增 `WanAdapter` + - 输入：`training_args` + `noise_scheduler` + `ModelBundle`（或 bundle role handles） + - 输出：实现 Phase 1 定义的 DMD2 primitives +- [x] 把以下逻辑从 legacy pipeline/helper 迁出到 adapter（Phase 1 做到“可跑通”即可） + - `_build_distill_input_kwargs`（transformer forward 的输入组装） + - `_get_real_score_transformer/_get_fake_score_transformer`（Phase 1 先保证 teacher 侧可选 transformer_2；critic MoE/optimizer 选择后续再补齐） + - `denoising_step_list` 构造、warp 逻辑、`min/max_timestep`、`timestep_shift` + - `ensure_negative_conditioning()`（Phase 0 已有，Phase 1 要确保能被复用） +- [ ] `WanPipelineAdapter` 继续保留（Phase 0 兜底），但在文档/代码里标注为“legacy-backed” + +### E. Builder 雏形（config -> instantiate） + +- [x] 新增 builder/registry（先落 `fastvideo/distillation/builder.py`，Phase 1 仅实现 Wan） + - `build_models(args) -> ModelBundle + Adapter`（Phase 1 先实现 Wan） + - `build_method(args, bundle, adapter) -> DistillMethod`（DMD2 先实现） + - 让 entrypoint 不再手写 `_build_bundle_from_wan_pipeline(...)` +- [x] 新增一个“Phase 1 entrypoint”（`fastvideo/training/wan_distillation_v3.py`） + - Phase 1 先支持：Wan + DMD2（student/teacher/critic） + - Phase 0 入口 `wan_distillation_v2.py` 不动（便于 A/B） + +### F. 示例脚本（Phase 1） + +- [x] 新增 `examples/distillation/phase1/` 的脚本 + - 目标：用户只改路径就能启动 “Wan 1.3B 学 14B，8 steps distill” 的 DMD2 + - 如果 Phase 1 引入 `--models_json`：提供一个最小 JSON 模板（写在脚本注释里） + +### G. 最小单测（CPU；可选但推荐） + +> 你说过 GPU test 你会自己跑；CPU test 主要用于锁定“调度/接口”不回归。 + +- [ ] 保留 Phase 0 的 schedule test（不改语义） +- [ ] 为 Phase 1 新增 1 个最小单测（不需要真实模型）： + - builder 能按 role 组装 optimizers/schedulers（或 method 能正确选择 opt/sched） + +--- + +## 关键设计决策点（出现风险就停下问你） + +### 决策点 1：`forward_context` 由谁管理？ + +背景：FastVideo 的 backward 可能触发 forward 重算，重算必须处于正确的 +`set_forward_context(current_timestep, attn_metadata)` 中。 + +**决定：Adapter 管理（你拍板）。** + +落地方式（Phase 1 采用）： + +- Method 不 import `fastvideo.forward_context`，也不直接调用 `set_forward_context(...)` +- Adapter 提供显式的上下文 API（例如 `adapter.student_context(...) / adapter.critic_context(...)`） + 以及必要时的 `adapter.backward_*()` 封装，确保 activation checkpointing 触发的 forward 重算也在 + 正确的 context 中执行 + +如果后续实现发现 adapter 侧会导致接口/实现“非常不优雅”（比如需要过多特殊 case），我会停下并 +汇报尝试与失败原因，再一起讨论是否回退到“Method 管理”或引入更好的抽象。 + +### 决策点 2：Builder 是否在 Phase 1 “彻底摆脱 legacy pipeline”？ + +Phase 1 的最小目标是“把胶水集中起来”，不必一次性重写所有加载/optimizer/dataloader 逻辑。 +如果我们发现不依赖 pipeline 会导致需要大规模复制 loader/optimizer 初始化逻辑， +我会建议 Phase 1 先做： + +- builder 内部仍可调用 `WanDistillationPipeline.from_pretrained(...)` 完成加载 +- 但 **method/adapter 不再依赖 pipeline 私有算法函数** + +是否要把“加载也完全重写”强行塞进 Phase 1，是一个风险点，需要你决定优先级。 + +--- + +## 代码落地点（具体到文件；Phase 1 预计会改/新增这些） + +- 新增： + - `fastvideo/distillation/methods/distribution_matching/dmd2.py` + - `fastvideo/distillation/methods/distribution_matching/__init__.py` + - `fastvideo/distillation/methods/consistency_model/__init__.py`（空壳） + - `fastvideo/distillation/methods/knowledge_distillation/__init__.py`（空壳） + - `fastvideo/distillation/methods/fine_tuning/__init__.py`（空壳） + - `fastvideo/distillation/builder.py`（或 `fastvideo/distillation/builders/*`） + - `fastvideo/training/distill_v3.py`（Phase 1 新入口，名字可再讨论） + - `examples/distillation/phase1/*` +- 修改： + - `fastvideo/distillation/adapters/base.py`（扩展/引入 protocol） + - `fastvideo/distillation/adapters/wan.py`（新增 `WanAdapter`，保留 `WanPipelineAdapter`） + - `fastvideo/distillation/methods/__init__.py`（导出新 method） + - （可选）`fastvideo/distillation/methods/wan_dmd2.py`（标注 deprecated + 逐步迁移） + +--- + +## Phase 1 完成标准（Definition of Done） + +- `DMD2Method` 存在且可运行，且不依赖 `fastvideo/training/distillation_pipeline.py` 的私有函数 +- Wan DMD2 的训练可以走 Phase 1 新入口（至少 smoke 跑通；GPU A/B 由你后续验证） +- Phase 0 入口仍可用（便于对齐/回滚） diff --git a/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh new file mode 100644 index 000000000..52fc2a0d4 --- /dev/null +++ b/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -e -x + +# Phase 1 example: run Wan DMD2 distillation via DMD2Method + WanAdapter entrypoint. +# Note: validation is currently best-effort; Phase 1 focuses on algorithm/model decoupling. + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_PORT=${MASTER_PORT:-29504} + +NUM_GPUS=${NUM_GPUS:-1} + +# Models +STUDENT_MODEL_PATH=${STUDENT_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} +# For best distillation, point TEACHER_MODEL_PATH to a stronger teacher (e.g. 14B). +# For a cheaper smoke run, set it to the same 1.3B model. +TEACHER_MODEL_PATH=${TEACHER_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-14B-Diffusers"} +CRITIC_MODEL_PATH=${CRITIC_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} + +# Data (parquet dataset folder) +DATA_DIR=${DATA_DIR:-"your_data_dir"} +VALIDATION_DATASET_FILE=${VALIDATION_DATASET_FILE:-"your_validation_dataset_file"} + +OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase1_wan2.1_t2v_1.3B_dmd2_8steps"} + +training_args=( + --tracker_project_name "phase1_wan_dmd2_8steps" + --output_dir "$OUTPUT_DIR" + --max_train_steps 4000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 21 + --num_height 480 + --num_width 832 + --num_frames 81 + --enable_gradient_checkpointing_type "full" + --simulate_generator_forward +) + +parallel_args=( + --num_gpus "$NUM_GPUS" + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim "$NUM_GPUS" +) + +model_args=( + --model_path "$STUDENT_MODEL_PATH" + --pretrained_model_name_or_path "$STUDENT_MODEL_PATH" + --real_score_model_path "$TEACHER_MODEL_PATH" + --fake_score_model_path "$CRITIC_MODEL_PATH" +) + +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 50 + --validation_sampling_steps "8" + --validation_guidance_scale "6.0" # not used for dmd inference +) + +optimizer_args=( + --learning_rate 1e-5 + --mixed_precision "bf16" + --weight_decay 0.01 + --betas '0.0,0.999' + --max_grad_norm 1.0 + --fake_score_learning_rate 8e-6 + --fake_score_betas '0.0,0.999' +) + +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.0 + --dit_precision "fp32" + --flow_shift 8 + --seed 1000 +) + +dmd_args=( + # 8-step schedule (same as Wan2.2 self-forcing examples) + --dmd_denoising_steps '1000,850,700,550,350,275,200,125' + --min_timestep_ratio 0.02 + --max_timestep_ratio 0.98 + --generator_update_interval 5 + --real_score_guidance_scale 3.5 +) + +torchrun \ +--nnodes 1 \ +--master_port "$MASTER_PORT" \ +--nproc_per_node "$NUM_GPUS" \ + fastvideo/training/wan_distillation_v3.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ + "${dmd_args[@]}" + diff --git a/fastvideo/distillation/adapters/base.py b/fastvideo/distillation/adapters/base.py index bb6d1db0d..a424eb3e8 100644 --- a/fastvideo/distillation/adapters/base.py +++ b/fastvideo/distillation/adapters/base.py @@ -3,9 +3,10 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Any +from typing import TYPE_CHECKING, Any -from fastvideo.pipelines import TrainingBatch +if TYPE_CHECKING: + from fastvideo.pipelines import TrainingBatch class DistillAdapter(ABC): @@ -15,6 +16,5 @@ def prepare_batch( raw_batch: dict[str, Any], *, current_vsa_sparsity: float = 0.0, - ) -> TrainingBatch: + ) -> "TrainingBatch": raise NotImplementedError - diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 5623d40e8..cd79f05b3 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -8,15 +8,38 @@ import torch +import fastvideo.envs as envs from fastvideo.configs.sample import SamplingParam -from fastvideo.distributed import get_local_torch_device, get_world_group +from fastvideo.distributed import ( + get_local_torch_device, + get_sp_group, + get_world_group, +) +from fastvideo.forward_context import set_forward_context +from fastvideo.models.utils import pred_noise_to_pred_video from fastvideo.pipelines import TrainingBatch from fastvideo.pipelines.pipeline_batch_info import ForwardBatch from fastvideo.pipelines.basic.wan.wan_dmd_pipeline import WanDMDPipeline from fastvideo.training.distillation_pipeline import DistillationPipeline +from fastvideo.training.training_utils import ( + compute_density_for_timestep_sampling, + get_sigmas, + normalize_dit_input, + shift_timestep, +) +from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed from fastvideo.distillation.adapters.base import DistillAdapter +try: + from fastvideo.attention.backends.video_sparse_attn import ( + VideoSparseAttentionMetadataBuilder, + ) + from fastvideo.attention.backends.vmoba import VideoMobaAttentionMetadataBuilder +except Exception: + VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] + VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] + class WanPipelineAdapter(DistillAdapter): def __init__(self, pipeline: DistillationPipeline) -> None: @@ -178,3 +201,686 @@ def prepare_batch( training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] return training_batch + + +class WanAdapter(DistillAdapter): + """ + Phase 1 target adapter: provide Wan-specific primitives without calling + legacy distillation pipeline algorithm helpers (e.g. `_dmd_forward`). + """ + + def __init__( + self, + *, + bundle: Any, + training_args: Any, + noise_scheduler: Any, + vae: Any, + validation_pipeline: DistillationPipeline | None = None, + ) -> None: + self.bundle = bundle + self.training_args = training_args + self.noise_scheduler = noise_scheduler + self.vae = vae + self._validation_pipeline_owner = validation_pipeline + + self.world_group = get_world_group() + self.sp_group = get_sp_group() + self.device = get_local_torch_device() + + self.noise_random_generator: torch.Generator | None = None + self.noise_gen_cuda: torch.Generator | None = None + + self.negative_prompt_embeds: torch.Tensor | None = None + self.negative_prompt_attention_mask: torch.Tensor | None = None + + self._init_dmd2_schedule() + + def _get_training_dtype(self) -> torch.dtype: + return torch.bfloat16 + + def _init_dmd2_schedule(self) -> None: + self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) + self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) + self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) + self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) + + self.denoising_step_list = torch.tensor( + self.training_args.pipeline_config.dmd_denoising_steps, + dtype=torch.long, + device=self.device, + ) + if getattr(self.training_args, "warp_denoising_step", False): + timesteps = torch.cat( + ( + self.noise_scheduler.timesteps.to("cpu"), + torch.tensor([0], dtype=torch.float32), + ) + ).to(self.device) + self.denoising_step_list = timesteps[1000 - self.denoising_step_list] + self.denoising_step_list = self.denoising_step_list.to(self.device) + + if getattr(self.training_args, "boundary_ratio", None) is not None: + self.boundary_timestep = float(self.training_args.boundary_ratio) * float( + self.num_train_timestep + ) + else: + self.boundary_timestep = None + + def on_train_start(self) -> None: + seed = self.training_args.seed + if seed is None: + raise ValueError("training_args.seed must be set for distillation") + + global_rank = int(getattr(self.world_group, "rank", 0)) + sp_world_size = int(getattr(self.training_args, "sp_size", 1) or 1) + if sp_world_size > 1: + sp_group_seed = int(seed) + (global_rank // sp_world_size) + set_random_seed(sp_group_seed) + else: + set_random_seed(int(seed) + global_rank) + + self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) + self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) + + self.ensure_negative_conditioning() + + def ensure_negative_conditioning(self) -> None: + if self.negative_prompt_embeds is not None: + return + + training_args = self.training_args + world_group = self.world_group + device = self.device + dtype = self._get_training_dtype() + + neg_embeds: torch.Tensor | None = None + neg_mask: torch.Tensor | None = None + + if world_group.rank_in_group == 0: + sampling_param = SamplingParam.from_pretrained(training_args.model_path) + negative_prompt = sampling_param.negative_prompt + + args_copy = copy.deepcopy(training_args) + args_copy.inference_mode = True + + student_transformer = self.bundle.role("student").require_module("transformer") + prompt_pipeline = WanDMDPipeline.from_pretrained( + training_args.model_path, + args=args_copy, + inference_mode=True, + loaded_modules={"transformer": student_transformer}, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + pin_cpu_memory=training_args.pin_cpu_memory, + dit_cpu_offload=True, + ) + + batch_negative = ForwardBatch( + data_type="video", + prompt=negative_prompt, + prompt_embeds=[], + prompt_attention_mask=[], + ) + result_batch = prompt_pipeline.prompt_encoding_stage( # type: ignore[attr-defined] + batch_negative, + training_args, + ) + + neg_embeds = result_batch.prompt_embeds[0].to(device=device, dtype=dtype) + neg_mask = result_batch.prompt_attention_mask[0].to(device=device, dtype=dtype) + + del prompt_pipeline + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + meta = torch.zeros((2,), device=device, dtype=torch.int64) + if world_group.rank_in_group == 0: + assert neg_embeds is not None + assert neg_mask is not None + meta[0] = neg_embeds.ndim + meta[1] = neg_mask.ndim + world_group.broadcast(meta, src=0) + embed_ndim, mask_ndim = (int(meta[0].item()), int(meta[1].item())) + + max_ndim = 8 + embed_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) + mask_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) + if world_group.rank_in_group == 0: + assert neg_embeds is not None + assert neg_mask is not None + embed_shape[:embed_ndim] = torch.tensor( + list(neg_embeds.shape), device=device, dtype=torch.int64 + ) + mask_shape[:mask_ndim] = torch.tensor( + list(neg_mask.shape), device=device, dtype=torch.int64 + ) + world_group.broadcast(embed_shape, src=0) + world_group.broadcast(mask_shape, src=0) + + embed_sizes = tuple(int(x) for x in embed_shape[:embed_ndim].tolist()) + mask_sizes = tuple(int(x) for x in mask_shape[:mask_ndim].tolist()) + + if world_group.rank_in_group != 0: + neg_embeds = torch.empty(embed_sizes, device=device, dtype=dtype) + neg_mask = torch.empty(mask_sizes, device=device, dtype=dtype) + assert neg_embeds is not None + assert neg_mask is not None + + world_group.broadcast(neg_embeds, src=0) + world_group.broadcast(neg_mask, src=0) + + self.negative_prompt_embeds = neg_embeds + self.negative_prompt_attention_mask = neg_mask + + def log_validation(self, iteration: int) -> None: + pipeline = self._validation_pipeline_owner + if pipeline is None: + return + training_args = pipeline.training_args + if not getattr(training_args, "log_validation", False): + return + + if getattr(pipeline, "validation_pipeline", None) is None: + pipeline.initialize_validation_pipeline(training_args) + + student_transformer = self.bundle.role("student").require_module("transformer") + + old_inference_mode = training_args.inference_mode + old_dit_cpu_offload = training_args.dit_cpu_offload + try: + pipeline._log_validation( + student_transformer, + training_args, + iteration, + ) + finally: + training_args.inference_mode = old_inference_mode + training_args.dit_cpu_offload = old_dit_cpu_offload + + def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: + if self.noise_random_generator is None: + raise RuntimeError("WanAdapter.on_train_start() must be called before prepare_batch()") + + u = compute_density_for_timestep_sampling( + weighting_scheme=self.training_args.weighting_scheme, + batch_size=batch_size, + generator=self.noise_random_generator, + logit_mean=self.training_args.logit_mean, + logit_std=self.training_args.logit_std, + mode_scale=self.training_args.mode_scale, + ) + indices = (u * self.noise_scheduler.config.num_train_timesteps).long() + return self.noise_scheduler.timesteps[indices].to(device=device) + + def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: + latents_shape = training_batch.raw_latent_shape + patch_size = self.training_args.pipeline_config.dit_config.patch_size + current_vsa_sparsity = training_batch.current_vsa_sparsity + assert latents_shape is not None + assert training_batch.timesteps is not None + + if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": + if not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None: + raise ImportError( + "FASTVIDEO_ATTENTION_BACKEND is VIDEO_SPARSE_ATTN, but fastvideo_kernel " + "is not correctly installed or detected." + ) + training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] + raw_latent_shape=latents_shape[2:5], + current_timestep=training_batch.timesteps, + patch_size=patch_size, + VSA_sparsity=current_vsa_sparsity, + device=self.device, + ) + elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": + if not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None: + raise ImportError( + "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, but fastvideo_kernel " + "(or flash_attn>=2.7.4) is not correctly installed." + ) + moba_params = self.training_args.moba_config.copy() + moba_params.update( + { + "current_timestep": training_batch.timesteps, + "raw_latent_shape": training_batch.raw_latent_shape[2:5], + "patch_size": patch_size, + "device": self.device, + } + ) + training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(**moba_params) # type: ignore[misc] + else: + training_batch.attn_metadata = None + + return training_batch + + def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: + latents = training_batch.latents + batch_size = latents.shape[0] + if self.noise_gen_cuda is None: + raise RuntimeError("WanAdapter.on_train_start() must be called before prepare_batch()") + + noise = torch.randn( + latents.shape, + generator=self.noise_gen_cuda, + device=latents.device, + dtype=latents.dtype, + ) + timesteps = self._sample_timesteps(batch_size, latents.device) + if int(getattr(self.training_args, "sp_size", 1) or 1) > 1: + self.sp_group.broadcast(timesteps, src=0) + + sigmas = get_sigmas( + self.noise_scheduler, + latents.device, + timesteps, + n_dim=latents.ndim, + dtype=latents.dtype, + ) + noisy_model_input = (1.0 - sigmas) * latents + sigmas * noise + + training_batch.noisy_model_input = noisy_model_input + training_batch.timesteps = timesteps + training_batch.sigmas = sigmas + training_batch.noise = noise + training_batch.raw_latent_shape = latents.shape + + training_batch.conditional_dict = { + "encoder_hidden_states": training_batch.encoder_hidden_states, + "encoder_attention_mask": training_batch.encoder_attention_mask, + } + + if self.negative_prompt_embeds is not None and self.negative_prompt_attention_mask is not None: + neg_embeds = self.negative_prompt_embeds + neg_mask = self.negative_prompt_attention_mask + if neg_embeds.shape[0] == 1 and batch_size > 1: + neg_embeds = neg_embeds.expand(batch_size, *neg_embeds.shape[1:]).contiguous() + if neg_mask.shape[0] == 1 and batch_size > 1: + neg_mask = neg_mask.expand(batch_size, *neg_mask.shape[1:]).contiguous() + training_batch.unconditional_dict = { + "encoder_hidden_states": neg_embeds, + "encoder_attention_mask": neg_mask, + } + + training_batch.dmd_latent_vis_dict = {} + training_batch.fake_score_latent_vis_dict = {} + + training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) + return training_batch + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + ) -> TrainingBatch: + self.ensure_negative_conditioning() + + dtype = self._get_training_dtype() + device = self.device + + training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + encoder_hidden_states = raw_batch["text_embedding"] + encoder_attention_mask = raw_batch["text_attention_mask"] + infos = raw_batch.get("info_list") + + if self.training_args.simulate_generator_forward: + batch_size = encoder_hidden_states.shape[0] + vae_config = self.training_args.pipeline_config.vae_config.arch_config + num_channels = vae_config.z_dim + spatial_compression_ratio = vae_config.spatial_compression_ratio + latent_height = self.training_args.num_height // spatial_compression_ratio + latent_width = self.training_args.num_width // spatial_compression_ratio + latents = torch.zeros( + batch_size, + num_channels, + self.training_args.num_latent_t, + latent_height, + latent_width, + device=device, + dtype=dtype, + ) + else: + if "vae_latent" not in raw_batch: + raise ValueError( + "vae_latent not found in batch and simulate_generator_forward is False" + ) + latents = raw_batch["vae_latent"] + latents = latents[:, :, : self.training_args.num_latent_t] + latents = latents.to(device, dtype=dtype) + + training_batch.latents = latents + training_batch.encoder_hidden_states = encoder_hidden_states.to(device, dtype=dtype) + training_batch.encoder_attention_mask = encoder_attention_mask.to(device, dtype=dtype) + training_batch.infos = infos + + training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch = self._prepare_dit_inputs(training_batch) + training_batch = self._build_attention_metadata(training_batch) + + training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) + if training_batch.attn_metadata is not None: + training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] + + return training_batch + + def sample_dmd_timestep(self, *, device: torch.device) -> torch.Tensor: + timestep = torch.randint( + 0, + self.num_train_timestep, + [1], + device=device, + dtype=torch.long, + ) + timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) + return timestep.clamp(self.min_timestep, self.max_timestep) + + def add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + b, t = clean_latents.shape[:2] + noisy = self.noise_scheduler.add_noise( + clean_latents.flatten(0, 1), + noise.flatten(0, 1), + timestep, + ).unflatten(0, (b, t)) + return noisy + + def _build_distill_input_kwargs( + self, + noise_input: torch.Tensor, + timestep: torch.Tensor, + text_dict: dict[str, torch.Tensor] | None, + ) -> dict[str, Any]: + if text_dict is None: + raise ValueError("text_dict cannot be None for Wan distillation") + return { + "hidden_states": noise_input.permute(0, 2, 1, 3, 4), + "encoder_hidden_states": text_dict["encoder_hidden_states"], + "encoder_attention_mask": text_dict["encoder_attention_mask"], + "timestep": timestep, + "return_dict": False, + } + + def _get_teacher_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: + role = self.bundle.role("teacher") + transformer = role.require_module("transformer") + transformer_2 = role.modules.get("transformer_2") + if transformer_2 is not None and self.boundary_timestep is not None: + if float(timestep.item()) < self.boundary_timestep: + return transformer_2 + return transformer + + def _get_critic_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: + # Phase 1: prefer a single critic transformer unless MoE is fully wired. + role = self.bundle.role("critic") + return role.require_module("transformer") + + def student_rollout(self, batch: TrainingBatch) -> tuple[torch.Tensor, Any]: + device_type = self.device.type + dtype = batch.latents.dtype + with torch.autocast(device_type, dtype=dtype): + with set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata_vsa, + ): + if self.training_args.simulate_generator_forward: + pred_x0 = self._student_multi_step_simulation(batch) + else: + pred_x0 = self._student_single_step(batch) + return pred_x0, (batch.timesteps, batch.attn_metadata_vsa) + + def _student_single_step(self, batch: TrainingBatch) -> torch.Tensor: + latents = batch.latents + b, t = latents.shape[:2] + index = torch.randint( + 0, + len(self.denoising_step_list), + [1], + device=self.device, + dtype=torch.long, + ) + timestep = self.denoising_step_list[index] + batch.dmd_latent_vis_dict["generator_timestep"] = timestep + + noise = torch.randn(latents.shape, device=self.device, dtype=latents.dtype) + noisy_latent = self.noise_scheduler.add_noise( + latents.flatten(0, 1), + noise.flatten(0, 1), + timestep, + ).unflatten(0, (b, t)) + + input_kwargs = self._build_distill_input_kwargs( + noisy_latent, + timestep, + batch.conditional_dict, + ) + transformer = self.bundle.role("student").require_module("transformer") + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latent.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + def _student_multi_step_simulation(self, batch: TrainingBatch) -> torch.Tensor: + latents = batch.latents + dtype = latents.dtype + + target_timestep_idx = torch.randint( + 0, + len(self.denoising_step_list), + [1], + device=self.device, + dtype=torch.long, + ) + target_timestep_idx_int = int(target_timestep_idx.item()) + target_timestep = self.denoising_step_list[target_timestep_idx] + + current_noise_latents = torch.randn(latents.shape, device=self.device, dtype=dtype) + current_noise_latents_copy = current_noise_latents.clone() + + max_target_idx = len(self.denoising_step_list) - 1 + noise_latents: list[torch.Tensor] = [] + noise_latent_index = target_timestep_idx_int - 1 + + transformer = self.bundle.role("student").require_module("transformer") + + if max_target_idx > 0: + with torch.no_grad(): + for step_idx in range(max_target_idx): + current_timestep = self.denoising_step_list[step_idx] + current_timestep_tensor = current_timestep * torch.ones( + 1, device=self.device, dtype=torch.long + ) + input_kwargs = self._build_distill_input_kwargs( + current_noise_latents, + current_timestep_tensor, + batch.conditional_dict, + ) + pred_flow = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_clean = pred_noise_to_pred_video( + pred_noise=pred_flow.flatten(0, 1), + noise_input_latent=current_noise_latents.flatten(0, 1), + timestep=current_timestep_tensor, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_flow.shape[:2]) + + next_timestep = self.denoising_step_list[step_idx + 1] + next_timestep_tensor = next_timestep * torch.ones( + 1, device=self.device, dtype=torch.long + ) + noise = torch.randn( + latents.shape, device=self.device, dtype=pred_clean.dtype + ) + b, t = pred_clean.shape[:2] + current_noise_latents = self.noise_scheduler.add_noise( + pred_clean.flatten(0, 1), + noise.flatten(0, 1), + next_timestep_tensor, + ).unflatten(0, (b, t)) + noise_latents.append(current_noise_latents.clone()) + + if noise_latent_index >= 0: + if noise_latent_index >= len(noise_latents): + raise RuntimeError("noise_latent_index is out of bounds") + noisy_input = noise_latents[noise_latent_index] + else: + noisy_input = current_noise_latents_copy + + input_kwargs = self._build_distill_input_kwargs( + noisy_input, + target_timestep, + batch.conditional_dict, + ) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_input.flatten(0, 1), + timestep=target_timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + + batch.dmd_latent_vis_dict["generator_timestep"] = target_timestep.float().detach() + return pred_x0 + + def teacher_predict_x0( + self, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + text_dict = batch.conditional_dict if conditional else getattr(batch, "unconditional_dict", None) + if text_dict is None: + raise RuntimeError("Missing unconditional_dict; ensure_negative_conditioning() may have failed") + + with torch.autocast(device_type, dtype=dtype): + with set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) + transformer = self._get_teacher_transformer(timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + def critic_predict_x0( + self, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + with torch.autocast(device_type, dtype=dtype): + with set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep, + batch.conditional_dict, + ) + transformer = self._get_critic_transformer(timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + def critic_flow_matching_loss( + self, + batch: TrainingBatch, + ) -> tuple[torch.Tensor, Any, dict[str, Any]]: + device_type = self.device.type + dtype = batch.latents.dtype + + with torch.no_grad(): + with torch.autocast(device_type, dtype=dtype): + with set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata_vsa, + ): + if self.training_args.simulate_generator_forward: + generator_pred_x0 = self._student_multi_step_simulation(batch) + else: + generator_pred_x0 = self._student_single_step(batch) + + fake_score_timestep = torch.randint( + 0, + self.num_train_timestep, + [1], + device=self.device, + dtype=torch.long, + ) + fake_score_timestep = shift_timestep( + fake_score_timestep, + self.timestep_shift, + self.num_train_timestep, + ).clamp(self.min_timestep, self.max_timestep) + + noise = torch.randn( + generator_pred_x0.shape, + device=self.device, + dtype=generator_pred_x0.dtype, + ) + b, t = generator_pred_x0.shape[:2] + noisy_x0 = self.noise_scheduler.add_noise( + generator_pred_x0.flatten(0, 1), + noise.flatten(0, 1), + fake_score_timestep, + ).unflatten(0, (b, t)) + + with torch.autocast(device_type, dtype=dtype): + with set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs( + noisy_x0, + fake_score_timestep, + batch.conditional_dict, + ) + transformer = self._get_critic_transformer(fake_score_timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + + target = noise - generator_pred_x0 + flow_matching_loss = torch.mean((pred_noise - target) ** 2) + + batch.fake_score_latent_vis_dict = { + "generator_pred_video": generator_pred_x0, + "fake_score_timestep": fake_score_timestep, + } + outputs = {"fake_score_latent_vis_dict": batch.fake_score_latent_vis_dict} + return flow_matching_loss, (batch.timesteps, batch.attn_metadata), outputs + + def backward_student(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + timesteps, attn_metadata = ctx + with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): + (loss / max(1, int(grad_accum_rounds))).backward() + + def backward_critic(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + timesteps, attn_metadata = ctx + with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): + (loss / max(1, int(grad_accum_rounds))).backward() diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py new file mode 100644 index 000000000..a4fdc0ab4 --- /dev/null +++ b/fastvideo/distillation/builder.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from fastvideo.distillation.adapters.wan import WanAdapter +from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method +from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline + + +def build_wan_dmd2_method( + pipeline: WanDistillationPipeline, +) -> DMD2Method: + roles: dict[str, RoleHandle] = { + "student": + RoleHandle( + modules={"transformer": pipeline.transformer}, + optimizers={"main": pipeline.optimizer}, + lr_schedulers={"main": pipeline.lr_scheduler}, + ), + "teacher": + RoleHandle( + modules={"transformer": pipeline.real_score_transformer}, + frozen=True, + ), + "critic": + RoleHandle( + modules={"transformer": pipeline.fake_score_transformer}, + optimizers={"main": pipeline.fake_score_optimizer}, + lr_schedulers={"main": pipeline.fake_score_lr_scheduler}, + ), + } + bundle = ModelBundle(roles=roles) + adapter = WanAdapter( + bundle=bundle, + training_args=pipeline.training_args, + noise_scheduler=pipeline.noise_scheduler, + vae=pipeline.get_module("vae"), + validation_pipeline=pipeline, + ) + return DMD2Method(bundle=bundle, adapter=adapter) diff --git a/fastvideo/distillation/methods/consistency_model/__init__.py b/fastvideo/distillation/methods/consistency_model/__init__.py new file mode 100644 index 000000000..57c85a008 --- /dev/null +++ b/fastvideo/distillation/methods/consistency_model/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 + +__all__: list[str] = [] + diff --git a/fastvideo/distillation/methods/distribution_matching/__init__.py b/fastvideo/distillation/methods/distribution_matching/__init__.py new file mode 100644 index 000000000..061614127 --- /dev/null +++ b/fastvideo/distillation/methods/distribution_matching/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method + +__all__ = [ + "DMD2Method", +] + diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py new file mode 100644 index 000000000..7aed5e9e1 --- /dev/null +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -0,0 +1,250 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any, Protocol + +import torch +import torch.nn.functional as F + +from fastvideo.training.training_utils import ( + clip_grad_norm_while_handling_failing_dtensor_cases, +) + +from fastvideo.distillation.bundle import ModelBundle +from fastvideo.distillation.methods.base import DistillMethod + + +class _DMD2Adapter(Protocol): + training_args: Any + + def on_train_start(self) -> None: + ... + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + ) -> Any: + ... + + def student_rollout(self, batch: Any) -> tuple[torch.Tensor, Any]: + ... + + def sample_dmd_timestep(self, *, device: torch.device) -> torch.Tensor: + ... + + def add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + ... + + def teacher_predict_x0( + self, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: Any, + *, + conditional: bool, + ) -> torch.Tensor: + ... + + def critic_predict_x0( + self, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: Any, + ) -> torch.Tensor: + ... + + def critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: + ... + + def backward_student(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + ... + + def backward_critic(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + ... + + def log_validation(self, iteration: int) -> None: + ... + + +class DMD2Method(DistillMethod): + def __init__( + self, + *, + bundle: ModelBundle, + adapter: _DMD2Adapter, + ) -> None: + super().__init__(bundle) + bundle.require_roles(["student", "teacher", "critic"]) + self.adapter = adapter + self.training_args = adapter.training_args + + def on_train_start(self) -> None: + self.adapter.on_train_start() + + def log_validation(self, iteration: int) -> None: + if hasattr(self.adapter, "log_validation"): + self.adapter.log_validation(iteration) + + def _should_update_student(self, iteration: int) -> bool: + interval = int(getattr(self.training_args, "generator_update_interval", 1) or 1) + if interval <= 0: + return True + return iteration % interval == 0 + + def _clip_grad_norm(self, module: torch.nn.Module) -> float: + max_grad_norm = getattr(self.training_args, "max_grad_norm", None) + if not max_grad_norm: + return 0.0 + grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in module.parameters()], + float(max_grad_norm), + foreach=None, + ) + return float(grad_norm.item()) if grad_norm is not None else 0.0 + + def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor: + guidance_scale = float(getattr(self.training_args, "real_score_guidance_scale", 1.0)) + device = generator_pred_x0.device + + with torch.no_grad(): + timestep = self.adapter.sample_dmd_timestep(device=device) + + noise = torch.randn( + generator_pred_x0.shape, + device=device, + dtype=generator_pred_x0.dtype, + ) + noisy_latents = self.adapter.add_noise(generator_pred_x0, noise, timestep) + + faker_x0 = self.adapter.critic_predict_x0(noisy_latents, timestep, batch) + real_cond_x0 = self.adapter.teacher_predict_x0( + noisy_latents, + timestep, + batch, + conditional=True, + ) + real_uncond_x0 = self.adapter.teacher_predict_x0( + noisy_latents, + timestep, + batch, + conditional=False, + ) + real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale + + denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() + grad = (faker_x0 - real_cfg_x0) / denom + grad = torch.nan_to_num(grad) + + loss = 0.5 * F.mse_loss( + generator_pred_x0.float(), + (generator_pred_x0.float() - grad.float()).detach(), + ) + return loss + + def single_train_step( + self, + batch: dict[str, Any], + iteration: int, + *, + current_vsa_sparsity: float = 0.0, + ) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + training_batch = self.adapter.prepare_batch( + batch, + current_vsa_sparsity=current_vsa_sparsity, + ) + + update_student = self._should_update_student(iteration) + + generator_loss = torch.zeros( + (), + device=training_batch.latents.device, + dtype=training_batch.latents.dtype, + ) + student_ctx = None + if update_student: + generator_pred_x0, student_ctx = self.adapter.student_rollout(training_batch) + generator_loss = self._dmd_loss(generator_pred_x0, training_batch) + + fake_score_loss, critic_ctx, critic_outputs = self.adapter.critic_flow_matching_loss( + training_batch + ) + + total_loss = generator_loss + fake_score_loss + loss_map = { + "total_loss": total_loss, + "generator_loss": generator_loss, + "fake_score_loss": fake_score_loss, + } + + outputs: dict[str, Any] = dict(critic_outputs) + outputs["_fv_backward"] = { + "update_student": update_student, + "student_ctx": student_ctx, + "critic_ctx": critic_ctx, + } + return loss_map, outputs + + def backward( + self, + loss_map: dict[str, torch.Tensor], + outputs: dict[str, Any], + *, + grad_accum_rounds: int = 1, + ) -> None: + grad_accum_rounds = max(1, int(grad_accum_rounds)) + backward_ctx = outputs.get("_fv_backward") + if not isinstance(backward_ctx, dict): + super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + return + + update_student = bool(backward_ctx.get("update_student", False)) + if update_student: + student_ctx = backward_ctx.get("student_ctx") + if student_ctx is None: + raise RuntimeError("Missing student backward context") + self.adapter.backward_student( + loss_map["generator_loss"], + student_ctx, + grad_accum_rounds=grad_accum_rounds, + ) + + critic_ctx = backward_ctx.get("critic_ctx") + if critic_ctx is None: + raise RuntimeError("Missing critic backward context") + self.adapter.backward_critic( + loss_map["fake_score_loss"], + critic_ctx, + grad_accum_rounds=grad_accum_rounds, + ) + + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + optimizers: list[torch.optim.Optimizer] = [] + optimizers.extend(self.bundle.role("critic").optimizers.values()) + if self._should_update_student(iteration): + optimizers.extend(self.bundle.role("student").optimizers.values()) + return optimizers + + def get_lr_schedulers(self, iteration: int) -> list[Any]: + schedulers: list[Any] = [] + schedulers.extend(self.bundle.role("critic").lr_schedulers.values()) + if self._should_update_student(iteration): + schedulers.extend(self.bundle.role("student").lr_schedulers.values()) + return schedulers + + def optimizers_schedulers_step(self, iteration: int) -> None: + if self._should_update_student(iteration): + for module in self.bundle.role("student").modules.values(): + self._clip_grad_norm(module) + for module in self.bundle.role("critic").modules.values(): + self._clip_grad_norm(module) + + super().optimizers_schedulers_step(iteration) + diff --git a/fastvideo/distillation/methods/fine_tuning/__init__.py b/fastvideo/distillation/methods/fine_tuning/__init__.py new file mode 100644 index 000000000..57c85a008 --- /dev/null +++ b/fastvideo/distillation/methods/fine_tuning/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 + +__all__: list[str] = [] + diff --git a/fastvideo/distillation/methods/knowledge_distillation/__init__.py b/fastvideo/distillation/methods/knowledge_distillation/__init__.py new file mode 100644 index 000000000..57c85a008 --- /dev/null +++ b/fastvideo/distillation/methods/knowledge_distillation/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 + +__all__: list[str] = [] + diff --git a/fastvideo/training/wan_distillation_v3.py b/fastvideo/training/wan_distillation_v3.py new file mode 100644 index 000000000..8ca81430b --- /dev/null +++ b/fastvideo/training/wan_distillation_v3.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys + +from fastvideo.distillation import DistillTrainer +from fastvideo.distillation.builder import build_wan_dmd2_method +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.logger import init_logger +from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline +from fastvideo.utils import FlexibleArgumentParser + +logger = init_logger(__name__) + + +def main(args) -> None: + logger.info("Starting Wan distillation v3 (DMD2Method + WanAdapter)...") + + pipeline = WanDistillationPipeline.from_pretrained( + args.pretrained_model_name_or_path, + args=args, + ) + training_args = pipeline.training_args + + method = build_wan_dmd2_method(pipeline) + + trainer = DistillTrainer(training_args, tracker=pipeline.tracker) + trainer.run( + method, + dataloader=pipeline.train_dataloader, + max_steps=training_args.max_train_steps, + start_step=pipeline.init_steps, + ) + + logger.info("Wan distillation v3 completed") + + +if __name__ == "__main__": + argv = sys.argv + parser = FlexibleArgumentParser() + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args(argv[1:]) + main(args) + From 7b2d8e5f48e82f78b08522f6896e5a87fb905e91 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 22:53:44 +0000 Subject: [PATCH 048/214] phase 1 init impl --- dev/phases/phase_1.md | 2 +- fastvideo/distillation/adapters/wan.py | 9 +++++++++ fastvideo/distillation/methods/__init__.py | 3 ++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dev/phases/phase_1.md b/dev/phases/phase_1.md index 9073816d6..06e10015a 100644 --- a/dev/phases/phase_1.md +++ b/dev/phases/phase_1.md @@ -83,7 +83,7 @@ Phase 1 的定位：**在不破坏 Phase 0 可跑性的前提下**，把算法 - `_get_real_score_transformer/_get_fake_score_transformer`（Phase 1 先保证 teacher 侧可选 transformer_2；critic MoE/optimizer 选择后续再补齐） - `denoising_step_list` 构造、warp 逻辑、`min/max_timestep`、`timestep_shift` - `ensure_negative_conditioning()`（Phase 0 已有，Phase 1 要确保能被复用） -- [ ] `WanPipelineAdapter` 继续保留（Phase 0 兜底），但在文档/代码里标注为“legacy-backed” +- [x] `WanPipelineAdapter` 继续保留（Phase 0 兜底），并在文档/代码里标注为“legacy-backed” ### E. Builder 雏形（config -> instantiate） diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index cd79f05b3..5e18bf622 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -42,6 +42,15 @@ class WanPipelineAdapter(DistillAdapter): + """Phase 0 adapter (legacy-backed). + + This adapter intentionally reuses helper methods on the legacy + `fastvideo.training.distillation_pipeline.DistillationPipeline` to keep the + Phase 0 change set low-risk. + + Prefer `WanAdapter` (Phase 1+) for algorithm/model decoupling. + """ + def __init__(self, pipeline: DistillationPipeline) -> None: self.pipeline = pipeline diff --git a/fastvideo/distillation/methods/__init__.py b/fastvideo/distillation/methods/__init__.py index 1b9de0b13..dd132ce2c 100644 --- a/fastvideo/distillation/methods/__init__.py +++ b/fastvideo/distillation/methods/__init__.py @@ -1,8 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 from fastvideo.distillation.methods.base import DistillMethod +from fastvideo.distillation.methods.distribution_matching import DMD2Method __all__ = [ "DistillMethod", + "DMD2Method", ] - From 613472194eb9b63cad07531dc0a4154503e0c911 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 23:01:09 +0000 Subject: [PATCH 049/214] general distill endpoint --- dev/phases/phase_1.md | 5 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.sh | 5 +- fastvideo/training/distill.py | 104 ++++++++++++++++++ fastvideo/training/wan_distillation_v3.py | 25 +---- 4 files changed, 115 insertions(+), 24 deletions(-) create mode 100644 fastvideo/training/distill.py diff --git a/dev/phases/phase_1.md b/dev/phases/phase_1.md index 06e10015a..0741b4319 100644 --- a/dev/phases/phase_1.md +++ b/dev/phases/phase_1.md @@ -91,7 +91,10 @@ Phase 1 的定位：**在不破坏 Phase 0 可跑性的前提下**，把算法 - `build_models(args) -> ModelBundle + Adapter`（Phase 1 先实现 Wan） - `build_method(args, bundle, adapter) -> DistillMethod`（DMD2 先实现） - 让 entrypoint 不再手写 `_build_bundle_from_wan_pipeline(...)` -- [x] 新增一个“Phase 1 entrypoint”（`fastvideo/training/wan_distillation_v3.py`） +- [x] 新增一个“通用 distill entrypoint”（`fastvideo/training/distill.py`） + - CLI 通过 `--distill_model/--distill_method` 选择并运行 + - Phase 1 先支持：`wan + dmd2` +- [x] 新增一个“Phase 1 entrypoint”（`fastvideo/training/wan_distillation_v3.py`，后续会变为 wrapper） - Phase 1 先支持：Wan + DMD2（student/teacher/critic） - Phase 0 入口 `wan_distillation_v2.py` 不动（便于 A/B） diff --git a/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh index 52fc2a0d4..f61f092f8 100644 --- a/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh +++ b/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh @@ -101,7 +101,9 @@ torchrun \ --nnodes 1 \ --master_port "$MASTER_PORT" \ --nproc_per_node "$NUM_GPUS" \ - fastvideo/training/wan_distillation_v3.py \ + fastvideo/training/distill.py \ + --distill_model "wan" \ + --distill_method "dmd2" \ "${parallel_args[@]}" \ "${model_args[@]}" \ "${dataset_args[@]}" \ @@ -110,4 +112,3 @@ torchrun \ "${validation_args[@]}" \ "${miscellaneous_args[@]}" \ "${dmd_args[@]}" - diff --git a/fastvideo/training/distill.py b/fastvideo/training/distill.py new file mode 100644 index 000000000..ebff5977d --- /dev/null +++ b/fastvideo/training/distill.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import sys +from typing import Any, Callable + +from fastvideo.distillation import DistillTrainer +from fastvideo.distillation.builder import build_wan_dmd2_method +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.logger import init_logger +from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline +from fastvideo.utils import FlexibleArgumentParser + +logger = init_logger(__name__) + +_PipelineFactory = Callable[[Any], Any] +_MethodBuilder = Callable[[Any], Any] + + +def _build_pipeline_factories() -> dict[str, _PipelineFactory]: + return { + "wan": + lambda args: WanDistillationPipeline.from_pretrained( + args.pretrained_model_name_or_path, + args=args, + ), + } + + +def _build_method_builders() -> dict[tuple[str, str], _MethodBuilder]: + return { + ("wan", "dmd2"): build_wan_dmd2_method, + } + + +def run_distillation( + args: Any, + *, + distill_model: str, + distill_method: str, +) -> None: + pipeline_factories = _build_pipeline_factories() + method_builders = _build_method_builders() + + if distill_model not in pipeline_factories: + raise ValueError( + f"Unknown distill_model={distill_model!r}. Supported: {sorted(pipeline_factories)}" + ) + + builder_key = (distill_model, distill_method) + if builder_key not in method_builders: + supported = sorted({m for (model, m) in method_builders if model == distill_model}) + raise ValueError( + f"Unknown distill_method={distill_method!r} for distill_model={distill_model!r}. " + f"Supported methods for {distill_model}: {supported}" + ) + + pipeline = pipeline_factories[distill_model](args) + training_args = pipeline.training_args + + method = method_builders[builder_key](pipeline) + + trainer = DistillTrainer(training_args, tracker=pipeline.tracker) + trainer.run( + method, + dataloader=pipeline.train_dataloader, + max_steps=training_args.max_train_steps, + start_step=pipeline.init_steps, + ) + + +def main(args: Any) -> None: + distill_model = str(getattr(args, "distill_model")) + distill_method = str(getattr(args, "distill_method")) + logger.info( + "Starting distillation: distill_model=%s, distill_method=%s", + distill_model, + distill_method, + ) + run_distillation(args, distill_model=distill_model, distill_method=distill_method) + logger.info("Distillation completed") + + +if __name__ == "__main__": + argv = sys.argv + parser = FlexibleArgumentParser() + parser.add_argument( + "--distill-model", + type=str, + default="wan", + help="Distillation model family (Phase 1 supports: wan).", + ) + parser.add_argument( + "--distill-method", + type=str, + default="dmd2", + help="Distillation method (Phase 1 supports: dmd2).", + ) + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args(argv[1:]) + main(args) + diff --git a/fastvideo/training/wan_distillation_v3.py b/fastvideo/training/wan_distillation_v3.py index 8ca81430b..7c4dd571f 100644 --- a/fastvideo/training/wan_distillation_v3.py +++ b/fastvideo/training/wan_distillation_v3.py @@ -4,35 +4,19 @@ import sys -from fastvideo.distillation import DistillTrainer -from fastvideo.distillation.builder import build_wan_dmd2_method from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs from fastvideo.logger import init_logger -from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline +from fastvideo.training.distill import run_distillation from fastvideo.utils import FlexibleArgumentParser logger = init_logger(__name__) def main(args) -> None: - logger.info("Starting Wan distillation v3 (DMD2Method + WanAdapter)...") - - pipeline = WanDistillationPipeline.from_pretrained( - args.pretrained_model_name_or_path, - args=args, - ) - training_args = pipeline.training_args - - method = build_wan_dmd2_method(pipeline) - - trainer = DistillTrainer(training_args, tracker=pipeline.tracker) - trainer.run( - method, - dataloader=pipeline.train_dataloader, - max_steps=training_args.max_train_steps, - start_step=pipeline.init_steps, + logger.info( + "Starting Wan distillation v3 (wrapper for training/distill.py: wan + dmd2)..." ) - + run_distillation(args, distill_model="wan", distill_method="dmd2") logger.info("Wan distillation v3 completed") @@ -43,4 +27,3 @@ def main(args) -> None: parser = FastVideoArgs.add_cli_args(parser) args = parser.parse_args(argv[1:]) main(args) - From 4a886065354513ebc0ad5765a1825125f00d10ed Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 23:09:04 +0000 Subject: [PATCH 050/214] distillation --- dev/phases/phase_1.md | 2 +- .../phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh | 2 +- fastvideo/training/{distill.py => distillation.py} | 1 - fastvideo/training/wan_distillation_v3.py | 4 ++-- 4 files changed, 4 insertions(+), 5 deletions(-) rename fastvideo/training/{distill.py => distillation.py} (99%) diff --git a/dev/phases/phase_1.md b/dev/phases/phase_1.md index 0741b4319..dfe1639dc 100644 --- a/dev/phases/phase_1.md +++ b/dev/phases/phase_1.md @@ -91,7 +91,7 @@ Phase 1 的定位：**在不破坏 Phase 0 可跑性的前提下**，把算法 - `build_models(args) -> ModelBundle + Adapter`（Phase 1 先实现 Wan） - `build_method(args, bundle, adapter) -> DistillMethod`（DMD2 先实现） - 让 entrypoint 不再手写 `_build_bundle_from_wan_pipeline(...)` -- [x] 新增一个“通用 distill entrypoint”（`fastvideo/training/distill.py`） +- [x] 新增一个“通用 distill entrypoint”（`fastvideo/training/distillation.py`） - CLI 通过 `--distill_model/--distill_method` 选择并运行 - Phase 1 先支持：`wan + dmd2` - [x] 新增一个“Phase 1 entrypoint”（`fastvideo/training/wan_distillation_v3.py`，后续会变为 wrapper） diff --git a/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh index f61f092f8..fe79dc58b 100644 --- a/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh +++ b/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh @@ -101,7 +101,7 @@ torchrun \ --nnodes 1 \ --master_port "$MASTER_PORT" \ --nproc_per_node "$NUM_GPUS" \ - fastvideo/training/distill.py \ + fastvideo/training/distillation.py \ --distill_model "wan" \ --distill_method "dmd2" \ "${parallel_args[@]}" \ diff --git a/fastvideo/training/distill.py b/fastvideo/training/distillation.py similarity index 99% rename from fastvideo/training/distill.py rename to fastvideo/training/distillation.py index ebff5977d..48382a62b 100644 --- a/fastvideo/training/distill.py +++ b/fastvideo/training/distillation.py @@ -101,4 +101,3 @@ def main(args: Any) -> None: parser = FastVideoArgs.add_cli_args(parser) args = parser.parse_args(argv[1:]) main(args) - diff --git a/fastvideo/training/wan_distillation_v3.py b/fastvideo/training/wan_distillation_v3.py index 7c4dd571f..f660ebe7f 100644 --- a/fastvideo/training/wan_distillation_v3.py +++ b/fastvideo/training/wan_distillation_v3.py @@ -6,7 +6,7 @@ from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs from fastvideo.logger import init_logger -from fastvideo.training.distill import run_distillation +from fastvideo.training.distillation import run_distillation from fastvideo.utils import FlexibleArgumentParser logger = init_logger(__name__) @@ -14,7 +14,7 @@ def main(args) -> None: logger.info( - "Starting Wan distillation v3 (wrapper for training/distill.py: wan + dmd2)..." + "Starting Wan distillation v3 (wrapper for training/distillation.py: wan + dmd2)..." ) run_distillation(args, distill_model="wan", distill_method="dmd2") logger.info("Wan distillation v3 completed") From ce22aeacb2d94a151f59b73717faf3e54a9b4109 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 23:14:31 +0000 Subject: [PATCH 051/214] temporary run script --- examples/distillation/phase1/temp.sh | 136 +++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 examples/distillation/phase1/temp.sh diff --git a/examples/distillation/phase1/temp.sh b/examples/distillation/phase1/temp.sh new file mode 100644 index 000000000..f1a617d48 --- /dev/null +++ b/examples/distillation/phase1/temp.sh @@ -0,0 +1,136 @@ +#!/bin/bash +set -e -x + +# One-shot launch script for Phase1 (DMD2Method + WanAdapter) Wan DMD2 few-step distillation. +# Uses the same local Wan-Syn parquet dataset + validation json as Phase0 temp.sh. +# +# Notes: +# - By default this runs W&B in offline mode (safer for overnight runs). +# If you want online logging: +# export WANDB_MODE=online +# export WANDB_API_KEY=... +# - Phase1 uses the general entrypoint: +# fastvideo/training/distillation.py --distill_model wan --distill_method dmd2 + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29504} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +if [[ -z "${NUM_GPUS:-}" ]]; then + if command -v nvidia-smi >/dev/null 2>&1; then + NUM_GPUS=$(nvidia-smi -L | wc -l) + else + NUM_GPUS=1 + fi +fi + +# Models +STUDENT_MODEL_PATH=${STUDENT_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} +TEACHER_MODEL_PATH=${TEACHER_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-14B-Diffusers"} +CRITIC_MODEL_PATH=${CRITIC_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} + +# Data (parquet dataset folder) +DATA_DIR=${DATA_DIR:-"data/Wan-Syn_77x448x832_600k"} +DEFAULT_VALIDATION_DATASET_FILE=\ +"examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json" +VALIDATION_DATASET_FILE=${VALIDATION_DATASET_FILE:-"$DEFAULT_VALIDATION_DATASET_FILE"} + +RUN_ID=${RUN_ID:-"$(date +%Y%m%d_%H%M%S)"} +OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase1_wan2.1_dmd2_8steps_wansyn_${RUN_ID}"} + +training_args=( + --tracker_project_name "phase1_wan_dmd2_8steps_wansyn" + --output_dir "$OUTPUT_DIR" + --max_train_steps 4000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 20 + --num_height 448 + --num_width 832 + --num_frames 77 + --enable_gradient_checkpointing_type "full" + --simulate_generator_forward +) + +parallel_args=( + --num_gpus "$NUM_GPUS" + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim "$NUM_GPUS" +) + +model_args=( + --model_path "$STUDENT_MODEL_PATH" + --pretrained_model_name_or_path "$STUDENT_MODEL_PATH" + --real_score_model_path "$TEACHER_MODEL_PATH" + --fake_score_model_path "$CRITIC_MODEL_PATH" +) + +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 50 + --validation_sampling_steps "8" + --validation_guidance_scale "6.0" +) + +optimizer_args=( + --learning_rate 2e-6 + --mixed_precision "bf16" + --weight_decay 0.01 + --betas '0.0,0.999' + --max_grad_norm 1.0 + --fake_score_learning_rate 8e-6 + --fake_score_betas '0.0,0.999' +) + +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.0 + --dit_precision "fp32" + --flow_shift 8 + --seed 1000 +) + +dmd_args=( + --dmd_denoising_steps '1000,850,700,550,350,275,200,125' + --min_timestep_ratio 0.02 + --max_timestep_ratio 0.98 + --generator_update_interval 5 + --real_score_guidance_scale 3.5 +) + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --distill_model "wan" \ + --distill_method "dmd2" \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ + "${dmd_args[@]}" From d20753bad2a369a61ebe01b26a151e13928c9fd8 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 21 Feb 2026 23:38:54 +0000 Subject: [PATCH 052/214] random generator fix --- fastvideo/distillation/adapters/wan.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 5e18bf622..52b6ac91e 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -292,6 +292,21 @@ def on_train_start(self) -> None: self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) + pipeline = self._validation_pipeline_owner + if pipeline is not None: + if not hasattr(pipeline, "validation_random_generator"): + pipeline.validation_random_generator = torch.Generator( # type: ignore[attr-defined] + device="cpu" + ).manual_seed(int(seed)) + if not hasattr(pipeline, "noise_random_generator"): + pipeline.noise_random_generator = torch.Generator( # type: ignore[attr-defined] + device="cpu" + ).manual_seed(int(seed)) + if not hasattr(pipeline, "noise_gen_cuda"): + pipeline.noise_gen_cuda = torch.Generator( # type: ignore[attr-defined] + device=self.device + ).manual_seed(int(seed)) + self.ensure_negative_conditioning() def ensure_negative_conditioning(self) -> None: From e36507b85b72fcbbf867c4341b1475936a8569e8 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 00:10:45 +0000 Subject: [PATCH 053/214] Phase 1 works very well on training. --- examples/distillation/phase1/run.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 examples/distillation/phase1/run.md diff --git a/examples/distillation/phase1/run.md b/examples/distillation/phase1/run.md new file mode 100644 index 000000000..929e462e4 --- /dev/null +++ b/examples/distillation/phase1/run.md @@ -0,0 +1 @@ +https://wandb.ai/alexzms-ucsd/phase0_wan_dmd2_8steps_wansyn/runs/cpw05inn?nw=nwuseralexzms \ No newline at end of file From bd24192c9d60ec5eb2d9e8208cb81b79e8a639c3 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 00:20:36 +0000 Subject: [PATCH 054/214] dmd2 adapter comments --- .../methods/distribution_matching/dmd2.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 7aed5e9e1..db66c9858 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -16,6 +16,18 @@ class _DMD2Adapter(Protocol): + """Algorithm-specific adapter contract for :class:`DMD2Method`. + + The method layer is intentionally model-agnostic: it should not import or + depend on any concrete pipeline/model implementation. Instead, all + model-specific primitives (batch preparation, noise schedule helpers, + forward-context management, and role-specific backward behavior) are + provided by an adapter (e.g. ``WanAdapter``). + + This ``Protocol`` documents the required surface area and helps static type + checkers/IDE tooling; it is not enforced at runtime (duck typing). + """ + training_args: Any def on_train_start(self) -> None: @@ -75,6 +87,19 @@ def log_validation(self, iteration: int) -> None: class DMD2Method(DistillMethod): + """DMD2 distillation algorithm (method layer). + + Owns the algorithmic orchestration (loss construction + update policy) and + stays independent of any specific model family. It requires a + :class:`~fastvideo.distillation.bundle.ModelBundle` containing at least the + roles ``student``, ``teacher``, and ``critic``. + + All model-family details (how to run student rollout, teacher CFG + prediction, critic loss, and how to safely run backward under activation + checkpointing/forward-context constraints) are delegated to the adapter + passed in at construction time. + """ + def __init__( self, *, @@ -247,4 +272,3 @@ def optimizers_schedulers_step(self, iteration: int) -> None: self._clip_grad_norm(module) super().optimizers_schedulers_step(iteration) - From b9590f80a2c7d39c7445cc266a2fdc83f5d1a44c Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 00:33:55 +0000 Subject: [PATCH 055/214] removing phase 0 dependency --- dev/phases/phase_1.md | 194 ++++++--------- examples/distillation/phase1/run.md | 4 +- examples/distillation/phase1/temp.sh | 4 +- fastvideo/distillation/adapters/wan.py | 171 -------------- fastvideo/distillation/methods/wan_dmd2.py | 220 ------------------ fastvideo/distillation/trainer.py | 2 +- ... => test_optimizer_scheduler_alignment.py} | 3 +- fastvideo/training/wan_distillation_v2.py | 72 ------ 8 files changed, 73 insertions(+), 597 deletions(-) delete mode 100644 fastvideo/distillation/methods/wan_dmd2.py rename fastvideo/tests/distillation/{test_phase0_schedule.py => test_optimizer_scheduler_alignment.py} (97%) delete mode 100644 fastvideo/training/wan_distillation_v2.py diff --git a/dev/phases/phase_1.md b/dev/phases/phase_1.md index dfe1639dc..d717148ed 100644 --- a/dev/phases/phase_1.md +++ b/dev/phases/phase_1.md @@ -1,172 +1,110 @@ -# Phase 1：把 DMD2 “抠出来”（算法与 Wan 解耦）+ Builder 雏形 +# Phase 1：算法/模型解耦落地（DMD2Method + WanAdapter）+ 通用入口 -Phase 0 已经证明：`Trainer/Method/Adapter/Bundle` 这套骨架是能跑通的（Wan DMD2 也已实测 -能收敛/validation 变清晰）。但 Phase 0 的核心问题仍然存在： +Phase 1 的定位：把 distillation 从 “pipeline god object” 拆成稳定边界，让 **算法** 与 +**模型家族** 解耦合，并且让训练入口开始具备 “选择 + instantiate” 的结构（对齐 FastGen +的 catalog 思路）。 -- `WanDMD2Method` 仍在调用 legacy pipeline 的私有函数（`_generator_forward/_dmd_forward/ - faker_score_forward/...`），本质还是 **Wan+方法耦合**。 -- `WanPipelineAdapter` 仍依赖 legacy pipeline 的 helper（normalize/prepare inputs/build metadata）。 -- 启动路径仍然是“先起 legacy pipeline，再手工拼 bundle+method+trainer”，builder 还只是散落在 - entrypoint 里的一段 glue code。 +目标边界（Phase 1 确认有效）： -Phase 1 的定位：**在不破坏 Phase 0 可跑性的前提下**，把算法层从 Wan/pipeline 中抽离出来， -建立稳定边界：`Method(算法)` 只依赖 `Adapter(模型 primitives)` + `ModelBundle(roles)`。 +- `DistillTrainer`：训练基础设施（accum/step/log hooks），不懂具体模型/算法细节 +- `DistillMethod`：算法编排（DMD2 / Self-Forcing / CM / ...），只依赖 adapter primitives +- `DistillAdapter`：模型家族适配（Wan / CogVideoX / ...），负责 forward/backward context 与 + pipeline 细节 +- `ModelBundle`：`roles -> handles`（modules/optimizers/schedulers），method 通过 role 取用 ---- - -## Phase 1 目标（可交付） - -1. 产出通用算法实现：`DMD2Method`（不再 import/调用 `fastvideo/training/distillation_pipeline.py` - 的任何私有函数）。 -2. 产出可复用的 Wan 适配层：`WanAdapter`（逐步替代 `WanPipelineAdapter`，目标是不再依赖 legacy - pipeline 的 helper；Phase 1 允许“过渡期双实现并存”）。 -3. 落地 Builder 雏形：从 FastVideo 现有 args/config（`TrainingArgs/FastVideoArgs`，或新增一个 - `--models_json`）构建： - - `ModelBundle(roles={...})` - - `Adapter` - - `Method` - 并让 entrypoint 变成“选择 + instantiate”，而不是手写胶水。 -4. 保持 Phase 0 路径可用：`wan_distillation_v2.py` + `WanDMD2Method` 暂不删除（仅标注过渡/弃用）。 +现状：Wan DMD2 的 Phase 1 训练结果已与旧 baseline 对齐，因此 Phase 0 的脚手架代码路径已移除 +（见 TODO H）。 --- -## Phase 1 非目标（明确不做 / 延后） +## Phase 1 非目标（明确延后） -- Self-forcing v2（以及 ODE-init）—— Phase 2 再做 -- role-based checkpoint/save/resume 协议—— Phase 2 再统一 -- 多模型家族混搭（例如 student=Wan、teacher=SDXL）—— 先不承诺 -- 完整的 validation 抽象（先保留 `method.log_validation` hook） +- Self-forcing v2（以及 ODE-init） +- role-based checkpoint/save/resume 协议统一 +- 完整的 validation 抽象（Phase 1 先保留 `method.log_validation()` hook） +- 多模型家族混搭（例如 student=Wan、teacher=SDXL） --- ## Phase 1 TODO List（Review Checklist） -> Phase 1 的 checklist 会在实施过程中持续更新、打钩（像 Phase 0 一样）。 - -### A. 方法目录结构（对齐 FastGen 的“分层 catalog”思路） +### A. 方法目录结构（catalog：多层级 methods） - [x] 建立 methods 分层目录（至少有目录与 `__init__.py`） - `fastvideo/distillation/methods/distribution_matching/` - `fastvideo/distillation/methods/consistency_model/` - `fastvideo/distillation/methods/knowledge_distillation/` - `fastvideo/distillation/methods/fine_tuning/` -- [x] 把 Phase 1 的 `DMD2Method` 放到 - `fastvideo/distillation/methods/distribution_matching/dmd2.py` +- [x] `DMD2Method` 放到 `fastvideo/distillation/methods/distribution_matching/dmd2.py` ### B. 通用 `DMD2Method`（算法层） -- [x] 新增 `DMD2Method`（算法实现），并显式 `bundle.require_roles([...])` - - 最小要求：`student/teacher/critic`（未来可扩展 role，但 Phase 1 先按 DMD2 固定需求） -- [x] `DMD2Method` 不持有 pipeline，不调用 legacy pipeline 私有函数 -- [x] 保留 Phase 0 的关键语义： - - generator update interval（`generator_update_interval`） - - backward 期 forward recompute 的 `forward_context` 约束（要么由 adapter 管，要么 method 管） - -### C. Adapter 接口升级（让算法真正可复用） - -- [x] 定义“DMD2 需要的 adapter primitives”（以 Protocol 的方式定义在 `DMD2Method` 内部） - - `prepare_batch(...)` - - `student_predict_x0(...)`（或“generator_pred_video”等等价语义） - - `teacher_predict_x0(cond/uncond, guidance_scale, ...)` - - `critic_flow_matching_loss(...)`（或拆成 critic forward + target 构造） - - `add_noise(...) / sample_timestep(...) / shift+clamp timestep(...)` - - `forward_context(...)`（如果决定由 adapter 托管 forward_context） -- [x] 让 `DMD2Method` 只依赖这些 primitives（而不是 Wan 细节） - -### D. `WanAdapter`（模型层，逐步摆脱 pipeline helper） - -- [x] 在 `fastvideo/distillation/adapters/wan.py` 中新增 `WanAdapter` - - 输入：`training_args` + `noise_scheduler` + `ModelBundle`（或 bundle role handles） - - 输出：实现 Phase 1 定义的 DMD2 primitives -- [x] 把以下逻辑从 legacy pipeline/helper 迁出到 adapter（Phase 1 做到“可跑通”即可） - - `_build_distill_input_kwargs`（transformer forward 的输入组装） - - `_get_real_score_transformer/_get_fake_score_transformer`（Phase 1 先保证 teacher 侧可选 transformer_2；critic MoE/optimizer 选择后续再补齐） - - `denoising_step_list` 构造、warp 逻辑、`min/max_timestep`、`timestep_shift` - - `ensure_negative_conditioning()`（Phase 0 已有，Phase 1 要确保能被复用） -- [x] `WanPipelineAdapter` 继续保留（Phase 0 兜底），并在文档/代码里标注为“legacy-backed” - -### E. Builder 雏形（config -> instantiate） - -- [x] 新增 builder/registry（先落 `fastvideo/distillation/builder.py`，Phase 1 仅实现 Wan） - - `build_models(args) -> ModelBundle + Adapter`（Phase 1 先实现 Wan） - - `build_method(args, bundle, adapter) -> DistillMethod`（DMD2 先实现） - - 让 entrypoint 不再手写 `_build_bundle_from_wan_pipeline(...)` -- [x] 新增一个“通用 distill entrypoint”（`fastvideo/training/distillation.py`） - - CLI 通过 `--distill_model/--distill_method` 选择并运行 - - Phase 1 先支持：`wan + dmd2` -- [x] 新增一个“Phase 1 entrypoint”（`fastvideo/training/wan_distillation_v3.py`，后续会变为 wrapper） - - Phase 1 先支持：Wan + DMD2（student/teacher/critic） - - Phase 0 入口 `wan_distillation_v2.py` 不动（便于 A/B） - -### F. 示例脚本（Phase 1） +- [x] 新增 `DMD2Method`（算法实现），并显式 `bundle.require_roles(["student","teacher","critic"])` +- [x] `DMD2Method` 不持有 legacy pipeline，不调用 legacy pipeline 私有算法函数 +- [x] 保留关键语义：`generator_update_interval`（只在该 step 更新 student） -- [x] 新增 `examples/distillation/phase1/` 的脚本 - - 目标：用户只改路径就能启动 “Wan 1.3B 学 14B，8 steps distill” 的 DMD2 - - 如果 Phase 1 引入 `--models_json`：提供一个最小 JSON 模板（写在脚本注释里） +### C. Adapter primitives 契约（让算法真正可复用） -### G. 最小单测（CPU；可选但推荐） +- [x] 在 `DMD2Method` 内通过 `Protocol` 定义 DMD2 所需 adapter surface（`_DMD2Adapter`） +- [x] `DMD2Method` 仅依赖 primitives，而不是 Wan 细节 -> 你说过 GPU test 你会自己跑；CPU test 主要用于锁定“调度/接口”不回归。 +### D. `WanAdapter`（模型层：forward/backward/context 全部在 adapter） -- [ ] 保留 Phase 0 的 schedule test（不改语义） -- [ ] 为 Phase 1 新增 1 个最小单测（不需要真实模型）： - - builder 能按 role 组装 optimizers/schedulers（或 method 能正确选择 opt/sched） +- [x] 新增 `fastvideo/distillation/adapters/wan.py::WanAdapter` +- [x] Wan 侧实现 DMD2 所需 primitives（batch prepare / teacher cfg / critic loss / backward 封装） +- [x] `forward_context` 由 adapter 托管（method 不直接触碰 `set_forward_context`） +- [x] `ensure_negative_conditioning()` 显式化（不依赖 validation 的副作用） ---- +### E. Builder + 通用入口（config -> instantiate） -## 关键设计决策点（出现风险就停下问你） +- [x] 新增 `fastvideo/distillation/builder.py::build_wan_dmd2_method` + - Phase 1 先实现：`wan + dmd2` +- [x] 新增通用 distill 入口：`fastvideo/training/distillation.py` + - CLI：`--distill-model` / `--distill-method` +- [x] 保留一个 Wan wrapper：`fastvideo/training/wan_distillation_v3.py` -### 决策点 1：`forward_context` 由谁管理？ +### F. 示例脚本（Phase 1） -背景：FastVideo 的 backward 可能触发 forward 重算，重算必须处于正确的 -`set_forward_context(current_timestep, attn_metadata)` 中。 +- [x] `examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh` +- [x] `examples/distillation/phase1/temp.sh`（可直接改路径启动训练） -**决定：Adapter 管理（你拍板）。** +### G. 最小单测（CPU；锁定调度语义） -落地方式（Phase 1 采用）： +- [x] 保留并重命名 optimizer/scheduler 对齐测试： + - `fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py` +- [ ] （可选）为 builder 增加 1 个最小单测（不需要真实模型） -- Method 不 import `fastvideo.forward_context`，也不直接调用 `set_forward_context(...)` -- Adapter 提供显式的上下文 API（例如 `adapter.student_context(...) / adapter.critic_context(...)`） - 以及必要时的 `adapter.backward_*()` 封装，确保 activation checkpointing 触发的 forward 重算也在 - 正确的 context 中执行 +### H. 移除 Phase 0 脚手架（去除旧路径依赖） -如果后续实现发现 adapter 侧会导致接口/实现“非常不优雅”（比如需要过多特殊 case），我会停下并 -汇报尝试与失败原因，再一起讨论是否回退到“Method 管理”或引入更好的抽象。 +- [x] 移除 legacy-backed `WanPipelineAdapter` +- [x] 移除旧的强耦合方法 `WanDMD2Method` +- [x] 移除旧入口 `fastvideo/training/wan_distillation_v2.py` +- [x] 移除 `examples/distillation/phase0/`（避免误用旧脚本） +- [x] 清理代码里残留的 `Phase 0/phase0` 调用与命名 -### 决策点 2：Builder 是否在 Phase 1 “彻底摆脱 legacy pipeline”？ +--- -Phase 1 的最小目标是“把胶水集中起来”，不必一次性重写所有加载/optimizer/dataloader 逻辑。 -如果我们发现不依赖 pipeline 会导致需要大规模复制 loader/optimizer 初始化逻辑， -我会建议 Phase 1 先做： +## 仍然依赖 legacy code 的部分（尚未解耦干净） -- builder 内部仍可调用 `WanDistillationPipeline.from_pretrained(...)` 完成加载 -- 但 **method/adapter 不再依赖 pipeline 私有算法函数** +Phase 1 已经把 **算法层** 从 legacy pipeline 私有函数中剥离出来，但当前仍保留 “复用 legacy +pipeline 做加载/数据/日志” 的过渡策略，主要耦合点是： -是否要把“加载也完全重写”强行塞进 Phase 1，是一个风险点，需要你决定优先级。 +1. **加载/数据/优化器仍依赖 legacy pipeline** + - `fastvideo/training/distillation.py` 仍通过 `WanDistillationPipeline.from_pretrained(...)` + 完成：模型加载、dataloader 构建、optimizer/scheduler 初始化、tracker 等。 + - `fastvideo/distillation/builder.py` 目前接受的输入还是 `WanDistillationPipeline`。 ---- +2. **validation 仍复用 legacy pipeline 的实现** + - `fastvideo/distillation/adapters/wan.py::WanAdapter.log_validation()` 仍调用 + legacy 的 `pipeline._log_validation(...)`（并需要 legacy pipeline 的 validation init/RNG）。 -## 代码落地点（具体到文件；Phase 1 预计会改/新增这些） - -- 新增： - - `fastvideo/distillation/methods/distribution_matching/dmd2.py` - - `fastvideo/distillation/methods/distribution_matching/__init__.py` - - `fastvideo/distillation/methods/consistency_model/__init__.py`（空壳） - - `fastvideo/distillation/methods/knowledge_distillation/__init__.py`（空壳） - - `fastvideo/distillation/methods/fine_tuning/__init__.py`（空壳） - - `fastvideo/distillation/builder.py`（或 `fastvideo/distillation/builders/*`） - - `fastvideo/training/distill_v3.py`（Phase 1 新入口，名字可再讨论） - - `examples/distillation/phase1/*` -- 修改： - - `fastvideo/distillation/adapters/base.py`（扩展/引入 protocol） - - `fastvideo/distillation/adapters/wan.py`（新增 `WanAdapter`，保留 `WanPipelineAdapter`） - - `fastvideo/distillation/methods/__init__.py`（导出新 method） - - （可选）`fastvideo/distillation/methods/wan_dmd2.py`（标注 deprecated + 逐步迁移） +这些耦合点将是 Phase 2 的主要清理对象（尤其是 validation + builder 输入从 “pipeline” 变为 +“roles/specs”）。 --- -## Phase 1 完成标准（Definition of Done） +## Definition of Done（Phase 1） -- `DMD2Method` 存在且可运行，且不依赖 `fastvideo/training/distillation_pipeline.py` 的私有函数 -- Wan DMD2 的训练可以走 Phase 1 新入口（至少 smoke 跑通；GPU A/B 由你后续验证） -- Phase 0 入口仍可用（便于对齐/回滚） +- `DMD2Method` 存在且可运行，且不依赖 legacy pipeline 私有算法函数 +- Wan DMD2 训练可走 `fastvideo/training/distillation.py --distill-model wan --distill-method dmd2` +- Phase 0 代码路径已移除，避免后续继续扩散耦合 diff --git a/examples/distillation/phase1/run.md b/examples/distillation/phase1/run.md index 929e462e4..45969dc43 100644 --- a/examples/distillation/phase1/run.md +++ b/examples/distillation/phase1/run.md @@ -1 +1,3 @@ -https://wandb.ai/alexzms-ucsd/phase0_wan_dmd2_8steps_wansyn/runs/cpw05inn?nw=nwuseralexzms \ No newline at end of file +# Phase 1 run link (optional) + +Paste your W&B run URL here for quick access. diff --git a/examples/distillation/phase1/temp.sh b/examples/distillation/phase1/temp.sh index f1a617d48..6559f46c8 100644 --- a/examples/distillation/phase1/temp.sh +++ b/examples/distillation/phase1/temp.sh @@ -1,8 +1,8 @@ #!/bin/bash set -e -x -# One-shot launch script for Phase1 (DMD2Method + WanAdapter) Wan DMD2 few-step distillation. -# Uses the same local Wan-Syn parquet dataset + validation json as Phase0 temp.sh. +# One-shot launch script for Phase 1 (DMD2Method + WanAdapter) Wan DMD2 few-step distillation. +# Uses the Wan-Syn parquet dataset + validation json (paths configurable below). # # Notes: # - By default this runs W&B in offline mode (safer for overnight runs). diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 52b6ac91e..730b8035f 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -41,177 +41,6 @@ VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] -class WanPipelineAdapter(DistillAdapter): - """Phase 0 adapter (legacy-backed). - - This adapter intentionally reuses helper methods on the legacy - `fastvideo.training.distillation_pipeline.DistillationPipeline` to keep the - Phase 0 change set low-risk. - - Prefer `WanAdapter` (Phase 1+) for algorithm/model decoupling. - """ - - def __init__(self, pipeline: DistillationPipeline) -> None: - self.pipeline = pipeline - - def _get_training_dtype(self) -> torch.dtype: - # Phase 0: match existing training pipelines (autocast bf16). - return torch.bfloat16 - - def ensure_negative_conditioning(self) -> None: - if getattr(self.pipeline, "negative_prompt_embeds", None) is not None: - return - - training_args = self.pipeline.training_args - world_group = get_world_group() - device = get_local_torch_device() - dtype = self._get_training_dtype() - - neg_embeds: torch.Tensor | None = None - neg_mask: torch.Tensor | None = None - - if world_group.rank_in_group == 0: - sampling_param = SamplingParam.from_pretrained(training_args.model_path) - negative_prompt = sampling_param.negative_prompt - - prompt_pipeline = getattr(self.pipeline, "validation_pipeline", None) - created_pipeline = False - if prompt_pipeline is None: - args_copy = copy.deepcopy(training_args) - args_copy.inference_mode = True - prompt_pipeline = WanDMDPipeline.from_pretrained( - training_args.model_path, - args=args_copy, - inference_mode=True, - loaded_modules={"transformer": self.pipeline.get_module("transformer")}, - tp_size=training_args.tp_size, - sp_size=training_args.sp_size, - num_gpus=training_args.num_gpus, - pin_cpu_memory=training_args.pin_cpu_memory, - dit_cpu_offload=True, - ) - created_pipeline = True - - batch_negative = ForwardBatch( - data_type="video", - prompt=negative_prompt, - prompt_embeds=[], - prompt_attention_mask=[], - ) - result_batch = prompt_pipeline.prompt_encoding_stage( # type: ignore[attr-defined] - batch_negative, - training_args, - ) - - neg_embeds = result_batch.prompt_embeds[0].to(device=device, dtype=dtype) - neg_mask = result_batch.prompt_attention_mask[0].to(device=device, dtype=dtype) - - if created_pipeline: - del prompt_pipeline - gc.collect() - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - meta = torch.zeros((2,), device=device, dtype=torch.int64) - if world_group.rank_in_group == 0: - assert neg_embeds is not None - assert neg_mask is not None - meta[0] = neg_embeds.ndim - meta[1] = neg_mask.ndim - world_group.broadcast(meta, src=0) - embed_ndim, mask_ndim = (int(meta[0].item()), int(meta[1].item())) - - max_ndim = 8 - embed_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) - mask_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) - if world_group.rank_in_group == 0: - assert neg_embeds is not None - assert neg_mask is not None - embed_shape[:embed_ndim] = torch.tensor( - list(neg_embeds.shape), device=device, dtype=torch.int64 - ) - mask_shape[:mask_ndim] = torch.tensor( - list(neg_mask.shape), device=device, dtype=torch.int64 - ) - world_group.broadcast(embed_shape, src=0) - world_group.broadcast(mask_shape, src=0) - - embed_sizes = tuple(int(x) for x in embed_shape[:embed_ndim].tolist()) - mask_sizes = tuple(int(x) for x in mask_shape[:mask_ndim].tolist()) - - if world_group.rank_in_group != 0: - neg_embeds = torch.empty(embed_sizes, device=device, dtype=dtype) - neg_mask = torch.empty(mask_sizes, device=device, dtype=dtype) - assert neg_embeds is not None - assert neg_mask is not None - - world_group.broadcast(neg_embeds, src=0) - world_group.broadcast(neg_mask, src=0) - - self.pipeline.negative_prompt_embeds = neg_embeds - self.pipeline.negative_prompt_attention_mask = neg_mask - - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - ) -> TrainingBatch: - self.ensure_negative_conditioning() - - device = get_local_torch_device() - dtype = self._get_training_dtype() - - training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) - encoder_hidden_states = raw_batch["text_embedding"] - encoder_attention_mask = raw_batch["text_attention_mask"] - infos = raw_batch.get("info_list") - - if self.pipeline.training_args.simulate_generator_forward: - batch_size = encoder_hidden_states.shape[0] - vae_config = self.pipeline.training_args.pipeline_config.vae_config.arch_config - num_channels = vae_config.z_dim - spatial_compression_ratio = vae_config.spatial_compression_ratio - latent_height = self.pipeline.training_args.num_height // spatial_compression_ratio - latent_width = self.pipeline.training_args.num_width // spatial_compression_ratio - latents = torch.zeros( - batch_size, - num_channels, - self.pipeline.training_args.num_latent_t, - latent_height, - latent_width, - device=device, - dtype=dtype, - ) - else: - if "vae_latent" not in raw_batch: - raise ValueError( - "vae_latent not found in batch and simulate_generator_forward is False" - ) - latents = raw_batch["vae_latent"] - latents = latents[:, :, : self.pipeline.training_args.num_latent_t] - latents = latents.to(device, dtype=dtype) - - training_batch.latents = latents - training_batch.encoder_hidden_states = encoder_hidden_states.to( - device, dtype=dtype - ) - training_batch.encoder_attention_mask = encoder_attention_mask.to( - device, dtype=dtype - ) - training_batch.infos = infos - - training_batch = self.pipeline._normalize_dit_input(training_batch) - training_batch = self.pipeline._prepare_dit_inputs(training_batch) - training_batch = self.pipeline._build_attention_metadata(training_batch) - - training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) - if training_batch.attn_metadata is not None: - training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] - - return training_batch - - class WanAdapter(DistillAdapter): """ Phase 1 target adapter: provide Wan-specific primitives without calling diff --git a/fastvideo/distillation/methods/wan_dmd2.py b/fastvideo/distillation/methods/wan_dmd2.py deleted file mode 100644 index ca5510b40..000000000 --- a/fastvideo/distillation/methods/wan_dmd2.py +++ /dev/null @@ -1,220 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import copy -from typing import Any - -import torch - -from fastvideo.distributed import get_world_group -from fastvideo.forward_context import set_forward_context -from fastvideo.training.training_utils import ( - clip_grad_norm_while_handling_failing_dtensor_cases, -) -from fastvideo.utils import set_random_seed - -from fastvideo.distillation.bundle import ModelBundle -from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.adapters.wan import WanPipelineAdapter - - -class WanDMD2Method(DistillMethod): - def __init__( - self, - *, - bundle: ModelBundle, - adapter: WanPipelineAdapter, - ) -> None: - super().__init__(bundle) - self.adapter = adapter - self.training_args = adapter.pipeline.training_args - self.world_group = get_world_group() - - def on_train_start(self) -> None: - seed = self.training_args.seed - if seed is None: - raise ValueError("training_args.seed must be set for distillation") - - pipeline = self.adapter.pipeline - if pipeline.sp_world_size > 1: - sp_group_seed = seed + (pipeline.global_rank // pipeline.sp_world_size) - set_random_seed(sp_group_seed) - else: - set_random_seed(seed + pipeline.global_rank) - - pipeline.noise_random_generator = torch.Generator( - device="cpu").manual_seed(seed) - pipeline.validation_random_generator = torch.Generator( - device="cpu").manual_seed(seed) - if pipeline.device.type == "cuda": - pipeline.noise_gen_cuda = torch.Generator(device="cuda").manual_seed(seed) - else: - pipeline.noise_gen_cuda = torch.Generator( - device=pipeline.device).manual_seed(seed) - - self.adapter.ensure_negative_conditioning() - - def log_validation(self, iteration: int) -> None: - pipeline = self.adapter.pipeline - training_args = pipeline.training_args - if not getattr(training_args, "log_validation", False): - return - - if getattr(pipeline, "validation_pipeline", None) is None: - pipeline.initialize_validation_pipeline(training_args) - - old_inference_mode = training_args.inference_mode - old_dit_cpu_offload = training_args.dit_cpu_offload - try: - pipeline._log_validation( - pipeline.transformer, - training_args, - iteration, - ) - finally: - training_args.inference_mode = old_inference_mode - training_args.dit_cpu_offload = old_dit_cpu_offload - - def _should_update_student(self, iteration: int) -> bool: - interval = int(self.training_args.generator_update_interval or 1) - if interval <= 0: - return True - return iteration % interval == 0 - - def _clip_grad_norm(self, module: torch.nn.Module) -> float: - max_grad_norm = self.training_args.max_grad_norm - if not max_grad_norm: - return 0.0 - grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( - [p for p in module.parameters()], - float(max_grad_norm), - foreach=None, - ) - return float(grad_norm.item()) if grad_norm is not None else 0.0 - - def single_train_step( - self, - batch: dict[str, Any], - iteration: int, - *, - current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: - pipeline = self.adapter.pipeline - pipeline.current_trainstep = iteration - - training_batch = self.adapter.prepare_batch( - batch, - current_vsa_sparsity=current_vsa_sparsity, - ) - - update_student = self._should_update_student(iteration) - device = pipeline.device - device_type = device.type - - generator_loss = torch.zeros( - (), - device=device, - dtype=training_batch.latents.dtype, - ) - batch_gen = None - student_backward_ctx = None - if update_student: - batch_gen = copy.deepcopy(training_batch) - with torch.autocast(device_type, dtype=batch_gen.latents.dtype): - with set_forward_context( - current_timestep=batch_gen.timesteps, - attn_metadata=batch_gen.attn_metadata_vsa, - ): - if self.training_args.simulate_generator_forward: - generator_pred_video = ( - pipeline._generator_multi_step_simulation_forward( - batch_gen)) - else: - generator_pred_video = pipeline._generator_forward(batch_gen) - - with set_forward_context( - current_timestep=batch_gen.timesteps, - attn_metadata=batch_gen.attn_metadata, - ): - generator_loss = pipeline._dmd_forward( - generator_pred_video=generator_pred_video, - training_batch=batch_gen, - ) - student_backward_ctx = (batch_gen.timesteps, batch_gen.attn_metadata_vsa) - - batch_fake = copy.deepcopy(training_batch) - with torch.autocast(device_type, dtype=batch_fake.latents.dtype): - _, fake_score_loss = pipeline.faker_score_forward(batch_fake) - - total_loss = generator_loss + fake_score_loss - loss_map = { - "total_loss": total_loss, - "generator_loss": generator_loss, - "fake_score_loss": fake_score_loss, - } - outputs = {} - if update_student and batch_gen is not None: - outputs["dmd_latent_vis_dict"] = batch_gen.dmd_latent_vis_dict - outputs["fake_score_latent_vis_dict"] = batch_fake.fake_score_latent_vis_dict - outputs["_fv_backward"] = { - "update_student": update_student, - "student_ctx": student_backward_ctx, - "critic_ctx": (batch_fake.timesteps, batch_fake.attn_metadata), - } - return loss_map, outputs - - def backward( - self, - loss_map: dict[str, torch.Tensor], - outputs: dict[str, Any], - *, - grad_accum_rounds: int = 1, - ) -> None: - grad_accum_rounds = max(1, int(grad_accum_rounds)) - backward_ctx = outputs.get("_fv_backward") - if not isinstance(backward_ctx, dict): - super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) - return - - update_student = bool(backward_ctx.get("update_student", False)) - if update_student: - student_ctx = backward_ctx.get("student_ctx") - if student_ctx is None: - raise RuntimeError("Missing student backward context") - timesteps, attn_metadata = student_ctx - with set_forward_context( - current_timestep=timesteps, - attn_metadata=attn_metadata, - ): - (loss_map["generator_loss"] / grad_accum_rounds).backward() - - timesteps, attn_metadata = backward_ctx["critic_ctx"] - with set_forward_context( - current_timestep=timesteps, - attn_metadata=attn_metadata, - ): - (loss_map["fake_score_loss"] / grad_accum_rounds).backward() - - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: - optimizers: list[torch.optim.Optimizer] = [] - optimizers.extend(self.bundle.role("critic").optimizers.values()) - if self._should_update_student(iteration): - optimizers.extend(self.bundle.role("student").optimizers.values()) - return optimizers - - def get_lr_schedulers(self, iteration: int) -> list[Any]: - schedulers: list[Any] = [] - schedulers.extend(self.bundle.role("critic").lr_schedulers.values()) - if self._should_update_student(iteration): - schedulers.extend(self.bundle.role("student").lr_schedulers.values()) - return schedulers - - def optimizers_schedulers_step(self, iteration: int) -> None: - if self._should_update_student(iteration): - for module in self.bundle.role("student").modules.values(): - self._clip_grad_norm(module) - for module in self.bundle.role("critic").modules.values(): - self._clip_grad_norm(module) - - super().optimizers_schedulers_step(iteration) diff --git a/fastvideo/distillation/trainer.py b/fastvideo/distillation/trainer.py index ba5a599f7..f26da9b2e 100644 --- a/fastvideo/distillation/trainer.py +++ b/fastvideo/distillation/trainer.py @@ -42,7 +42,7 @@ def _iter_dataloader(self, dataloader: Any) -> Iterator[dict[str, Any]]: yield batch def _get_current_vsa_sparsity(self, step: int) -> float: - # Phase 0: keep behavior close to existing pipelines. + # Keep behavior close to existing pipelines. vsa_sparsity = self.training_args.VSA_sparsity vsa_decay_rate = self.training_args.VSA_decay_rate vsa_decay_interval_steps = self.training_args.VSA_decay_interval_steps diff --git a/fastvideo/tests/distillation/test_phase0_schedule.py b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py similarity index 97% rename from fastvideo/tests/distillation/test_phase0_schedule.py rename to fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py index 7c9666725..e37412406 100644 --- a/fastvideo/tests/distillation/test_phase0_schedule.py +++ b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py @@ -69,7 +69,7 @@ def get_lr_schedulers(self, iteration): # noqa: ANN001, ANN201 return schedulers -def test_phase0_optimizer_scheduler_alignment() -> None: +def test_optimizer_scheduler_alignment() -> None: method = _ScheduleMethod(interval=5) for step in range(1, 11): @@ -79,4 +79,3 @@ def test_phase0_optimizer_scheduler_alignment() -> None: assert method.critic_sched.step_calls == 10 assert method.student_opt.step_calls == 2 assert method.student_sched.step_calls == 2 - diff --git a/fastvideo/training/wan_distillation_v2.py b/fastvideo/training/wan_distillation_v2.py deleted file mode 100644 index e5c928886..000000000 --- a/fastvideo/training/wan_distillation_v2.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import sys - -from fastvideo.distillation import DistillTrainer, ModelBundle, RoleHandle -from fastvideo.distillation.adapters.wan import WanPipelineAdapter -from fastvideo.distillation.methods.wan_dmd2 import WanDMD2Method -from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.logger import init_logger -from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline -from fastvideo.utils import FlexibleArgumentParser - -logger = init_logger(__name__) - - -def _build_bundle_from_wan_pipeline( - pipeline: WanDistillationPipeline, -) -> ModelBundle: - roles: dict[str, RoleHandle] = { - "student": - RoleHandle( - modules={"transformer": pipeline.transformer}, - optimizers={"main": pipeline.optimizer}, - lr_schedulers={"main": pipeline.lr_scheduler}, - ), - "teacher": - RoleHandle( - modules={"transformer": pipeline.real_score_transformer}, - frozen=True, - ), - "critic": - RoleHandle( - modules={"transformer": pipeline.fake_score_transformer}, - optimizers={"main": pipeline.fake_score_optimizer}, - lr_schedulers={"main": pipeline.fake_score_lr_scheduler}, - ), - } - return ModelBundle(roles=roles) - - -def main(args) -> None: - logger.info("Starting Wan distillation v2 (Method/Trainer)...") - - pipeline = WanDistillationPipeline.from_pretrained( - args.pretrained_model_name_or_path, args=args) - training_args = pipeline.training_args - - adapter = WanPipelineAdapter(pipeline) - bundle = _build_bundle_from_wan_pipeline(pipeline) - method = WanDMD2Method(bundle=bundle, adapter=adapter) - - trainer = DistillTrainer(training_args, tracker=pipeline.tracker) - trainer.run( - method, - dataloader=pipeline.train_dataloader, - max_steps=training_args.max_train_steps, - start_step=pipeline.init_steps, - ) - - logger.info("Wan distillation v2 completed") - - -if __name__ == "__main__": - argv = sys.argv - parser = FlexibleArgumentParser() - parser = TrainingArgs.add_cli_args(parser) - parser = FastVideoArgs.add_cli_args(parser) - args = parser.parse_args(argv[1:]) - main(args) - From 8461d68f6767386cb46887f7da56e0144de5489d Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 01:15:00 +0000 Subject: [PATCH 056/214] design phase 2 --- dev/design.md | 105 +++++++++++++++------ dev/phases/phase_1.md | 2 +- dev/phases/phase_2.md | 209 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 287 insertions(+), 29 deletions(-) create mode 100644 dev/phases/phase_2.md diff --git a/dev/design.md b/dev/design.md index b0e4bd37d..2acc22f07 100644 --- a/dev/design.md +++ b/dev/design.md @@ -362,10 +362,17 @@ FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step` ### 6.1 最小可用（建议先落地） -- `--models.student ` -- `--models.teacher ` -- `--models.critic `（可选） -- `--distill.method dmd2|self_forcing|teacher_only` +Phase 1 已经落地的最小形态是：**复用 FastVideo 现有 TrainingArgs/FastVideoArgs**， +再加一个 “选择 distill 组合” 的入口参数： + +- `--distill-model wan|...` +- `--distill-method dmd2|...` + +其中 “roles -> model path” 暂时仍沿用现有 WAN distill 参数（Phase 2 会统一成 role-based 形态）： + +- student：`--model_path` / `--pretrained_model_name_or_path` +- teacher：`--real_score_model_path` +- critic：`--fake_score_model_path`（可选，取决于 method 需求） distill 专有参数建议用 namespace： @@ -397,9 +404,11 @@ Phase 0 的实践表明：先把新框架以 **additive** 方式落地到一个 建议结构（已部分实现）： - `fastvideo/distillation/bundle.py`：`ModelBundle/RoleHandle` -- `fastvideo/distillation/adapters/`：`WanPipelineAdapter`（Phase 0 过渡版）→ `WanAdapter`（目标） -- `fastvideo/distillation/methods/`：`base.py`、（目标）`dmd2.py`、（目标）`self_forcing.py` +- `fastvideo/distillation/adapters/`：`WanAdapter`（Phase 1 已落地；后续新增更多 adapter） +- `fastvideo/distillation/methods/`：`base.py`、`distribution_matching/dmd2.py`、（目标）`self_forcing.py` - `fastvideo/distillation/trainer.py`：`DistillTrainer` +- `fastvideo/distillation/builder.py`：把 “config -> roles -> bundle/adapter/method” 的胶水集中起来 +- `fastvideo/training/distillation.py`：通用入口（选择 distill_model + distill_method） - （后续）`fastvideo/distillation/checkpoint.py`：role-based `CheckpointManager`（先兼容旧格式） - （后续）`fastvideo/distillation/callbacks/`：EMA/clip/log/profiler 等 @@ -415,17 +424,10 @@ Phase 0 的实践表明：先把新框架以 **additive** 方式落地到一个 Phase 0 的定位在实践中更明确了：它是“**把旧 Wan distill pipeline 包一层新框架壳**”， 先把训练循环/多 optimizer 调度/validation hook 等基础设施固定下来，再逐步解耦。 -- ✅ 新增 `DistillTrainer/DistillMethod/ModelBundle/(pipeline-backed) WanAdapter` -- ✅ 新增一个 additive 入口：`fastvideo/training/wan_distillation_v2.py` - - 复用 legacy `WanDistillationPipeline` 完成模型加载/optimizer/dataloader/tracker - - 再把 student/teacher/critic 打包为 `ModelBundle(roles={...})` -- ✅ 跑通 Wan DMD2（student + teacher + critic） - - 过渡命名：`WanDMD2Method`（刻意暴露耦合，避免误解为通用 DMD2） -- ✅ 消除一个关键隐式耦合：训练前显式初始化 `neg/uncond conditioning` - - 不再依赖 validation 的副作用（见 `ensure_negative_conditioning()`） -- ✅ 修正并用单测锁定一个关键语义：scheduler step 与 optimizer step 对齐 +- ✅ 新增 `DistillTrainer/DistillMethod/ModelBundle` 的骨架，并跑通 WAN distill +- ✅ 用单测锁定关键语义：scheduler step 与 optimizer step 对齐 - `generator_update_interval > 1` 时不会“空 step scheduler” -- ✅ 提供 few-step distill 示例脚本 + 可直接跑的 temp 脚本 +- ✅ 为后续解耦铺路：把 “roles={student,teacher,critic}” 显式化到 bundle Phase 0 明确没有做（刻意延期）： @@ -433,22 +435,69 @@ Phase 0 明确没有做（刻意延期）： - ❌ `DMD2Method` 的真正算法解耦（目前仍调用旧 pipeline 内部函数） - ❌ Self-forcing v2 迁移 -### Phase 1（建议开启）：算法与模型真正解耦（先把 DMD2 “抠出来”） +### Phase 1（已完成）：算法与模型真正解耦（先把 DMD2 “抠出来”） Phase 1 的核心目标：把 Phase 0 的“脚手架耦合”逐步替换为 **Method(算法) + Adapter(模型)** 的稳定边界，让其它模型/其它方法可以复用 Trainer。 -- 产出通用算法：`fastvideo/distillation/methods/dmd2.py::DMD2Method` - - 不再依赖 `fastvideo/training/distillation_pipeline.py` 的私有函数 - - 只依赖 adapter 提供的 primitives（noise/add_noise/pred_to_x0/teacher_cfg/critic_forward 等） -- 产出真正模型适配：`WanAdapter`（替换 `WanPipelineAdapter`） - - 逐步把 normalize/layout/attention metadata/输入 kwargs 组装等从 legacy pipeline 迁出 -- Builder 层雏形：从 `TrainingArgs/FastVideoArgs`（或 `--models_json`）直接构建 - `ModelBundle + Adapter + Method` - - 目标：最终不再依赖 legacy pipeline 才能启动 v2 -- Validation 进一步抽象（可选）：把“怎么验证”从 method 里抽走，变成通用 hook/组件 - -### Phase 2：清理旧实现 + 扩展新模型/新算法 +Phase 1 的“辉煌”（落地与收益）： + +- ✅ 通用算法 method：`fastvideo/distillation/methods/distribution_matching/dmd2.py::DMD2Method` + - 算法层不再调用 legacy pipeline 私有算法函数 + - 依赖面缩到 adapter primitives（通过 `Protocol` 约束 surface） +- ✅ 真正的 WAN 适配层：`fastvideo/distillation/adapters/wan.py::WanAdapter` + - `forward_context` 与 backward 重算约束收敛到 adapter（method 只实现算法） + - `ensure_negative_conditioning()` 显式化（不再依赖 validation 的隐式副作用） +- ✅ Builder 雏形：`fastvideo/distillation/builder.py` + - 把 “roles -> bundle -> method” 的胶水集中在一处，便于扩展新 method/new model +- ✅ 通用入口：`fastvideo/training/distillation.py` + - CLI 选择：`--distill-model` + `--distill-method` +- ✅ 训练效果对齐：Phase 1 跑出来的 WAN DMD2 与 Phase 0/baseline 行为一致（已实测） + +### Phase 2（建议重点推进）：彻底脱离 legacy distill pipeline（让新框架可独立存在） + +你提的建议我同意：Phase 2 应该把 Phase 1 仍然残留的 legacy 依赖清干净，让新的 distill +代码路径可以 **不依赖** `fastvideo/training/*distillation_pipeline.py` 和 +`WanDistillationPipeline` 仍可运行训练与验证。 + +为了降低风险，建议 Phase 2 按 “先 validation、再 builder/runtime、最后清理入口” 的顺序推进。 + +#### Phase 2.1：Validation 独立化（优先级最高，收益最大） + +- 目标：`WanAdapter.log_validation()` 不再调用 legacy `pipeline._log_validation(...)` +- 建议实现： + - 新增 `fastvideo/distillation/validation/`（或 `fastvideo/distillation/validators/`） + - 由 adapter 提供 `build_validator(...)` 或直接实现 `adapter.sample(...)` + - 复用模块化 inference pipeline（例如 `fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py`） + 来生成视频并交给 tracker 记录 +- 收益：彻底消除 “validation 初始化副作用/属性缺失” 这类隐式耦合与脆弱点 + +#### Phase 2.2：Builder/Runtime 脱离 pipeline（roles/spec -> instantiate） + +- 目标：`fastvideo/training/distillation.py` 不再先 instantiate `WanDistillationPipeline` +- 建议实现： + - 定义结构化 spec：`RoleSpec/ModelSpec`（role -> {family, path, precision, frozen/trainable,...}） + - CLI 形态落地（择一）： + - `--models_json path/to/models.json`（推荐） + - 或 `--models.student ... --models.teacher ...`（人类可读但可扩展性较弱） + - builder 根据 spec： + - 加载 modules（student/teacher/critic） + - 构建 role-based optimizers/schedulers + - 组装 `ModelBundle + Adapter + Method` + - 构建 dataloader（直接复用 dataset 代码，不经由 legacy pipeline class） +- 收益：distill 路径具备真正的“模型/算法 catalog + instantiate”，开始能支持更多模型家族 + +#### Phase 2.3：role-based checkpoint/save/resume（新框架自洽） + +- 目标：新框架训练可 save/resume，且协议围绕 role 命名空间（不再绑死 WAN pipeline） +- 建议实现： + - `fastvideo/distillation/checkpoint.py`：保存/加载 modules + optimizers + schedulers + RNG states + - 明确兼容策略：兼容旧格式（若必要）或提供一次性转换脚本 + +#### Phase 2.4（Deferred）：收敛与清理（暂不做；完全解耦后手动处理） + +本轮 Phase 2 采用 **非侵入式** 策略：只新增新路径所需的代码，不做 legacy 代码搬家/清理。 +当 Phase 2.1/2.2/2.3 全部完成、并且新框架可以独立运行后，再由你手动清理旧入口/旧实现。 在 Phase 1 的稳定边界之上，Phase 2 再做“功能扩展 + 旧实现收敛”： diff --git a/dev/phases/phase_1.md b/dev/phases/phase_1.md index d717148ed..d36c1ed61 100644 --- a/dev/phases/phase_1.md +++ b/dev/phases/phase_1.md @@ -79,7 +79,7 @@ Phase 1 的定位：把 distillation 从 “pipeline god object” 拆成稳定 - [x] 移除 legacy-backed `WanPipelineAdapter` - [x] 移除旧的强耦合方法 `WanDMD2Method` - [x] 移除旧入口 `fastvideo/training/wan_distillation_v2.py` -- [x] 移除 `examples/distillation/phase0/`（避免误用旧脚本） +- [ ] 保留 `examples/distillation/phase0/`（暂存脚本用于对照；最终会统一清理） - [x] 清理代码里残留的 `Phase 0/phase0` 调用与命名 --- diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md new file mode 100644 index 000000000..70ad87523 --- /dev/null +++ b/dev/phases/phase_2.md @@ -0,0 +1,209 @@ +# Phase 2：让新 Distill 框架 **独立运行**（摆脱 legacy distill pipeline） + +Phase 2 的定位：在 Phase 1 已经验证 “Wan DMD2 训练行为对齐 baseline” 的前提下， +把当前仍然依赖 legacy pipeline 的部分逐步替换掉，使 **新 distill 代码路径可以独立运行** +（训练 / validation / checkpoint-resume），同时 **旧代码仍然可跑**。 + +> 约束：本 phase 采用 **非侵入式** 策略 —— 优先新增代码路径，不强行重构/迁移 legacy 文件。 +> 等到完全解耦之后，旧代码的清理由你手动完成（不在 Phase 2 做“删除/搬家”）。 + +--- + +## Phase 2 目标（可交付） + +1. **Validation 独立化**：不再调用 legacy `pipeline._log_validation(...)`，避免隐式依赖与脆弱属性。 +2. **Builder/Runtime 脱离 pipeline**：不再依赖 `WanDistillationPipeline.from_pretrained(...)` 来启动训练； + 改为从 `models={role -> spec}` 直接构建 `ModelBundle + Adapter + Method + DataLoader + Tracker`。 +3. **role-based checkpoint/save/resume**：新框架自洽地保存/恢复： + - per-role modules / optimizers / schedulers + - RNG states（含用于噪声采样的 generator） + - StatefulDataLoader（若使用） + +--- + +## Phase 2 非目标（明确不做） + +- 清理/删除 legacy distill code（你会在完全解耦后手动清理） +- 将 `fastvideo/training/*` 的通用函数迁移到更中立目录（设计里叫 Phase 2.4，先不做） +- 迁移 Self-forcing v2 / ODE-init（等 distill runtime 自洽后再做） + +--- + +## 当前“仍依赖 legacy”的点（Phase 2 要消灭） + +以 Phase 1 的 Wan DMD2 为例，当前仍有两类耦合： + +1. **启动/加载耦合**： + - `fastvideo/training/distillation.py` 仍通过 `WanDistillationPipeline.from_pretrained(...)` + 完成模型加载、optimizer/scheduler、dataloader、tracker。 +2. **validation 耦合**： + - `WanAdapter.log_validation()` 仍复用 legacy `pipeline._log_validation(...)` + +Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 + +--- + +## Phase 2 TODO List（Review Checklist） + +### A. Validation 独立化（Phase 2.1） + +- [ ] 定义通用接口：`DistillValidator` + - 输入：`bundle + adapter + training_args + tracker` + - 输出：`log_validation(step)`（只在 rank0 真正写 artifact） +- [ ] 实现 `WanValidator`（Wan + T2V 的最小版本） + - 复用 `fastvideo/dataset/validation_dataset.py::ValidationDataset` + - 复用模块化 **inference pipeline**（建议：`fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py::WanDMDPipeline`） + - 支持关键参数： + - `validation_sampling_steps`（few-step） + - `validation_guidance_scale` + - seed / RNG（不依赖 legacy pipeline 初始化副作用） +- [ ] 将 Phase 1 的 `WanAdapter.log_validation()` 切到新 validator（不再依赖 legacy pipeline） + +### B. Builder/Runtime 脱离 pipeline（Phase 2.2） + +- [ ] 定义结构化 spec（角色驱动）：`ModelSpec / RoleSpec / DistillSpec` + - 目标：`models={role -> spec}` 成为唯一真相 + - method 自己声明需要哪些 roles（缺失则报错） +- [ ] 支持配置入口（择一优先落地） + - [ ] `--models_json path/to/models.json`（推荐） + - [ ] （可选）`--models.student ... --models.teacher ...`（快捷但可扩展性较弱） +- [ ] 实现 “standalone runtime builder” + - 直接加载 modules（student/teacher/critic）并构建 `ModelBundle` + - 构建 per-role optimizers/schedulers（复用现有 TrainingArgs 超参） + - 构建 dataloader（直接调用 `fastvideo/dataset/parquet_dataset_map_style.py::build_parquet_map_style_dataloader`） + - 初始化 tracker（复用 `fastvideo/training/trackers/`） +- [ ] 新增一个 **standalone entrypoint**（不替换旧入口） + - 例如：`fastvideo/training/distillation_standalone.py` + - 和 Phase 1 `distillation.py` 并存（用于 A/B 与回滚） + +### C. role-based checkpoint/save/resume（Phase 2.3） + +- [ ] 新增 `DistillCheckpointManager` + - 保存内容： + - `bundle.roles[*].modules`（仅 trainable params 或全量可配置） + - `bundle.roles[*].optimizers / lr_schedulers` + - `StatefulDataLoader`（如果使用） + - `RandomStateWrapper`（torch/numpy/python/cuda + noise generators） + - 恢复内容： + - `start_step`、dataloader iterator state、optimizer/scheduler state、RNG +- [ ] 将 checkpoint manager 接入 `DistillTrainer` + - `--resume_from_checkpoint` + - `--checkpointing_steps`（或复用现有 args） + - `--checkpoints_total_limit` + +### D. 示例脚本（Phase 2） + +- [ ] `examples/distillation/phase2/`： + - 最小 smoke：Wan 1.3B 学 14B（DMD2）+ few-step validation + save/resume + - 提供 `models.json` 模板（role-based） + +### E. 最小单测（可选但建议） + +- [ ] `models_json` schema 解析 + role 校验 +- [ ] checkpoint roundtrip（mock modules + optimizer）不 crash + +--- + +## 关键设计（具体到代码） + +### 2.1 Validation 独立化：`DistillValidator` / `WanValidator` + +**核心原则**：validation 只是一段 “inference + decode + log”，它应当： + +- 不依赖训练 pipeline 的内部属性（例如 `validation_random_generator`） +- 不要求 `pipeline.train()` 被调用才“初始化齐全” + +建议落地： + +- 新增目录：`fastvideo/distillation/validation/` + - `base.py`：`DistillValidator` 抽象 + - `wan.py`：`WanValidator` + +`WanValidator` 的实现思路（最小版本）： + +1. 从 `training_args.validation_dataset_file` 构建 `ValidationDataset` +2. 取若干条 prompt（rank0） +3. 构建/复用一个 inference pipeline： + - `WanDMDPipeline.from_pretrained(student_model_path, loaded_modules=...)` + - 或者直接用 loader 加载 `text_encoder/tokenizer/vae/scheduler`，并注入 student transformer +4. 生成视频并交给 tracker（wandb）记录 + +**风险点（如遇到需要你决策会停下讨论）** + +- validation 要不要严格对齐 legacy `_log_validation` 的所有输出格式（latent vis dict 等）？ + - 建议 Phase 2 先只对齐“视频 artifact + caption”，其余可后续补齐。 + +### 2.2 Builder/Runtime 独立化：roles/spec -> instantiate + +**目标**：从 “先构建 legacy pipeline，再拆 roles” 转成 “roles/spec 直接构建 bundle”。 + +建议新增： + +- `fastvideo/distillation/specs.py` + - `ModelSpec`：`family/path/revision/precision/...` + - `RoleSpec`：`role/frozen/optimizer/scheduler/...` + - `DistillSpec`：`method + models{role->RoleSpec} + adapter_family(optional)` + +一个最小的 `models.json` 示例（Wan 1.3B 学 14B + critic=1.3B）： + +```json +{ + "adapter_family": "wan", + "method": "dmd2", + "models": { + "student": {"family": "wan", "path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "trainable": true}, + "teacher": {"family": "wan", "path": "Wan-AI/Wan2.1-T2V-14B-Diffusers", "frozen": true}, + "critic": {"family": "wan", "path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "trainable": true} + } +} +``` + +**加载实现建议（Wan）** + +尽量复用已经存在的、与 distill training pipeline 无关的 loader 工具链： + +- `fastvideo/utils.py::maybe_download_model / verify_model_config_and_directory` +- `fastvideo/models/loader/component_loader.py::PipelineComponentLoader` + +这样我们无需实例化 `WanDistillationPipeline`，也能加载： + +- `transformer`（student/teacher/critic） +- （可选）`transformer_2`（MoE 支持，Phase 2 先保持 optional） +- `vae/text_encoder/tokenizer/scheduler`（用于 adapter + validation） + +**构建 optimizer/scheduler** + +- 在 builder 内按 role 创建 optimizer/scheduler，并放进 `RoleHandle.optimizers/lr_schedulers` +- method 继续用 `get_optimizers/get_lr_schedulers` 做 update policy（Trainer 不关心 role） + +**入口文件** + +- 新增 `fastvideo/training/distillation_standalone.py` + - 解析 args（复用 TrainingArgs/FastVideoArgs） + - 读取 `models_json`（若存在） + - 调用 standalone builder 构建： + - `method` + - `dataloader` + - `tracker` + - `validator` + - `checkpoint_manager` + - `DistillTrainer.run(method, ...)` + +> 这条路径与 `fastvideo/training/distillation.py` 并存，旧路径不动。 + +### 2.3 role-based checkpoint：`DistillCheckpointManager` + +建议新增： + +- `fastvideo/distillation/checkpoint.py` + - `DistillCheckpointManager` + - 内部复用 `fastvideo/training/checkpointing_utils.py` 的 wrappers： + - `ModelWrapper/OptimizerWrapper/SchedulerWrapper/RandomStateWrapper` + +建议 checkpoint 的 “state dict key” 命名空间： + +- `models/{role}/{module_name}` +- `optimizers/{role}/{name}` +- `schedulers/{role}/{name}` +- `random/{name}`（全局 RNG + per-role noise generator） + From c9681cee51e5fb011df0a24e79365d4fbb10b73f Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 01:20:51 +0000 Subject: [PATCH 057/214] designing phase 2: config --- dev/design.md | 96 ++++++++++++++++++++++++++++++++++++++++--- dev/phases/phase_2.md | 69 ++++++++++++++++--------------- 2 files changed, 128 insertions(+), 37 deletions(-) diff --git a/dev/design.md b/dev/design.md index 2acc22f07..b73544239 100644 --- a/dev/design.md +++ b/dev/design.md @@ -360,9 +360,18 @@ FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step` ## 6. 配置与 CLI 形态（渐进式） +> Phase 2 开始，我们需要把 “如何启动一次 distill” 从大量 CLI 参数，演进为 **YAML 驱动** +> 的结构化配置（可读、可复现、可审查）。同时为了不破坏现有用法，CLI 只作为 override。 + ### 6.1 最小可用（建议先落地） -Phase 1 已经落地的最小形态是：**复用 FastVideo 现有 TrainingArgs/FastVideoArgs**， +**推荐（Phase 2 目标）**：一个 YAML 配置文件描述一次 distill 运行，入口只需要： + +- `fastvideo/training/distillation.py --config path/to/distill.yaml` + +并允许少量 CLI overrides（只覆盖显式提供的参数）。 + +**当前（Phase 1 已落地）**：仍然复用 FastVideo 现有 TrainingArgs/FastVideoArgs 的 CLI， 再加一个 “选择 distill 组合” 的入口参数： - `--distill-model wan|...` @@ -385,7 +394,80 @@ distill 专有参数建议用 namespace： - `--models_json path/to/models.json` - per-role precision/offload/trainable/fsdp_policy/ckpt_path 等 -### 6.3 配置系统演进（可选吸收 FastGen 的优点） +### 6.3 YAML 配置（Phase 2 必做）：结构化训练参数 + roles 选择 + +我们希望最终的 “单次运行” 配置长这样（示意；字段可迭代）： + +```yaml +distill: + model: wan + method: dmd2 + +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + frozen: true + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + output_dir: outputs/... + max_train_steps: 4000 + seed: 1000 + # ... (TrainingArgs/FastVideoArgs 的字段) + +pipeline_config: + # 支持直接内联覆盖，也支持只给 pipeline_config_path + # pipeline_config_path: fastvideo/configs/wan_1.3B_t2v_pipeline.json + flow_shift: 8 +``` + +**解析策略（最优雅且低风险）** + +- 入口 parser 仍然保留（便于 torchrun/集群 launch），但只保留： + - `--config distill.yaml` + - 以及少量 override（可选） +- 若提供 `--config`： + 1) `yaml.safe_load` 得到 dict + 2) 用现有 `clean_cli_args(args)` 收集“显式提供的 CLI 参数” + 3) 做 merge：`yaml_cfg` <- `cli_overrides` + 4) 最终用 `TrainingArgs.from_kwargs(**merged)` 实例化（由现有 PipelineConfig/PreprocessConfig 负责子配置） + +这样不需要推翻现有 TrainingArgs/FastVideoArgs 体系，只是把 “输入源” 从 CLI 扩展为 YAML。 + +### 6.4 `outside/` overlay（Phase 2 约束下的 workaround） + +我们不能直接修改大项目里的 `fastvideo/configs/`（避免冲突/合并成本）。 +因此 Phase 2 建议在 distillation 侧新增一个 overlay 根目录： + +- `fastvideo/distillation/outside/` + +并约定： + +- 把“本应在外部 repo 存在的新增/改版配置”放进： + - `fastvideo/distillation/outside/fastvideo/configs/...` +- distillation 的配置加载器在解析任何 config 路径时： + - **先查 outside overlay 是否存在同路径文件** + - 若不存在，再 fallback 到 repo 内的 `fastvideo/configs/...` + +这让我们可以在不侵入主仓库配置的情况下，迭代 YAML/JSON config、做实验性变更， +同时不影响 legacy 代码路径。 + +**实现注意** + +- 不建议把 `outside/` 直接插入 `sys.path` 去 shadow 整个 `fastvideo` 包（风险太高、调试困难）。 +- 推荐把 `outside/` 仅作为 **配置文件 overlay**（YAML/JSON）来做路径解析。 +- 如果确实需要覆盖 Python config（`.py`）： + - 用 `importlib` 的“按文件路径加载模块”方式加载为独立 module name，避免影响全局 import。 + +### 6.5 配置系统演进（可选吸收 FastGen 的优点） FastGen 的 python config + instantiate + override 很优秀，但 FastVideo 现阶段可以先： @@ -477,14 +559,18 @@ Phase 1 的“辉煌”（落地与收益）： - 目标：`fastvideo/training/distillation.py` 不再先 instantiate `WanDistillationPipeline` - 建议实现： - 定义结构化 spec：`RoleSpec/ModelSpec`（role -> {family, path, precision, frozen/trainable,...}） - - CLI 形态落地（择一）： - - `--models_json path/to/models.json`（推荐） - - 或 `--models.student ... --models.teacher ...`（人类可读但可扩展性较弱） + - 配置形态落地（Phase 2 必做）： + - `--config path/to/distill.yaml`（YAML 为 single source of truth；CLI 只做 override） + - `outside/` overlay：解析 `pipeline_config_path` 等文件路径时 outside 优先、repo fallback + - （可选）保留 `--models_json` 作为“程序生成配置”的接口 - builder 根据 spec： - 加载 modules（student/teacher/critic） - 构建 role-based optimizers/schedulers - 组装 `ModelBundle + Adapter + Method` - 构建 dataloader（直接复用 dataset 代码，不经由 legacy pipeline class） + - 不新增入口文件：直接增强 `fastvideo/training/distillation.py` + - 有 `--config` 时走新 builder/runtime + - 无 `--config` 时保留旧 pipeline 路径（legacy 仍可跑） - 收益：distill 路径具备真正的“模型/算法 catalog + instantiate”，开始能支持更多模型家族 #### Phase 2.3：role-based checkpoint/save/resume（新框架自洽） diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md index 70ad87523..4766089df 100644 --- a/dev/phases/phase_2.md +++ b/dev/phases/phase_2.md @@ -7,6 +7,9 @@ Phase 2 的定位：在 Phase 1 已经验证 “Wan DMD2 训练行为对齐 base > 约束：本 phase 采用 **非侵入式** 策略 —— 优先新增代码路径，不强行重构/迁移 legacy 文件。 > 等到完全解耦之后，旧代码的清理由你手动完成（不在 Phase 2 做“删除/搬家”）。 +> 额外约束（你拍板）：**不新增任何入口文件**。 +> Phase 2 的新能力直接落在 `fastvideo/training/distillation.py` 内，通过配置选择走新/旧路径。 + --- ## Phase 2 目标（可交付） @@ -18,6 +21,8 @@ Phase 2 的定位：在 Phase 1 已经验证 “Wan DMD2 训练行为对齐 base - per-role modules / optimizers / schedulers - RNG states（含用于噪声采样的 generator） - StatefulDataLoader（若使用） +4. **YAML 驱动的训练参数解析**：用 `distill.yaml` 描述一次运行，CLI 仅做 override。 +5. **`outside/` overlay workaround**：不修改主仓库 `fastvideo/configs/`，在 distillation 内提供可覆盖的“外部配置根”。 --- @@ -64,17 +69,23 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 - [ ] 定义结构化 spec（角色驱动）：`ModelSpec / RoleSpec / DistillSpec` - 目标：`models={role -> spec}` 成为唯一真相 - method 自己声明需要哪些 roles（缺失则报错） -- [ ] 支持配置入口（择一优先落地） - - [ ] `--models_json path/to/models.json`（推荐） - - [ ] （可选）`--models.student ... --models.teacher ...`（快捷但可扩展性较弱） +- [ ] 新增 YAML 配置解析（Phase 2 必做） + - [ ] `fastvideo/training/distillation.py` 支持 `--config path/to/distill.yaml` + - [ ] 解析策略：`yaml.safe_load` + `clean_cli_args(args)` merge（CLI 仅覆盖显式传入字段） + - [ ] YAML schema 最小包含：`distill.{model,method}` + `models{role->spec}` + `training{...}` + `pipeline_config{...}` +- [ ] 支持 `outside/` overlay（Phase 2 workaround） + - [ ] 新增目录：`fastvideo/distillation/outside/`（视作外部 repo root） + - [ ] 约定覆盖路径：`fastvideo/distillation/outside/fastvideo/configs/...` + - [ ] config loader 在解析 `pipeline_config_path`/yaml include 时：outside 优先、repo fallback + - [ ] 不通过 `sys.path` shadow `fastvideo` 包；outside 仅做文件 overlay（必要时用 `importlib` 按路径加载 .py config） - [ ] 实现 “standalone runtime builder” - 直接加载 modules（student/teacher/critic）并构建 `ModelBundle` - 构建 per-role optimizers/schedulers（复用现有 TrainingArgs 超参） - 构建 dataloader（直接调用 `fastvideo/dataset/parquet_dataset_map_style.py::build_parquet_map_style_dataloader`） - 初始化 tracker（复用 `fastvideo/training/trackers/`） -- [ ] 新增一个 **standalone entrypoint**（不替换旧入口） - - 例如：`fastvideo/training/distillation_standalone.py` - - 和 Phase 1 `distillation.py` 并存（用于 A/B 与回滚） +- [ ] 修改现有入口 `fastvideo/training/distillation.py`（不新增入口文件） + - 若提供 `--config`（或未来 `--models_json`）：走 standalone builder 路径 + - 否则：保留现有 legacy pipeline 路径（确保旧 code 可跑） ### C. role-based checkpoint/save/resume（Phase 2.3） @@ -95,7 +106,7 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 - [ ] `examples/distillation/phase2/`： - 最小 smoke：Wan 1.3B 学 14B（DMD2）+ few-step validation + save/resume - - 提供 `models.json` 模板（role-based） + - 提供 `distill.yaml` 模板（role-based） ### E. 最小单测（可选但建议） @@ -144,18 +155,22 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 - `RoleSpec`：`role/frozen/optimizer/scheduler/...` - `DistillSpec`：`method + models{role->RoleSpec} + adapter_family(optional)` -一个最小的 `models.json` 示例（Wan 1.3B 学 14B + critic=1.3B）： - -```json -{ - "adapter_family": "wan", - "method": "dmd2", - "models": { - "student": {"family": "wan", "path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "trainable": true}, - "teacher": {"family": "wan", "path": "Wan-AI/Wan2.1-T2V-14B-Diffusers", "frozen": true}, - "critic": {"family": "wan", "path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "trainable": true} - } -} +一个最小的 `distill.yaml` 示例（Wan 1.3B 学 14B + critic=1.3B；示意）： + +```yaml +distill: + model: wan + method: dmd2 + +models: + student: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} + teacher: {family: wan, path: Wan-AI/Wan2.1-T2V-14B-Diffusers, frozen: true} + critic: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} + +training: + output_dir: outputs/phase2_wan_dmd2 + max_train_steps: 4000 + seed: 1000 ``` **加载实现建议（Wan）** @@ -178,18 +193,9 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 **入口文件** -- 新增 `fastvideo/training/distillation_standalone.py` - - 解析 args（复用 TrainingArgs/FastVideoArgs） - - 读取 `models_json`（若存在） - - 调用 standalone builder 构建： - - `method` - - `dataloader` - - `tracker` - - `validator` - - `checkpoint_manager` - - `DistillTrainer.run(method, ...)` - -> 这条路径与 `fastvideo/training/distillation.py` 并存，旧路径不动。 +- 不新增入口文件；直接增强 `fastvideo/training/distillation.py`： + - `--config distill.yaml` 走 standalone builder + - 不传 `--config` 则走 legacy pipeline（保持旧代码可跑） ### 2.3 role-based checkpoint：`DistillCheckpointManager` @@ -206,4 +212,3 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 - `optimizers/{role}/{name}` - `schedulers/{role}/{name}` - `random/{name}`（全局 RNG + per-role noise generator） - From 889c1c55511cfef9564971a728de52d60c533cd1 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 01:32:32 +0000 Subject: [PATCH 058/214] designing phase 2: config 2 --- dev/design.md | 58 +++++++++++++++---------------------------- dev/phases/phase_2.md | 22 ++++++++-------- 2 files changed, 32 insertions(+), 48 deletions(-) diff --git a/dev/design.md b/dev/design.md index b73544239..5e568255e 100644 --- a/dev/design.md +++ b/dev/design.md @@ -123,7 +123,7 @@ ModelBundle RoleHandle modules: dict[str, nn.Module] # e.g. {"transformer": ..., "transformer_2": ...} - frozen: bool + trainable: bool precision: optional # bf16/fp16/fp32 fsdp_policy: optional # shard strategy / ignored modules ema: optional @@ -360,34 +360,16 @@ FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step` ## 6. 配置与 CLI 形态（渐进式） -> Phase 2 开始，我们需要把 “如何启动一次 distill” 从大量 CLI 参数，演进为 **YAML 驱动** -> 的结构化配置（可读、可复现、可审查）。同时为了不破坏现有用法，CLI 只作为 override。 +> Phase 2 开始，新的 distillation 入口 **不再兼容旧式 CLI 传参**。 +> 我们只接受新的结构化配置（YAML），让一次运行可读、可复现、可审查。 ### 6.1 最小可用（建议先落地） -**推荐（Phase 2 目标）**：一个 YAML 配置文件描述一次 distill 运行，入口只需要： +**Phase 2 目标**：一个 YAML 配置文件描述一次 distill 运行，入口只需要： - `fastvideo/training/distillation.py --config path/to/distill.yaml` -并允许少量 CLI overrides（只覆盖显式提供的参数）。 - -**当前（Phase 1 已落地）**：仍然复用 FastVideo 现有 TrainingArgs/FastVideoArgs 的 CLI， -再加一个 “选择 distill 组合” 的入口参数： - -- `--distill-model wan|...` -- `--distill-method dmd2|...` - -其中 “roles -> model path” 暂时仍沿用现有 WAN distill 参数（Phase 2 会统一成 role-based 形态）： - -- student：`--model_path` / `--pretrained_model_name_or_path` -- teacher：`--real_score_model_path` -- critic：`--fake_score_model_path`（可选，取决于 method 需求） - -distill 专有参数建议用 namespace： - -- `--distill.dmd2.student_update_freq 5` -- `--distill.dmd2.guidance_scale 3.5` -- `--distill.sf.num_frame_per_block 3` +除此之外的训练参数/模型选择/方法选择，都写入 YAML。 ### 6.2 复杂配置（建议支持） @@ -411,7 +393,7 @@ models: teacher: family: wan path: Wan-AI/Wan2.1-T2V-14B-Diffusers - frozen: true + trainable: false critic: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers @@ -431,16 +413,15 @@ pipeline_config: **解析策略（最优雅且低风险）** -- 入口 parser 仍然保留（便于 torchrun/集群 launch），但只保留： - - `--config distill.yaml` - - 以及少量 override（可选） -- 若提供 `--config`： +- 新入口的 parser 只保留 `--config distill.yaml`（以及少量 meta flags，如 `--dry-run`）。 +- 训练相关的所有参数（TrainingArgs/FastVideoArgs/pipeline_config/method/models）都来自 YAML。 +- 解析流程： 1) `yaml.safe_load` 得到 dict - 2) 用现有 `clean_cli_args(args)` 收集“显式提供的 CLI 参数” - 3) 做 merge：`yaml_cfg` <- `cli_overrides` - 4) 最终用 `TrainingArgs.from_kwargs(**merged)` 实例化（由现有 PipelineConfig/PreprocessConfig 负责子配置） + 2) 规范化/校验 schema（distill/models/training/pipeline_config/...） + 3) 将 `training:` 与 `pipeline_config:` 合成 kwargs，调用 `TrainingArgs.from_kwargs(**kwargs)` + （由现有 PipelineConfig/PreprocessConfig 负责子配置实例化与校验） -这样不需要推翻现有 TrainingArgs/FastVideoArgs 体系，只是把 “输入源” 从 CLI 扩展为 YAML。 +这样不需要推翻现有 TrainingArgs/FastVideoArgs 体系，但从入口层面彻底摒弃旧式 CLI 传参方式。 ### 6.4 `outside/` overlay（Phase 2 约束下的 workaround） @@ -533,7 +514,8 @@ Phase 1 的“辉煌”（落地与收益）： - ✅ Builder 雏形：`fastvideo/distillation/builder.py` - 把 “roles -> bundle -> method” 的胶水集中在一处，便于扩展新 method/new model - ✅ 通用入口：`fastvideo/training/distillation.py` - - CLI 选择：`--distill-model` + `--distill-method` + - Phase 1 仍是 CLI 选择：`--distill-model` + `--distill-method` + - Phase 2 起将切换为 **YAML-only**（见第 6 节），并逐步废弃这套 CLI - ✅ 训练效果对齐：Phase 1 跑出来的 WAN DMD2 与 Phase 0/baseline 行为一致（已实测） ### Phase 2（建议重点推进）：彻底脱离 legacy distill pipeline（让新框架可独立存在） @@ -558,9 +540,9 @@ Phase 1 的“辉煌”（落地与收益）： - 目标：`fastvideo/training/distillation.py` 不再先 instantiate `WanDistillationPipeline` - 建议实现： - - 定义结构化 spec：`RoleSpec/ModelSpec`（role -> {family, path, precision, frozen/trainable,...}） + - 定义结构化 spec：`RoleSpec/ModelSpec`（role -> {family, path, precision, trainable,...}） - 配置形态落地（Phase 2 必做）： - - `--config path/to/distill.yaml`（YAML 为 single source of truth；CLI 只做 override） + - `--config path/to/distill.yaml`（YAML 为 single source of truth；CLI 仅指定配置路径） - `outside/` overlay：解析 `pipeline_config_path` 等文件路径时 outside 优先、repo fallback - （可选）保留 `--models_json` 作为“程序生成配置”的接口 - builder 根据 spec： @@ -568,9 +550,9 @@ Phase 1 的“辉煌”（落地与收益）： - 构建 role-based optimizers/schedulers - 组装 `ModelBundle + Adapter + Method` - 构建 dataloader（直接复用 dataset 代码，不经由 legacy pipeline class） - - 不新增入口文件：直接增强 `fastvideo/training/distillation.py` - - 有 `--config` 时走新 builder/runtime - - 无 `--config` 时保留旧 pipeline 路径（legacy 仍可跑） + - 不新增入口文件：直接增强 `fastvideo/training/distillation.py`，并把它定义为 **YAML-only distill entrypoint** + - 仅支持 `--config distill.yaml`（以及少量 meta flags），不再兼容旧式 CLI configs + - legacy distill 继续通过原有 `fastvideo/training/*distillation_pipeline.py` 入口运行（两套路径并存） - 收益：distill 路径具备真正的“模型/算法 catalog + instantiate”，开始能支持更多模型家族 #### Phase 2.3：role-based checkpoint/save/resume（新框架自洽） diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md index 4766089df..9bb7623d4 100644 --- a/dev/phases/phase_2.md +++ b/dev/phases/phase_2.md @@ -2,13 +2,15 @@ Phase 2 的定位：在 Phase 1 已经验证 “Wan DMD2 训练行为对齐 baseline” 的前提下， 把当前仍然依赖 legacy pipeline 的部分逐步替换掉，使 **新 distill 代码路径可以独立运行** -（训练 / validation / checkpoint-resume），同时 **旧代码仍然可跑**。 +（训练 / validation / checkpoint-resume），同时 **旧代码仍然可跑**（通过保留旧入口文件）。 > 约束：本 phase 采用 **非侵入式** 策略 —— 优先新增代码路径，不强行重构/迁移 legacy 文件。 > 等到完全解耦之后，旧代码的清理由你手动完成（不在 Phase 2 做“删除/搬家”）。 > 额外约束（你拍板）：**不新增任何入口文件**。 -> Phase 2 的新能力直接落在 `fastvideo/training/distillation.py` 内，通过配置选择走新/旧路径。 +> 新 distill 入口直接落在 `fastvideo/training/distillation.py`，并且 **仅接受新的 YAML configs** +>（不兼容旧式 CLI configs）。legacy distill 继续通过现有 +> `fastvideo/training/*distillation_pipeline.py` 入口运行（两套路径并存）。 --- @@ -21,7 +23,7 @@ Phase 2 的定位：在 Phase 1 已经验证 “Wan DMD2 训练行为对齐 base - per-role modules / optimizers / schedulers - RNG states（含用于噪声采样的 generator） - StatefulDataLoader（若使用） -4. **YAML 驱动的训练参数解析**：用 `distill.yaml` 描述一次运行，CLI 仅做 override。 +4. **YAML 驱动的训练参数解析**：用 `distill.yaml` 描述一次运行；入口只接受新 config（不做 legacy CLI merge）。 5. **`outside/` overlay workaround**：不修改主仓库 `fastvideo/configs/`，在 distillation 内提供可覆盖的“外部配置根”。 --- @@ -71,7 +73,7 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 - method 自己声明需要哪些 roles（缺失则报错） - [ ] 新增 YAML 配置解析（Phase 2 必做） - [ ] `fastvideo/training/distillation.py` 支持 `--config path/to/distill.yaml` - - [ ] 解析策略：`yaml.safe_load` + `clean_cli_args(args)` merge（CLI 仅覆盖显式传入字段） + - [ ] 解析策略：`yaml.safe_load` + schema 校验（不做 legacy CLI merge） - [ ] YAML schema 最小包含：`distill.{model,method}` + `models{role->spec}` + `training{...}` + `pipeline_config{...}` - [ ] 支持 `outside/` overlay（Phase 2 workaround） - [ ] 新增目录：`fastvideo/distillation/outside/`（视作外部 repo root） @@ -84,8 +86,8 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 - 构建 dataloader（直接调用 `fastvideo/dataset/parquet_dataset_map_style.py::build_parquet_map_style_dataloader`） - 初始化 tracker（复用 `fastvideo/training/trackers/`） - [ ] 修改现有入口 `fastvideo/training/distillation.py`（不新增入口文件） - - 若提供 `--config`（或未来 `--models_json`）：走 standalone builder 路径 - - 否则：保留现有 legacy pipeline 路径（确保旧 code 可跑） + - `--config` 为必需参数：只跑 standalone builder 路径（YAML-only） + - legacy distill 通过旧入口文件继续可跑（不在该入口做 fallback） ### C. role-based checkpoint/save/resume（Phase 2.3） @@ -152,7 +154,7 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 - `fastvideo/distillation/specs.py` - `ModelSpec`：`family/path/revision/precision/...` - - `RoleSpec`：`role/frozen/optimizer/scheduler/...` + - `RoleSpec`：`role/trainable/optimizer/scheduler/...` - `DistillSpec`：`method + models{role->RoleSpec} + adapter_family(optional)` 一个最小的 `distill.yaml` 示例（Wan 1.3B 学 14B + critic=1.3B；示意）： @@ -164,7 +166,7 @@ distill: models: student: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} - teacher: {family: wan, path: Wan-AI/Wan2.1-T2V-14B-Diffusers, frozen: true} + teacher: {family: wan, path: Wan-AI/Wan2.1-T2V-14B-Diffusers, trainable: false} critic: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} training: @@ -194,8 +196,8 @@ training: **入口文件** - 不新增入口文件；直接增强 `fastvideo/training/distillation.py`： - - `--config distill.yaml` 走 standalone builder - - 不传 `--config` 则走 legacy pipeline（保持旧代码可跑） + - 仅支持 `--config distill.yaml`（YAML-only），不再兼容旧式 CLI configs + - legacy pipeline 继续通过现有 `fastvideo/training/*distillation_pipeline.py` 入口运行 ### 2.3 role-based checkpoint：`DistillCheckpointManager` From 7431e9561f4eac44034dc701342762daf2cbdcfa Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 01:47:45 +0000 Subject: [PATCH 059/214] progressing phase 2 --- fastvideo/distillation/builder.py | 4 +- fastvideo/distillation/bundle.py | 3 +- fastvideo/distillation/outside/README.md | 14 ++ fastvideo/distillation/specs.py | 29 ++++ fastvideo/distillation/yaml_config.py | 176 +++++++++++++++++++++++ 5 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 fastvideo/distillation/outside/README.md create mode 100644 fastvideo/distillation/specs.py create mode 100644 fastvideo/distillation/yaml_config.py diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index a4fdc0ab4..05f6c466a 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -17,17 +17,19 @@ def build_wan_dmd2_method( modules={"transformer": pipeline.transformer}, optimizers={"main": pipeline.optimizer}, lr_schedulers={"main": pipeline.lr_scheduler}, + trainable=True, ), "teacher": RoleHandle( modules={"transformer": pipeline.real_score_transformer}, - frozen=True, + trainable=False, ), "critic": RoleHandle( modules={"transformer": pipeline.fake_score_transformer}, optimizers={"main": pipeline.fake_score_optimizer}, lr_schedulers={"main": pipeline.fake_score_lr_scheduler}, + trainable=True, ), } bundle = ModelBundle(roles=roles) diff --git a/fastvideo/distillation/bundle.py b/fastvideo/distillation/bundle.py index 47e6016bb..a971665bb 100644 --- a/fastvideo/distillation/bundle.py +++ b/fastvideo/distillation/bundle.py @@ -15,7 +15,7 @@ class RoleHandle: modules: dict[str, torch.nn.Module] = field(default_factory=dict) optimizers: dict[str, torch.optim.Optimizer] = field(default_factory=dict) lr_schedulers: dict[str, Any] = field(default_factory=dict) - frozen: bool = False + trainable: bool = True def require_module(self, name: str) -> torch.nn.Module: if name not in self.modules: @@ -36,4 +36,3 @@ def role(self, role: RoleName) -> RoleHandle: if role not in self.roles: raise KeyError(f"Unknown role: {role}") return self.roles[role] - diff --git a/fastvideo/distillation/outside/README.md b/fastvideo/distillation/outside/README.md new file mode 100644 index 000000000..5855410f0 --- /dev/null +++ b/fastvideo/distillation/outside/README.md @@ -0,0 +1,14 @@ +# `outside/` overlay + +This directory is a Phase 2 workaround for iterating on new YAML/JSON configs +without modifying the main repository's `fastvideo/configs/` tree. + +The distillation config loader resolves file paths via an overlay lookup: + +- if a run config references `fastvideo/configs/foo.json`, it will first check + `fastvideo/distillation/outside/fastvideo/configs/foo.json`; +- otherwise it falls back to the original path. + +Keep `outside/` for **data/config files only** (do not place importable Python +code here). + diff --git a/fastvideo/distillation/specs.py b/fastvideo/distillation/specs.py new file mode 100644 index 000000000..09048918e --- /dev/null +++ b/fastvideo/distillation/specs.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from dataclasses import dataclass + +RoleName = str + + +@dataclass(slots=True) +class DistillSpec: + """Selects the model family + distillation method. + + This is intentionally small: everything else (roles, training args, and + pipeline config) lives in the run config. + """ + + model: str + method: str + + +@dataclass(slots=True) +class RoleSpec: + """Describes a role's model source and whether it should be trained.""" + + family: str + path: str + trainable: bool = True + diff --git a/fastvideo/distillation/yaml_config.py b/fastvideo/distillation/yaml_config.py new file mode 100644 index 000000000..44a2dd563 --- /dev/null +++ b/fastvideo/distillation/yaml_config.py @@ -0,0 +1,176 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from fastvideo.fastvideo_args import ExecutionMode, TrainingArgs +from fastvideo.logger import init_logger + +from fastvideo.distillation.specs import DistillSpec, RoleName, RoleSpec + +logger = init_logger(__name__) + + +@dataclass(slots=True) +class DistillRunConfig: + distill: DistillSpec + roles: dict[RoleName, RoleSpec] + training_args: TrainingArgs + raw: dict[str, Any] + + +def _distillation_root() -> Path: + # .../fastvideo/distillation/yaml_config.py -> .../fastvideo/distillation + return Path(__file__).resolve().parent + + +def _repo_root() -> Path: + # .../fastvideo/distillation -> .../ + return _distillation_root().parent.parent + + +def _outside_root() -> Path: + return _distillation_root() / "outside" + + +def resolve_outside_overlay(path: str) -> str: + """Resolve ``path`` via the distillation ``outside/`` overlay if present. + + The overlay root is ``fastvideo/distillation/outside/`` and mirrors the + repository layout. For example, if the run config references: + + fastvideo/configs/foo.json + + then we first check: + + fastvideo/distillation/outside/fastvideo/configs/foo.json + + and fall back to the original path when the overlay file does not exist. + """ + + if not path: + return path + + expanded = os.path.expanduser(path) + candidate: Path | None = None + + p = Path(expanded) + if p.is_absolute(): + try: + rel = p.resolve().relative_to(_repo_root()) + except Exception: + rel = None + if rel is not None: + candidate = _outside_root() / rel + else: + candidate = _outside_root() / p + + if candidate is not None and candidate.exists(): + logger.info("Using outside overlay for %s -> %s", path, candidate) + return str(candidate) + + return expanded + + +def _require_mapping(raw: Any, *, where: str) -> dict[str, Any]: + if not isinstance(raw, dict): + raise ValueError(f"Expected mapping at {where}, got {type(raw).__name__}") + return raw + + +def _require_str(raw: Any, *, where: str) -> str: + if not isinstance(raw, str) or not raw.strip(): + raise ValueError(f"Expected non-empty string at {where}") + return raw + + +def _get_bool(raw: Any, *, where: str, default: bool) -> bool: + if raw is None: + return default + if isinstance(raw, bool): + return raw + raise ValueError(f"Expected bool at {where}, got {type(raw).__name__}") + + +def load_distill_run_config(path: str) -> DistillRunConfig: + """Load a Phase 2 distillation run config from YAML. + + This loader intentionally does **not** merge with legacy CLI args. The YAML + file is the single source of truth for a run. + """ + + path = resolve_outside_overlay(path) + with open(path, "r", encoding="utf-8") as f: + raw = yaml.safe_load(f) + cfg = _require_mapping(raw, where=path) + + distill_raw = _require_mapping(cfg.get("distill"), where="distill") + distill_model = _require_str(distill_raw.get("model"), where="distill.model") + distill_method = _require_str(distill_raw.get("method"), where="distill.method") + distill = DistillSpec(model=distill_model, method=distill_method) + + roles_raw = _require_mapping(cfg.get("models"), where="models") + roles: dict[RoleName, RoleSpec] = {} + for role, role_cfg_raw in roles_raw.items(): + role_str = _require_str(role, where="models.") + role_cfg = _require_mapping(role_cfg_raw, where=f"models.{role_str}") + family = role_cfg.get("family") or distill_model + family = _require_str(family, where=f"models.{role_str}.family") + model_path = _require_str(role_cfg.get("path"), where=f"models.{role_str}.path") + trainable = _get_bool( + role_cfg.get("trainable"), + where=f"models.{role_str}.trainable", + default=True, + ) + roles[role_str] = RoleSpec(family=family, path=model_path, trainable=trainable) + + training_raw = _require_mapping(cfg.get("training"), where="training") + + pipeline_cfg_raw = cfg.get("pipeline_config", None) + pipeline_cfg_path = cfg.get("pipeline_config_path", None) + if pipeline_cfg_raw is not None and pipeline_cfg_path is not None: + raise ValueError("Provide either pipeline_config or pipeline_config_path, not both") + + training_kwargs: dict[str, Any] = dict(training_raw) + + # Entrypoint invariants. + training_kwargs["mode"] = ExecutionMode.DISTILLATION + training_kwargs["inference_mode"] = False + + # Use student path as the default base model_path. This is needed for + # PipelineConfig registry lookup. + if "model_path" not in training_kwargs: + student = roles.get("student") + if student is None: + raise ValueError("training.model_path is missing and models.student is not provided") + training_kwargs["model_path"] = student.path + + if "pretrained_model_name_or_path" not in training_kwargs: + training_kwargs["pretrained_model_name_or_path"] = training_kwargs["model_path"] + + if pipeline_cfg_path is not None: + pipeline_cfg_path = _require_str(pipeline_cfg_path, where="pipeline_config_path") + training_kwargs["pipeline_config"] = resolve_outside_overlay(pipeline_cfg_path) + elif pipeline_cfg_raw is not None: + if isinstance(pipeline_cfg_raw, str): + training_kwargs["pipeline_config"] = resolve_outside_overlay(pipeline_cfg_raw) + elif isinstance(pipeline_cfg_raw, dict): + training_kwargs["pipeline_config"] = pipeline_cfg_raw + else: + raise ValueError("pipeline_config must be a mapping or a path string") + + training_args = TrainingArgs.from_kwargs(**training_kwargs) + + return DistillRunConfig( + distill=distill, + roles=roles, + training_args=training_args, + raw=cfg, + ) + From f8029ad6a96ffd8b1f15cbc6f3174ecc20d990c8 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 01:55:26 +0000 Subject: [PATCH 060/214] progressing phase 2. 2 --- fastvideo/distillation/adapters/wan.py | 7 + fastvideo/distillation/builder.py | 266 +++++++++++++++++++++++++ fastvideo/distillation/yaml_config.py | 5 +- fastvideo/training/distillation.py | 76 +++++-- 4 files changed, 332 insertions(+), 22 deletions(-) diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 730b8035f..021e72285 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -55,12 +55,14 @@ def __init__( noise_scheduler: Any, vae: Any, validation_pipeline: DistillationPipeline | None = None, + validator: Any | None = None, ) -> None: self.bundle = bundle self.training_args = training_args self.noise_scheduler = noise_scheduler self.vae = vae self._validation_pipeline_owner = validation_pipeline + self._validator = validator self.world_group = get_world_group() self.sp_group = get_sp_group() @@ -229,6 +231,11 @@ def ensure_negative_conditioning(self) -> None: self.negative_prompt_attention_mask = neg_mask def log_validation(self, iteration: int) -> None: + validator = getattr(self, "_validator", None) + if validator is not None: + validator.log_validation(iteration) + return + pipeline = self._validation_pipeline_owner if pipeline is None: return diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index 05f6c466a..c03461910 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -2,10 +2,276 @@ from __future__ import annotations +import os +from dataclasses import dataclass +from typing import Any + +import torch + from fastvideo.distillation.adapters.wan import WanAdapter from fastvideo.distillation.bundle import ModelBundle, RoleHandle from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method +from fastvideo.distillation.yaml_config import DistillRunConfig +from fastvideo.models.loader.component_loader import PipelineComponentLoader +from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from fastvideo.training.activation_checkpoint import ( + apply_activation_checkpointing, +) +from fastvideo.training.trackers import initialize_trackers, Trackers +from fastvideo.training.training_utils import get_scheduler from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline +from fastvideo.utils import maybe_download_model, verify_model_config_and_directory +from fastvideo.distributed import get_world_group + + +@dataclass(slots=True) +class DistillRuntime: + training_args: Any + method: DMD2Method + dataloader: Any + tracker: Any + start_step: int = 0 + + +def _parse_betas(betas: str) -> tuple[float, float]: + beta1, beta2 = (float(x.strip()) for x in betas.split(",")) + return beta1, beta2 + + +def _load_module_from_path( + *, + model_path: str, + module_type: str, + training_args: Any, + mark_teacher_critic: bool = False, +) -> torch.nn.Module: + local_model_path = maybe_download_model(model_path) + config = verify_model_config_and_directory(local_model_path) + + if module_type not in config: + raise ValueError(f"Module {module_type!r} not found in config at {local_model_path}") + + module_info = config[module_type] + if module_info is None: + raise ValueError(f"Module {module_type!r} has null value in config at {local_model_path}") + + transformers_or_diffusers, _architecture = module_info + component_path = os.path.join(local_model_path, module_type) + + if mark_teacher_critic: + setattr(training_args, "_loading_teacher_critic_model", True) + try: + module = PipelineComponentLoader.load_module( + module_name=module_type, + component_model_path=component_path, + transformers_or_diffusers=transformers_or_diffusers, + fastvideo_args=training_args, + ) + finally: + if mark_teacher_critic and hasattr(training_args, "_loading_teacher_critic_model"): + delattr(training_args, "_loading_teacher_critic_model") + + if not isinstance(module, torch.nn.Module): + raise TypeError(f"Loaded {module_type!r} is not a torch.nn.Module: {type(module)}") + return module + + +def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Module: + module.requires_grad_(trainable) + if trainable: + module.train() + else: + module.eval() + return module + + +def _build_optimizer_and_scheduler( + *, + role: str, + role_modules: dict[str, torch.nn.Module], + training_args: Any, +) -> tuple[dict[str, torch.optim.Optimizer], dict[str, Any]]: + if role == "critic": + lr = float(getattr(training_args, "fake_score_learning_rate", 0.0) or 0.0) + if lr == 0.0: + lr = float(training_args.learning_rate) + betas = _parse_betas(str(getattr(training_args, "fake_score_betas", training_args.betas))) + scheduler_name = str(getattr(training_args, "fake_score_lr_scheduler", training_args.lr_scheduler)) + else: + lr = float(training_args.learning_rate) + betas = _parse_betas(str(training_args.betas)) + scheduler_name = str(training_args.lr_scheduler) + + params = [] + for module in role_modules.values(): + params.extend([p for p in module.parameters() if p.requires_grad]) + + if not params: + raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") + + optimizer = torch.optim.AdamW( + params, + lr=lr, + betas=betas, + weight_decay=float(getattr(training_args, "weight_decay", 0.0) or 0.0), + eps=1e-8, + ) + + scheduler = get_scheduler( + scheduler_name, + optimizer=optimizer, + num_warmup_steps=int(getattr(training_args, "lr_warmup_steps", 0) or 0), + num_training_steps=int(getattr(training_args, "max_train_steps", 0) or 0), + num_cycles=int(getattr(training_args, "lr_num_cycles", 0) or 0), + power=float(getattr(training_args, "lr_power", 0.0) or 0.0), + min_lr_ratio=float(getattr(training_args, "min_lr_ratio", 0.5) or 0.5), + last_epoch=-1, + ) + + return {"main": optimizer}, {"main": scheduler} + + +def _build_tracker(training_args: Any, *, config: dict[str, Any] | None) -> Any: + world_group = get_world_group() + trackers = list(getattr(training_args, "trackers", [])) + if not trackers and getattr(training_args, "tracker_project_name", ""): + trackers.append(Trackers.WANDB.value) + if world_group.rank != 0: + trackers = [] + + tracker_log_dir = getattr(training_args, "output_dir", "") or os.getcwd() + if trackers: + tracker_log_dir = os.path.join(tracker_log_dir, "tracker") + + tracker_config = config if trackers else None + tracker_run_name = getattr(training_args, "wandb_run_name", "") or None + project = getattr(training_args, "tracker_project_name", "") or "fastvideo" + return initialize_trackers( + trackers, + experiment_name=project, + config=tracker_config, + log_dir=tracker_log_dir, + run_name=tracker_run_name, + ) + + +def build_wan_dmd2_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: + training_args = cfg.training_args + roles_cfg = cfg.roles + + if getattr(training_args, "seed", None) is None: + raise ValueError("training.seed must be set for distillation") + if not getattr(training_args, "data_path", ""): + raise ValueError("training.data_path must be set for distillation") + + for required in ("student", "teacher", "critic"): + if required not in roles_cfg: + raise ValueError(f"Missing required role {required!r} for wan+dmd2") + + # Load shared components (student base path). + training_args.override_transformer_cls_name = "WanTransformer3DModel" + vae = _load_module_from_path( + model_path=str(training_args.model_path), + module_type="vae", + training_args=training_args, + ) + noise_scheduler = FlowMatchEulerDiscreteScheduler( + shift=float(training_args.pipeline_config.flow_shift or 0.0) + ) + + role_handles: dict[str, RoleHandle] = {} + for role, role_spec in roles_cfg.items(): + if role_spec.family != "wan": + raise ValueError( + f"Phase 2 builder currently supports only wan roles; got {role}={role_spec.family!r}" + ) + + mark_teacher_critic = role in ("teacher", "critic") + transformer = _load_module_from_path( + model_path=role_spec.path, + module_type="transformer", + training_args=training_args, + mark_teacher_critic=mark_teacher_critic, + ) + modules: dict[str, torch.nn.Module] = {"transformer": transformer} + + # Optional MoE support: allow teacher transformer_2 if present. + if role == "teacher": + try: + transformer_2 = _load_module_from_path( + model_path=role_spec.path, + module_type="transformer_2", + training_args=training_args, + mark_teacher_critic=mark_teacher_critic, + ) + except Exception: + transformer_2 = None + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + + for name, module in list(modules.items()): + module = _apply_trainable(module, trainable=bool(role_spec.trainable)) + if role_spec.trainable and getattr(training_args, "enable_gradient_checkpointing_type", None): + module = apply_activation_checkpointing( + module, + checkpointing_type=training_args.enable_gradient_checkpointing_type, + ) + modules[name] = module + + optimizers: dict[str, torch.optim.Optimizer] = {} + lr_schedulers: dict[str, Any] = {} + if role_spec.trainable: + optimizers, lr_schedulers = _build_optimizer_and_scheduler( + role=role, + role_modules=modules, + training_args=training_args, + ) + + role_handles[role] = RoleHandle( + modules=modules, + optimizers=optimizers, + lr_schedulers=lr_schedulers, + trainable=bool(role_spec.trainable), + ) + + bundle = ModelBundle(roles=role_handles) + + adapter = WanAdapter( + bundle=bundle, + training_args=training_args, + noise_scheduler=noise_scheduler, + vae=vae, + validation_pipeline=None, + ) + + method = DMD2Method(bundle=bundle, adapter=adapter) + + from fastvideo.dataset import build_parquet_map_style_dataloader + from fastvideo.dataset.dataloader.schema import pyarrow_schema_t2v + + text_len = training_args.pipeline_config.text_encoder_configs[0].arch_config.text_len # type: ignore[attr-defined] + _dataset, dataloader = build_parquet_map_style_dataloader( + training_args.data_path, + training_args.train_batch_size, + num_data_workers=training_args.dataloader_num_workers, + parquet_schema=pyarrow_schema_t2v, + cfg_rate=training_args.training_cfg_rate, + drop_last=True, + text_padding_length=int(text_len), + seed=int(training_args.seed or 0), + ) + + tracker = _build_tracker(training_args, config=cfg.raw) + + return DistillRuntime( + training_args=training_args, + method=method, + dataloader=dataloader, + tracker=tracker, + start_step=0, + ) def build_wan_dmd2_method( diff --git a/fastvideo/distillation/yaml_config.py b/fastvideo/distillation/yaml_config.py index 44a2dd563..1d46772d3 100644 --- a/fastvideo/distillation/yaml_config.py +++ b/fastvideo/distillation/yaml_config.py @@ -142,6 +142,10 @@ def load_distill_run_config(path: str) -> DistillRunConfig: # Entrypoint invariants. training_kwargs["mode"] = ExecutionMode.DISTILLATION training_kwargs["inference_mode"] = False + # Match the training-mode loader behavior in `ComposedPipelineBase`: + # training uses fp32 master weights and should not CPU-offload DiT weights. + training_kwargs.setdefault("dit_precision", "fp32") + training_kwargs["dit_cpu_offload"] = False # Use student path as the default base model_path. This is needed for # PipelineConfig registry lookup. @@ -173,4 +177,3 @@ def load_distill_run_config(path: str) -> DistillRunConfig: training_args=training_args, raw=cfg, ) - diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 48382a62b..58380963c 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -5,11 +5,7 @@ import sys from typing import Any, Callable -from fastvideo.distillation import DistillTrainer -from fastvideo.distillation.builder import build_wan_dmd2_method -from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs from fastvideo.logger import init_logger -from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline from fastvideo.utils import FlexibleArgumentParser logger = init_logger(__name__) @@ -19,6 +15,8 @@ def _build_pipeline_factories() -> dict[str, _PipelineFactory]: + from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline + return { "wan": lambda args: WanDistillationPipeline.from_pretrained( @@ -29,6 +27,8 @@ def _build_pipeline_factories() -> dict[str, _PipelineFactory]: def _build_method_builders() -> dict[tuple[str, str], _MethodBuilder]: + from fastvideo.distillation.builder import build_wan_dmd2_method + return { ("wan", "dmd2"): build_wan_dmd2_method, } @@ -40,6 +40,10 @@ def run_distillation( distill_model: str, distill_method: str, ) -> None: + """Legacy Phase 1 entrypoint (CLI-driven, uses legacy pipelines).""" + + from fastvideo.distillation import DistillTrainer + pipeline_factories = _build_pipeline_factories() method_builders = _build_method_builders() @@ -70,15 +74,48 @@ def run_distillation( ) -def main(args: Any) -> None: - distill_model = str(getattr(args, "distill_model")) - distill_method = str(getattr(args, "distill_method")) - logger.info( - "Starting distillation: distill_model=%s, distill_method=%s", - distill_model, - distill_method, +def run_distillation_from_config(config_path: str, *, dry_run: bool = False) -> None: + """Phase 2 entrypoint (YAML-only, standalone runtime builder).""" + + from fastvideo.distributed import maybe_init_distributed_environment_and_model_parallel + from fastvideo.distillation import DistillTrainer + from fastvideo.distillation.builder import build_wan_dmd2_runtime_from_config + from fastvideo.distillation.yaml_config import load_distill_run_config + + cfg = load_distill_run_config(config_path) + training_args = cfg.training_args + + maybe_init_distributed_environment_and_model_parallel( + training_args.tp_size, + training_args.sp_size, + ) + + if cfg.distill.model == "wan" and cfg.distill.method == "dmd2": + runtime = build_wan_dmd2_runtime_from_config(cfg) + else: + raise ValueError( + f"Unsupported distillation config: model={cfg.distill.model!r}, " + f"method={cfg.distill.method!r}" + ) + + if dry_run: + logger.info("Dry-run: config parsed and runtime built successfully.") + return + + trainer = DistillTrainer(training_args, tracker=runtime.tracker) + trainer.run( + runtime.method, + dataloader=runtime.dataloader, + max_steps=training_args.max_train_steps, + start_step=runtime.start_step, ) - run_distillation(args, distill_model=distill_model, distill_method=distill_method) + + +def main(args: Any) -> None: + config_path = str(getattr(args, "config")) + dry_run = bool(getattr(args, "dry_run", False)) + logger.info("Starting Phase 2 distillation from config=%s", config_path) + run_distillation_from_config(config_path, dry_run=dry_run) logger.info("Distillation completed") @@ -86,18 +123,15 @@ def main(args: Any) -> None: argv = sys.argv parser = FlexibleArgumentParser() parser.add_argument( - "--distill-model", + "--config", type=str, - default="wan", - help="Distillation model family (Phase 1 supports: wan).", + required=True, + help="Path to distillation YAML config (Phase 2 entrypoint).", ) parser.add_argument( - "--distill-method", - type=str, - default="dmd2", - help="Distillation method (Phase 1 supports: dmd2).", + "--dry-run", + action="store_true", + help="Parse config and build runtime, but do not start training.", ) - parser = TrainingArgs.add_cli_args(parser) - parser = FastVideoArgs.add_cli_args(parser) args = parser.parse_args(argv[1:]) main(args) From cef57efc97ffc9f6607506a08c2f7e3e96884143 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 02:06:26 +0000 Subject: [PATCH 061/214] phase 2 init impl --- dev/phases/phase_2.md | 2 +- examples/distillation/phase2/README.md | 14 ++ .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 78 ++++++ .../phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh | 21 ++ fastvideo/distillation/builder.py | 19 +- .../methods/distribution_matching/dmd2.py | 2 + fastvideo/distillation/validators/__init__.py | 9 + fastvideo/distillation/validators/base.py | 11 + fastvideo/distillation/validators/wan.py | 231 ++++++++++++++++++ fastvideo/distillation/yaml_config.py | 12 +- fastvideo/training/distillation.py | 7 +- 11 files changed, 397 insertions(+), 9 deletions(-) create mode 100644 examples/distillation/phase2/README.md create mode 100644 examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml create mode 100644 examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh create mode 100644 fastvideo/distillation/validators/__init__.py create mode 100644 fastvideo/distillation/validators/base.py create mode 100644 fastvideo/distillation/validators/wan.py diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md index 9bb7623d4..b376707a0 100644 --- a/dev/phases/phase_2.md +++ b/dev/phases/phase_2.md @@ -128,7 +128,7 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 建议落地： -- 新增目录：`fastvideo/distillation/validation/` +- 新增目录：`fastvideo/distillation/validators/` - `base.py`：`DistillValidator` 抽象 - `wan.py`：`WanValidator` diff --git a/examples/distillation/phase2/README.md b/examples/distillation/phase2/README.md new file mode 100644 index 000000000..9edcc0d76 --- /dev/null +++ b/examples/distillation/phase2/README.md @@ -0,0 +1,14 @@ +# Phase 2 examples + +This folder contains **YAML-only** distillation examples for the Phase 2 +entrypoint: + +- `fastvideo/training/distillation.py --config path/to/distill.yaml` + +Start from: + +- `distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` + +and edit `training.data_path` + `training.validation_dataset_file` before +running. + diff --git a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml new file mode 100644 index 000000000..c7dbecb77 --- /dev/null +++ b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -0,0 +1,78 @@ +distill: + model: wan + method: dmd2 + +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # Distributed + num_gpus: 1 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 1 + + # Data + data_path: your_data_dir + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 21 + num_height: 480 + num_width: 832 + num_frames: 81 + + # Output / steps + output_dir: outputs/phase2_wan2.1_t2v_1.3B_dmd2_8steps + max_train_steps: 4000 + seed: 1000 + + # Optimizer + learning_rate: 1.0e-5 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + simulate_generator_forward: true + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: phase2_wan_dmd2_8steps + wandb_run_name: phase2_wan_dmd2_8steps + log_validation: true + validation_dataset_file: your_validation_dataset_file + validation_steps: 50 + validation_sampling_steps: "8" + validation_guidance_scale: "6.0" + +pipeline_config: + flow_shift: 8 + # 8-step schedule (same as Wan2.2 self-forcing examples) + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + diff --git a/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh new file mode 100644 index 000000000..ec32bf1ce --- /dev/null +++ b/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -euo pipefail +set -x + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_PORT=${MASTER_PORT:-29506} + +NUM_GPUS=${NUM_GPUS:-1} +CONFIG=${CONFIG:-"examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} + +torchrun \ + --nnodes 1 \ + --master_port "$MASTER_PORT" \ + --nproc_per_node "$NUM_GPUS" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" + diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index c03461910..f019f9d94 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -61,7 +61,7 @@ def _load_module_from_path( component_path = os.path.join(local_model_path, module_type) if mark_teacher_critic: - setattr(training_args, "_loading_teacher_critic_model", True) + training_args._loading_teacher_critic_model = True try: module = PipelineComponentLoader.load_module( module_name=module_type, @@ -71,7 +71,7 @@ def _load_module_from_path( ) finally: if mark_teacher_critic and hasattr(training_args, "_loading_teacher_critic_model"): - delattr(training_args, "_loading_teacher_critic_model") + del training_args._loading_teacher_critic_model if not isinstance(module, torch.nn.Module): raise TypeError(f"Loaded {module_type!r} is not a torch.nn.Module: {type(module)}") @@ -238,12 +238,25 @@ def build_wan_dmd2_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: bundle = ModelBundle(roles=role_handles) + tracker = _build_tracker(training_args, config=cfg.raw) + + validator = None + if getattr(training_args, "log_validation", False): + from fastvideo.distillation.validators.wan import WanValidator + + validator = WanValidator( + bundle=bundle, + training_args=training_args, + tracker=tracker, + ) + adapter = WanAdapter( bundle=bundle, training_args=training_args, noise_scheduler=noise_scheduler, vae=vae, validation_pipeline=None, + validator=validator, ) method = DMD2Method(bundle=bundle, adapter=adapter) @@ -263,8 +276,6 @@ def build_wan_dmd2_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: seed=int(training_args.seed or 0), ) - tracker = _build_tracker(training_args, config=cfg.raw) - return DistillRuntime( training_args=training_args, method=method, diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index db66c9858..19d80cf17 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -108,6 +108,8 @@ def __init__( ) -> None: super().__init__(bundle) bundle.require_roles(["student", "teacher", "critic"]) + if getattr(bundle.role("teacher"), "trainable", False): + raise ValueError("DMD2Method requires models.teacher.trainable=false") self.adapter = adapter self.training_args = adapter.training_args diff --git a/fastvideo/distillation/validators/__init__.py b/fastvideo/distillation/validators/__init__.py new file mode 100644 index 000000000..571c5aa2f --- /dev/null +++ b/fastvideo/distillation/validators/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 + +from fastvideo.distillation.validators.base import DistillValidator +from fastvideo.distillation.validators.wan import WanValidator + +__all__ = [ + "DistillValidator", + "WanValidator", +] diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py new file mode 100644 index 000000000..cb732ee07 --- /dev/null +++ b/fastvideo/distillation/validators/base.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from abc import ABC, abstractmethod + + +class DistillValidator(ABC): + @abstractmethod + def log_validation(self, step: int) -> None: + raise NotImplementedError diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py new file mode 100644 index 000000000..25257435a --- /dev/null +++ b/fastvideo/distillation/validators/wan.py @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import copy +import os +from dataclasses import dataclass +from typing import Any + +import imageio +import numpy as np +import torch +import torchvision +from einops import rearrange +from torch.utils.data import DataLoader + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.validation_dataset import ValidationDataset +from fastvideo.distributed import get_sp_group, get_world_group +from fastvideo.logger import init_logger +from fastvideo.pipelines import ForwardBatch +from fastvideo.pipelines.basic.wan.wan_dmd_pipeline import WanDMDPipeline +from fastvideo.utils import shallow_asdict + +logger = init_logger(__name__) + + +@dataclass(slots=True) +class _ValidationStepResult: + videos: list[list[np.ndarray]] + captions: list[str] + + +class WanValidator: + """Phase 2 standalone validator for Wan distillation.""" + + def __init__( + self, + *, + bundle: Any, + training_args: Any, + tracker: Any, + ) -> None: + self.bundle = bundle + self.training_args = training_args + self.tracker = tracker + + self.world_group = get_world_group() + self.sp_group = get_sp_group() + self.global_rank = self.world_group.rank + self.rank_in_sp_group = self.sp_group.rank_in_group + self.sp_world_size = self.sp_group.world_size + + seed = getattr(training_args, "seed", None) + if seed is None: + raise ValueError("training_args.seed must be set for validation") + self.seed = int(seed) + self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) + + self._pipeline: WanDMDPipeline | None = None + self._sampling_param: SamplingParam | None = None + + def _get_sampling_param(self) -> SamplingParam: + if self._sampling_param is None: + self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) + return self._sampling_param + + def _get_pipeline(self) -> WanDMDPipeline: + if self._pipeline is not None: + return self._pipeline + + args_copy = copy.deepcopy(self.training_args) + args_copy.inference_mode = True + + student_transformer = self.bundle.role("student").require_module("transformer") + self._pipeline = WanDMDPipeline.from_pretrained( + self.training_args.model_path, + args=args_copy, # ignored in inference branch but keeps parity + inference_mode=True, + loaded_modules={"transformer": student_transformer}, + tp_size=self.training_args.tp_size, + sp_size=self.training_args.sp_size, + num_gpus=self.training_args.num_gpus, + pin_cpu_memory=self.training_args.pin_cpu_memory, + dit_cpu_offload=True, + ) + return self._pipeline + + def _parse_validation_steps(self) -> list[int]: + raw = str(getattr(self.training_args, "validation_sampling_steps", "") or "") + steps = [int(s) for s in raw.split(",") if s.strip()] + return [s for s in steps if s > 0] + + def _prepare_validation_batch( + self, + sampling_param: SamplingParam, + validation_batch: dict[str, Any], + num_inference_steps: int, + ) -> ForwardBatch: + sampling_param.prompt = validation_batch["prompt"] + sampling_param.height = self.training_args.num_height + sampling_param.width = self.training_args.num_width + sampling_param.num_inference_steps = num_inference_steps + sampling_param.data_type = "video" + if getattr(self.training_args, "validation_guidance_scale", ""): + sampling_param.guidance_scale = float(self.training_args.validation_guidance_scale) + sampling_param.seed = self.seed + + latents_size = [ + (sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, + sampling_param.width // 8, + ] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + + temporal_compression_factor = ( + self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_factor + 1 + sampling_param.num_frames = int(num_frames) + + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=self.validation_random_generator, + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=self.training_args.VSA_sparsity, + ) + return batch + + def _run_validation_for_steps(self, num_inference_steps: int) -> _ValidationStepResult: + training_args = self.training_args + pipeline = self._get_pipeline() + sampling_param = self._get_sampling_param() + + dataset = ValidationDataset(training_args.validation_dataset_file) + dataloader = DataLoader(dataset, batch_size=None, num_workers=0) + + videos: list[list[np.ndarray]] = [] + captions: list[str] = [] + + for validation_batch in dataloader: + batch = self._prepare_validation_batch(sampling_param, validation_batch, num_inference_steps) + + assert batch.prompt is not None and isinstance(batch.prompt, str) + captions.append(batch.prompt) + + with torch.no_grad(): + output_batch = pipeline.forward(batch, training_args) + + samples = output_batch.output.cpu() + if self.rank_in_sp_group != 0: + continue + + video = rearrange(samples, "b c t h w -> t b c h w") + frames: list[np.ndarray] = [] + for x in video: + x = torchvision.utils.make_grid(x, nrow=6) + x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) + frames.append((x * 255).numpy().astype(np.uint8)) + videos.append(frames) + + return _ValidationStepResult(videos=videos, captions=captions) + + def log_validation(self, step: int) -> None: + training_args = self.training_args + if not getattr(training_args, "log_validation", False): + return + if not getattr(training_args, "validation_dataset_file", ""): + raise ValueError("validation_dataset_file must be set when log_validation is enabled") + + validation_steps = self._parse_validation_steps() + if not validation_steps: + return + + student_transformer = self.bundle.role("student").require_module("transformer") + was_training = bool(getattr(student_transformer, "training", False)) + + old_inference_mode = training_args.inference_mode + old_dit_cpu_offload = training_args.dit_cpu_offload + try: + training_args.inference_mode = True + training_args.dit_cpu_offload = True + student_transformer.eval() + + num_sp_groups = self.world_group.world_size // self.sp_group.world_size + + for num_inference_steps in validation_steps: + result = self._run_validation_for_steps(num_inference_steps) + + if self.rank_in_sp_group != 0: + continue + + if self.global_rank == 0: + all_videos = list(result.videos) + all_captions = list(result.captions) + for sp_group_idx in range(1, num_sp_groups): + src_rank = sp_group_idx * self.sp_world_size + recv_videos = self.world_group.recv_object(src=src_rank) + recv_captions = self.world_group.recv_object(src=src_rank) + all_videos.extend(recv_videos) + all_captions.extend(recv_captions) + + os.makedirs(training_args.output_dir, exist_ok=True) + video_filenames: list[str] = [] + sampling_param = self._get_sampling_param() + for i, video in enumerate(all_videos): + filename = os.path.join( + training_args.output_dir, + f"validation_step_{step}_inference_steps_{num_inference_steps}_video_{i}.mp4", + ) + imageio.mimsave(filename, video, fps=sampling_param.fps) + video_filenames.append(filename) + + artifacts = [] + for filename, caption in zip(video_filenames, all_captions, strict=True): + video_artifact = self.tracker.video(filename, caption=caption) + if video_artifact is not None: + artifacts.append(video_artifact) + if artifacts: + logs = {f"validation_videos_{num_inference_steps}_steps": artifacts} + self.tracker.log_artifacts(logs, step) + else: + self.world_group.send_object(result.videos, dst=0) + self.world_group.send_object(result.captions, dst=0) + finally: + training_args.inference_mode = old_inference_mode + training_args.dit_cpu_offload = old_dit_cpu_offload + if was_training: + student_transformer.train() diff --git a/fastvideo/distillation/yaml_config.py b/fastvideo/distillation/yaml_config.py index 1d46772d3..5737f60fb 100644 --- a/fastvideo/distillation/yaml_config.py +++ b/fastvideo/distillation/yaml_config.py @@ -106,7 +106,7 @@ def load_distill_run_config(path: str) -> DistillRunConfig: """ path = resolve_outside_overlay(path) - with open(path, "r", encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: raw = yaml.safe_load(f) cfg = _require_mapping(raw, where=path) @@ -147,6 +147,16 @@ def load_distill_run_config(path: str) -> DistillRunConfig: training_kwargs.setdefault("dit_precision", "fp32") training_kwargs["dit_cpu_offload"] = False + # Default distributed sizes. These must be set *before* TrainingArgs + # construction because `check_fastvideo_args()` asserts they are not -1 in + # training mode. + num_gpus = int(training_kwargs.get("num_gpus", 1) or 1) + training_kwargs.setdefault("num_gpus", num_gpus) + training_kwargs.setdefault("tp_size", 1) + training_kwargs.setdefault("sp_size", num_gpus) + training_kwargs.setdefault("hsdp_replicate_dim", 1) + training_kwargs.setdefault("hsdp_shard_dim", num_gpus) + # Use student path as the default base model_path. This is needed for # PipelineConfig registry lookup. if "model_path" not in training_kwargs: diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 58380963c..e20ea3699 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -3,7 +3,8 @@ from __future__ import annotations import sys -from typing import Any, Callable +from collections.abc import Callable +from typing import Any from fastvideo.logger import init_logger from fastvideo.utils import FlexibleArgumentParser @@ -112,8 +113,8 @@ def run_distillation_from_config(config_path: str, *, dry_run: bool = False) -> def main(args: Any) -> None: - config_path = str(getattr(args, "config")) - dry_run = bool(getattr(args, "dry_run", False)) + config_path = str(args.config) + dry_run = bool(args.dry_run) logger.info("Starting Phase 2 distillation from config=%s", config_path) run_distillation_from_config(config_path, dry_run=dry_run) logger.info("Distillation completed") From 7f14865ba37affa2835e00e2c514cefc1fa0af12 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 05:50:59 +0000 Subject: [PATCH 062/214] phase 2 config. training code --- dev/phases/phase_2.md | 71 ++++++++--------- examples/distillation/phase2/README.md | 11 ++- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 33 +++++--- .../phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh | 13 ++- examples/distillation/phase2/temp.sh | 49 ++++++++++++ fastvideo/distillation/outside/README.md | 13 +-- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 79 +++++++++++++++++++ fastvideo/distillation/yaml_config.py | 56 ++++--------- 8 files changed, 222 insertions(+), 103 deletions(-) create mode 100644 examples/distillation/phase2/temp.sh create mode 100644 fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md index b376707a0..2204f1824 100644 --- a/dev/phases/phase_2.md +++ b/dev/phases/phase_2.md @@ -38,15 +38,15 @@ Phase 2 的定位：在 Phase 1 已经验证 “Wan DMD2 训练行为对齐 base ## 当前“仍依赖 legacy”的点（Phase 2 要消灭） -以 Phase 1 的 Wan DMD2 为例，当前仍有两类耦合： +以 Wan DMD2 为例，我们现在同时存在两条路径： -1. **启动/加载耦合**： - - `fastvideo/training/distillation.py` 仍通过 `WanDistillationPipeline.from_pretrained(...)` - 完成模型加载、optimizer/scheduler、dataloader、tracker。 -2. **validation 耦合**： - - `WanAdapter.log_validation()` 仍复用 legacy `pipeline._log_validation(...)` +1. **Phase 1（legacy pipeline path）**：仍然依赖 `WanDistillationPipeline.from_pretrained(...)`， + validation 也会走 legacy `_log_validation(...)`（因为 Phase 1 需要保持与 baseline 的对齐与可用性）。 +2. **Phase 2（standalone runtime path）**：已替换为 **YAML-only + builder/runtime + standalone validator**， + 不再依赖 legacy pipeline。 -Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 +Phase 2 的 deliverable 是让 Phase 2 路径完全自洽（训练/validation/checkpoint-resume），同时 Phase 1 +路径仍可跑（兼容历史脚本）。 --- @@ -54,40 +54,38 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 ### A. Validation 独立化（Phase 2.1） -- [ ] 定义通用接口：`DistillValidator` - - 输入：`bundle + adapter + training_args + tracker` - - 输出：`log_validation(step)`（只在 rank0 真正写 artifact） -- [ ] 实现 `WanValidator`（Wan + T2V 的最小版本） - - 复用 `fastvideo/dataset/validation_dataset.py::ValidationDataset` - - 复用模块化 **inference pipeline**（建议：`fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py::WanDMDPipeline`） - - 支持关键参数： - - `validation_sampling_steps`（few-step） - - `validation_guidance_scale` - - seed / RNG（不依赖 legacy pipeline 初始化副作用） -- [ ] 将 Phase 1 的 `WanAdapter.log_validation()` 切到新 validator（不再依赖 legacy pipeline） +- [x] 定义通用接口：`fastvideo/distillation/validators/base.py::DistillValidator` + - 入口：`log_validation(step)` + - 行为：rank0 写 artifacts（其余 rank 走 gather/send） +- [x] 实现 `fastvideo/distillation/validators/wan.py::WanValidator`（Wan + T2V 最小版本） + - 复用 `ValidationDataset` + - 使用模块化 inference pipeline：`WanDMDPipeline` + - 支持 few-step：`validation_sampling_steps` + `validation_guidance_scale` + seed/RNG（不依赖 legacy pipeline） +- [x] `fastvideo/distillation/adapters/wan.py::WanAdapter.log_validation()` 支持注入 validator + - Phase 2 路径：走新 validator + - Phase 1 路径：未提供 validator 时，仍可回退到 legacy pipeline（保留兼容） ### B. Builder/Runtime 脱离 pipeline（Phase 2.2） -- [ ] 定义结构化 spec（角色驱动）：`ModelSpec / RoleSpec / DistillSpec` +- [x] 定义结构化 spec（角色驱动）：`DistillSpec / RoleSpec` - 目标：`models={role -> spec}` 成为唯一真相 - method 自己声明需要哪些 roles（缺失则报错） -- [ ] 新增 YAML 配置解析（Phase 2 必做） - - [ ] `fastvideo/training/distillation.py` 支持 `--config path/to/distill.yaml` - - [ ] 解析策略：`yaml.safe_load` + schema 校验（不做 legacy CLI merge） - - [ ] YAML schema 最小包含：`distill.{model,method}` + `models{role->spec}` + `training{...}` + `pipeline_config{...}` -- [ ] 支持 `outside/` overlay（Phase 2 workaround） - - [ ] 新增目录：`fastvideo/distillation/outside/`（视作外部 repo root） - - [ ] 约定覆盖路径：`fastvideo/distillation/outside/fastvideo/configs/...` - - [ ] config loader 在解析 `pipeline_config_path`/yaml include 时：outside 优先、repo fallback - - [ ] 不通过 `sys.path` shadow `fastvideo` 包；outside 仅做文件 overlay（必要时用 `importlib` 按路径加载 .py config） -- [ ] 实现 “standalone runtime builder” +- [x] 新增 YAML 配置解析：`fastvideo/distillation/yaml_config.py::load_distill_run_config` + - `yaml.safe_load` + 最小 schema 校验（不做 legacy CLI merge） + - schema：`distill + models + training + (pipeline_config|pipeline_config_path)` +- [x] 修改入口：`fastvideo/training/distillation.py` + - `--config` 为必需参数：Phase 2 路径 **YAML-only** + - legacy distill 仍通过旧入口文件可跑（两套路径并存） +- [x] 支持 `outside/`（你拍板的 Phase 2 workaround） + - 新增目录：`fastvideo/distillation/outside/`（视作外部 repo root） + - 覆盖路径：`fastvideo/distillation/outside/` + - **无自动补全/overlay**：config loader 不做路径重写；运行时传入 outside YAML 的真实路径（无 fallback） +- [x] 实现 standalone runtime builder：`fastvideo/distillation/builder.py::build_wan_dmd2_runtime_from_config` - 直接加载 modules（student/teacher/critic）并构建 `ModelBundle` - - 构建 per-role optimizers/schedulers（复用现有 TrainingArgs 超参） - - 构建 dataloader（直接调用 `fastvideo/dataset/parquet_dataset_map_style.py::build_parquet_map_style_dataloader`） + - 构建 per-role optimizers/schedulers（复用 TrainingArgs 超参） + - 构建 dataloader（`build_parquet_map_style_dataloader`） - 初始化 tracker（复用 `fastvideo/training/trackers/`） -- [ ] 修改现有入口 `fastvideo/training/distillation.py`（不新增入口文件） - - `--config` 为必需参数：只跑 standalone builder 路径（YAML-only） - - legacy distill 通过旧入口文件继续可跑（不在该入口做 fallback） + - 通过 `WanAdapter(validator=...)` 接入独立 validation ### C. role-based checkpoint/save/resume（Phase 2.3） @@ -106,9 +104,8 @@ Phase 2 的 deliverable 就是把这两类耦合点都替换掉。 ### D. 示例脚本（Phase 2） -- [ ] `examples/distillation/phase2/`： - - 最小 smoke：Wan 1.3B 学 14B（DMD2）+ few-step validation + save/resume - - 提供 `distill.yaml` 模板（role-based） +- [x] 最小 smoke（训练 + few-step validation）：`examples/distillation/phase2/temp.sh` +- [ ] Save/Resume 示例：等 Phase 2.3 checkpoint manager 完成后再补 ### E. 最小单测（可选但建议） diff --git a/examples/distillation/phase2/README.md b/examples/distillation/phase2/README.md index 9edcc0d76..dd4984684 100644 --- a/examples/distillation/phase2/README.md +++ b/examples/distillation/phase2/README.md @@ -5,10 +5,17 @@ entrypoint: - `fastvideo/training/distillation.py --config path/to/distill.yaml` +Important: + +- Phase 2 does not rewrite config paths automatically. Pass the explicit YAML + path (we keep runnable YAMLs under `fastvideo/distillation/outside/`). + Start from: - `distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` -and edit `training.data_path` + `training.validation_dataset_file` before -running. +Recommended: +- Edit the runnable YAML in: + `fastvideo/distillation/outside/fastvideo/configs/distillation/` +- `temp.sh` (runs the config above; same dataset + validation defaults as Phase0/Phase1). diff --git a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index c7dbecb77..062a2fc45 100644 --- a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -1,3 +1,10 @@ +# NOTE: +# Phase 2 does not rewrite config paths automatically. Put runnable YAML under: +# fastvideo/distillation/outside/fastvideo/configs/distillation/ +# +# The runnable copy for this template lives at: +# fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +# distill: model: wan method: dmd2 @@ -24,26 +31,27 @@ training: hsdp_replicate_dim: 1 hsdp_shard_dim: 1 - # Data - data_path: your_data_dir + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k dataloader_num_workers: 4 - # Batch / shape + # Batch / shape (matches Wan-Syn 77x448x832) train_batch_size: 1 train_sp_batch_size: 1 gradient_accumulation_steps: 1 - num_latent_t: 21 - num_height: 480 + num_latent_t: 20 + num_height: 448 num_width: 832 - num_frames: 81 + num_frames: 77 # Output / steps - output_dir: outputs/phase2_wan2.1_t2v_1.3B_dmd2_8steps + output_dir: outputs/phase2_wan2.1_dmd2_8steps_wansyn max_train_steps: 4000 seed: 1000 # Optimizer - learning_rate: 1.0e-5 + learning_rate: 2.0e-6 + mixed_precision: bf16 betas: "0.0,0.999" weight_decay: 0.01 lr_scheduler: constant @@ -55,6 +63,7 @@ training: fake_score_lr_scheduler: constant # Distillation + training_cfg_rate: 0.0 min_timestep_ratio: 0.02 max_timestep_ratio: 0.98 generator_update_interval: 5 @@ -63,16 +72,14 @@ training: enable_gradient_checkpointing_type: full # Tracking / validation - tracker_project_name: phase2_wan_dmd2_8steps - wandb_run_name: phase2_wan_dmd2_8steps + tracker_project_name: phase2_wan_dmd2_8steps_wansyn + wandb_run_name: phase2_wan_dmd2_8steps_wansyn log_validation: true - validation_dataset_file: your_validation_dataset_file + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json validation_steps: 50 validation_sampling_steps: "8" validation_guidance_scale: "6.0" pipeline_config: flow_shift: 8 - # 8-step schedule (same as Wan2.2 self-forcing examples) dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] - diff --git a/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh index ec32bf1ce..78eb4494a 100644 --- a/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh +++ b/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh @@ -2,6 +2,10 @@ set -euo pipefail set -x +# NOTE: +# Phase 2 expects an explicit YAML path (we keep runnable YAML under outside/): +# fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml + export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} @@ -9,8 +13,12 @@ export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} export WANDB_MODE=${WANDB_MODE:-offline} export MASTER_PORT=${MASTER_PORT:-29506} -NUM_GPUS=${NUM_GPUS:-1} -CONFIG=${CONFIG:-"examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} +CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} +if [[ ! -f "$CONFIG" ]]; then + echo "Missing Phase 2 YAML config at: $CONFIG" >&2 + exit 1 +fi +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") torchrun \ --nnodes 1 \ @@ -18,4 +26,3 @@ torchrun \ --nproc_per_node "$NUM_GPUS" \ fastvideo/training/distillation.py \ --config "$CONFIG" - diff --git a/examples/distillation/phase2/temp.sh b/examples/distillation/phase2/temp.sh new file mode 100644 index 000000000..cc4ed0048 --- /dev/null +++ b/examples/distillation/phase2/temp.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -e -x + +# One-shot launch script for Phase 2 (YAML-only, standalone runtime) Wan DMD2 +# few-step distillation. +# +# Uses the same defaults as Phase0/Phase1 temp.sh: +# - parquet dataset folder: data/Wan-Syn_77x448x832_600k +# - validation json: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json +# +# Notes: +# - Phase 2 expects an explicit YAML path (we keep runnable YAML under outside/). +# Put your YAML under: +# fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml +# - By default this runs W&B in offline mode (safer for overnight runs). +# If you want online logging: +# export WANDB_MODE=online +# export WANDB_API_KEY=... + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29507} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing Phase 2 YAML config at: $CONFIG" >&2 + exit 1 +fi + +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" diff --git a/fastvideo/distillation/outside/README.md b/fastvideo/distillation/outside/README.md index 5855410f0..fbd6b0310 100644 --- a/fastvideo/distillation/outside/README.md +++ b/fastvideo/distillation/outside/README.md @@ -1,14 +1,15 @@ -# `outside/` overlay +# `outside/` (Phase 2 config root) This directory is a Phase 2 workaround for iterating on new YAML/JSON configs without modifying the main repository's `fastvideo/configs/` tree. -The distillation config loader resolves file paths via an overlay lookup: +Phase 2 does **not** rewrite config paths automatically. Put configs under this +tree and pass the real path to the entrypoint (e.g. `--config +fastvideo/distillation/outside/fastvideo/configs/distillation/foo.yaml`). -- if a run config references `fastvideo/configs/foo.json`, it will first check - `fastvideo/distillation/outside/fastvideo/configs/foo.json`; -- otherwise it falls back to the original path. +Recommended layout: + +- `fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml` Keep `outside/` for **data/config files only** (do not place importable Python code here). - diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml new file mode 100644 index 000000000..83ef73078 --- /dev/null +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -0,0 +1,79 @@ +distill: + model: wan + method: dmd2 + +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # Distributed + num_gpus: 1 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase2_wan2.1_dmd2_8steps_wansyn + max_train_steps: 4000 + seed: 1000 + + # Optimizer + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + simulate_generator_forward: true + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: phase2_wan_dmd2_8steps_wansyn + wandb_run_name: phase2_wan_dmd2_8steps_wansyn + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "8" + validation_guidance_scale: "6.0" + +pipeline_config: + flow_shift: 8 + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + diff --git a/fastvideo/distillation/yaml_config.py b/fastvideo/distillation/yaml_config.py index 5737f60fb..a2bdd1558 100644 --- a/fastvideo/distillation/yaml_config.py +++ b/fastvideo/distillation/yaml_config.py @@ -30,52 +30,24 @@ def _distillation_root() -> Path: return Path(__file__).resolve().parent -def _repo_root() -> Path: - # .../fastvideo/distillation -> .../ - return _distillation_root().parent.parent +def _resolve_existing_file(path: str) -> str: + """Resolve a user-provided config path and require it exists. - -def _outside_root() -> Path: - return _distillation_root() / "outside" - - -def resolve_outside_overlay(path: str) -> str: - """Resolve ``path`` via the distillation ``outside/`` overlay if present. - - The overlay root is ``fastvideo/distillation/outside/`` and mirrors the - repository layout. For example, if the run config references: - - fastvideo/configs/foo.json - - then we first check: - - fastvideo/distillation/outside/fastvideo/configs/foo.json - - and fall back to the original path when the overlay file does not exist. + Phase 2 intentionally does not perform any "overlay" path rewriting. The + caller must pass the real path (typically under + ``fastvideo/distillation/outside/``). """ if not path: return path expanded = os.path.expanduser(path) - candidate: Path | None = None - - p = Path(expanded) - if p.is_absolute(): - try: - rel = p.resolve().relative_to(_repo_root()) - except Exception: - rel = None - if rel is not None: - candidate = _outside_root() / rel - else: - candidate = _outside_root() / p - - if candidate is not None and candidate.exists(): - logger.info("Using outside overlay for %s -> %s", path, candidate) - return str(candidate) - - return expanded + resolved = Path(expanded).resolve() + if not resolved.exists(): + raise FileNotFoundError(f"Config file not found: {resolved}") + if not resolved.is_file(): + raise ValueError(f"Expected a file path, got: {resolved}") + return str(resolved) def _require_mapping(raw: Any, *, where: str) -> dict[str, Any]: @@ -105,7 +77,7 @@ def load_distill_run_config(path: str) -> DistillRunConfig: file is the single source of truth for a run. """ - path = resolve_outside_overlay(path) + path = _resolve_existing_file(path) with open(path, encoding="utf-8") as f: raw = yaml.safe_load(f) cfg = _require_mapping(raw, where=path) @@ -170,10 +142,10 @@ def load_distill_run_config(path: str) -> DistillRunConfig: if pipeline_cfg_path is not None: pipeline_cfg_path = _require_str(pipeline_cfg_path, where="pipeline_config_path") - training_kwargs["pipeline_config"] = resolve_outside_overlay(pipeline_cfg_path) + training_kwargs["pipeline_config"] = _resolve_existing_file(pipeline_cfg_path) elif pipeline_cfg_raw is not None: if isinstance(pipeline_cfg_raw, str): - training_kwargs["pipeline_config"] = resolve_outside_overlay(pipeline_cfg_raw) + training_kwargs["pipeline_config"] = _resolve_existing_file(pipeline_cfg_raw) elif isinstance(pipeline_cfg_raw, dict): training_kwargs["pipeline_config"] = pipeline_cfg_raw else: From 99acff3b69b3481367d99b9d610791ae231730f1 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 06:32:05 +0000 Subject: [PATCH 063/214] remove all legacy dependency --- dev/phases/phase_1.md | 10 +- dev/phases/phase_2.md | 21 ++- fastvideo/distillation/adapters/wan.py | 175 ++++++++-------------- fastvideo/distillation/builder.py | 37 ----- fastvideo/training/distillation.py | 73 +-------- fastvideo/training/wan_distillation_v3.py | 29 ---- 6 files changed, 89 insertions(+), 256 deletions(-) delete mode 100644 fastvideo/training/wan_distillation_v3.py diff --git a/dev/phases/phase_1.md b/dev/phases/phase_1.md index d36c1ed61..ee0c48d51 100644 --- a/dev/phases/phase_1.md +++ b/dev/phases/phase_1.md @@ -57,11 +57,11 @@ Phase 1 的定位：把 distillation 从 “pipeline god object” 拆成稳定 ### E. Builder + 通用入口（config -> instantiate） -- [x] 新增 `fastvideo/distillation/builder.py::build_wan_dmd2_method` - - Phase 1 先实现：`wan + dmd2` -- [x] 新增通用 distill 入口：`fastvideo/training/distillation.py` - - CLI：`--distill-model` / `--distill-method` -- [x] 保留一个 Wan wrapper：`fastvideo/training/wan_distillation_v3.py` +- [x] （Phase 1 过渡实现；Phase 2 已移除）`fastvideo/distillation/builder.py::build_wan_dmd2_method` + - 说明：Phase 1 用 legacy pipeline 启动，快速对齐 baseline +- [x] （Phase 2 已改为 YAML-only）通用 distill 入口：`fastvideo/training/distillation.py` + - Phase 1：CLI 驱动；Phase 2：`--config /real/path/to/distill.yaml` +- [x] （Phase 1 过渡实现；Phase 2 已移除）Wan wrapper：`fastvideo/training/wan_distillation_v3.py` ### F. 示例脚本（Phase 1） diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md index 2204f1824..921d5129d 100644 --- a/dev/phases/phase_2.md +++ b/dev/phases/phase_2.md @@ -36,17 +36,18 @@ Phase 2 的定位：在 Phase 1 已经验证 “Wan DMD2 训练行为对齐 base --- -## 当前“仍依赖 legacy”的点（Phase 2 要消灭） +## 当前与 legacy 的关系（Phase 2 目标） -以 Wan DMD2 为例，我们现在同时存在两条路径： +Phase 2 的目标是：**新 distill 代码路径在 import 与 runtime 两个层面都不再依赖 legacy +distillation pipeline**（`fastvideo/training/*distillation_pipeline.py`）。 -1. **Phase 1（legacy pipeline path）**：仍然依赖 `WanDistillationPipeline.from_pretrained(...)`， - validation 也会走 legacy `_log_validation(...)`（因为 Phase 1 需要保持与 baseline 的对齐与可用性）。 -2. **Phase 2（standalone runtime path）**：已替换为 **YAML-only + builder/runtime + standalone validator**， - 不再依赖 legacy pipeline。 +当前状态（已达成）： -Phase 2 的 deliverable 是让 Phase 2 路径完全自洽（训练/validation/checkpoint-resume），同时 Phase 1 -路径仍可跑（兼容历史脚本）。 +- Phase 2 entrypoint：`fastvideo/training/distillation.py --config ` +- runtime：`build_wan_dmd2_runtime_from_config(...)` +- validation：`fastvideo/distillation/validators/wan.py::WanValidator` + +以上链路不再实例化/调用 legacy `WanDistillationPipeline` / `DistillationPipeline._log_validation(...)`。 --- @@ -86,6 +87,10 @@ Phase 2 的 deliverable 是让 Phase 2 路径完全自洽（训练/validation/ch - 构建 dataloader（`build_parquet_map_style_dataloader`） - 初始化 tracker（复用 `fastvideo/training/trackers/`） - 通过 `WanAdapter(validator=...)` 接入独立 validation +- [x] 移除 Phase 1 legacy bridge（不影响 Phase 2） + - `fastvideo/distillation/builder.py::build_wan_dmd2_method` 已移除 + - `fastvideo/distillation/adapters/wan.py` 已移除 legacy pipeline fallback + - `fastvideo/training/wan_distillation_v3.py` 已移除 ### C. role-based checkpoint/save/resume（Phase 2.3） diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 021e72285..0be13c9b8 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -20,7 +20,6 @@ from fastvideo.pipelines import TrainingBatch from fastvideo.pipelines.pipeline_batch_info import ForwardBatch from fastvideo.pipelines.basic.wan.wan_dmd_pipeline import WanDMDPipeline -from fastvideo.training.distillation_pipeline import DistillationPipeline from fastvideo.training.training_utils import ( compute_density_for_timestep_sampling, get_sigmas, @@ -54,14 +53,12 @@ def __init__( training_args: Any, noise_scheduler: Any, vae: Any, - validation_pipeline: DistillationPipeline | None = None, validator: Any | None = None, ) -> None: self.bundle = bundle self.training_args = training_args self.noise_scheduler = noise_scheduler self.vae = vae - self._validation_pipeline_owner = validation_pipeline self._validator = validator self.world_group = get_world_group() @@ -123,21 +120,6 @@ def on_train_start(self) -> None: self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) - pipeline = self._validation_pipeline_owner - if pipeline is not None: - if not hasattr(pipeline, "validation_random_generator"): - pipeline.validation_random_generator = torch.Generator( # type: ignore[attr-defined] - device="cpu" - ).manual_seed(int(seed)) - if not hasattr(pipeline, "noise_random_generator"): - pipeline.noise_random_generator = torch.Generator( # type: ignore[attr-defined] - device="cpu" - ).manual_seed(int(seed)) - if not hasattr(pipeline, "noise_gen_cuda"): - pipeline.noise_gen_cuda = torch.Generator( # type: ignore[attr-defined] - device=self.device - ).manual_seed(int(seed)) - self.ensure_negative_conditioning() def ensure_negative_conditioning(self) -> None: @@ -232,33 +214,9 @@ def ensure_negative_conditioning(self) -> None: def log_validation(self, iteration: int) -> None: validator = getattr(self, "_validator", None) - if validator is not None: - validator.log_validation(iteration) - return - - pipeline = self._validation_pipeline_owner - if pipeline is None: - return - training_args = pipeline.training_args - if not getattr(training_args, "log_validation", False): + if validator is None: return - - if getattr(pipeline, "validation_pipeline", None) is None: - pipeline.initialize_validation_pipeline(training_args) - - student_transformer = self.bundle.role("student").require_module("transformer") - - old_inference_mode = training_args.inference_mode - old_dit_cpu_offload = training_args.dit_cpu_offload - try: - pipeline._log_validation( - student_transformer, - training_args, - iteration, - ) - finally: - training_args.inference_mode = old_inference_mode - training_args.dit_cpu_offload = old_dit_cpu_offload + validator.log_validation(iteration) def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: if self.noise_random_generator is None: @@ -471,9 +429,12 @@ def _get_teacher_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: role = self.bundle.role("teacher") transformer = role.require_module("transformer") transformer_2 = role.modules.get("transformer_2") - if transformer_2 is not None and self.boundary_timestep is not None: - if float(timestep.item()) < self.boundary_timestep: - return transformer_2 + if ( + transformer_2 is not None + and self.boundary_timestep is not None + and float(timestep.item()) < self.boundary_timestep + ): + return transformer_2 return transformer def _get_critic_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: @@ -484,15 +445,14 @@ def _get_critic_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: def student_rollout(self, batch: TrainingBatch) -> tuple[torch.Tensor, Any]: device_type = self.device.type dtype = batch.latents.dtype - with torch.autocast(device_type, dtype=dtype): - with set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata_vsa, - ): - if self.training_args.simulate_generator_forward: - pred_x0 = self._student_multi_step_simulation(batch) - else: - pred_x0 = self._student_single_step(batch) + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata_vsa, + ): + if self.training_args.simulate_generator_forward: + pred_x0 = self._student_multi_step_simulation(batch) + else: + pred_x0 = self._student_single_step(batch) return pred_x0, (batch.timesteps, batch.attn_metadata_vsa) def _student_single_step(self, batch: TrainingBatch) -> torch.Tensor: @@ -625,20 +585,19 @@ def teacher_predict_x0( if text_dict is None: raise RuntimeError("Missing unconditional_dict; ensure_negative_conditioning() may have failed") - with torch.autocast(device_type, dtype=dtype): - with set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata, - ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) - transformer = self._get_teacher_transformer(timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) + transformer = self._get_teacher_transformer(timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) return pred_x0 def critic_predict_x0( @@ -649,24 +608,23 @@ def critic_predict_x0( ) -> torch.Tensor: device_type = self.device.type dtype = noisy_latents.dtype - with torch.autocast(device_type, dtype=dtype): - with set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata, - ): - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, - timestep, - batch.conditional_dict, - ) - transformer = self._get_critic_transformer(timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep, + batch.conditional_dict, + ) + transformer = self._get_critic_transformer(timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) return pred_x0 def critic_flow_matching_loss( @@ -676,16 +634,14 @@ def critic_flow_matching_loss( device_type = self.device.type dtype = batch.latents.dtype - with torch.no_grad(): - with torch.autocast(device_type, dtype=dtype): - with set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata_vsa, - ): - if self.training_args.simulate_generator_forward: - generator_pred_x0 = self._student_multi_step_simulation(batch) - else: - generator_pred_x0 = self._student_single_step(batch) + with torch.no_grad(), torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata_vsa, + ): + if self.training_args.simulate_generator_forward: + generator_pred_x0 = self._student_multi_step_simulation(batch) + else: + generator_pred_x0 = self._student_single_step(batch) fake_score_timestep = torch.randint( 0, @@ -712,18 +668,17 @@ def critic_flow_matching_loss( fake_score_timestep, ).unflatten(0, (b, t)) - with torch.autocast(device_type, dtype=dtype): - with set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata, - ): - input_kwargs = self._build_distill_input_kwargs( - noisy_x0, - fake_score_timestep, - batch.conditional_dict, - ) - transformer = self._get_critic_transformer(fake_score_timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=batch.attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs( + noisy_x0, + fake_score_timestep, + batch.conditional_dict, + ) + transformer = self._get_critic_transformer(fake_score_timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) target = noise - generator_pred_x0 flow_matching_loss = torch.mean((pred_noise - target) ** 2) diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index f019f9d94..fc5bdff87 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -21,7 +21,6 @@ ) from fastvideo.training.trackers import initialize_trackers, Trackers from fastvideo.training.training_utils import get_scheduler -from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline from fastvideo.utils import maybe_download_model, verify_model_config_and_directory from fastvideo.distributed import get_world_group @@ -255,7 +254,6 @@ def build_wan_dmd2_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: training_args=training_args, noise_scheduler=noise_scheduler, vae=vae, - validation_pipeline=None, validator=validator, ) @@ -283,38 +281,3 @@ def build_wan_dmd2_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: tracker=tracker, start_step=0, ) - - -def build_wan_dmd2_method( - pipeline: WanDistillationPipeline, -) -> DMD2Method: - roles: dict[str, RoleHandle] = { - "student": - RoleHandle( - modules={"transformer": pipeline.transformer}, - optimizers={"main": pipeline.optimizer}, - lr_schedulers={"main": pipeline.lr_scheduler}, - trainable=True, - ), - "teacher": - RoleHandle( - modules={"transformer": pipeline.real_score_transformer}, - trainable=False, - ), - "critic": - RoleHandle( - modules={"transformer": pipeline.fake_score_transformer}, - optimizers={"main": pipeline.fake_score_optimizer}, - lr_schedulers={"main": pipeline.fake_score_lr_scheduler}, - trainable=True, - ), - } - bundle = ModelBundle(roles=roles) - adapter = WanAdapter( - bundle=bundle, - training_args=pipeline.training_args, - noise_scheduler=pipeline.noise_scheduler, - vae=pipeline.get_module("vae"), - validation_pipeline=pipeline, - ) - return DMD2Method(bundle=bundle, adapter=adapter) diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index e20ea3699..0734e5a9a 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -2,79 +2,14 @@ from __future__ import annotations +import argparse import sys -from collections.abc import Callable from typing import Any from fastvideo.logger import init_logger -from fastvideo.utils import FlexibleArgumentParser logger = init_logger(__name__) -_PipelineFactory = Callable[[Any], Any] -_MethodBuilder = Callable[[Any], Any] - - -def _build_pipeline_factories() -> dict[str, _PipelineFactory]: - from fastvideo.training.wan_distillation_pipeline import WanDistillationPipeline - - return { - "wan": - lambda args: WanDistillationPipeline.from_pretrained( - args.pretrained_model_name_or_path, - args=args, - ), - } - - -def _build_method_builders() -> dict[tuple[str, str], _MethodBuilder]: - from fastvideo.distillation.builder import build_wan_dmd2_method - - return { - ("wan", "dmd2"): build_wan_dmd2_method, - } - - -def run_distillation( - args: Any, - *, - distill_model: str, - distill_method: str, -) -> None: - """Legacy Phase 1 entrypoint (CLI-driven, uses legacy pipelines).""" - - from fastvideo.distillation import DistillTrainer - - pipeline_factories = _build_pipeline_factories() - method_builders = _build_method_builders() - - if distill_model not in pipeline_factories: - raise ValueError( - f"Unknown distill_model={distill_model!r}. Supported: {sorted(pipeline_factories)}" - ) - - builder_key = (distill_model, distill_method) - if builder_key not in method_builders: - supported = sorted({m for (model, m) in method_builders if model == distill_model}) - raise ValueError( - f"Unknown distill_method={distill_method!r} for distill_model={distill_model!r}. " - f"Supported methods for {distill_model}: {supported}" - ) - - pipeline = pipeline_factories[distill_model](args) - training_args = pipeline.training_args - - method = method_builders[builder_key](pipeline) - - trainer = DistillTrainer(training_args, tracker=pipeline.tracker) - trainer.run( - method, - dataloader=pipeline.train_dataloader, - max_steps=training_args.max_train_steps, - start_step=pipeline.init_steps, - ) - - def run_distillation_from_config(config_path: str, *, dry_run: bool = False) -> None: """Phase 2 entrypoint (YAML-only, standalone runtime builder).""" @@ -122,7 +57,11 @@ def main(args: Any) -> None: if __name__ == "__main__": argv = sys.argv - parser = FlexibleArgumentParser() + # NOTE: do not use `FlexibleArgumentParser` here. + # It treats `--config` specially (loads and inlines CLI args from YAML), + # which conflicts with Phase 2 distillation where `--config` points to the + # distillation run YAML itself. + parser = argparse.ArgumentParser() parser.add_argument( "--config", type=str, diff --git a/fastvideo/training/wan_distillation_v3.py b/fastvideo/training/wan_distillation_v3.py deleted file mode 100644 index f660ebe7f..000000000 --- a/fastvideo/training/wan_distillation_v3.py +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import sys - -from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.logger import init_logger -from fastvideo.training.distillation import run_distillation -from fastvideo.utils import FlexibleArgumentParser - -logger = init_logger(__name__) - - -def main(args) -> None: - logger.info( - "Starting Wan distillation v3 (wrapper for training/distillation.py: wan + dmd2)..." - ) - run_distillation(args, distill_model="wan", distill_method="dmd2") - logger.info("Wan distillation v3 completed") - - -if __name__ == "__main__": - argv = sys.argv - parser = FlexibleArgumentParser() - parser = TrainingArgs.add_cli_args(parser) - parser = FastVideoArgs.add_cli_args(parser) - args = parser.parse_args(argv[1:]) - main(args) From 0a3be304fb615acf9b95c1469fc680c5462956d9 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 08:03:10 +0000 Subject: [PATCH 064/214] fix gpu num --- .../phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 2 -- .../distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index 062a2fc45..e51486d1a 100644 --- a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -28,8 +28,6 @@ training: num_gpus: 1 sp_size: 1 tp_size: 1 - hsdp_replicate_dim: 1 - hsdp_shard_dim: 1 # Data (parquet dataset folder) data_path: data/Wan-Syn_77x448x832_600k diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index 83ef73078..99a706171 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -18,11 +18,9 @@ models: training: # Distributed - num_gpus: 1 + num_gpus: 8 sp_size: 1 tp_size: 1 - hsdp_replicate_dim: 1 - hsdp_shard_dim: 1 # Data (parquet dataset folder) data_path: data/Wan-Syn_77x448x832_600k @@ -76,4 +74,3 @@ training: pipeline_config: flow_shift: 8 dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] - From 225be11cb116d3b662325054aa43fad620d4ec52 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 20:20:05 +0000 Subject: [PATCH 065/214] ckpt manager for phase 2 --- dev/phases/phase_2.md | 36 ++- examples/distillation/phase2/README.md | 7 + .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 1 + fastvideo/distillation/adapters/wan.py | 16 + fastvideo/distillation/checkpoint.py | 277 ++++++++++++++++++ .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 1 + fastvideo/distillation/trainer.py | 15 + fastvideo/training/distillation.py | 60 +++- 8 files changed, 397 insertions(+), 16 deletions(-) create mode 100644 fastvideo/distillation/checkpoint.py diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md index 921d5129d..b0a21160c 100644 --- a/dev/phases/phase_2.md +++ b/dev/phases/phase_2.md @@ -64,7 +64,7 @@ distillation pipeline**（`fastvideo/training/*distillation_pipeline.py`）。 - 支持 few-step：`validation_sampling_steps` + `validation_guidance_scale` + seed/RNG（不依赖 legacy pipeline） - [x] `fastvideo/distillation/adapters/wan.py::WanAdapter.log_validation()` 支持注入 validator - Phase 2 路径：走新 validator - - Phase 1 路径：未提供 validator 时，仍可回退到 legacy pipeline（保留兼容） + - 未提供 validator 时：不做 validation（不再回退到 legacy pipeline） ### B. Builder/Runtime 脱离 pipeline（Phase 2.2） @@ -94,23 +94,31 @@ distillation pipeline**（`fastvideo/training/*distillation_pipeline.py`）。 ### C. role-based checkpoint/save/resume（Phase 2.3） -- [ ] 新增 `DistillCheckpointManager` - - 保存内容： - - `bundle.roles[*].modules`（仅 trainable params 或全量可配置） - - `bundle.roles[*].optimizers / lr_schedulers` - - `StatefulDataLoader`（如果使用） - - `RandomStateWrapper`（torch/numpy/python/cuda + noise generators） - - 恢复内容： - - `start_step`、dataloader iterator state、optimizer/scheduler state、RNG -- [ ] 将 checkpoint manager 接入 `DistillTrainer` - - `--resume_from_checkpoint` - - `--checkpointing_steps`（或复用现有 args） - - `--checkpoints_total_limit` +- [x] 新增 `fastvideo/distillation/checkpoint.py::DistillCheckpointManager` + - 保存内容（Phase 2 路径）： + - **trainable roles** 的 `modules/optimizers/lr_schedulers`（teacher 默认 frozen，不保存） + - `StatefulDataLoader`（Wan-Syn parquet loader 是 `torchdata.stateful_dataloader.StatefulDataLoader`） + - RNG states： + - 全局 torch/numpy/python/cuda + - adapter 暴露的 generators（`WanAdapter.get_rng_generators()`：noise_cpu/noise_cuda/validation_cpu） + - 保存位置： + - `${output_dir}/checkpoint-/dcp/`（torch.distributed.checkpoint） + - 旧 checkpoint 清理： + - 由 `training.checkpoints_total_limit` 控制（只保留最近 N 个） +- [x] 将 checkpoint manager 接入 `fastvideo/distillation/trainer.py::DistillTrainer` + - resume 参数来自 CLI：`fastvideo/training/distillation.py --resume-from-checkpoint ` + - 支持传入：`checkpoint-` / `checkpoint-/dcp` / `output_dir`（自动选最新） + - checkpoint 策略来自 YAML（TrainingArgs 字段）： + - `training.training_state_checkpointing_steps`: 训练态 checkpoint 保存间隔（<=0 关闭保存） + - `training.checkpoints_total_limit`: 保留最近 N 个 checkpoint（<=0 不清理） + - 行为： + - `on_train_start()` 之后再 `dcp.load(...)`，确保 adapter generators 已创建，可恢复其状态 + - checkpoint 保存发生在 **每步训练完成**（optim step + zero_grad）之后、validation 之前 ### D. 示例脚本（Phase 2） - [x] 最小 smoke（训练 + few-step validation）：`examples/distillation/phase2/temp.sh` -- [ ] Save/Resume 示例：等 Phase 2.3 checkpoint manager 完成后再补 +- [x] Save/Resume 用法说明：`examples/distillation/phase2/README.md` ### E. 最小单测（可选但建议） diff --git a/examples/distillation/phase2/README.md b/examples/distillation/phase2/README.md index dd4984684..29cf0df62 100644 --- a/examples/distillation/phase2/README.md +++ b/examples/distillation/phase2/README.md @@ -19,3 +19,10 @@ Recommended: - Edit the runnable YAML in: `fastvideo/distillation/outside/fastvideo/configs/distillation/` - `temp.sh` (runs the config above; same dataset + validation defaults as Phase0/Phase1). + +Resume: + +- Checkpoints are saved under `${output_dir}/checkpoint-/dcp/` when + `training.training_state_checkpointing_steps > 0` in YAML. +- To resume, pass a checkpoint directory (or an output_dir to auto-pick latest): + - `fastvideo/training/distillation.py --config --resume-from-checkpoint ` diff --git a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index e51486d1a..dd273fb6b 100644 --- a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -46,6 +46,7 @@ training: output_dir: outputs/phase2_wan2.1_dmd2_8steps_wansyn max_train_steps: 4000 seed: 1000 + checkpoints_total_limit: 3 # Optimizer learning_rate: 2.0e-6 diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 0be13c9b8..d7ae5d3fd 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -122,6 +122,22 @@ def on_train_start(self) -> None: self.ensure_negative_conditioning() + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + generators: dict[str, torch.Generator] = {} + if self.noise_random_generator is not None: + generators["noise_cpu"] = self.noise_random_generator + if self.noise_gen_cuda is not None: + generators["noise_cuda"] = self.noise_gen_cuda + + validator = getattr(self, "_validator", None) + validation_gen = getattr(validator, "validation_random_generator", None) + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen + + return generators + def ensure_negative_conditioning(self) -> None: if self.negative_prompt_embeds is not None: return diff --git a/fastvideo/distillation/checkpoint.py b/fastvideo/distillation/checkpoint.py new file mode 100644 index 000000000..5b09f9eab --- /dev/null +++ b/fastvideo/distillation/checkpoint.py @@ -0,0 +1,277 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +import re +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + +import torch +import torch.distributed as dist +import torch.distributed.checkpoint as dcp + +from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.logger import init_logger +from fastvideo.training.checkpointing_utils import ( + ModelWrapper, + OptimizerWrapper, + RandomStateWrapper, + SchedulerWrapper, +) + +logger = init_logger(__name__) + + +_CHECKPOINT_DIR_RE = re.compile(r"^checkpoint-(\d+)$") + + +def _is_stateful(obj: Any) -> bool: + return callable(getattr(obj, "state_dict", None)) and callable( + getattr(obj, "load_state_dict", None) + ) + + +def _rank() -> int: + if dist.is_available() and dist.is_initialized(): + return int(dist.get_rank()) + return 0 + + +def _barrier() -> None: + if dist.is_available() and dist.is_initialized(): + dist.barrier() + + +def _parse_step_from_dir(checkpoint_dir: Path) -> int: + match = _CHECKPOINT_DIR_RE.match(checkpoint_dir.name) + if not match: + raise ValueError( + f"Invalid checkpoint directory name {checkpoint_dir.name!r}; " + "expected 'checkpoint-'" + ) + return int(match.group(1)) + + +def _find_latest_checkpoint(output_dir: Path) -> Path | None: + if not output_dir.exists(): + return None + + candidates: list[tuple[int, Path]] = [] + for child in output_dir.iterdir(): + if not child.is_dir(): + continue + if not _CHECKPOINT_DIR_RE.match(child.name): + continue + if not (child / "dcp").is_dir(): + continue + try: + step = _parse_step_from_dir(child) + except Exception: + continue + candidates.append((step, child)) + + if not candidates: + return None + candidates.sort(key=lambda x: x[0]) + return candidates[-1][1] + + +def _resolve_resume_checkpoint(resume_from_checkpoint: str, *, output_dir: str) -> Path: + """Resolve a user-provided resume path to a concrete checkpoint dir. + + Accepted values: + - /path/to/output_dir/checkpoint- + - /path/to/output_dir/checkpoint-/dcp + - /path/to/output_dir (auto-pick latest checkpoint-*/dcp) + """ + + raw = os.path.expanduser(str(resume_from_checkpoint)) + path = Path(raw).resolve() + if not path.exists(): + raise FileNotFoundError(f"resume_from_checkpoint not found: {path}") + + if path.is_dir() and path.name == "dcp": + path = path.parent + + if path.is_dir() and _CHECKPOINT_DIR_RE.match(path.name): + if not (path / "dcp").is_dir(): + raise FileNotFoundError(f"Missing dcp dir under checkpoint: {path / 'dcp'}") + return path + + # Treat as output_dir -> pick latest. + latest = _find_latest_checkpoint(path) + if latest is not None: + return latest + + # Give a clearer error message. + out = Path(os.path.expanduser(str(output_dir))).resolve() + raise ValueError( + "Could not resolve resume checkpoint. Expected a checkpoint directory " + f"named 'checkpoint-' (with 'dcp/' inside), or an output_dir " + f"containing such checkpoints. Got: {path} (output_dir={out})." + ) + + +class _RoleModuleContainer(torch.nn.Module): + """Ephemeral container to expose multiple role modules as a single module. + + Needed because `OptimizerWrapper` expects a single root module covering all + parameters owned by the optimizer. + """ + + def __init__(self, modules: dict[str, torch.nn.Module]) -> None: + super().__init__() + for name, module in modules.items(): + self.add_module(name, module) + + +@dataclass(slots=True) +class DistillCheckpointConfig: + save_steps: int + keep_last: int + + +class DistillCheckpointManager: + """Role-based checkpoint manager for Phase 2 distillation runtime. + + - Checkpoint policy lives in YAML (via TrainingArgs fields). + - Resume path is typically provided via CLI (`--resume-from-checkpoint`). + """ + + def __init__( + self, + *, + bundle: ModelBundle, + dataloader: Any, + output_dir: str, + config: DistillCheckpointConfig, + get_rng_generators: Callable[[], dict[str, torch.Generator]] | None = None, + ) -> None: + self.bundle = bundle + self.dataloader = dataloader + self.output_dir = str(output_dir) + self.config = config + self._get_rng_generators = get_rng_generators + self._last_saved_step: int | None = None + + def _build_role_states(self, role: str, handle: RoleHandle) -> dict[str, Any]: + if not handle.trainable: + return {} + + states: dict[str, Any] = {} + container = _RoleModuleContainer(handle.modules) + + for module_name, module in handle.modules.items(): + states[f"models.{role}.{module_name}"] = ModelWrapper(module) + + for name, optimizer in handle.optimizers.items(): + states[f"optimizers.{role}.{name}"] = OptimizerWrapper(container, optimizer) + + for name, scheduler in handle.lr_schedulers.items(): + states[f"schedulers.{role}.{name}"] = SchedulerWrapper(scheduler) + + return states + + def _build_states(self) -> dict[str, Any]: + states: dict[str, Any] = {} + + # Models/opts/schedulers are role-scoped. + for role, handle in self.bundle.roles.items(): + states.update(self._build_role_states(role, handle)) + + # Dataloader (optional but recommended for exact resume). + if _is_stateful(self.dataloader): + states["dataloader"] = self.dataloader + + # RNG states: always save global RNG; also save adapter-provided generators. + states["random_state"] = RandomStateWrapper(None) + if self._get_rng_generators is not None: + for name, gen in (self._get_rng_generators() or {}).items(): + if gen is None: + continue + states[f"random_state.{name}"] = RandomStateWrapper(gen) + + return states + + def _checkpoint_dir(self, step: int) -> Path: + return Path(self.output_dir) / f"checkpoint-{step}" + + def _dcp_dir(self, step: int) -> Path: + return self._checkpoint_dir(step) / "dcp" + + def maybe_save(self, step: int) -> None: + save_steps = int(self.config.save_steps or 0) + if save_steps <= 0: + return + if step % save_steps != 0: + return + if self._last_saved_step == step: + return + self.save(step) + + def save_final(self, step: int) -> None: + save_steps = int(self.config.save_steps or 0) + if save_steps <= 0: + return + self.save(step) + + def save(self, step: int) -> None: + checkpoint_dir = self._checkpoint_dir(step) + dcp_dir = self._dcp_dir(step) + os.makedirs(dcp_dir, exist_ok=True) + + states = self._build_states() + if _rank() == 0: + logger.info("Saving Phase 2 checkpoint to %s", checkpoint_dir) + dcp.save(states, checkpoint_id=str(dcp_dir)) + _barrier() + self._last_saved_step = step + + self._cleanup_old_checkpoints() + + def maybe_resume(self, *, resume_from_checkpoint: str | None) -> int | None: + if not resume_from_checkpoint: + return None + + resolved = _resolve_resume_checkpoint(resume_from_checkpoint, output_dir=self.output_dir) + step = _parse_step_from_dir(resolved) + + states = self._build_states() + logger.info("Loading Phase 2 checkpoint from %s", resolved) + dcp.load(states, checkpoint_id=str(resolved / "dcp")) + _barrier() + logger.info("Checkpoint loaded; resuming from step=%s", step) + return step + + def _cleanup_old_checkpoints(self) -> None: + keep_last = int(self.config.keep_last or 0) + if keep_last <= 0: + return + + if _rank() != 0: + _barrier() + return + + output_dir = Path(self.output_dir) + candidates: list[tuple[int, Path]] = [] + for child in output_dir.iterdir(): + if not child.is_dir(): + continue + if not _CHECKPOINT_DIR_RE.match(child.name): + continue + try: + step = _parse_step_from_dir(child) + except Exception: + continue + candidates.append((step, child)) + + candidates.sort(key=lambda x: x[0]) + to_delete = candidates[:-keep_last] if len(candidates) > keep_last else [] + for step, path in to_delete: + logger.info("Removing old checkpoint (keep_last=%s): %s", keep_last, path) + shutil.rmtree(path, ignore_errors=True) + + _barrier() diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index 99a706171..314872e03 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -39,6 +39,7 @@ training: output_dir: outputs/phase2_wan2.1_dmd2_8steps_wansyn max_train_steps: 4000 seed: 1000 + checkpoints_total_limit: 3 # Optimizer learning_rate: 2.0e-6 diff --git a/fastvideo/distillation/trainer.py b/fastvideo/distillation/trainer.py index f26da9b2e..7dbaabac8 100644 --- a/fastvideo/distillation/trainer.py +++ b/fastvideo/distillation/trainer.py @@ -59,6 +59,7 @@ def run( dataloader: Any, max_steps: int, start_step: int = 0, + checkpoint_manager: Any | None = None, ) -> None: grad_accum = max(1, int(self.training_args.gradient_accumulation_steps or 1)) @@ -66,6 +67,14 @@ def run( if hasattr(method, "on_train_start"): method.on_train_start() # type: ignore[attr-defined] + resume_from_checkpoint = getattr(self.training_args, "resume_from_checkpoint", "") or "" + if checkpoint_manager is not None: + resumed_step = checkpoint_manager.maybe_resume( + resume_from_checkpoint=resume_from_checkpoint + ) + if resumed_step is not None: + start_step = int(resumed_step) + validation_interval = int(self.training_args.validation_steps or 0) if (getattr(self.training_args, "log_validation", False) and validation_interval > 0 and hasattr(method, @@ -124,10 +133,16 @@ def run( if self.global_rank == 0 and metrics: self.tracker.log(metrics, step) + if checkpoint_manager is not None: + checkpoint_manager.maybe_save(step) + if (getattr(self.training_args, "log_validation", False) and validation_interval > 0 and step % validation_interval == 0 and hasattr(method, "log_validation")): method.log_validation(step) # type: ignore[attr-defined] + if checkpoint_manager is not None: + checkpoint_manager.save_final(max_steps) + self.tracker.finish() diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 0734e5a9a..9079f59c5 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -10,17 +10,32 @@ logger = init_logger(__name__) -def run_distillation_from_config(config_path: str, *, dry_run: bool = False) -> None: +def run_distillation_from_config( + config_path: str, + *, + dry_run: bool = False, + resume_from_checkpoint: str | None = None, + override_output_dir: str | None = None, +) -> None: """Phase 2 entrypoint (YAML-only, standalone runtime builder).""" from fastvideo.distributed import maybe_init_distributed_environment_and_model_parallel from fastvideo.distillation import DistillTrainer + from fastvideo.distillation.checkpoint import ( + DistillCheckpointConfig, + DistillCheckpointManager, + ) from fastvideo.distillation.builder import build_wan_dmd2_runtime_from_config from fastvideo.distillation.yaml_config import load_distill_run_config cfg = load_distill_run_config(config_path) training_args = cfg.training_args + if resume_from_checkpoint is not None: + training_args.resume_from_checkpoint = str(resume_from_checkpoint) + if override_output_dir is not None: + training_args.output_dir = str(override_output_dir) + maybe_init_distributed_environment_and_model_parallel( training_args.tp_size, training_args.sp_size, @@ -38,20 +53,46 @@ def run_distillation_from_config(config_path: str, *, dry_run: bool = False) -> logger.info("Dry-run: config parsed and runtime built successfully.") return + ckpt_config = DistillCheckpointConfig( + save_steps=int(getattr(training_args, "training_state_checkpointing_steps", 0) or 0), + keep_last=int(getattr(training_args, "checkpoints_total_limit", 0) or 0), + ) + + adapter = getattr(runtime.method, "adapter", None) + get_rng_generators = getattr(adapter, "get_rng_generators", None) + if not callable(get_rng_generators): + get_rng_generators = None + + checkpoint_manager = DistillCheckpointManager( + bundle=runtime.method.bundle, + dataloader=runtime.dataloader, + output_dir=training_args.output_dir, + config=ckpt_config, + get_rng_generators=get_rng_generators, + ) + trainer = DistillTrainer(training_args, tracker=runtime.tracker) trainer.run( runtime.method, dataloader=runtime.dataloader, max_steps=training_args.max_train_steps, start_step=runtime.start_step, + checkpoint_manager=checkpoint_manager, ) def main(args: Any) -> None: config_path = str(args.config) dry_run = bool(args.dry_run) + resume_from_checkpoint = getattr(args, "resume_from_checkpoint", None) + override_output_dir = getattr(args, "override_output_dir", None) logger.info("Starting Phase 2 distillation from config=%s", config_path) - run_distillation_from_config(config_path, dry_run=dry_run) + run_distillation_from_config( + config_path, + dry_run=dry_run, + resume_from_checkpoint=resume_from_checkpoint, + override_output_dir=override_output_dir, + ) logger.info("Distillation completed") @@ -73,5 +114,20 @@ def main(args: Any) -> None: action="store_true", help="Parse config and build runtime, but do not start training.", ) + parser.add_argument( + "--resume-from-checkpoint", + type=str, + default=None, + help=( + "Path to a checkpoint directory (checkpoint-), its 'dcp/' subdir, " + "or an output_dir containing checkpoints (auto-picks latest)." + ), + ) + parser.add_argument( + "--override-output-dir", + type=str, + default=None, + help="Override training.output_dir from YAML (useful for repeated runs).", + ) args = parser.parse_args(argv[1:]) main(args) From b58d5cd58c64248b9bf3c361589482e5d2866481 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 23:02:46 +0000 Subject: [PATCH 066/214] config design --- dev/config.md | 143 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 dev/config.md diff --git a/dev/config.md b/dev/config.md new file mode 100644 index 000000000..72c0a8948 --- /dev/null +++ b/dev/config.md @@ -0,0 +1,143 @@ +# Phase 2 Distillation YAML Config + +本文档描述当前 **Phase 2 distillation** 入口所使用的 YAML 配置结构、字段含义，以及为什么采用这种设计。 + +相关实现： +- YAML loader：`fastvideo/distillation/yaml_config.py` +- Phase 2 入口：`fastvideo/training/distillation.py` +- Phase 2 示例 YAML：`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` + +## 1) 入口与约束（非常重要） + +Phase 2 distillation **只接受一个真实存在的 YAML 文件路径**，不会 fallback 到 legacy CLI/configs，也不会做 “overlay/补全 outside 路径” 的魔法。 + +运行方式（示意）： +```bash +python -m fastvideo.training.distillation \ + --config /abs/path/to/fastvideo/distillation/outside/fastvideo/configs/distillation/xxx.yaml +``` + +CLI 只保留少量 **runtime override**（不属于“实验定义”的内容）： +- `--resume-from-checkpoint`：从 checkpoint 恢复 +- `--override-output-dir`：临时覆盖输出目录（方便重复跑实验） +- `--dry-run`：只 parse + build runtime，不启动训练 + +## 2) YAML 顶层结构 + +目前 YAML 顶层包含 4 个部分： + +```yaml +distill: # 选择 “模型家族” + “蒸馏方法” +models: # 以 role 为 key 的模型参与者配置 +training: # 训练参数（直接映射到 TrainingArgs） +pipeline_config: # pipeline_config 的内联配置（dict） +# 或者 pipeline_config_path: /path/to/pipeline_config.json|yaml +``` + +loader 规则： +- `pipeline_config` 与 `pipeline_config_path` **二选一**，不能同时提供。 +- `training` 会被传入 `TrainingArgs.from_kwargs(**training_kwargs)`；Phase 2 不重复造一套训练参数体系。 +- Phase 2 会强制一些 invariants（见第 5 节）。 + +## 3) `distill`: 选择模型家族与蒸馏方法 + +```yaml +distill: + model: wan + method: dmd2 +``` + +用途： +- 让入口决定用哪个 **runtime builder**（当前仅支持 `wan + dmd2`）。 +- 未来扩展时，用同样的 schema 接入更多 `model/method` 组合，而不需要为每个组合写一个新的 training entry file。 + +设计原因： +- 把 “选择什么（model/method）” 与 “如何训练（training）/谁参与（models）” 分开，结构更稳定。 +- 更接近 FastGen：config 选择 `method` + `network`，训练逻辑由 method/adapter 决定。 + +## 4) `models`: role-based 模型参与者 + +```yaml +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true +``` + +字段含义（见 `fastvideo/distillation/specs.py`）： +- `family`：模型家族（可省略，默认等于 `distill.model`） +- `path`：模型路径/Hub 名称（由 builder/loader 负责加载） +- `trainable`：该 role 的参数是否参与训练（默认 `true`） + +设计原因（为什么要 role-based）： +- distillation setup 千差万别，不应 hard-code “只有 student/teacher/critic 才是核心角色”。**role 只是 key**，method 决定它需要哪些 role。 +- role-based bundle 让 Method 可以泛化：例如 CM/KD/SFT 可能需要 reward/refiner/aux_teacher 等，都可以用同一个结构表达。 + +关于 `trainable` 应由谁决定： +- YAML 里的 `trainable` 表示 **“训练配置的意图/策略”**（policy）。 +- Method 仍然可以施加 **算法不变量（invariants）**。例如 `DMD2Method` 会强制要求 + `models.teacher.trainable=false`（否则直接报错），因为 DMD2 默认 teacher 作为固定 reference。 + +## 5) `training`: 直接映射到 `TrainingArgs` + +`training:` 下的 key 基本上就是 `TrainingArgs` 的字段（`fastvideo/fastvideo_args.py`），例如： +- 分布式：`num_gpus`, `sp_size`, `tp_size`（以及内部需要的 hsdp 维度默认值） +- 数据：`data_path`, `dataloader_num_workers`, batch/shape 相关字段 +- 输出：`output_dir`, `max_train_steps`, `seed`, `checkpoints_total_limit` +- 优化器/调度器：`learning_rate`, `betas`, `lr_scheduler`, `fake_score_learning_rate`, ... +- distill 相关：`generator_update_interval`, `real_score_guidance_scale`, ... +- tracking/validation：`log_validation`, `validation_*`, `tracker_project_name`, ... + +Phase 2 loader 会强制/补全的关键 invariants（见 `fastvideo/distillation/yaml_config.py`）： +- `mode = ExecutionMode.DISTILLATION` +- `inference_mode = False` +- `dit_cpu_offload = False` +- `dit_precision` 默认 `fp32`（保持 training-mode loader 语义：fp32 master weights） +- 若未显式提供，补默认分布式尺寸（例如 `num_gpus`、`sp_size/tp_size`、hsdp 维度） +- 若 `training.model_path` 未提供，则默认取 `models.student.path` + - 这是为了让 FastVideo 的 pipeline/pipeline_config registry 能正常工作（以 student 为 base）。 + +设计原因（为什么 training 直接复用 TrainingArgs）： +- FastVideo 的大量组件（loader、pipeline config、distributed init、各种 utils）都以 `TrainingArgs` 为中心。 +- Phase 2 的目标是 **解耦 legacy pipeline**，但不等于要立刻重造整个 training 参数系统；复用 TrainingArgs 能显著降低迁移成本与风险。 + +## 6) `pipeline_config` / `pipeline_config_path` + +两种等价写法（二选一）： + +1) 直接内联（推荐用于少量关键字段）： +```yaml +pipeline_config: + flow_shift: 8 + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] +``` + +2) 使用外部文件路径（适合复用现有 pipeline config 文件）： +```yaml +pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json +``` + +设计原因： +- 把 “运行一个 distillation 实验需要的最小 pipeline 配置” 放在 distill YAML 附近，便于复现实验。 +- 但也允许用 path 复用大型 config 文件，避免在 YAML 中塞进过多模型细节。 +- 同时与我们 “outside/ 非侵入式新增 config” 的策略兼容：不必修改上游 `fastvideo/configs/*`。 + +## 7) 最小可运行示例（Wan few-step DMD2） + +完整示例参考： +`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` + +其核心要点是： +- `distill: {model: wan, method: dmd2}` +- `models` 至少包含 `student/teacher/critic` +- `training.data_path / output_dir / max_train_steps / seed` 等训练必须项 +- `pipeline_config.flow_shift` + `pipeline_config.dmd_denoising_steps`（8 steps）用于 few-step schedule From 7d20269b9cb90a4444d73167afdf2b5f6dd0e5be Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 22 Feb 2026 23:46:41 +0000 Subject: [PATCH 067/214] designing phase 3 --- dev/config.md | 80 +++++++++++++++++++++++++++ dev/design.md | 146 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 209 insertions(+), 17 deletions(-) diff --git a/dev/config.md b/dev/config.md index 72c0a8948..21e103c1f 100644 --- a/dev/config.md +++ b/dev/config.md @@ -141,3 +141,83 @@ pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json - `models` 至少包含 `student/teacher/critic` - `training.data_path / output_dir / max_train_steps / seed` 等训练必须项 - `pipeline_config.flow_shift` + `pipeline_config.dmd_denoising_steps`（8 steps）用于 few-step schedule + +## 8) Phase 3 计划：`recipe` 顶层 + `method_config` + +Phase 2 的 YAML schema 使用 `distill:` 作为顶层选择（历史原因：当时入口只跑 distillation）。 +但随着我们计划把 **finetuning 也纳入同一框架**，`distill.method=finetune` 的语义会显得别扭。 + +因此 Phase 3 计划升级 schema： + +```yaml +recipe: {family: wan, method: dmd2} # 只负责选择（更通用） +models: {...} # role -> {family/path/trainable} +training: {...} # infra 参数（映射到 TrainingArgs） +pipeline_config: {...} # pipeline/backbone config（模型侧） +method_config: {...} # method-specific 超参（方法侧） +``` + +同时保持与 FastVideo 的 `ExecutionMode` 语义对齐（Phase 3 计划）： +- `recipe.method=finetune` 时：入口层设置 `training.mode=FINETUNING` +- 其它 distillation methods：入口层设置 `training.mode=DISTILLATION` + +### 8.1 为什么需要 `method_config`？ + +动机是把语义分清楚： +- `training:`（TrainingArgs）应该尽量只承载 **基础设施**：分布式、优化器、ckpt、logging、数据路径等 +- `method_config:` 承载 **算法/recipe** 的超参：DMD2 / Self-Forcing / Finetune 各自不同 + +这样未来 method 变多时，不会出现所有参数都混在 `training:` 里，导致配置难读、难 review、难复现。 + +### 8.2 示例：DMD2（新 schema） + +```yaml +recipe: + family: wan + method: dmd2 + +models: + student: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} + teacher: {family: wan, path: Wan-AI/Wan2.1-T2V-14B-Diffusers, trainable: false} + critic: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} + +training: + # ... TrainingArgs fields ... + output_dir: outputs/... + max_train_steps: 4000 + seed: 1000 + +pipeline_config: + flow_shift: 8 + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + +method_config: + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + simulate_generator_forward: true +``` + +### 8.3 示例：Finetuning（only student） + +```yaml +recipe: + family: wan + method: finetune + +models: + student: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} + +training: + # ... TrainingArgs fields ... + data_path: ... + output_dir: outputs/... + max_train_steps: 4000 + seed: 1000 + +pipeline_config: + flow_shift: 8 + +method_config: + pred_type: x0 + loss: flow_matching +``` diff --git a/dev/design.md b/dev/design.md index 5e568255e..b1210dd2f 100644 --- a/dev/design.md +++ b/dev/design.md @@ -365,9 +365,9 @@ FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step` ### 6.1 最小可用（建议先落地） -**Phase 2 目标**：一个 YAML 配置文件描述一次 distill 运行，入口只需要： +**Phase 2+ 目标**：一个 YAML 配置文件描述一次运行（distill/finetune/…），入口只需要： -- `fastvideo/training/distillation.py --config path/to/distill.yaml` +- `fastvideo/training/distillation.py --config path/to/run.yaml` 除此之外的训练参数/模型选择/方法选择，都写入 YAML。 @@ -376,13 +376,17 @@ FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step` - `--models_json path/to/models.json` - per-role precision/offload/trainable/fsdp_policy/ckpt_path 等 -### 6.3 YAML 配置（Phase 2 必做）：结构化训练参数 + roles 选择 +### 6.3 YAML schema v2（Phase 3）：`recipe` + `method_config` -我们希望最终的 “单次运行” 配置长这样（示意；字段可迭代）： +说明： +- Phase 2 的 YAML schema v1 使用 `distill:` 顶层（历史原因） +- Phase 3 将升级为 schema v2：用 `recipe:` 顶层，并引入 `method_config:`（语义更通用） + +schema v2 的 “单次运行” 配置示意（字段可迭代）： ```yaml -distill: - model: wan +recipe: + family: wan method: dmd2 models: @@ -409,15 +413,20 @@ pipeline_config: # 支持直接内联覆盖，也支持只给 pipeline_config_path # pipeline_config_path: fastvideo/configs/wan_1.3B_t2v_pipeline.json flow_shift: 8 + +method_config: + # method-specific 超参（不进入 TrainingArgs；由 method/adapter 自行解析） + generator_update_interval: 5 + real_score_guidance_scale: 3.5 ``` **解析策略（最优雅且低风险）** -- 新入口的 parser 只保留 `--config distill.yaml`（以及少量 meta flags，如 `--dry-run`）。 +- 新入口的 parser 只保留 `--config run.yaml`（以及少量 meta flags，如 `--dry-run`）。 - 训练相关的所有参数（TrainingArgs/FastVideoArgs/pipeline_config/method/models）都来自 YAML。 - 解析流程： 1) `yaml.safe_load` 得到 dict - 2) 规范化/校验 schema（distill/models/training/pipeline_config/...） + 2) 规范化/校验 schema（recipe/models/training/pipeline_config/method_config/...） 3) 将 `training:` 与 `pipeline_config:` 合成 kwargs，调用 `TrainingArgs.from_kwargs(**kwargs)` （由现有 PipelineConfig/PreprocessConfig 负责子配置实例化与校验） @@ -434,9 +443,9 @@ pipeline_config: - 把“本应在外部 repo 存在的新增/改版配置”放进： - `fastvideo/distillation/outside/fastvideo/configs/...` -- distillation 的配置加载器在解析任何 config 路径时： - - **先查 outside overlay 是否存在同路径文件** - - 若不存在，再 fallback 到 repo 内的 `fastvideo/configs/...` +- distillation 入口 **不做任何自动补全/overlay 重写**： + - 用户传入的 `--config` 必须是一个真实存在的文件路径（通常位于 `outside/` 下） + - config 内引用的其它路径（如 `pipeline_config_path`）也必须是 **真实路径** 这让我们可以在不侵入主仓库配置的情况下，迭代 YAML/JSON config、做实验性变更， 同时不影响 legacy 代码路径。 @@ -444,7 +453,7 @@ pipeline_config: **实现注意** - 不建议把 `outside/` 直接插入 `sys.path` 去 shadow 整个 `fastvideo` 包（风险太高、调试困难）。 -- 推荐把 `outside/` 仅作为 **配置文件 overlay**（YAML/JSON）来做路径解析。 +- 推荐把 `outside/` 仅作为 **外部配置存放目录**（YAML/JSON），避免运行时“魔法寻路”。 - 如果确实需要覆盖 Python config（`.py`）： - 用 `importlib` 的“按文件路径加载模块”方式加载为独立 module name，避免影响全局 import。 @@ -471,7 +480,7 @@ Phase 0 的实践表明：先把新框架以 **additive** 方式落地到一个 - `fastvideo/distillation/methods/`：`base.py`、`distribution_matching/dmd2.py`、（目标）`self_forcing.py` - `fastvideo/distillation/trainer.py`：`DistillTrainer` - `fastvideo/distillation/builder.py`：把 “config -> roles -> bundle/adapter/method” 的胶水集中起来 -- `fastvideo/training/distillation.py`：通用入口（选择 distill_model + distill_method） +- `fastvideo/training/distillation.py`：通用入口（YAML-only：`--config path/to/run.yaml`） - （后续）`fastvideo/distillation/checkpoint.py`：role-based `CheckpointManager`（先兼容旧格式） - （后续）`fastvideo/distillation/callbacks/`：EMA/clip/log/profiler 等 @@ -518,7 +527,7 @@ Phase 1 的“辉煌”（落地与收益）： - Phase 2 起将切换为 **YAML-only**（见第 6 节），并逐步废弃这套 CLI - ✅ 训练效果对齐：Phase 1 跑出来的 WAN DMD2 与 Phase 0/baseline 行为一致（已实测） -### Phase 2（建议重点推进）：彻底脱离 legacy distill pipeline（让新框架可独立存在） +### Phase 2（已完成）：彻底脱离 legacy distill pipeline（让新框架可独立存在） 你提的建议我同意：Phase 2 应该把 Phase 1 仍然残留的 legacy 依赖清干净，让新的 distill 代码路径可以 **不依赖** `fastvideo/training/*distillation_pipeline.py` 和 @@ -542,8 +551,8 @@ Phase 1 的“辉煌”（落地与收益）： - 建议实现： - 定义结构化 spec：`RoleSpec/ModelSpec`（role -> {family, path, precision, trainable,...}） - 配置形态落地（Phase 2 必做）： - - `--config path/to/distill.yaml`（YAML 为 single source of truth；CLI 仅指定配置路径） - - `outside/` overlay：解析 `pipeline_config_path` 等文件路径时 outside 优先、repo fallback + - `--config path/to/run.yaml`（YAML 为 single source of truth；CLI 仅指定配置路径） + - `outside/` workaround：把新增/实验性 configs 放在 `outside/`，入口只接受真实路径（不做 overlay 寻路） - （可选）保留 `--models_json` 作为“程序生成配置”的接口 - builder 根据 spec： - 加载 modules（student/teacher/critic） @@ -551,7 +560,7 @@ Phase 1 的“辉煌”（落地与收益）： - 组装 `ModelBundle + Adapter + Method` - 构建 dataloader（直接复用 dataset 代码，不经由 legacy pipeline class） - 不新增入口文件：直接增强 `fastvideo/training/distillation.py`，并把它定义为 **YAML-only distill entrypoint** - - 仅支持 `--config distill.yaml`（以及少量 meta flags），不再兼容旧式 CLI configs + - 仅支持 `--config run.yaml`（以及少量 meta flags），不再兼容旧式 CLI configs - legacy distill 继续通过原有 `fastvideo/training/*distillation_pipeline.py` 入口运行（两套路径并存） - 收益：distill 路径具备真正的“模型/算法 catalog + instantiate”，开始能支持更多模型家族 @@ -576,6 +585,109 @@ Phase 1 的“辉煌”（落地与收益）： - 新增更多 method（teacher-only、多 teacher、KD 轨迹蒸馏等） - 逐步冻结或移除旧 distill pipeline（保留兼容入口亦可） +### Phase 3（计划）：优雅 dispatch + Recipe config + Finetuning（统一到同一框架） + +Phase 3 的定位：在 Phase 2 已经证明“新 distill 框架可独立运行”的基础上，解决两个长期 +扩展的核心问题： + +1) **真正优雅的 dispatch（避免 N×M builder 组合爆炸）** +2) **配置语义升级（`distill` -> `recipe`，引入 `method_config`）** +3) **把 finetuning 作为一种 method 接入框架**（只需要 `student` + dataset） + +#### Phase 3.1：真正优雅的 dispatch（N+M，而不是 N×M） + +目标：新增第 5 个模型家族 + 第 5 个算法时，不需要写 25 个 `build__()`。 + +核心思路：把 “可组合的变化” 拆成两类 registry，然后用 adapter capability/protocol 做约束： + +- **Model family registry**（按 `recipe.family` 注册） + - 负责：按 role 加载 modules、构建 adapter、构建 validator、构建 dataloader（或 data hooks） +- **Method registry**（按 `recipe.method` 注册） + - 负责：构建 method（算法）；声明 `required_roles`；声明需要的 adapter primitives（Protocol 或 capability） + +入口层只做组合（伪代码）： + +```text +cfg = load_run_config(...) +family = FAMILY_REGISTRY[cfg.recipe.family] +method = METHOD_REGISTRY[cfg.recipe.method] + +bundle = family.build_bundle(cfg.models, cfg.training, cfg.pipeline_config) +adapter = family.build_adapter(bundle, cfg.training, cfg.pipeline_config, cfg.method_config) +validator = family.build_validator(...) # optional +dataloader = family.build_dataloader(cfg.training, cfg.data?) # optional + +distill_method = method.build(bundle=bundle, adapter=adapter, method_config=cfg.method_config) +trainer.run(distill_method, dataloader, ...) +``` + +这样新增扩展的成本是： +- 新模型家族：新增 1 个 family plugin（N） +- 新算法：新增 1 个 method plugin（M） +- 组合不需要额外代码（不再写 N×M） + +实现落点（建议，Phase 3 落地到代码时再细化）： +- `fastvideo/distillation/registry.py` + - `register_family(name)(cls)` / `register_method(name)(cls)` 装饰器 + - `get_family(name)` / `get_method(name)` + “可用项”错误提示 +- `fastvideo/distillation/builder.py` + - 收敛为 `build_runtime_from_config(cfg)`（通用），内部查 registry + - Wan 的加载逻辑迁移为 `WanFamily` plugin（保留当前 Phase2 的 loader 复用） + +#### Phase 3.2：配置语义升级（`distill` -> `recipe`，引入 `method_config`） + +动机： +- `distill.method=finetune` 语义别扭，因为 finetune 是一种训练 recipe，不一定是“蒸馏”。 +- method-specific 参数长期塞进 `training:`（TrainingArgs）会让配置语义越来越混杂。 + +Phase 3 计划把 YAML schema 升级为： + +```yaml +recipe: {family: wan, method: dmd2} # 只负责 “选什么” +models: {student: ..., teacher: ...} # 参与者 +training: {...} # infra 参数（映射到 TrainingArgs） +pipeline_config: {...} # pipeline/backbone config（模型侧） +method_config: {...} # algorithm/method 超参（方法侧） +``` + +同时保持与 FastVideo 现有语义对齐： +- 入口层会根据 `recipe.method` 推导 `TrainingArgs.mode` + - `finetune` -> `ExecutionMode.FINETUNING` + - 其它 distillation methods -> `ExecutionMode.DISTILLATION` + +迁移策略（建议）： +- Phase 3 先把 `method_config` 作为新增字段引入，并逐步把以下参数从 `training:` 挪过去： + - DMD2：`generator_update_interval`, `real_score_guidance_scale`, `simulate_generator_forward`, ... + - Self-forcing：ODE-init / cache / rollout 策略相关参数 + - Finetune：loss/target/pred_type 等 +- `training:` 保持 “trainer/infra” 语义（分布式、优化器、ckpt、logging、数据路径等）。 + +#### Phase 3.3：Finetuning 作为一种 method 接入（only student） + +目标：让 finetuning 跟 distillation 一样走同一套： +`ModelBundle + Adapter + Method + Trainer + (Validator/Checkpoint)`。 + +建议落地形态（Phase 3 落地到代码时）： +- 新增 method：`fastvideo/distillation/methods/fine_tuning/finetune.py::FineTuneMethod` + - `bundle.require_roles(["student"])` + - 复用 trainer 的 step/ckpt/validation + - 通过 adapter 提供的 primitives 完成 forward/loss/backward（避免 method 管 forward_context） +- 为 finetune 定义 adapter contract（类似 `_DMD2Adapter` 的做法）： + - `_FineTuneAdapter(Protocol)`：`prepare_batch()` + `sample_train_timestep()` + `student_predict()` + `training_loss()` 等 + - Wan 侧由 `WanAdapter` 实现该 contract（或拆出 `WanAdapterBase + WanFineTuneOps` 以避免 adapter 过度膨胀） + +Finetune 的 config（示意）： +```yaml +recipe: {family: wan, method: finetune} +models: + student: {family: wan, path: ..., trainable: true} +training: {...} +pipeline_config: {...} +method_config: + pred_type: x0 + loss: flow_matching +``` + --- ## 9. Guardrails / 测试建议（避免重构“跑得通但不可维护”） From 02e948b266245836bb72a18842a330de1b2080cb Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 23 Feb 2026 00:40:24 +0000 Subject: [PATCH 068/214] designing phase 2.9: decoupling adapter --- dev/design.md | 58 +++------- dev/phases/phase_2_9.md | 199 +++++++++++++++++++++++++++++++++ dev/phases/phase_3.md | 237 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 453 insertions(+), 41 deletions(-) create mode 100644 dev/phases/phase_2_9.md create mode 100644 dev/phases/phase_3.md diff --git a/dev/design.md b/dev/design.md index b1210dd2f..aafc6cc04 100644 --- a/dev/design.md +++ b/dev/design.md @@ -585,56 +585,32 @@ Phase 1 的“辉煌”（落地与收益）： - 新增更多 method（teacher-only、多 teacher、KD 轨迹蒸馏等） - 逐步冻结或移除旧 distill pipeline（保留兼容入口亦可） -### Phase 3（计划）：优雅 dispatch + Recipe config + Finetuning（统一到同一框架） +### Phase 2.9（计划）：A+B+Families 语义收敛（为 Phase 3 铺路） -Phase 3 的定位：在 Phase 2 已经证明“新 distill 框架可独立运行”的基础上，解决两个长期 -扩展的核心问题： +动机：Phase 2 已经完成“独立可跑”，但仍存在几个会阻碍长期扩展的结构性问题： -1) **真正优雅的 dispatch（避免 N×M builder 组合爆炸）** -2) **配置语义升级（`distill` -> `recipe`，引入 `method_config`）** -3) **把 finetuning 作为一种 method 接入框架**（只需要 `student` + dataset） +- adapter API 仍有一定 **role-centric / method-specific** 倾向（例如 `sample_dmd_timestep()`、`teacher/critic` 分叉函数） +- entrypoint/builder 的组合 dispatch 未来如果扩展，会走向 N×M（写大量组合函数或 if/else） -#### Phase 3.1：真正优雅的 dispatch（N+M，而不是 N×M） +因此引入一个 **Phase 2.9**，先把语义边界收敛好，再开始 Phase 3（配置 schema 升级 + finetune）。 -目标：新增第 5 个模型家族 + 第 5 个算法时，不需要写 25 个 `build__()`。 +Phase 2.9 目标（简称 A+B+Families）： -核心思路：把 “可组合的变化” 拆成两类 registry，然后用 adapter capability/protocol 做约束： +- A) 把 adapter API 收敛为 **operation-centric**（role 只是 key，不暴露 teacher/critic/student 专用函数） +- B) 把 “timestep sampling policy” 等 **算法策略** 挪回 method（adapter 只保留 scheduler 语义转换） +- Families) 引入 family registry + method registry，完成 **优雅 dispatch（N+M）** + - 这一步做完后，Phase 3 不再需要改 entrypoint/builder 的 dispatch 逻辑 -- **Model family registry**（按 `recipe.family` 注册） - - 负责：按 role 加载 modules、构建 adapter、构建 validator、构建 dataloader（或 data hooks） -- **Method registry**（按 `recipe.method` 注册） - - 负责：构建 method（算法）；声明 `required_roles`；声明需要的 adapter primitives（Protocol 或 capability） +详细执行清单见：`dev/phases/phase_2_9.md` -入口层只做组合（伪代码）： +### Phase 3（计划）：Recipe config + method_config + Finetuning（统一到同一框架） -```text -cfg = load_run_config(...) -family = FAMILY_REGISTRY[cfg.recipe.family] -method = METHOD_REGISTRY[cfg.recipe.method] - -bundle = family.build_bundle(cfg.models, cfg.training, cfg.pipeline_config) -adapter = family.build_adapter(bundle, cfg.training, cfg.pipeline_config, cfg.method_config) -validator = family.build_validator(...) # optional -dataloader = family.build_dataloader(cfg.training, cfg.data?) # optional - -distill_method = method.build(bundle=bundle, adapter=adapter, method_config=cfg.method_config) -trainer.run(distill_method, dataloader, ...) -``` - -这样新增扩展的成本是： -- 新模型家族：新增 1 个 family plugin（N） -- 新算法：新增 1 个 method plugin（M） -- 组合不需要额外代码（不再写 N×M） +Phase 3 的定位：在 Phase 2.9 已经完成“优雅 dispatch + adapter/method 语义收敛”的基础上： -实现落点（建议，Phase 3 落地到代码时再细化）： -- `fastvideo/distillation/registry.py` - - `register_family(name)(cls)` / `register_method(name)(cls)` 装饰器 - - `get_family(name)` / `get_method(name)` + “可用项”错误提示 -- `fastvideo/distillation/builder.py` - - 收敛为 `build_runtime_from_config(cfg)`（通用），内部查 registry - - Wan 的加载逻辑迁移为 `WanFamily` plugin（保留当前 Phase2 的 loader 复用） +1) **配置语义升级（`distill` -> `recipe`，引入 `method_config`）** +2) **把 finetuning 作为一种 method 接入框架**（只需要 `student` + dataset） -#### Phase 3.2：配置语义升级（`distill` -> `recipe`，引入 `method_config`） +#### Phase 3.1：配置语义升级（`distill` -> `recipe`，引入 `method_config`） 动机： - `distill.method=finetune` 语义别扭，因为 finetune 是一种训练 recipe，不一定是“蒸馏”。 @@ -662,7 +638,7 @@ method_config: {...} # algorithm/method 超参（方法侧） - Finetune：loss/target/pred_type 等 - `training:` 保持 “trainer/infra” 语义（分布式、优化器、ckpt、logging、数据路径等）。 -#### Phase 3.3：Finetuning 作为一种 method 接入（only student） +#### Phase 3.2：Finetuning 作为一种 method 接入（only student） 目标：让 finetuning 跟 distillation 一样走同一套： `ModelBundle + Adapter + Method + Trainer + (Validator/Checkpoint)`。 diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md new file mode 100644 index 000000000..a48ff4f0b --- /dev/null +++ b/dev/phases/phase_2_9.md @@ -0,0 +1,199 @@ +# Phase 2.9：A+B+Families 语义收敛（operation-centric adapter + policy 回归 method + 优雅 dispatch） + +Phase 2 已经实现了“新框架可独立运行（不依赖 legacy distill pipeline）”。但从长期扩展的角度， +我们仍有三类结构性问题需要先解决，否则 Phase 3（配置语义升级 + finetune）会被迫在不稳的语义边界上继续堆功能。 + +本 Phase 2.9 的目标是先把语义边界收敛好（A+B+Families），并把 dispatch 做到真正优雅（N+M）， +让 Phase 3 只需要在此基础上加 config schema 与新 method，而不需要再动 entrypoint/builder 的组合逻辑。 + +本文件只做**代码层面的设计**（不写代码），后续实现过程中如果有小调整会回填；遇到重大风险会停下讨论。 + +--- + +## 0) Phase 2.9 的 “A+B+Families” 是什么？ + +### A) operation-centric adapter API（避免 role 爆炸） + +把 adapter API 从： +- `teacher_predict_x0(...) / critic_predict_x0(...) / backward_student(...) / backward_critic(...)` + +收敛为： +- `predict_x0(handle, ...)` +- `backward(loss, ctx, ...)`（ctx 自带所需信息，不需要 role） + +核心原则： +- adapter **不接触 role 字符串**，只接收 `RoleHandle`（handle）来取 module/optimizer 等资源。 +- method 才持有 “role -> handle” 的语义映射（teacher/critic/student/reward/... 的意义只存在于 method）。 + +为什么要用 handle 而不是 role 字符串？ +- 防止 adapter 内部出现 `if role == "teacher": ...` 这类语义泄漏，长期演化会再次耦合/role 爆炸。 +- 让 adapter 更像 “family runtime primitives”，method 更像 “algorithm orchestration”。 + +### B) policy 回归 method（adapter 只保留 mechanics） + +把 DMD2 的“策略”从 adapter 挪回 method，例如： +- timestep sampling strategy（uniform/课程/分段等） + +adapter 只保留 scheduler 相关的 mechanics： +- `num_train_timesteps` +- `shift/clamp timestep` 的语义转换 + +### Families) build-time 语义收敛 + 优雅 dispatch（N+M） + +引入 `families/` + registry 的目的： +- adapter 专注 runtime/step-time +- family 插件专注 build-time（加载 modules / shared components / dataloader / validator / tracker） +- builder/entrypoint 不写组合 if/else；新增 family 或 method 的成本为 N+M，而不是 N×M + +--- + +## 1) Phase 2.9 交付目标（Definition of Done） + +- `fastvideo/training/distillation.py` 不再硬编码 `wan + dmd2` 分支； + 改为调用通用 `build_runtime_from_config(cfg)`，并通过 registry resolve family/method。 +- `WanAdapter` 对 DMD2 的暴露接口变为 operation-centric： + - 不再暴露 `teacher_*` / `critic_*` / `student_*` 专用函数给 method 使用 + - DMD2Method 通过通用操作（如 `predict_x0(handle=...)`）完成 teacher/critic/student 的调用 +- DMD2 的 timestep sampling policy 从 adapter 迁移到 method（最少把 `sample_dmd_timestep()` 挪走）。 +- Phase 2 的训练行为/结果应尽可能保持一致（同 config 下 loss 形态、validation 产物趋势不应漂移）。 + +--- + +## 2) 非目标（明确不做） + +- 不做 YAML schema v2（`recipe` + `method_config`）升级（留到 Phase 3）。 +- 不新增 finetune method（留到 Phase 3）。 +- 不新增新模型家族（Phase 2.9 只整理 Wan）。 +- 不追求把所有 DMD2 逻辑从 adapter 中抠干净（例如 critic loss 里 student rollout 的复用）； + Phase 2.9 先解决“role-centric API + policy 泄漏”这两个最大痛点。 + +--- + +## 3) TODO List（Review Checklist） + +### 3.1 Registry + Families（优雅 dispatch，N+M） + +- [ ] 新增 `fastvideo/distillation/registry.py` + - `register_family(name)` / `register_method(name)` 装饰器 + - `get_family(name)` / `get_method(name)`（错误信息包含可用项） + - `ensure_builtin_registrations()`：导入内置 family/method 以完成注册 +- [ ] 新增 `fastvideo/distillation/families/` + - `fastvideo/distillation/families/__init__.py` + - `fastvideo/distillation/families/wan.py`：`WanFamily` + - 从 Phase 2 builder 迁移 Wan-specific build-time 逻辑： + - 加载 role modules（transformer/transformer_2/vae 等） + - shared components（scheduler/noise_scheduler） + - dataloader（parquet + schema） + - tracker + - validator（WanValidator） + - adapter 实例化（WanAdapter） +- [ ] 改造 `fastvideo/distillation/builder.py` + - 新增/收敛为 `build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime` + - `DistillRuntime.method` 类型改为 `DistillMethod`（而不是 `DMD2Method`） + - builder 内部逻辑： + - `family = registry.get_family(cfg.distill.model)`（Phase 2.9 暂用 `distill.model` 作为 family） + - `method = registry.get_method(cfg.distill.method)` + - 调用 family 构建 bundle/adapter/dataloader/tracker + - 调用 method factory 构建 DistillMethod +- [ ] 改造入口 `fastvideo/training/distillation.py` + - 删除 `if cfg.distill.model == "wan" and cfg.distill.method == "dmd2": ...` + - 统一走：`runtime = build_runtime_from_config(cfg)` + +### 3.2 Adapter API：从 role-centric 收敛到 operation-centric（A） + +- [ ] 更新 `fastvideo/distillation/methods/distribution_matching/dmd2.py` + - `_DMD2Adapter` Protocol 改为 operation-centric： + - `predict_x0(handle, noisy_latents, timestep, batch, *, conditional, attn_kind)` + - `backward(loss, ctx, ...)` + - `select_module(handle, module_name, timestep)`（可选：用于 transformer_2/boundary） + - `timestep_ops`（见 3.3） + - 移除对 `teacher_predict_x0/critic_predict_x0/backward_student/backward_critic` 的直接依赖 +- [ ] 更新 `fastvideo/distillation/adapters/wan.py` + - 把 `teacher_predict_x0/critic_predict_x0` 合并为 `predict_x0(handle=...)` + - 把 `backward_student/backward_critic` 合并为 `backward(loss, ctx, ...)` + - 将 `get_teacher_transformer/get_critic_transformer` 改为 `get_transformer(handle, timestep)` + - handle 不包含“语义” + +### 3.3 Timestep sampling policy 回归 method（B） + +- [ ] DMD2Method 内实现 timestep sampling policy： + - `t = torch.randint(0, adapter.num_train_timesteps, ...)`（policy：uniform） + - 然后调用 adapter 的 mechanics： + - `t = adapter.shift_and_clamp_timestep(t)`（mechanics：shift/clamp 语义） +- [ ] `WanAdapter` 去掉 `sample_dmd_timestep()`（或保留为 deprecated wrapper，直到 DMD2Method 完成迁移） + +### 3.4 兼容性与安全落地（降低风险） + +- [ ] 允许“过渡期双接口”以降低一次性重构风险： + - adapter 新增 operation-centric API + - 旧 API 暂时保留为薄 wrapper（在一个 PR 内完成迁移后再删除） +- [ ] 明确哪些行为必须保持一致（不引入训练 drift）： + - forward_context 的 ctx 捕获/恢复方式不改变 + - teacher 的 `transformer_2` boundary 逻辑不变 + - validation 路径不回退到 legacy + +--- + +## 4) 关键接口草案（更具体一些） + +### 4.1 `predict_x0(...)` 的建议签名 + +```text +predict_x0( + handle: RoleHandle, + noisy_latents: Tensor, + timestep: Tensor, + batch: TrainingBatch, + *, + conditional: bool, + attn_kind: Literal["dense", "vsa"], +) -> Tensor +``` + +解释： +- `handle`：由 method 从 bundle 解析得到（`handle = bundle.role("teacher")` 等），adapter 只使用 handle 取 transformer（以及可选 transformer_2） +- `conditional`：选择 `batch.conditional_dict` 或 `batch.unconditional_dict` +- `attn_kind`：选择 `batch.attn_metadata` 或 `batch.attn_metadata_vsa`（以及对应 ctx） + +### 4.2 `backward(...)` 的建议形态 + +```text +ctx = AdapterBackwardContext(timesteps, attn_metadata) +adapter.backward(loss, ctx, grad_accum_rounds=...) +``` + +ctx 不需要包含 role（role 语义由 method 管理；backward 只需要 forward_context 信息）。 + +### 4.3 timestep mechanics（adapter 提供） + +```text +adapter.num_train_timesteps -> int +adapter.shift_and_clamp_timestep(t: Tensor) -> Tensor +``` + +method 决定如何 sample（policy），adapter 负责把 sample 结果转换为本模型家族的 scheduler 语义（mechanics）。 + +--- + +## 5) 风险点 / 需要你参与决策的地方 + +1) **一次性改动范围**：operation-centric API 迁移是否允许保留旧 API wrapper 一段时间？ + - 我建议允许，以降低重构风险；但最终目标是删除旧 API，避免长期双语义。 + +2) **timestep policy 的抽象边界**： + - Phase 2.9 最小只迁 `sample_dmd_timestep`（uniform + shift/clamp） + - 未来如果要更复杂的采样（课程学习），应该放在 `method_config`（Phase 3） + +3) **registry 的注册方式**：使用显式 `ensure_builtin_registrations()` 还是 import side-effect？ + - 我建议显式 `ensure_builtin_registrations()`，避免 import 顺序导致“没注册”这类隐式 bug。 + +--- + +## 6) 备注：为什么 Phase 2.9 不做 `recipe/method_config`？ + +因为本 phase 的核心风险来自“语义边界调整”： +- adapter/method API 变更 +- dispatch/build 结构变更 + +如果同时改 YAML schema（`distill` -> `recipe`）会叠加变量，出现问题时很难定位。 +因此 Phase 2.9 先保证内部语义正确，再在 Phase 3 做 schema 升级与 finetune 接入。 diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md new file mode 100644 index 000000000..26d0110f8 --- /dev/null +++ b/dev/phases/phase_3.md @@ -0,0 +1,237 @@ +# Phase 3：优雅 Dispatch（N+M）+ `recipe` 配置语义升级 + Finetuning 接入 + +Phase 3 的定位：在 Phase 2 已经证明“新 distill 框架可独立运行”的基础上，把它推进到 +“长期可扩展、可承载更多训练 recipe（包括 finetune）”的状态。 + +本 phase 目标（你已拍板）： + +1) **彻底优雅的 dispatch**：避免 `wan+dmd2` 这种硬编码分支；扩展成本从 N×M 降到 N+M。 +2) **YAML schema 升级**：顶层从 `distill` 改为 `recipe`，并新增 `method_config`。 +3) **新增 finetuning 支持**：把 finetune 作为一种 method 接入框架（only student + dataset）。 + +约束： +- 不新增 entry file：继续使用 `fastvideo/training/distillation.py` 作为统一入口。 +- Phase 3 仍遵循边界：**forward/backward context 由 adapter 托管**，method 只实现算法/编排。 + +--- + +## A) Phase 3 交付目标（Definition of Done） + +- `fastvideo/training/distillation.py` 不再出现 `if family==... and method==...` 的组合硬编码； + 而是通过 registry 查找 family/method 组件来构建 runtime。 +- YAML schema 支持 `recipe + models + training + pipeline_config + method_config`： + - `distill:` schema v1 可以选择性兼容（见“风险/决策点”）。 + - 文档与 examples 更新到 schema v2（`recipe`）。 +- 能跑通两个 recipe： + 1) `recipe.method=dmd2`（现有 few-step wan DMD2） + 2) `recipe.method=finetune`（wan finetune 最小版本，only student） +- `method_config` 在代码中**真实生效**（至少 DMD2 的 update policy + guidance scale 使用它）。 + +--- + +## B) TODO List（Review Checklist） + +### B1. 彻底优雅的 dispatch（registry + 通用 builder） + +- [ ] 新增 `fastvideo/distillation/registry.py` + - `register_family(name)` / `register_method(name)` 装饰器 + - `get_family(name)` / `get_method(name)`：带可用项提示的错误信息 + - `ensure_builtin_registrations()`：导入内置 family/method 以完成注册 +- [ ] 新增 `fastvideo/distillation/families/`（model family 插件） + - `fastvideo/distillation/families/__init__.py` + - `fastvideo/distillation/families/wan.py`：`WanFamily`（复用 Phase 2 的加载与数据构建逻辑） +- [ ] 改造 `fastvideo/distillation/builder.py` + - 把当前 `build_wan_dmd2_runtime_from_config()` 收敛为： + - `build_runtime_from_config(cfg: RunConfig) -> DistillRuntime` + - `DistillRuntime.method` 类型从 `DMD2Method` 泛化为 `DistillMethod` +- [ ] 改造入口 `fastvideo/training/distillation.py` + - 入口只做：`cfg = load_run_config()` → `runtime = build_runtime_from_config(cfg)` → `trainer.run(...)` + - 不再写组合分支（`wan+dmd2` 等） + +### B2. YAML schema v2：`recipe` + `method_config` + +- [ ] 更新 spec：`fastvideo/distillation/specs.py` + - 新增 `RecipeSpec(family: str, method: str)` + - 保留 `RoleSpec(family/path/trainable)`，与 Phase 2 兼容 +- [ ] 更新 YAML loader：`fastvideo/distillation/yaml_config.py` + - 新 schema： + - `recipe: {family, method}` + - `method_config: { ... }`（可选，默认 `{}`） + - 入口层推导 `TrainingArgs.mode`： + - `recipe.method == "finetune"` → `ExecutionMode.FINETUNING` + - 其它 method → `ExecutionMode.DISTILLATION` + - （可选）兼容 v1：若仅提供 `distill: {model, method}`： + - 转换为 `recipe.family = distill.model`，`recipe.method = distill.method` + - 打 warning 提示迁移（见“风险/决策点”） + +### B3. `method_config` 接入到 DMD2（让它真正生效） + +- [ ] 更新 `fastvideo/distillation/methods/distribution_matching/dmd2.py` + - `DMD2Method` 构造函数增加 `method_config: dict[str, Any] | None` + - 读取优先级：`method_config` > `training_args` 默认值（用于平滑迁移） + - `generator_update_interval` + - `real_score_guidance_scale` + - （可选）`simulate_generator_forward`（若继续存在） + +### B4. Finetuning method（only student） + +- [ ] 新增 `fastvideo/distillation/methods/fine_tuning/finetune.py` + - `FineTuneMethod(DistillMethod)` + - `bundle.require_roles(["student"])` + - `single_train_step()`： + - `training_batch = adapter.prepare_batch(...)` + - `pred = adapter.student_predict(...)` + - `loss = adapter.finetune_loss(...)` + - update policy： + - `get_optimizers()` / `get_lr_schedulers()` 始终返回 student 的 +- [ ] 在 `fastvideo/distillation/methods/fine_tuning/__init__.py` 暴露该 method（并注册） + +### B5. WanAdapter 增强 finetune primitives（不让 method 管 forward_context） + +- [ ] 更新 `fastvideo/distillation/adapters/wan.py` + - 新增 `_FineTuneAdapter(Protocol)` 所需 primitives： + - `student_predict_for_finetune(batch) -> torch.Tensor` + - `finetune_loss(batch, pred) -> torch.Tensor` + - （如 activation checkpoint/backward 重算需要）`backward_student(loss, ctx, ...)` + - 复用现有的： + - `prepare_batch()`（要求 finetune 路径必须提供真实 `vae_latent`） + - `set_forward_context(...)` 的管理继续留在 adapter + +### B6. examples / outside configs 更新到 schema v2 + +- [ ] 更新 DMD2 YAML（schema v2）： + - `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` + - `distill -> recipe` + - 新增 `method_config`（至少包含 DMD2 的 interval + guidance scale + simulate flag） +- [ ] 新增 finetune YAML（schema v2）： + - `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B.yaml`（命名可再讨论） +- [ ] 更新 `examples/distillation/phase2/README.md` 与脚本说明（指向 `recipe` schema） + +### B7.（可选但推荐）最小单测 + +- [ ] `fastvideo/tests/distillation/test_yaml_schema_v2.py` + - schema v2 能 parse + - （如兼容 v1）schema v1 → v2 conversion 能 parse +- [ ] `fastvideo/tests/distillation/test_registry_dispatch.py` + - registry 能注册并 resolve `wan` + `dmd2` / `finetune` + +--- + +## C) 核心代码设计（具体到类/函数） + +### C1. `registry.py`：N+M 组合的关键 + +目标：入口/Builder 只写一次组合逻辑，不写 `build__()`。 + +建议接口（简化版）： + +- `DistillFamily`（family 插件） + - `build_bundle(cfg) -> ModelBundle` + - `build_shared_components(cfg) -> ...`（如 vae/scheduler） + - `build_adapter(bundle, cfg, shared) -> DistillAdapter` + - `build_validator(bundle, cfg, tracker, shared) -> DistillValidator | None` + - `build_dataloader(cfg) -> DataLoaderLike` + - `build_optimizers_schedulers(bundle, cfg) -> None`（写入 RoleHandle） + +- `DistillMethodFactory`（method 插件） + - `build(bundle, adapter, cfg) -> DistillMethod` + +注册方式： +- `@register_family("wan") class WanFamily: ...` +- `@register_method("dmd2") def build_dmd2(...): ...` +- `@register_method("finetune") def build_finetune(...): ...` + +> 这样新增一个 family 或 method 是一次注册（N+M），不会产生 N×M 组合函数。 + +### C2. 通用 `build_runtime_from_config(cfg)` + +`fastvideo/distillation/builder.py` 收敛为一个通用入口： + +1) `ensure_builtin_registrations()` +2) `family = get_family(cfg.recipe.family)` +3) `method_factory = get_method(cfg.recipe.method)` +4) `bundle = family.build_bundle(cfg)` +5) `family.build_optimizers_schedulers(bundle, cfg)`（只给 trainable roles 创建） +6) `shared = family.build_shared_components(cfg)` +7) `tracker = family.build_tracker(cfg)`（或 builder 复用 training/trackers） +8) `validator = family.build_validator(...)`（可选） +9) `adapter = family.build_adapter(bundle, cfg, shared, validator)` +10) `method = method_factory.build(bundle=bundle, adapter=adapter, cfg=cfg)` +11) `dataloader = family.build_dataloader(cfg)` +12) 返回 `DistillRuntime(training_args, method, dataloader, tracker, start_step=0)` + +### C3. YAML loader（schema v2） + +`fastvideo/distillation/yaml_config.py` 产出新的 `RunConfig`（命名可沿用 `DistillRunConfig`）： + +- `recipe: RecipeSpec` +- `roles/models: dict[str, RoleSpec]` +- `training_args: TrainingArgs` +- `method_config: dict[str, Any]` +- `raw: dict[str, Any]` + +并将 `training_args.mode` 与 `inference_mode` 强制一致（Phase 2 的经验）： +- `finetune` → `FINETUNING` + `inference_mode=False` +- distill methods → `DISTILLATION` + `inference_mode=False` + +### C4. DMD2：method_config 生效点 + +Phase 3 最小要求：以下两个参数从 `method_config` 读取（否则 `method_config` 只是摆设）： +- `generator_update_interval` +- `real_score_guidance_scale` + +兼容策略（建议）：如果 `method_config` 缺失，则回落到 `training_args` 字段，避免破坏旧 YAML。 + +### C5. Finetune：method + adapter contract + +Finetune 的边界： +- method 负责算法编排（loss/optim schedule） +- adapter 负责 forward_context + model/pipeline 差异 + +建议 `_FineTuneAdapter` 的最小 surface： +- `prepare_batch(raw_batch, current_vsa_sparsity=...) -> TrainingBatch` +- `student_predict_for_finetune(batch) -> torch.Tensor` +- `finetune_loss(batch, pred) -> torch.Tensor` +- `backward_student(loss, ctx, grad_accum_rounds=...)`（如果需要） + +> Wan 侧可以复用现有 `TrainingBatch` 字段与 `normalize_dit_input / get_sigmas` 等工具， +> 目标是对齐 legacy `TrainingPipeline._transformer_forward_and_compute_loss()` 的 loss 语义。 + +--- + +## D) 风险点 / 需要你参与决策的地方（遇到会停下讨论） + +1) **schema v1 兼容性**：Phase 3 是否需要继续支持 `distill:`？ + - 选项 A：完全切到 `recipe:`（更干净，但需要一次性改所有 YAML） + - 选项 B：兼容 v1 → v2 conversion + warning（更平滑，但多一点维护负担） + +2) **`method_config` 的形态**：是否需要做 method namespace？ + - 选项 A：flat mapping（推荐）：`method_config: {generator_update_interval: 5, ...}` + - 选项 B：namespaced：`method_config: {dmd2: {...}}`（更显式但更啰嗦） + +3) **finetune 的 validation**：Phase 3 是否要求 finetune 也能 video validation？ + - 最小交付可以先让 finetune 训练跑通，validation 可先关（`log_validation=false`） + - 若要开启，需要确认 `WanValidator` 的 sampling pipeline 是否应随 recipe.method 切换 + +--- + +## E) YAML 小结：flow mapping vs block mapping + +YAML 语法上： + +```yaml +student: {family: wan, path: ..., trainable: true} +``` + +与 + +```yaml +student: + family: wan + path: ... + trainable: true +``` + +在语义上是等价的（都是一个 mapping）。Phase 3 的 examples/docs 我们将统一采用 **block +style**（你偏好的后者），因为更可读、diff 更友好。 + From c6707c813352f7fca64cbc08b25bedcf6713e377 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Mon, 23 Feb 2026 00:45:41 +0000 Subject: [PATCH 069/214] restart thread --- .../finetune_ode_init.slurm | 22 +- .../launch_preprocess_slurm.sh | 4 +- .../preprocess_worker.slurm | 4 +- .../causal_wangame_ode_init/validation.json | 378 ++++++++++++++++-- fastvideo/training/distillation_pipeline.py | 3 +- 5 files changed, 352 insertions(+), 59 deletions(-) diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm index c226012b1..52bcc4ff8 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm @@ -1,8 +1,8 @@ #!/bin/bash #SBATCH --job-name=wg-ode #SBATCH --partition=main -#SBATCH --nodes=8 -#SBATCH --ntasks=8 +#SBATCH --nodes=4 +#SBATCH --ntasks=4 #SBATCH --ntasks-per-node=1 #SBATCH --gres=gpu:8 #SBATCH --cpus-per-task=128 @@ -30,18 +30,21 @@ export TOKENIZERS_PARALLELISM=false echo "MASTER_ADDR: $MASTER_ADDR" echo "NODE_RANK: $NODE_RANK" +RUN_NAME=$(date +"%m%d_%H%M") +echo "RUN_NAME: $RUN_NAME" + # Configs MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" -DATA_DIR="../traindata_0209_1500/ode_init/preprocessed" +DATA_DIR="../traindata_0222_0030/ode_init_mc_with_mouse/preprocessed_wangame" VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" -CKPT_SAFETENSOR="/mnt/weka/home/hao.zhang/mhuo/FastVideo/wangame_1.3b_with_warmup_lr_1e-5/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" +CKPT_SAFETENSOR="/mnt/weka/home/hao.zhang/mhuo/FastVideo/wangame_1.3b_1action_rand_from_scratch/checkpoint-9000/transformer/diffusion_pytorch_model.safetensors" # Training arguments training_args=( --tracker_project_name "wangame_ode_init" - --output_dir "checkpoints/wangame_ode_init" + --output_dir "checkpoints/wangame_ode_init_${RUN_NAME}" --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" - --wandb_run_name "wangame_ode_init_64gpu" + --wandb_run_name "${RUN_NAME}_bs32" --max_train_steps 5000 --train_batch_size 1 --train_sp_batch_size 1 @@ -57,11 +60,11 @@ training_args=( # Parallel arguments parallel_args=( - --num_gpus 64 + --num_gpus 32 --sp_size 1 --tp_size 1 --hsdp_replicate_dim 1 - --hsdp_shard_dim 64 + --hsdp_shard_dim 32 ) # Model arguments @@ -79,8 +82,10 @@ dataset_args=( # Validation arguments validation_args=( --log_validation + --log-visualization --validation_dataset_file "$VALIDATION_DATASET_FILE" --validation_steps 100 + --visualization-steps 100 --validation_sampling_steps "50" --validation_guidance_scale "6.0" ) @@ -104,7 +109,6 @@ miscellaneous_args=( --not_apply_cfg_solver --dit_precision "fp32" --num_euler_timesteps 50 - --ema_start_step 0 ) mkdir -p ode_train_output diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh index 30d8f13f8..a07f03760 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh @@ -5,7 +5,7 @@ mkdir -p preprocess_output # Launch 8 jobs, one for each node (Total 64 GPUs) # Each node processes 8 consecutive files (64 total files / 8 nodes = 8 files per node) -for node_id in {0..2}; do +for node_id in {0..3}; do # Calculate the starting file number for this node start_file=$((node_id * 8)) @@ -17,4 +17,4 @@ for node_id in {0..2}; do $(pwd)/FastVideo/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm $start_file $node_id done -echo "All 3 nodes (24 GPUs) launched successfully!" +echo "All 4 nodes (32 GPUs) launched successfully!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm index 50e3ef126..2aac00253 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm @@ -17,7 +17,7 @@ START_FILE=${1:-1} # Starting file number for this node NODE_ID=${2:-0} # Node identifier (0-7) MODEL_PATH="./Wan2.1-Fun-1.3B-InP-Diffusers" -OUTPUT_BASE="traindata_0209_1500/ode_init/preprocessed" +OUTPUT_BASE="traindata_0222_0030/ode_init_mc_with_mouse/preprocessed_wangame" # Port range calculation base_port=$((29500 + NODE_ID * 100)) @@ -28,7 +28,7 @@ for i in {1..8}; do gpu=${gpu_ids[((i-1))]} file_num=$((START_FILE + i - 1)) - DATA_MERGE_PATH="traindata_0209_1500/mg_ode_init/merge_${file_num}.txt" + DATA_MERGE_PATH="traindata_0222_0030/ode_init_mc_with_mouse/merge_${file_num}.txt" OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" echo "DATA_MERGE_PATH: $DATA_MERGE_PATH" echo "OUTPUT_DIR: $OUTPUT_DIR" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json index 1425f2fef..d151bfc07 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json @@ -1,44 +1,334 @@ -{ - "data": [ - { - "caption": "", - "image_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000000.jpg", - "action_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000000_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "", - "image_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000001.jpg", - "action_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000001_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "00: A", - "image_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000002.jpg", - "action_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000002_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "00: D", - "image_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000003.jpg", - "action_path": "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000003_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - } - ] -} +{ + "data": [ + { + "caption": "00: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000000_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "01: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000001_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "02: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000002_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "03: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000003_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "04: Idle", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000004_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "04: Idle", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000004_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "05: camera down", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000005_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "06: camera up", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000006_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "07: camera right", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000007_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "08: camera left", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000008.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000008_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "09: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000009.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000009_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "10: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000010.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000010_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "11: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000011.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000011_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "12: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000012.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000012_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "13: Idle", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000013.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000013_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "14: Idle", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000014.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000014_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "15: camera down", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000015.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000015_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "16: camera up", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000016.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000016_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "17: camera right", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000017.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000017_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "18: camera left", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000018.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000018_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "19: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000019.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000019_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "20: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000020.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000020_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "21: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000021.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000021_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "22: D", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000022.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000022_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "23: Idle", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000023.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000023_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "24: Idle", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000024.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000024_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "25: camera down", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000025.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000025_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "26: camera up", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000026.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000026_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "27: camera right", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000027.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000027_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "28: camera left", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000028.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000028_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "29: W", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000029.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000029_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "30: S", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000030.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000030_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "31: A", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000031.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000031_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + } + ] +} diff --git a/fastvideo/training/distillation_pipeline.py b/fastvideo/training/distillation_pipeline.py index f1b95a446..d699f7b10 100644 --- a/fastvideo/training/distillation_pipeline.py +++ b/fastvideo/training/distillation_pipeline.py @@ -1453,8 +1453,7 @@ def visualize_intermediate_latents(self, training_batch: TrainingBatch, imageio.mimsave(video_filename, video_frames, fps=24) video_artifact = self.tracker.video( - video, fps=24, format="mp4", - video_filename, caption=latent_key) # change to 16 for Wan2.1 + video, fps=24, format="mp4", caption=latent_key) # change to 16 for Wan2.1 if video_artifact is not None: tracker_loss_dict[latent_key] = video_artifact # Clean up references From 11709a4d51a2f74816a26286fd07c9a3c085939e Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 23 Feb 2026 00:52:32 +0000 Subject: [PATCH 070/214] designing phase 2.9: explain why families registry --- dev/phases/phase_2_9.md | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md index a48ff4f0b..c4a85d082 100644 --- a/dev/phases/phase_2_9.md +++ b/dev/phases/phase_2_9.md @@ -47,6 +47,57 @@ adapter 只保留 scheduler 相关的 mechanics： --- +## 0.5) 常见疑问（为什么这么拆？） + +### backward 为什么不需要传 handle？ + +因为 backward 本质上只做两件事： +1) 让 autograd 通过 **已有的计算图** 反传（参数由计算图决定，不需要再“选择模块”） +2) 在 FastVideo/Wan 这种依赖全局 forward_context 的实现里，在 backward 期间 **恢复必要的上下文** + +因此 adapter 的 backward 更像： +- `adapter.backward(loss, ctx)`：ctx 里带着 forward_context 所需的 timesteps/attn_metadata 等信息 + +handle 是为 **forward/select module** 服务的：比如选择哪个 transformer（以及 transformer_2/boundary）。而 backward 只需要“把当时 forward 用的上下文恢复回来”，不需要知道 handle。 + +如果未来真的出现 role/handle 相关的 backward 特殊逻辑（例如不同模块需要不同 backward context），也更推荐： +- 把 handle 放进 ctx（由 forward 返回），而不是让 adapter.backward(handle, ...) 成为常态 API。 + +### Families 和 Adapter 的关系是什么？ + +- `Family`（build-time）：负责“装配/构建” + - 从 config 解析 roles -> 加载 modules -> 构建 `ModelBundle` + - 构建 shared components（scheduler/vae/tokenizer/...） + - 构建 dataloader/validator/tracker（可选） + - **实例化 adapter**（把 bundle + shared components 注入进去） + +- `Adapter`（runtime/step-time）：负责“操作/执行” + - prepare_batch / forward_context / attention metadata + - `predict_x0(handle, ...)` 这类 operation-centric primitives + - backward 时恢复 forward_context + +一句话：**Family 负责把一堆零件装成可运行的 runtime；Adapter 负责在训练 step 里把 FastVideo 的模型/管线差异变成可复用的操作。** + +### 每个 family 都一定要有自己的 adapter 吗？能否复用？ + +通常会是 “一个模型家族（Wan/SDXL/Flux/...）至少有一个 adapter”，因为： +- raw batch schema 不同 +- forward_context / attention metadata 的机制不同 +- 预测目标（pred noise / pred x0 / flow-matching 等）的转换不同 + +但并不是强制 1:1： +- 一个 family 可能有多个 adapter（例如不同 pipeline 类型） +- 不同 family 也可能共享一个 adapter 基类或部分 mixin（当 runtime mechanics 很像时） + +### 为什么不把 Family 和 Adapter 合并成一个东西？ + +可以合并，但会带来三个长期问题（这也是我们保留 Families 的核心原因）： +1) **高内聚被破坏**：build-time（下载/加载/优化器/dataloader）和 step-time（forward/backward/context）混在一起，类会膨胀成 God object。 +2) **扩展成本回退到 N×M**：新增 method 时更容易被迫去改“合体的 adapter/builder”，最终又会出现各种 if/else 组合分支。 +3) **语义边界变模糊**：method 想保持纯算法；adapter 想保持纯 runtime；family 想保持纯装配。合并后这些边界会互相污染，导致未来很难做 finetune/新 method 的接入。 + +--- + ## 1) Phase 2.9 交付目标（Definition of Done） - `fastvideo/training/distillation.py` 不再硬编码 `wan + dmd2` 分支； From 1e1e11f71ef04b9096da8f9d30e3c55dca94fa0b Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 23 Feb 2026 01:10:44 +0000 Subject: [PATCH 071/214] phase 2.9 init impl --- dev/phases/phase_2_9.md | 24 +- fastvideo/distillation/adapters/wan.py | 127 +++----- fastvideo/distillation/builder.py | 290 ++---------------- fastvideo/distillation/families/__init__.py | 4 + fastvideo/distillation/families/wan.py | 277 +++++++++++++++++ .../methods/distribution_matching/dmd2.py | 93 ++++-- fastvideo/distillation/registry.py | 96 ++++++ fastvideo/distillation/runtime.py | 40 +++ fastvideo/training/distillation.py | 10 +- 9 files changed, 569 insertions(+), 392 deletions(-) create mode 100644 fastvideo/distillation/families/__init__.py create mode 100644 fastvideo/distillation/families/wan.py create mode 100644 fastvideo/distillation/registry.py create mode 100644 fastvideo/distillation/runtime.py diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md index c4a85d082..4d1e97553 100644 --- a/dev/phases/phase_2_9.md +++ b/dev/phases/phase_2_9.md @@ -124,13 +124,13 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform ### 3.1 Registry + Families（优雅 dispatch，N+M） -- [ ] 新增 `fastvideo/distillation/registry.py` +- [x] 新增 `fastvideo/distillation/registry.py` - `register_family(name)` / `register_method(name)` 装饰器 - `get_family(name)` / `get_method(name)`（错误信息包含可用项） - `ensure_builtin_registrations()`：导入内置 family/method 以完成注册 -- [ ] 新增 `fastvideo/distillation/families/` +- [x] 新增 `fastvideo/distillation/families/` - `fastvideo/distillation/families/__init__.py` - - `fastvideo/distillation/families/wan.py`：`WanFamily` + - `fastvideo/distillation/families/wan.py`：`build_wan_family_artifacts` - 从 Phase 2 builder 迁移 Wan-specific build-time 逻辑： - 加载 role modules（transformer/transformer_2/vae 等） - shared components（scheduler/noise_scheduler） @@ -138,7 +138,7 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - tracker - validator（WanValidator） - adapter 实例化（WanAdapter） -- [ ] 改造 `fastvideo/distillation/builder.py` +- [x] 改造 `fastvideo/distillation/builder.py` - 新增/收敛为 `build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime` - `DistillRuntime.method` 类型改为 `DistillMethod`（而不是 `DMD2Method`） - builder 内部逻辑： @@ -146,20 +146,20 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - `method = registry.get_method(cfg.distill.method)` - 调用 family 构建 bundle/adapter/dataloader/tracker - 调用 method factory 构建 DistillMethod -- [ ] 改造入口 `fastvideo/training/distillation.py` +- [x] 改造入口 `fastvideo/training/distillation.py` - 删除 `if cfg.distill.model == "wan" and cfg.distill.method == "dmd2": ...` - 统一走：`runtime = build_runtime_from_config(cfg)` ### 3.2 Adapter API：从 role-centric 收敛到 operation-centric（A） -- [ ] 更新 `fastvideo/distillation/methods/distribution_matching/dmd2.py` +- [x] 更新 `fastvideo/distillation/methods/distribution_matching/dmd2.py` - `_DMD2Adapter` Protocol 改为 operation-centric： - `predict_x0(handle, noisy_latents, timestep, batch, *, conditional, attn_kind)` - `backward(loss, ctx, ...)` - `select_module(handle, module_name, timestep)`（可选：用于 transformer_2/boundary） - `timestep_ops`（见 3.3） - 移除对 `teacher_predict_x0/critic_predict_x0/backward_student/backward_critic` 的直接依赖 -- [ ] 更新 `fastvideo/distillation/adapters/wan.py` +- [x] 更新 `fastvideo/distillation/adapters/wan.py` - 把 `teacher_predict_x0/critic_predict_x0` 合并为 `predict_x0(handle=...)` - 把 `backward_student/backward_critic` 合并为 `backward(loss, ctx, ...)` - 将 `get_teacher_transformer/get_critic_transformer` 改为 `get_transformer(handle, timestep)` @@ -167,18 +167,16 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform ### 3.3 Timestep sampling policy 回归 method（B） -- [ ] DMD2Method 内实现 timestep sampling policy： +- [x] DMD2Method 内实现 timestep sampling policy： - `t = torch.randint(0, adapter.num_train_timesteps, ...)`（policy：uniform） - 然后调用 adapter 的 mechanics： - `t = adapter.shift_and_clamp_timestep(t)`（mechanics：shift/clamp 语义） -- [ ] `WanAdapter` 去掉 `sample_dmd_timestep()`（或保留为 deprecated wrapper，直到 DMD2Method 完成迁移） +- [x] `WanAdapter` 去掉 `sample_dmd_timestep()`（改为提供 `shift_and_clamp_timestep()`） ### 3.4 兼容性与安全落地（降低风险） -- [ ] 允许“过渡期双接口”以降低一次性重构风险： - - adapter 新增 operation-centric API - - 旧 API 暂时保留为薄 wrapper（在一个 PR 内完成迁移后再删除） -- [ ] 明确哪些行为必须保持一致（不引入训练 drift）： +- [x] （选择 direct cut）一次性迁移到 operation-centric API（不保留旧 role-centric API wrapper） +- [x] 明确哪些行为必须保持一致（不引入训练 drift）： - forward_context 的 ctx 捕获/恢复方式不改变 - teacher 的 `transformer_2` boundary 逻辑不变 - validation 路径不回退到 legacy diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index d7ae5d3fd..cc3402f61 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -4,7 +4,7 @@ import copy import gc -from typing import Any +from typing import Any, Literal import torch @@ -29,6 +29,7 @@ from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed from fastvideo.distillation.adapters.base import DistillAdapter +from fastvideo.distillation.bundle import RoleHandle try: from fastvideo.attention.backends.video_sparse_attn import ( @@ -49,13 +50,13 @@ class WanAdapter(DistillAdapter): def __init__( self, *, - bundle: Any, + prompt_handle: RoleHandle, training_args: Any, noise_scheduler: Any, vae: Any, validator: Any | None = None, ) -> None: - self.bundle = bundle + self.prompt_handle = prompt_handle self.training_args = training_args self.noise_scheduler = noise_scheduler self.vae = vae @@ -97,13 +98,20 @@ def _init_dmd2_schedule(self) -> None: self.denoising_step_list = timesteps[1000 - self.denoising_step_list] self.denoising_step_list = self.denoising_step_list.to(self.device) - if getattr(self.training_args, "boundary_ratio", None) is not None: - self.boundary_timestep = float(self.training_args.boundary_ratio) * float( - self.num_train_timestep - ) + boundary_ratio = getattr(self.training_args, "boundary_ratio", None) + if boundary_ratio is not None: + self.boundary_timestep = float(boundary_ratio) * float(self.num_train_timestep) else: self.boundary_timestep = None + @property + def num_train_timesteps(self) -> int: + return int(self.num_train_timestep) + + def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: + timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) + return timestep.clamp(self.min_timestep, self.max_timestep) + def on_train_start(self) -> None: seed = self.training_args.seed if seed is None: @@ -157,7 +165,7 @@ def ensure_negative_conditioning(self) -> None: args_copy = copy.deepcopy(training_args) args_copy.inference_mode = True - student_transformer = self.bundle.role("student").require_module("transformer") + student_transformer = self.prompt_handle.require_module("transformer") prompt_pipeline = WanDMDPipeline.from_pretrained( training_args.model_path, args=args_copy, @@ -400,17 +408,6 @@ def prepare_batch( return training_batch - def sample_dmd_timestep(self, *, device: torch.device) -> torch.Tensor: - timestep = torch.randint( - 0, - self.num_train_timestep, - [1], - device=device, - dtype=torch.long, - ) - timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) - return timestep.clamp(self.min_timestep, self.max_timestep) - def add_noise( self, clean_latents: torch.Tensor, @@ -441,24 +438,18 @@ def _build_distill_input_kwargs( "return_dict": False, } - def _get_teacher_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: - role = self.bundle.role("teacher") - transformer = role.require_module("transformer") - transformer_2 = role.modules.get("transformer_2") + def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: + transformer = handle.require_module("transformer") + transformer_2 = handle.modules.get("transformer_2") if ( transformer_2 is not None and self.boundary_timestep is not None - and float(timestep.item()) < self.boundary_timestep + and float(timestep.item()) < float(self.boundary_timestep) ): return transformer_2 return transformer - def _get_critic_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: - # Phase 1: prefer a single critic transformer unless MoE is fully wired. - role = self.bundle.role("critic") - return role.require_module("transformer") - - def student_rollout(self, batch: TrainingBatch) -> tuple[torch.Tensor, Any]: + def student_rollout(self, handle: RoleHandle, batch: TrainingBatch) -> tuple[torch.Tensor, Any]: device_type = self.device.type dtype = batch.latents.dtype with torch.autocast(device_type, dtype=dtype), set_forward_context( @@ -466,12 +457,12 @@ def student_rollout(self, batch: TrainingBatch) -> tuple[torch.Tensor, Any]: attn_metadata=batch.attn_metadata_vsa, ): if self.training_args.simulate_generator_forward: - pred_x0 = self._student_multi_step_simulation(batch) + pred_x0 = self._student_multi_step_simulation(handle, batch) else: - pred_x0 = self._student_single_step(batch) + pred_x0 = self._student_single_step(handle, batch) return pred_x0, (batch.timesteps, batch.attn_metadata_vsa) - def _student_single_step(self, batch: TrainingBatch) -> torch.Tensor: + def _student_single_step(self, handle: RoleHandle, batch: TrainingBatch) -> torch.Tensor: latents = batch.latents b, t = latents.shape[:2] index = torch.randint( @@ -496,7 +487,7 @@ def _student_single_step(self, batch: TrainingBatch) -> torch.Tensor: timestep, batch.conditional_dict, ) - transformer = self.bundle.role("student").require_module("transformer") + transformer = handle.require_module("transformer") pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) pred_x0 = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), @@ -506,7 +497,7 @@ def _student_single_step(self, batch: TrainingBatch) -> torch.Tensor: ).unflatten(0, pred_noise.shape[:2]) return pred_x0 - def _student_multi_step_simulation(self, batch: TrainingBatch) -> torch.Tensor: + def _student_multi_step_simulation(self, handle: RoleHandle, batch: TrainingBatch) -> torch.Tensor: latents = batch.latents dtype = latents.dtype @@ -527,7 +518,7 @@ def _student_multi_step_simulation(self, batch: TrainingBatch) -> torch.Tensor: noise_latents: list[torch.Tensor] = [] noise_latent_index = target_timestep_idx_int - 1 - transformer = self.bundle.role("student").require_module("transformer") + transformer = handle.require_module("transformer") if max_target_idx > 0: with torch.no_grad(): @@ -587,13 +578,15 @@ def _student_multi_step_simulation(self, batch: TrainingBatch) -> torch.Tensor: batch.dmd_latent_vis_dict["generator_timestep"] = target_timestep.float().detach() return pred_x0 - def teacher_predict_x0( + def predict_x0( self, + handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, *, conditional: bool, + attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: device_type = self.device.type dtype = noisy_latents.dtype @@ -601,39 +594,19 @@ def teacher_predict_x0( if text_dict is None: raise RuntimeError("Missing unconditional_dict; ensure_negative_conditioning() may have failed") - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata, - ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) - transformer = self._get_teacher_transformer(timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) - return pred_x0 + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - def critic_predict_x0( - self, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype with torch.autocast(device_type, dtype=dtype), set_forward_context( current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata, + attn_metadata=attn_metadata, ): - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, - timestep, - batch.conditional_dict, - ) - transformer = self._get_critic_transformer(timestep) + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) + transformer = self._get_transformer(handle, timestep) pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) pred_x0 = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), @@ -645,6 +618,9 @@ def critic_predict_x0( def critic_flow_matching_loss( self, + *, + student: RoleHandle, + critic: RoleHandle, batch: TrainingBatch, ) -> tuple[torch.Tensor, Any, dict[str, Any]]: device_type = self.device.type @@ -655,9 +631,9 @@ def critic_flow_matching_loss( attn_metadata=batch.attn_metadata_vsa, ): if self.training_args.simulate_generator_forward: - generator_pred_x0 = self._student_multi_step_simulation(batch) + generator_pred_x0 = self._student_multi_step_simulation(student, batch) else: - generator_pred_x0 = self._student_single_step(batch) + generator_pred_x0 = self._student_single_step(student, batch) fake_score_timestep = torch.randint( 0, @@ -666,11 +642,7 @@ def critic_flow_matching_loss( device=self.device, dtype=torch.long, ) - fake_score_timestep = shift_timestep( - fake_score_timestep, - self.timestep_shift, - self.num_train_timestep, - ).clamp(self.min_timestep, self.max_timestep) + fake_score_timestep = self.shift_and_clamp_timestep(fake_score_timestep) noise = torch.randn( generator_pred_x0.shape, @@ -693,7 +665,7 @@ def critic_flow_matching_loss( fake_score_timestep, batch.conditional_dict, ) - transformer = self._get_critic_transformer(fake_score_timestep) + transformer = self._get_transformer(critic, fake_score_timestep) pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) target = noise - generator_pred_x0 @@ -706,12 +678,7 @@ def critic_flow_matching_loss( outputs = {"fake_score_latent_vis_dict": batch.fake_score_latent_vis_dict} return flow_matching_loss, (batch.timesteps, batch.attn_metadata), outputs - def backward_student(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: - timesteps, attn_metadata = ctx - with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): - (loss / max(1, int(grad_accum_rounds))).backward() - - def backward_critic(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: timesteps, attn_metadata = ctx with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): (loss / max(1, int(grad_accum_rounds))).backward() diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index fc5bdff87..49e80586c 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -2,282 +2,42 @@ from __future__ import annotations -import os -from dataclasses import dataclass -from typing import Any - -import torch - -from fastvideo.distillation.adapters.wan import WanAdapter -from fastvideo.distillation.bundle import ModelBundle, RoleHandle -from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method +from fastvideo.distillation.registry import get_family, get_method +from fastvideo.distillation.runtime import DistillRuntime from fastvideo.distillation.yaml_config import DistillRunConfig -from fastvideo.models.loader.component_loader import PipelineComponentLoader -from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, -) -from fastvideo.training.activation_checkpoint import ( - apply_activation_checkpointing, -) -from fastvideo.training.trackers import initialize_trackers, Trackers -from fastvideo.training.training_utils import get_scheduler -from fastvideo.utils import maybe_download_model, verify_model_config_and_directory -from fastvideo.distributed import get_world_group - - -@dataclass(slots=True) -class DistillRuntime: - training_args: Any - method: DMD2Method - dataloader: Any - tracker: Any - start_step: int = 0 - - -def _parse_betas(betas: str) -> tuple[float, float]: - beta1, beta2 = (float(x.strip()) for x in betas.split(",")) - return beta1, beta2 - - -def _load_module_from_path( - *, - model_path: str, - module_type: str, - training_args: Any, - mark_teacher_critic: bool = False, -) -> torch.nn.Module: - local_model_path = maybe_download_model(model_path) - config = verify_model_config_and_directory(local_model_path) - - if module_type not in config: - raise ValueError(f"Module {module_type!r} not found in config at {local_model_path}") - - module_info = config[module_type] - if module_info is None: - raise ValueError(f"Module {module_type!r} has null value in config at {local_model_path}") - - transformers_or_diffusers, _architecture = module_info - component_path = os.path.join(local_model_path, module_type) - - if mark_teacher_critic: - training_args._loading_teacher_critic_model = True - try: - module = PipelineComponentLoader.load_module( - module_name=module_type, - component_model_path=component_path, - transformers_or_diffusers=transformers_or_diffusers, - fastvideo_args=training_args, - ) - finally: - if mark_teacher_critic and hasattr(training_args, "_loading_teacher_critic_model"): - del training_args._loading_teacher_critic_model - if not isinstance(module, torch.nn.Module): - raise TypeError(f"Loaded {module_type!r} is not a torch.nn.Module: {type(module)}") - return module +def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: + """Build a distillation runtime from a Phase 2 YAML config. -def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Module: - module.requires_grad_(trainable) - if trainable: - module.train() - else: - module.eval() - return module + This is the Phase 2.9 "elegant dispatch" entry for assembling: + - model family artifacts (bundle/adapter/dataloader/tracker) + - method implementation (algorithm) on top of those artifacts + """ + family_builder = get_family(str(cfg.distill.model)) + artifacts = family_builder(cfg=cfg) -def _build_optimizer_and_scheduler( - *, - role: str, - role_modules: dict[str, torch.nn.Module], - training_args: Any, -) -> tuple[dict[str, torch.optim.Optimizer], dict[str, Any]]: - if role == "critic": - lr = float(getattr(training_args, "fake_score_learning_rate", 0.0) or 0.0) - if lr == 0.0: - lr = float(training_args.learning_rate) - betas = _parse_betas(str(getattr(training_args, "fake_score_betas", training_args.betas))) - scheduler_name = str(getattr(training_args, "fake_score_lr_scheduler", training_args.lr_scheduler)) - else: - lr = float(training_args.learning_rate) - betas = _parse_betas(str(training_args.betas)) - scheduler_name = str(training_args.lr_scheduler) + method_builder = get_method(str(cfg.distill.method)) + method = method_builder(cfg=cfg, bundle=artifacts.bundle, adapter=artifacts.adapter) - params = [] - for module in role_modules.values(): - params.extend([p for p in module.parameters() if p.requires_grad]) - - if not params: - raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") - - optimizer = torch.optim.AdamW( - params, - lr=lr, - betas=betas, - weight_decay=float(getattr(training_args, "weight_decay", 0.0) or 0.0), - eps=1e-8, - ) - - scheduler = get_scheduler( - scheduler_name, - optimizer=optimizer, - num_warmup_steps=int(getattr(training_args, "lr_warmup_steps", 0) or 0), - num_training_steps=int(getattr(training_args, "max_train_steps", 0) or 0), - num_cycles=int(getattr(training_args, "lr_num_cycles", 0) or 0), - power=float(getattr(training_args, "lr_power", 0.0) or 0.0), - min_lr_ratio=float(getattr(training_args, "min_lr_ratio", 0.5) or 0.5), - last_epoch=-1, - ) - - return {"main": optimizer}, {"main": scheduler} - - -def _build_tracker(training_args: Any, *, config: dict[str, Any] | None) -> Any: - world_group = get_world_group() - trackers = list(getattr(training_args, "trackers", [])) - if not trackers and getattr(training_args, "tracker_project_name", ""): - trackers.append(Trackers.WANDB.value) - if world_group.rank != 0: - trackers = [] - - tracker_log_dir = getattr(training_args, "output_dir", "") or os.getcwd() - if trackers: - tracker_log_dir = os.path.join(tracker_log_dir, "tracker") - - tracker_config = config if trackers else None - tracker_run_name = getattr(training_args, "wandb_run_name", "") or None - project = getattr(training_args, "tracker_project_name", "") or "fastvideo" - return initialize_trackers( - trackers, - experiment_name=project, - config=tracker_config, - log_dir=tracker_log_dir, - run_name=tracker_run_name, + return DistillRuntime( + training_args=artifacts.training_args, + method=method, + dataloader=artifacts.dataloader, + tracker=artifacts.tracker, + start_step=int(getattr(artifacts, "start_step", 0) or 0), ) def build_wan_dmd2_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: - training_args = cfg.training_args - roles_cfg = cfg.roles + """Legacy Phase 2 helper kept for compatibility during Phase 2.9 rollout.""" - if getattr(training_args, "seed", None) is None: - raise ValueError("training.seed must be set for distillation") - if not getattr(training_args, "data_path", ""): - raise ValueError("training.data_path must be set for distillation") - - for required in ("student", "teacher", "critic"): - if required not in roles_cfg: - raise ValueError(f"Missing required role {required!r} for wan+dmd2") - - # Load shared components (student base path). - training_args.override_transformer_cls_name = "WanTransformer3DModel" - vae = _load_module_from_path( - model_path=str(training_args.model_path), - module_type="vae", - training_args=training_args, - ) - noise_scheduler = FlowMatchEulerDiscreteScheduler( - shift=float(training_args.pipeline_config.flow_shift or 0.0) - ) - - role_handles: dict[str, RoleHandle] = {} - for role, role_spec in roles_cfg.items(): - if role_spec.family != "wan": - raise ValueError( - f"Phase 2 builder currently supports only wan roles; got {role}={role_spec.family!r}" - ) - - mark_teacher_critic = role in ("teacher", "critic") - transformer = _load_module_from_path( - model_path=role_spec.path, - module_type="transformer", - training_args=training_args, - mark_teacher_critic=mark_teacher_critic, + if cfg.distill.model != "wan" or cfg.distill.method != "dmd2": + raise ValueError( + "build_wan_dmd2_runtime_from_config expects distill.model='wan' " + f"and distill.method='dmd2', got model={cfg.distill.model!r}, " + f"method={cfg.distill.method!r}" ) - modules: dict[str, torch.nn.Module] = {"transformer": transformer} - - # Optional MoE support: allow teacher transformer_2 if present. - if role == "teacher": - try: - transformer_2 = _load_module_from_path( - model_path=role_spec.path, - module_type="transformer_2", - training_args=training_args, - mark_teacher_critic=mark_teacher_critic, - ) - except Exception: - transformer_2 = None - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 - - for name, module in list(modules.items()): - module = _apply_trainable(module, trainable=bool(role_spec.trainable)) - if role_spec.trainable and getattr(training_args, "enable_gradient_checkpointing_type", None): - module = apply_activation_checkpointing( - module, - checkpointing_type=training_args.enable_gradient_checkpointing_type, - ) - modules[name] = module - - optimizers: dict[str, torch.optim.Optimizer] = {} - lr_schedulers: dict[str, Any] = {} - if role_spec.trainable: - optimizers, lr_schedulers = _build_optimizer_and_scheduler( - role=role, - role_modules=modules, - training_args=training_args, - ) - - role_handles[role] = RoleHandle( - modules=modules, - optimizers=optimizers, - lr_schedulers=lr_schedulers, - trainable=bool(role_spec.trainable), - ) - - bundle = ModelBundle(roles=role_handles) - - tracker = _build_tracker(training_args, config=cfg.raw) + return build_runtime_from_config(cfg) - validator = None - if getattr(training_args, "log_validation", False): - from fastvideo.distillation.validators.wan import WanValidator - - validator = WanValidator( - bundle=bundle, - training_args=training_args, - tracker=tracker, - ) - - adapter = WanAdapter( - bundle=bundle, - training_args=training_args, - noise_scheduler=noise_scheduler, - vae=vae, - validator=validator, - ) - - method = DMD2Method(bundle=bundle, adapter=adapter) - - from fastvideo.dataset import build_parquet_map_style_dataloader - from fastvideo.dataset.dataloader.schema import pyarrow_schema_t2v - - text_len = training_args.pipeline_config.text_encoder_configs[0].arch_config.text_len # type: ignore[attr-defined] - _dataset, dataloader = build_parquet_map_style_dataloader( - training_args.data_path, - training_args.train_batch_size, - num_data_workers=training_args.dataloader_num_workers, - parquet_schema=pyarrow_schema_t2v, - cfg_rate=training_args.training_cfg_rate, - drop_last=True, - text_padding_length=int(text_len), - seed=int(training_args.seed or 0), - ) - - return DistillRuntime( - training_args=training_args, - method=method, - dataloader=dataloader, - tracker=tracker, - start_step=0, - ) diff --git a/fastvideo/distillation/families/__init__.py b/fastvideo/distillation/families/__init__.py new file mode 100644 index 000000000..869a647a1 --- /dev/null +++ b/fastvideo/distillation/families/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Model-family build plugins for Phase 2/2.9 distillation.""" + diff --git a/fastvideo/distillation/families/wan.py b/fastvideo/distillation/families/wan.py new file mode 100644 index 000000000..8d8b29e85 --- /dev/null +++ b/fastvideo/distillation/families/wan.py @@ -0,0 +1,277 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from typing import Any + +import torch + +from fastvideo.distributed import get_world_group +from fastvideo.distillation.adapters.wan import WanAdapter +from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.registry import register_family +from fastvideo.distillation.runtime import FamilyArtifacts +from fastvideo.distillation.yaml_config import DistillRunConfig +from fastvideo.models.loader.component_loader import PipelineComponentLoader +from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from fastvideo.training.activation_checkpoint import apply_activation_checkpointing +from fastvideo.training.trackers import initialize_trackers, Trackers +from fastvideo.training.training_utils import get_scheduler +from fastvideo.utils import maybe_download_model, verify_model_config_and_directory + + +def _parse_betas(betas: str) -> tuple[float, float]: + beta1, beta2 = (float(x.strip()) for x in betas.split(",")) + return beta1, beta2 + + +def _load_module_from_path( + *, + model_path: str, + module_type: str, + training_args: Any, + mark_teacher_critic: bool = False, +) -> torch.nn.Module: + local_model_path = maybe_download_model(model_path) + config = verify_model_config_and_directory(local_model_path) + + if module_type not in config: + raise ValueError(f"Module {module_type!r} not found in config at {local_model_path}") + + module_info = config[module_type] + if module_info is None: + raise ValueError(f"Module {module_type!r} has null value in config at {local_model_path}") + + transformers_or_diffusers, _architecture = module_info + component_path = os.path.join(local_model_path, module_type) + + if mark_teacher_critic: + training_args._loading_teacher_critic_model = True + try: + module = PipelineComponentLoader.load_module( + module_name=module_type, + component_model_path=component_path, + transformers_or_diffusers=transformers_or_diffusers, + fastvideo_args=training_args, + ) + finally: + if mark_teacher_critic and hasattr(training_args, "_loading_teacher_critic_model"): + del training_args._loading_teacher_critic_model + + if not isinstance(module, torch.nn.Module): + raise TypeError(f"Loaded {module_type!r} is not a torch.nn.Module: {type(module)}") + return module + + +def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Module: + module.requires_grad_(trainable) + if trainable: + module.train() + else: + module.eval() + return module + + +def _build_optimizer_and_scheduler( + *, + role: str, + role_modules: dict[str, torch.nn.Module], + training_args: Any, +) -> tuple[dict[str, torch.optim.Optimizer], dict[str, Any]]: + if role == "critic": + lr = float(getattr(training_args, "fake_score_learning_rate", 0.0) or 0.0) + if lr == 0.0: + lr = float(training_args.learning_rate) + betas = _parse_betas(str(getattr(training_args, "fake_score_betas", training_args.betas))) + scheduler_name = str( + getattr(training_args, "fake_score_lr_scheduler", training_args.lr_scheduler) + ) + else: + lr = float(training_args.learning_rate) + betas = _parse_betas(str(training_args.betas)) + scheduler_name = str(training_args.lr_scheduler) + + params = [] + for module in role_modules.values(): + params.extend([p for p in module.parameters() if p.requires_grad]) + + if not params: + raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") + + optimizer = torch.optim.AdamW( + params, + lr=lr, + betas=betas, + weight_decay=float(getattr(training_args, "weight_decay", 0.0) or 0.0), + eps=1e-8, + ) + + scheduler = get_scheduler( + scheduler_name, + optimizer=optimizer, + num_warmup_steps=int(getattr(training_args, "lr_warmup_steps", 0) or 0), + num_training_steps=int(getattr(training_args, "max_train_steps", 0) or 0), + num_cycles=int(getattr(training_args, "lr_num_cycles", 0) or 0), + power=float(getattr(training_args, "lr_power", 0.0) or 0.0), + min_lr_ratio=float(getattr(training_args, "min_lr_ratio", 0.5) or 0.5), + last_epoch=-1, + ) + + return {"main": optimizer}, {"main": scheduler} + + +def _build_tracker(training_args: Any, *, config: dict[str, Any] | None) -> Any: + world_group = get_world_group() + trackers = list(getattr(training_args, "trackers", [])) + if not trackers and getattr(training_args, "tracker_project_name", ""): + trackers.append(Trackers.WANDB.value) + if world_group.rank != 0: + trackers = [] + + tracker_log_dir = getattr(training_args, "output_dir", "") or os.getcwd() + if trackers: + tracker_log_dir = os.path.join(tracker_log_dir, "tracker") + + tracker_config = config if trackers else None + tracker_run_name = getattr(training_args, "wandb_run_name", "") or None + project = getattr(training_args, "tracker_project_name", "") or "fastvideo" + return initialize_trackers( + trackers, + experiment_name=project, + config=tracker_config, + log_dir=tracker_log_dir, + run_name=tracker_run_name, + ) + + +@register_family("wan") +def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: + training_args = cfg.training_args + roles_cfg = cfg.roles + + if getattr(training_args, "seed", None) is None: + raise ValueError("training.seed must be set for distillation") + if not getattr(training_args, "data_path", ""): + raise ValueError("training.data_path must be set for distillation") + + # Load shared components (student base path). + training_args.override_transformer_cls_name = "WanTransformer3DModel" + vae = _load_module_from_path( + model_path=str(training_args.model_path), + module_type="vae", + training_args=training_args, + ) + noise_scheduler = FlowMatchEulerDiscreteScheduler( + shift=float(training_args.pipeline_config.flow_shift or 0.0) + ) + + role_handles: dict[str, RoleHandle] = {} + for role, role_spec in roles_cfg.items(): + if role_spec.family != "wan": + raise ValueError( + "Wan family only supports wan roles; " + f"got {role}={role_spec.family!r}" + ) + + mark_teacher_critic = role in ("teacher", "critic") + transformer = _load_module_from_path( + model_path=role_spec.path, + module_type="transformer", + training_args=training_args, + mark_teacher_critic=mark_teacher_critic, + ) + modules: dict[str, torch.nn.Module] = {"transformer": transformer} + + # Optional MoE support: allow teacher transformer_2 if present. + if role == "teacher": + try: + transformer_2 = _load_module_from_path( + model_path=role_spec.path, + module_type="transformer_2", + training_args=training_args, + mark_teacher_critic=mark_teacher_critic, + ) + except Exception: + transformer_2 = None + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + + for name, module in list(modules.items()): + module = _apply_trainable(module, trainable=bool(role_spec.trainable)) + if role_spec.trainable and getattr( + training_args, "enable_gradient_checkpointing_type", None + ): + module = apply_activation_checkpointing( + module, + checkpointing_type=training_args.enable_gradient_checkpointing_type, + ) + modules[name] = module + + optimizers: dict[str, torch.optim.Optimizer] = {} + lr_schedulers: dict[str, Any] = {} + if role_spec.trainable: + optimizers, lr_schedulers = _build_optimizer_and_scheduler( + role=role, + role_modules=modules, + training_args=training_args, + ) + + role_handles[role] = RoleHandle( + modules=modules, + optimizers=optimizers, + lr_schedulers=lr_schedulers, + trainable=bool(role_spec.trainable), + ) + + bundle = ModelBundle(roles=role_handles) + tracker = _build_tracker(training_args, config=cfg.raw) + + validator = None + if getattr(training_args, "log_validation", False): + from fastvideo.distillation.validators.wan import WanValidator + + validator = WanValidator( + bundle=bundle, + training_args=training_args, + tracker=tracker, + ) + + # NOTE: adapter is the family runtime boundary; it may implement multiple + # method-specific protocols via duck typing. + prompt_handle = role_handles.get("student") + if prompt_handle is None: + raise ValueError("Wan family requires a 'student' role for prompt encoding") + adapter = WanAdapter( + prompt_handle=prompt_handle, + training_args=training_args, + noise_scheduler=noise_scheduler, + vae=vae, + validator=validator, + ) + + from fastvideo.dataset import build_parquet_map_style_dataloader + from fastvideo.dataset.dataloader.schema import pyarrow_schema_t2v + + text_len = training_args.pipeline_config.text_encoder_configs[0].arch_config.text_len # type: ignore[attr-defined] + _dataset, dataloader = build_parquet_map_style_dataloader( + training_args.data_path, + training_args.train_batch_size, + num_data_workers=training_args.dataloader_num_workers, + parquet_schema=pyarrow_schema_t2v, + cfg_rate=training_args.training_cfg_rate, + drop_last=True, + text_padding_length=int(text_len), + seed=int(training_args.seed or 0), + ) + + return FamilyArtifacts( + training_args=training_args, + bundle=bundle, + adapter=adapter, + dataloader=dataloader, + tracker=tracker, + start_step=0, + ) diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 19d80cf17..be66253cf 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any, Protocol +from typing import Any, Literal, Protocol import torch import torch.nn.functional as F @@ -12,7 +12,10 @@ ) from fastvideo.distillation.bundle import ModelBundle +from fastvideo.distillation.bundle import RoleHandle from fastvideo.distillation.methods.base import DistillMethod +from fastvideo.distillation.registry import register_method +from fastvideo.distillation.yaml_config import DistillRunConfig class _DMD2Adapter(Protocol): @@ -41,10 +44,14 @@ def prepare_batch( ) -> Any: ... - def student_rollout(self, batch: Any) -> tuple[torch.Tensor, Any]: + def student_rollout(self, handle: RoleHandle, batch: Any) -> tuple[torch.Tensor, Any]: ... - def sample_dmd_timestep(self, *, device: torch.device) -> torch.Tensor: + @property + def num_train_timesteps(self) -> int: + ... + + def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: ... def add_noise( @@ -55,31 +62,28 @@ def add_noise( ) -> torch.Tensor: ... - def teacher_predict_x0( + def predict_x0( self, + handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: Any, *, conditional: bool, + attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: ... - def critic_predict_x0( + def critic_flow_matching_loss( self, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, + *, + student: RoleHandle, + critic: RoleHandle, batch: Any, - ) -> torch.Tensor: - ... - - def critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: - ... - - def backward_student(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + ) -> tuple[torch.Tensor, Any, dict[str, Any]]: ... - def backward_critic(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: ... def log_validation(self, iteration: int) -> None: @@ -108,7 +112,10 @@ def __init__( ) -> None: super().__init__(bundle) bundle.require_roles(["student", "teacher", "critic"]) - if getattr(bundle.role("teacher"), "trainable", False): + self.student = bundle.role("student") + self.teacher = bundle.role("teacher") + self.critic = bundle.role("critic") + if getattr(self.teacher, "trainable", False): raise ValueError("DMD2Method requires models.teacher.trainable=false") self.adapter = adapter self.training_args = adapter.training_args @@ -142,7 +149,14 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor device = generator_pred_x0.device with torch.no_grad(): - timestep = self.adapter.sample_dmd_timestep(device=device) + timestep = torch.randint( + 0, + int(self.adapter.num_train_timesteps), + [1], + device=device, + dtype=torch.long, + ) + timestep = self.adapter.shift_and_clamp_timestep(timestep) noise = torch.randn( generator_pred_x0.shape, @@ -151,18 +165,29 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor ) noisy_latents = self.adapter.add_noise(generator_pred_x0, noise, timestep) - faker_x0 = self.adapter.critic_predict_x0(noisy_latents, timestep, batch) - real_cond_x0 = self.adapter.teacher_predict_x0( + faker_x0 = self.adapter.predict_x0( + self.critic, + noisy_latents, + timestep, + batch, + conditional=True, + attn_kind="dense", + ) + real_cond_x0 = self.adapter.predict_x0( + self.teacher, noisy_latents, timestep, batch, conditional=True, + attn_kind="dense", ) - real_uncond_x0 = self.adapter.teacher_predict_x0( + real_uncond_x0 = self.adapter.predict_x0( + self.teacher, noisy_latents, timestep, batch, conditional=False, + attn_kind="dense", ) real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale @@ -197,11 +222,16 @@ def single_train_step( ) student_ctx = None if update_student: - generator_pred_x0, student_ctx = self.adapter.student_rollout(training_batch) + generator_pred_x0, student_ctx = self.adapter.student_rollout( + self.student, + training_batch, + ) generator_loss = self._dmd_loss(generator_pred_x0, training_batch) fake_score_loss, critic_ctx, critic_outputs = self.adapter.critic_flow_matching_loss( - training_batch + student=self.student, + critic=self.critic, + batch=training_batch, ) total_loss = generator_loss + fake_score_loss @@ -237,7 +267,7 @@ def backward( student_ctx = backward_ctx.get("student_ctx") if student_ctx is None: raise RuntimeError("Missing student backward context") - self.adapter.backward_student( + self.adapter.backward( loss_map["generator_loss"], student_ctx, grad_accum_rounds=grad_accum_rounds, @@ -246,7 +276,7 @@ def backward( critic_ctx = backward_ctx.get("critic_ctx") if critic_ctx is None: raise RuntimeError("Missing critic backward context") - self.adapter.backward_critic( + self.adapter.backward( loss_map["fake_score_loss"], critic_ctx, grad_accum_rounds=grad_accum_rounds, @@ -254,9 +284,9 @@ def backward( def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: optimizers: list[torch.optim.Optimizer] = [] - optimizers.extend(self.bundle.role("critic").optimizers.values()) + optimizers.extend(self.critic.optimizers.values()) if self._should_update_student(iteration): - optimizers.extend(self.bundle.role("student").optimizers.values()) + optimizers.extend(self.student.optimizers.values()) return optimizers def get_lr_schedulers(self, iteration: int) -> list[Any]: @@ -274,3 +304,14 @@ def optimizers_schedulers_step(self, iteration: int) -> None: self._clip_grad_norm(module) super().optimizers_schedulers_step(iteration) + + +@register_method("dmd2") +def build_dmd2_method( + *, + cfg: DistillRunConfig, + bundle: ModelBundle, + adapter: _DMD2Adapter, +) -> DistillMethod: + del cfg + return DMD2Method(bundle=bundle, adapter=adapter) diff --git a/fastvideo/distillation/registry.py b/fastvideo/distillation/registry.py new file mode 100644 index 000000000..1ecd1808f --- /dev/null +++ b/fastvideo/distillation/registry.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, Protocol + +from fastvideo.distillation.bundle import ModelBundle +from fastvideo.distillation.methods.base import DistillMethod +from fastvideo.distillation.runtime import FamilyArtifacts +from fastvideo.distillation.yaml_config import DistillRunConfig + + +class FamilyBuilder(Protocol): + def __call__(self, *, cfg: DistillRunConfig) -> FamilyArtifacts: + ... + + +class MethodBuilder(Protocol): + def __call__( + self, + *, + cfg: DistillRunConfig, + bundle: ModelBundle, + adapter: Any, + ) -> DistillMethod: + ... + + +_FAMILIES: dict[str, FamilyBuilder] = {} +_METHODS: dict[str, MethodBuilder] = {} +_BUILTINS_REGISTERED = False + + +def register_family(name: str) -> Callable[[FamilyBuilder], FamilyBuilder]: + name = str(name).strip() + if not name: + raise ValueError("family name cannot be empty") + + def decorator(builder: FamilyBuilder) -> FamilyBuilder: + if name in _FAMILIES: + raise KeyError(f"Family already registered: {name!r}") + _FAMILIES[name] = builder + return builder + + return decorator + + +def register_method(name: str) -> Callable[[MethodBuilder], MethodBuilder]: + name = str(name).strip() + if not name: + raise ValueError("method name cannot be empty") + + def decorator(builder: MethodBuilder) -> MethodBuilder: + if name in _METHODS: + raise KeyError(f"Method already registered: {name!r}") + _METHODS[name] = builder + return builder + + return decorator + + +def ensure_builtin_registrations() -> None: + global _BUILTINS_REGISTERED + if _BUILTINS_REGISTERED: + return + + # NOTE: keep these imports explicit (no wildcard scanning) so registration + # order is stable and failures are debuggable. + from fastvideo.distillation.families import wan as _wan # noqa: F401 + from fastvideo.distillation.methods.distribution_matching import dmd2 as _dmd2 # noqa: F401 + + _BUILTINS_REGISTERED = True + + +def available_families() -> list[str]: + return sorted(_FAMILIES.keys()) + + +def available_methods() -> list[str]: + return sorted(_METHODS.keys()) + + +def get_family(name: str) -> FamilyBuilder: + ensure_builtin_registrations() + if name not in _FAMILIES: + raise KeyError(f"Unknown family {name!r}. Available: {available_families()}") + return _FAMILIES[name] + + +def get_method(name: str) -> MethodBuilder: + ensure_builtin_registrations() + if name not in _METHODS: + raise KeyError(f"Unknown method {name!r}. Available: {available_methods()}") + return _METHODS[name] + diff --git a/fastvideo/distillation/runtime.py b/fastvideo/distillation/runtime.py new file mode 100644 index 000000000..1051002bb --- /dev/null +++ b/fastvideo/distillation/runtime.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from fastvideo.fastvideo_args import TrainingArgs + +from fastvideo.distillation.bundle import ModelBundle +from fastvideo.distillation.methods.base import DistillMethod + + +@dataclass(slots=True) +class FamilyArtifacts: + """Build-time outputs produced by a model family plugin. + + A family is responsible for loading modules, constructing a `ModelBundle`, + and assembling shared components needed by runtime adapters, dataloaders, + validators, and trackers. + """ + + training_args: TrainingArgs + bundle: ModelBundle + adapter: Any + dataloader: Any + tracker: Any + start_step: int = 0 + + +@dataclass(slots=True) +class DistillRuntime: + """Fully assembled runtime for `DistillTrainer.run()`.""" + + training_args: TrainingArgs + method: DistillMethod + dataloader: Any + tracker: Any + start_step: int = 0 + diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 9079f59c5..e3720754f 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -25,7 +25,7 @@ def run_distillation_from_config( DistillCheckpointConfig, DistillCheckpointManager, ) - from fastvideo.distillation.builder import build_wan_dmd2_runtime_from_config + from fastvideo.distillation.builder import build_runtime_from_config from fastvideo.distillation.yaml_config import load_distill_run_config cfg = load_distill_run_config(config_path) @@ -41,13 +41,7 @@ def run_distillation_from_config( training_args.sp_size, ) - if cfg.distill.model == "wan" and cfg.distill.method == "dmd2": - runtime = build_wan_dmd2_runtime_from_config(cfg) - else: - raise ValueError( - f"Unsupported distillation config: model={cfg.distill.model!r}, " - f"method={cfg.distill.method!r}" - ) + runtime = build_runtime_from_config(cfg) if dry_run: logger.info("Dry-run: config parsed and runtime built successfully.") From 70157b41eacb79002ed51a95136e54f312ac1d58 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 23 Feb 2026 01:31:13 +0000 Subject: [PATCH 072/214] wan adapter decouple --- fastvideo/distillation/adapters/wan.py | 222 ++---------------- .../methods/distribution_matching/dmd2.py | 205 ++++++++++++++-- 2 files changed, 212 insertions(+), 215 deletions(-) diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index cc3402f61..e59208656 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -72,32 +72,17 @@ def __init__( self.negative_prompt_embeds: torch.Tensor | None = None self.negative_prompt_attention_mask: torch.Tensor | None = None - self._init_dmd2_schedule() + self._init_timestep_mechanics() def _get_training_dtype(self) -> torch.dtype: return torch.bfloat16 - def _init_dmd2_schedule(self) -> None: + def _init_timestep_mechanics(self) -> None: self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) - self.denoising_step_list = torch.tensor( - self.training_args.pipeline_config.dmd_denoising_steps, - dtype=torch.long, - device=self.device, - ) - if getattr(self.training_args, "warp_denoising_step", False): - timesteps = torch.cat( - ( - self.noise_scheduler.timesteps.to("cpu"), - torch.tensor([0], dtype=torch.float32), - ) - ).to(self.device) - self.denoising_step_list = timesteps[1000 - self.denoising_step_list] - self.denoising_step_list = self.denoising_step_list.to(self.device) - boundary_ratio = getattr(self.training_args, "boundary_ratio", None) if boundary_ratio is not None: self.boundary_timestep = float(boundary_ratio) * float(self.num_train_timestep) @@ -449,135 +434,6 @@ def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch. return transformer_2 return transformer - def student_rollout(self, handle: RoleHandle, batch: TrainingBatch) -> tuple[torch.Tensor, Any]: - device_type = self.device.type - dtype = batch.latents.dtype - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata_vsa, - ): - if self.training_args.simulate_generator_forward: - pred_x0 = self._student_multi_step_simulation(handle, batch) - else: - pred_x0 = self._student_single_step(handle, batch) - return pred_x0, (batch.timesteps, batch.attn_metadata_vsa) - - def _student_single_step(self, handle: RoleHandle, batch: TrainingBatch) -> torch.Tensor: - latents = batch.latents - b, t = latents.shape[:2] - index = torch.randint( - 0, - len(self.denoising_step_list), - [1], - device=self.device, - dtype=torch.long, - ) - timestep = self.denoising_step_list[index] - batch.dmd_latent_vis_dict["generator_timestep"] = timestep - - noise = torch.randn(latents.shape, device=self.device, dtype=latents.dtype) - noisy_latent = self.noise_scheduler.add_noise( - latents.flatten(0, 1), - noise.flatten(0, 1), - timestep, - ).unflatten(0, (b, t)) - - input_kwargs = self._build_distill_input_kwargs( - noisy_latent, - timestep, - batch.conditional_dict, - ) - transformer = handle.require_module("transformer") - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latent.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) - return pred_x0 - - def _student_multi_step_simulation(self, handle: RoleHandle, batch: TrainingBatch) -> torch.Tensor: - latents = batch.latents - dtype = latents.dtype - - target_timestep_idx = torch.randint( - 0, - len(self.denoising_step_list), - [1], - device=self.device, - dtype=torch.long, - ) - target_timestep_idx_int = int(target_timestep_idx.item()) - target_timestep = self.denoising_step_list[target_timestep_idx] - - current_noise_latents = torch.randn(latents.shape, device=self.device, dtype=dtype) - current_noise_latents_copy = current_noise_latents.clone() - - max_target_idx = len(self.denoising_step_list) - 1 - noise_latents: list[torch.Tensor] = [] - noise_latent_index = target_timestep_idx_int - 1 - - transformer = handle.require_module("transformer") - - if max_target_idx > 0: - with torch.no_grad(): - for step_idx in range(max_target_idx): - current_timestep = self.denoising_step_list[step_idx] - current_timestep_tensor = current_timestep * torch.ones( - 1, device=self.device, dtype=torch.long - ) - input_kwargs = self._build_distill_input_kwargs( - current_noise_latents, - current_timestep_tensor, - batch.conditional_dict, - ) - pred_flow = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_clean = pred_noise_to_pred_video( - pred_noise=pred_flow.flatten(0, 1), - noise_input_latent=current_noise_latents.flatten(0, 1), - timestep=current_timestep_tensor, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_flow.shape[:2]) - - next_timestep = self.denoising_step_list[step_idx + 1] - next_timestep_tensor = next_timestep * torch.ones( - 1, device=self.device, dtype=torch.long - ) - noise = torch.randn( - latents.shape, device=self.device, dtype=pred_clean.dtype - ) - b, t = pred_clean.shape[:2] - current_noise_latents = self.noise_scheduler.add_noise( - pred_clean.flatten(0, 1), - noise.flatten(0, 1), - next_timestep_tensor, - ).unflatten(0, (b, t)) - noise_latents.append(current_noise_latents.clone()) - - if noise_latent_index >= 0: - if noise_latent_index >= len(noise_latents): - raise RuntimeError("noise_latent_index is out of bounds") - noisy_input = noise_latents[noise_latent_index] - else: - noisy_input = current_noise_latents_copy - - input_kwargs = self._build_distill_input_kwargs( - noisy_input, - target_timestep, - batch.conditional_dict, - ) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_input.flatten(0, 1), - timestep=target_timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) - - batch.dmd_latent_vis_dict["generator_timestep"] = target_timestep.float().detach() - return pred_x0 - def predict_x0( self, handle: RoleHandle, @@ -616,67 +472,37 @@ def predict_x0( ).unflatten(0, pred_noise.shape[:2]) return pred_x0 - def critic_flow_matching_loss( + def predict_noise( self, - *, - student: RoleHandle, - critic: RoleHandle, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, batch: TrainingBatch, - ) -> tuple[torch.Tensor, Any, dict[str, Any]]: + *, + conditional: bool, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: device_type = self.device.type - dtype = batch.latents.dtype - - with torch.no_grad(), torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata_vsa, - ): - if self.training_args.simulate_generator_forward: - generator_pred_x0 = self._student_multi_step_simulation(student, batch) - else: - generator_pred_x0 = self._student_single_step(student, batch) - - fake_score_timestep = torch.randint( - 0, - self.num_train_timestep, - [1], - device=self.device, - dtype=torch.long, - ) - fake_score_timestep = self.shift_and_clamp_timestep(fake_score_timestep) + dtype = noisy_latents.dtype + text_dict = batch.conditional_dict if conditional else getattr(batch, "unconditional_dict", None) + if text_dict is None: + raise RuntimeError("Missing unconditional_dict; ensure_negative_conditioning() may have failed") - noise = torch.randn( - generator_pred_x0.shape, - device=self.device, - dtype=generator_pred_x0.dtype, - ) - b, t = generator_pred_x0.shape[:2] - noisy_x0 = self.noise_scheduler.add_noise( - generator_pred_x0.flatten(0, 1), - noise.flatten(0, 1), - fake_score_timestep, - ).unflatten(0, (b, t)) + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") with torch.autocast(device_type, dtype=dtype), set_forward_context( current_timestep=batch.timesteps, - attn_metadata=batch.attn_metadata, + attn_metadata=attn_metadata, ): - input_kwargs = self._build_distill_input_kwargs( - noisy_x0, - fake_score_timestep, - batch.conditional_dict, - ) - transformer = self._get_transformer(critic, fake_score_timestep) + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) + transformer = self._get_transformer(handle, timestep) pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - - target = noise - generator_pred_x0 - flow_matching_loss = torch.mean((pred_noise - target) ** 2) - - batch.fake_score_latent_vis_dict = { - "generator_pred_video": generator_pred_x0, - "fake_score_timestep": fake_score_timestep, - } - outputs = {"fake_score_latent_vis_dict": batch.fake_score_latent_vis_dict} - return flow_matching_loss, (batch.timesteps, batch.attn_metadata), outputs + return pred_noise def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: timesteps, attn_metadata = ctx diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index be66253cf..e4b5301c8 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -44,9 +44,6 @@ def prepare_batch( ) -> Any: ... - def student_rollout(self, handle: RoleHandle, batch: Any) -> tuple[torch.Tensor, Any]: - ... - @property def num_train_timesteps(self) -> int: ... @@ -74,13 +71,16 @@ def predict_x0( ) -> torch.Tensor: ... - def critic_flow_matching_loss( + def predict_noise( self, - *, - student: RoleHandle, - critic: RoleHandle, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, batch: Any, - ) -> tuple[torch.Tensor, Any, dict[str, Any]]: + *, + conditional: bool, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: ... def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: @@ -119,6 +119,10 @@ def __init__( raise ValueError("DMD2Method requires models.teacher.trainable=false") self.adapter = adapter self.training_args = adapter.training_args + self._simulate_generator_forward = bool( + getattr(self.training_args, "simulate_generator_forward", False) + ) + self._denoising_step_list: torch.Tensor | None = None def on_train_start(self) -> None: self.adapter.on_train_start() @@ -144,6 +148,179 @@ def _clip_grad_norm(self, module: torch.nn.Module) -> float: ) return float(grad_norm.item()) if grad_norm is not None else 0.0 + def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: + if self._denoising_step_list is not None and self._denoising_step_list.device == device: + return self._denoising_step_list + + raw = getattr(self.training_args.pipeline_config, "dmd_denoising_steps", None) + if not raw: + raise ValueError("pipeline_config.dmd_denoising_steps must be set for DMD2 distillation") + + steps = torch.tensor(raw, dtype=torch.long, device=device) + + if getattr(self.training_args, "warp_denoising_step", False): + noise_scheduler = getattr(self.adapter, "noise_scheduler", None) + if noise_scheduler is None: + raise ValueError("warp_denoising_step requires adapter.noise_scheduler.timesteps") + + timesteps = torch.cat( + ( + noise_scheduler.timesteps.to("cpu"), + torch.tensor([0], dtype=torch.float32), + ) + ).to(device) + steps = timesteps[1000 - steps] + + self._denoising_step_list = steps + return steps + + def _sample_rollout_timestep(self, device: torch.device) -> torch.Tensor: + step_list = self._get_denoising_step_list(device) + index = torch.randint( + 0, + len(step_list), + [1], + device=device, + dtype=torch.long, + ) + return step_list[index] + + def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: + latents = batch.latents + device = latents.device + dtype = latents.dtype + step_list = self._get_denoising_step_list(device) + + if not self._simulate_generator_forward: + timestep = self._sample_rollout_timestep(device) + noise = torch.randn(latents.shape, device=device, dtype=dtype) + noisy_latents = self.adapter.add_noise(latents, noise, timestep) + pred_x0 = self.adapter.predict_x0( + self.student, + noisy_latents, + timestep, + batch, + conditional=True, + attn_kind="vsa", + ) + batch.dmd_latent_vis_dict["generator_timestep"] = timestep + return pred_x0 + + target_timestep_idx = torch.randint( + 0, + len(step_list), + [1], + device=device, + dtype=torch.long, + ) + target_timestep_idx_int = int(target_timestep_idx.item()) + target_timestep = step_list[target_timestep_idx] + + current_noise_latents = torch.randn(latents.shape, device=device, dtype=dtype) + current_noise_latents_copy = current_noise_latents.clone() + + max_target_idx = len(step_list) - 1 + noise_latents: list[torch.Tensor] = [] + noise_latent_index = target_timestep_idx_int - 1 + + if max_target_idx > 0: + with torch.no_grad(): + for step_idx in range(max_target_idx): + current_timestep = step_list[step_idx] + current_timestep_tensor = current_timestep * torch.ones( + 1, device=device, dtype=torch.long + ) + + pred_clean = self.adapter.predict_x0( + self.student, + current_noise_latents, + current_timestep_tensor, + batch, + conditional=True, + attn_kind="vsa", + ) + + next_timestep = step_list[step_idx + 1] + next_timestep_tensor = next_timestep * torch.ones( + 1, device=device, dtype=torch.long + ) + noise = torch.randn(latents.shape, device=device, dtype=pred_clean.dtype) + current_noise_latents = self.adapter.add_noise( + pred_clean, + noise, + next_timestep_tensor, + ) + noise_latents.append(current_noise_latents.clone()) + + if noise_latent_index >= 0: + if noise_latent_index >= len(noise_latents): + raise RuntimeError("noise_latent_index is out of bounds") + noisy_input = noise_latents[noise_latent_index] + else: + noisy_input = current_noise_latents_copy + + if with_grad: + pred_x0 = self.adapter.predict_x0( + self.student, + noisy_input, + target_timestep, + batch, + conditional=True, + attn_kind="vsa", + ) + else: + with torch.no_grad(): + pred_x0 = self.adapter.predict_x0( + self.student, + noisy_input, + target_timestep, + batch, + conditional=True, + attn_kind="vsa", + ) + + batch.dmd_latent_vis_dict["generator_timestep"] = target_timestep.float().detach() + return pred_x0 + + def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: + with torch.no_grad(): + generator_pred_x0 = self._student_rollout(batch, with_grad=False) + + device = generator_pred_x0.device + fake_score_timestep = torch.randint( + 0, + int(self.adapter.num_train_timesteps), + [1], + device=device, + dtype=torch.long, + ) + fake_score_timestep = self.adapter.shift_and_clamp_timestep(fake_score_timestep) + + noise = torch.randn( + generator_pred_x0.shape, + device=device, + dtype=generator_pred_x0.dtype, + ) + noisy_x0 = self.adapter.add_noise(generator_pred_x0, noise, fake_score_timestep) + + pred_noise = self.adapter.predict_noise( + self.critic, + noisy_x0, + fake_score_timestep, + batch, + conditional=True, + attn_kind="dense", + ) + target = noise - generator_pred_x0 + flow_matching_loss = torch.mean((pred_noise - target) ** 2) + + batch.fake_score_latent_vis_dict = { + "generator_pred_video": generator_pred_x0, + "fake_score_timestep": fake_score_timestep, + } + outputs = {"fake_score_latent_vis_dict": batch.fake_score_latent_vis_dict} + return flow_matching_loss, (batch.timesteps, batch.attn_metadata), outputs + def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor: guidance_scale = float(getattr(self.training_args, "real_score_guidance_scale", 1.0)) device = generator_pred_x0.device @@ -222,17 +399,11 @@ def single_train_step( ) student_ctx = None if update_student: - generator_pred_x0, student_ctx = self.adapter.student_rollout( - self.student, - training_batch, - ) + generator_pred_x0 = self._student_rollout(training_batch, with_grad=True) + student_ctx = (training_batch.timesteps, training_batch.attn_metadata_vsa) generator_loss = self._dmd_loss(generator_pred_x0, training_batch) - fake_score_loss, critic_ctx, critic_outputs = self.adapter.critic_flow_matching_loss( - student=self.student, - critic=self.critic, - batch=training_batch, - ) + fake_score_loss, critic_ctx, critic_outputs = self._critic_flow_matching_loss(training_batch) total_loss = generator_loss + fake_score_loss loss_map = { From f12732c0a429228cbba6c8716497421fcf8e4721 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 23 Feb 2026 01:37:39 +0000 Subject: [PATCH 073/214] removing dmd in wan adapters --- dev/phases/phase_2_9.md | 16 +++++++++++++--- fastvideo/distillation/adapters/wan.py | 9 +++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md index 4d1e97553..3ff5b9753 100644 --- a/dev/phases/phase_2_9.md +++ b/dev/phases/phase_2_9.md @@ -6,7 +6,8 @@ Phase 2 已经实现了“新框架可独立运行（不依赖 legacy distill pi 本 Phase 2.9 的目标是先把语义边界收敛好（A+B+Families），并把 dispatch 做到真正优雅（N+M）， 让 Phase 3 只需要在此基础上加 config schema 与新 method，而不需要再动 entrypoint/builder 的组合逻辑。 -本文件只做**代码层面的设计**（不写代码），后续实现过程中如果有小调整会回填；遇到重大风险会停下讨论。 +本文件最初用于 **Phase 2.9 的代码层面设计**；目前 Phase 2.9 已实现，下面 checklist 已同步打勾。 +后续如果在实践过程中发现有小调整，会继续回填；遇到重大风险会停下讨论。 --- @@ -106,6 +107,8 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - 不再暴露 `teacher_*` / `critic_*` / `student_*` 专用函数给 method 使用 - DMD2Method 通过通用操作（如 `predict_x0(handle=...)`）完成 teacher/critic/student 的调用 - DMD2 的 timestep sampling policy 从 adapter 迁移到 method（最少把 `sample_dmd_timestep()` 挪走）。 +- few-step rollout 的 step list / simulate 逻辑从 adapter 迁移到 method（未来应进一步移到 `method_config`）。 +- `WanAdapter` 不应包含 method-specific 命名/概念（例如不应依赖 `*DMD*Pipeline` 这类算法命名）。 - Phase 2 的训练行为/结果应尽可能保持一致（同 config 下 loss 形态、validation 产物趋势不应漂移）。 --- @@ -115,8 +118,8 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - 不做 YAML schema v2（`recipe` + `method_config`）升级（留到 Phase 3）。 - 不新增 finetune method（留到 Phase 3）。 - 不新增新模型家族（Phase 2.9 只整理 Wan）。 -- 不追求把所有 DMD2 逻辑从 adapter 中抠干净（例如 critic loss 里 student rollout 的复用）； - Phase 2.9 先解决“role-centric API + policy 泄漏”这两个最大痛点。 +- 不追求把所有“validation / sample / prompt encode”的实现都完全脱离 pipeline（Phase 2.9 先保证训练路径独立可跑）； + 但 adapter 层需要避免 method-specific policy/rollout 泄漏，并避免 method-specific 命名耦合。 --- @@ -164,6 +167,9 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - 把 `backward_student/backward_critic` 合并为 `backward(loss, ctx, ...)` - 将 `get_teacher_transformer/get_critic_transformer` 改为 `get_transformer(handle, timestep)` - handle 不包含“语义” +- [x] `fastvideo/distillation/adapters/wan.py` 不再出现 method-specific 命名 + - 移除 `WanDMDPipeline` 依赖，prompt encoding 改用 `WanPipeline` + - adapter 内不再出现 `dmd/DMD` 字眼（避免“命名耦合”） ### 3.3 Timestep sampling policy 回归 method（B） @@ -172,6 +178,10 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - 然后调用 adapter 的 mechanics： - `t = adapter.shift_and_clamp_timestep(t)`（mechanics：shift/clamp 语义） - [x] `WanAdapter` 去掉 `sample_dmd_timestep()`（改为提供 `shift_and_clamp_timestep()`） +- [x] few-step rollout policy 回归 method + - denoising step list / warp mapping / multi-step simulate 全部由 `DMD2Method` 管理 + - adapter 只提供单步 primitives（如 `predict_x0` / `predict_noise` / `add_noise`） + - TODO(Phase 3): 将 `pipeline_config.dmd_denoising_steps` 迁移到 `method_config`，避免“pipeline_config 承载算法语义” ### 3.4 兼容性与安全落地（降低风险） diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index e59208656..75dab6fe1 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -19,7 +19,7 @@ from fastvideo.models.utils import pred_noise_to_pred_video from fastvideo.pipelines import TrainingBatch from fastvideo.pipelines.pipeline_batch_info import ForwardBatch -from fastvideo.pipelines.basic.wan.wan_dmd_pipeline import WanDMDPipeline +from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline from fastvideo.training.training_utils import ( compute_density_for_timestep_sampling, get_sigmas, @@ -44,7 +44,7 @@ class WanAdapter(DistillAdapter): """ Phase 1 target adapter: provide Wan-specific primitives without calling - legacy distillation pipeline algorithm helpers (e.g. `_dmd_forward`). + legacy distillation pipeline algorithm helpers (e.g. pipeline-private forward wrappers). """ def __init__( @@ -151,7 +151,7 @@ def ensure_negative_conditioning(self) -> None: args_copy.inference_mode = True student_transformer = self.prompt_handle.require_module("transformer") - prompt_pipeline = WanDMDPipeline.from_pretrained( + prompt_pipeline = WanPipeline.from_pretrained( training_args.model_path, args=args_copy, inference_mode=True, @@ -331,9 +331,6 @@ def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: "encoder_attention_mask": neg_mask, } - training_batch.dmd_latent_vis_dict = {} - training_batch.fake_score_latent_vis_dict = {} - training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) return training_batch From 8853110b2f58dfb8c34a0ec4157326760eae7c14 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 23 Feb 2026 02:22:34 +0000 Subject: [PATCH 074/214] phase2.9: adapter ang families decouple from dmd --- dev/phases/phase_2_9.md | 4 + fastvideo/distillation/families/wan.py | 60 ------------ .../methods/distribution_matching/dmd2.py | 93 +++++++++++++++++++ 3 files changed, 97 insertions(+), 60 deletions(-) diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md index 3ff5b9753..1b495ff62 100644 --- a/dev/phases/phase_2_9.md +++ b/dev/phases/phase_2_9.md @@ -109,6 +109,7 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - DMD2 的 timestep sampling policy 从 adapter 迁移到 method（最少把 `sample_dmd_timestep()` 挪走）。 - few-step rollout 的 step list / simulate 逻辑从 adapter 迁移到 method（未来应进一步移到 `method_config`）。 - `WanAdapter` 不应包含 method-specific 命名/概念（例如不应依赖 `*DMD*Pipeline` 这类算法命名）。 +- optimizer/scheduler 的创建归属 method（update policy），family 不再出现 DMD2/critic 专属超参（例如 `fake_score_*`）。 - Phase 2 的训练行为/结果应尽可能保持一致（同 config 下 loss 形态、validation 产物趋势不应漂移）。 --- @@ -190,6 +191,9 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - forward_context 的 ctx 捕获/恢复方式不改变 - teacher 的 `transformer_2` boundary 逻辑不变 - validation 路径不回退到 legacy +- [x] Wan family 不再创建 optimizers/schedulers + - `fastvideo/distillation/families/wan.py` 只负责加载 modules + 构建 `ModelBundle` + - `DMD2Method` 在 init 时为 student/critic 创建 optimizers/schedulers（复用 TrainingArgs 字段，未来迁移到 `method_config`） --- diff --git a/fastvideo/distillation/families/wan.py b/fastvideo/distillation/families/wan.py index 8d8b29e85..cac4ed209 100644 --- a/fastvideo/distillation/families/wan.py +++ b/fastvideo/distillation/families/wan.py @@ -19,15 +19,9 @@ ) from fastvideo.training.activation_checkpoint import apply_activation_checkpointing from fastvideo.training.trackers import initialize_trackers, Trackers -from fastvideo.training.training_utils import get_scheduler from fastvideo.utils import maybe_download_model, verify_model_config_and_directory -def _parse_betas(betas: str) -> tuple[float, float]: - beta1, beta2 = (float(x.strip()) for x in betas.split(",")) - return beta1, beta2 - - def _load_module_from_path( *, model_path: str, @@ -75,54 +69,6 @@ def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Mo return module -def _build_optimizer_and_scheduler( - *, - role: str, - role_modules: dict[str, torch.nn.Module], - training_args: Any, -) -> tuple[dict[str, torch.optim.Optimizer], dict[str, Any]]: - if role == "critic": - lr = float(getattr(training_args, "fake_score_learning_rate", 0.0) or 0.0) - if lr == 0.0: - lr = float(training_args.learning_rate) - betas = _parse_betas(str(getattr(training_args, "fake_score_betas", training_args.betas))) - scheduler_name = str( - getattr(training_args, "fake_score_lr_scheduler", training_args.lr_scheduler) - ) - else: - lr = float(training_args.learning_rate) - betas = _parse_betas(str(training_args.betas)) - scheduler_name = str(training_args.lr_scheduler) - - params = [] - for module in role_modules.values(): - params.extend([p for p in module.parameters() if p.requires_grad]) - - if not params: - raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") - - optimizer = torch.optim.AdamW( - params, - lr=lr, - betas=betas, - weight_decay=float(getattr(training_args, "weight_decay", 0.0) or 0.0), - eps=1e-8, - ) - - scheduler = get_scheduler( - scheduler_name, - optimizer=optimizer, - num_warmup_steps=int(getattr(training_args, "lr_warmup_steps", 0) or 0), - num_training_steps=int(getattr(training_args, "max_train_steps", 0) or 0), - num_cycles=int(getattr(training_args, "lr_num_cycles", 0) or 0), - power=float(getattr(training_args, "lr_power", 0.0) or 0.0), - min_lr_ratio=float(getattr(training_args, "min_lr_ratio", 0.5) or 0.5), - last_epoch=-1, - ) - - return {"main": optimizer}, {"main": scheduler} - - def _build_tracker(training_args: Any, *, config: dict[str, Any] | None) -> Any: world_group = get_world_group() trackers = list(getattr(training_args, "trackers", [])) @@ -212,12 +158,6 @@ def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: optimizers: dict[str, torch.optim.Optimizer] = {} lr_schedulers: dict[str, Any] = {} - if role_spec.trainable: - optimizers, lr_schedulers = _build_optimizer_and_scheduler( - role=role, - role_modules=modules, - training_args=training_args, - ) role_handles[role] = RoleHandle( modules=modules, diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index e4b5301c8..fbbc09e8f 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -9,6 +9,7 @@ from fastvideo.training.training_utils import ( clip_grad_norm_while_handling_failing_dtensor_cases, + get_scheduler, ) from fastvideo.distillation.bundle import ModelBundle @@ -115,14 +116,106 @@ def __init__( self.student = bundle.role("student") self.teacher = bundle.role("teacher") self.critic = bundle.role("critic") + if not getattr(self.student, "trainable", False): + raise ValueError("DMD2Method requires models.student.trainable=true") if getattr(self.teacher, "trainable", False): raise ValueError("DMD2Method requires models.teacher.trainable=false") + if not getattr(self.critic, "trainable", False): + raise ValueError("DMD2Method requires models.critic.trainable=true") self.adapter = adapter self.training_args = adapter.training_args self._simulate_generator_forward = bool( getattr(self.training_args, "simulate_generator_forward", False) ) self._denoising_step_list: torch.Tensor | None = None + self._init_optimizers_and_schedulers() + + def _parse_betas(self, raw: Any, *, where: str) -> tuple[float, float]: + if raw is None: + raise ValueError(f"Missing betas for {where}") + if isinstance(raw, (tuple, list)) and len(raw) == 2: + return float(raw[0]), float(raw[1]) + if isinstance(raw, str): + parts = [p.strip() for p in raw.split(",") if p.strip()] + if len(parts) != 2: + raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {raw!r}") + return float(parts[0]), float(parts[1]) + raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {type(raw).__name__}") + + def _build_role_optimizer_and_scheduler( + self, + *, + role: str, + handle: RoleHandle, + learning_rate: float, + betas: tuple[float, float], + scheduler_name: str, + ) -> None: + modules = handle.modules + params: list[torch.nn.Parameter] = [] + for module in modules.values(): + params.extend([p for p in module.parameters() if p.requires_grad]) + if not params: + raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") + + optimizer = torch.optim.AdamW( + params, + lr=float(learning_rate), + betas=betas, + weight_decay=float(getattr(self.training_args, "weight_decay", 0.0) or 0.0), + eps=1e-8, + ) + + scheduler = get_scheduler( + str(scheduler_name), + optimizer=optimizer, + num_warmup_steps=int(getattr(self.training_args, "lr_warmup_steps", 0) or 0), + num_training_steps=int(getattr(self.training_args, "max_train_steps", 0) or 0), + num_cycles=int(getattr(self.training_args, "lr_num_cycles", 0) or 0), + power=float(getattr(self.training_args, "lr_power", 0.0) or 0.0), + min_lr_ratio=float(getattr(self.training_args, "min_lr_ratio", 0.5) or 0.5), + last_epoch=-1, + ) + + handle.optimizers = {"main": optimizer} + handle.lr_schedulers = {"main": scheduler} + + def _init_optimizers_and_schedulers(self) -> None: + training_args = self.training_args + + # Student optimizer/scheduler (default training hyperparams). + student_lr = float(getattr(training_args, "learning_rate", 0.0) or 0.0) + student_betas = self._parse_betas( + getattr(training_args, "betas", None), + where="training.betas", + ) + student_sched = str(getattr(training_args, "lr_scheduler", "constant")) + self._build_role_optimizer_and_scheduler( + role="student", + handle=self.student, + learning_rate=student_lr, + betas=student_betas, + scheduler_name=student_sched, + ) + + # Critic optimizer/scheduler (DMD2-specific overrides). + critic_lr = float(getattr(training_args, "fake_score_learning_rate", 0.0) or 0.0) + if critic_lr == 0.0: + critic_lr = student_lr + + critic_betas_raw = getattr(training_args, "fake_score_betas", None) + if critic_betas_raw is None: + critic_betas_raw = getattr(training_args, "betas", None) + critic_betas = self._parse_betas(critic_betas_raw, where="training.fake_score_betas") + + critic_sched = str(getattr(training_args, "fake_score_lr_scheduler", None) or student_sched) + self._build_role_optimizer_and_scheduler( + role="critic", + handle=self.critic, + learning_rate=critic_lr, + betas=critic_betas, + scheduler_name=critic_sched, + ) def on_train_start(self) -> None: self.adapter.on_train_start() From 074f559a4b9af02817f6f150f282f0a8c14457c7 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 23 Feb 2026 02:32:54 +0000 Subject: [PATCH 075/214] doc for every file --- fastvideo/distillation/doc/README.md | 58 +++++++++++++++++++ fastvideo/distillation/doc/__init__.md | 14 +++++ .../distillation/doc/adapters/__init__.md | 8 +++ fastvideo/distillation/doc/adapters/base.md | 17 ++++++ fastvideo/distillation/doc/adapters/wan.md | 32 ++++++++++ fastvideo/distillation/doc/builder.md | 19 ++++++ fastvideo/distillation/doc/bundle.md | 22 +++++++ fastvideo/distillation/doc/checkpoint.md | 28 +++++++++ .../distillation/doc/families/__init__.md | 15 +++++ fastvideo/distillation/doc/families/wan.md | 35 +++++++++++ .../distillation/doc/methods/__init__.md | 13 +++++ fastvideo/distillation/doc/methods/base.md | 22 +++++++ .../doc/methods/consistency_model/__init__.md | 9 +++ .../methods/distribution_matching/__init__.md | 14 +++++ .../doc/methods/distribution_matching/dmd2.md | 45 ++++++++++++++ .../doc/methods/fine_tuning/__init__.md | 14 +++++ .../knowledge_distillation/__init__.md | 11 ++++ fastvideo/distillation/doc/outside/README.md | 15 +++++ .../distill_wan2.1_t2v_1.3B_dmd2_8steps.md | 51 ++++++++++++++++ fastvideo/distillation/doc/registry.md | 22 +++++++ fastvideo/distillation/doc/runtime.md | 27 +++++++++ fastvideo/distillation/doc/specs.md | 20 +++++++ fastvideo/distillation/doc/trainer.md | 27 +++++++++ .../distillation/doc/validators/__init__.md | 9 +++ fastvideo/distillation/doc/validators/base.md | 13 +++++ fastvideo/distillation/doc/validators/wan.md | 23 ++++++++ fastvideo/distillation/doc/yaml_config.md | 30 ++++++++++ 27 files changed, 613 insertions(+) create mode 100644 fastvideo/distillation/doc/README.md create mode 100644 fastvideo/distillation/doc/__init__.md create mode 100644 fastvideo/distillation/doc/adapters/__init__.md create mode 100644 fastvideo/distillation/doc/adapters/base.md create mode 100644 fastvideo/distillation/doc/adapters/wan.md create mode 100644 fastvideo/distillation/doc/builder.md create mode 100644 fastvideo/distillation/doc/bundle.md create mode 100644 fastvideo/distillation/doc/checkpoint.md create mode 100644 fastvideo/distillation/doc/families/__init__.md create mode 100644 fastvideo/distillation/doc/families/wan.md create mode 100644 fastvideo/distillation/doc/methods/__init__.md create mode 100644 fastvideo/distillation/doc/methods/base.md create mode 100644 fastvideo/distillation/doc/methods/consistency_model/__init__.md create mode 100644 fastvideo/distillation/doc/methods/distribution_matching/__init__.md create mode 100644 fastvideo/distillation/doc/methods/distribution_matching/dmd2.md create mode 100644 fastvideo/distillation/doc/methods/fine_tuning/__init__.md create mode 100644 fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md create mode 100644 fastvideo/distillation/doc/outside/README.md create mode 100644 fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md create mode 100644 fastvideo/distillation/doc/registry.md create mode 100644 fastvideo/distillation/doc/runtime.md create mode 100644 fastvideo/distillation/doc/specs.md create mode 100644 fastvideo/distillation/doc/trainer.md create mode 100644 fastvideo/distillation/doc/validators/__init__.md create mode 100644 fastvideo/distillation/doc/validators/base.md create mode 100644 fastvideo/distillation/doc/validators/wan.md create mode 100644 fastvideo/distillation/doc/yaml_config.md diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md new file mode 100644 index 000000000..f746874c4 --- /dev/null +++ b/fastvideo/distillation/doc/README.md @@ -0,0 +1,58 @@ +# Distillation docs (file-by-file) + +本目录用于帮助 reviewer/贡献者快速理解 `fastvideo/distillation/` 的 Phase 2/2.9 架构。 + +设计原则（对应 Phase 2.9）： +- **Trainer** 只做 infra（loop/accum/日志/ckpt/validate 调用），不包含算法策略。 +- **Method** 只做算法（loss + update policy + 需要哪些 roles）。 +- **Family** 只做装配（build-time：加载 modules、构建 bundle/adapter/dataloader/validator/tracker）。 +- **Adapter** 只做运行时 primitive（step-time：prepare_batch/forward_context/predict/backward 等）， + API 以 operation 为中心，不以 role 为中心（避免 role 爆炸）。 + +快速入口（从运行到训练）： +`fastvideo/training/distillation.py` → `yaml_config.load_distill_run_config()` → +`builder.build_runtime_from_config()` → `registry.get_family()/get_method()` → +`FamilyArtifacts + DistillMethod` → `DistillTrainer.run()` + +--- + +## Index + +### Core +- `__init__.md` +- `yaml_config.md` +- `specs.md` +- `registry.md` +- `builder.md` +- `runtime.md` +- `bundle.md` +- `trainer.md` +- `checkpoint.md` + +### adapters/ +- `adapters/__init__.md` +- `adapters/base.md` +- `adapters/wan.md` + +### families/ +- `families/__init__.md` +- `families/wan.md` + +### methods/ +- `methods/__init__.md` +- `methods/base.md` +- `methods/distribution_matching/__init__.md` +- `methods/distribution_matching/dmd2.md` +- `methods/consistency_model/__init__.md` +- `methods/knowledge_distillation/__init__.md` +- `methods/fine_tuning/__init__.md` + +### validators/ +- `validators/__init__.md` +- `validators/base.md` +- `validators/wan.md` + +### outside/ +- `outside/README.md` +- `outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md` + diff --git a/fastvideo/distillation/doc/__init__.md b/fastvideo/distillation/doc/__init__.md new file mode 100644 index 000000000..126834da2 --- /dev/null +++ b/fastvideo/distillation/doc/__init__.md @@ -0,0 +1,14 @@ +# `fastvideo/distillation/__init__.py` + +**目的** +- 提供 distillation 子系统的最小“公共入口”，避免上层到处 import 内部实现细节。 + +**当前导出** +- `DistillTrainer`：训练 loop（infra only） +- `ModelBundle` / `RoleHandle`：multi-role 模型与训练状态容器 + +**设计意图** +- 让上层（例如 `fastvideo/training/distillation.py`）只依赖稳定 API： + - `DistillTrainer.run(method, dataloader, ...)` + - `bundle.role("student")` 等 role 访问模式 + diff --git a/fastvideo/distillation/doc/adapters/__init__.md b/fastvideo/distillation/doc/adapters/__init__.md new file mode 100644 index 000000000..6f061234d --- /dev/null +++ b/fastvideo/distillation/doc/adapters/__init__.md @@ -0,0 +1,8 @@ +# `fastvideo/distillation/adapters/__init__.py` + +**目的** +- 统一导出 adapter 抽象基类 `DistillAdapter`，避免上层直接 import 具体实现。 + +**当前导出** +- `DistillAdapter`（见 `adapters/base.md`） + diff --git a/fastvideo/distillation/doc/adapters/base.md b/fastvideo/distillation/doc/adapters/base.md new file mode 100644 index 000000000..deee3abb2 --- /dev/null +++ b/fastvideo/distillation/doc/adapters/base.md @@ -0,0 +1,17 @@ +# `fastvideo/distillation/adapters/base.py` + +**目的** +- 定义 adapter 的最小抽象接口。 + +**当前最小契约** +- `prepare_batch(raw_batch, current_vsa_sparsity=...) -> TrainingBatch` + +**为什么接口这么小？** +- adapter 的“完整能力”通常是 **method-specific protocol**（duck typing），例如： + - `predict_x0(handle, ...)` + - `predict_noise(handle, ...)` + - `add_noise(...)` + - `backward(loss, ctx, ...)` +- 这些能力应由具体 method 在自己的 `Protocol` 里声明（例如 `DMD2Method` 的 `_DMD2Adapter`）， + 从而保持 adapter 的基类稳定、method 的需求显式可读。 + diff --git a/fastvideo/distillation/doc/adapters/wan.md b/fastvideo/distillation/doc/adapters/wan.md new file mode 100644 index 000000000..cf8d0b264 --- /dev/null +++ b/fastvideo/distillation/doc/adapters/wan.md @@ -0,0 +1,32 @@ +# `fastvideo/distillation/adapters/wan.py` + +**定位** +- `WanAdapter` 是 Wan family 的 runtime 边界： + - 把 FastVideo/Wan 的 batch schema、forward_context、attention metadata 等细节 + 封装为一组 **operation-centric primitives** + - 不实现任何 method 的 rollout policy / step list / loss（这些属于 method） + +**关键依赖** +- `RoleHandle`：adapter 不认识 role 字符串，method 传 handle 进来，adapter 只用 handle 拿模块。 +- `fastvideo.forward_context.set_forward_context`：Wan forward/backward 依赖全局上下文。 +- attention metadata builder（VSA / VMOBA）与 `envs.FASTVIDEO_ATTENTION_BACKEND`。 + +**主要 API（被 method 通过 protocol 调用）** +- `prepare_batch(...) -> TrainingBatch` + - 处理 raw_batch → latents/noise/timesteps/sigmas + - 构建 `conditional_dict` / `unconditional_dict`（含 negative prompt embeds） + - 构建 attention metadata（dense / vsa 两套） +- `predict_x0(handle, noisy_latents, timestep, batch, conditional, attn_kind) -> Tensor` +- `predict_noise(handle, noisy_latents, timestep, batch, conditional, attn_kind) -> Tensor` +- `add_noise(clean_latents, noise, timestep) -> Tensor` +- `shift_and_clamp_timestep(t) -> Tensor` + `num_train_timesteps` +- `backward(loss, ctx, grad_accum_rounds=...)` + +**关于 negative/unconditional conditioning** +- `ensure_negative_conditioning()` 只做 prompt encoding（无 denoise）。 +- 为避免算法命名耦合，prompt encoding 使用 `WanPipeline`（而不是带 method 语义的 pipeline 名称）。 + +**边界（Phase 2.9）** +- ✅ adapter 不保存/管理 few-step 的 denoising step list，也不决定 rollout 策略。 +- ✅ adapter 不区分 `student/teacher/critic` 的专用方法；只提供通用操作，role 语义由 method 管理。 + diff --git a/fastvideo/distillation/doc/builder.md b/fastvideo/distillation/doc/builder.md new file mode 100644 index 000000000..9f8b080b5 --- /dev/null +++ b/fastvideo/distillation/doc/builder.md @@ -0,0 +1,19 @@ +# `fastvideo/distillation/builder.py` + +**目的** +- 把 YAML config（`DistillRunConfig`）装配成一个可运行的 `DistillRuntime`： + - `family` 负责 build-time 产物（`FamilyArtifacts`） + - `method` 负责算法（`DistillMethod`） + +**关键 API** +- `build_runtime_from_config(cfg) -> DistillRuntime` + - `family_builder = registry.get_family(cfg.distill.model)` + - `method_builder = registry.get_method(cfg.distill.method)` + - `method = method_builder(cfg=cfg, bundle=artifacts.bundle, adapter=artifacts.adapter)` +- `build_wan_dmd2_runtime_from_config(cfg)` + - Phase 2.9 期间保留的兼容函数（最终可删除） + +**边界** +- ✅ 这里不写 `if model==... and method==...` 的 N×M 组合逻辑。 +- ✅ 这里只做“装配”，不包含训练 loop / loss / rollout / optimizer policy。 + diff --git a/fastvideo/distillation/doc/bundle.md b/fastvideo/distillation/doc/bundle.md new file mode 100644 index 000000000..c46416fb0 --- /dev/null +++ b/fastvideo/distillation/doc/bundle.md @@ -0,0 +1,22 @@ +# `fastvideo/distillation/bundle.py` + +**目的** +- 用统一的数据结构表达 “多角色（roles）参与的训练/蒸馏”： + - roles 是字符串 key（`"student"`, `"teacher"`, `"critic"`, `"reward"`, ...） + - 每个 role 对应一个 `RoleHandle` + +**关键类型** +- `RoleHandle` + - `modules: dict[str, nn.Module]`：该 role 持有的模块（例如 `transformer`） + - `optimizers: dict[str, Optimizer]` / `lr_schedulers: dict[str, Any]` + - `trainable: bool` + - `require_module(name)`：强制获取模块（缺失则报错） +- `ModelBundle` + - `roles: dict[str, RoleHandle]` + - `require_roles([...])`：method 在构造时校验依赖的 role 是否齐全 + - `role(name)`：获取 handle + +**Phase 2.9 约定** +- family 负责 **load modules + 设置 trainable** 并创建 `ModelBundle` +- method 负责 **(按算法) 创建 optimizers/schedulers** 并写回对应的 `RoleHandle` + diff --git a/fastvideo/distillation/doc/checkpoint.md b/fastvideo/distillation/doc/checkpoint.md new file mode 100644 index 000000000..27478d32a --- /dev/null +++ b/fastvideo/distillation/doc/checkpoint.md @@ -0,0 +1,28 @@ +# `fastvideo/distillation/checkpoint.py` + +**目的** +- Phase 2 的 role-based checkpoint/save-resume 管理： + - 按 role 保存/恢复 modules、optimizers、schedulers + - 可选保存 dataloader 状态（如果 dataloader 是 stateful） + - 保存 RNG（全局 RNG + adapter/validator 暴露的额外 generator） + +**关键类型** +- `DistillCheckpointConfig` + - `save_steps` / `keep_last` +- `DistillCheckpointManager` + - `maybe_resume(resume_from_checkpoint=...) -> step | None` + - `maybe_save(step)` + - `save_final(step)` + +**关键机制** +- 只对 `handle.trainable == True` 的 role 保存 optimizer/scheduler 状态。 +- 使用 `torch.distributed.checkpoint (dcp)` 做分布式 checkpoint。 +- `resume_from_checkpoint` 支持： + - `checkpoint-` 目录 + - `checkpoint-/dcp` + - `output_dir`（自动选择最新 checkpoint） + +**与 Method 的关系** +- 该文件假设：训练开始前 `RoleHandle.optimizers/lr_schedulers` 已经就绪。 + Phase 2.9 开始，它们通常由 method（例如 `DMD2Method`）在构造时创建并写回 handle。 + diff --git a/fastvideo/distillation/doc/families/__init__.md b/fastvideo/distillation/doc/families/__init__.md new file mode 100644 index 000000000..c04261efa --- /dev/null +++ b/fastvideo/distillation/doc/families/__init__.md @@ -0,0 +1,15 @@ +# `fastvideo/distillation/families/__init__.py` + +**目的** +- families 是 build-time 插件层： + - 从 YAML config 读取 role spec + - 加载模型模块（transformer/vae/...） + - 构建 `ModelBundle` + - 构建 adapter / dataloader / tracker / validator + +**为什么需要 families？** +- 把 “装配/加载/数据/分布式细节” 与 “算法/rollout/loss/update policy” 分离： + - family 专注 build-time 高内聚 + - method 专注算法高内聚 + - entrypoint/builder 不需要 N×M 组合逻辑 + diff --git a/fastvideo/distillation/doc/families/wan.md b/fastvideo/distillation/doc/families/wan.md new file mode 100644 index 000000000..bbffa1095 --- /dev/null +++ b/fastvideo/distillation/doc/families/wan.md @@ -0,0 +1,35 @@ +# `fastvideo/distillation/families/wan.py` + +**定位** +- `@register_family("wan")` 的 build-time 插件： + - 负责把 YAML config 装配成 `FamilyArtifacts` + - 包含 Wan 特有的模块加载、shared components、dataloader schema 等逻辑 + +**产物** +- `FamilyArtifacts(training_args, bundle, adapter, dataloader, tracker, start_step)` + +**主要职责** +1) **加载 shared components** + - `vae`：从 student 的 base `model_path` 加载 + - `noise_scheduler`：`FlowMatchEulerDiscreteScheduler(shift=flow_shift)` +2) **按 roles 加载 transformer 模块** + - 对每个 role：加载 `transformer`（teacher 可选 `transformer_2`） + - 根据 `RoleSpec.trainable` 设置 `requires_grad` + `train()/eval()` + - 可选开启 activation checkpoint（仅对 trainable role） +3) **构建 bundle / adapter / dataloader** + - `bundle = ModelBundle(roles=role_handles)` + - `adapter = WanAdapter(prompt_handle=student_handle, ...)` + - dataloader：parquet + `pyarrow_schema_t2v` +4) **tracker / validator（可选）** + - tracker：`initialize_trackers(...)`（rank0 才启用） + - validator：`WanValidator`（当 `training_args.log_validation=true`） + +**Phase 2.9 的关键变化** +- ✅ family 不再创建 optimizers/schedulers。 + - 这类 update policy（哪些 role 训练、各自超参）属于 method/算法语义。 + - 当前由 `DMD2Method` 在初始化时创建并写回 `RoleHandle.optimizers/lr_schedulers`。 + +**注意 / TODO** +- YAML 中目前仍使用 `training.fake_score_*` 这类字段作为 DMD2 的 critic 超参来源； + Phase 3 计划把它们迁移到 `method_config`，进一步减少 “training_args 承载算法语义”。 + diff --git a/fastvideo/distillation/doc/methods/__init__.md b/fastvideo/distillation/doc/methods/__init__.md new file mode 100644 index 000000000..509718d50 --- /dev/null +++ b/fastvideo/distillation/doc/methods/__init__.md @@ -0,0 +1,13 @@ +# `fastvideo/distillation/methods/__init__.py` + +**目的** +- 提供 method 层（算法层）的统一入口与可发现性。 + +**当前导出** +- `DistillMethod`：算法基类（抽象） +- `DMD2Method`：distribution matching 目录下的一个具体方法实现 + +**设计意图** +- method 层应当是 **模型无关** 的（不 import 具体 pipeline/模型实现）； + 任何 family 细节都通过 adapter primitives（protocol）注入。 + diff --git a/fastvideo/distillation/doc/methods/base.md b/fastvideo/distillation/doc/methods/base.md new file mode 100644 index 000000000..199e110f1 --- /dev/null +++ b/fastvideo/distillation/doc/methods/base.md @@ -0,0 +1,22 @@ +# `fastvideo/distillation/methods/base.py` + +**定位** +- `DistillMethod` 是算法层抽象： + - 负责实现 `single_train_step()`（loss 构造） + - 负责定义 update policy（哪些 optimizer/scheduler 在何时 step） + - 不负责训练 loop（由 `DistillTrainer` 承担） + +**关键点** +- `DistillMethod` 持有 `bundle: ModelBundle`，并把所有 role 的 modules 放进 + `self.role_modules: ModuleDict`，便于 DDP/FSDP/ckpt 系统统一发现参数。 + +**需要子类实现的抽象方法** +- `single_train_step(batch, iteration, current_vsa_sparsity=...)` +- `get_optimizers(iteration)` +- `get_lr_schedulers(iteration)` + +**默认实现** +- `backward()`：对 `loss_map["total_loss"]` 做 backward（子类可覆写以处理多 ctx） +- `optimizers_schedulers_step()`：按 `get_optimizers/get_lr_schedulers` 的结果 step +- `optimizers_zero_grad()`：对当前 iteration 的 optimizers 清梯度 + diff --git a/fastvideo/distillation/doc/methods/consistency_model/__init__.md b/fastvideo/distillation/doc/methods/consistency_model/__init__.md new file mode 100644 index 000000000..e4ce40650 --- /dev/null +++ b/fastvideo/distillation/doc/methods/consistency_model/__init__.md @@ -0,0 +1,9 @@ +# `fastvideo/distillation/methods/consistency_model/__init__.py` + +**状态** +- 当前是占位目录（`__all__ = []`），用于未来加入 Consistency Model（CM）相关方法。 + +**期望的演进方向** +- 通过 `@register_method("cm")`（示例）注册具体实现。 +- method 只包含算法与 update policy；family/adapter 提供运行时 primitives。 + diff --git a/fastvideo/distillation/doc/methods/distribution_matching/__init__.md b/fastvideo/distillation/doc/methods/distribution_matching/__init__.md new file mode 100644 index 000000000..09dd705db --- /dev/null +++ b/fastvideo/distillation/doc/methods/distribution_matching/__init__.md @@ -0,0 +1,14 @@ +# `fastvideo/distillation/methods/distribution_matching/__init__.py` + +**定位** +- distribution matching 类方法的集合目录。 + +**当前实现** +- `DMD2Method`（见 `dmd2.md`） + +**扩展** +- 新增方法时建议保持： + - 算法逻辑在 method + - family/model 细节通过 adapter protocol 注入 + - 注册通过 `@register_method("")` + diff --git a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md new file mode 100644 index 000000000..201942c75 --- /dev/null +++ b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md @@ -0,0 +1,45 @@ +# `fastvideo/distillation/methods/distribution_matching/dmd2.py` + +**定位** +- `DMD2Method`：DMD2 distillation 的算法实现（method layer）。 +- 该文件可以出现 DMD2/critic/fake_score 等算法术语；这些语义不应泄漏到 adapter/family。 + +**依赖与边界** +- ✅ 不 import 任何具体模型/管线实现（Wan/SDXL/...）。 +- ✅ 只依赖： + - `ModelBundle`/`RoleHandle`（获取 student/teacher/critic） + - adapter 提供的 primitives（通过 `_DMD2Adapter` Protocol） + +**需要的 roles** +- `student`：可训练（trainable=true） +- `teacher`：冻结（trainable=false） +- `critic`：可训练（trainable=true） + +**算法结构（高层）** +1) `prepare_batch()`：交给 adapter +2) student 更新（按 `generator_update_interval` 控制频率） + - 先做 student rollout 得到 `generator_pred_x0` + - 再计算 DMD loss（teacher CFG 引导 + critic 输出构造梯度） +3) critic 更新（每 step） + - 使用 student rollout（no-grad）构造 flow matching loss +4) backward + - 由于 Wan 的 forward_context 约束，需要 adapter.backward(loss, ctx) + +**few-step rollout policy（Phase 2.9）** +- rollout 的 step list / simulate 逻辑由 method 管理： + - `pipeline_config.dmd_denoising_steps` → `_get_denoising_step_list()` + - `simulate_generator_forward` 控制单步/多步模拟路径 + - 可选 `warp_denoising_step`（通过 adapter.noise_scheduler.timesteps duck-typing） +- adapter 只提供单步 primitives： + - `predict_x0()` / `predict_noise()` / `add_noise()` + +**optimizer/scheduler 的归属（Phase 2.9）** +- `DMD2Method` 在初始化时创建并写回： + - student 的 optimizer/scheduler：使用 `training.learning_rate/betas/lr_scheduler` + - critic 的 optimizer/scheduler：优先使用 `training.fake_score_*` 覆盖（否则回退到 student） +- 这样 Wan family 可以完全不“懂” DMD2 的 critic 超参，从 build-time 层面解耦。 + +**配置语义的 TODO（Phase 3）** +- 目前仍从 `training_args` 读取 DMD2/critic 专属字段（例如 `fake_score_*`、`dmd_denoising_steps`）。 + Phase 3 计划引入 `method_config`，把这些算法超参从 `training:` / `pipeline_config:` 中迁移出去。 + diff --git a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md new file mode 100644 index 000000000..3d0362cd3 --- /dev/null +++ b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md @@ -0,0 +1,14 @@ +# `fastvideo/distillation/methods/fine_tuning/__init__.py` + +**状态** +- 当前是占位目录（`__all__ = []`），用于未来加入 finetuning（可视为只有 student 的特殊训练 recipe）。 + +**与 distillation 框架的关系** +- finetune 可以复用同一套： + - YAML config（models 里只提供 student） + - family（加载 student 模块与数据） + - trainer（infra） +- method 则实现： + - loss（SFT/finetune objective） + - optimizer/scheduler policy + diff --git a/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md b/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md new file mode 100644 index 000000000..793d3cdef --- /dev/null +++ b/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md @@ -0,0 +1,11 @@ +# `fastvideo/distillation/methods/knowledge_distillation/__init__.py` + +**状态** +- 当前是占位目录（`__all__ = []`），用于未来加入经典 KD 类方法（logit/feature matching 等）。 + +**期望的扩展方式** +- 新增 KD method 时： + - method 定义需要哪些 roles（student/teacher/aux_teacher/...） + - family 只负责加载这些 roles 的 modules 并构建 bundle/adapter + - optimizer/scheduler 由 method 创建并写回 handle + diff --git a/fastvideo/distillation/doc/outside/README.md b/fastvideo/distillation/doc/outside/README.md new file mode 100644 index 000000000..1bc818db5 --- /dev/null +++ b/fastvideo/distillation/doc/outside/README.md @@ -0,0 +1,15 @@ +# `fastvideo/distillation/outside/README.md` + +**目的** +- 解释 `fastvideo/distillation/outside/` 的存在理由： + - Phase 2/2.9 期间，我们需要引入新的 YAML config，但又不想直接改动主 repo 的 + `fastvideo/configs/` 树（避免冲突/侵入式修改）。 + +**约定** +- Phase 2 entrypoint（`fastvideo/training/distillation.py`）只接受**真实路径**： + - `--config fastvideo/distillation/outside/fastvideo/configs/distillation/.yaml` +- `outside/` 只放 data/config（不要放可 import 的 Python 代码）。 + +**推荐目录结构** +- `fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml` + diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md new file mode 100644 index 000000000..6687eaea5 --- /dev/null +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md @@ -0,0 +1,51 @@ +# `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` + +这是一个 Phase 2/2.9 的可运行示例：**Wan few-step distillation（8 steps）+ DMD2**。 + +--- + +## 顶层结构 + +- `distill:` + - `model: wan` → registry dispatch 到 `families/wan.py` + - `method: dmd2` → registry dispatch 到 `methods/distribution_matching/dmd2.py` +- `models:` + - `student / teacher / critic` 三个 roles（role 名称本身由 method 解释语义） + - 每个 role 指定： + - `family`（默认可省略，继承 `distill.model`） + - `path`（权重路径） + - `trainable`（是否训练） +- `training:` + - 主要复用 `TrainingArgs.from_kwargs()` 的字段集合（batch/shape/steps/logging 等） +- `pipeline_config:` + - Phase 2 允许 inline 提供（也可用 `pipeline_config_path` 指向文件） + +--- + +## 关键语义归属（Phase 2.9 视角） + +**Family（Wan）关心：** +- `models.*.path / trainable` +- `training.data_path / dataloader_num_workers / train_batch_size / seed / output_dir` +- Wan 相关的 shape 信息（`num_latent_t/num_height/num_width/...`） + +**Method（DMD2）关心：** +- update policy：`generator_update_interval` +- student rollout 相关：`simulate_generator_forward` +- optimizer/scheduler（Phase 2.9 已迁移到 method 创建）： + - student：`learning_rate / betas / lr_scheduler` + - critic（DMD2 专属覆盖）：`fake_score_learning_rate / fake_score_betas / fake_score_lr_scheduler` +- few-step step list（目前仍放在 `pipeline_config`）： + - `pipeline_config.dmd_denoising_steps` + +**Adapter（WanAdapter）关心：** +- 把 FastVideo/Wan 的 forward primitives 暴露给 method（不包含 step list/policy） + +--- + +## TODO（Phase 3） + +为进一步减少 “training/pipeline_config 承载算法语义”，建议迁移： +- `fake_score_*` → `method_config.optimizers.critic.*` +- `dmd_denoising_steps` → `method_config.rollout.steps`（或类似命名） + diff --git a/fastvideo/distillation/doc/registry.md b/fastvideo/distillation/doc/registry.md new file mode 100644 index 000000000..d1e9f7d84 --- /dev/null +++ b/fastvideo/distillation/doc/registry.md @@ -0,0 +1,22 @@ +# `fastvideo/distillation/registry.py` + +**目的** +- 为 distillation 的 “family / method” 提供轻量 registry： + - 新增 family 的成本 ≈ N + - 新增 method 的成本 ≈ M + - build 组合不需要写 N×M 的 if/else + +**关键概念** +- `FamilyBuilder(cfg) -> FamilyArtifacts` +- `MethodBuilder(cfg, bundle, adapter) -> DistillMethod` + +**关键 API** +- `register_family(name)` / `register_method(name)`：装饰器注册 +- `get_family(name)` / `get_method(name)`：查询（会触发内置注册） +- `ensure_builtin_registrations()`：显式 import 内置实现，避免 import 顺序隐式 bug +- `available_families()` / `available_methods()` + +**扩展方式** +- 新增 family：实现 `fastvideo/distillation/families/.py` 并用 `@register_family("")` +- 新增 method：实现 `fastvideo/distillation/methods/...` 并用 `@register_method("")` + diff --git a/fastvideo/distillation/doc/runtime.md b/fastvideo/distillation/doc/runtime.md new file mode 100644 index 000000000..e604bc2d8 --- /dev/null +++ b/fastvideo/distillation/doc/runtime.md @@ -0,0 +1,27 @@ +# `fastvideo/distillation/runtime.py` + +**目的** +- 定义 distillation builder 的结构化输出类型，明确 build-time 与 run-time 的边界。 + +**关键类型** +- `FamilyArtifacts` + - build-time family 插件的产物集合： + - `training_args` + - `bundle` + - `adapter` + - `dataloader` + - `tracker` + - `start_step`（用于 resume / warm-start） +- `DistillRuntime` + - `DistillTrainer.run()` 所需的最小集合： + - `training_args` + - `method`（`DistillMethod`） + - `dataloader` + - `tracker` + - `start_step` + +**设计意图** +- family 负责把 “零件” 装配成 `FamilyArtifacts` +- method 负责把算法绑定到（bundle + adapter）上 +- trainer 只接收 `method` 并开始训练 + diff --git a/fastvideo/distillation/doc/specs.md b/fastvideo/distillation/doc/specs.md new file mode 100644 index 000000000..a3662f2c5 --- /dev/null +++ b/fastvideo/distillation/doc/specs.md @@ -0,0 +1,20 @@ +# `fastvideo/distillation/specs.py` + +**目的** +- 把 config 里的“选择项”做成轻量 dataclass，便于： + - YAML 解析（`yaml_config.py`） + - builder/registry dispatch（`builder.py` / `registry.py`） + +**关键类型** +- `DistillSpec` + - `model`: family 名称（例如 `"wan"`） + - `method`: method 名称（例如 `"dmd2"`） +- `RoleSpec` + - `family`: 该 role 的 family（默认可继承 `distill.model`） + - `path`: 模型权重路径（HF repo 或本地目录） + - `trainable`: 是否训练该 role（只影响 `requires_grad`/模式；具体 optimizer 由 method 决定） + +**注意** +- role 名称本身（`student/teacher/critic/...`）是字符串。 + framework 不强行规定“canonical roles”，由 method 决定语义与依赖。 + diff --git a/fastvideo/distillation/doc/trainer.md b/fastvideo/distillation/doc/trainer.md new file mode 100644 index 000000000..51f8222aa --- /dev/null +++ b/fastvideo/distillation/doc/trainer.md @@ -0,0 +1,27 @@ +# `fastvideo/distillation/trainer.py` + +**目的** +- 提供与算法无关的训练 loop（infra only），把 “怎么训练” 固定下来，把 + “训练什么（loss/rollout/update policy）” 留给 method。 + +**关键类型** +- `DistillTrainer` + - `run(method, dataloader, max_steps, ...)` + - 支持： + - grad accumulation + - tracker logging（rank0） + - validation hook（`method.log_validation(step)`） + - checkpoint hook（通过 `checkpoint_manager` 注入） + +**与 Method 的契约** +`run()` 通过 duck-typing 调用（存在则调用）： +- `method.on_train_start()` +- `method.single_train_step(batch, step, current_vsa_sparsity=...)` +- `method.backward(loss_map, outputs, grad_accum_rounds=...)` +- `method.optimizers_schedulers_step(step)` +- `method.optimizers_zero_grad(step)` + +**重要边界** +- trainer 不应知道 roles（student/teacher/critic/...）也不应知道具体算法； + optimizer cadence、multi-optimizer 更新策略都应由 method 决定并暴露为 hook。 + diff --git a/fastvideo/distillation/doc/validators/__init__.md b/fastvideo/distillation/doc/validators/__init__.md new file mode 100644 index 000000000..00b263900 --- /dev/null +++ b/fastvideo/distillation/doc/validators/__init__.md @@ -0,0 +1,9 @@ +# `fastvideo/distillation/validators/__init__.py` + +**目的** +- 统一导出 validator 接口与 Wan validator 实现。 + +**当前导出** +- `DistillValidator`：最小 validation 接口 +- `WanValidator`：Wan family 的 validation 采样与记录实现 + diff --git a/fastvideo/distillation/doc/validators/base.md b/fastvideo/distillation/doc/validators/base.md new file mode 100644 index 000000000..14569a701 --- /dev/null +++ b/fastvideo/distillation/doc/validators/base.md @@ -0,0 +1,13 @@ +# `fastvideo/distillation/validators/base.py` + +**目的** +- 定义 distillation validator 的最小抽象接口。 + +**接口** +- `log_validation(step: int) -> None` + +**设计意图** +- trainer 与 method 不需要知道 validator 的实现细节： + - adapter 可以持有 validator，并在合适的时机调用 + - 或 method/trainer 通过 hook 调用 `log_validation()` + diff --git a/fastvideo/distillation/doc/validators/wan.md b/fastvideo/distillation/doc/validators/wan.md new file mode 100644 index 000000000..7255fed71 --- /dev/null +++ b/fastvideo/distillation/doc/validators/wan.md @@ -0,0 +1,23 @@ +# `fastvideo/distillation/validators/wan.py` + +**定位** +- `WanValidator`：Wan distillation 的 validation/sampling 实现（Phase 2 standalone）。 +- 负责： + - 读取 validation dataset（json） + - 调用 Wan pipeline 生成视频样本 + - 保存 mp4 到 `output_dir` + - 通过 tracker（例如 wandb）记录 artifacts + +**关键点** +- validator 运行在分布式环境下： + - 以 SP group 为单位做采样，最终由 global rank0 聚合写文件与 log +- 通过 `loaded_modules={"transformer": student_transformer}` 复用训练中的 student 模块权重。 + +**依赖** +- 当前使用 `WanDMDPipeline` 做采样推理（FlowMatch scheduler + DmdDenoisingStage）。 + 这属于 validation 选择（并不影响 adapter 的“无算法命名耦合”约束）。 + +**可演进方向（Phase 3+）** +- 将 validation steps/guidance 等采样配置从 `TrainingArgs` 迁移到更明确的配置块（例如 `validation:`）。 +- 进一步抽象 validator API，使其更容易被不同 family/method 复用。 + diff --git a/fastvideo/distillation/doc/yaml_config.md b/fastvideo/distillation/doc/yaml_config.md new file mode 100644 index 000000000..3b0effb2f --- /dev/null +++ b/fastvideo/distillation/doc/yaml_config.md @@ -0,0 +1,30 @@ +# `fastvideo/distillation/yaml_config.py` + +**目的** +- Phase 2 distillation 的 YAML-only 配置加载器（不兼容/不 merge legacy CLI）。 + +**核心产物** +- `DistillRunConfig` + - `distill: DistillSpec`（选择 family + method） + - `roles: dict[str, RoleSpec]`（来自 YAML 的 `models:`） + - `training_args: TrainingArgs`（来自 YAML 的 `training:`，并注入 entrypoint invariants） + - `raw: dict`（原始 YAML，便于 tracker 记录） + +**YAML 结构（Phase 2）** +- `distill: {model: ..., method: ...}` +- `models: {: {family?, path, trainable?}, ...}` +- `training: {...}`（大部分字段复用 `TrainingArgs.from_kwargs()`） +- `pipeline_config` 或 `pipeline_config_path` + +**实现要点** +- `_resolve_existing_file()`：要求传入真实存在的路径（Phase 2 不做 overlay/fallback） +- 默认分布式 size： + - `num_gpus` 默认 1 + - `tp_size` 默认 1 + - `sp_size` 默认 `num_gpus`（保持与现有 pipeline 的期望一致） +- 训练模式 invariants（由入口强制注入）： + - `mode = DISTILLATION` + - `inference_mode = False` + - `dit_precision` 默认 `fp32`（master weights） + - `dit_cpu_offload = False` + From e8a83712e0717fbc4a8ecb225a6e3b4b8486390e Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Sun, 22 Feb 2026 19:46:12 -0800 Subject: [PATCH 076/214] update wangame sf --- examples/distill/SFWanGame2.1/distill_dmd.sh | 140 +++ .../distill/SFWanGame2.1/distill_dmd.slurm | 160 +++ examples/distill/SFWanGame2.1/validation.json | 164 +++ ...game_self_forcing_distillation_pipeline.py | 943 ++++++++++++++++++ 4 files changed, 1407 insertions(+) create mode 100644 examples/distill/SFWanGame2.1/distill_dmd.sh create mode 100644 examples/distill/SFWanGame2.1/distill_dmd.slurm create mode 100644 examples/distill/SFWanGame2.1/validation.json create mode 100644 fastvideo/training/wangame_self_forcing_distillation_pipeline.py diff --git a/examples/distill/SFWanGame2.1/distill_dmd.sh b/examples/distill/SFWanGame2.1/distill_dmd.sh new file mode 100644 index 000000000..c819ef1aa --- /dev/null +++ b/examples/distill/SFWanGame2.1/distill_dmd.sh @@ -0,0 +1,140 @@ +#!/bin/bash +#SBATCH --job-name=t2v +#SBATCH --partition=main +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:1 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=dmd_t2v_output/t2v_%j.out +#SBATCH --error=dmd_t2v_output/t2v_%j.err +#SBATCH --exclusive + +# Basic Info +export NCCL_P2P_DISABLE=1 +export TORCH_NCCL_ENABLE_MONITORING=0 +# different cache dir for different processes +# export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} +export MASTER_PORT=29503 +export TOKENIZERS_PARALLELISM=false +export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" +export WANDB_BASE_URL="https://api.wandb.ai" +export WANDB_MODE=online +export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN + +# Configs +NUM_GPUS=64 + +# Model paths for Self-Forcing DMD distillation: +GENERATOR_MODEL_PATH="../WanGame-2.1" +REAL_SCORE_MODEL_PATH="../WanGame-2.1" # Teacher model +FAKE_SCORE_MODEL_PATH="../WanGame-2.1-" # Critic model + +DATA_DIR="../FastvideoWorldModel-MC/preprocessed" +VALIDATION_DATASET_FILE="examples/distill/SFWanGame2.1/validation.json" + +training_args=( + --tracker_project_name wangame_distill_self_forcing_dmd + --output_dir "checkpoints/wangame_distill_self_forcing_dmd" + --wandb_run_name "0202_1010_steps2000_bs_64" + --max_train_steps 500 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 21 + --num_height 352 + --num_width 640 + --enable_gradient_checkpointing_type "full" + --log_visualization + --simulate_generator_forward + --num_frames 81 + --num_frame_per_block 3 # Frame generation block size for self-forcing + --enable_gradient_masking + --gradient_mask_last_n_frames 21 + # --resume_from_checkpoint "checkpoints/wangame_distill_self_forcing_dmd/checkpoint-100" +) + +parallel_args=( + --num_gpus $NUM_GPUS # 64 + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 # 64 + --hsdp_shard_dim $NUM_GPUS +) + +model_args=( + --model_path $GENERATOR_MODEL_PATH # TODO: check if you can remove this in this script + --pretrained_model_name_or_path $GENERATOR_MODEL_PATH + --real_score_model_path $REAL_SCORE_MODEL_PATH + --fake_score_model_path $FAKE_SCORE_MODEL_PATH +) + +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 50 + --validation_sampling_steps "4" + --validation_guidance_scale "6.0" # not used for dmd inference +) + +optimizer_args=( + --learning_rate 6e-6 + --mixed_precision "bf16" + --training_state_checkpointing_steps 50 + --weight_only_checkpointing_steps 50 + --weight_decay 0.01 + --betas '0.0,0.999' + --max_grad_norm 1.0 +) + +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.0 + --dit_precision "fp32" + --flow_shift 5 + --seed 1000 + --use_ema True + --ema_decay 0.99 + --ema_start_step 100 + --init_weights_from_safetensors "checkpoints/wangame_ode_init_64gpu/checkpoint-2000/transformer" +) + +dmd_args=( + --dmd_denoising_steps '1000,750,500,250' + --min_timestep_ratio 0.02 + --max_timestep_ratio 0.98 + --dfake_gen_update_ratio 5 + --real_score_guidance_scale 3.0 + --fake_score_learning_rate 8e-6 + --fake_score_betas '0.0,0.999' + --warp_denoising_step +) + +self_forcing_args=( + --independent_first_frame False # Whether to treat first frame independently + --same_step_across_blocks True # Whether to use same denoising step across all blocks + --last_step_only False # Whether to only use the last denoising step + --context_noise 0 # Amount of noise to add during context caching (0 = no noise) +) + +torchrun \ +--nnodes 1 \ +--master_port $MASTER_PORT \ +--nproc_per_node $NUM_GPUS \ + fastvideo/training/wangame_self_forcing_distillation_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ + "${dmd_args[@]}" \ + "${self_forcing_args[@]}" diff --git a/examples/distill/SFWanGame2.1/distill_dmd.slurm b/examples/distill/SFWanGame2.1/distill_dmd.slurm new file mode 100644 index 000000000..7c6b19812 --- /dev/null +++ b/examples/distill/SFWanGame2.1/distill_dmd.slurm @@ -0,0 +1,160 @@ +#!/bin/bash +#SBATCH --job-name=wg-sf +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=sf_train_output/ode_%j.out +#SBATCH --error=sf_train_output/ode_%j.err +#SBATCH --exclusive + +set -e -x + +# Environment Setup +source ~/conda/miniconda/bin/activate +conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv +export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin:$PYTHONPATH" + +# Basic Info +export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" +export WANDB_MODE="online" +export NCCL_P2P_DISABLE=1 +export MASTER_PORT=29500 +export NODE_RANK=$SLURM_PROCID +nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) +export MASTER_ADDR=${nodes[0]} +export TOKENIZERS_PARALLELISM=false + +echo "MASTER_ADDR: $MASTER_ADDR" +echo "NODE_RANK: $NODE_RANK" + +RUN_NAME=$(date +"%m%d_%H%M") +echo "RUN_NAME: $RUN_NAME" + +# Model paths for Self-Forcing DMD distillation: +# GENERATOR_MODEL_PATH="../wg_models/WanGame-2.1-Student-VizDoom1k-1000steps-Diffusers" +GENERATOR_MODEL_PATH="../wg_models/WanGame-2.1-Student-VizDoom1k-1000steps-Diffusers" +REAL_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-Foundation-VizDoom1k-1000steps-Diffusers" # Teacher model +FAKE_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-Foundation-VizDoom1k-1000steps-Diffusers" # Critic model + +DATA_DIR="../traindata_0209_1500/wg_ode_init/preprocessed" +VALIDATION_DATASET_FILE="examples/distill/SFWanGame2.1/validation.json" +CKPT_SAFETENSOR="checkpoints/wangame_ode_init_vizdoom_new/checkpoint-6000/transformer/diffusion_pytorch_model.safetensors" + +# Training arguments +training_args=( + --tracker_project_name "wangame_sf" + --output_dir "checkpoints/wangame_sf_${RUN_NAME}" + --wandb_run_name "${RUN_NAME}_bs32" + --max_train_steps 3000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 21 + --num_height 352 + --num_width 640 + --enable_gradient_checkpointing_type "full" + --log_visualization + --simulate_generator_forward + --num_frames 81 + --num_frame_per_block 3 # Frame generation block size for self-forcing + --enable_gradient_masking + --gradient_mask_last_n_frames 21 + # --init_weights_from_safetensors $CKPT_SAFETENSOR +) + +# Parallel arguments +parallel_args=( + --num_gpus 32 + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim 32 +) + +model_args=( + --model_path $GENERATOR_MODEL_PATH # TODO: check if you can remove this in this script + --pretrained_model_name_or_path $GENERATOR_MODEL_PATH + --real_score_model_path $REAL_SCORE_MODEL_PATH + --fake_score_model_path $FAKE_SCORE_MODEL_PATH +) + +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +# Validation arguments +validation_args=( + --log_validation + --log_visualization + --visualization-steps 100 + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 100 + --validation_sampling_steps "4" + --validation_guidance_scale "6.0" +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 6e-6 + --mixed_precision "bf16" + --weight_only_checkpointing_steps 100 + --training_state_checkpointing_steps 100 + --weight_decay 0.01 + --betas '0.0,0.999' + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.0 + --dit_precision "fp32" + --flow_shift 5 + --seed 1000 + --use_ema True + --ema_decay 0.99 + --ema_start_step 200 +) + +dmd_args=( + --dmd_denoising_steps '1000,750,500,250' + --min_timestep_ratio 0.02 + --max_timestep_ratio 0.98 + --dfake_gen_update_ratio 5 + --real_score_guidance_scale 3.0 + --fake_score_learning_rate 8e-6 + --fake_score_betas '0.0,0.999' + --warp_denoising_step +) + +self_forcing_args=( + --independent_first_frame False # Whether to treat first frame independently + --same_step_across_blocks True # Whether to use same denoising step across all blocks + --last_step_only False # Whether to only use the last denoising step + --context_noise 0 # Amount of noise to add during context caching (0 = no noise) +) + +mkdir -p sf_train_output + +srun torchrun \ +--nnodes $SLURM_JOB_NUM_NODES \ +--nproc_per_node 8 \ +--node_rank $SLURM_PROCID \ +--rdzv_backend=c10d \ +--rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ + fastvideo/training/wangame_self_forcing_distillation_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ + "${dmd_args[@]}" \ + "${self_forcing_args[@]}" diff --git a/examples/distill/SFWanGame2.1/validation.json b/examples/distill/SFWanGame2.1/validation.json new file mode 100644 index 000000000..d97352dd7 --- /dev/null +++ b/examples/distill/SFWanGame2.1/validation.json @@ -0,0 +1,164 @@ +{ + "data": [ + { + "caption": "Hold [W] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000000_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [S] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000001_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [A] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000002_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [D] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000003_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [W] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000000_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [S] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000001_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [A] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000002_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [D] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000003_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [W] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000000_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [S] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000001_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [A] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000002_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [D] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000003_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [W] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000000_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [S] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000001_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [A] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000002_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "Hold [D] + Static", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000003_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + } + ] +} \ No newline at end of file diff --git a/fastvideo/training/wangame_self_forcing_distillation_pipeline.py b/fastvideo/training/wangame_self_forcing_distillation_pipeline.py new file mode 100644 index 000000000..8723b2cb1 --- /dev/null +++ b/fastvideo/training/wangame_self_forcing_distillation_pipeline.py @@ -0,0 +1,943 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys +from copy import deepcopy +from typing import Any + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F +from einops import rearrange + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_ode_trajectory_wangame) +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.forward_context import set_forward_context +from fastvideo.logger import init_logger +from fastvideo.models.dits.hyworld.pose import process_custom_actions +from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( + SelfForcingFlowMatchScheduler) +from fastvideo.models.utils import pred_noise_to_pred_video +from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( + WanGameCausalDMDPipeline) +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch +from fastvideo.training.self_forcing_distillation_pipeline import ( + SelfForcingDistillationPipeline) +from fastvideo.training.training_utils import shift_timestep +from fastvideo.utils import is_vsa_available, shallow_asdict + +vsa_available = is_vsa_available() + +logger = init_logger(__name__) + + +class WanGameSelfForcingDistillationPipeline(SelfForcingDistillationPipeline): + """ + A self-forcing distillation pipeline for WanGame that uses the self-forcing methodology + with DMD for video generation. + """ + _required_config_modules = [ + "scheduler", + "transformer", + "vae", + ] + + def set_schemas(self): + self.train_dataset_schema = pyarrow_schema_ode_trajectory_wangame + + def _initialize_simulation_caches( + self, + batch_size: int, + dtype: torch.dtype, + device: torch.device, + *, + max_num_frames: int | None = None, + ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Initialize KV cache and cross-attention cache for multi-step simulation.""" + num_transformer_blocks = len(self.transformer.blocks) + latent_shape = self.video_latent_shape_sp + _, num_frames, _, height, width = latent_shape + + _, p_h, p_w = self.transformer.patch_size + post_patch_height = height // p_h + post_patch_width = width // p_w + + frame_seq_length = post_patch_height * post_patch_width + self.frame_seq_length = frame_seq_length + + # Get model configuration parameters - handle FSDP wrapping + num_attention_heads = getattr(self.transformer, 'num_attention_heads', + None) + attention_head_dim = getattr(self.transformer, 'attention_head_dim', + None) + + # 1 CLS token + 256 patch tokens = 257 + text_len = 257 + + if max_num_frames is None: + max_num_frames = num_frames + num_max_frames = max(max_num_frames, num_frames) + kv_cache_size = num_max_frames * frame_seq_length + # WanGame causal attention stores both RoPE and PRoPE branches in cache. + cache_head_dim = attention_head_dim * 2 + + kv_cache = [] + for _ in range(num_transformer_blocks): + kv_cache.append({ + "k": + torch.zeros([ + batch_size, kv_cache_size, num_attention_heads, + cache_head_dim + ], + dtype=dtype, + device=device), + "v": + torch.zeros([ + batch_size, kv_cache_size, num_attention_heads, + cache_head_dim + ], + dtype=dtype, + device=device), + "global_end_index": + torch.tensor([0], dtype=torch.long, device=device), + "local_end_index": + torch.tensor([0], dtype=torch.long, device=device) + }) + + # Initialize cross-attention cache + crossattn_cache = [] + for _ in range(num_transformer_blocks): + crossattn_cache.append({"is_init": False}) + + return kv_cache, crossattn_cache + + def _reset_simulation_caches( + self, kv_cache: list[dict[str, + Any]], crossattn_cache: list[dict[str, + Any]] + ) -> None: + """Reset KV cache and cross-attention cache to clean state.""" + if kv_cache is not None: + for cache_dict in kv_cache: + cache_dict["global_end_index"].fill_(0) + cache_dict["local_end_index"].fill_(0) + cache_dict["k"].zero_() + cache_dict["v"].zero_() + + if crossattn_cache is not None: + for cache_dict in crossattn_cache: + cache_dict["is_init"] = False + if "k" in cache_dict: + cache_dict["k"].zero_() + if "v" in cache_dict: + cache_dict["v"].zero_() + + def _generator_multi_step_simulation_forward( + self, + training_batch: TrainingBatch, + return_sim_steps: bool = False) -> torch.Tensor: + """Forward pass through student transformer matching inference procedure with KV cache management. + + This function is adapted from the reference self-forcing implementation's inference_with_trajectory + and includes gradient masking logic for dynamic frame generation. + """ + latents = training_batch.latents + dtype = latents.dtype + batch_size = latents.shape[0] + initial_latent = getattr(training_batch, 'image_latent', None) + + # Dynamic frame generation logic (adapted from _run_generator) + num_training_frames = getattr(self.training_args, 'num_latent_t', 21) + + # During training, the number of generated frames should be uniformly sampled from + # [21, self.num_training_frames], but still being a multiple of self.num_frame_per_block + min_num_frames = 20 if self.independent_first_frame else 21 + max_num_frames = num_training_frames - 1 if self.independent_first_frame else num_training_frames + assert max_num_frames % self.num_frame_per_block == 0 + assert min_num_frames % self.num_frame_per_block == 0 + max_num_blocks = max_num_frames // self.num_frame_per_block + min_num_blocks = min_num_frames // self.num_frame_per_block + + # Sample number of blocks and sync across processes + num_generated_blocks = torch.randint(min_num_blocks, + max_num_blocks + 1, (1, ), + device=self.device) + if dist.is_initialized(): + dist.broadcast(num_generated_blocks, src=0) + num_generated_blocks = num_generated_blocks.item() + num_generated_frames = num_generated_blocks * self.num_frame_per_block + if self.independent_first_frame and initial_latent is None: + num_generated_frames += 1 + min_num_frames += 1 + + # Create noise with dynamic shape + if initial_latent is not None: + noise_shape = [ + batch_size, num_generated_frames - 1, + *self.video_latent_shape[2:] + ] + else: + noise_shape = [ + batch_size, num_generated_frames, *self.video_latent_shape[2:] + ] + + noise = torch.randn(noise_shape, device=self.device, dtype=dtype) + if self.sp_world_size > 1: + noise = rearrange(noise, + "b (n t) c h w -> b n t c h w", + n=self.sp_world_size).contiguous() + noise = noise[:, self.rank_in_sp_group, :, :, :, :] + + batch_size, num_frames, num_channels, height, width = noise.shape + + # Block size calculation + if not self.independent_first_frame or (self.independent_first_frame + and initial_latent is not None): + assert num_frames % self.num_frame_per_block == 0 + num_blocks = num_frames // self.num_frame_per_block + else: + assert (num_frames - 1) % self.num_frame_per_block == 0 + num_blocks = (num_frames - 1) // self.num_frame_per_block + + num_input_frames = initial_latent.shape[ + 1] if initial_latent is not None else 0 + num_output_frames = num_frames + num_input_frames + output = torch.zeros( + [batch_size, num_output_frames, num_channels, height, width], + device=noise.device, + dtype=noise.dtype) + + def get_model_device(model): + if model is None: + return "None" + try: + return next(model.parameters()).device + except (StopIteration, AttributeError): + return "Unknown" + + # Step 1: Initialize KV cache to all zeros + cache_frames = num_generated_frames + num_input_frames + (self.kv_cache1, + self.crossattn_cache) = self._initialize_simulation_caches( + batch_size, dtype, self.device, max_num_frames=cache_frames) + + # Step 2: Cache context feature + current_start_frame = 0 + if initial_latent is not None: + timestep = torch.ones( + [batch_size, 1], device=noise.device, dtype=torch.int64) * 0 + output[:, :1] = initial_latent + with torch.no_grad(): + # Build input kwargs for initial latent + training_batch_temp = self._build_distill_input_kwargs( + initial_latent, + timestep * 0, + training_batch.conditional_dict, + training_batch, + frame_start=0, + frame_end=1, + num_frame_per_block=1) + + # we process the image latent with self.transformer_2 (low-noise expert) + current_model = self.transformer_2 if self.transformer_2 is not None else self.transformer + current_model( + **training_batch_temp.input_kwargs, + kv_cache=self.kv_cache1, + crossattn_cache=self.crossattn_cache, + current_start=current_start_frame * self.frame_seq_length, + start_frame=current_start_frame) + current_start_frame += 1 + + # Step 3: Temporal denoising loop + all_num_frames = [self.num_frame_per_block] * num_blocks + if self.independent_first_frame and initial_latent is None: + all_num_frames = [1] + all_num_frames + num_denoising_steps = len(self.denoising_step_list) + exit_flags = self.generate_and_sync_list(len(all_num_frames), + num_denoising_steps, + device=noise.device) + start_gradient_frame_index = max(0, num_output_frames - 21) + + for block_index, current_num_frames in enumerate(all_num_frames): + noisy_input = noise[:, current_start_frame - + num_input_frames:current_start_frame + + current_num_frames - num_input_frames] + + # Step 3.1: Spatial denoising loop + for index, current_timestep in enumerate(self.denoising_step_list): + if self.same_step_across_blocks: + exit_flag = (index == exit_flags[0]) + else: + exit_flag = (index == exit_flags[block_index]) + + timestep = torch.ones([batch_size, current_num_frames], + device=noise.device, + dtype=torch.int64) * current_timestep + + if self.boundary_timestep is not None and current_timestep < self.boundary_timestep and self.transformer_2 is not None: + current_model = self.transformer_2 + else: + current_model = self.transformer + + if not exit_flag: + with torch.no_grad(): + # Build input kwargs + training_batch_temp = self._build_distill_input_kwargs( + noisy_input, + timestep, + training_batch.conditional_dict, + training_batch, + frame_start=current_start_frame, + frame_end=current_start_frame + current_num_frames, + num_frame_per_block=current_num_frames) + + pred_flow = current_model( + **training_batch_temp.input_kwargs, + kv_cache=self.kv_cache1, + crossattn_cache=self.crossattn_cache, + current_start=current_start_frame * + self.frame_seq_length, + start_frame=current_start_frame).permute( + 0, 2, 1, 3, 4) + + denoised_pred = pred_noise_to_pred_video( + pred_noise=pred_flow.flatten(0, 1), + noise_input_latent=noisy_input.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler).unflatten( + 0, pred_flow.shape[:2]) + + next_timestep = self.denoising_step_list[index + 1] + noisy_input = self.noise_scheduler.add_noise( + denoised_pred.flatten(0, 1), + torch.randn_like(denoised_pred.flatten(0, 1)), + next_timestep * + torch.ones([batch_size * current_num_frames], + device=noise.device, + dtype=torch.long)).unflatten( + 0, denoised_pred.shape[:2]) + else: + # Final prediction with gradient control + if current_start_frame < start_gradient_frame_index: + with torch.no_grad(): + training_batch_temp = self._build_distill_input_kwargs( + noisy_input, + timestep, + training_batch.conditional_dict, + training_batch, + frame_start=current_start_frame, + frame_end=current_start_frame + + current_num_frames, + num_frame_per_block=current_num_frames) + + pred_flow = current_model( + **training_batch_temp.input_kwargs, + kv_cache=self.kv_cache1, + crossattn_cache=self.crossattn_cache, + current_start=current_start_frame * + self.frame_seq_length, + start_frame=current_start_frame).permute( + 0, 2, 1, 3, 4) + else: + training_batch_temp = self._build_distill_input_kwargs( + noisy_input, + timestep, + training_batch.conditional_dict, + training_batch, + frame_start=current_start_frame, + frame_end=current_start_frame + current_num_frames, + num_frame_per_block=current_num_frames) + + pred_flow = current_model( + **training_batch_temp.input_kwargs, + kv_cache=self.kv_cache1, + crossattn_cache=self.crossattn_cache, + current_start=current_start_frame * + self.frame_seq_length, + start_frame=current_start_frame).permute( + 0, 2, 1, 3, 4) + + denoised_pred = pred_noise_to_pred_video( + pred_noise=pred_flow.flatten(0, 1), + noise_input_latent=noisy_input.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler).unflatten( + 0, pred_flow.shape[:2]) + break + + # Step 3.2: record the model's output + output[:, current_start_frame:current_start_frame + + current_num_frames] = denoised_pred + + # Step 3.3: rerun with timestep zero to update the cache + context_timestep = torch.ones_like(timestep) * self.context_noise + denoised_pred = self.noise_scheduler.add_noise( + denoised_pred.flatten(0, 1), + torch.randn_like(denoised_pred.flatten(0, 1)), + context_timestep).unflatten(0, denoised_pred.shape[:2]) + + with torch.no_grad(): + training_batch_temp = self._build_distill_input_kwargs( + denoised_pred, + context_timestep, + training_batch.conditional_dict, + training_batch, + frame_start=current_start_frame, + frame_end=current_start_frame + current_num_frames, + num_frame_per_block=current_num_frames) + + # context_timestep is 0 so we use transformer_2 + current_model = self.transformer_2 if self.transformer_2 is not None else self.transformer + current_model( + **training_batch_temp.input_kwargs, + kv_cache=self.kv_cache1, + crossattn_cache=self.crossattn_cache, + current_start=current_start_frame * self.frame_seq_length, + start_frame=current_start_frame) + + # Step 3.4: update the start and end frame indices + current_start_frame += current_num_frames + + # Handle last 21 frames logic + pred_image_or_video = output + if num_input_frames > 0: + pred_image_or_video = output[:, num_input_frames:] + + # Slice last 21 frames if we generated more + gradient_mask = None + if pred_image_or_video.shape[1] > 21: + with torch.no_grad(): + # Re-encode to get image latent + latent_to_decode = pred_image_or_video[:, :-20, ...] + # Decode to video + latent_to_decode = latent_to_decode.permute( + 0, 2, 1, 3, 4) # [B, C, F, H, W] + + # Apply VAE scaling and shift factors + if isinstance(self.vae.scaling_factor, torch.Tensor): + latent_to_decode = latent_to_decode / self.vae.scaling_factor.to( + latent_to_decode.device, latent_to_decode.dtype) + else: + latent_to_decode = latent_to_decode / self.vae.scaling_factor + + if hasattr( + self.vae, + "shift_factor") and self.vae.shift_factor is not None: + if isinstance(self.vae.shift_factor, torch.Tensor): + latent_to_decode += self.vae.shift_factor.to( + latent_to_decode.device, latent_to_decode.dtype) + else: + latent_to_decode += self.vae.shift_factor + + # Decode to pixels + pixels = self.vae.decode(latent_to_decode) + frame = pixels[:, :, -1:, :, :].to( + dtype) # Last frame [B, C, 1, H, W] + + # Encode frame back to get image latent + image_latent = self.vae.encode(frame).to(dtype) + image_latent = image_latent.permute(0, 2, 1, 3, + 4) # [B, F, C, H, W] + + pred_image_or_video_last_21 = torch.cat( + [image_latent, pred_image_or_video[:, -20:, ...]], dim=1) + else: + pred_image_or_video_last_21 = pred_image_or_video + + # Set up gradient mask if we generated more than minimum frames + if num_generated_frames != min_num_frames: + # Currently, we do not use gradient for the first chunk, since it contains image latents + gradient_mask = torch.ones_like(pred_image_or_video_last_21, + dtype=torch.bool) + if self.independent_first_frame: + gradient_mask[:, :1] = False + else: + gradient_mask[:, :self.num_frame_per_block] = False + + # Apply gradient masking if needed + final_output = pred_image_or_video_last_21.to(dtype) + if gradient_mask is not None: + # Apply gradient masking: detach frames that shouldn't contribute gradients + final_output = torch.where( + gradient_mask, + pred_image_or_video_last_21, # Keep original values where gradient_mask is True + pred_image_or_video_last_21.detach( + ) # Detach where gradient_mask is False + ) + + # Store visualization data + training_batch.dmd_latent_vis_dict["generator_timestep"] = torch.tensor( + self.denoising_step_list[exit_flags[0]], + dtype=torch.float32, + device=self.device) + + # Store gradient mask information for debugging + if gradient_mask is not None: + training_batch.dmd_latent_vis_dict[ + "gradient_mask"] = gradient_mask.float() + training_batch.dmd_latent_vis_dict[ + "num_generated_frames"] = torch.tensor(num_generated_frames, + dtype=torch.float32, + device=self.device) + training_batch.dmd_latent_vis_dict["min_num_frames"] = torch.tensor( + min_num_frames, dtype=torch.float32, device=self.device) + + # Clean up caches + assert self.kv_cache1 is not None + assert self.crossattn_cache is not None + self._reset_simulation_caches(self.kv_cache1, self.crossattn_cache) + + return final_output if gradient_mask is not None else pred_image_or_video + + def initialize_validation_pipeline(self, training_args: TrainingArgs): + logger.info("Initializing validation pipeline...") + args_copy = deepcopy(training_args) + args_copy.inference_mode = True + # Use the same flow-matching scheduler as training for consistent validation. + validation_scheduler = SelfForcingFlowMatchScheduler( + shift=args_copy.pipeline_config.flow_shift, + sigma_min=0.0, + extra_one_step=True) + validation_scheduler.set_timesteps(num_inference_steps=1000, + training=True) + # Warm start validation with current transformer + self.validation_pipeline = WanGameCausalDMDPipeline.from_pretrained( + training_args.model_path, + args=args_copy, # type: ignore + inference_mode=True, + loaded_modules={ + "transformer": self.get_module("transformer"), + "vae": self.get_module("vae"), + "scheduler": validation_scheduler, + }, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + pin_cpu_memory=training_args.pin_cpu_memory, + dit_cpu_offload=True) + + def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: + batch = next(self.train_loader_iter, None) # type: ignore + if batch is None: + self.current_epoch += 1 + # Reset iterator for next epoch + self.train_loader_iter = iter(self.train_dataloader) + # Get first batch of new epoch + batch = next(self.train_loader_iter) + + clip_feature = batch['clip_feature'] + first_frame_latent = batch['first_frame_latent'] + keyboard_cond = batch.get('keyboard_cond', None) + mouse_cond = batch.get('mouse_cond', None) + infos = batch['info_list'] + + batch_size = clip_feature.shape[0] + vae_config = self.training_args.pipeline_config.vae_config.arch_config + num_channels = vae_config.z_dim + spatial_compression_ratio = vae_config.spatial_compression_ratio + + latent_height = self.training_args.num_height // spatial_compression_ratio + latent_width = self.training_args.num_width // spatial_compression_ratio + + latents = torch.randn(batch_size, num_channels, + self.training_args.num_latent_t, latent_height, + latent_width).to(get_local_torch_device(), + dtype=torch.bfloat16) + + training_batch.latents = latents.to(get_local_torch_device(), + dtype=torch.bfloat16) + training_batch.encoder_hidden_states = None + training_batch.encoder_attention_mask = None + training_batch.image_embeds = clip_feature.to(get_local_torch_device(), + dtype=torch.bfloat16) + training_batch.image_latents = first_frame_latent.to( + get_local_torch_device(), dtype=torch.bfloat16) + # Action conditioning + if keyboard_cond is not None and keyboard_cond.numel() > 0: + keyboard_cond_full = keyboard_cond.to(get_local_torch_device(), + dtype=torch.bfloat16) + training_batch.keyboard_cond = keyboard_cond_full # For Teacher/Critic (dim=6) + else: + training_batch.keyboard_cond = None + if mouse_cond is not None and mouse_cond.numel() > 0: + training_batch.mouse_cond = mouse_cond.to(get_local_torch_device(), + dtype=torch.bfloat16) + else: + training_batch.mouse_cond = None + training_batch.infos = infos + return training_batch + + def _prepare_dit_inputs(self, + training_batch: TrainingBatch) -> TrainingBatch: + """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" + # First, call parent method to prepare noise, timesteps, etc. for video latents + training_batch = super()._prepare_dit_inputs(training_batch) + + assert isinstance(training_batch.image_latents, torch.Tensor) + image_latents = training_batch.image_latents.to( + get_local_torch_device(), dtype=torch.bfloat16) + + # cond_concat = [mask(4), image_latent(16)] with 20 channels. + expected_cond_channels = 20 + if image_latents.shape[1] != expected_cond_channels: + raise ValueError( + "Unexpected first_frame_latent channels, " + "Expected {expected_cond_channels} (cond_concat), got {image_latents.shape[1]}." + ) + + if self.sp_world_size > 1: + total_frames = image_latents.shape[2] + # Split cond latents to local SP shard only when tensor is still global. + if total_frames == self.training_args.num_latent_t: + if total_frames % self.sp_world_size != 0: + raise ValueError( + "image_latents temporal dim is not divisible by SP world size: " + f"frames={total_frames}, sp_world_size={self.sp_world_size}" + ) + image_latents = rearrange(image_latents, + "b c (n t) h w -> b c n t h w", + n=self.sp_world_size).contiguous() + image_latents = image_latents[:, :, self.rank_in_sp_group, :, :, + :] + + training_batch.image_latents = image_latents + + return training_batch + + def _build_distill_input_kwargs( + self, + noise_input: torch.Tensor, + timestep: torch.Tensor, + text_dict: dict[str, torch.Tensor] | None, + training_batch: TrainingBatch, + frame_start: int | None = None, + frame_end: int | None = None, + num_frame_per_block: int | None = None) -> TrainingBatch: + # Image Embeds for conditioning + image_embeds = training_batch.image_embeds + assert torch.isnan(image_embeds).sum() == 0 + image_embeds = image_embeds.to(get_local_torch_device(), + dtype=torch.bfloat16) + + image_latents = training_batch.image_latents + if frame_start is not None and frame_end is not None: + image_latents = image_latents[:, :, frame_start:frame_end, :, :] + + vae_temporal_compression_ratio = 4 + if frame_end is not None: + action_frame_end = (frame_end - + 1) * vae_temporal_compression_ratio + 1 + keyboard_cond_sliced = training_batch.keyboard_cond[:, : + action_frame_end, :] if training_batch.keyboard_cond is not None else None + mouse_cond_sliced = training_batch.mouse_cond[:, : + action_frame_end, :] if training_batch.mouse_cond is not None else None + else: + keyboard_cond_sliced = training_batch.keyboard_cond + mouse_cond_sliced = training_batch.mouse_cond + + if keyboard_cond_sliced is not None and mouse_cond_sliced is not None: + viewmats_list = [] + intrinsics_list = [] + action_labels_list = [] + for b in range(noise_input.shape[0]): + viewmats, intrinsics, action_labels = process_custom_actions( + keyboard_cond_sliced[b], mouse_cond_sliced[b]) + viewmats_list.append(viewmats) + intrinsics_list.append(intrinsics) + action_labels_list.append(action_labels) + + viewmats = torch.stack(viewmats_list, dim=0).to( + device=get_local_torch_device(), dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, dim=0).to( + device=get_local_torch_device(), dtype=torch.bfloat16) + action_labels = torch.stack(action_labels_list, dim=0).to( + device=get_local_torch_device(), dtype=torch.bfloat16) + else: + viewmats = None + intrinsics = None + action_labels = None + + noisy_model_input = torch.cat( + [noise_input, image_latents.permute(0, 2, 1, 3, 4)], dim=2) + + training_batch.input_kwargs = { + "hidden_states": noisy_model_input.permute(0, 2, 1, 3, + 4), # bs, c, t, h, w + "encoder_hidden_states": None, + "timestep": timestep, + "encoder_hidden_states_image": image_embeds, + "viewmats": viewmats, + "Ks": intrinsics, + "action": action_labels, + "num_frame_per_block": num_frame_per_block if num_frame_per_block is not None else self.num_frame_per_block, + } + training_batch.noise_latents = noise_input + + return training_batch + + def _dmd_forward(self, generator_pred_video: torch.Tensor, + training_batch: TrainingBatch) -> torch.Tensor: + """Compute DMD (Diffusion Model Distillation) loss for WanGame.""" + original_latent = generator_pred_video + with torch.no_grad(): + timestep = torch.randint(0, + self.num_train_timestep, [1], + device=self.device, + dtype=torch.long) + + timestep = shift_timestep(timestep, self.timestep_shift, + self.num_train_timestep) + + timestep = timestep.clamp(self.min_timestep, self.max_timestep) + + noise = torch.randn(self.video_latent_shape, + device=self.device, + dtype=generator_pred_video.dtype) + + noisy_latent = self.noise_scheduler.add_noise( + generator_pred_video.flatten(0, 1), noise.flatten(0, 1), + timestep).detach().unflatten(0, (generator_pred_video.shape[0], + generator_pred_video.shape[1])) + + # Non-causal models expect 1D timestep (batch_size,) + critic_timestep = timestep.expand(noisy_latent.shape[0]) + + self._build_distill_input_kwargs( + noisy_latent, critic_timestep, None, training_batch + ) + + # fake_score_transformer forward + current_fake_score_transformer = self._get_fake_score_transformer( + timestep) + fake_score_pred_noise = current_fake_score_transformer( + **training_batch.input_kwargs + ).permute(0, 2, 1, 3, 4) + + faker_score_pred_video = pred_noise_to_pred_video( + pred_noise=fake_score_pred_noise.flatten(0, 1), + noise_input_latent=noisy_latent.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler).unflatten( + 0, fake_score_pred_noise.shape[:2]) + + # real_score_transformer forward + current_real_score_transformer = self._get_real_score_transformer( + timestep) + real_score_pred_noise = current_real_score_transformer( + **training_batch.input_kwargs + ).permute(0, 2, 1, 3, 4) + + real_score_pred_video = pred_noise_to_pred_video( + pred_noise=real_score_pred_noise.flatten(0, 1), + noise_input_latent=noisy_latent.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler).unflatten( + 0, real_score_pred_noise.shape[:2]) + + # No CFG for WanGame - use real_score_pred_video directly + grad = (faker_score_pred_video - real_score_pred_video) / torch.abs( + original_latent - real_score_pred_video).mean() + grad = torch.nan_to_num(grad) + + dmd_loss = 0.5 * F.mse_loss( + original_latent.float(), + (original_latent.float() - grad.float()).detach()) + + training_batch.dmd_latent_vis_dict.update({ + "training_batch_dmd_fwd_clean_latent": + training_batch.latents, + "generator_pred_video": + original_latent.detach(), + "real_score_pred_video": + real_score_pred_video.detach(), + "faker_score_pred_video": + faker_score_pred_video.detach(), + "dmd_timestep": + timestep.detach(), + }) + + return dmd_loss + + def faker_score_forward( + self, training_batch: TrainingBatch + ) -> tuple[TrainingBatch, torch.Tensor]: + """Forward pass for critic training with WanGame action conditioning.""" + with torch.no_grad(), set_forward_context( + current_timestep=training_batch.timesteps, + attn_metadata=training_batch.attn_metadata_vsa): + if self.training_args.simulate_generator_forward: + generator_pred_video = self._generator_multi_step_simulation_forward( + training_batch) + else: + generator_pred_video = self._generator_forward(training_batch) + + fake_score_timestep = torch.randint(0, + self.num_train_timestep, [1], + device=self.device, + dtype=torch.long) + + fake_score_timestep = shift_timestep(fake_score_timestep, + self.timestep_shift, + self.num_train_timestep) + + fake_score_timestep = fake_score_timestep.clamp(self.min_timestep, + self.max_timestep) + + fake_score_noise = torch.randn(self.video_latent_shape, + device=self.device, + dtype=generator_pred_video.dtype) + + noisy_generator_pred_video = self.noise_scheduler.add_noise( + generator_pred_video.flatten(0, 1), + fake_score_noise.flatten(0, 1), fake_score_timestep).unflatten( + 0, + (generator_pred_video.shape[0], generator_pred_video.shape[1])) + + # Non-causal critic expects 1D timestep (batch_size,), not 2D (batch_size, num_frames). + expanded_fake_score_timestep = fake_score_timestep.expand( + noisy_generator_pred_video.shape[0]) + + self._build_distill_input_kwargs( + noisy_generator_pred_video, expanded_fake_score_timestep, None, training_batch + ) + + with set_forward_context(current_timestep=training_batch.timesteps, + attn_metadata=training_batch.attn_metadata): + current_fake_score_transformer = self._get_fake_score_transformer(fake_score_timestep) + fake_score_pred_noise = current_fake_score_transformer( + **training_batch.input_kwargs + ).permute(0, 2, 1, 3, 4) + + target = fake_score_noise - generator_pred_video + flow_matching_loss = torch.mean((fake_score_pred_noise - target)**2) + + training_batch.fake_score_latent_vis_dict = { + "training_batch_fakerscore_fwd_clean_latent": + training_batch.latents, + "generator_pred_video": generator_pred_video, + "fake_score_timestep": fake_score_timestep, + } + + return training_batch, flow_matching_loss + + def _prepare_validation_batch(self, sampling_param: SamplingParam, + training_args: TrainingArgs, + validation_batch: dict[str, Any], + num_inference_steps: int) -> ForwardBatch: + sampling_param.prompt = validation_batch['prompt'] + sampling_param.height = training_args.num_height + sampling_param.width = training_args.num_width + sampling_param.image_path = validation_batch.get( + 'image_path') or validation_batch.get('video_path') + sampling_param.num_inference_steps = num_inference_steps + sampling_param.data_type = "video" + assert self.seed is not None + sampling_param.seed = self.seed + + latents_size = [(sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, sampling_param.width // 8] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (training_args.num_latent_t - + 1) * temporal_compression_factor + 1 + sampling_param.num_frames = num_frames + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=torch.Generator(device="cpu").manual_seed(self.seed), + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=training_args.VSA_sparsity, + ) + if "image" in validation_batch and validation_batch["image"] is not None: + batch.pil_image = validation_batch["image"] + + if "keyboard_cond" in validation_batch and validation_batch[ + "keyboard_cond"] is not None: + keyboard_cond = validation_batch["keyboard_cond"] + keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) + keyboard_cond = keyboard_cond.unsqueeze(0) + batch.keyboard_cond = keyboard_cond + + if "mouse_cond" in validation_batch and validation_batch[ + "mouse_cond"] is not None: + mouse_cond = validation_batch["mouse_cond"] + mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) + mouse_cond = mouse_cond.unsqueeze(0) + batch.mouse_cond = mouse_cond + + return batch + + def _post_process_validation_frames( + self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: + """Apply action overlay to validation frames for WanGame. + + Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. + """ + # Check if action data is available + keyboard_cond = getattr(batch, 'keyboard_cond', None) + mouse_cond = getattr(batch, 'mouse_cond', None) + + if keyboard_cond is None and mouse_cond is None: + return frames + + # Import overlay functions + from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, + draw_mouse_on_frame) + + # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) + if keyboard_cond is not None: + keyboard_cond = keyboard_cond.squeeze( + 0).cpu().float().numpy() # (T, 6) + if mouse_cond is not None: + mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) + + # WanGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] + key_names = ["W", "S", "A", "D", "left", "right"] + + processed_frames = [] + for frame_idx, frame in enumerate(frames): + frame = np.ascontiguousarray(frame.copy()) + + # Draw keyboard overlay + if keyboard_cond is not None and frame_idx < len(keyboard_cond): + keys = { + key_names[i]: bool(keyboard_cond[frame_idx, i]) + for i in range(min(len(key_names), keyboard_cond.shape[1])) + } + draw_keys_on_frame(frame, keys, mode='universal') + + # Draw mouse overlay + if mouse_cond is not None and frame_idx < len(mouse_cond): + pitch = float(mouse_cond[frame_idx, 0]) + yaw = float(mouse_cond[frame_idx, 1]) + draw_mouse_on_frame(frame, pitch, yaw) + + processed_frames.append(frame) + + return processed_frames + + +def main(args) -> None: + logger.info("Starting WanGame self-forcing distillation pipeline...") + + pipeline = WanGameSelfForcingDistillationPipeline.from_pretrained( + args.pretrained_model_name_or_path, args=args) + + args = pipeline.training_args + pipeline.train() + logger.info("WanGame self-forcing distillation pipeline completed") + + +if __name__ == "__main__": + argv = sys.argv + from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.utils import FlexibleArgumentParser + parser = FlexibleArgumentParser() + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args() + main(args) From baac257fc498697682f0d658158d8a2d15f45609 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 23 Feb 2026 22:08:37 +0000 Subject: [PATCH 077/214] validation decouple from dmd and role --- dev/phases/phase_2_9.md | 5 ++ fastvideo/distillation/adapters/wan.py | 13 ---- fastvideo/distillation/builder.py | 8 ++- fastvideo/distillation/doc/builder.md | 3 +- fastvideo/distillation/doc/checkpoint.md | 3 +- fastvideo/distillation/doc/families/wan.md | 7 +- .../doc/methods/distribution_matching/dmd2.md | 7 +- fastvideo/distillation/doc/registry.md | 3 +- fastvideo/distillation/doc/runtime.md | 2 +- .../distillation/doc/validators/__init__.md | 2 +- fastvideo/distillation/doc/validators/base.md | 14 ++-- fastvideo/distillation/doc/validators/wan.md | 9 +-- fastvideo/distillation/families/wan.py | 3 +- .../methods/distribution_matching/dmd2.py | 53 ++++++++++++-- fastvideo/distillation/registry.py | 2 +- fastvideo/distillation/runtime.py | 2 +- fastvideo/distillation/validators/__init__.py | 3 +- fastvideo/distillation/validators/base.py | 20 +++++- fastvideo/distillation/validators/wan.py | 72 +++++++++++++------ fastvideo/training/distillation.py | 8 ++- 20 files changed, 166 insertions(+), 73 deletions(-) diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md index 1b495ff62..cbc418101 100644 --- a/dev/phases/phase_2_9.md +++ b/dev/phases/phase_2_9.md @@ -110,6 +110,7 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - few-step rollout 的 step list / simulate 逻辑从 adapter 迁移到 method（未来应进一步移到 `method_config`）。 - `WanAdapter` 不应包含 method-specific 命名/概念（例如不应依赖 `*DMD*Pipeline` 这类算法命名）。 - optimizer/scheduler 的创建归属 method（update policy），family 不再出现 DMD2/critic 专属超参（例如 `fake_score_*`）。 +- validation 归属 method：family 构建 `WanValidator`，method 负责决定是否/如何调用（通过 `ValidationRequest` 传参）。 - Phase 2 的训练行为/结果应尽可能保持一致（同 config 下 loss 形态、validation 产物趋势不应漂移）。 --- @@ -194,6 +195,10 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - [x] Wan family 不再创建 optimizers/schedulers - `fastvideo/distillation/families/wan.py` 只负责加载 modules + 构建 `ModelBundle` - `DMD2Method` 在 init 时为 student/critic 创建 optimizers/schedulers（复用 TrainingArgs 字段，未来迁移到 `method_config`） +- [x] Wan validator 不引入 method-specific 依赖 + - `fastvideo/distillation/validators/wan.py` 使用 `WanPipeline` 进行 validation 采样 + - validator 不 hardcode `bundle.role("student")` 等 role 语义；由 method 通过 `ValidationRequest.sample_handle` 指定采样模型 + - validator 不由 adapter 持有/调用；trainer 只调用 `method.log_validation(step)` --- diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 75dab6fe1..09bc851ca 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -54,13 +54,11 @@ def __init__( training_args: Any, noise_scheduler: Any, vae: Any, - validator: Any | None = None, ) -> None: self.prompt_handle = prompt_handle self.training_args = training_args self.noise_scheduler = noise_scheduler self.vae = vae - self._validator = validator self.world_group = get_world_group() self.sp_group = get_sp_group() @@ -124,11 +122,6 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: if self.noise_gen_cuda is not None: generators["noise_cuda"] = self.noise_gen_cuda - validator = getattr(self, "_validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen - return generators def ensure_negative_conditioning(self) -> None: @@ -221,12 +214,6 @@ def ensure_negative_conditioning(self) -> None: self.negative_prompt_embeds = neg_embeds self.negative_prompt_attention_mask = neg_mask - def log_validation(self, iteration: int) -> None: - validator = getattr(self, "_validator", None) - if validator is None: - return - validator.log_validation(iteration) - def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: if self.noise_random_generator is None: raise RuntimeError("WanAdapter.on_train_start() must be called before prepare_batch()") diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index 49e80586c..932d76140 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -19,7 +19,12 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: artifacts = family_builder(cfg=cfg) method_builder = get_method(str(cfg.distill.method)) - method = method_builder(cfg=cfg, bundle=artifacts.bundle, adapter=artifacts.adapter) + method = method_builder( + cfg=cfg, + bundle=artifacts.bundle, + adapter=artifacts.adapter, + validator=artifacts.validator, + ) return DistillRuntime( training_args=artifacts.training_args, @@ -40,4 +45,3 @@ def build_wan_dmd2_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: f"method={cfg.distill.method!r}" ) return build_runtime_from_config(cfg) - diff --git a/fastvideo/distillation/doc/builder.md b/fastvideo/distillation/doc/builder.md index 9f8b080b5..4e3a50e11 100644 --- a/fastvideo/distillation/doc/builder.md +++ b/fastvideo/distillation/doc/builder.md @@ -9,11 +9,10 @@ - `build_runtime_from_config(cfg) -> DistillRuntime` - `family_builder = registry.get_family(cfg.distill.model)` - `method_builder = registry.get_method(cfg.distill.method)` - - `method = method_builder(cfg=cfg, bundle=artifacts.bundle, adapter=artifacts.adapter)` + - `method = method_builder(cfg=cfg, bundle=artifacts.bundle, adapter=artifacts.adapter, validator=artifacts.validator)` - `build_wan_dmd2_runtime_from_config(cfg)` - Phase 2.9 期间保留的兼容函数（最终可删除） **边界** - ✅ 这里不写 `if model==... and method==...` 的 N×M 组合逻辑。 - ✅ 这里只做“装配”，不包含训练 loop / loss / rollout / optimizer policy。 - diff --git a/fastvideo/distillation/doc/checkpoint.md b/fastvideo/distillation/doc/checkpoint.md index 27478d32a..52612cd57 100644 --- a/fastvideo/distillation/doc/checkpoint.md +++ b/fastvideo/distillation/doc/checkpoint.md @@ -4,7 +4,7 @@ - Phase 2 的 role-based checkpoint/save-resume 管理： - 按 role 保存/恢复 modules、optimizers、schedulers - 可选保存 dataloader 状态（如果 dataloader 是 stateful） - - 保存 RNG（全局 RNG + adapter/validator 暴露的额外 generator） + - 保存 RNG（全局 RNG + method 暴露的额外 generators，例如 adapter/validator 的 RNG） **关键类型** - `DistillCheckpointConfig` @@ -25,4 +25,3 @@ **与 Method 的关系** - 该文件假设：训练开始前 `RoleHandle.optimizers/lr_schedulers` 已经就绪。 Phase 2.9 开始，它们通常由 method（例如 `DMD2Method`）在构造时创建并写回 handle。 - diff --git a/fastvideo/distillation/doc/families/wan.md b/fastvideo/distillation/doc/families/wan.md index bbffa1095..c4413a1cd 100644 --- a/fastvideo/distillation/doc/families/wan.md +++ b/fastvideo/distillation/doc/families/wan.md @@ -6,7 +6,7 @@ - 包含 Wan 特有的模块加载、shared components、dataloader schema 等逻辑 **产物** -- `FamilyArtifacts(training_args, bundle, adapter, dataloader, tracker, start_step)` +- `FamilyArtifacts(training_args, bundle, adapter, dataloader, tracker, validator, start_step)` **主要职责** 1) **加载 shared components** @@ -23,6 +23,10 @@ 4) **tracker / validator（可选）** - tracker：`initialize_trackers(...)`（rank0 才启用） - validator：`WanValidator`（当 `training_args.log_validation=true`） + - family 只负责构建并返回 `validator` + - validator 本身不应 hardcode `bundle.role("student")` 等角色语义； + method 通过 `ValidationRequest.sample_handle` 指定要采样的模型 + - 是否调用、用什么采样配置由 method 决定（method-managed validation） **Phase 2.9 的关键变化** - ✅ family 不再创建 optimizers/schedulers。 @@ -32,4 +36,3 @@ **注意 / TODO** - YAML 中目前仍使用 `training.fake_score_*` 这类字段作为 DMD2 的 critic 超参来源； Phase 3 计划把它们迁移到 `method_config`，进一步减少 “training_args 承载算法语义”。 - diff --git a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md index 201942c75..42cc180ca 100644 --- a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md +++ b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md @@ -39,7 +39,12 @@ - critic 的 optimizer/scheduler：优先使用 `training.fake_score_*` 覆盖（否则回退到 student） - 这样 Wan family 可以完全不“懂” DMD2 的 critic 超参，从 build-time 层面解耦。 +**validation 的归属（Phase 2.9）** +- `DMD2Method` 持有 family-specific `validator`（build-time 注入），并在 `log_validation()` 中调用。 +- method 通过 `ValidationRequest` 明确传入 sampling steps/guidance 等参数； + 同时通过 `ValidationRequest.sample_handle` 指定要采样的模型（通常是 student）， + validator 负责执行采样与记录，保持 method-agnostic。 + **配置语义的 TODO（Phase 3）** - 目前仍从 `training_args` 读取 DMD2/critic 专属字段（例如 `fake_score_*`、`dmd_denoising_steps`）。 Phase 3 计划引入 `method_config`，把这些算法超参从 `training:` / `pipeline_config:` 中迁移出去。 - diff --git a/fastvideo/distillation/doc/registry.md b/fastvideo/distillation/doc/registry.md index d1e9f7d84..83ebd75a2 100644 --- a/fastvideo/distillation/doc/registry.md +++ b/fastvideo/distillation/doc/registry.md @@ -8,7 +8,7 @@ **关键概念** - `FamilyBuilder(cfg) -> FamilyArtifacts` -- `MethodBuilder(cfg, bundle, adapter) -> DistillMethod` +- `MethodBuilder(cfg, bundle, adapter, validator) -> DistillMethod` **关键 API** - `register_family(name)` / `register_method(name)`：装饰器注册 @@ -19,4 +19,3 @@ **扩展方式** - 新增 family：实现 `fastvideo/distillation/families/.py` 并用 `@register_family("")` - 新增 method：实现 `fastvideo/distillation/methods/...` 并用 `@register_method("")` - diff --git a/fastvideo/distillation/doc/runtime.md b/fastvideo/distillation/doc/runtime.md index e604bc2d8..6e33ffe80 100644 --- a/fastvideo/distillation/doc/runtime.md +++ b/fastvideo/distillation/doc/runtime.md @@ -11,6 +11,7 @@ - `adapter` - `dataloader` - `tracker` + - `validator`（可选；family-specific） - `start_step`（用于 resume / warm-start） - `DistillRuntime` - `DistillTrainer.run()` 所需的最小集合： @@ -24,4 +25,3 @@ - family 负责把 “零件” 装配成 `FamilyArtifacts` - method 负责把算法绑定到（bundle + adapter）上 - trainer 只接收 `method` 并开始训练 - diff --git a/fastvideo/distillation/doc/validators/__init__.md b/fastvideo/distillation/doc/validators/__init__.md index 00b263900..bc96924b3 100644 --- a/fastvideo/distillation/doc/validators/__init__.md +++ b/fastvideo/distillation/doc/validators/__init__.md @@ -5,5 +5,5 @@ **当前导出** - `DistillValidator`：最小 validation 接口 +- `ValidationRequest`：method 提供的 validation overrides - `WanValidator`：Wan family 的 validation 采样与记录实现 - diff --git a/fastvideo/distillation/doc/validators/base.md b/fastvideo/distillation/doc/validators/base.md index 14569a701..55c800391 100644 --- a/fastvideo/distillation/doc/validators/base.md +++ b/fastvideo/distillation/doc/validators/base.md @@ -4,10 +4,14 @@ - 定义 distillation validator 的最小抽象接口。 **接口** -- `log_validation(step: int) -> None` +- `log_validation(step: int, request: ValidationRequest | None = None) -> None` -**设计意图** -- trainer 与 method 不需要知道 validator 的实现细节： - - adapter 可以持有 validator，并在合适的时机调用 - - 或 method/trainer 通过 hook 调用 `log_validation()` +`ValidationRequest` 用于 method 覆盖关键采样配置（steps/guidance/output_dir 等），让 validator +保持 family-specific、但 method-agnostic。 + +`ValidationRequest.sample_handle` 用于由 method 明确指定“本次 validation 要采样哪个模型/权重” +（例如 student / student_ema / refiner / ...）。validator 不应自行 hardcode 角色语义。 +**设计意图** +- trainer 只调用 `method.log_validation(step)`。 +- method 决定是否做 validation，并把 `ValidationRequest` 传给 family-specific validator。 diff --git a/fastvideo/distillation/doc/validators/wan.md b/fastvideo/distillation/doc/validators/wan.md index 7255fed71..6e807e4fb 100644 --- a/fastvideo/distillation/doc/validators/wan.md +++ b/fastvideo/distillation/doc/validators/wan.md @@ -11,13 +11,14 @@ **关键点** - validator 运行在分布式环境下： - 以 SP group 为单位做采样，最终由 global rank0 聚合写文件与 log -- 通过 `loaded_modules={"transformer": student_transformer}` 复用训练中的 student 模块权重。 +- 通过 `ValidationRequest.sample_handle` 获取本次要采样的 transformer， + 并以 `loaded_modules={"transformer": transformer}` 复用训练中的权重。 +- method 通过 `ValidationRequest` 覆盖采样配置（例如 sampling steps / guidance / output_dir）。 **依赖** -- 当前使用 `WanDMDPipeline` 做采样推理（FlowMatch scheduler + DmdDenoisingStage）。 - 这属于 validation 选择（并不影响 adapter 的“无算法命名耦合”约束）。 +- 当前使用 `WanPipeline` 做采样推理（Wan 通用 inference pipeline）。 + Phase 2.9 约束：validator 应保持 method-agnostic（不依赖具体 distillation method）。 **可演进方向（Phase 3+）** - 将 validation steps/guidance 等采样配置从 `TrainingArgs` 迁移到更明确的配置块（例如 `validation:`）。 - 进一步抽象 validator API，使其更容易被不同 family/method 复用。 - diff --git a/fastvideo/distillation/families/wan.py b/fastvideo/distillation/families/wan.py index cac4ed209..21c5dc6f4 100644 --- a/fastvideo/distillation/families/wan.py +++ b/fastvideo/distillation/families/wan.py @@ -174,7 +174,6 @@ def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: from fastvideo.distillation.validators.wan import WanValidator validator = WanValidator( - bundle=bundle, training_args=training_args, tracker=tracker, ) @@ -189,7 +188,6 @@ def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: training_args=training_args, noise_scheduler=noise_scheduler, vae=vae, - validator=validator, ) from fastvideo.dataset import build_parquet_map_style_dataloader @@ -213,5 +211,6 @@ def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: adapter=adapter, dataloader=dataloader, tracker=tracker, + validator=validator, start_step=0, ) diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index fbbc09e8f..612870ea5 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -16,6 +16,7 @@ from fastvideo.distillation.bundle import RoleHandle from fastvideo.distillation.methods.base import DistillMethod from fastvideo.distillation.registry import register_method +from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.distillation.yaml_config import DistillRunConfig @@ -87,9 +88,6 @@ def predict_noise( def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: ... - def log_validation(self, iteration: int) -> None: - ... - class DMD2Method(DistillMethod): """DMD2 distillation algorithm (method layer). @@ -110,6 +108,7 @@ def __init__( *, bundle: ModelBundle, adapter: _DMD2Adapter, + validator: Any | None = None, ) -> None: super().__init__(bundle) bundle.require_roles(["student", "teacher", "critic"]) @@ -123,6 +122,7 @@ def __init__( if not getattr(self.critic, "trainable", False): raise ValueError("DMD2Method requires models.critic.trainable=true") self.adapter = adapter + self.validator = validator self.training_args = adapter.training_args self._simulate_generator_forward = bool( getattr(self.training_args, "simulate_generator_forward", False) @@ -221,8 +221,48 @@ def on_train_start(self) -> None: self.adapter.on_train_start() def log_validation(self, iteration: int) -> None: - if hasattr(self.adapter, "log_validation"): - self.adapter.log_validation(iteration) + validator = getattr(self, "validator", None) + if validator is None: + return + if not getattr(self.training_args, "log_validation", False): + return + + raw_steps = str(getattr(self.training_args, "validation_sampling_steps", "") or "") + sampling_steps = [int(s) for s in raw_steps.split(",") if s.strip()] + sampling_steps = [s for s in sampling_steps if s > 0] + if not sampling_steps: + # Default to the few-step student rollout step count for DMD2. + raw_rollout = getattr(self.training_args.pipeline_config, "dmd_denoising_steps", None) + if not raw_rollout: + return + sampling_steps = [int(len(raw_rollout))] + + raw_guidance = getattr(self.training_args, "validation_guidance_scale", None) + guidance_scale = float(raw_guidance) if raw_guidance not in (None, "") else None + + request = ValidationRequest( + sample_handle=self.student, + sampling_steps=sampling_steps, + guidance_scale=guidance_scale, + ) + validator.log_validation(iteration, request=request) + + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + generators: dict[str, torch.Generator] = {} + + adapter = getattr(self, "adapter", None) + get_adapter_generators = getattr(adapter, "get_rng_generators", None) + if callable(get_adapter_generators): + generators.update(get_adapter_generators()) + + validator = getattr(self, "validator", None) + validation_gen = getattr(validator, "validation_random_generator", None) + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen + + return generators def _should_update_student(self, iteration: int) -> bool: interval = int(getattr(self.training_args, "generator_update_interval", 1) or 1) @@ -576,6 +616,7 @@ def build_dmd2_method( cfg: DistillRunConfig, bundle: ModelBundle, adapter: _DMD2Adapter, + validator: Any | None, ) -> DistillMethod: del cfg - return DMD2Method(bundle=bundle, adapter=adapter) + return DMD2Method(bundle=bundle, adapter=adapter, validator=validator) diff --git a/fastvideo/distillation/registry.py b/fastvideo/distillation/registry.py index 1ecd1808f..2e4cd9091 100644 --- a/fastvideo/distillation/registry.py +++ b/fastvideo/distillation/registry.py @@ -23,6 +23,7 @@ def __call__( cfg: DistillRunConfig, bundle: ModelBundle, adapter: Any, + validator: Any | None, ) -> DistillMethod: ... @@ -93,4 +94,3 @@ def get_method(name: str) -> MethodBuilder: if name not in _METHODS: raise KeyError(f"Unknown method {name!r}. Available: {available_methods()}") return _METHODS[name] - diff --git a/fastvideo/distillation/runtime.py b/fastvideo/distillation/runtime.py index 1051002bb..e0aa2c712 100644 --- a/fastvideo/distillation/runtime.py +++ b/fastvideo/distillation/runtime.py @@ -25,6 +25,7 @@ class FamilyArtifacts: adapter: Any dataloader: Any tracker: Any + validator: Any | None = None start_step: int = 0 @@ -37,4 +38,3 @@ class DistillRuntime: dataloader: Any tracker: Any start_step: int = 0 - diff --git a/fastvideo/distillation/validators/__init__.py b/fastvideo/distillation/validators/__init__.py index 571c5aa2f..84dcab5c6 100644 --- a/fastvideo/distillation/validators/__init__.py +++ b/fastvideo/distillation/validators/__init__.py @@ -1,9 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 -from fastvideo.distillation.validators.base import DistillValidator +from fastvideo.distillation.validators.base import DistillValidator, ValidationRequest from fastvideo.distillation.validators.wan import WanValidator __all__ = [ "DistillValidator", + "ValidationRequest", "WanValidator", ] diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py index cb732ee07..631b1f1d7 100644 --- a/fastvideo/distillation/validators/base.py +++ b/fastvideo/distillation/validators/base.py @@ -2,10 +2,28 @@ from __future__ import annotations +from dataclasses import dataclass from abc import ABC, abstractmethod +from fastvideo.distillation.bundle import RoleHandle + + +@dataclass(slots=True) +class ValidationRequest: + """Method-provided validation configuration overrides. + + Validators are family-specific (e.g. Wan sampling), but should remain + method-agnostic. A method may override key sampling parameters by passing a + request object here. + """ + + sample_handle: RoleHandle | None = None + sampling_steps: list[int] | None = None + guidance_scale: float | None = None + output_dir: str | None = None + class DistillValidator(ABC): @abstractmethod - def log_validation(self, step: int) -> None: + def log_validation(self, step: int, *, request: ValidationRequest | None = None) -> None: raise NotImplementedError diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 25257435a..1252a8631 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -19,7 +19,8 @@ from fastvideo.distributed import get_sp_group, get_world_group from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch -from fastvideo.pipelines.basic.wan.wan_dmd_pipeline import WanDMDPipeline +from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline +from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.utils import shallow_asdict logger = init_logger(__name__) @@ -37,11 +38,9 @@ class WanValidator: def __init__( self, *, - bundle: Any, training_args: Any, tracker: Any, ) -> None: - self.bundle = bundle self.training_args = training_args self.tracker = tracker @@ -57,7 +56,8 @@ def __init__( self.seed = int(seed) self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) - self._pipeline: WanDMDPipeline | None = None + self._pipeline: WanPipeline | None = None + self._pipeline_transformer_id: int | None = None self._sampling_param: SamplingParam | None = None def _get_sampling_param(self) -> SamplingParam: @@ -65,25 +65,26 @@ def _get_sampling_param(self) -> SamplingParam: self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) return self._sampling_param - def _get_pipeline(self) -> WanDMDPipeline: - if self._pipeline is not None: + def _get_pipeline(self, *, transformer: torch.nn.Module) -> WanPipeline: + transformer_id = id(transformer) + if self._pipeline is not None and self._pipeline_transformer_id == transformer_id: return self._pipeline args_copy = copy.deepcopy(self.training_args) args_copy.inference_mode = True - student_transformer = self.bundle.role("student").require_module("transformer") - self._pipeline = WanDMDPipeline.from_pretrained( + self._pipeline = WanPipeline.from_pretrained( self.training_args.model_path, - args=args_copy, # ignored in inference branch but keeps parity + args=args_copy, # inference_mode=True uses FastVideoArgs branch inference_mode=True, - loaded_modules={"transformer": student_transformer}, + loaded_modules={"transformer": transformer}, tp_size=self.training_args.tp_size, sp_size=self.training_args.sp_size, num_gpus=self.training_args.num_gpus, pin_cpu_memory=self.training_args.pin_cpu_memory, dit_cpu_offload=True, ) + self._pipeline_transformer_id = transformer_id return self._pipeline def _parse_validation_steps(self) -> list[int]: @@ -96,13 +97,17 @@ def _prepare_validation_batch( sampling_param: SamplingParam, validation_batch: dict[str, Any], num_inference_steps: int, + *, + guidance_scale: float | None = None, ) -> ForwardBatch: sampling_param.prompt = validation_batch["prompt"] sampling_param.height = self.training_args.num_height sampling_param.width = self.training_args.num_width sampling_param.num_inference_steps = num_inference_steps sampling_param.data_type = "video" - if getattr(self.training_args, "validation_guidance_scale", ""): + if guidance_scale is not None: + sampling_param.guidance_scale = float(guidance_scale) + elif getattr(self.training_args, "validation_guidance_scale", ""): sampling_param.guidance_scale = float(self.training_args.validation_guidance_scale) sampling_param.seed = self.seed @@ -129,9 +134,15 @@ def _prepare_validation_batch( ) return batch - def _run_validation_for_steps(self, num_inference_steps: int) -> _ValidationStepResult: + def _run_validation_for_steps( + self, + num_inference_steps: int, + *, + transformer: torch.nn.Module, + guidance_scale: float | None = None, + ) -> _ValidationStepResult: training_args = self.training_args - pipeline = self._get_pipeline() + pipeline = self._get_pipeline(transformer=transformer) sampling_param = self._get_sampling_param() dataset = ValidationDataset(training_args.validation_dataset_file) @@ -141,7 +152,12 @@ def _run_validation_for_steps(self, num_inference_steps: int) -> _ValidationStep captions: list[str] = [] for validation_batch in dataloader: - batch = self._prepare_validation_batch(sampling_param, validation_batch, num_inference_steps) + batch = self._prepare_validation_batch( + sampling_param, + validation_batch, + num_inference_steps, + guidance_scale=guidance_scale, + ) assert batch.prompt is not None and isinstance(batch.prompt, str) captions.append(batch.prompt) @@ -163,31 +179,41 @@ def _run_validation_for_steps(self, num_inference_steps: int) -> _ValidationStep return _ValidationStepResult(videos=videos, captions=captions) - def log_validation(self, step: int) -> None: + def log_validation(self, step: int, *, request: ValidationRequest | None = None) -> None: training_args = self.training_args if not getattr(training_args, "log_validation", False): return if not getattr(training_args, "validation_dataset_file", ""): raise ValueError("validation_dataset_file must be set when log_validation is enabled") - validation_steps = self._parse_validation_steps() + guidance_scale = getattr(request, "guidance_scale", None) + validation_steps = getattr(request, "sampling_steps", None) or self._parse_validation_steps() if not validation_steps: return - student_transformer = self.bundle.role("student").require_module("transformer") - was_training = bool(getattr(student_transformer, "training", False)) + sample_handle = getattr(request, "sample_handle", None) + if sample_handle is None: + raise ValueError("ValidationRequest.sample_handle must be provided by the method") + transformer = sample_handle.require_module("transformer") + was_training = bool(getattr(transformer, "training", False)) + + output_dir = getattr(request, "output_dir", None) or training_args.output_dir old_inference_mode = training_args.inference_mode old_dit_cpu_offload = training_args.dit_cpu_offload try: training_args.inference_mode = True training_args.dit_cpu_offload = True - student_transformer.eval() + transformer.eval() num_sp_groups = self.world_group.world_size // self.sp_group.world_size for num_inference_steps in validation_steps: - result = self._run_validation_for_steps(num_inference_steps) + result = self._run_validation_for_steps( + num_inference_steps, + transformer=transformer, + guidance_scale=guidance_scale, + ) if self.rank_in_sp_group != 0: continue @@ -202,12 +228,12 @@ def log_validation(self, step: int) -> None: all_videos.extend(recv_videos) all_captions.extend(recv_captions) - os.makedirs(training_args.output_dir, exist_ok=True) + os.makedirs(output_dir, exist_ok=True) video_filenames: list[str] = [] sampling_param = self._get_sampling_param() for i, video in enumerate(all_videos): filename = os.path.join( - training_args.output_dir, + output_dir, f"validation_step_{step}_inference_steps_{num_inference_steps}_video_{i}.mp4", ) imageio.mimsave(filename, video, fps=sampling_param.fps) @@ -228,4 +254,4 @@ def log_validation(self, step: int) -> None: training_args.inference_mode = old_inference_mode training_args.dit_cpu_offload = old_dit_cpu_offload if was_training: - student_transformer.train() + transformer.train() diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index e3720754f..8bc4b01d2 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -52,10 +52,12 @@ def run_distillation_from_config( keep_last=int(getattr(training_args, "checkpoints_total_limit", 0) or 0), ) - adapter = getattr(runtime.method, "adapter", None) - get_rng_generators = getattr(adapter, "get_rng_generators", None) + get_rng_generators = getattr(runtime.method, "get_rng_generators", None) if not callable(get_rng_generators): - get_rng_generators = None + adapter = getattr(runtime.method, "adapter", None) + get_rng_generators = getattr(adapter, "get_rng_generators", None) + if not callable(get_rng_generators): + get_rng_generators = None checkpoint_manager = DistillCheckpointManager( bundle=runtime.method.bundle, From 88c946d6c34d90bc3bd6a0da4282031440dfd04b Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Mon, 23 Feb 2026 22:19:12 +0000 Subject: [PATCH 078/214] some fix --- .gitignore | 6 +++++ .../distill/SFWanGame2.1/distill_dmd.slurm | 13 +++++------ .../causal_wangame_ode_init/validation.json | 10 -------- fastvideo/training/distillation_pipeline.py | 12 ++++++---- ...game_self_forcing_distillation_pipeline.py | 23 +++++++++++++------ 5 files changed, 36 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index ca1d6d319..a4cb0e2e0 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,9 @@ docs/distillation/examples/ dmd_t2v_output/ preprocess_output_text/ +wangame_1.3b_overfit +wangame_1.3b_overfit_output +wangame_lingbot_test + +ode_train_output +sf_train_output diff --git a/examples/distill/SFWanGame2.1/distill_dmd.slurm b/examples/distill/SFWanGame2.1/distill_dmd.slurm index 7c6b19812..e2acd5460 100644 --- a/examples/distill/SFWanGame2.1/distill_dmd.slurm +++ b/examples/distill/SFWanGame2.1/distill_dmd.slurm @@ -16,7 +16,7 @@ set -e -x # Environment Setup source ~/conda/miniconda/bin/activate conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv -export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin:$PYTHONPATH" +export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" # Basic Info export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" @@ -36,13 +36,12 @@ echo "RUN_NAME: $RUN_NAME" # Model paths for Self-Forcing DMD distillation: # GENERATOR_MODEL_PATH="../wg_models/WanGame-2.1-Student-VizDoom1k-1000steps-Diffusers" -GENERATOR_MODEL_PATH="../wg_models/WanGame-2.1-Student-VizDoom1k-1000steps-Diffusers" -REAL_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-Foundation-VizDoom1k-1000steps-Diffusers" # Teacher model -FAKE_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-Foundation-VizDoom1k-1000steps-Diffusers" # Critic model +GENERATOR_MODEL_PATH="../wg_models/SFWanGame-2.1-0223-9000steps" +REAL_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Teacher model +FAKE_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Critic model -DATA_DIR="../traindata_0209_1500/wg_ode_init/preprocessed" -VALIDATION_DATASET_FILE="examples/distill/SFWanGame2.1/validation.json" -CKPT_SAFETENSOR="checkpoints/wangame_ode_init_vizdoom_new/checkpoint-6000/transformer/diffusion_pytorch_model.safetensors" +DATA_DIR="../traindata_0222_0030/ode_init_mc_with_mouse/preprocessed_wangame" +VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" # Training arguments training_args=( diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json index d151bfc07..b796d925f 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json @@ -50,16 +50,6 @@ "width": 640, "num_frames": 81 }, - { - "caption": "04: Idle", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000004_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, { "caption": "05: camera down", "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000005.jpg", diff --git a/fastvideo/training/distillation_pipeline.py b/fastvideo/training/distillation_pipeline.py index d699f7b10..24bc92b39 100644 --- a/fastvideo/training/distillation_pipeline.py +++ b/fastvideo/training/distillation_pipeline.py @@ -1251,10 +1251,14 @@ def run_validation_with_ema( prompt_embeds=[], prompt_attention_mask=[], ) - result_batch = self.validation_pipeline.prompt_encoding_stage( # type: ignore - batch_negative, training_args) - self.negative_prompt_embeds, self.negative_prompt_attention_mask = result_batch.prompt_embeds[ - 0], result_batch.prompt_attention_mask[0] + if hasattr(self.validation_pipeline, "prompt_encoding_stage"): + result_batch = self.validation_pipeline.prompt_encoding_stage( # type: ignore + batch_negative, training_args) + self.negative_prompt_embeds, self.negative_prompt_attention_mask = result_batch.prompt_embeds[ + 0], result_batch.prompt_attention_mask[0] + else: + self.negative_prompt_embeds = None + self.negative_prompt_attention_mask = None logger.info( "rank: %s: rank_in_sp_group: %s, batch.prompt: %s", diff --git a/fastvideo/training/wangame_self_forcing_distillation_pipeline.py b/fastvideo/training/wangame_self_forcing_distillation_pipeline.py index 8723b2cb1..0325826ac 100644 --- a/fastvideo/training/wangame_self_forcing_distillation_pipeline.py +++ b/fastvideo/training/wangame_self_forcing_distillation_pipeline.py @@ -626,13 +626,16 @@ def _build_distill_input_kwargs( image_latents = image_latents[:, :, frame_start:frame_end, :, :] vae_temporal_compression_ratio = 4 - if frame_end is not None: + if frame_start is not None and frame_end is not None: + action_frame_start = frame_start * vae_temporal_compression_ratio action_frame_end = (frame_end - 1) * vae_temporal_compression_ratio + 1 - keyboard_cond_sliced = training_batch.keyboard_cond[:, : - action_frame_end, :] if training_batch.keyboard_cond is not None else None - mouse_cond_sliced = training_batch.mouse_cond[:, : - action_frame_end, :] if training_batch.mouse_cond is not None else None + if frame_start == 0: + action_frame_start = 0 + keyboard_cond_sliced = training_batch.keyboard_cond[:, + action_frame_start:action_frame_end, :] if training_batch.keyboard_cond is not None else None + mouse_cond_sliced = training_batch.mouse_cond[:, + action_frame_start:action_frame_end, :] if training_batch.mouse_cond is not None else None else: keyboard_cond_sliced = training_batch.keyboard_cond mouse_cond_sliced = training_batch.mouse_cond @@ -857,14 +860,20 @@ def _prepare_validation_batch(self, sampling_param: SamplingParam, if "keyboard_cond" in validation_batch and validation_batch[ "keyboard_cond"] is not None: keyboard_cond = validation_batch["keyboard_cond"] - keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) + if isinstance(keyboard_cond, torch.Tensor): + keyboard_cond = keyboard_cond.detach().clone().to(dtype=torch.bfloat16) + else: + keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) keyboard_cond = keyboard_cond.unsqueeze(0) batch.keyboard_cond = keyboard_cond if "mouse_cond" in validation_batch and validation_batch[ "mouse_cond"] is not None: mouse_cond = validation_batch["mouse_cond"] - mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) + if isinstance(mouse_cond, torch.Tensor): + mouse_cond = mouse_cond.detach().clone().to(dtype=torch.bfloat16) + else: + mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) mouse_cond = mouse_cond.unsqueeze(0) batch.mouse_cond = mouse_cond From cb09285b43cab26524520c04f270ddf6997fdf1f Mon Sep 17 00:00:00 2001 From: mignonjia Date: Mon, 23 Feb 2026 22:50:55 +0000 Subject: [PATCH 079/214] freeze action slurm: Doom from MC --- .../finetune_wangame_freeze_action.slurm | 190 ++++++++++++++++++ fastvideo/fastvideo_args.py | 15 +- 2 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm new file mode 100644 index 000000000..767bb797a --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm @@ -0,0 +1,190 @@ +#!/bin/bash +#SBATCH --job-name=wangame_1.3b +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=wangame_1.3b_output/wangame_1.3b_%j.out +#SBATCH --error=wangame_1.3b_output/wangame_1.3b_%j.err +#SBATCH --exclusive + +# Basic Info +export NCCL_P2P_DISABLE=1 +export TORCH_NCCL_ENABLE_MONITORING=0 +export NCCL_DEBUG_SUBSYS=INIT,NET +# different cache dir for different processes +export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} +export MASTER_PORT=29501 +export NODE_RANK=$SLURM_PROCID +nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) +export MASTER_ADDR=${nodes[0]} +export TOKENIZERS_PARALLELISM=false +export WANDB_API_KEY="d5b02b05e30d8cb34c7b31c6ae10416fc26dcb66" +export WANDB_BASE_URL="https://api.wandb.ai" +export WANDB_MODE=online +export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN +export FASTVIDEO_MAP_STYLE_CACHE_DIR="/mnt/weka/home/hao.zhang/mhuo/FastVideo/map_style_cache" + +source ~/conda/miniconda/bin/activate +conda activate mhuo-fv +export HOME="/mnt/weka/home/hao.zhang/mhuo" + +# Configs +NUM_GPUS=8 +NUM_NODES=4 # TODO: change this to 1 to debug +NUM_TOTAL_GPUS=$((NUM_GPUS * NUM_NODES)) +BS_PER_GPU=1 +GRADIENT_ACCUMULATION_STEPS=1 +WANDB_RUN_NAME="Doom from MC freeze action" +RUN_DIR="wangame_1.3b" +CHECKPOINTING_STEPS=100000 +ACTION_WARMUP_STEPS=100000 +LEARNING_RATE=1e-5 +# Freeze base DiT, only train action modules +Freeze_DiT=false + +MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" +# CKPT_SAFETENSOR="wangame_1.3b_wsad_random_lr_1e-5/checkpoint-2000/transformer/diffusion_pytorch_model.safetensors" +# CKPT_SAFETENSOR="wangame_1.3b_with_warmup_lr_1e-5/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" +CKPT_SAFETENSOR="wangame_1.3b_1action_rand_from_scratch/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" + +# Data dirs (use one of the following): +# - DATA_DIR_ALL: all datasets below combined (comma-separated) +# - Or a single path / subset; optional ":N" = repeat, ":0" = skip +# +DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0" # Random +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:1" # Doom +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:0" # Static + w only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:0" # w/s/a/d only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:0" # wasd only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:0" # camera l-only and r-only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:0" # camera only +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:0" # key_camera_excl_1_action_rand +DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/alex/wm-lab/datas/cache/zelda_overfit:0" # zelda + +VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json" # TODO: double check this, you can remove some MC data and add more doom data +# +# Single-dir / validation alternatives (comment out DATA_DIR above and uncomment one block): +# MC wasd only: +# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json" +# MC random: +# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" +# Doom: +# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_doom.json" +# Overfit: +# DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" +# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_overfit.json" + + +# Training arguments +training_args=( + --tracker_project_name "wangame_1.3b" + --output_dir $RUN_DIR + --wandb_run_name "$WANDB_RUN_NAME" + --max_train_steps 10000 + --train_batch_size $BS_PER_GPU + --train_sp_batch_size $BS_PER_GPU + --gradient_accumulation_steps $GRADIENT_ACCUMULATION_STEPS + --num_latent_t 20 + --num_height 352 + --num_width 640 + --num_frames 77 + --enable_gradient_checkpointing_type "full" + --action_warmup_steps $ACTION_WARMUP_STEPS + --init_weights_from_safetensors $CKPT_SAFETENSOR + # TODO: check terminal log or log file xxx.err, whether there is a output line saying "Starting training with 1.53 B trainable parameters (total)". If both action modules are frozen, it should be around 1.53B, otherwise it could be 1.6B. +) + +if [ "$FREEZE_DiT" = "true" ]; then + training_args+=(--train_action_only) +fi + +# Parallel arguments +parallel_args=( + --num_gpus $NUM_TOTAL_GPUS + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim $NUM_TOTAL_GPUS +) + +# Model arguments +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +# Dataset arguments +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 1 +) + +# Validation arguments +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 100 + --validation_sampling_steps "40" + --validation_guidance_scale "1.0" + --validation_num_samples $NUM_TOTAL_GPUS +) + +# Optimizer arguments +optimizer_args=( + --learning_rate $LEARNING_RATE + --mixed_precision "bf16" + --weight_only_checkpointing_steps 100000 + --training_state_checkpointing_steps $CHECKPOINTING_STEPS + --weight_decay 1e-4 + --max_grad_norm 1.0 +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 2 + --training_cfg_rate 0.1 + --multi_phased_distill_schedule "4000-1" + --not_apply_cfg_solver + --dit_precision "fp32" + --num_euler_timesteps 50 + --ema_start_step 0 +) + +# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t + +if [ $NUM_NODES -eq 1 ]; then + torchrun \ + --nnodes $NUM_NODES \ + --nproc_per_node $NUM_GPUS \ + fastvideo/training/wangame_training_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" +else + srun torchrun \ + --nnodes $NUM_NODES \ + --nproc_per_node $NUM_GPUS \ + --rdzv_backend c10d \ + --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT \ + --node_rank $SLURM_PROCID \ + fastvideo/training/wangame_training_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" +fi \ No newline at end of file diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index 4f10969fb..1e651d293 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -907,9 +907,15 @@ class TrainingArgs(FastVideoArgs): lora_training: bool = False ltx2_first_frame_conditioning_p: float = 0.1 - # Action-only training (freeze base model, only train action params) + # Action-only training: freeze base DiT, only train action modules train_action_only: bool = False + # Which action modules to train (only effective when train_action_only=True): + # "both" – action_embedder + prope_proj (default) + # "action_mlp" – action_embedder only + # "prope" – prope_proj only + action_train_target: str = "both" + # Action warmup: keep action modules (action_embedder, to_out_prope) at zero # for this many steps to let the base model stabilize first, then enable them. action_warmup_steps: int = 0 @@ -1392,6 +1398,13 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: type=int, default=TrainingArgs.context_noise, help="Context noise level for cache updates") + parser.add_argument( + "--action-train-target", + type=str, + default=TrainingArgs.action_train_target, + choices=["both", "action_mlp", "prope"], + help="Which action modules to train while freezing the base model", + ) return parser From 743b04d049b960550d8b403f4070e8a2fd7de802 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Mon, 23 Feb 2026 22:54:12 +0000 Subject: [PATCH 080/214] some comment to the slurm --- .../finetune_wangame_freeze_action.slurm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm index 767bb797a..fb0d30b24 100644 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm @@ -40,8 +40,9 @@ BS_PER_GPU=1 GRADIENT_ACCUMULATION_STEPS=1 WANDB_RUN_NAME="Doom from MC freeze action" RUN_DIR="wangame_1.3b" -CHECKPOINTING_STEPS=100000 -ACTION_WARMUP_STEPS=100000 +CHECKPOINTING_STEPS=100000 # This means checkpoint every 100000 steps, effectively no ckpt will be saved +ACTION_WARMUP_STEPS=100000 # This means the action modules will be frozen for the first 100000 steps. +# Effectively, during the total 100000 steps, action modules are always frozen. LEARNING_RATE=1e-5 # Freeze base DiT, only train action modules Freeze_DiT=false @@ -65,7 +66,7 @@ DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camer DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:0" # key_camera_excl_1_action_rand DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/alex/wm-lab/datas/cache/zelda_overfit:0" # zelda -VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json" # TODO: double check this, you can remove some MC data and add more doom data +VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json" # TODO: double check this, you may remove some MC image and add more doom image # # Single-dir / validation alternatives (comment out DATA_DIR above and uncomment one block): # MC wasd only: From c3c75c2f59a1ce35d30fa214b920c27bbae8a13f Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 00:21:56 +0000 Subject: [PATCH 081/214] fix circular import and designing phase 2.9: validation config --- dev/phases/phase_2_9.md | 80 +++++++++++++++++++ dev/phases/phase_3.md | 18 ++++- examples/distillation/phase2_9/temp.sh | 42 ++++++++++ fastvideo/distillation/adapters/wan.py | 9 ++- .../distillation/doc/methods/__init__.md | 3 + fastvideo/distillation/methods/__init__.py | 17 +++- 6 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 examples/distillation/phase2_9/temp.sh diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md index cbc418101..412969a7e 100644 --- a/dev/phases/phase_2_9.md +++ b/dev/phases/phase_2_9.md @@ -265,3 +265,83 @@ method 决定如何 sample（policy），adapter 负责把 sample 结果转换 如果同时改 YAML schema（`distill` -> `recipe`）会叠加变量，出现问题时很难定位。 因此 Phase 2.9 先保证内部语义正确，再在 Phase 3 做 schema 升级与 finetune 接入。 + +--- + +## 7) End-to-end 验证（建议） + +- 训练脚本：`examples/distillation/phase2_9/temp.sh` + - 默认读取：`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` + - 输出建议：`bash examples/distillation/phase2_9/temp.sh |& tee examples/distillation/phase2_9/temp.log` + +--- + +## 8) TODO（Phase 2.9 follow-up）：validation 采样语义彻底回归 method（解决 WanPipeline vs WanDMDPipeline drift） + +### 8.1 背景：为什么会 drift？ + +当前 Phase 2.9 的 `WanValidator` 为了保持 “validator method-agnostic / 不出现 DMD 依赖”， +使用的是 `WanPipeline`（默认 scheduler=UniPC）。而 legacy pipeline 的 validation 使用的是 +`WanDMDPipeline`（scheduler=FlowMatchEulerDiscrete + 固定 `dmd_denoising_steps` 的 rollout）。 + +即便训练完全一致，这两条 sampling 路径也可能产生 **不同的 validation video**，导致： +- 难以与 legacy 结果 apples-to-apples 对比（尤其是 few-step 的 timesteps schedule） +- method 的 sampling policy（timesteps/solver）被隐式地固定在 validator 的实现选择里 + +因此：**“用什么 schedule / solver 做 validation sampling” 必须由 method 决定**（或未来的 `method_config` 决定）， +validator 只负责执行与记录。 + +### 8.2 目标（Design Constraints） + +- validator 仍然 **family-specific + method-agnostic** + - 不 import / 不依赖 `WanDMDPipeline` / `DmdDenoisingStage` / 任何 DMD 语义 +- method 负责提供 sampling policy（而不是 validator 自己猜） + - explicit timesteps（例如 DMD few-step schedule） + - 使用的 scheduler/solver 类型（以中性的术语描述，例如 `flowmatch_euler` vs `unipc`） + - guidance/output_dir/sample_handle 等 + +### 8.3 建议方案（最小可落地） + +扩展 `ValidationRequest`，让 method 显式描述 sampling policy： + +- `sampling_timesteps: list[int] | None` + - 若提供，则 **优先于** `sampling_steps`（num_inference_steps） + - 用于表达 few-step 的固定 schedule（例如 `[1000, 850, ...]`） +- `scheduler_kind: Literal["unipc", "flowmatch_euler"] | None` + - 以“solver/scheduler”术语描述，而不是 “dmd” 术语 + +`WanValidator` 执行策略： +- 默认（保持现状）：`scheduler_kind="unipc"` → `WanPipeline`（UniPC）+ `num_inference_steps` +- 当 `scheduler_kind="flowmatch_euler"` 且 `sampling_timesteps` 非空： + - 使用 `WanPipeline` 但注入 `FlowMatchEulerDiscreteScheduler(shift=flow_shift)` + - 在 `ForwardBatch` 里填 `timesteps=sampling_timesteps` + - 让 `TimestepPreparationStage` 走 “custom timesteps” 分支（`scheduler.set_timesteps(timesteps=...)`） + +> 备注：这条路径不引入任何 DMD pipeline/stage 依赖；但可以让 validation 的 timesteps/scheduler +> 由 method 精确控制，从而减少 drift，并便于对齐 legacy 的 few-step schedule。 + +### 8.4 文件 TODO（实现清单） + +- [ ] `fastvideo/distillation/validators/base.py` + - 扩展 `ValidationRequest`：增加 `sampling_timesteps` / `scheduler_kind` +- [ ] `fastvideo/distillation/methods/distribution_matching/dmd2.py` + - `log_validation()` 构造 request 时： + - `request.sampling_timesteps = pipeline_config.dmd_denoising_steps` + - `request.scheduler_kind = "flowmatch_euler"` +- [ ] `fastvideo/distillation/validators/wan.py` + - 支持根据 request 选择 scheduler/policy（保持默认 unipc） + - 当指定 flowmatch_euler + timesteps 时，走 “custom timesteps + FlowMatchEuler scheduler” 路径 +- [ ] docs + - `fastvideo/distillation/doc/validators/base.md` + - `fastvideo/distillation/doc/validators/wan.md` + - `fastvideo/distillation/doc/methods/distribution_matching/dmd2.md` + +### 8.5 风险与后续（如需 1:1 复刻 legacy） + +即使统一 timesteps + scheduler，`WanPipeline` 的 denoising loop 仍可能与 legacy 的 +`DmdDenoisingStage` 不完全一致（legacy 有自己的 loop / noise reinjection 语义）。 + +如果我们需要 **严格复刻** legacy sampling 的每一个细节，建议在 Phase 3 做更系统的抽象： +- method（或 `method_config`）提供 “rollout plan” +- validator 仅做 dataset + logging +- 具体 rollout 执行由一个中性组件实现（例如 `sampling/rollout.py`），并且不携带 DMD 命名 diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 26d0110f8..c4d589ba3 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -97,6 +97,23 @@ Phase 3 的定位：在 Phase 2 已经证明“新 distill 框架可独立运行 - `prepare_batch()`（要求 finetune 路径必须提供真实 `vae_latent`） - `set_forward_context(...)` 的管理继续留在 adapter +### B5.5 彻底去除 adapter 对 method knob 的读取（例如 `simulate_generator_forward`） + +动机：当前 `WanAdapter.prepare_batch()` 仍读取 `training_args.simulate_generator_forward` 来决定 +是否需要 `vae_latent`（或构造 placeholder）。这会把 DMD2/student-rollout 的语义泄漏到 adapter， +违背 Phase 2.9 的 operation-centric 边界。 + +- [ ] 调整 `WanAdapter` 的 batch API，使其不再读取 `training_args.simulate_generator_forward` + - 选项 A（推荐）：拆成两个显式入口 + - `prepare_batch_with_vae_latent(raw_batch, ...)`：必须有 `vae_latent` + - `prepare_batch_text_only(raw_batch, ...)`：不依赖 `vae_latent`，只根据 family 规则构造 latents/shape + - 选项 B：保留单入口但显式参数化 + - `prepare_batch(raw_batch, *, latents_source=...)` + - 由 method/`method_config` 决定使用 data latents 还是 placeholder latents +- [ ] 将 “是否 simulate / rollout 模式” 的开关迁移到 `method_config` + - DMD2：例如 `method_config.rollout_mode = "data_latent" | "simulate"` + - adapter 不读取该开关；method 选择调用哪个 adapter API + ### B6. examples / outside configs 更新到 schema v2 - [ ] 更新 DMD2 YAML（schema v2）： @@ -234,4 +251,3 @@ student: 在语义上是等价的（都是一个 mapping）。Phase 3 的 examples/docs 我们将统一采用 **block style**（你偏好的后者），因为更可读、diff 更友好。 - diff --git a/examples/distillation/phase2_9/temp.sh b/examples/distillation/phase2_9/temp.sh new file mode 100644 index 000000000..cf67c49c1 --- /dev/null +++ b/examples/distillation/phase2_9/temp.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -euo pipefail +if [[ "${DEBUG:-0}" == "1" ]]; then + set -x +fi + +# One-shot launch script for Phase 2.9 (Families + registry dispatch + +# operation-centric adapter + method-managed validation). +# +# Uses the same dataset/validation defaults as phase0/phase1/phase2; the main +# difference is internal wiring/decoupling, not hyperparameters. + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29509} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 09bc851ca..8683970e8 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -82,10 +82,11 @@ def _init_timestep_mechanics(self) -> None: self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) boundary_ratio = getattr(self.training_args, "boundary_ratio", None) - if boundary_ratio is not None: - self.boundary_timestep = float(boundary_ratio) * float(self.num_train_timestep) - else: - self.boundary_timestep = None + self.boundary_timestep: float | None = ( + float(boundary_ratio) * float(self.num_train_timestep) + if boundary_ratio is not None + else None + ) @property def num_train_timesteps(self) -> int: diff --git a/fastvideo/distillation/doc/methods/__init__.md b/fastvideo/distillation/doc/methods/__init__.md index 509718d50..f62cb6be8 100644 --- a/fastvideo/distillation/doc/methods/__init__.md +++ b/fastvideo/distillation/doc/methods/__init__.md @@ -11,3 +11,6 @@ - method 层应当是 **模型无关** 的（不 import 具体 pipeline/模型实现）； 任何 family 细节都通过 adapter primitives（protocol）注入。 +**实现细节** +- 该模块对 `DMD2Method` 使用 lazy import（`__getattr__`），避免 registry/builder 在 + import 时触发循环依赖（circular import）。 diff --git a/fastvideo/distillation/methods/__init__.py b/fastvideo/distillation/methods/__init__.py index dd132ce2c..00ab6b5fc 100644 --- a/fastvideo/distillation/methods/__init__.py +++ b/fastvideo/distillation/methods/__init__.py @@ -1,9 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from typing import TYPE_CHECKING + from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.methods.distribution_matching import DMD2Method + +if TYPE_CHECKING: + from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method __all__ = [ "DistillMethod", "DMD2Method", ] + + +def __getattr__(name: str) -> object: + # Lazy import to avoid circular imports during registry bring-up. + if name == "DMD2Method": + from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method + + return DMD2Method + raise AttributeError(name) From 601da9d0da22fbda3280b110ddafd799e4f15a42 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 01:41:17 +0000 Subject: [PATCH 082/214] validator still use WanDMDPipeline. Future decoupling will be done in phase 3 --- dev/design.md | 18 ++++- dev/phases/phase_2_9.md | 84 ++++++-------------- dev/phases/phase_3.md | 39 +++++++++ fastvideo/distillation/doc/validators/wan.md | 5 +- fastvideo/distillation/validators/wan.py | 8 +- 5 files changed, 86 insertions(+), 68 deletions(-) diff --git a/dev/design.md b/dev/design.md index aafc6cc04..2c6e2f1bb 100644 --- a/dev/design.md +++ b/dev/design.md @@ -585,7 +585,7 @@ Phase 1 的“辉煌”（落地与收益）： - 新增更多 method（teacher-only、多 teacher、KD 轨迹蒸馏等） - 逐步冻结或移除旧 distill pipeline（保留兼容入口亦可） -### Phase 2.9（计划）：A+B+Families 语义收敛（为 Phase 3 铺路） +### Phase 2.9（已完成）：A+B+Families 语义收敛（为 Phase 3 铺路） 动机：Phase 2 已经完成“独立可跑”，但仍存在几个会阻碍长期扩展的结构性问题： @@ -603,6 +603,15 @@ Phase 2.9 目标（简称 A+B+Families）： 详细执行清单见：`dev/phases/phase_2_9.md` +补充：Phase 2.9 实践过程中暴露了一个之前设计阶段没充分考虑的点—— +**distillation/sampling method 可能不仅仅改变 timesteps/scheduler，而是会改变 denoising loop 本身** +（例如 DMD2 的 SDE 风格 `pred_x0 -> add_noise(next_t, eps)` 与常规 ODE/solver 风格 `scheduler.step(...)`）。 +这会直接导致 validation sampling 的 drift：即使训练一致，选不同 loop 的 pipeline 也可能出不同视频。 + +Phase 2.9 为了端到端与 legacy apples-to-apples，对 Wan DMD2 的 validation 暂时使用 +`WanDMDPipeline`（SDE rollout）以避免漂移；Phase 3 会把它升级为可插拔的 ODE/SDE sampler， +从而淘汰 `Pipeline` 这种耦合。 + ### Phase 3（计划）：Recipe config + method_config + Finetuning（统一到同一框架） Phase 3 的定位：在 Phase 2.9 已经完成“优雅 dispatch + adapter/method 语义收敛”的基础上： @@ -610,6 +619,13 @@ Phase 3 的定位：在 Phase 2.9 已经完成“优雅 dispatch + adapter/metho 1) **配置语义升级（`distill` -> `recipe`，引入 `method_config`）** 2) **把 finetuning 作为一种 method 接入框架**（只需要 `student` + dataset） +3) **统一 sampling 语义（ODE/SDE sampler 可插拔）** + - 背景：DMD2/Consistency 等方法可能需要不同的 denoising loop；如果靠 “每个 method 一个 pipeline 变体” + 会重新回到 N×M 的组合爆炸。 + - 目标：让 `WanPipeline`（以及未来其它 family 的 pipeline）通过参数选择 sampler/integrator（ODE vs SDE）， + validation 由 method/method_config 显式指定 sampler+steps，validator 只做 dataset+logging。 + - 结果：新框架不再需要依赖 `WanDMDPipeline`（保留 legacy 兼容即可）。 + #### Phase 3.1：配置语义升级（`distill` -> `recipe`，引入 `method_config`） 动机： diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md index 412969a7e..16ff29bc6 100644 --- a/dev/phases/phase_2_9.md +++ b/dev/phases/phase_2_9.md @@ -195,10 +195,11 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - [x] Wan family 不再创建 optimizers/schedulers - `fastvideo/distillation/families/wan.py` 只负责加载 modules + 构建 `ModelBundle` - `DMD2Method` 在 init 时为 student/critic 创建 optimizers/schedulers（复用 TrainingArgs 字段，未来迁移到 `method_config`） -- [x] Wan validator 不引入 method-specific 依赖 - - `fastvideo/distillation/validators/wan.py` 使用 `WanPipeline` 进行 validation 采样 +- [x] Wan validator 归属 method（method 决定是否/如何调用 validation） + - `fastvideo/distillation/validators/wan.py` 当前使用 `WanDMDPipeline` 进行 validation 采样（对齐 DMD2 的 SDE rollout，便于与 legacy apples-to-apples 对比） - validator 不 hardcode `bundle.role("student")` 等 role 语义；由 method 通过 `ValidationRequest.sample_handle` 指定采样模型 - validator 不由 adapter 持有/调用；trainer 只调用 `method.log_validation(step)` + - TODO(Phase 3)：通过 “ODE/SDE sampler 可插拔” 的方式淘汰 `WanDMDPipeline`（validator 回到 method-agnostic） --- @@ -276,72 +277,33 @@ method 决定如何 sample（policy），adapter 负责把 sample 结果转换 --- -## 8) TODO（Phase 2.9 follow-up）：validation 采样语义彻底回归 method（解决 WanPipeline vs WanDMDPipeline drift） +## 8) TODO（Phase 3）：统一 ODE/SDE sampling 语义（淘汰 `WanDMDPipeline`） -### 8.1 背景：为什么会 drift？ +### 8.1 背景：为什么 WanPipeline 和 DMD2 sampling 会不一致？ -当前 Phase 2.9 的 `WanValidator` 为了保持 “validator method-agnostic / 不出现 DMD 依赖”， -使用的是 `WanPipeline`（默认 scheduler=UniPC）。而 legacy pipeline 的 validation 使用的是 -`WanDMDPipeline`（scheduler=FlowMatchEulerDiscrete + 固定 `dmd_denoising_steps` 的 rollout）。 +- DMD2 的 few-step rollout（以及 legacy `WanDMDPipeline`）本质是 **SDE 风格**： + 每一步先得到 `pred_x0`，再用 `add_noise(pred_x0, eps, next_t)` **重加噪**进入下一步。 +- `WanPipeline` 的默认采样 loop 是 **ODE/solver 风格**： + 每一步用 `scheduler.step(noise_pred, t, latents)` 直接更新 latents（并包含 CFG 双 forward + mix）。 -即便训练完全一致，这两条 sampling 路径也可能产生 **不同的 validation video**，导致： -- 难以与 legacy 结果 apples-to-apples 对比（尤其是 few-step 的 timesteps schedule） -- method 的 sampling policy（timesteps/solver）被隐式地固定在 validator 的实现选择里 +因此即使我们把 timesteps/scheduler “看起来”统一，**只要 loop 更新公式不同，采样结果就可能 drift**。 -因此：**“用什么 schedule / solver 做 validation sampling” 必须由 method 决定**（或未来的 `method_config` 决定）， -validator 只负责执行与记录。 +### 8.2 Phase 2.9 的取舍（为了端到端对齐） -### 8.2 目标（Design Constraints） +为了确认 Phase 2.9 的新训练链路与 legacy 能 apples-to-apples 对比， +当前 `WanValidator` 直接改为使用 `WanDMDPipeline` 做采样（对齐 SDE rollout）。 -- validator 仍然 **family-specific + method-agnostic** - - 不 import / 不依赖 `WanDMDPipeline` / `DmdDenoisingStage` / 任何 DMD 语义 -- method 负责提供 sampling policy（而不是 validator 自己猜） - - explicit timesteps（例如 DMD few-step schedule） - - 使用的 scheduler/solver 类型（以中性的术语描述，例如 `flowmatch_euler` vs `unipc`） - - guidance/output_dir/sample_handle 等 +缺点：validator 仍依赖一个带 method 命名的 pipeline 变体（不够优雅）。 -### 8.3 建议方案（最小可落地） +### 8.3 Phase 3 的目标：让 sampling 语义可插拔（ODE/SDE），而不是靠 “Pipeline” -扩展 `ValidationRequest`，让 method 显式描述 sampling policy： +在 Phase 3 我们应把 “denoising loop” 抽象成可注入的 sampler/integrator： +- `OdeSampler`：`scheduler.step(...)`（当前 `DenoisingStage` 的语义） +- `SdeSampler`：`pred_x0 -> add_noise(next_t, eps)`（当前 `DmdDenoisingStage` 的语义） -- `sampling_timesteps: list[int] | None` - - 若提供，则 **优先于** `sampling_steps`（num_inference_steps） - - 用于表达 few-step 的固定 schedule（例如 `[1000, 850, ...]`） -- `scheduler_kind: Literal["unipc", "flowmatch_euler"] | None` - - 以“solver/scheduler”术语描述，而不是 “dmd” 术语 +然后： +- `WanPipeline` 通过参数选择 `sampler_kind: ode|sde`（默认 ode） +- method（或 `method_config`）在 validation 时显式选择 sampler + step list +- validator 回到 method-agnostic：只做 dataset + logging（不再 import `WanDMDPipeline`） -`WanValidator` 执行策略： -- 默认（保持现状）：`scheduler_kind="unipc"` → `WanPipeline`（UniPC）+ `num_inference_steps` -- 当 `scheduler_kind="flowmatch_euler"` 且 `sampling_timesteps` 非空： - - 使用 `WanPipeline` 但注入 `FlowMatchEulerDiscreteScheduler(shift=flow_shift)` - - 在 `ForwardBatch` 里填 `timesteps=sampling_timesteps` - - 让 `TimestepPreparationStage` 走 “custom timesteps” 分支（`scheduler.set_timesteps(timesteps=...)`） - -> 备注：这条路径不引入任何 DMD pipeline/stage 依赖；但可以让 validation 的 timesteps/scheduler -> 由 method 精确控制，从而减少 drift，并便于对齐 legacy 的 few-step schedule。 - -### 8.4 文件 TODO（实现清单） - -- [ ] `fastvideo/distillation/validators/base.py` - - 扩展 `ValidationRequest`：增加 `sampling_timesteps` / `scheduler_kind` -- [ ] `fastvideo/distillation/methods/distribution_matching/dmd2.py` - - `log_validation()` 构造 request 时： - - `request.sampling_timesteps = pipeline_config.dmd_denoising_steps` - - `request.scheduler_kind = "flowmatch_euler"` -- [ ] `fastvideo/distillation/validators/wan.py` - - 支持根据 request 选择 scheduler/policy（保持默认 unipc） - - 当指定 flowmatch_euler + timesteps 时，走 “custom timesteps + FlowMatchEuler scheduler” 路径 -- [ ] docs - - `fastvideo/distillation/doc/validators/base.md` - - `fastvideo/distillation/doc/validators/wan.md` - - `fastvideo/distillation/doc/methods/distribution_matching/dmd2.md` - -### 8.5 风险与后续（如需 1:1 复刻 legacy） - -即使统一 timesteps + scheduler，`WanPipeline` 的 denoising loop 仍可能与 legacy 的 -`DmdDenoisingStage` 不完全一致（legacy 有自己的 loop / noise reinjection 语义）。 - -如果我们需要 **严格复刻** legacy sampling 的每一个细节，建议在 Phase 3 做更系统的抽象： -- method（或 `method_config`）提供 “rollout plan” -- validator 仅做 dataset + logging -- 具体 rollout 执行由一个中性组件实现（例如 `sampling/rollout.py`），并且不携带 DMD 命名 +> 这个设计会被写入 `dev/phases/phase_3.md` 并在 Phase 3 实施。 diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index c4d589ba3..4720149f7 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -8,6 +8,8 @@ Phase 3 的定位：在 Phase 2 已经证明“新 distill 框架可独立运行 1) **彻底优雅的 dispatch**：避免 `wan+dmd2` 这种硬编码分支；扩展成本从 N×M 降到 N+M。 2) **YAML schema 升级**：顶层从 `distill` 改为 `recipe`，并新增 `method_config`。 3) **新增 finetuning 支持**：把 finetune 作为一种 method 接入框架（only student + dataset）。 +4) **统一 sampling 语义（ODE/SDE）**：把 “validation 用哪个 pipeline/loop” 从 `Pipeline` + 的耦合里解放出来，避免未来出现 25 个 pipeline 变体。 约束： - 不新增 entry file：继续使用 `fastvideo/training/distillation.py` 作为统一入口。 @@ -132,6 +134,23 @@ Phase 3 的定位：在 Phase 2 已经证明“新 distill 框架可独立运行 - [ ] `fastvideo/tests/distillation/test_registry_dispatch.py` - registry 能注册并 resolve `wan` + `dmd2` / `finetune` +### B8.（建议纳入 Phase 3）ODE/SDE sampler 可插拔（淘汰 `WanDMDPipeline`） + +背景：Phase 2.9 暴露了一个关键事实——即使统一 timesteps/scheduler，`WanPipeline`（ODE/solver-style +`scheduler.step`）与 DMD2/legacy `WanDMDPipeline`（SDE-style `pred_x0 -> add_noise(next_t, eps)`） +仍可能产生不同的 sampling 结果。validation 若选错 loop，就无法与训练/legacy apples-to-apples。 + +- [ ] 在 pipeline 层引入 “denoising loop / sampler” 抽象（中性命名，不出现 DMD） + - `OdeSampler`：使用 `scheduler.step(...)` + - `SdeSampler`：使用 `pred_x0 -> add_noise(next_t, eps)`（noise reinjection） +- [ ] `WanPipeline` 支持选择 sampler（默认 ODE） + - 选项 A：`WanPipeline` 通过参数注入 sampler（更通用） + - 选项 B：保留 stage 选择（`DenoisingStage` vs `SdeDenoisingStage`），但把命名去 DMD 化 +- [ ] 更新 `WanValidator`：不再 import `WanDMDPipeline` + - method（或 `method_config`）在 `ValidationRequest` 里显式指定 `sampler_kind: ode|sde` + - `WanValidator` 仍保持 family-specific + method-agnostic：只负责 dataset + logging +- [ ] 最终目标：`WanDMDPipeline` 仅作为 legacy 兼容（或被完全替代），新框架不依赖它 + --- ## C) 核心代码设计（具体到类/函数） @@ -214,6 +233,26 @@ Finetune 的边界： > Wan 侧可以复用现有 `TrainingBatch` 字段与 `normalize_dit_input / get_sigmas` 等工具， > 目标是对齐 legacy `TrainingPipeline._transformer_forward_and_compute_loss()` 的 loss 语义。 +### C6. Phase 3 sampling：把 “ODE vs SDE” 变成可配置的 sampler/integrator + +目标：避免出现 `Pipeline` / `Pipeline` 这类组合爆炸，同时让 method 能精确控制 +validation 的 solver/loop 语义（与训练一致）。 + +建议最小接口（示意）： + +- `Sampler`（pipeline 层抽象，不携带 method 命名） + - `run(transformer, batch, *, timesteps, guidance, rng, ...) -> latents` + +Wan 侧落地： +- `OdeSampler` = 复用现有 `DenoisingStage` 语义（`scheduler.step` + CFG） +- `SdeSampler` = 复用现有 `DmdDenoisingStage` 的语义，但把： + - `pipeline_config.dmd_denoising_steps` 改成显式输入的 `timesteps` + - class/变量命名去 DMD 化（例如 `SdeDenoisingStage` 或 `SdeSampler`） + +与 distillation 的边界： +- method（或 `method_config`）决定 validation 用 `ode|sde`，以及用哪一组 timesteps。 +- validator 只执行与记录，不再隐式选择 `WanPipeline` 或 `WanDMDPipeline`。 + --- ## D) 风险点 / 需要你参与决策的地方（遇到会停下讨论） diff --git a/fastvideo/distillation/doc/validators/wan.md b/fastvideo/distillation/doc/validators/wan.md index 6e807e4fb..ebdb051e3 100644 --- a/fastvideo/distillation/doc/validators/wan.md +++ b/fastvideo/distillation/doc/validators/wan.md @@ -16,8 +16,9 @@ - method 通过 `ValidationRequest` 覆盖采样配置（例如 sampling steps / guidance / output_dir）。 **依赖** -- 当前使用 `WanPipeline` 做采样推理（Wan 通用 inference pipeline）。 - Phase 2.9 约束：validator 应保持 method-agnostic（不依赖具体 distillation method）。 +- 当前使用 `WanDMDPipeline` 做采样推理（对齐 DMD2/SDE rollout，便于与 legacy apples-to-apples 对比）。 +- 这不是最终形态：Phase 3 计划把 `WanPipeline` 的 denoising loop 抽象成可插拔的 sampler（ODE/SDE）， + 从而淘汰 `WanDMDPipeline`，让 validator 回到 method-agnostic。 **可演进方向（Phase 3+）** - 将 validation steps/guidance 等采样配置从 `TrainingArgs` 迁移到更明确的配置块（例如 `validation:`）。 diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 1252a8631..0c3498309 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -19,7 +19,7 @@ from fastvideo.distributed import get_sp_group, get_world_group from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch -from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline +from fastvideo.pipelines.basic.wan.wan_dmd_pipeline import WanDMDPipeline from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.utils import shallow_asdict @@ -56,7 +56,7 @@ def __init__( self.seed = int(seed) self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) - self._pipeline: WanPipeline | None = None + self._pipeline: WanDMDPipeline | None = None self._pipeline_transformer_id: int | None = None self._sampling_param: SamplingParam | None = None @@ -65,7 +65,7 @@ def _get_sampling_param(self) -> SamplingParam: self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) return self._sampling_param - def _get_pipeline(self, *, transformer: torch.nn.Module) -> WanPipeline: + def _get_pipeline(self, *, transformer: torch.nn.Module) -> WanDMDPipeline: transformer_id = id(transformer) if self._pipeline is not None and self._pipeline_transformer_id == transformer_id: return self._pipeline @@ -73,7 +73,7 @@ def _get_pipeline(self, *, transformer: torch.nn.Module) -> WanPipeline: args_copy = copy.deepcopy(self.training_args) args_copy.inference_mode = True - self._pipeline = WanPipeline.from_pretrained( + self._pipeline = WanDMDPipeline.from_pretrained( self.training_args.model_path, args=args_copy, # inference_mode=True uses FastVideoArgs branch inference_mode=True, From a8e728bce0d1a70bb108131f7470f50b1d88f65d Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 01:43:19 +0000 Subject: [PATCH 083/214] phase 2.9 config --- examples/distillation/phase2_9/temp.sh | 2 +- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml | 77 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml diff --git a/examples/distillation/phase2_9/temp.sh b/examples/distillation/phase2_9/temp.sh index cf67c49c1..cc3474ec1 100644 --- a/examples/distillation/phase2_9/temp.sh +++ b/examples/distillation/phase2_9/temp.sh @@ -24,7 +24,7 @@ if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then exit 1 fi -CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} +CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml"} if [[ ! -f "$CONFIG" ]]; then echo "Missing distillation YAML config at: $CONFIG" >&2 diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml new file mode 100644 index 000000000..0f86de9eb --- /dev/null +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml @@ -0,0 +1,77 @@ +distill: + model: wan + method: dmd2 + +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase2_wan2.1_dmd2_8steps_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + simulate_generator_forward: true + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: phase2.9_wan_dmd2_8steps_wansyn + wandb_run_name: phase2.9_wan_dmd2_8steps_wansyn + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "8" + validation_guidance_scale: "6.0" + +pipeline_config: + flow_shift: 8 + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] From 8ed38699f7d637fc99012b5843a592914b5a3457 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 02:15:27 +0000 Subject: [PATCH 084/214] sheidewenti? --- fastvideo/distillation/families/wan.py | 2 +- fastvideo/distillation/methods/distribution_matching/dmd2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fastvideo/distillation/families/wan.py b/fastvideo/distillation/families/wan.py index 21c5dc6f4..57e496b8d 100644 --- a/fastvideo/distillation/families/wan.py +++ b/fastvideo/distillation/families/wan.py @@ -72,7 +72,7 @@ def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Mo def _build_tracker(training_args: Any, *, config: dict[str, Any] | None) -> Any: world_group = get_world_group() trackers = list(getattr(training_args, "trackers", [])) - if not trackers and getattr(training_args, "tracker_project_name", ""): + if not trackers and str(getattr(training_args, "tracker_project_name", "")): trackers.append(Trackers.WANDB.value) if world_group.rank != 0: trackers = [] diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 612870ea5..0567d6e11 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -238,7 +238,7 @@ def log_validation(self, iteration: int) -> None: sampling_steps = [int(len(raw_rollout))] raw_guidance = getattr(self.training_args, "validation_guidance_scale", None) - guidance_scale = float(raw_guidance) if raw_guidance not in (None, "") else None + guidance_scale = float(str(raw_guidance)) if raw_guidance not in (None, "") else None request = ValidationRequest( sample_handle=self.student, From 7220ebf97c43d4ac0d092a96ce3981ca9f24f1f9 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 03:26:39 +0000 Subject: [PATCH 085/214] phase 3 design: decouple simulate_generator_forward --- dev/design.md | 64 ++-- dev/phases/phase_3.md | 388 ++++++++----------------- fastvideo/distillation/adapters/wan.py | 4 +- 3 files changed, 156 insertions(+), 300 deletions(-) diff --git a/dev/design.md b/dev/design.md index 2c6e2f1bb..c3e77d9aa 100644 --- a/dev/design.md +++ b/dev/design.md @@ -612,67 +612,65 @@ Phase 2.9 为了端到端与 legacy apples-to-apples，对 Wan DMD2 的 validati `WanDMDPipeline`（SDE rollout）以避免漂移；Phase 3 会把它升级为可插拔的 ODE/SDE sampler， 从而淘汰 `Pipeline` 这种耦合。 -### Phase 3（计划）：Recipe config + method_config + Finetuning（统一到同一框架） +### Phase 3（计划）：3.1 Config schema v2 + 3.2 ODE/SDE sampler + 3.3 Finetuning Phase 3 的定位：在 Phase 2.9 已经完成“优雅 dispatch + adapter/method 语义收敛”的基础上： -1) **配置语义升级（`distill` -> `recipe`，引入 `method_config`）** -2) **把 finetuning 作为一种 method 接入框架**（只需要 `student` + dataset） - -3) **统一 sampling 语义（ODE/SDE sampler 可插拔）** - - 背景：DMD2/Consistency 等方法可能需要不同的 denoising loop；如果靠 “每个 method 一个 pipeline 变体” - 会重新回到 N×M 的组合爆炸。 - - 目标：让 `WanPipeline`（以及未来其它 family 的 pipeline）通过参数选择 sampler/integrator（ODE vs SDE）， - validation 由 method/method_config 显式指定 sampler+steps，validator 只做 dataset+logging。 - - 结果：新框架不再需要依赖 `WanDMDPipeline`（保留 legacy 兼容即可）。 - #### Phase 3.1：配置语义升级（`distill` -> `recipe`，引入 `method_config`） 动机： - `distill.method=finetune` 语义别扭，因为 finetune 是一种训练 recipe，不一定是“蒸馏”。 -- method-specific 参数长期塞进 `training:`（TrainingArgs）会让配置语义越来越混杂。 +- method-specific 参数长期塞进 `training:`（TrainingArgs）/`pipeline_config:` 会让配置语义越来越混杂。 +- Phase 2.9 还残留少量 “method knob 泄漏到 adapter” 的问题（例如 `simulate_generator_forward`），需要借助 + `method_config` 做干净的边界收敛。 -Phase 3 计划把 YAML schema 升级为： +Phase 3.1 计划把 YAML schema 升级为： ```yaml recipe: {family: wan, method: dmd2} # 只负责 “选什么” models: {student: ..., teacher: ...} # 参与者 training: {...} # infra 参数（映射到 TrainingArgs） -pipeline_config: {...} # pipeline/backbone config（模型侧） +pipeline_config: {...} # backbone/pipeline config（模型侧） method_config: {...} # algorithm/method 超参（方法侧） ``` -同时保持与 FastVideo 现有语义对齐： -- 入口层会根据 `recipe.method` 推导 `TrainingArgs.mode` - - `finetune` -> `ExecutionMode.FINETUNING` - - 其它 distillation methods -> `ExecutionMode.DISTILLATION` - 迁移策略（建议）： -- Phase 3 先把 `method_config` 作为新增字段引入，并逐步把以下参数从 `training:` 挪过去： - - DMD2：`generator_update_interval`, `real_score_guidance_scale`, `simulate_generator_forward`, ... - - Self-forcing：ODE-init / cache / rollout 策略相关参数 - - Finetune：loss/target/pred_type 等 -- `training:` 保持 “trainer/infra” 语义（分布式、优化器、ckpt、logging、数据路径等）。 +- DMD2：把 `generator_update_interval`, `real_score_guidance_scale`, + `dmd_denoising_steps`, `simulate_generator_forward`（或替代字段）迁移到 `method_config`。 +- `training:` 保持纯 infra（分布式、优化器默认值、ckpt、logging、数据路径等）。 + +#### Phase 3.2：统一 sampling 语义（ODE/SDE sampler 可插拔） + +背景：DMD2/Consistency 等方法可能需要不同的 denoising loop（不仅是 timesteps/scheduler），如果靠 +“每个 method 一个 pipeline 变体（例如 `WanDMDPipeline`）”会重新回到 N×M 的组合爆炸。 -#### Phase 3.2：Finetuning 作为一种 method 接入（only student） +目标： +- `WanPipeline`（以及未来其它 family）通过参数选择 sampler/integrator（`ode|sde`）。 +- validation 由 method/method_config 显式指定 sampler + steps/timesteps；validator 只做 dataset+logging。 +- 新框架不再依赖 `WanDMDPipeline`（可保留 legacy 兼容）。 -目标：让 finetuning 跟 distillation 一样走同一套： -`ModelBundle + Adapter + Method + Trainer + (Validator/Checkpoint)`。 +#### Phase 3.3：把 finetuning 作为一种 method 接入框架 + +目标：把 finetune 作为一种 method（only `student` + dataset）接入同一套 +`ModelBundle + Adapter + Method + Trainer + (Validator/Checkpoint)` 基础设施，并让其配置语义与 +Phase 3.1 的 `recipe/method_config` 对齐。 建议落地形态（Phase 3 落地到代码时）： - 新增 method：`fastvideo/distillation/methods/fine_tuning/finetune.py::FineTuneMethod` - `bundle.require_roles(["student"])` - - 复用 trainer 的 step/ckpt/validation - - 通过 adapter 提供的 primitives 完成 forward/loss/backward（避免 method 管 forward_context） + - update policy 永远只 step student 的 optimizer/scheduler - 为 finetune 定义 adapter contract（类似 `_DMD2Adapter` 的做法）： - - `_FineTuneAdapter(Protocol)`：`prepare_batch()` + `sample_train_timestep()` + `student_predict()` + `training_loss()` 等 - - Wan 侧由 `WanAdapter` 实现该 contract（或拆出 `WanAdapterBase + WanFineTuneOps` 以避免 adapter 过度膨胀） + - `_FineTuneAdapter(Protocol)`：`prepare_batch()` + `predict_*()` + `compute_loss()` + `backward()` + - Wan 侧由 `WanAdapter` 实现该 contract（或拆出 mixin，避免 adapter 膨胀） -Finetune 的 config（示意）： +Finetune 的 config（示意，schema v2）： ```yaml recipe: {family: wan, method: finetune} models: - student: {family: wan, path: ..., trainable: true} + student: + family: wan + path: ... + trainable: true training: {...} pipeline_config: {...} method_config: diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 4720149f7..2b6ce5100 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -1,292 +1,150 @@ -# Phase 3：优雅 Dispatch（N+M）+ `recipe` 配置语义升级 + Finetuning 接入 +# Phase 3：3.1 Config schema v2 + 3.2 ODE/SDE sampler + 3.3 Finetuning -Phase 3 的定位：在 Phase 2 已经证明“新 distill 框架可独立运行”的基础上，把它推进到 -“长期可扩展、可承载更多训练 recipe（包括 finetune）”的状态。 +Phase 2.9 已完成三件关键事情（为 Phase 3 铺路）： +- operation-centric adapter（adapter 不看 role string，只收 `RoleHandle`） +- policy 回归 method（few-step rollout / step list 等在 method 里） +- families + registry + builder（优雅 dispatch：新增 family 或 method 是 N+M，不是 N×M） -本 phase 目标（你已拍板）： +因此 Phase 3 不再聚焦 dispatch；Phase 3 的新增工作按顺序拆成三个子阶段： -1) **彻底优雅的 dispatch**：避免 `wan+dmd2` 这种硬编码分支；扩展成本从 N×M 降到 N+M。 -2) **YAML schema 升级**：顶层从 `distill` 改为 `recipe`，并新增 `method_config`。 -3) **新增 finetuning 支持**：把 finetune 作为一种 method 接入框架（only student + dataset）。 -4) **统一 sampling 语义（ODE/SDE）**：把 “validation 用哪个 pipeline/loop” 从 `Pipeline` - 的耦合里解放出来，避免未来出现 25 个 pipeline 变体。 +- **Phase 3.1：Config schema v2（`recipe` + `method_config`）** +- **Phase 3.2：ODE/SDE sampler 可插拔（淘汰 `Pipeline`）** +- **Phase 3.3：Finetuning method 接入（only student + dataset）** -约束： -- 不新增 entry file：继续使用 `fastvideo/training/distillation.py` 作为统一入口。 -- Phase 3 仍遵循边界：**forward/backward context 由 adapter 托管**，method 只实现算法/编排。 +约束（延续前几个 phase）： +- 不新增 entry file：继续使用 `fastvideo/training/distillation.py`。 +- forward/backward context 仍由 adapter 托管；method 只负责算法编排。 --- -## A) Phase 3 交付目标（Definition of Done） +## Phase 3.1：Config schema v2（`recipe` + `method_config`） -- `fastvideo/training/distillation.py` 不再出现 `if family==... and method==...` 的组合硬编码； - 而是通过 registry 查找 family/method 组件来构建 runtime。 -- YAML schema 支持 `recipe + models + training + pipeline_config + method_config`： - - `distill:` schema v1 可以选择性兼容（见“风险/决策点”）。 - - 文档与 examples 更新到 schema v2（`recipe`）。 -- 能跑通两个 recipe： - 1) `recipe.method=dmd2`（现有 few-step wan DMD2） - 2) `recipe.method=finetune`（wan finetune 最小版本，only student） -- `method_config` 在代码中**真实生效**（至少 DMD2 的 update policy + guidance scale 使用它）。 +### 目标（DoD） +- distillation 入口只接受 **schema v2** 的 YAML（顶层 `recipe:`），并作为 single source of truth。 +- `method_config` 能被解析并传入 method；至少 DMD2 的关键超参从 `method_config` 生效。 +- 将 “method knob” 从 `training_args/pipeline_config` 中剥离出来（逐步迁移），并修复 Phase 2.9 + 残留的语义泄漏： + - `WanAdapter.prepare_batch()` 不再读取 `training_args.simulate_generator_forward`。 ---- - -## B) TODO List（Review Checklist） - -### B1. 彻底优雅的 dispatch（registry + 通用 builder） - -- [ ] 新增 `fastvideo/distillation/registry.py` - - `register_family(name)` / `register_method(name)` 装饰器 - - `get_family(name)` / `get_method(name)`：带可用项提示的错误信息 - - `ensure_builtin_registrations()`：导入内置 family/method 以完成注册 -- [ ] 新增 `fastvideo/distillation/families/`（model family 插件） - - `fastvideo/distillation/families/__init__.py` - - `fastvideo/distillation/families/wan.py`：`WanFamily`（复用 Phase 2 的加载与数据构建逻辑） -- [ ] 改造 `fastvideo/distillation/builder.py` - - 把当前 `build_wan_dmd2_runtime_from_config()` 收敛为： - - `build_runtime_from_config(cfg: RunConfig) -> DistillRuntime` - - `DistillRuntime.method` 类型从 `DMD2Method` 泛化为 `DistillMethod` -- [ ] 改造入口 `fastvideo/training/distillation.py` - - 入口只做：`cfg = load_run_config()` → `runtime = build_runtime_from_config(cfg)` → `trainer.run(...)` - - 不再写组合分支（`wan+dmd2` 等） - -### B2. YAML schema v2：`recipe` + `method_config` +### Schema v2（示意） +```yaml +recipe: + family: wan + method: dmd2 + +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: {...} # infra（映射到 TrainingArgs） +pipeline_config: {...} # backbone/pipeline（模型侧） +method_config: {...} # algorithm（方法侧） +``` -- [ ] 更新 spec：`fastvideo/distillation/specs.py` +### 文件 TODO（实现清单） +- [ ] `fastvideo/distillation/specs.py` - 新增 `RecipeSpec(family: str, method: str)` - - 保留 `RoleSpec(family/path/trainable)`，与 Phase 2 兼容 -- [ ] 更新 YAML loader：`fastvideo/distillation/yaml_config.py` - - 新 schema： - - `recipe: {family, method}` - - `method_config: { ... }`（可选，默认 `{}`） - - 入口层推导 `TrainingArgs.mode`： - - `recipe.method == "finetune"` → `ExecutionMode.FINETUNING` - - 其它 method → `ExecutionMode.DISTILLATION` - - （可选）兼容 v1：若仅提供 `distill: {model, method}`： - - 转换为 `recipe.family = distill.model`，`recipe.method = distill.method` - - 打 warning 提示迁移（见“风险/决策点”） - -### B3. `method_config` 接入到 DMD2（让它真正生效） - -- [ ] 更新 `fastvideo/distillation/methods/distribution_matching/dmd2.py` - - `DMD2Method` 构造函数增加 `method_config: dict[str, Any] | None` - - 读取优先级：`method_config` > `training_args` 默认值（用于平滑迁移） + - `DistillRunConfig` 增加 `recipe` 与 `method_config` +- [ ] `fastvideo/distillation/yaml_config.py` + - 解析 `recipe:` 与 `method_config:`（默认 `{}`） + - 将 v1 的 `distill:` 视为不再支持（或仅保留一次性迁移期的兼容分支，需你拍板） +- [ ] `fastvideo/distillation/builder.py` + - 从 `cfg.recipe` 取 family/method（不再读 `cfg.distill`） + - build method 时传入 `method_config` +- [ ] `fastvideo/distillation/methods/distribution_matching/dmd2.py` + - `DMD2Method(..., method_config=...)` + - 关键参数读取优先级：`method_config` > `training_args`（迁移期平滑） - `generator_update_interval` - `real_score_guidance_scale` - - （可选）`simulate_generator_forward`（若继续存在） - -### B4. Finetuning method（only student） - -- [ ] 新增 `fastvideo/distillation/methods/fine_tuning/finetune.py` - - `FineTuneMethod(DistillMethod)` - - `bundle.require_roles(["student"])` - - `single_train_step()`： - - `training_batch = adapter.prepare_batch(...)` - - `pred = adapter.student_predict(...)` - - `loss = adapter.finetune_loss(...)` - - update policy： - - `get_optimizers()` / `get_lr_schedulers()` 始终返回 student 的 -- [ ] 在 `fastvideo/distillation/methods/fine_tuning/__init__.py` 暴露该 method（并注册） - -### B5. WanAdapter 增强 finetune primitives（不让 method 管 forward_context） - -- [ ] 更新 `fastvideo/distillation/adapters/wan.py` - - 新增 `_FineTuneAdapter(Protocol)` 所需 primitives： - - `student_predict_for_finetune(batch) -> torch.Tensor` - - `finetune_loss(batch, pred) -> torch.Tensor` - - （如 activation checkpoint/backward 重算需要）`backward_student(loss, ctx, ...)` - - 复用现有的： - - `prepare_batch()`（要求 finetune 路径必须提供真实 `vae_latent`） - - `set_forward_context(...)` 的管理继续留在 adapter - -### B5.5 彻底去除 adapter 对 method knob 的读取（例如 `simulate_generator_forward`） - -动机：当前 `WanAdapter.prepare_batch()` 仍读取 `training_args.simulate_generator_forward` 来决定 -是否需要 `vae_latent`（或构造 placeholder）。这会把 DMD2/student-rollout 的语义泄漏到 adapter， -违背 Phase 2.9 的 operation-centric 边界。 - -- [ ] 调整 `WanAdapter` 的 batch API，使其不再读取 `training_args.simulate_generator_forward` - - 选项 A（推荐）：拆成两个显式入口 - - `prepare_batch_with_vae_latent(raw_batch, ...)`：必须有 `vae_latent` - - `prepare_batch_text_only(raw_batch, ...)`：不依赖 `vae_latent`，只根据 family 规则构造 latents/shape - - 选项 B：保留单入口但显式参数化 - - `prepare_batch(raw_batch, *, latents_source=...)` - - 由 method/`method_config` 决定使用 data latents 还是 placeholder latents -- [ ] 将 “是否 simulate / rollout 模式” 的开关迁移到 `method_config` - - DMD2：例如 `method_config.rollout_mode = "data_latent" | "simulate"` - - adapter 不读取该开关；method 选择调用哪个 adapter API - -### B6. examples / outside configs 更新到 schema v2 - -- [ ] 更新 DMD2 YAML（schema v2）： - - `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` - - `distill -> recipe` - - 新增 `method_config`（至少包含 DMD2 的 interval + guidance scale + simulate flag） -- [ ] 新增 finetune YAML（schema v2）： - - `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B.yaml`（命名可再讨论） -- [ ] 更新 `examples/distillation/phase2/README.md` 与脚本说明（指向 `recipe` schema） - -### B7.（可选但推荐）最小单测 - -- [ ] `fastvideo/tests/distillation/test_yaml_schema_v2.py` - - schema v2 能 parse - - （如兼容 v1）schema v1 → v2 conversion 能 parse -- [ ] `fastvideo/tests/distillation/test_registry_dispatch.py` - - registry 能注册并 resolve `wan` + `dmd2` / `finetune` - -### B8.（建议纳入 Phase 3）ODE/SDE sampler 可插拔（淘汰 `WanDMDPipeline`） - -背景：Phase 2.9 暴露了一个关键事实——即使统一 timesteps/scheduler，`WanPipeline`（ODE/solver-style -`scheduler.step`）与 DMD2/legacy `WanDMDPipeline`（SDE-style `pred_x0 -> add_noise(next_t, eps)`） -仍可能产生不同的 sampling 结果。validation 若选错 loop，就无法与训练/legacy apples-to-apples。 - -- [ ] 在 pipeline 层引入 “denoising loop / sampler” 抽象（中性命名，不出现 DMD） - - `OdeSampler`：使用 `scheduler.step(...)` - - `SdeSampler`：使用 `pred_x0 -> add_noise(next_t, eps)`（noise reinjection） -- [ ] `WanPipeline` 支持选择 sampler（默认 ODE） - - 选项 A：`WanPipeline` 通过参数注入 sampler（更通用） - - 选项 B：保留 stage 选择（`DenoisingStage` vs `SdeDenoisingStage`），但把命名去 DMD 化 -- [ ] 更新 `WanValidator`：不再 import `WanDMDPipeline` - - method（或 `method_config`）在 `ValidationRequest` 里显式指定 `sampler_kind: ode|sde` - - `WanValidator` 仍保持 family-specific + method-agnostic：只负责 dataset + logging -- [ ] 最终目标：`WanDMDPipeline` 仅作为 legacy 兼容（或被完全替代），新框架不依赖它 + - `dmd_denoising_steps`（few-step step list） + - `rollout_mode`（替代 `simulate_generator_forward`） +- [ ] `fastvideo/distillation/adapters/wan.py` + - 移除 `training_args.simulate_generator_forward` 的读取（这是 Phase 2.9 的残留耦合） + - 把 batch 形态做成显式 API/参数，让 method 决定： + - 选项 A（推荐）：拆分显式入口 + - `prepare_batch_from_data_latent(raw_batch, ...)`（必须有 `vae_latent`） + - `prepare_batch_from_placeholder_latent(raw_batch, ...)`（不依赖 `vae_latent`） + - 选项 B：保留单入口但显式参数化：`prepare_batch(..., latents_source=...)` +- [ ] configs / docs + - [ ] `fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml` 全部升级到 schema v2 + - [ ] 更新 `dev/config.md`（描述 schema v2 与迁移策略） --- -## C) 核心代码设计（具体到类/函数） - -### C1. `registry.py`：N+M 组合的关键 - -目标：入口/Builder 只写一次组合逻辑，不写 `build__()`。 - -建议接口（简化版）： - -- `DistillFamily`（family 插件） - - `build_bundle(cfg) -> ModelBundle` - - `build_shared_components(cfg) -> ...`（如 vae/scheduler） - - `build_adapter(bundle, cfg, shared) -> DistillAdapter` - - `build_validator(bundle, cfg, tracker, shared) -> DistillValidator | None` - - `build_dataloader(cfg) -> DataLoaderLike` - - `build_optimizers_schedulers(bundle, cfg) -> None`（写入 RoleHandle） - -- `DistillMethodFactory`（method 插件） - - `build(bundle, adapter, cfg) -> DistillMethod` - -注册方式： -- `@register_family("wan") class WanFamily: ...` -- `@register_method("dmd2") def build_dmd2(...): ...` -- `@register_method("finetune") def build_finetune(...): ...` - -> 这样新增一个 family 或 method 是一次注册（N+M），不会产生 N×M 组合函数。 - -### C2. 通用 `build_runtime_from_config(cfg)` - -`fastvideo/distillation/builder.py` 收敛为一个通用入口： - -1) `ensure_builtin_registrations()` -2) `family = get_family(cfg.recipe.family)` -3) `method_factory = get_method(cfg.recipe.method)` -4) `bundle = family.build_bundle(cfg)` -5) `family.build_optimizers_schedulers(bundle, cfg)`（只给 trainable roles 创建） -6) `shared = family.build_shared_components(cfg)` -7) `tracker = family.build_tracker(cfg)`（或 builder 复用 training/trackers） -8) `validator = family.build_validator(...)`（可选） -9) `adapter = family.build_adapter(bundle, cfg, shared, validator)` -10) `method = method_factory.build(bundle=bundle, adapter=adapter, cfg=cfg)` -11) `dataloader = family.build_dataloader(cfg)` -12) 返回 `DistillRuntime(training_args, method, dataloader, tracker, start_step=0)` - -### C3. YAML loader（schema v2） - -`fastvideo/distillation/yaml_config.py` 产出新的 `RunConfig`（命名可沿用 `DistillRunConfig`）： - -- `recipe: RecipeSpec` -- `roles/models: dict[str, RoleSpec]` -- `training_args: TrainingArgs` -- `method_config: dict[str, Any]` -- `raw: dict[str, Any]` - -并将 `training_args.mode` 与 `inference_mode` 强制一致（Phase 2 的经验）： -- `finetune` → `FINETUNING` + `inference_mode=False` -- distill methods → `DISTILLATION` + `inference_mode=False` - -### C4. DMD2：method_config 生效点 - -Phase 3 最小要求：以下两个参数从 `method_config` 读取（否则 `method_config` 只是摆设）： -- `generator_update_interval` -- `real_score_guidance_scale` - -兼容策略（建议）：如果 `method_config` 缺失，则回落到 `training_args` 字段，避免破坏旧 YAML。 - -### C5. Finetune：method + adapter contract - -Finetune 的边界： -- method 负责算法编排（loss/optim schedule） -- adapter 负责 forward_context + model/pipeline 差异 - -建议 `_FineTuneAdapter` 的最小 surface： -- `prepare_batch(raw_batch, current_vsa_sparsity=...) -> TrainingBatch` -- `student_predict_for_finetune(batch) -> torch.Tensor` -- `finetune_loss(batch, pred) -> torch.Tensor` -- `backward_student(loss, ctx, grad_accum_rounds=...)`（如果需要） - -> Wan 侧可以复用现有 `TrainingBatch` 字段与 `normalize_dit_input / get_sigmas` 等工具， -> 目标是对齐 legacy `TrainingPipeline._transformer_forward_and_compute_loss()` 的 loss 语义。 - -### C6. Phase 3 sampling：把 “ODE vs SDE” 变成可配置的 sampler/integrator - -目标：避免出现 `Pipeline` / `Pipeline` 这类组合爆炸，同时让 method 能精确控制 -validation 的 solver/loop 语义（与训练一致）。 - -建议最小接口（示意）： - -- `Sampler`（pipeline 层抽象，不携带 method 命名） - - `run(transformer, batch, *, timesteps, guidance, rng, ...) -> latents` - -Wan 侧落地： -- `OdeSampler` = 复用现有 `DenoisingStage` 语义（`scheduler.step` + CFG） -- `SdeSampler` = 复用现有 `DmdDenoisingStage` 的语义，但把： - - `pipeline_config.dmd_denoising_steps` 改成显式输入的 `timesteps` - - class/变量命名去 DMD 化（例如 `SdeDenoisingStage` 或 `SdeSampler`） - -与 distillation 的边界： -- method（或 `method_config`）决定 validation 用 `ode|sde`，以及用哪一组 timesteps。 -- validator 只执行与记录，不再隐式选择 `WanPipeline` 或 `WanDMDPipeline`。 +## Phase 3.2：ODE/SDE sampler 可插拔（统一 sampling loop 语义） + +### 背景（为什么需要） +Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop 不同**，sampling 结果仍会 drift： +- ODE/solver 风格：`scheduler.step(noise_pred, t, latents)`（当前 `WanPipeline`） +- SDE 风格：`pred_x0 -> add_noise(next_t, eps)`（DMD2/legacy `WanDMDPipeline`） + +如果继续靠 `Pipeline` 这类 pipeline 变体来选择 loop，会重新走向 N×M 组合爆炸。 + +### 目标（DoD） +- `WanPipeline` 支持通过参数/配置选择 sampler（`ode|sde`），默认 `ode`。 +- distillation 的 validation 由 method/method_config 显式指定 sampler + step list； + validator 回到 method-agnostic，不再 import `WanDMDPipeline`。 +- `WanDMDPipeline` 保留为 legacy 兼容（可选），但新框架不依赖它。 + +### 文件 TODO（实现清单） +- [ ] 抽象 sampler（中性命名，不出现 DMD） + - 选项 A：新增 `fastvideo/pipelines/samplers/`（推荐） + - 选项 B：在 `fastvideo/pipelines/stages/denoising.py` 内做 `OdeSampler/SdeSampler` +- [ ] `fastvideo/pipelines/stages/denoising.py` + - 把 `DmdDenoisingStage` 的语义迁移为 `SdeSampler`（或 `SdeDenoisingStage`） + - `SdeSampler` 接受显式 `timesteps`（不再读 `pipeline_config.dmd_denoising_steps`） + - 继续使用 `batch.generator` 来生成每一步注入的 `eps`（保证可复现实验） +- [ ] `fastvideo/pipelines/basic/wan/wan_pipeline.py` + - `initialize_pipeline/create_pipeline_stages` 支持选择 sampler +- [ ] `fastvideo/distillation/validators/base.py` + - 扩展 `ValidationRequest`： + - `sampler_kind: Literal["ode", "sde"] | None` + - `sampling_timesteps: list[int] | None`（用于 few-step schedule） +- [ ] `fastvideo/distillation/validators/wan.py` + - 改回使用 `WanPipeline`（不再 import `WanDMDPipeline`） + - 根据 request 选择 sampler + timesteps +- [ ] `fastvideo/distillation/methods/distribution_matching/dmd2.py` + - validation request 里指定 `sampler_kind="sde"` + `sampling_timesteps=` --- -## D) 风险点 / 需要你参与决策的地方（遇到会停下讨论） +## Phase 3.3：Finetuning method（only student） -1) **schema v1 兼容性**：Phase 3 是否需要继续支持 `distill:`？ - - 选项 A：完全切到 `recipe:`（更干净，但需要一次性改所有 YAML） - - 选项 B：兼容 v1 → v2 conversion + warning（更平滑，但多一点维护负担） +### 目标（DoD） +- 新增 `finetune` method，复用同一套 Trainer/Bundle/Adapter/Family/Validator 基础设施。 +- 最小可运行：只需 `models.student` + dataset 即可训练。 +- finetune 的 method 参数进入 `method_config`（与 Phase 3.1 schema 一致）。 -2) **`method_config` 的形态**：是否需要做 method namespace？ - - 选项 A：flat mapping（推荐）：`method_config: {generator_update_interval: 5, ...}` - - 选项 B：namespaced：`method_config: {dmd2: {...}}`（更显式但更啰嗦） - -3) **finetune 的 validation**：Phase 3 是否要求 finetune 也能 video validation？ - - 最小交付可以先让 finetune 训练跑通，validation 可先关（`log_validation=false`） - - 若要开启，需要确认 `WanValidator` 的 sampling pipeline 是否应随 recipe.method 切换 +### 文件 TODO（实现清单） +- [ ] `fastvideo/distillation/methods/fine_tuning/finetune.py` + - `FineTuneMethod(DistillMethod)` + `@register_method("finetune")` + - `bundle.require_roles(["student"])` + - `single_train_step()` 只更新 student +- [ ] `fastvideo/distillation/adapters/wan.py` + - 增加 finetune 所需 primitives（operation-centric）： + - `predict_noise(handle, ...)` / `predict_x0(handle, ...)`（可复用已有） + - （按 legacy loss 语义）增加最小 `finetune_loss(...)` 或 `compute_training_loss(...)` +- [ ] configs/examples + - [ ] `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_*.yaml` + - [ ] `examples/distillation/phase3/`（或更新现有 examples） --- -## E) YAML 小结：flow mapping vs block mapping +## 备注：关于 `simulate_generator_forward`（Phase 2.9 的残留） -YAML 语法上： +目前 `WanAdapter.prepare_batch()` 仍读取 `training_args.simulate_generator_forward` 来决定是否需要 `vae_latent`。 +这不是“功能 bug”，但确实是语义耦合（把 DMD2 rollout 的开关泄漏到了 adapter）。 -```yaml -student: {family: wan, path: ..., trainable: true} -``` - -与 - -```yaml -student: - family: wan - path: ... - trainable: true -``` +结论：它 **应该在 Phase 3.1** 通过 `method_config.rollout_mode` + 显式 batch API 来解决； +Phase 2.9 没处理是因为当时我们刻意不引入 schema v2/method_config，避免变量叠加导致定位困难。 -在语义上是等价的（都是一个 mapping）。Phase 3 的 examples/docs 我们将统一采用 **block -style**（你偏好的后者），因为更可读、diff 更友好。 diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 8683970e8..7b4ede936 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -144,12 +144,12 @@ def ensure_negative_conditioning(self) -> None: args_copy = copy.deepcopy(training_args) args_copy.inference_mode = True - student_transformer = self.prompt_handle.require_module("transformer") + prompt_transformer = self.prompt_handle.require_module("transformer") prompt_pipeline = WanPipeline.from_pretrained( training_args.model_path, args=args_copy, inference_mode=True, - loaded_modules={"transformer": student_transformer}, + loaded_modules={"transformer": prompt_transformer}, tp_size=training_args.tp_size, sp_size=training_args.sp_size, num_gpus=training_args.num_gpus, From 5ee043c6db73102931af68a331c909fe43940720 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 08:05:03 +0000 Subject: [PATCH 086/214] phase 3.1 impl --- dev/config.md | 205 ++++++------------ dev/design.md | 18 +- dev/phases/phase_3.md | 34 +-- examples/distillation/phase3_1/temp.sh | 43 ++++ fastvideo/distillation/adapters/base.py | 5 +- fastvideo/distillation/adapters/wan.py | 9 +- fastvideo/distillation/builder.py | 14 +- fastvideo/distillation/doc/builder.md | 4 +- .../doc/methods/distribution_matching/dmd2.md | 16 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.md | 22 +- ...ll_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md | 14 ++ ...ll_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md | 15 ++ fastvideo/distillation/doc/specs.md | 10 +- fastvideo/distillation/doc/yaml_config.md | 12 +- fastvideo/distillation/families/wan.py | 52 +++-- .../methods/distribution_matching/dmd2.py | 110 ++++++++-- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 19 +- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml | 19 +- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml | 86 ++++++++ fastvideo/distillation/specs.py | 8 +- fastvideo/distillation/yaml_config.py | 38 +++- fastvideo/training/distillation.py | 6 +- 22 files changed, 491 insertions(+), 268 deletions(-) create mode 100644 examples/distillation/phase3_1/temp.sh create mode 100644 fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md create mode 100644 fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md create mode 100644 fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml diff --git a/dev/config.md b/dev/config.md index 21e103c1f..a5da8d336 100644 --- a/dev/config.md +++ b/dev/config.md @@ -1,223 +1,144 @@ -# Phase 2 Distillation YAML Config +# Distillation YAML config (schema v2) -本文档描述当前 **Phase 2 distillation** 入口所使用的 YAML 配置结构、字段含义，以及为什么采用这种设计。 +本文档描述当前 distillation 入口所使用的 **YAML schema v2**、字段含义与设计取舍。 相关实现： - YAML loader：`fastvideo/distillation/yaml_config.py` -- Phase 2 入口：`fastvideo/training/distillation.py` -- Phase 2 示例 YAML：`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` +- Entrypoint：`fastvideo/training/distillation.py` +- Schema 定义：`fastvideo/distillation/specs.py` +- 示例 YAML（outside）：`fastvideo/distillation/outside/fastvideo/configs/distillation/` ## 1) 入口与约束（非常重要） -Phase 2 distillation **只接受一个真实存在的 YAML 文件路径**，不会 fallback 到 legacy CLI/configs，也不会做 “overlay/补全 outside 路径” 的魔法。 +distillation 入口 **只接受一个真实存在的 YAML 文件路径**（不 merge legacy CLI/configs， +也不做 outside 路径补全/overlay）。YAML 是 single source of truth。 运行方式（示意）： ```bash -python -m fastvideo.training.distillation \ +python fastvideo/training/distillation.py \ --config /abs/path/to/fastvideo/distillation/outside/fastvideo/configs/distillation/xxx.yaml ``` -CLI 只保留少量 **runtime override**（不属于“实验定义”的内容）： +CLI 仅保留少量 **runtime override**（不属于“实验定义”的内容）： - `--resume-from-checkpoint`：从 checkpoint 恢复 - `--override-output-dir`：临时覆盖输出目录（方便重复跑实验） - `--dry-run`：只 parse + build runtime，不启动训练 -## 2) YAML 顶层结构 - -目前 YAML 顶层包含 4 个部分： +## 2) YAML 顶层结构（schema v2） ```yaml -distill: # 选择 “模型家族” + “蒸馏方法” -models: # 以 role 为 key 的模型参与者配置 -training: # 训练参数（直接映射到 TrainingArgs） -pipeline_config: # pipeline_config 的内联配置（dict） -# 或者 pipeline_config_path: /path/to/pipeline_config.json|yaml +recipe: # 选择 family + method（只负责“选什么”） +models: # role -> role spec（谁参与） +training: # infra 参数（直接映射到 TrainingArgs） +pipeline_config: # 模型/pipeline 侧 config（可 inline） +method_config: # method/algorithm 超参（方法侧 single source of truth） +# 或者 pipeline_config_path: /abs/path/to/pipeline_config.json|yaml ``` loader 规则： - `pipeline_config` 与 `pipeline_config_path` **二选一**，不能同时提供。 -- `training` 会被传入 `TrainingArgs.from_kwargs(**training_kwargs)`；Phase 2 不重复造一套训练参数体系。 -- Phase 2 会强制一些 invariants（见第 5 节）。 +- `training` 会被传入 `TrainingArgs.from_kwargs(**training_kwargs)`；我们不重造一套训练参数体系。 +- 缺少 `recipe:` 会直接报错（schema v1 的 `distill:` 不再支持）。 -## 3) `distill`: 选择模型家族与蒸馏方法 +## 3) `recipe`: 选择 family 与 method ```yaml -distill: - model: wan +recipe: + family: wan method: dmd2 ``` 用途： -- 让入口决定用哪个 **runtime builder**（当前仅支持 `wan + dmd2`）。 -- 未来扩展时，用同样的 schema 接入更多 `model/method` 组合，而不需要为每个组合写一个新的 training entry file。 +- registry dispatch：选择 `families/.py` + `methods/.py` 的组合（N+M，而非 N×M）。 +- 语义更通用：未来把 finetuning 也纳入时不会出现 `distill.method=finetune` 的别扭表达。 -设计原因： -- 把 “选择什么（model/method）” 与 “如何训练（training）/谁参与（models）” 分开，结构更稳定。 -- 更接近 FastGen：config 选择 `method` + `network`，训练逻辑由 method/adapter 决定。 - -## 4) `models`: role-based 模型参与者 +## 4) `models`: role-based 参与者 ```yaml models: student: - family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers trainable: true teacher: - family: wan path: Wan-AI/Wan2.1-T2V-14B-Diffusers trainable: false + disable_custom_init_weights: true critic: - family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers trainable: true + disable_custom_init_weights: true ``` 字段含义（见 `fastvideo/distillation/specs.py`）： -- `family`：模型家族（可省略，默认等于 `distill.model`） -- `path`：模型路径/Hub 名称（由 builder/loader 负责加载） -- `trainable`：该 role 的参数是否参与训练（默认 `true`） - -设计原因（为什么要 role-based）： -- distillation setup 千差万别，不应 hard-code “只有 student/teacher/critic 才是核心角色”。**role 只是 key**，method 决定它需要哪些 role。 -- role-based bundle 让 Method 可以泛化：例如 CM/KD/SFT 可能需要 reward/refiner/aux_teacher 等，都可以用同一个结构表达。 +- `family`：可选；默认继承 `recipe.family` +- `path`：模型路径 / hub 名称（由 family 负责加载） +- `trainable`：该 role 的参数是否参与训练（影响 `requires_grad`/train/eval） +- `disable_custom_init_weights`：可选；禁用 family 的 “加载时自定义 init weights 逻辑” -关于 `trainable` 应由谁决定： -- YAML 里的 `trainable` 表示 **“训练配置的意图/策略”**（policy）。 -- Method 仍然可以施加 **算法不变量（invariants）**。例如 `DMD2Method` 会强制要求 - `models.teacher.trainable=false`（否则直接报错），因为 DMD2 默认 teacher 作为固定 reference。 +设计原因： +- role 只是 key；framework 不强行规定 “canonical roles”。method 决定它需要哪些 roles。 +- `trainable` 表示训练意图；method 仍可施加算法不变量（例如 DMD2 强制 teacher frozen）。 ## 5) `training`: 直接映射到 `TrainingArgs` -`training:` 下的 key 基本上就是 `TrainingArgs` 的字段（`fastvideo/fastvideo_args.py`），例如： -- 分布式：`num_gpus`, `sp_size`, `tp_size`（以及内部需要的 hsdp 维度默认值） -- 数据：`data_path`, `dataloader_num_workers`, batch/shape 相关字段 +`training:` 下的 key 基本上就是 `TrainingArgs` 字段（`fastvideo/fastvideo_args.py`），例如： +- 分布式：`num_gpus`, `sp_size`, `tp_size` +- 数据：`data_path`, `dataloader_num_workers`, shape/batch 相关字段 - 输出：`output_dir`, `max_train_steps`, `seed`, `checkpoints_total_limit` -- 优化器/调度器：`learning_rate`, `betas`, `lr_scheduler`, `fake_score_learning_rate`, ... -- distill 相关：`generator_update_interval`, `real_score_guidance_scale`, ... +- 优化器默认值：`learning_rate`, `betas`, `lr_scheduler`, ... - tracking/validation：`log_validation`, `validation_*`, `tracker_project_name`, ... -Phase 2 loader 会强制/补全的关键 invariants（见 `fastvideo/distillation/yaml_config.py`）： +loader 会注入/补全的 invariants（见 `fastvideo/distillation/yaml_config.py`）： - `mode = ExecutionMode.DISTILLATION` - `inference_mode = False` +- `dit_precision` 默认 `fp32`（master weights） - `dit_cpu_offload = False` -- `dit_precision` 默认 `fp32`（保持 training-mode loader 语义：fp32 master weights） -- 若未显式提供，补默认分布式尺寸（例如 `num_gpus`、`sp_size/tp_size`、hsdp 维度） -- 若 `training.model_path` 未提供，则默认取 `models.student.path` - - 这是为了让 FastVideo 的 pipeline/pipeline_config registry 能正常工作（以 student 为 base）。 - -设计原因（为什么 training 直接复用 TrainingArgs）： -- FastVideo 的大量组件（loader、pipeline config、distributed init、各种 utils）都以 `TrainingArgs` 为中心。 -- Phase 2 的目标是 **解耦 legacy pipeline**，但不等于要立刻重造整个 training 参数系统；复用 TrainingArgs 能显著降低迁移成本与风险。 +- 分布式尺寸默认值（`num_gpus/tp_size/sp_size/hsdp_*`） +- `training.model_path` 若缺失，默认使用 `models.student.path`（供 pipeline_config registry 使用） ## 6) `pipeline_config` / `pipeline_config_path` -两种等价写法（二选一）： +两种写法（二选一）： -1) 直接内联（推荐用于少量关键字段）： +1) inline（适合少量 override）： ```yaml pipeline_config: flow_shift: 8 dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] ``` -2) 使用外部文件路径（适合复用现有 pipeline config 文件）： +2) path（适合复用大型 config 文件）： ```yaml pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json ``` -设计原因： -- 把 “运行一个 distillation 实验需要的最小 pipeline 配置” 放在 distill YAML 附近，便于复现实验。 -- 但也允许用 path 复用大型 config 文件，避免在 YAML 中塞进过多模型细节。 -- 同时与我们 “outside/ 非侵入式新增 config” 的策略兼容：不必修改上游 `fastvideo/configs/*`。 - -## 7) 最小可运行示例（Wan few-step DMD2） - -完整示例参考： -`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` - -其核心要点是： -- `distill: {model: wan, method: dmd2}` -- `models` 至少包含 `student/teacher/critic` -- `training.data_path / output_dir / max_train_steps / seed` 等训练必须项 -- `pipeline_config.flow_shift` + `pipeline_config.dmd_denoising_steps`（8 steps）用于 few-step schedule - -## 8) Phase 3 计划：`recipe` 顶层 + `method_config` - -Phase 2 的 YAML schema 使用 `distill:` 作为顶层选择（历史原因：当时入口只跑 distillation）。 -但随着我们计划把 **finetuning 也纳入同一框架**，`distill.method=finetune` 的语义会显得别扭。 - -因此 Phase 3 计划升级 schema： - -```yaml -recipe: {family: wan, method: dmd2} # 只负责选择（更通用） -models: {...} # role -> {family/path/trainable} -training: {...} # infra 参数（映射到 TrainingArgs） -pipeline_config: {...} # pipeline/backbone config（模型侧） -method_config: {...} # method-specific 超参（方法侧） -``` - -同时保持与 FastVideo 的 `ExecutionMode` 语义对齐（Phase 3 计划）： -- `recipe.method=finetune` 时：入口层设置 `training.mode=FINETUNING` -- 其它 distillation methods：入口层设置 `training.mode=DISTILLATION` - -### 8.1 为什么需要 `method_config`？ - -动机是把语义分清楚： -- `training:`（TrainingArgs）应该尽量只承载 **基础设施**：分布式、优化器、ckpt、logging、数据路径等 -- `method_config:` 承载 **算法/recipe** 的超参：DMD2 / Self-Forcing / Finetune 各自不同 - -这样未来 method 变多时，不会出现所有参数都混在 `training:` 里，导致配置难读、难 review、难复现。 +备注（重要）： +- 从语义上讲，`dmd_denoising_steps` 是 algorithm knob，不应长期存在于 pipeline_config。 +- 但当前 validation 仍使用 legacy SDE sampler（`WanDMDPipeline` / `DmdDenoisingStage`）， + 它会读取 `pipeline_config.dmd_denoising_steps`。 +- Phase 3.2 会把 sampling timesteps 变成显式 request 参数，从而移除该重复字段。 -### 8.2 示例：DMD2（新 schema） +## 7) `method_config`: method/algorithm 专属超参 +`method_config` 由 method 自己解释。以 DMD2 为例： ```yaml -recipe: - family: wan - method: dmd2 - -models: - student: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} - teacher: {family: wan, path: Wan-AI/Wan2.1-T2V-14B-Diffusers, trainable: false} - critic: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} - -training: - # ... TrainingArgs fields ... - output_dir: outputs/... - max_train_steps: 4000 - seed: 1000 - -pipeline_config: - flow_shift: 8 - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] - method_config: + rollout_mode: simulate # {simulate|data_latent} generator_update_interval: 5 real_score_guidance_scale: 3.5 - simulate_generator_forward: true + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] ``` -### 8.3 示例：Finetuning（only student） - -```yaml -recipe: - family: wan - method: finetune +其中： +- `rollout_mode` 替代 legacy 的 `training.simulate_generator_forward`： + - `simulate`：adapter 用零 latents 构造 batch（不依赖 `vae_latent`） + - `data_latent`：dataset batch 必须提供 `vae_latent` +- `dmd_denoising_steps` 是 method 的 few-step schedule single source of truth。 -models: - student: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} +## 8) 最小可运行示例（Wan few-step DMD2） -training: - # ... TrainingArgs fields ... - data_path: ... - output_dir: outputs/... - max_train_steps: 4000 - seed: 1000 +参考 outside 下的三个可运行 YAML： +- `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` +- `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml` +- `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` -pipeline_config: - flow_shift: 8 - -method_config: - pred_type: x0 - loss: flow_matching -``` diff --git a/dev/design.md b/dev/design.md index c3e77d9aa..f14bfc77d 100644 --- a/dev/design.md +++ b/dev/design.md @@ -380,7 +380,8 @@ FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step` 说明： - Phase 2 的 YAML schema v1 使用 `distill:` 顶层（历史原因） -- Phase 3 将升级为 schema v2：用 `recipe:` 顶层，并引入 `method_config:`（语义更通用） +- Phase 3.1 已升级为 schema v2：用 `recipe:` 顶层，并引入 `method_config:`（语义更通用） + - 入口只接受 schema v2（不再兼容 `distill:`） schema v2 的 “单次运行” 配置示意（字段可迭代）： @@ -398,10 +399,12 @@ models: family: wan path: Wan-AI/Wan2.1-T2V-14B-Diffusers trainable: false + disable_custom_init_weights: true critic: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers trainable: true + disable_custom_init_weights: true training: output_dir: outputs/... @@ -413,9 +416,14 @@ pipeline_config: # 支持直接内联覆盖，也支持只给 pipeline_config_path # pipeline_config_path: fastvideo/configs/wan_1.3B_t2v_pipeline.json flow_shift: 8 + # NOTE: 当前 legacy SDE sampling（`WanDMDPipeline`）仍读取此字段； + # Phase 3.2 会把 sampling timesteps 变成显式 request 参数，从而移除依赖。 + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] method_config: - # method-specific 超参（不进入 TrainingArgs；由 method/adapter 自行解析） + # method-specific 超参（不进入 TrainingArgs；由 method 自行解析） + rollout_mode: simulate + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] generator_update_interval: 5 real_score_guidance_scale: 3.5 ``` @@ -621,8 +629,8 @@ Phase 3 的定位：在 Phase 2.9 已经完成“优雅 dispatch + adapter/metho 动机： - `distill.method=finetune` 语义别扭，因为 finetune 是一种训练 recipe，不一定是“蒸馏”。 - method-specific 参数长期塞进 `training:`（TrainingArgs）/`pipeline_config:` 会让配置语义越来越混杂。 -- Phase 2.9 还残留少量 “method knob 泄漏到 adapter” 的问题（例如 `simulate_generator_forward`），需要借助 - `method_config` 做干净的边界收敛。 +- Phase 2.9 暴露过 “method knob 泄漏到 adapter” 的问题（例如 `simulate_generator_forward`），因此需要引入 + `method_config` 来做干净的边界收敛（Phase 3.1 已解决该耦合）。 Phase 3.1 计划把 YAML schema 升级为： @@ -636,7 +644,7 @@ method_config: {...} # algorithm/method 超参（方法侧） 迁移策略（建议）： - DMD2：把 `generator_update_interval`, `real_score_guidance_scale`, - `dmd_denoising_steps`, `simulate_generator_forward`（或替代字段）迁移到 `method_config`。 + `dmd_denoising_steps`, `rollout_mode`（替代 `simulate_generator_forward`）迁移到 `method_config`。 - `training:` 保持纯 infra（分布式、优化器默认值、ckpt、logging、数据路径等）。 #### Phase 3.2：统一 sampling 语义（ODE/SDE sampler 可插拔） diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 2b6ce5100..29132f207 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -52,32 +52,36 @@ method_config: {...} # algorithm（方法侧） ``` ### 文件 TODO（实现清单） -- [ ] `fastvideo/distillation/specs.py` +- [x] `fastvideo/distillation/specs.py` - 新增 `RecipeSpec(family: str, method: str)` - `DistillRunConfig` 增加 `recipe` 与 `method_config` -- [ ] `fastvideo/distillation/yaml_config.py` +- [x] `fastvideo/distillation/yaml_config.py` - 解析 `recipe:` 与 `method_config:`（默认 `{}`） - - 将 v1 的 `distill:` 视为不再支持（或仅保留一次性迁移期的兼容分支，需你拍板） -- [ ] `fastvideo/distillation/builder.py` + - 将 v1 的 `distill:` 视为不再支持（breaking change，直接推进 schema v2） +- [x] `fastvideo/distillation/builder.py` - 从 `cfg.recipe` 取 family/method（不再读 `cfg.distill`） - build method 时传入 `method_config` -- [ ] `fastvideo/distillation/methods/distribution_matching/dmd2.py` +- [x] `fastvideo/distillation/methods/distribution_matching/dmd2.py` - `DMD2Method(..., method_config=...)` - 关键参数读取优先级：`method_config` > `training_args`（迁移期平滑） - `generator_update_interval` - `real_score_guidance_scale` - `dmd_denoising_steps`（few-step step list） - `rollout_mode`（替代 `simulate_generator_forward`） -- [ ] `fastvideo/distillation/adapters/wan.py` +- [x] `fastvideo/distillation/adapters/wan.py` - 移除 `training_args.simulate_generator_forward` 的读取（这是 Phase 2.9 的残留耦合） - 把 batch 形态做成显式 API/参数，让 method 决定： - 选项 A（推荐）：拆分显式入口 - `prepare_batch_from_data_latent(raw_batch, ...)`（必须有 `vae_latent`） - `prepare_batch_from_placeholder_latent(raw_batch, ...)`（不依赖 `vae_latent`） - - 选项 B：保留单入口但显式参数化：`prepare_batch(..., latents_source=...)` -- [ ] configs / docs - - [ ] `fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml` 全部升级到 schema v2 - - [ ] 更新 `dev/config.md`（描述 schema v2 与迁移策略） + - 选项 B：保留单入口但显式参数化：`prepare_batch(..., latents_source=...)`（本阶段采用） +- [x] configs / docs + - [x] `fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml` 全部升级到 schema v2 + - [x] 更新 `dev/config.md`（描述 schema v2 与迁移策略） + +### 可运行产物 +- Phase 3.1 YAML：`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` +- One-shot 脚本：`examples/distillation/phase3_1/temp.sh` --- @@ -142,9 +146,7 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop ## 备注：关于 `simulate_generator_forward`（Phase 2.9 的残留） -目前 `WanAdapter.prepare_batch()` 仍读取 `training_args.simulate_generator_forward` 来决定是否需要 `vae_latent`。 -这不是“功能 bug”，但确实是语义耦合（把 DMD2 rollout 的开关泄漏到了 adapter）。 - -结论：它 **应该在 Phase 3.1** 通过 `method_config.rollout_mode` + 显式 batch API 来解决； -Phase 2.9 没处理是因为当时我们刻意不引入 schema v2/method_config，避免变量叠加导致定位困难。 - +该耦合已在 Phase 3.1 解决： +- `WanAdapter.prepare_batch()` 不再读取 `training_args.simulate_generator_forward` +- `DMD2Method` 通过 `method_config.rollout_mode` 决定 `latents_source={zeros|data}`， + 并把它作为参数传给 adapter（adapter 只处理 batch 形态，不解释 DMD2 语义） diff --git a/examples/distillation/phase3_1/temp.sh b/examples/distillation/phase3_1/temp.sh new file mode 100644 index 000000000..c46aa1e4b --- /dev/null +++ b/examples/distillation/phase3_1/temp.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -euo pipefail +if [[ "${DEBUG:-0}" == "1" ]]; then + set -x +fi + +# One-shot launch script for Phase 3.1 (schema v2: `recipe` + `method_config`). +# +# Uses the same dataset/validation defaults as phase0/phase1/phase2/phase2_9; +# the main difference is config semantics (method knobs moved into +# `method_config`) and the removal of `simulate_generator_forward` coupling in +# adapters. + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29510} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" diff --git a/fastvideo/distillation/adapters/base.py b/fastvideo/distillation/adapters/base.py index a424eb3e8..fa40ea280 100644 --- a/fastvideo/distillation/adapters/base.py +++ b/fastvideo/distillation/adapters/base.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal if TYPE_CHECKING: from fastvideo.pipelines import TrainingBatch @@ -16,5 +16,6 @@ def prepare_batch( raw_batch: dict[str, Any], *, current_vsa_sparsity: float = 0.0, - ) -> "TrainingBatch": + latents_source: Literal["data", "zeros"] = "data", + ) -> TrainingBatch: raise NotImplementedError diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 7b4ede936..6716a4cd3 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -327,6 +327,7 @@ def prepare_batch( raw_batch: dict[str, Any], *, current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", ) -> TrainingBatch: self.ensure_negative_conditioning() @@ -338,7 +339,7 @@ def prepare_batch( encoder_attention_mask = raw_batch["text_attention_mask"] infos = raw_batch.get("info_list") - if self.training_args.simulate_generator_forward: + if latents_source == "zeros": batch_size = encoder_hidden_states.shape[0] vae_config = self.training_args.pipeline_config.vae_config.arch_config num_channels = vae_config.z_dim @@ -354,14 +355,16 @@ def prepare_batch( device=device, dtype=dtype, ) - else: + elif latents_source == "data": if "vae_latent" not in raw_batch: raise ValueError( - "vae_latent not found in batch and simulate_generator_forward is False" + "vae_latent not found in batch and latents_source='data'" ) latents = raw_batch["vae_latent"] latents = latents[:, :, : self.training_args.num_latent_t] latents = latents.to(device, dtype=dtype) + else: + raise ValueError(f"Unknown latents_source: {latents_source!r}") training_batch.latents = latents training_batch.encoder_hidden_states = encoder_hidden_states.to(device, dtype=dtype) diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index 932d76140..ee886d76f 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -8,17 +8,17 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: - """Build a distillation runtime from a Phase 2 YAML config. + """Build a distillation runtime from a YAML config. This is the Phase 2.9 "elegant dispatch" entry for assembling: - model family artifacts (bundle/adapter/dataloader/tracker) - method implementation (algorithm) on top of those artifacts """ - family_builder = get_family(str(cfg.distill.model)) + family_builder = get_family(str(cfg.recipe.family)) artifacts = family_builder(cfg=cfg) - method_builder = get_method(str(cfg.distill.method)) + method_builder = get_method(str(cfg.recipe.method)) method = method_builder( cfg=cfg, bundle=artifacts.bundle, @@ -38,10 +38,10 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: def build_wan_dmd2_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: """Legacy Phase 2 helper kept for compatibility during Phase 2.9 rollout.""" - if cfg.distill.model != "wan" or cfg.distill.method != "dmd2": + if cfg.recipe.family != "wan" or cfg.recipe.method != "dmd2": raise ValueError( - "build_wan_dmd2_runtime_from_config expects distill.model='wan' " - f"and distill.method='dmd2', got model={cfg.distill.model!r}, " - f"method={cfg.distill.method!r}" + "build_wan_dmd2_runtime_from_config expects recipe.family='wan' " + f"and recipe.method='dmd2', got family={cfg.recipe.family!r}, " + f"method={cfg.recipe.method!r}" ) return build_runtime_from_config(cfg) diff --git a/fastvideo/distillation/doc/builder.md b/fastvideo/distillation/doc/builder.md index 4e3a50e11..5a659b41e 100644 --- a/fastvideo/distillation/doc/builder.md +++ b/fastvideo/distillation/doc/builder.md @@ -7,8 +7,8 @@ **关键 API** - `build_runtime_from_config(cfg) -> DistillRuntime` - - `family_builder = registry.get_family(cfg.distill.model)` - - `method_builder = registry.get_method(cfg.distill.method)` + - `family_builder = registry.get_family(cfg.recipe.family)` + - `method_builder = registry.get_method(cfg.recipe.method)` - `method = method_builder(cfg=cfg, bundle=artifacts.bundle, adapter=artifacts.adapter, validator=artifacts.validator)` - `build_wan_dmd2_runtime_from_config(cfg)` - Phase 2.9 期间保留的兼容函数（最终可删除） diff --git a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md index 42cc180ca..2791241eb 100644 --- a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md +++ b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md @@ -27,8 +27,10 @@ **few-step rollout policy（Phase 2.9）** - rollout 的 step list / simulate 逻辑由 method 管理： - - `pipeline_config.dmd_denoising_steps` → `_get_denoising_step_list()` - - `simulate_generator_forward` 控制单步/多步模拟路径 + - `method_config.dmd_denoising_steps` → `_get_denoising_step_list()` + - `method_config.rollout_mode={simulate|data_latent}` + - `simulate`：batch 不要求 `vae_latent`（adapter 用零 latents 构造形状） + - `data_latent`：batch 必须带 `vae_latent` - 可选 `warp_denoising_step`（通过 adapter.noise_scheduler.timesteps duck-typing） - adapter 只提供单步 primitives： - `predict_x0()` / `predict_noise()` / `add_noise()` @@ -46,5 +48,11 @@ validator 负责执行采样与记录，保持 method-agnostic。 **配置语义的 TODO（Phase 3）** -- 目前仍从 `training_args` 读取 DMD2/critic 专属字段（例如 `fake_score_*`、`dmd_denoising_steps`）。 - Phase 3 计划引入 `method_config`，把这些算法超参从 `training:` / `pipeline_config:` 中迁移出去。 +- Phase 3.1 已引入 `method_config`，并将 rollout 的关键 knob 迁移到 method 层： + - `rollout_mode` + - `dmd_denoising_steps` + - `generator_update_interval` + - `real_score_guidance_scale` +- 仍有一个过渡期现实：legacy SDE sampler（`WanDMDPipeline` / `DmdDenoisingStage`） + 目前读取 `pipeline_config.dmd_denoising_steps`，因此 config 会短期出现重复字段。 + Phase 3.2 会把 sampling timesteps 变成显式 request 参数，从而移除该重复。 diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md index 6687eaea5..599fba2d7 100644 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md @@ -1,20 +1,21 @@ # `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` -这是一个 Phase 2/2.9 的可运行示例：**Wan few-step distillation（8 steps）+ DMD2**。 +这是一个可运行示例（schema v2）：**Wan few-step distillation（8 steps）+ DMD2**。 --- ## 顶层结构 -- `distill:` - - `model: wan` → registry dispatch 到 `families/wan.py` +- `recipe:` + - `family: wan` → registry dispatch 到 `families/wan.py` - `method: dmd2` → registry dispatch 到 `methods/distribution_matching/dmd2.py` - `models:` - `student / teacher / critic` 三个 roles（role 名称本身由 method 解释语义） - 每个 role 指定： - - `family`（默认可省略，继承 `distill.model`） + - `family`（默认可省略，继承 `recipe.family`） - `path`（权重路径） - `trainable`（是否训练） + - `disable_custom_init_weights`（可选；用于 teacher/critic 等 auxiliary roles 的加载语义） - `training:` - 主要复用 `TrainingArgs.from_kwargs()` 的字段集合（batch/shape/steps/logging 等） - `pipeline_config:` @@ -31,12 +32,12 @@ **Method（DMD2）关心：** - update policy：`generator_update_interval` -- student rollout 相关：`simulate_generator_forward` +- student rollout 相关：`method_config.rollout_mode` - optimizer/scheduler（Phase 2.9 已迁移到 method 创建）： - student：`learning_rate / betas / lr_scheduler` - critic（DMD2 专属覆盖）：`fake_score_learning_rate / fake_score_betas / fake_score_lr_scheduler` -- few-step step list（目前仍放在 `pipeline_config`）： - - `pipeline_config.dmd_denoising_steps` +- few-step step list（single source of truth 在 `method_config`）： + - `method_config.dmd_denoising_steps` **Adapter（WanAdapter）关心：** - 把 FastVideo/Wan 的 forward primitives 暴露给 method（不包含 step list/policy） @@ -45,7 +46,6 @@ ## TODO（Phase 3） -为进一步减少 “training/pipeline_config 承载算法语义”，建议迁移： -- `fake_score_*` → `method_config.optimizers.critic.*` -- `dmd_denoising_steps` → `method_config.rollout.steps`（或类似命名） - +Phase 3.2 会把 validation sampling 的 ODE/SDE loop 做成可插拔 sampler， +从而淘汰对 `WanDMDPipeline` 的依赖，并移除 `pipeline_config.dmd_denoising_steps` +这类 “algorithm knob” 在 pipeline config 里的残留。 diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md new file mode 100644 index 000000000..f4c3ee56c --- /dev/null +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md @@ -0,0 +1,14 @@ +# `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml` + +这是一个 **Phase 2.9** 的可运行示例（schema v2）： +Wan few-step distillation（8 steps）+ DMD2。 + +它与 `distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` 的结构基本一致，主要差异在于： +- `training.output_dir / tracker_project_name / wandb_run_name` 用于区分实验阶段。 + +备注： +- few-step step list 的 single source of truth 在 `method_config.dmd_denoising_steps`。 +- 由于当前 validation 仍使用 legacy SDE sampling（`WanDMDPipeline`），pipeline 会读取 + `pipeline_config.dmd_denoising_steps`，因此该字段短期会与 `method_config` 重复； + Phase 3.2 会移除此重复（sampler 可插拔 + 显式 timesteps request）。 + diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md new file mode 100644 index 000000000..f0ff6995f --- /dev/null +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md @@ -0,0 +1,15 @@ +# `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` + +这是一个 **Phase 3.1** 的可运行示例（schema v2：`recipe` + `method_config`）： +Wan few-step distillation（8 steps）+ DMD2。 + +它与 `distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` 的结构基本一致，主要用于强调 Phase 3.1 的配置语义： +- 顶层选择从 `distill` 升级为 `recipe`。 +- method knobs（例如 `rollout_mode` / `dmd_denoising_steps`）进入 `method_config`。 +- `WanAdapter.prepare_batch()` 不再读取 legacy 的 `training.simulate_generator_forward`。 + +备注： +- 由于当前 validation 仍使用 legacy SDE sampling（`WanDMDPipeline`），pipeline 会读取 + `pipeline_config.dmd_denoising_steps`，因此该字段短期会与 `method_config` 重复； + Phase 3.2 会移除此重复（sampler 可插拔 + 显式 timesteps request）。 + diff --git a/fastvideo/distillation/doc/specs.md b/fastvideo/distillation/doc/specs.md index a3662f2c5..2c9694322 100644 --- a/fastvideo/distillation/doc/specs.md +++ b/fastvideo/distillation/doc/specs.md @@ -6,15 +6,17 @@ - builder/registry dispatch（`builder.py` / `registry.py`） **关键类型** -- `DistillSpec` - - `model`: family 名称（例如 `"wan"`） +- `RecipeSpec` + - `family`: family 名称（例如 `"wan"`） - `method`: method 名称（例如 `"dmd2"`） - `RoleSpec` - - `family`: 该 role 的 family（默认可继承 `distill.model`） + - `family`: 该 role 的 family（默认可继承 `recipe.family`） - `path`: 模型权重路径（HF repo 或本地目录） - `trainable`: 是否训练该 role（只影响 `requires_grad`/模式；具体 optimizer 由 method 决定） + - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” + - 这是一个 build-time/loader 语义开关（用于 teacher/critic 等 auxiliary roles） + - 角色名不应在 family 内被 hard-code；由 YAML 显式声明更清晰 **注意** - role 名称本身（`student/teacher/critic/...`）是字符串。 framework 不强行规定“canonical roles”，由 method 决定语义与依赖。 - diff --git a/fastvideo/distillation/doc/yaml_config.md b/fastvideo/distillation/doc/yaml_config.md index 3b0effb2f..83d1f97f2 100644 --- a/fastvideo/distillation/doc/yaml_config.md +++ b/fastvideo/distillation/doc/yaml_config.md @@ -1,20 +1,22 @@ # `fastvideo/distillation/yaml_config.py` **目的** -- Phase 2 distillation 的 YAML-only 配置加载器（不兼容/不 merge legacy CLI）。 +- distillation 的 YAML-only 配置加载器（schema v2；不兼容/不 merge legacy CLI）。 **核心产物** - `DistillRunConfig` - - `distill: DistillSpec`（选择 family + method） + - `recipe: RecipeSpec`（选择 family + method） - `roles: dict[str, RoleSpec]`（来自 YAML 的 `models:`） - `training_args: TrainingArgs`（来自 YAML 的 `training:`，并注入 entrypoint invariants） + - `method_config: dict`（来自 YAML 的 `method_config:`，传给 method 解释） - `raw: dict`（原始 YAML，便于 tracker 记录） -**YAML 结构（Phase 2）** -- `distill: {model: ..., method: ...}` +**YAML 结构（schema v2）** +- `recipe: {family: ..., method: ...}` - `models: {: {family?, path, trainable?}, ...}` - `training: {...}`（大部分字段复用 `TrainingArgs.from_kwargs()`） - `pipeline_config` 或 `pipeline_config_path` +- `method_config: {...}`（算法/recipe 专属超参） **实现要点** - `_resolve_existing_file()`：要求传入真实存在的路径（Phase 2 不做 overlay/fallback） @@ -28,3 +30,5 @@ - `dit_precision` 默认 `fp32`（master weights） - `dit_cpu_offload = False` +**兼容性** +- loader 仅接受 schema v2：缺少 `recipe:` 会直接报错。 diff --git a/fastvideo/distillation/families/wan.py b/fastvideo/distillation/families/wan.py index 57e496b8d..ad5d91177 100644 --- a/fastvideo/distillation/families/wan.py +++ b/fastvideo/distillation/families/wan.py @@ -27,7 +27,7 @@ def _load_module_from_path( model_path: str, module_type: str, training_args: Any, - mark_teacher_critic: bool = False, + disable_custom_init_weights: bool = False, ) -> torch.nn.Module: local_model_path = maybe_download_model(model_path) config = verify_model_config_and_directory(local_model_path) @@ -42,7 +42,10 @@ def _load_module_from_path( transformers_or_diffusers, _architecture = module_info component_path = os.path.join(local_model_path, module_type) - if mark_teacher_critic: + if disable_custom_init_weights: + # NOTE: This flag is used by PipelineComponentLoader to skip applying + # `init_weights_from_safetensors*` overrides when loading auxiliary + # roles (teacher/critic/etc). The attribute name is legacy. training_args._loading_teacher_critic_model = True try: module = PipelineComponentLoader.load_module( @@ -52,7 +55,9 @@ def _load_module_from_path( fastvideo_args=training_args, ) finally: - if mark_teacher_critic and hasattr(training_args, "_loading_teacher_critic_model"): + if disable_custom_init_weights and hasattr( + training_args, "_loading_teacher_critic_model" + ): del training_args._loading_teacher_critic_model if not isinstance(module, torch.nn.Module): @@ -71,19 +76,19 @@ def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Mo def _build_tracker(training_args: Any, *, config: dict[str, Any] | None) -> Any: world_group = get_world_group() - trackers = list(getattr(training_args, "trackers", [])) - if not trackers and str(getattr(training_args, "tracker_project_name", "")): + trackers = list(training_args.trackers) + if not trackers and str(training_args.tracker_project_name): trackers.append(Trackers.WANDB.value) if world_group.rank != 0: trackers = [] - tracker_log_dir = getattr(training_args, "output_dir", "") or os.getcwd() + tracker_log_dir = training_args.output_dir or os.getcwd() if trackers: tracker_log_dir = os.path.join(tracker_log_dir, "tracker") tracker_config = config if trackers else None - tracker_run_name = getattr(training_args, "wandb_run_name", "") or None - project = getattr(training_args, "tracker_project_name", "") or "fastvideo" + tracker_run_name = training_args.wandb_run_name or None + project = training_args.tracker_project_name or "fastvideo" return initialize_trackers( trackers, experiment_name=project, @@ -122,28 +127,29 @@ def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: f"got {role}={role_spec.family!r}" ) - mark_teacher_critic = role in ("teacher", "critic") + disable_custom_init_weights = bool( + getattr(role_spec, "disable_custom_init_weights", False) + ) transformer = _load_module_from_path( model_path=role_spec.path, module_type="transformer", training_args=training_args, - mark_teacher_critic=mark_teacher_critic, + disable_custom_init_weights=disable_custom_init_weights, ) modules: dict[str, torch.nn.Module] = {"transformer": transformer} - # Optional MoE support: allow teacher transformer_2 if present. - if role == "teacher": - try: - transformer_2 = _load_module_from_path( - model_path=role_spec.path, - module_type="transformer_2", - training_args=training_args, - mark_teacher_critic=mark_teacher_critic, - ) - except Exception: - transformer_2 = None - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 + # Optional MoE support: load transformer_2 if present in the model. + try: + transformer_2 = _load_module_from_path( + model_path=role_spec.path, + module_type="transformer_2", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + ) + except ValueError: + transformer_2 = None + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 for name, module in list(modules.items()): module = _apply_trainable(module, trainable=bool(role_spec.trainable)) diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 0567d6e11..231a571a3 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -43,6 +43,7 @@ def prepare_batch( raw_batch: dict[str, Any], *, current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", ) -> Any: ... @@ -108,6 +109,7 @@ def __init__( *, bundle: ModelBundle, adapter: _DMD2Adapter, + method_config: dict[str, Any] | None = None, validator: Any | None = None, ) -> None: super().__init__(bundle) @@ -124,12 +126,59 @@ def __init__( self.adapter = adapter self.validator = validator self.training_args = adapter.training_args - self._simulate_generator_forward = bool( - getattr(self.training_args, "simulate_generator_forward", False) - ) + self.method_config: dict[str, Any] = dict(method_config or {}) + self._rollout_mode = self._parse_rollout_mode() self._denoising_step_list: torch.Tensor | None = None self._init_optimizers_and_schedulers() + def _parse_rollout_mode(self) -> Literal["simulate", "data_latent"]: + raw = self.method_config.get("rollout_mode", None) + if raw is None: + raise ValueError("method_config.rollout_mode must be set for DMD2") + if not isinstance(raw, str): + raise ValueError( + "method_config.rollout_mode must be a string, got " + f"{type(raw).__name__}" + ) + mode = raw.strip().lower() + if mode in ("simulate", "sim"): + return "simulate" + if mode in ("data_latent", "data", "vae_latent"): + return "data_latent" + raise ValueError( + "method_config.rollout_mode must be one of " + "{simulate, data_latent}, got " + f"{raw!r}" + ) + + def _get_method_int(self, key: str) -> int | None: + raw = self.method_config.get(key, None) + if raw is None: + return None + if isinstance(raw, bool): + raise ValueError(f"method_config.{key} must be an int, got bool") + if isinstance(raw, int): + return int(raw) + if isinstance(raw, float) and raw.is_integer(): + return int(raw) + if isinstance(raw, str) and raw.strip(): + return int(raw) + raise ValueError(f"method_config.{key} must be an int, got {type(raw).__name__}") + + def _get_method_float(self, key: str) -> float | None: + raw = self.method_config.get(key, None) + if raw is None: + return None + if isinstance(raw, bool): + raise ValueError(f"method_config.{key} must be a float, got bool") + if isinstance(raw, (int, float)): + return float(raw) + if isinstance(raw, str) and raw.strip(): + return float(raw) + raise ValueError( + f"method_config.{key} must be a float, got {type(raw).__name__}" + ) + def _parse_betas(self, raw: Any, *, where: str) -> tuple[float, float]: if raw is None: raise ValueError(f"Missing betas for {where}") @@ -232,8 +281,8 @@ def log_validation(self, iteration: int) -> None: sampling_steps = [s for s in sampling_steps if s > 0] if not sampling_steps: # Default to the few-step student rollout step count for DMD2. - raw_rollout = getattr(self.training_args.pipeline_config, "dmd_denoising_steps", None) - if not raw_rollout: + raw_rollout = self.method_config.get("dmd_denoising_steps", None) + if not isinstance(raw_rollout, list) or not raw_rollout: return sampling_steps = [int(len(raw_rollout))] @@ -265,18 +314,29 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: return generators def _should_update_student(self, iteration: int) -> bool: - interval = int(getattr(self.training_args, "generator_update_interval", 1) or 1) + interval = self._get_method_int("generator_update_interval") + if interval is None: + interval = int(getattr(self.training_args, "generator_update_interval", 1) or 1) if interval <= 0: return True return iteration % interval == 0 def _clip_grad_norm(self, module: torch.nn.Module) -> float: - max_grad_norm = getattr(self.training_args, "max_grad_norm", None) - if not max_grad_norm: + max_grad_norm_raw = getattr(self.training_args, "max_grad_norm", None) + if max_grad_norm_raw is None: + return 0.0 + try: + max_grad_norm = float(max_grad_norm_raw) + except (TypeError, ValueError) as e: + raise ValueError( + "training.max_grad_norm must be a number when set, got " + f"{max_grad_norm_raw!r}" + ) from e + if max_grad_norm <= 0.0: return 0.0 grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( [p for p in module.parameters()], - float(max_grad_norm), + max_grad_norm, foreach=None, ) return float(grad_norm.item()) if grad_norm is not None else 0.0 @@ -285,13 +345,16 @@ def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: if self._denoising_step_list is not None and self._denoising_step_list.device == device: return self._denoising_step_list - raw = getattr(self.training_args.pipeline_config, "dmd_denoising_steps", None) - if not raw: - raise ValueError("pipeline_config.dmd_denoising_steps must be set for DMD2 distillation") + raw = self.method_config.get("dmd_denoising_steps", None) + if not isinstance(raw, list) or not raw: + raise ValueError("method_config.dmd_denoising_steps must be set for DMD2 distillation") - steps = torch.tensor(raw, dtype=torch.long, device=device) + steps = torch.tensor([int(s) for s in raw], dtype=torch.long, device=device) - if getattr(self.training_args, "warp_denoising_step", False): + warp = self.method_config.get("warp_denoising_step", None) + if warp is None: + warp = getattr(self.training_args, "warp_denoising_step", False) + if bool(warp): noise_scheduler = getattr(self.adapter, "noise_scheduler", None) if noise_scheduler is None: raise ValueError("warp_denoising_step requires adapter.noise_scheduler.timesteps") @@ -324,7 +387,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: dtype = latents.dtype step_list = self._get_denoising_step_list(device) - if not self._simulate_generator_forward: + if self._rollout_mode != "simulate": timestep = self._sample_rollout_timestep(device) noise = torch.randn(latents.shape, device=device, dtype=dtype) noisy_latents = self.adapter.add_noise(latents, noise, timestep) @@ -455,7 +518,9 @@ def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dic return flow_matching_loss, (batch.timesteps, batch.attn_metadata), outputs def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor: - guidance_scale = float(getattr(self.training_args, "real_score_guidance_scale", 1.0)) + guidance_scale = self._get_method_float("real_score_guidance_scale") + if guidance_scale is None: + guidance_scale = float(getattr(self.training_args, "real_score_guidance_scale", 1.0)) device = generator_pred_x0.device with torch.no_grad(): @@ -518,9 +583,14 @@ def single_train_step( *, current_vsa_sparsity: float = 0.0, ) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + latents_source: Literal["data", "zeros"] = "data" + if self._rollout_mode == "simulate": + latents_source = "zeros" + training_batch = self.adapter.prepare_batch( batch, current_vsa_sparsity=current_vsa_sparsity, + latents_source=latents_source, ) update_student = self._should_update_student(iteration) @@ -618,5 +688,9 @@ def build_dmd2_method( adapter: _DMD2Adapter, validator: Any | None, ) -> DistillMethod: - del cfg - return DMD2Method(bundle=bundle, adapter=adapter, validator=validator) + return DMD2Method( + bundle=bundle, + adapter=adapter, + method_config=cfg.method_config, + validator=validator, + ) diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index 314872e03..d5a5cc575 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -1,5 +1,5 @@ -distill: - model: wan +recipe: + family: wan method: dmd2 models: @@ -11,10 +11,12 @@ models: family: wan path: Wan-AI/Wan2.1-T2V-14B-Diffusers trainable: false + disable_custom_init_weights: true critic: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers trainable: true + disable_custom_init_weights: true training: # Distributed @@ -58,9 +60,6 @@ training: training_cfg_rate: 0.0 min_timestep_ratio: 0.02 max_timestep_ratio: 0.98 - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - simulate_generator_forward: true enable_gradient_checkpointing_type: full # Tracking / validation @@ -74,4 +73,14 @@ training: pipeline_config: flow_shift: 8 + # NOTE: legacy SDE sampling (`WanDMDPipeline`) reads this field today. + # Phase 3.2 will make sampling timesteps an explicit request parameter. + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + +method_config: + # Replace legacy `training.simulate_generator_forward`. + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + # Few-step schedule (kept here as the method single source of truth). dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml index 0f86de9eb..9af959d0f 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml @@ -1,5 +1,5 @@ -distill: - model: wan +recipe: + family: wan method: dmd2 models: @@ -11,10 +11,12 @@ models: family: wan path: Wan-AI/Wan2.1-T2V-14B-Diffusers trainable: false + disable_custom_init_weights: true critic: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers trainable: true + disable_custom_init_weights: true training: # Distributed @@ -58,9 +60,6 @@ training: training_cfg_rate: 0.0 min_timestep_ratio: 0.02 max_timestep_ratio: 0.98 - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - simulate_generator_forward: true enable_gradient_checkpointing_type: full # Tracking / validation @@ -74,4 +73,14 @@ training: pipeline_config: flow_shift: 8 + # NOTE: legacy SDE sampling (`WanDMDPipeline`) reads this field today. + # Phase 3.2 will make sampling timesteps an explicit request parameter. + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + +method_config: + # Replace legacy `training.simulate_generator_forward`. + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + # Few-step schedule (kept here as the method single source of truth). dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml new file mode 100644 index 000000000..ede0272fe --- /dev/null +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml @@ -0,0 +1,86 @@ +recipe: + family: wan + method: dmd2 + +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + disable_custom_init_weights: true + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + disable_custom_init_weights: true + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.1_wan2.1_dmd2_8steps_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation (method-agnostic knobs) + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: phase3.1_wan_dmd2_8steps_wansyn + wandb_run_name: phase3.1_wan_dmd2_8steps_wansyn + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "8" + validation_guidance_scale: "6.0" + +pipeline_config: + flow_shift: 8 + # NOTE: legacy SDE sampling (`WanDMDPipeline`) reads this field today. + # Phase 3.2 will make sampling timesteps an explicit request parameter. + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + +method_config: + # Replace legacy `training.simulate_generator_forward`. + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + # Few-step schedule (single source of truth for the method). + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/fastvideo/distillation/specs.py b/fastvideo/distillation/specs.py index 09048918e..f971d65b2 100644 --- a/fastvideo/distillation/specs.py +++ b/fastvideo/distillation/specs.py @@ -8,14 +8,14 @@ @dataclass(slots=True) -class DistillSpec: - """Selects the model family + distillation method. +class RecipeSpec: + """Selects the model family + training method. This is intentionally small: everything else (roles, training args, and pipeline config) lives in the run config. """ - model: str + family: str method: str @@ -26,4 +26,4 @@ class RoleSpec: family: str path: str trainable: bool = True - + disable_custom_init_weights: bool = False diff --git a/fastvideo/distillation/yaml_config.py b/fastvideo/distillation/yaml_config.py index a2bdd1558..b1174ed29 100644 --- a/fastvideo/distillation/yaml_config.py +++ b/fastvideo/distillation/yaml_config.py @@ -12,16 +12,17 @@ from fastvideo.fastvideo_args import ExecutionMode, TrainingArgs from fastvideo.logger import init_logger -from fastvideo.distillation.specs import DistillSpec, RoleName, RoleSpec +from fastvideo.distillation.specs import RecipeSpec, RoleName, RoleSpec logger = init_logger(__name__) @dataclass(slots=True) class DistillRunConfig: - distill: DistillSpec + recipe: RecipeSpec roles: dict[RoleName, RoleSpec] training_args: TrainingArgs + method_config: dict[str, Any] raw: dict[str, Any] @@ -71,7 +72,7 @@ def _get_bool(raw: Any, *, where: str, default: bool) -> bool: def load_distill_run_config(path: str) -> DistillRunConfig: - """Load a Phase 2 distillation run config from YAML. + """Load a distillation run config from YAML (schema v2). This loader intentionally does **not** merge with legacy CLI args. The YAML file is the single source of truth for a run. @@ -82,17 +83,17 @@ def load_distill_run_config(path: str) -> DistillRunConfig: raw = yaml.safe_load(f) cfg = _require_mapping(raw, where=path) - distill_raw = _require_mapping(cfg.get("distill"), where="distill") - distill_model = _require_str(distill_raw.get("model"), where="distill.model") - distill_method = _require_str(distill_raw.get("method"), where="distill.method") - distill = DistillSpec(model=distill_model, method=distill_method) + recipe_raw = _require_mapping(cfg.get("recipe"), where="recipe") + recipe_family = _require_str(recipe_raw.get("family"), where="recipe.family") + recipe_method = _require_str(recipe_raw.get("method"), where="recipe.method") + recipe = RecipeSpec(family=recipe_family, method=recipe_method) roles_raw = _require_mapping(cfg.get("models"), where="models") roles: dict[RoleName, RoleSpec] = {} for role, role_cfg_raw in roles_raw.items(): role_str = _require_str(role, where="models.") role_cfg = _require_mapping(role_cfg_raw, where=f"models.{role_str}") - family = role_cfg.get("family") or distill_model + family = role_cfg.get("family") or recipe_family family = _require_str(family, where=f"models.{role_str}.family") model_path = _require_str(role_cfg.get("path"), where=f"models.{role_str}.path") trainable = _get_bool( @@ -100,10 +101,26 @@ def load_distill_run_config(path: str) -> DistillRunConfig: where=f"models.{role_str}.trainable", default=True, ) - roles[role_str] = RoleSpec(family=family, path=model_path, trainable=trainable) + disable_custom_init_weights = _get_bool( + role_cfg.get("disable_custom_init_weights"), + where=f"models.{role_str}.disable_custom_init_weights", + default=False, + ) + roles[role_str] = RoleSpec( + family=family, + path=model_path, + trainable=trainable, + disable_custom_init_weights=disable_custom_init_weights, + ) training_raw = _require_mapping(cfg.get("training"), where="training") + method_config_raw = cfg.get("method_config", None) + if method_config_raw is None: + method_config: dict[str, Any] = {} + else: + method_config = _require_mapping(method_config_raw, where="method_config") + pipeline_cfg_raw = cfg.get("pipeline_config", None) pipeline_cfg_path = cfg.get("pipeline_config_path", None) if pipeline_cfg_raw is not None and pipeline_cfg_path is not None: @@ -154,8 +171,9 @@ def load_distill_run_config(path: str) -> DistillRunConfig: training_args = TrainingArgs.from_kwargs(**training_kwargs) return DistillRunConfig( - distill=distill, + recipe=recipe, roles=roles, training_args=training_args, + method_config=method_config, raw=cfg, ) diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 8bc4b01d2..254e50fa9 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -17,7 +17,7 @@ def run_distillation_from_config( resume_from_checkpoint: str | None = None, override_output_dir: str | None = None, ) -> None: - """Phase 2 entrypoint (YAML-only, standalone runtime builder).""" + """YAML-only distillation entrypoint (schema v2).""" from fastvideo.distributed import maybe_init_distributed_environment_and_model_parallel from fastvideo.distillation import DistillTrainer @@ -82,7 +82,7 @@ def main(args: Any) -> None: dry_run = bool(args.dry_run) resume_from_checkpoint = getattr(args, "resume_from_checkpoint", None) override_output_dir = getattr(args, "override_output_dir", None) - logger.info("Starting Phase 2 distillation from config=%s", config_path) + logger.info("Starting distillation from config=%s", config_path) run_distillation_from_config( config_path, dry_run=dry_run, @@ -103,7 +103,7 @@ def main(args: Any) -> None: "--config", type=str, required=True, - help="Path to distillation YAML config (Phase 2 entrypoint).", + help="Path to distillation YAML config (schema v2).", ) parser.add_argument( "--dry-run", From b985942d2a42b93f368ad83bffff937ff4f8ac52 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 21:32:17 +0000 Subject: [PATCH 087/214] phase 3.2 impl --- dev/config.md | 15 ++-- dev/design.md | 14 ++-- dev/phases/phase_3.md | 34 ++++----- fastvideo/configs/pipelines/base.py | 13 ++++ .../doc/methods/distribution_matching/dmd2.md | 6 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.md | 9 +-- ...ll_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md | 7 +- ...ll_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md | 7 +- fastvideo/distillation/doc/validators/wan.md | 7 +- .../methods/distribution_matching/dmd2.py | 13 +++- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 3 - ..._wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml | 3 - ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml | 3 - fastvideo/distillation/validators/base.py | 3 + fastvideo/distillation/validators/wan.py | 47 +++++++++--- .../pipelines/basic/wan/wan_dmd_pipeline.py | 72 ++++--------------- .../basic/wan/wan_i2v_dmd_pipeline.py | 6 +- fastvideo/pipelines/basic/wan/wan_pipeline.py | 48 ++++++++----- fastvideo/pipelines/pipeline_batch_info.py | 4 ++ fastvideo/pipelines/samplers/__init__.py | 8 +++ fastvideo/pipelines/samplers/base.py | 26 +++++++ fastvideo/pipelines/samplers/wan.py | 29 ++++++++ fastvideo/pipelines/stages/__init__.py | 3 +- fastvideo/pipelines/stages/denoising.py | 70 ++++++++++-------- 24 files changed, 269 insertions(+), 181 deletions(-) create mode 100644 fastvideo/pipelines/samplers/__init__.py create mode 100644 fastvideo/pipelines/samplers/base.py create mode 100644 fastvideo/pipelines/samplers/wan.py diff --git a/dev/config.md b/dev/config.md index a5da8d336..742404ca8 100644 --- a/dev/config.md +++ b/dev/config.md @@ -104,7 +104,6 @@ loader 会注入/补全的 invariants（见 `fastvideo/distillation/yaml_config. ```yaml pipeline_config: flow_shift: 8 - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] ``` 2) path（适合复用大型 config 文件）： @@ -112,11 +111,16 @@ pipeline_config: pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json ``` +常见字段（非穷举）： +- `flow_shift`：Wan 的 flow-matching shift（影响 noise schedule）。 +- `sampler_kind`：`ode|sde`，选择 sampling loop 语义（`WanPipeline` 内部切换）。 + 备注（重要）： -- 从语义上讲，`dmd_denoising_steps` 是 algorithm knob，不应长期存在于 pipeline_config。 -- 但当前 validation 仍使用 legacy SDE sampler（`WanDMDPipeline` / `DmdDenoisingStage`）， - 它会读取 `pipeline_config.dmd_denoising_steps`。 -- Phase 3.2 会把 sampling timesteps 变成显式 request 参数，从而移除该重复字段。 +- 从语义上讲，`dmd_denoising_steps` 是 algorithm knob，应当只存在于 `method_config`。 +- Phase 3.2 已将 sampling loop 语义显式化： + - method 通过 `ValidationRequest(sampler_kind=..., sampling_timesteps=...)` 指定采样方式与 few-step schedule + - `WanValidator` 将 timesteps 写入 `ForwardBatch.sampling_timesteps`，并使用 `WanPipeline` 执行采样 + - `pipeline_config.dmd_denoising_steps` 不再是 distillation 的必需字段（仅保留为 inference/legacy 兼容） ## 7) `method_config`: method/algorithm 专属超参 @@ -141,4 +145,3 @@ method_config: - `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` - `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml` - `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` - diff --git a/dev/design.md b/dev/design.md index f14bfc77d..5f691ea2e 100644 --- a/dev/design.md +++ b/dev/design.md @@ -416,9 +416,6 @@ pipeline_config: # 支持直接内联覆盖，也支持只给 pipeline_config_path # pipeline_config_path: fastvideo/configs/wan_1.3B_t2v_pipeline.json flow_shift: 8 - # NOTE: 当前 legacy SDE sampling（`WanDMDPipeline`）仍读取此字段； - # Phase 3.2 会把 sampling timesteps 变成显式 request 参数，从而移除依赖。 - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] method_config: # method-specific 超参（不进入 TrainingArgs；由 method 自行解析） @@ -617,10 +614,12 @@ Phase 2.9 目标（简称 A+B+Families）： 这会直接导致 validation sampling 的 drift：即使训练一致，选不同 loop 的 pipeline 也可能出不同视频。 Phase 2.9 为了端到端与 legacy apples-to-apples，对 Wan DMD2 的 validation 暂时使用 -`WanDMDPipeline`（SDE rollout）以避免漂移；Phase 3 会把它升级为可插拔的 ODE/SDE sampler， -从而淘汰 `Pipeline` 这种耦合。 +`WanDMDPipeline`（SDE rollout）以避免漂移；Phase 3.2 已将其升级为可插拔的 ODE/SDE sampler： +- `WanPipeline` 通过 `pipeline_config.sampler_kind={ode|sde}` 选择 sampling loop +- distillation validation 由 method 通过 `ValidationRequest` 显式指定 sampler + timesteps +- `WanDMDPipeline` 退化为兼容 wrapper（新框架不再依赖） -### Phase 3（计划）：3.1 Config schema v2 + 3.2 ODE/SDE sampler + 3.3 Finetuning +### Phase 3：3.1 Config schema v2 + 3.2 ODE/SDE sampler + 3.3 Finetuning Phase 3 的定位：在 Phase 2.9 已经完成“优雅 dispatch + adapter/method 语义收敛”的基础上： @@ -652,6 +651,9 @@ method_config: {...} # algorithm/method 超参（方法侧） 背景：DMD2/Consistency 等方法可能需要不同的 denoising loop（不仅是 timesteps/scheduler），如果靠 “每个 method 一个 pipeline 变体（例如 `WanDMDPipeline`）”会重新回到 N×M 的组合爆炸。 +状态：**已完成**（`WanPipeline` 支持 `pipeline_config.sampler_kind={ode|sde}`，distillation validation +通过 `ValidationRequest` 显式传入 sampler + timesteps，`WanDMDPipeline` 仅作为兼容 wrapper 保留）。 + 目标： - `WanPipeline`（以及未来其它 family）通过参数选择 sampler/integrator（`ode|sde`）。 - validation 由 method/method_config 显式指定 sampler + steps/timesteps；validator 只做 dataset+logging。 diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 29132f207..6f0b06c82 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -101,24 +101,24 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop - `WanDMDPipeline` 保留为 legacy 兼容（可选），但新框架不依赖它。 ### 文件 TODO（实现清单） -- [ ] 抽象 sampler（中性命名，不出现 DMD） - - 选项 A：新增 `fastvideo/pipelines/samplers/`（推荐） - - 选项 B：在 `fastvideo/pipelines/stages/denoising.py` 内做 `OdeSampler/SdeSampler` -- [ ] `fastvideo/pipelines/stages/denoising.py` - - 把 `DmdDenoisingStage` 的语义迁移为 `SdeSampler`（或 `SdeDenoisingStage`） - - `SdeSampler` 接受显式 `timesteps`（不再读 `pipeline_config.dmd_denoising_steps`） - - 继续使用 `batch.generator` 来生成每一步注入的 `eps`（保证可复现实验） -- [ ] `fastvideo/pipelines/basic/wan/wan_pipeline.py` - - `initialize_pipeline/create_pipeline_stages` 支持选择 sampler -- [ ] `fastvideo/distillation/validators/base.py` - - 扩展 `ValidationRequest`： +- [x] 抽象 sampler（中性命名，不出现 DMD） + - `fastvideo/pipelines/samplers/`：`SamplerKind` + Wan sampler helpers + - `pipeline_config.sampler_kind={ode|sde}`：`WanPipeline` 通过该参数选择 sampling loop +- [x] `fastvideo/pipelines/stages/denoising.py` + - `SdeDenoisingStage`：SDE 风格 rollout（`pred_x0 -> add_noise(next_t, eps)`） + - `SdeDenoisingStage` 接受显式 `batch.sampling_timesteps`（来自 ValidationRequest） + - 继续使用 `batch.generator` 生成每一步注入的 `eps`（可复现） + - 保留 `DmdDenoisingStage = SdeDenoisingStage` alias（legacy pipeline 兼容） +- [x] `fastvideo/pipelines/basic/wan/wan_pipeline.py` + - `WanPipeline` 支持 `sampler_kind={ode|sde}`（单一 pipeline 覆盖两种 loop） +- [x] `fastvideo/distillation/validators/base.py` + - `ValidationRequest` 新增： - `sampler_kind: Literal["ode", "sde"] | None` - - `sampling_timesteps: list[int] | None`（用于 few-step schedule） -- [ ] `fastvideo/distillation/validators/wan.py` - - 改回使用 `WanPipeline`（不再 import `WanDMDPipeline`） - - 根据 request 选择 sampler + timesteps -- [ ] `fastvideo/distillation/methods/distribution_matching/dmd2.py` - - validation request 里指定 `sampler_kind="sde"` + `sampling_timesteps=` + - `sampling_timesteps: list[int] | None` +- [x] `fastvideo/distillation/validators/wan.py` + - 使用 `WanPipeline` + request 的 sampler/timesteps（不再 import `WanDMDPipeline`） +- [x] `fastvideo/distillation/methods/distribution_matching/dmd2.py` + - validation request 指定 `sampler_kind="sde"` + `sampling_timesteps=` --- diff --git a/fastvideo/configs/pipelines/base.py b/fastvideo/configs/pipelines/base.py index 7a6e623ed..39c77ad47 100644 --- a/fastvideo/configs/pipelines/base.py +++ b/fastvideo/configs/pipelines/base.py @@ -89,6 +89,11 @@ class PipelineConfig: # DMD parameters dmd_denoising_steps: list[int] | None = field(default=None) + # Sampler kind (controls the denoising loop semantics). + # - "ode": deterministic solver-style loop (default) + # - "sde": stochastic loop with noise injection + sampler_kind: str = "ode" + # Wan2.2 TI2V parameters ti2v_task: bool = False boundary_ratio: float | None = None @@ -218,6 +223,14 @@ def add_cli_args(parser: FlexibleArgumentParser, help= "Comma-separated list of denoising steps (e.g., '1000,757,522')", ) + parser.add_argument( + f"--{prefix_with_dot}sampler-kind", + type=str, + choices=["ode", "sde"], + dest=f"{prefix_with_dot.replace('-', '_')}sampler_kind", + default=PipelineConfig.sampler_kind, + help="Sampling loop kind: ode (default) or sde.", + ) # STA (Sliding Tile Attention) parameters parser.add_argument( diff --git a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md index 2791241eb..a1fca6415 100644 --- a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md +++ b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md @@ -53,6 +53,6 @@ - `dmd_denoising_steps` - `generator_update_interval` - `real_score_guidance_scale` -- 仍有一个过渡期现实：legacy SDE sampler（`WanDMDPipeline` / `DmdDenoisingStage`） - 目前读取 `pipeline_config.dmd_denoising_steps`，因此 config 会短期出现重复字段。 - Phase 3.2 会把 sampling timesteps 变成显式 request 参数，从而移除该重复。 +- Phase 3.2 已完成：sampling loop/timesteps 由 method 在 `ValidationRequest` 中显式指定 + （`sampler_kind` + `sampling_timesteps`），validator 使用 `WanPipeline` 执行采样； + distillation config 不再需要 `pipeline_config.dmd_denoising_steps` 这种重复字段。 diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md index 599fba2d7..421e2ccaf 100644 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md @@ -44,8 +44,9 @@ --- -## TODO（Phase 3） +## 备注（Phase 3.2 已完成） -Phase 3.2 会把 validation sampling 的 ODE/SDE loop 做成可插拔 sampler， -从而淘汰对 `WanDMDPipeline` 的依赖，并移除 `pipeline_config.dmd_denoising_steps` -这类 “algorithm knob” 在 pipeline config 里的残留。 +Phase 3.2 已将 validation sampling 的 ODE/SDE loop 做成可插拔 sampler： +- method 通过 `ValidationRequest(sampler_kind=..., sampling_timesteps=...)` 显式指定采样方式与 timesteps +- validator 使用统一的 `WanPipeline` 执行采样（不再依赖 `WanDMDPipeline`） +- 因此该 YAML 不再需要 `pipeline_config.dmd_denoising_steps` 这类重复字段 diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md index f4c3ee56c..63eb3ba88 100644 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md @@ -8,7 +8,6 @@ Wan few-step distillation（8 steps）+ DMD2。 备注： - few-step step list 的 single source of truth 在 `method_config.dmd_denoising_steps`。 -- 由于当前 validation 仍使用 legacy SDE sampling（`WanDMDPipeline`），pipeline 会读取 - `pipeline_config.dmd_denoising_steps`，因此该字段短期会与 `method_config` 重复； - Phase 3.2 会移除此重复（sampler 可插拔 + 显式 timesteps request）。 - +- Phase 3.2 已将 validation 采样升级为可插拔的 ODE/SDE sampler： + - method 通过 `ValidationRequest` 显式指定 `sampler_kind` 与 `sampling_timesteps` + - validator 使用统一的 `WanPipeline` 执行采样（不再依赖 `WanDMDPipeline`） diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md index f0ff6995f..832c69595 100644 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md @@ -9,7 +9,6 @@ Wan few-step distillation（8 steps）+ DMD2。 - `WanAdapter.prepare_batch()` 不再读取 legacy 的 `training.simulate_generator_forward`。 备注： -- 由于当前 validation 仍使用 legacy SDE sampling（`WanDMDPipeline`），pipeline 会读取 - `pipeline_config.dmd_denoising_steps`，因此该字段短期会与 `method_config` 重复； - Phase 3.2 会移除此重复（sampler 可插拔 + 显式 timesteps request）。 - +- Phase 3.2 已将 validation 采样升级为可插拔的 ODE/SDE sampler： + - method 通过 `ValidationRequest` 显式指定 `sampler_kind` 与 `sampling_timesteps` + - validator 使用统一的 `WanPipeline` 执行采样（不再依赖 `WanDMDPipeline`） diff --git a/fastvideo/distillation/doc/validators/wan.md b/fastvideo/distillation/doc/validators/wan.md index ebdb051e3..4767be116 100644 --- a/fastvideo/distillation/doc/validators/wan.md +++ b/fastvideo/distillation/doc/validators/wan.md @@ -16,9 +16,10 @@ - method 通过 `ValidationRequest` 覆盖采样配置（例如 sampling steps / guidance / output_dir）。 **依赖** -- 当前使用 `WanDMDPipeline` 做采样推理（对齐 DMD2/SDE rollout，便于与 legacy apples-to-apples 对比）。 -- 这不是最终形态：Phase 3 计划把 `WanPipeline` 的 denoising loop 抽象成可插拔的 sampler（ODE/SDE）， - 从而淘汰 `WanDMDPipeline`，让 validator 回到 method-agnostic。 +- 使用统一的 `WanPipeline` 做采样推理： + - `ValidationRequest.sampler_kind={ode|sde}` 选择 denoising loop + - `ValidationRequest.sampling_timesteps` 提供 few-step schedule（写入 `ForwardBatch.sampling_timesteps`） +- 这样 validator 不再依赖 `Pipeline`（例如 `WanDMDPipeline`），保持 method-agnostic。 **可演进方向（Phase 3+）** - 将 validation steps/guidance 等采样配置从 `TrainingArgs` 迁移到更明确的配置块（例如 `validation:`）。 diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 231a571a3..84cbfbaff 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -279,12 +279,17 @@ def log_validation(self, iteration: int) -> None: raw_steps = str(getattr(self.training_args, "validation_sampling_steps", "") or "") sampling_steps = [int(s) for s in raw_steps.split(",") if s.strip()] sampling_steps = [s for s in sampling_steps if s > 0] + + raw_rollout = self.method_config.get("dmd_denoising_steps", None) + sampling_timesteps: list[int] | None = None + if isinstance(raw_rollout, list) and raw_rollout: + sampling_timesteps = [int(s) for s in raw_rollout] + if not sampling_steps: # Default to the few-step student rollout step count for DMD2. - raw_rollout = self.method_config.get("dmd_denoising_steps", None) - if not isinstance(raw_rollout, list) or not raw_rollout: + if sampling_timesteps is None: return - sampling_steps = [int(len(raw_rollout))] + sampling_steps = [int(len(sampling_timesteps))] raw_guidance = getattr(self.training_args, "validation_guidance_scale", None) guidance_scale = float(str(raw_guidance)) if raw_guidance not in (None, "") else None @@ -292,6 +297,8 @@ def log_validation(self, iteration: int) -> None: request = ValidationRequest( sample_handle=self.student, sampling_steps=sampling_steps, + sampler_kind="sde", + sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, ) validator.log_validation(iteration, request=request) diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index d5a5cc575..db56dac31 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -73,9 +73,6 @@ training: pipeline_config: flow_shift: 8 - # NOTE: legacy SDE sampling (`WanDMDPipeline`) reads this field today. - # Phase 3.2 will make sampling timesteps an explicit request parameter. - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] method_config: # Replace legacy `training.simulate_generator_forward`. diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml index 9af959d0f..2ecbfa501 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml @@ -73,9 +73,6 @@ training: pipeline_config: flow_shift: 8 - # NOTE: legacy SDE sampling (`WanDMDPipeline`) reads this field today. - # Phase 3.2 will make sampling timesteps an explicit request parameter. - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] method_config: # Replace legacy `training.simulate_generator_forward`. diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml index ede0272fe..f3e116031 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml @@ -73,9 +73,6 @@ training: pipeline_config: flow_shift: 8 - # NOTE: legacy SDE sampling (`WanDMDPipeline`) reads this field today. - # Phase 3.2 will make sampling timesteps an explicit request parameter. - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] method_config: # Replace legacy `training.simulate_generator_forward`. diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py index 631b1f1d7..38d094a8f 100644 --- a/fastvideo/distillation/validators/base.py +++ b/fastvideo/distillation/validators/base.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from abc import ABC, abstractmethod +from typing import Literal from fastvideo.distillation.bundle import RoleHandle @@ -19,6 +20,8 @@ class ValidationRequest: sample_handle: RoleHandle | None = None sampling_steps: list[int] | None = None + sampler_kind: Literal["ode", "sde"] | None = None + sampling_timesteps: list[int] | None = None guidance_scale: float | None = None output_dir: str | None = None diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 0c3498309..51bf0ce15 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -19,8 +19,8 @@ from fastvideo.distributed import get_sp_group, get_world_group from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch -from fastvideo.pipelines.basic.wan.wan_dmd_pipeline import WanDMDPipeline from fastvideo.distillation.validators.base import ValidationRequest +from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline from fastvideo.utils import shallow_asdict logger = init_logger(__name__) @@ -56,8 +56,8 @@ def __init__( self.seed = int(seed) self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) - self._pipeline: WanDMDPipeline | None = None - self._pipeline_transformer_id: int | None = None + self._pipeline: WanPipeline | None = None + self._pipeline_key: tuple[int, str] | None = None self._sampling_param: SamplingParam | None = None def _get_sampling_param(self) -> SamplingParam: @@ -65,15 +65,21 @@ def _get_sampling_param(self) -> SamplingParam: self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) return self._sampling_param - def _get_pipeline(self, *, transformer: torch.nn.Module) -> WanDMDPipeline: - transformer_id = id(transformer) - if self._pipeline is not None and self._pipeline_transformer_id == transformer_id: + def _get_pipeline( + self, + *, + transformer: torch.nn.Module, + sampler_kind: str, + ) -> WanPipeline: + key = (id(transformer), str(sampler_kind)) + if self._pipeline is not None and self._pipeline_key == key: return self._pipeline args_copy = copy.deepcopy(self.training_args) args_copy.inference_mode = True + args_copy.pipeline_config.sampler_kind = str(sampler_kind) - self._pipeline = WanDMDPipeline.from_pretrained( + self._pipeline = WanPipeline.from_pretrained( self.training_args.model_path, args=args_copy, # inference_mode=True uses FastVideoArgs branch inference_mode=True, @@ -84,7 +90,7 @@ def _get_pipeline(self, *, transformer: torch.nn.Module) -> WanDMDPipeline: pin_cpu_memory=self.training_args.pin_cpu_memory, dit_cpu_offload=True, ) - self._pipeline_transformer_id = transformer_id + self._pipeline_key = key return self._pipeline def _parse_validation_steps(self) -> list[int]: @@ -98,6 +104,7 @@ def _prepare_validation_batch( validation_batch: dict[str, Any], num_inference_steps: int, *, + sampling_timesteps: list[int] | None = None, guidance_scale: float | None = None, ) -> ForwardBatch: sampling_param.prompt = validation_batch["prompt"] @@ -131,6 +138,11 @@ def _prepare_validation_batch( n_tokens=n_tokens, eta=0.0, VSA_sparsity=self.training_args.VSA_sparsity, + sampling_timesteps=( + torch.tensor([int(s) for s in sampling_timesteps], dtype=torch.long) + if sampling_timesteps is not None + else None + ), ) return batch @@ -139,10 +151,12 @@ def _run_validation_for_steps( num_inference_steps: int, *, transformer: torch.nn.Module, + sampler_kind: str, + sampling_timesteps: list[int] | None = None, guidance_scale: float | None = None, ) -> _ValidationStepResult: training_args = self.training_args - pipeline = self._get_pipeline(transformer=transformer) + pipeline = self._get_pipeline(transformer=transformer, sampler_kind=sampler_kind) sampling_param = self._get_sampling_param() dataset = ValidationDataset(training_args.validation_dataset_file) @@ -156,6 +170,7 @@ def _run_validation_for_steps( sampling_param, validation_batch, num_inference_steps, + sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, ) @@ -190,6 +205,18 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) validation_steps = getattr(request, "sampling_steps", None) or self._parse_validation_steps() if not validation_steps: return + sampler_kind = getattr(request, "sampler_kind", None) or "ode" + sampling_timesteps = getattr(request, "sampling_timesteps", None) + if sampling_timesteps is not None: + expected = int(len(sampling_timesteps)) + for steps in validation_steps: + if int(steps) != expected: + raise ValueError( + "validation_sampling_steps must match " + f"len(request.sampling_timesteps)={expected} when " + "sampling_timesteps is provided, got " + f"{validation_steps!r}." + ) sample_handle = getattr(request, "sample_handle", None) if sample_handle is None: @@ -212,6 +239,8 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) result = self._run_validation_for_steps( num_inference_steps, transformer=transformer, + sampler_kind=str(sampler_kind), + sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, ) diff --git a/fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py index 7cffcb0f3..2729d3988 100644 --- a/fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py @@ -1,74 +1,28 @@ # SPDX-License-Identifier: Apache-2.0 """ -Wan video diffusion pipeline implementation. +Legacy Wan DMD pipeline entrypoint. -This module contains an implementation of the Wan video diffusion pipeline -using the modular pipeline architecture. +Historically FastVideo exposed a dedicated `WanDMDPipeline` class that wired a +stochastic (SDE-style) denoising loop. Phase 3.2 makes sampling loop selection +explicit via `pipeline_config.sampler_kind`, so this file becomes a thin +compatibility wrapper around `WanPipeline`. """ from fastvideo.fastvideo_args import FastVideoArgs -from fastvideo.logger import init_logger -from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler) -from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline +from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline -# isort: off -from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, - DmdDenoisingStage, InputValidationStage, - LatentPreparationStage, - TextEncodingStage, - TimestepPreparationStage) -# isort: on -logger = init_logger(__name__) - - -class WanDMDPipeline(LoRAPipeline, ComposedPipelineBase): - """ - Wan video diffusion pipeline with LoRA support. - """ - - _required_config_modules = [ - "text_encoder", "tokenizer", "vae", "transformer", "scheduler" - ] +class WanDMDPipeline(WanPipeline): + """Compatibility wrapper for SDE sampling on Wan.""" def initialize_pipeline(self, fastvideo_args: FastVideoArgs): - - self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler( - shift=fastvideo_args.pipeline_config.flow_shift) + fastvideo_args.pipeline_config.sampler_kind = "sde" + return super().initialize_pipeline(fastvideo_args) def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: - """Set up pipeline stages with proper dependency injection.""" - - self.add_stage(stage_name="input_validation_stage", - stage=InputValidationStage()) - - self.add_stage(stage_name="prompt_encoding_stage", - stage=TextEncodingStage( - text_encoders=[self.get_module("text_encoder")], - tokenizers=[self.get_module("tokenizer")], - )) - - self.add_stage(stage_name="conditioning_stage", - stage=ConditioningStage()) - - self.add_stage(stage_name="timestep_preparation_stage", - stage=TimestepPreparationStage( - scheduler=self.get_module("scheduler"))) - - self.add_stage(stage_name="latent_preparation_stage", - stage=LatentPreparationStage( - scheduler=self.get_module("scheduler"), - transformer=self.get_module("transformer", None), - use_btchw_layout=True)) - - self.add_stage(stage_name="denoising_stage", - stage=DmdDenoisingStage( - transformer=self.get_module("transformer"), - scheduler=self.get_module("scheduler"))) - - self.add_stage(stage_name="decoding_stage", - stage=DecodingStage(vae=self.get_module("vae"))) + fastvideo_args.pipeline_config.sampler_kind = "sde" + return super().create_pipeline_stages(fastvideo_args) EntryClass = WanDMDPipeline + diff --git a/fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py index ed4d870c6..d092e7c65 100644 --- a/fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py @@ -15,7 +15,7 @@ from fastvideo.pipelines.stages import ( ImageEncodingStage, ConditioningStage, DecodingStage, DmdDenoisingStage, ImageVAEEncodingStage, InputValidationStage, LatentPreparationStage, - TextEncodingStage, TimestepPreparationStage) + TextEncodingStage) # isort: on from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( FlowMatchEulerDiscreteScheduler) @@ -55,10 +55,6 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) - self.add_stage(stage_name="timestep_preparation_stage", - stage=TimestepPreparationStage( - scheduler=self.get_module("scheduler"))) - self.add_stage(stage_name="latent_preparation_stage", stage=LatentPreparationStage( scheduler=self.get_module("scheduler"), diff --git a/fastvideo/pipelines/basic/wan/wan_pipeline.py b/fastvideo/pipelines/basic/wan/wan_pipeline.py index 64c4a0685..0f0c5e9c1 100644 --- a/fastvideo/pipelines/basic/wan/wan_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_pipeline.py @@ -8,12 +8,16 @@ from fastvideo.fastvideo_args import FastVideoArgs from fastvideo.logger import init_logger -from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( - FlowUniPCMultistepScheduler) +from fastvideo.pipelines.samplers.wan import ( + build_wan_scheduler, + get_wan_sampler_kind, + wan_use_btchw_layout, +) from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, DenoisingStage, InputValidationStage, LatentPreparationStage, + SdeDenoisingStage, TextEncodingStage, TimestepPreparationStage) @@ -30,12 +34,13 @@ class WanPipeline(LoRAPipeline, ComposedPipelineBase): ] def initialize_pipeline(self, fastvideo_args: FastVideoArgs): - # We use UniPCMScheduler from Wan2.1 official repo, not the one in diffusers. - self.modules["scheduler"] = FlowUniPCMultistepScheduler( - shift=fastvideo_args.pipeline_config.flow_shift) + sampler_kind = get_wan_sampler_kind(fastvideo_args) + self.modules["scheduler"] = build_wan_scheduler(fastvideo_args, sampler_kind) def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: """Set up pipeline stages with proper dependency injection.""" + sampler_kind = get_wan_sampler_kind(fastvideo_args) + use_btchw_layout = wan_use_btchw_layout(sampler_kind) self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) @@ -49,22 +54,31 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) - self.add_stage(stage_name="timestep_preparation_stage", - stage=TimestepPreparationStage( - scheduler=self.get_module("scheduler"))) + if sampler_kind == "ode": + self.add_stage(stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"))) self.add_stage(stage_name="latent_preparation_stage", stage=LatentPreparationStage( scheduler=self.get_module("scheduler"), - transformer=self.get_module("transformer", None))) - - self.add_stage(stage_name="denoising_stage", - stage=DenoisingStage( - transformer=self.get_module("transformer"), - transformer_2=self.get_module("transformer_2", None), - scheduler=self.get_module("scheduler"), - vae=self.get_module("vae"), - pipeline=self)) + transformer=self.get_module("transformer", None), + use_btchw_layout=use_btchw_layout)) + + if sampler_kind == "sde": + self.add_stage(stage_name="denoising_stage", + stage=SdeDenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + )) + else: + self.add_stage(stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + transformer_2=self.get_module("transformer_2", None), + scheduler=self.get_module("scheduler"), + vae=self.get_module("vae"), + pipeline=self)) self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae"), diff --git a/fastvideo/pipelines/pipeline_batch_info.py b/fastvideo/pipelines/pipeline_batch_info.py index 3ec7340ed..b9064b0cb 100644 --- a/fastvideo/pipelines/pipeline_batch_info.py +++ b/fastvideo/pipelines/pipeline_batch_info.py @@ -162,6 +162,10 @@ class ForwardBatch: # Timesteps timesteps: torch.Tensor | None = None + # Optional explicit denoising-loop timesteps (sampler-specific). + # When set, some samplers (e.g. SDE-style rollout) will iterate this list + # instead of `timesteps` produced by `TimestepPreparationStage`. + sampling_timesteps: torch.Tensor | None = None timestep: torch.Tensor | float | int | None = None step_index: int | None = None boundary_ratio: float | None = None diff --git a/fastvideo/pipelines/samplers/__init__.py b/fastvideo/pipelines/samplers/__init__.py new file mode 100644 index 000000000..3ae85ae51 --- /dev/null +++ b/fastvideo/pipelines/samplers/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 + +from fastvideo.pipelines.samplers.base import SamplerKind + +__all__ = [ + "SamplerKind", +] + diff --git a/fastvideo/pipelines/samplers/base.py b/fastvideo/pipelines/samplers/base.py new file mode 100644 index 000000000..571713295 --- /dev/null +++ b/fastvideo/pipelines/samplers/base.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Literal + +SamplerKind = Literal["ode", "sde"] + + +def normalize_sampler_kind( + raw: str | None, + *, + where: str, + default: SamplerKind = "ode", +) -> SamplerKind: + if raw is None: + return default + + kind = str(raw).strip().lower() + if kind == "ode": + return "ode" + if kind == "sde": + return "sde" + + raise ValueError(f"Unknown sampler kind at {where}: {raw!r} (expected ode|sde)") + diff --git a/fastvideo/pipelines/samplers/wan.py b/fastvideo/pipelines/samplers/wan.py new file mode 100644 index 000000000..f749147be --- /dev/null +++ b/fastvideo/pipelines/samplers/wan.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from fastvideo.fastvideo_args import FastVideoArgs +from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler, +) +from fastvideo.pipelines.samplers.base import SamplerKind, normalize_sampler_kind + + +def get_wan_sampler_kind(fastvideo_args: FastVideoArgs) -> SamplerKind: + raw = getattr(fastvideo_args.pipeline_config, "sampler_kind", None) + return normalize_sampler_kind(raw, where="pipeline_config.sampler_kind") + + +def build_wan_scheduler(fastvideo_args: FastVideoArgs, kind: SamplerKind): + shift = fastvideo_args.pipeline_config.flow_shift + if kind == "sde": + return FlowMatchEulerDiscreteScheduler(shift=shift) + return FlowUniPCMultistepScheduler(shift=shift) + + +def wan_use_btchw_layout(kind: SamplerKind) -> bool: + return kind == "sde" + diff --git a/fastvideo/pipelines/stages/__init__.py b/fastvideo/pipelines/stages/__init__.py index 52a8c9ef3..4f5efc294 100644 --- a/fastvideo/pipelines/stages/__init__.py +++ b/fastvideo/pipelines/stages/__init__.py @@ -13,7 +13,7 @@ from fastvideo.pipelines.stages.denoising import ( Cosmos25AutoDenoisingStage, Cosmos25DenoisingStage, Cosmos25V2WDenoisingStage, Cosmos25T2WDenoisingStage, CosmosDenoisingStage, - DenoisingStage, DmdDenoisingStage) + DenoisingStage, DmdDenoisingStage, SdeDenoisingStage) from fastvideo.pipelines.stages.sr_denoising import SRDenoisingStage from fastvideo.pipelines.stages.encoding import EncodingStage from fastvideo.pipelines.stages.image_encoding import ( @@ -63,6 +63,7 @@ "LTX2AudioDecodingStage", "ConditioningStage", "DenoisingStage", + "SdeDenoisingStage", "DmdDenoisingStage", "CausalDMDDenosingStage", "MatrixGameCausalDenoisingStage", diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index 0630fdc5b..5e801f0fe 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -18,8 +18,6 @@ from fastvideo.forward_context import set_forward_context from fastvideo.logger import init_logger from fastvideo.models.loader.component_loader import TransformerLoader -from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler) from fastvideo.models.utils import pred_noise_to_pred_video from fastvideo.pipelines.pipeline_batch_info import ForwardBatch from fastvideo.pipelines.stages.base import PipelineStage @@ -1330,14 +1328,16 @@ def verify_output(self, batch: ForwardBatch, return self._t2w.verify_output(batch, fastvideo_args) -class DmdDenoisingStage(DenoisingStage): - """ - Denoising stage for DMD. +class SdeDenoisingStage(DenoisingStage): + """Denoising stage for SDE-style sampling. + + This stage runs a stochastic rollout loop: + - predict x0 at timestep t + - inject fresh noise to reach the next timestep """ def __init__(self, transformer, scheduler) -> None: super().__init__(transformer, scheduler) - self.scheduler = FlowMatchEulerDiscreteScheduler(shift=8.0) def forward( self, @@ -1361,16 +1361,6 @@ def forward( autocast_enabled = (target_dtype != torch.float32 ) and not fastvideo_args.disable_autocast - # Get timesteps and calculate warmup steps - timesteps = batch.timesteps - - # TODO(will): remove this once we add input/output validation for stages - if timesteps is None: - raise ValueError("Timesteps must be provided") - num_inference_steps = batch.num_inference_steps - num_warmup_steps = len( - timesteps) - num_inference_steps * self.scheduler.order - # Prepare image latents and embeddings for I2V generation image_embeds = batch.image_embeds if len(image_embeds) > 0: @@ -1408,14 +1398,31 @@ def forward( prompt_embeds = batch.prompt_embeds assert not torch.isnan( prompt_embeds[0]).any(), "prompt_embeds contains nan" - timesteps = torch.tensor( - fastvideo_args.pipeline_config.dmd_denoising_steps, - dtype=torch.long, - device=get_local_torch_device()) + loop_timesteps = batch.sampling_timesteps + if loop_timesteps is None: + legacy = getattr(fastvideo_args.pipeline_config, "dmd_denoising_steps", None) + if legacy is not None: + loop_timesteps = torch.tensor(legacy, dtype=torch.long) + else: + loop_timesteps = batch.timesteps + + if loop_timesteps is None: + raise ValueError( + "SDE sampling requires `batch.sampling_timesteps` (preferred) " + "or `pipeline_config.dmd_denoising_steps`." + ) + if not isinstance(loop_timesteps, torch.Tensor): + loop_timesteps = torch.tensor(loop_timesteps, dtype=torch.long) + if loop_timesteps.ndim != 1: + raise ValueError( + "Expected 1D `sampling_timesteps`, got shape " + f"{tuple(loop_timesteps.shape)}" + ) + loop_timesteps = loop_timesteps.to(get_local_torch_device()) # Run denoising loop - with self.progress_bar(total=len(timesteps)) as progress_bar: - for i, t in enumerate(timesteps): + with self.progress_bar(total=len(loop_timesteps)) as progress_bar: + for i, t in enumerate(loop_timesteps): # Skip if interrupted if hasattr(self, 'interrupt') and self.interrupt: continue @@ -1499,13 +1506,14 @@ def forward( scheduler=self.scheduler).unflatten( 0, pred_noise.shape[:2]) - if i < len(timesteps) - 1: - next_timestep = timesteps[i + 1] * torch.ones( + if i < len(loop_timesteps) - 1: + next_timestep = loop_timesteps[i + 1] * torch.ones( [1], dtype=torch.long, device=pred_video.device) noise = torch.randn(video_raw_latent_shape, dtype=pred_video.dtype, - generator=batch.generator[0]).to( - self.device) + generator=batch.generator[0] + if isinstance(batch.generator, list) + else batch.generator).to(self.device) latents = self.scheduler.add_noise( pred_video.flatten(0, 1), noise.flatten(0, 1), next_timestep).unflatten(0, pred_video.shape[:2]) @@ -1513,11 +1521,7 @@ def forward( latents = pred_video # Update progress bar - if i == len(timesteps) - 1 or ( - (i + 1) > num_warmup_steps and - (i + 1) % self.scheduler.order == 0 - and progress_bar is not None): - progress_bar.update() + progress_bar.update() # Gather results if using sequence parallelism latents = latents.permute(0, 2, 1, 3, 4) @@ -1525,3 +1529,7 @@ def forward( batch.latents = latents return batch + + +# Backwards-compatible alias (legacy pipelines still import this symbol). +DmdDenoisingStage = SdeDenoisingStage From dd80a97bb7b0b9d44d6e42f55294f8dd6cbb0c19 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 22:17:27 +0000 Subject: [PATCH 088/214] fix validator not using sampler. --- dev/phases/phase_3.md | 4 + examples/distillation/phase3_2/temp.sh | 43 ++++++++++ ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml | 85 +++++++++++++++++++ fastvideo/distillation/validators/wan.py | 10 +-- 4 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 examples/distillation/phase3_2/temp.sh create mode 100644 fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 6f0b06c82..2549dacf8 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -120,6 +120,10 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop - [x] `fastvideo/distillation/methods/distribution_matching/dmd2.py` - validation request 指定 `sampler_kind="sde"` + `sampling_timesteps=` +### 可运行产物 +- Phase 3.2 YAML：`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml` +- One-shot 脚本：`examples/distillation/phase3_2/temp.sh` + --- ## Phase 3.3：Finetuning method（only student） diff --git a/examples/distillation/phase3_2/temp.sh b/examples/distillation/phase3_2/temp.sh new file mode 100644 index 000000000..69aa3d6d2 --- /dev/null +++ b/examples/distillation/phase3_2/temp.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -euo pipefail +if [[ "${DEBUG:-0}" == "1" ]]; then + set -x +fi + +# One-shot launch script for Phase 3.2 (ODE/SDE sampler selection in `WanPipeline`). +# +# Compared to phase3_1, validation no longer needs `WanDMDPipeline`; the method +# supplies `sampler_kind + sampling_timesteps` via `ValidationRequest`, and +# `WanValidator` runs `WanPipeline` with the requested loop. + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29512} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" + diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml new file mode 100644 index 000000000..3bf0ab689 --- /dev/null +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml @@ -0,0 +1,85 @@ +recipe: + family: wan + method: dmd2 + +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + disable_custom_init_weights: true + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + disable_custom_init_weights: true + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.2_wan2.1_dmd2_8steps_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation (method-agnostic knobs) + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: phase3.2_wan_dmd2_8steps_wansyn + wandb_run_name: phase3.2_wan_dmd2_8steps_data_latent + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "8" + validation_guidance_scale: "6.0" + +pipeline_config: + flow_shift: 8 + sampler_kind: ode + +method_config: + # Replace legacy `training.simulate_generator_forward`. + rollout_mode: data_latent + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + # Few-step schedule (single source of truth for the method). + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 51bf0ce15..c6a026b8f 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -2,7 +2,6 @@ from __future__ import annotations -import copy import os from dataclasses import dataclass from typing import Any @@ -75,14 +74,15 @@ def _get_pipeline( if self._pipeline is not None and self._pipeline_key == key: return self._pipeline - args_copy = copy.deepcopy(self.training_args) - args_copy.inference_mode = True - args_copy.pipeline_config.sampler_kind = str(sampler_kind) + # NOTE: `ComposedPipelineBase.from_pretrained()` ignores `args` when + # `inference_mode=True`, so we must pass pipeline knobs via kwargs. + flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) self._pipeline = WanPipeline.from_pretrained( self.training_args.model_path, - args=args_copy, # inference_mode=True uses FastVideoArgs branch inference_mode=True, + sampler_kind=str(sampler_kind), + flow_shift=float(flow_shift) if flow_shift is not None else None, loaded_modules={"transformer": transformer}, tp_size=self.training_args.tp_size, sp_size=self.training_args.sp_size, From 167ddb9294da3ec54153aaf1d3f0c3201863ca94 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 22:36:10 +0000 Subject: [PATCH 089/214] fix timestep --- fastvideo/distillation/validators/wan.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index c6a026b8f..5dcedcb6b 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -131,6 +131,11 @@ def _prepare_validation_batch( num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_factor + 1 sampling_param.num_frames = int(num_frames) + sampling_timesteps_tensor = ( + torch.tensor([int(s) for s in sampling_timesteps], dtype=torch.long) + if sampling_timesteps is not None + else None + ) batch = ForwardBatch( **shallow_asdict(sampling_param), latents=None, @@ -138,11 +143,10 @@ def _prepare_validation_batch( n_tokens=n_tokens, eta=0.0, VSA_sparsity=self.training_args.VSA_sparsity, - sampling_timesteps=( - torch.tensor([int(s) for s in sampling_timesteps], dtype=torch.long) - if sampling_timesteps is not None - else None - ), + # SDE-style sampling iterates `sampling_timesteps`. Some stages still + # expect `timesteps` to be set, so we mirror the same tensor there. + timesteps=sampling_timesteps_tensor, + sampling_timesteps=sampling_timesteps_tensor, ) return batch From 72a91e847d08b1be4688222e1d750963006698a0 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 23:21:47 +0000 Subject: [PATCH 090/214] deisigning phase 3.3 finetuning --- dev/design.md | 5 +++-- dev/phases/phase_3.md | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/dev/design.md b/dev/design.md index 5f691ea2e..3e46e53a7 100644 --- a/dev/design.md +++ b/dev/design.md @@ -670,8 +670,9 @@ Phase 3.1 的 `recipe/method_config` 对齐。 - `bundle.require_roles(["student"])` - update policy 永远只 step student 的 optimizer/scheduler - 为 finetune 定义 adapter contract（类似 `_DMD2Adapter` 的做法）： - - `_FineTuneAdapter(Protocol)`：`prepare_batch()` + `predict_*()` + `compute_loss()` + `backward()` - - Wan 侧由 `WanAdapter` 实现该 contract（或拆出 mixin，避免 adapter 膨胀） + - `_FineTuneAdapter(Protocol)`：`prepare_batch()` + `predict_*()` + `backward()` + - **不建议把 `compute_loss()` 放进 adapter**：loss/policy 属于 method（参考 FastGen 的 `SFTModel.single_train_step()`） + - Wan 侧由 `WanAdapter` 实现该 contract；若未来出现 family-specific 分支，优先在 adapter 内部消化而不是膨胀 method Finetune 的 config（示意，schema v2）： ```yaml diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 2549dacf8..d23e917fe 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -138,10 +138,12 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop - `FineTuneMethod(DistillMethod)` + `@register_method("finetune")` - `bundle.require_roles(["student"])` - `single_train_step()` 只更新 student -- [ ] `fastvideo/distillation/adapters/wan.py` - - 增加 finetune 所需 primitives（operation-centric）： - - `predict_noise(handle, ...)` / `predict_x0(handle, ...)`（可复用已有） - - （按 legacy loss 语义）增加最小 `finetune_loss(...)` 或 `compute_training_loss(...)` +- [ ] （如有必要）为 finetune 定义 adapter contract（类似 `_DMD2Adapter` 的做法） + - 重点：**loss 仍由 method 计算**；adapter 只提供 operation-centric primitives + - `_FineTuneAdapter(Protocol)` 推荐只包含： + - `prepare_batch(...)`（产出 latents/noise/timesteps/sigmas/conditioning） + - `predict_noise(handle, ...)`（以及可选 `predict_x0`） + - `backward(loss, ctx, ...)`（forward-context/activation ckpt 相关） - [ ] configs/examples - [ ] `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_*.yaml` - [ ] `examples/distillation/phase3/`（或更新现有 examples） From 97534d81c05d0c5a611f3f81fb870cc49f43a2a6 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 23:47:24 +0000 Subject: [PATCH 091/214] phase 3.3 init impl --- dev/design.md | 2 + dev/phases/phase_3.md | 8 +- examples/distillation/phase3_3/temp.sh | 43 +++ fastvideo/distillation/doc/README.md | 3 +- .../distillation/doc/methods/__init__.md | 1 + .../doc/methods/fine_tuning/__init__.md | 22 +- .../doc/methods/fine_tuning/finetune.md | 41 +++ .../finetune_wan2.1_t2v_1.3B_phase3.3.md | 9 + fastvideo/distillation/methods/__init__.py | 6 + .../methods/fine_tuning/__init__.py | 19 +- .../methods/fine_tuning/finetune.py | 330 ++++++++++++++++++ .../finetune_wan2.1_t2v_1.3B_phase3.3.yaml | 65 ++++ fastvideo/distillation/registry.py | 1 + 13 files changed, 533 insertions(+), 17 deletions(-) create mode 100644 examples/distillation/phase3_3/temp.sh create mode 100644 fastvideo/distillation/doc/methods/fine_tuning/finetune.md create mode 100644 fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md create mode 100644 fastvideo/distillation/methods/fine_tuning/finetune.py create mode 100644 fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml diff --git a/dev/design.md b/dev/design.md index 3e46e53a7..612b1fc2a 100644 --- a/dev/design.md +++ b/dev/design.md @@ -665,6 +665,8 @@ method_config: {...} # algorithm/method 超参（方法侧） `ModelBundle + Adapter + Method + Trainer + (Validator/Checkpoint)` 基础设施，并让其配置语义与 Phase 3.1 的 `recipe/method_config` 对齐。 +状态：**已完成**（`FineTuneMethod` + Phase 3.3 示例 YAML + one-shot 脚本）。 + 建议落地形态（Phase 3 落地到代码时）： - 新增 method：`fastvideo/distillation/methods/fine_tuning/finetune.py::FineTuneMethod` - `bundle.require_roles(["student"])` diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index d23e917fe..bd367bab3 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -134,7 +134,7 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop - finetune 的 method 参数进入 `method_config`（与 Phase 3.1 schema 一致）。 ### 文件 TODO（实现清单） -- [ ] `fastvideo/distillation/methods/fine_tuning/finetune.py` +- [x] `fastvideo/distillation/methods/fine_tuning/finetune.py` - `FineTuneMethod(DistillMethod)` + `@register_method("finetune")` - `bundle.require_roles(["student"])` - `single_train_step()` 只更新 student @@ -144,9 +144,9 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop - `prepare_batch(...)`（产出 latents/noise/timesteps/sigmas/conditioning） - `predict_noise(handle, ...)`（以及可选 `predict_x0`） - `backward(loss, ctx, ...)`（forward-context/activation ckpt 相关） -- [ ] configs/examples - - [ ] `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_*.yaml` - - [ ] `examples/distillation/phase3/`（或更新现有 examples） +- [x] configs/examples + - [x] `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` + - [x] `examples/distillation/phase3_3/temp.sh` --- diff --git a/examples/distillation/phase3_3/temp.sh b/examples/distillation/phase3_3/temp.sh new file mode 100644 index 000000000..037e8cd94 --- /dev/null +++ b/examples/distillation/phase3_3/temp.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -euo pipefail +if [[ "${DEBUG:-0}" == "1" ]]; then + set -x +fi + +# One-shot launch script for Phase 3.3 (finetune method on the distillation scaffold). +# +# - Same trainer/bundle/adapter/family infrastructure as distillation. +# - Only `models.student` is required; the method updates student weights only. +# - Validation is still supported via `ValidationRequest` + `WanValidator`. + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29513} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" + diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index f746874c4..d0b7a938d 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -46,6 +46,7 @@ - `methods/consistency_model/__init__.md` - `methods/knowledge_distillation/__init__.md` - `methods/fine_tuning/__init__.md` +- `methods/fine_tuning/finetune.md` ### validators/ - `validators/__init__.md` @@ -55,4 +56,4 @@ ### outside/ - `outside/README.md` - `outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md` - +- `outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md` diff --git a/fastvideo/distillation/doc/methods/__init__.md b/fastvideo/distillation/doc/methods/__init__.md index f62cb6be8..10126797f 100644 --- a/fastvideo/distillation/doc/methods/__init__.md +++ b/fastvideo/distillation/doc/methods/__init__.md @@ -6,6 +6,7 @@ **当前导出** - `DistillMethod`：算法基类（抽象） - `DMD2Method`：distribution matching 目录下的一个具体方法实现 +- `FineTuneMethod`：fine tuning 目录下的一个具体方法实现（only student） **设计意图** - method 层应当是 **模型无关** 的（不 import 具体 pipeline/模型实现）； diff --git a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md index 3d0362cd3..aaa6fb646 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md @@ -1,14 +1,14 @@ -# `fastvideo/distillation/methods/fine_tuning/__init__.py` +# `fastvideo/distillation/methods/fine_tuning/` -**状态** -- 当前是占位目录（`__all__ = []`），用于未来加入 finetuning（可视为只有 student 的特殊训练 recipe）。 +**目的** +- 以 “method” 的形式实现 finetuning / SFT：把它看作一种特殊的 distillation recipe（只有 `student` + dataset）。 -**与 distillation 框架的关系** -- finetune 可以复用同一套： - - YAML config（models 里只提供 student） - - family（加载 student 模块与数据） - - trainer（infra） -- method 则实现： - - loss（SFT/finetune objective） - - optimizer/scheduler policy +**当前实现** +- `finetune.py`：`FineTuneMethod` + - 只要求 `models.student` + - loss/policy 在 method 层 + - 复用同一套 trainer/bundle/adapter/family/validator/checkpoint 基础设施 +**设计要点** +- adapter 仍保持 operation-centric（`prepare_batch / predict_* / backward`），不内置 finetune 的 loss 语义。 +- family 负责 build-time：加载 student modules、shared components（VAE/scheduler）、dataloader、validator。 diff --git a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md new file mode 100644 index 000000000..1b6ebe61c --- /dev/null +++ b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md @@ -0,0 +1,41 @@ +# `fastvideo/distillation/methods/fine_tuning/finetune.py` + +## 目的 +- 将 “finetuning / SFT” 以 `DistillMethod` 的方式接入 Phase 2+ 架构： + - 复用 `DistillTrainer`（infra loop / accum / step / ckpt / validate 调用） + - 复用 `ModelBundle`（角色容器，finetune 只需要 `student`） + - 复用 family/adapter（加载与 primitives） + +finetune 可以被视为一种特殊的 distillation recipe：**只有 student + dataset**。 + +## 角色依赖 +- 必需：`student` +- 不需要：`teacher / critic / reward / ...` + +方法会强制： +- `models.student.trainable=true` + +## 核心训练逻辑 +`FineTuneMethod.single_train_step()`： +1. `adapter.prepare_batch(..., latents_source="data")` +2. 用 student 做 `adapter.predict_noise(student, noisy_latents, timesteps, batch, conditional=True)` +3. 计算 loss（与 legacy `training_pipeline.py` 对齐）： + - 默认（`training.precondition_outputs=false`）： + - target = `noise - x0` + - loss = `mse(pred, target)` + - 若 `training.precondition_outputs=true`： + - 先 precondition 到 `x0`：`pred_x0 = x_t - sigma * pred` + - loss = `mse(pred_x0, x0)` +4. backward 通过 `adapter.backward(loss, ctx, ...)` 执行（确保 forward-context/activation ckpt 兼容） + +## Optimizer / Scheduler +- 由 method 创建（而非 family）： + - 使用 `training.learning_rate / training.betas / training.lr_scheduler / ...` + - 只为 `student` role 创建 `optimizer + lr_scheduler` + +## Validation +- `FineTuneMethod.log_validation()` 构造 `ValidationRequest(sample_handle=student, ...)` +- 具体 pipeline 与采样 loop 由 validator + `pipeline_config.sampler_kind` 决定（默认 `ode`） + +## 配置示例 +- `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md new file mode 100644 index 000000000..569136c33 --- /dev/null +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md @@ -0,0 +1,9 @@ +# `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` + +这是一个 **Phase 3.3** 的可运行示例：把 finetuning 作为一种 method 接入 Phase 2+ distillation scaffold。 + +关键点： +- `recipe.method = finetune` +- `models` 里只提供 `student`（no teacher/critic） +- 训练 loss 由 `FineTuneMethod` 实现（与 legacy `training_pipeline.py` 的目标对齐） +- validation 通过 `ValidationRequest + WanValidator + WanPipeline` 执行（默认走 `ode` sampler） diff --git a/fastvideo/distillation/methods/__init__.py b/fastvideo/distillation/methods/__init__.py index 00ab6b5fc..f1d11ff6c 100644 --- a/fastvideo/distillation/methods/__init__.py +++ b/fastvideo/distillation/methods/__init__.py @@ -8,10 +8,12 @@ if TYPE_CHECKING: from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method + from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod __all__ = [ "DistillMethod", "DMD2Method", + "FineTuneMethod", ] @@ -21,4 +23,8 @@ def __getattr__(name: str) -> object: from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method return DMD2Method + if name == "FineTuneMethod": + from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod + + return FineTuneMethod raise AttributeError(name) diff --git a/fastvideo/distillation/methods/fine_tuning/__init__.py b/fastvideo/distillation/methods/fine_tuning/__init__.py index 57c85a008..730415100 100644 --- a/fastvideo/distillation/methods/fine_tuning/__init__.py +++ b/fastvideo/distillation/methods/fine_tuning/__init__.py @@ -1,4 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 -__all__: list[str] = [] +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod + +__all__ = [ + "FineTuneMethod", +] + + +def __getattr__(name: str) -> object: + # Lazy import to avoid circular imports during registry bring-up. + if name == "FineTuneMethod": + from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod + + return FineTuneMethod + raise AttributeError(name) diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py new file mode 100644 index 000000000..ea5f6e3cd --- /dev/null +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -0,0 +1,330 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any, Literal, Protocol + +import torch +import torch.nn.functional as F + +from fastvideo.training.training_utils import ( + clip_grad_norm_while_handling_failing_dtensor_cases, + get_scheduler, +) + +from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.methods.base import DistillMethod +from fastvideo.distillation.registry import register_method +from fastvideo.distillation.validators.base import ValidationRequest +from fastvideo.distillation.yaml_config import DistillRunConfig + + +class _FineTuneAdapter(Protocol): + """Adapter contract for :class:`FineTuneMethod`. + + Finetuning is implemented as a method (algorithm layer) on top of the + family-provided adapter. The method must remain model-family agnostic, so + it consumes only operation-centric primitives exposed by the adapter. + """ + + training_args: Any + + def on_train_start(self) -> None: + ... + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", + ) -> Any: + ... + + def predict_noise( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: Any, + *, + conditional: bool, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + ... + + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + ... + + +class FineTuneMethod(DistillMethod): + """Supervised finetuning as a method: only `student` participates. + + The loss follows the same objective used by the legacy training pipeline: + - default: flow-matching target `noise - x0` + - optional (if `training.precondition_outputs=true`): precondition to `x0` + and regress `x0` directly. + """ + + def __init__( + self, + *, + bundle: ModelBundle, + adapter: _FineTuneAdapter, + method_config: dict[str, Any] | None = None, + validator: Any | None = None, + ) -> None: + super().__init__(bundle) + bundle.require_roles(["student"]) + self.student = bundle.role("student") + if not getattr(self.student, "trainable", False): + raise ValueError("FineTuneMethod requires models.student.trainable=true") + + self.adapter = adapter + self.validator = validator + self.training_args = adapter.training_args + self.method_config: dict[str, Any] = dict(method_config or {}) + + self._init_optimizers_and_schedulers() + + def _parse_betas(self, raw: Any, *, where: str) -> tuple[float, float]: + if raw is None: + raise ValueError(f"Missing betas for {where}") + if isinstance(raw, (tuple, list)) and len(raw) == 2: + return float(raw[0]), float(raw[1]) + if isinstance(raw, str): + parts = [p.strip() for p in raw.split(",") if p.strip()] + if len(parts) != 2: + raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {raw!r}") + return float(parts[0]), float(parts[1]) + raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {type(raw).__name__}") + + def _build_role_optimizer_and_scheduler( + self, + *, + role: str, + handle: RoleHandle, + learning_rate: float, + betas: tuple[float, float], + scheduler_name: str, + ) -> None: + modules = handle.modules + params: list[torch.nn.Parameter] = [] + for module in modules.values(): + params.extend([p for p in module.parameters() if p.requires_grad]) + if not params: + raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") + + optimizer = torch.optim.AdamW( + params, + lr=float(learning_rate), + betas=betas, + weight_decay=float(getattr(self.training_args, "weight_decay", 0.0) or 0.0), + eps=1e-8, + ) + + scheduler = get_scheduler( + str(scheduler_name), + optimizer=optimizer, + num_warmup_steps=int(getattr(self.training_args, "lr_warmup_steps", 0) or 0), + num_training_steps=int(getattr(self.training_args, "max_train_steps", 0) or 0), + num_cycles=int(getattr(self.training_args, "lr_num_cycles", 0) or 0), + power=float(getattr(self.training_args, "lr_power", 0.0) or 0.0), + min_lr_ratio=float(getattr(self.training_args, "min_lr_ratio", 0.5) or 0.5), + last_epoch=-1, + ) + + handle.optimizers = {"main": optimizer} + handle.lr_schedulers = {"main": scheduler} + + def _init_optimizers_and_schedulers(self) -> None: + training_args = self.training_args + + student_lr = float(getattr(training_args, "learning_rate", 0.0) or 0.0) + if student_lr <= 0.0: + raise ValueError("training.learning_rate must be > 0 for finetune") + + student_betas = self._parse_betas( + getattr(training_args, "betas", None), + where="training.betas", + ) + student_sched = str(getattr(training_args, "lr_scheduler", "constant")) + self._build_role_optimizer_and_scheduler( + role="student", + handle=self.student, + learning_rate=student_lr, + betas=student_betas, + scheduler_name=student_sched, + ) + + def on_train_start(self) -> None: + self.adapter.on_train_start() + + def log_validation(self, iteration: int) -> None: + validator = getattr(self, "validator", None) + if validator is None: + return + if not getattr(self.training_args, "log_validation", False): + return + + raw_steps = str(getattr(self.training_args, "validation_sampling_steps", "") or "") + sampling_steps = [int(s) for s in raw_steps.split(",") if s.strip()] + sampling_steps = [s for s in sampling_steps if s > 0] + if not sampling_steps: + return + + raw_guidance = getattr(self.training_args, "validation_guidance_scale", None) + guidance_scale = float(str(raw_guidance)) if raw_guidance not in (None, "") else None + + pipeline_sampler_kind = getattr( + getattr(self.training_args, "pipeline_config", None), "sampler_kind", None + ) + sampler_kind = str(pipeline_sampler_kind) if pipeline_sampler_kind else "ode" + + request = ValidationRequest( + sample_handle=self.student, + sampling_steps=sampling_steps, + sampler_kind=sampler_kind, + sampling_timesteps=None, + guidance_scale=guidance_scale, + ) + validator.log_validation(iteration, request=request) + + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + generators: dict[str, torch.Generator] = {} + + adapter = getattr(self, "adapter", None) + get_adapter_generators = getattr(adapter, "get_rng_generators", None) + if callable(get_adapter_generators): + generators.update(get_adapter_generators()) + + validator = getattr(self, "validator", None) + validation_gen = getattr(validator, "validation_random_generator", None) + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen + + return generators + + def _clip_grad_norm(self, module: torch.nn.Module) -> float: + max_grad_norm_raw = getattr(self.training_args, "max_grad_norm", None) + if max_grad_norm_raw is None: + return 0.0 + try: + max_grad_norm = float(max_grad_norm_raw) + except (TypeError, ValueError) as e: + raise ValueError( + "training.max_grad_norm must be a number when set, got " + f"{max_grad_norm_raw!r}" + ) from e + if max_grad_norm <= 0.0: + return 0.0 + grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in module.parameters()], + max_grad_norm, + foreach=None, + ) + return float(grad_norm.item()) if grad_norm is not None else 0.0 + + def single_train_step( + self, + batch: dict[str, Any], + iteration: int, + *, + current_vsa_sparsity: float = 0.0, + ) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + del iteration + training_batch = self.adapter.prepare_batch( + batch, + current_vsa_sparsity=current_vsa_sparsity, + latents_source="data", + ) + + if training_batch.latents is None: + raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.latents") + if training_batch.noisy_model_input is None: + raise RuntimeError( + "adapter.prepare_batch() must set TrainingBatch.noisy_model_input" + ) + if training_batch.noise is None: + raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.noise") + if training_batch.sigmas is None: + raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.sigmas") + if training_batch.timesteps is None: + raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.timesteps") + + clean_latents = training_batch.latents + noisy_latents = training_batch.noisy_model_input.permute(0, 2, 1, 3, 4) + noise = training_batch.noise.permute(0, 2, 1, 3, 4) + sigmas = training_batch.sigmas + timesteps = training_batch.timesteps + + pred = self.adapter.predict_noise( + self.student, + noisy_latents, + timesteps, + training_batch, + conditional=True, + attn_kind="dense", + ) + + if bool(getattr(self.training_args, "precondition_outputs", False)): + pred_x0 = noisy_latents - pred * sigmas + loss = F.mse_loss(pred_x0.float(), clean_latents.float()) + else: + target = noise - clean_latents + loss = F.mse_loss(pred.float(), target.float()) + + loss_map = {"total_loss": loss, "finetune_loss": loss} + outputs: dict[str, Any] = { + "_fv_backward": (training_batch.timesteps, training_batch.attn_metadata) + } + return loss_map, outputs + + def backward( + self, + loss_map: dict[str, torch.Tensor], + outputs: dict[str, Any], + *, + grad_accum_rounds: int = 1, + ) -> None: + grad_accum_rounds = max(1, int(grad_accum_rounds)) + ctx = outputs.get("_fv_backward") + if ctx is None: + super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + return + self.adapter.backward( + loss_map["total_loss"], + ctx, + grad_accum_rounds=grad_accum_rounds, + ) + + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + del iteration + return list(self.student.optimizers.values()) + + def get_lr_schedulers(self, iteration: int) -> list[Any]: + del iteration + return list(self.student.lr_schedulers.values()) + + def optimizers_schedulers_step(self, iteration: int) -> None: + for module in self.student.modules.values(): + self._clip_grad_norm(module) + super().optimizers_schedulers_step(iteration) + + +@register_method("finetune") +def build_finetune_method( + *, + cfg: DistillRunConfig, + bundle: ModelBundle, + adapter: _FineTuneAdapter, + validator: Any | None, +) -> DistillMethod: + return FineTuneMethod( + bundle=bundle, + adapter=adapter, + method_config=cfg.method_config, + validator=validator, + ) diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml new file mode 100644 index 000000000..1f02bb0b8 --- /dev/null +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml @@ -0,0 +1,65 @@ +recipe: + family: wan + method: finetune + +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.3_wan2.1_finetune_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Method-agnostic knobs + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: phase3.3_wan_finetune_wansyn + wandb_run_name: phase3.3_wan_finetune + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "8" + validation_guidance_scale: "6.0" + +pipeline_config: + flow_shift: 8 + sampler_kind: ode + +method_config: {} + diff --git a/fastvideo/distillation/registry.py b/fastvideo/distillation/registry.py index 2e4cd9091..646d51ef5 100644 --- a/fastvideo/distillation/registry.py +++ b/fastvideo/distillation/registry.py @@ -70,6 +70,7 @@ def ensure_builtin_registrations() -> None: # order is stable and failures are debuggable. from fastvideo.distillation.families import wan as _wan # noqa: F401 from fastvideo.distillation.methods.distribution_matching import dmd2 as _dmd2 # noqa: F401 + from fastvideo.distillation.methods.fine_tuning import finetune as _finetune # noqa: F401 _BUILTINS_REGISTERED = True From 2a68748e7306e13092da7a1450485ceeb90f6cf1 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 24 Feb 2026 23:52:47 +0000 Subject: [PATCH 092/214] upload wandb file --- fastvideo/distillation/doc/yaml_config.md | 2 + fastvideo/training/distillation.py | 5 +++ fastvideo/training/trackers.py | 54 ++++++++++++++++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/fastvideo/distillation/doc/yaml_config.md b/fastvideo/distillation/doc/yaml_config.md index 83d1f97f2..679bcb9e6 100644 --- a/fastvideo/distillation/doc/yaml_config.md +++ b/fastvideo/distillation/doc/yaml_config.md @@ -10,6 +10,8 @@ - `training_args: TrainingArgs`（来自 YAML 的 `training:`，并注入 entrypoint invariants） - `method_config: dict`（来自 YAML 的 `method_config:`，传给 method 解释） - `raw: dict`（原始 YAML，便于 tracker 记录） + - `wan` family 默认会把 `raw` 作为 W&B config 传给 `wandb.init(config=...)` + - 入口还会把 YAML 文件本身以 `run.yaml` 的形式上传到 tracker（如 W&B Files） **YAML 结构（schema v2）** - `recipe: {family: ..., method: ...}` diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 254e50fa9..c3f18e197 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import os import sys from typing import Any @@ -47,6 +48,10 @@ def run_distillation_from_config( logger.info("Dry-run: config parsed and runtime built successfully.") return + # Attach the exact YAML used for this run to the tracker (e.g., W&B Files). + # This helps reproducibility and makes runs easy to inspect later. + runtime.tracker.log_file(os.path.abspath(os.path.expanduser(config_path)), name="run.yaml") + ckpt_config = DistillCheckpointConfig( save_steps=int(getattr(training_args, "training_state_checkpointing_steps", 0) or 0), keep_last=int(getattr(training_args, "checkpoints_total_limit", 0) or 0), diff --git a/fastvideo/training/trackers.py b/fastvideo/training/trackers.py index 281d79325..93f48ddd2 100644 --- a/fastvideo/training/trackers.py +++ b/fastvideo/training/trackers.py @@ -11,6 +11,7 @@ import copy import os import pathlib +import shutil import time from dataclasses import dataclass from enum import Enum @@ -92,6 +93,21 @@ def log_artifacts(self, artifacts: dict[str, Any], step: int) -> None: def finish(self) -> None: # pragma: no cover - interface """Finalize the tracker session.""" + def log_file( + self, + path: str, + *, + name: str | None = None, + ) -> None: + """Log a local file to the tracker run (best-effort). + + Useful for attaching the exact YAML config used for a run. + + Trackers that do not support files should treat this as a no-op. + """ + + del path, name + def video( self, data: Any, @@ -134,12 +150,13 @@ def __init__( import wandb - pathlib.Path(log_dir).mkdir(parents=True, exist_ok=True) + self._log_dir = os.path.abspath(str(log_dir)) + pathlib.Path(self._log_dir).mkdir(parents=True, exist_ok=True) self._wandb = wandb self._run = wandb.init( project=experiment_name, - dir=log_dir, + dir=self._log_dir, config=config, name=run_name, ) @@ -154,6 +171,35 @@ def log(self, metrics: dict[str, Any], step: int) -> None: def finish(self) -> None: self._run.finish() + def log_file(self, path: str, *, name: str | None = None) -> None: + resolved = os.path.abspath(os.path.expanduser(str(path))) + if not os.path.isfile(resolved): + logger.warning("W&B log_file skipped; file not found: %s", resolved) + return + + save_path = resolved + if name is not None and str(name).strip(): + dest_path = os.path.join(self._log_dir, str(name).strip()) + try: + shutil.copyfile(resolved, dest_path) + except Exception: + logger.exception( + "Failed to copy file for W&B upload: %s -> %s", + resolved, + dest_path, + ) + else: + save_path = dest_path + + try: + self._run.save( + save_path, + base_path=os.path.dirname(save_path), + policy="now", + ) + except Exception: + logger.exception("Failed to upload file to W&B: %s", save_path) + def video( self, data: Any, @@ -201,6 +247,10 @@ def log_artifacts(self, artifacts: dict[str, Any], step: int) -> None: tracker.log_artifacts(artifacts, step) self._timed_metrics = {} + def log_file(self, path: str, *, name: str | None = None) -> None: + for tracker in self._trackers: + tracker.log_file(path, name=name) + def finish(self) -> None: for tracker in self._trackers: tracker.finish() From 4d2acfc1acd70b70b1215000e5fecc79c4ab8a7d Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 00:46:15 +0000 Subject: [PATCH 093/214] vsa finetune --- examples/distillation/phase3_3/temp-vsa.sh | 44 +++++++++++ .../methods/fine_tuning/finetune.py | 25 +++++- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml | 4 +- .../finetune_wan2.1_t2v_1.3B_phase3.3.yaml | 2 +- ...finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml | 77 +++++++++++++++++++ 5 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 examples/distillation/phase3_3/temp-vsa.sh create mode 100644 fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml diff --git a/examples/distillation/phase3_3/temp-vsa.sh b/examples/distillation/phase3_3/temp-vsa.sh new file mode 100644 index 000000000..0055fca08 --- /dev/null +++ b/examples/distillation/phase3_3/temp-vsa.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -euo pipefail +if [[ "${DEBUG:-0}" == "1" ]]; then + set -x +fi + +# One-shot launch script for Phase 3.3 finetune (SFT) with VSA backend. +# +# This mirrors legacy finetune scripts: +# - FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN +# - VSA_* knobs live in the YAML (VSA_sparsity / VSA_decay_rate / VSA_decay_interval_steps) + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-VIDEO_SPARSE_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29514} +export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-"/tmp/triton_cache_${USER}_$$"} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" + diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index ea5f6e3cd..1ec224e0f 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any, Literal, Protocol +from typing import Any, Literal, Protocol, cast import torch import torch.nn.functional as F @@ -84,9 +84,23 @@ def __init__( self.validator = validator self.training_args = adapter.training_args self.method_config: dict[str, Any] = dict(method_config or {}) + self._attn_kind: Literal["dense", "vsa"] = self._parse_attn_kind( + self.method_config.get("attn_kind", None) + ) self._init_optimizers_and_schedulers() + def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: + if raw in (None, ""): + return "dense" + kind = str(raw).strip().lower() + if kind not in {"dense", "vsa"}: + raise ValueError( + "method_config.attn_kind must be one of {'dense', 'vsa'}, " + f"got {raw!r}." + ) + return cast(Literal["dense", "vsa"], kind) + def _parse_betas(self, raw: Any, *, where: str) -> tuple[float, float]: if raw is None: raise ValueError(f"Missing betas for {where}") @@ -266,7 +280,7 @@ def single_train_step( timesteps, training_batch, conditional=True, - attn_kind="dense", + attn_kind=self._attn_kind, ) if bool(getattr(self.training_args, "precondition_outputs", False)): @@ -276,9 +290,14 @@ def single_train_step( target = noise - clean_latents loss = F.mse_loss(pred.float(), target.float()) + if self._attn_kind == "vsa": + attn_metadata = training_batch.attn_metadata_vsa + else: + attn_metadata = training_batch.attn_metadata + loss_map = {"total_loss": loss, "finetune_loss": loss} outputs: dict[str, Any] = { - "_fv_backward": (training_batch.timesteps, training_batch.attn_metadata) + "_fv_backward": (training_batch.timesteps, attn_metadata) } return loss_map, outputs diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml index 3bf0ab689..f89f96ab3 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml @@ -64,7 +64,7 @@ training: # Tracking / validation tracker_project_name: phase3.2_wan_dmd2_8steps_wansyn - wandb_run_name: phase3.2_wan_dmd2_8steps_data_latent + wandb_run_name: phase3.2_wan_dmd2_8steps_simulate log_validation: true validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json validation_steps: 50 @@ -77,7 +77,7 @@ pipeline_config: method_config: # Replace legacy `training.simulate_generator_forward`. - rollout_mode: data_latent + rollout_mode: simulate generator_update_interval: 5 real_score_guidance_scale: 3.5 # Few-step schedule (single source of truth for the method). diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml index 1f02bb0b8..b3cab3778 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml @@ -54,7 +54,7 @@ training: log_validation: true validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json validation_steps: 50 - validation_sampling_steps: "8" + validation_sampling_steps: "50" validation_guidance_scale: "6.0" pipeline_config: diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml new file mode 100644 index 000000000..74d436e39 --- /dev/null +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml @@ -0,0 +1,77 @@ +recipe: + family: wan + method: finetune + +models: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # Distributed (mirror legacy Wan VSA finetune scripts) + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.3_wan2.1_finetune_vsa_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer (match legacy defaults as closely as possible) + learning_rate: 1.0e-6 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.1 + enable_gradient_checkpointing_type: full + + # VSA knobs (schedule ramps up sparsity during training) + VSA_sparsity: 0.9 + VSA_decay_rate: 0.03 + VSA_decay_interval_steps: 50 + + # Tracking / validation + tracker_project_name: phase3.3_wan_finetune_vsa_wansyn + wandb_run_name: phase3.3_wan_finetune_vsa + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 200 + validation_sampling_steps: "50" + validation_guidance_scale: "5.0" + +pipeline_config: + # Match legacy Wan VSA finetune scripts. + flow_shift: 1 + sampler_kind: ode + +method_config: + # Use the VSA attention metadata when FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN. + attn_kind: vsa + From be92a57c1f78b5f8b46819d5929f053b7627b712 Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Wed, 25 Feb 2026 07:54:44 +0000 Subject: [PATCH 094/214] add wangame dmd distillation --- .gitignore | 1 + examples/distill/WanGame2.1/distill_dmd.slurm | 146 +++++ examples/distill/WanGame2.1/validation.json | 324 +++++++++++ .../finetune_ode_init.slurm | 10 +- .../preprocess_worker.slurm | 6 +- .../causal_wangame_ode_init/validation.json | 194 +++---- .../validation_same.json | 324 +++++++++++ .../training/wangame_distillation_pipeline.py | 517 ++++++++++++++++++ 8 files changed, 1419 insertions(+), 103 deletions(-) create mode 100644 examples/distill/WanGame2.1/distill_dmd.slurm create mode 100644 examples/distill/WanGame2.1/validation.json create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json create mode 100644 fastvideo/training/wangame_distillation_pipeline.py diff --git a/.gitignore b/.gitignore index a4cb0e2e0..e8025446e 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,4 @@ wangame_lingbot_test ode_train_output sf_train_output +log diff --git a/examples/distill/WanGame2.1/distill_dmd.slurm b/examples/distill/WanGame2.1/distill_dmd.slurm new file mode 100644 index 000000000..6e4a800e0 --- /dev/null +++ b/examples/distill/WanGame2.1/distill_dmd.slurm @@ -0,0 +1,146 @@ +#!/bin/bash +#SBATCH --job-name=wg-dmd +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=log/dmd_train_output/dmd_%j.out +#SBATCH --error=log/dmd_train_output/dmd_%j.err +#SBATCH --exclusive + +set -e -x + +# Environment Setup +source ~/conda/miniconda/bin/activate +conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv +export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" + +# Basic Info +export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" +export WANDB_MODE="online" +export NCCL_P2P_DISABLE=1 +export MASTER_PORT=29500 +export NODE_RANK=$SLURM_PROCID +nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) +export MASTER_ADDR=${nodes[0]} +export TOKENIZERS_PARALLELISM=false + +echo "MASTER_ADDR: $MASTER_ADDR" +echo "NODE_RANK: $NODE_RANK" + +RUN_NAME=$(date +"%m%d_%H%M") +echo "RUN_NAME: $RUN_NAME" + +GENERATOR_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" +REAL_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Teacher model +FAKE_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Critic model + +DATA_DIR="../traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed" +VALIDATION_DATASET_FILE="examples/distill/WanGame2.1/validation.json" + +# Training arguments +training_args=( + --tracker_project_name "wangame_dmd" + --output_dir "checkpoints/wangame_dmd_${RUN_NAME}" + --wandb_run_name "${RUN_NAME}_dmd" + --max_train_steps 3000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 20 + --num_height 352 + --num_width 640 + --num_frames 77 + --enable_gradient_checkpointing_type "full" + --training_state_checkpointing_steps 500 + --weight_only_checkpointing_steps 500 +) + +# Parallel arguments +parallel_args=( + --num_gpus 32 + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim 32 +) + +model_args=( + --model_path $GENERATOR_MODEL_PATH + --pretrained_model_name_or_path $GENERATOR_MODEL_PATH + --real_score_model_path $REAL_SCORE_MODEL_PATH + --fake_score_model_path $FAKE_SCORE_MODEL_PATH +) + +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +# Validation arguments +validation_args=( + --log_validation + --log_visualization + --visualization-steps 200 + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 200 + --validation_sampling_steps "4" + --validation_guidance_scale "6.0" +) + +# Optimizer arguments +optimizer_args=( + --learning_rate 2e-6 + --mixed_precision "bf16" + --weight_decay 0.01 + --betas '0.0,0.999' + --max_grad_norm 1.0 + --fake_score_learning_rate 8e-6 + --fake_score_betas '0.0,0.999' +) + +# Miscellaneous arguments +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.0 + --dit_precision "fp32" + --flow_shift 5 + --seed 1000 + --use_ema True + --ema_decay 0.99 + --ema_start_step 200 +) + +# DMD-specific arguments +dmd_args=( + --dmd_denoising_steps '1000,750,500,250' + --min_timestep_ratio 0.02 + --max_timestep_ratio 0.98 + --dfake_gen_update_ratio 5 + --real_score_guidance_scale 3.0 + --fake_score_learning_rate 8e-6 + --fake_score_betas '0.0,0.999' + --warp_denoising_step +) + +mkdir -p log/dmd_train_output + +srun torchrun \ +--nnodes $SLURM_JOB_NUM_NODES \ +--nproc_per_node 8 \ +--node_rank $SLURM_PROCID \ +--rdzv_backend=c10d \ +--rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ + fastvideo/training/wangame_distillation_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" \ + "${dmd_args[@]}" diff --git a/examples/distill/WanGame2.1/validation.json b/examples/distill/WanGame2.1/validation.json new file mode 100644 index 000000000..2012d50fe --- /dev/null +++ b/examples/distill/WanGame2.1/validation.json @@ -0,0 +1,324 @@ +{ + "data": [ + { + "caption": "51", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000051.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000051_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "229", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000229.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000229_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "250", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000250.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000250_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "380", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000380.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000380_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "382", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000382.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000382_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "387", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000387.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000387_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "418", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000418.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000418_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "505", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000505.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000505_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "515", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000515.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000515_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "534", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000534.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000534_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "599", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000599.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000599_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "613", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000613.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000613_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "745", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000745.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000745_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "861", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000861.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000861_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "940", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000940.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000940_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "946", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000946.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000946_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "996", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000996.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000996_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1011", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001011.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001011_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1037", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001037.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001037_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1057", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001057.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001057_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1195", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001195.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001195_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1236", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001236.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001236_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1276", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001276.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001276_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1368", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001368.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001368_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1403", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001403.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001403_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1417", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001417.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001417_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1481", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001481.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001481_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1489", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001489.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001489_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1618", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001618.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001618_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1779", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001779.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001779_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1867", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001867.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001867_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "1949", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001949.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001949_action.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + } + ] +} \ No newline at end of file diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm index 52bcc4ff8..cd02796fe 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm @@ -35,8 +35,10 @@ echo "RUN_NAME: $RUN_NAME" # Configs MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" -DATA_DIR="../traindata_0222_0030/ode_init_mc_with_mouse/preprocessed_wangame" -VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" +# DATA_DIR="../traindata_0222_0030/ode_init_mc_Xonly_3k/preprocessed" +DATA_DIR="../traindata_0222_0030/ode_init_mc_random/preprocessed_wangame" +# VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" +VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json" CKPT_SAFETENSOR="/mnt/weka/home/hao.zhang/mhuo/FastVideo/wangame_1.3b_1action_rand_from_scratch/checkpoint-9000/transformer/diffusion_pytorch_model.safetensors" # Training arguments @@ -44,11 +46,11 @@ training_args=( --tracker_project_name "wangame_ode_init" --output_dir "checkpoints/wangame_ode_init_${RUN_NAME}" --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" - --wandb_run_name "${RUN_NAME}_bs32" + --wandb_run_name "${RUN_NAME}_bs64_random" --max_train_steps 5000 --train_batch_size 1 --train_sp_batch_size 1 - --gradient_accumulation_steps 1 + --gradient_accumulation_steps 2 --num_latent_t 21 --num_height 352 --num_width 640 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm index 2aac00253..79a57304e 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm @@ -17,7 +17,8 @@ START_FILE=${1:-1} # Starting file number for this node NODE_ID=${2:-0} # Node identifier (0-7) MODEL_PATH="./Wan2.1-Fun-1.3B-InP-Diffusers" -OUTPUT_BASE="traindata_0222_0030/ode_init_mc_with_mouse/preprocessed_wangame" +# OUTPUT_BASE="traindata_0222_0030/ode_init_mc_Xonly_3k/preprocessed" +OUTPUT_BASE="traindata_0222_0030/ode_init_mc_same/preprocessed" # Port range calculation base_port=$((29500 + NODE_ID * 100)) @@ -28,7 +29,8 @@ for i in {1..8}; do gpu=${gpu_ids[((i-1))]} file_num=$((START_FILE + i - 1)) - DATA_MERGE_PATH="traindata_0222_0030/ode_init_mc_with_mouse/merge_${file_num}.txt" + # DATA_MERGE_PATH="traindata_0222_0030/ode_init_mc_Xonly_3k/merge_${file_num}.txt" + DATA_MERGE_PATH="traindata_0222_0030/ode_init_mc_same/merge_${file_num}.txt" OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" echo "DATA_MERGE_PATH: $DATA_MERGE_PATH" echo "OUTPUT_DIR: $OUTPUT_DIR" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json index b796d925f..bdd9b6e45 100644 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json @@ -1,9 +1,9 @@ { "data": [ { - "caption": "00: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000000_action.npy", + "caption": "51", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000051.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000051_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -11,9 +11,9 @@ "num_frames": 81 }, { - "caption": "01: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000001_action.npy", + "caption": "229", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000229.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000229_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -21,9 +21,9 @@ "num_frames": 81 }, { - "caption": "02: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000002_action.npy", + "caption": "250", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000250.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000250_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -31,9 +31,9 @@ "num_frames": 81 }, { - "caption": "03: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000003_action.npy", + "caption": "380", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000380.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000380_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -41,9 +41,9 @@ "num_frames": 81 }, { - "caption": "04: Idle", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000004_action.npy", + "caption": "382", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000382.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000382_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -51,9 +51,9 @@ "num_frames": 81 }, { - "caption": "05: camera down", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000005_action.npy", + "caption": "387", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000387.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000387_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -61,9 +61,9 @@ "num_frames": 81 }, { - "caption": "06: camera up", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000006.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000006_action.npy", + "caption": "418", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000418.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000418_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -71,9 +71,9 @@ "num_frames": 81 }, { - "caption": "07: camera right", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000007.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000007_action.npy", + "caption": "505", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000505.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000505_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -81,9 +81,9 @@ "num_frames": 81 }, { - "caption": "08: camera left", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000008.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000008_action.npy", + "caption": "515", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000515.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000515_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -91,9 +91,9 @@ "num_frames": 81 }, { - "caption": "09: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000009.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000009_action.npy", + "caption": "534", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000534.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000534_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -101,9 +101,9 @@ "num_frames": 81 }, { - "caption": "10: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000010.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000010_action.npy", + "caption": "599", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000599.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000599_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -111,9 +111,9 @@ "num_frames": 81 }, { - "caption": "11: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000011.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000011_action.npy", + "caption": "613", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000613.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000613_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -121,9 +121,9 @@ "num_frames": 81 }, { - "caption": "12: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000012.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000012_action.npy", + "caption": "745", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000745.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000745_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -131,9 +131,9 @@ "num_frames": 81 }, { - "caption": "13: Idle", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000013.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000013_action.npy", + "caption": "861", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000861.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000861_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -141,9 +141,9 @@ "num_frames": 81 }, { - "caption": "14: Idle", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000014.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000014_action.npy", + "caption": "940", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000940.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000940_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -151,9 +151,9 @@ "num_frames": 81 }, { - "caption": "15: camera down", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000015.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000015_action.npy", + "caption": "946", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000946.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000946_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -161,9 +161,9 @@ "num_frames": 81 }, { - "caption": "16: camera up", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000016.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000016_action.npy", + "caption": "996", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000996.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000996_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -171,9 +171,9 @@ "num_frames": 81 }, { - "caption": "17: camera right", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000017.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000017_action.npy", + "caption": "1011", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001011.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001011_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -181,9 +181,9 @@ "num_frames": 81 }, { - "caption": "18: camera left", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000018.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000018_action.npy", + "caption": "1037", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001037.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001037_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -191,9 +191,9 @@ "num_frames": 81 }, { - "caption": "19: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000019.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000019_action.npy", + "caption": "1057", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001057.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001057_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -201,9 +201,9 @@ "num_frames": 81 }, { - "caption": "20: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000020.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000020_action.npy", + "caption": "1195", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001195.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001195_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -211,9 +211,9 @@ "num_frames": 81 }, { - "caption": "21: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000021.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000021_action.npy", + "caption": "1236", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001236.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001236_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -221,9 +221,9 @@ "num_frames": 81 }, { - "caption": "22: D", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000022.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000022_action.npy", + "caption": "1276", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001276.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001276_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -231,9 +231,9 @@ "num_frames": 81 }, { - "caption": "23: Idle", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000023.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000023_action.npy", + "caption": "1368", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001368.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001368_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -241,9 +241,9 @@ "num_frames": 81 }, { - "caption": "24: Idle", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000024.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000024_action.npy", + "caption": "1403", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001403.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001403_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -251,9 +251,9 @@ "num_frames": 81 }, { - "caption": "25: camera down", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000025.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000025_action.npy", + "caption": "1417", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001417.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001417_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -261,9 +261,9 @@ "num_frames": 81 }, { - "caption": "26: camera up", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000026.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000026_action.npy", + "caption": "1481", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001481.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001481_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -271,9 +271,9 @@ "num_frames": 81 }, { - "caption": "27: camera right", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000027.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000027_action.npy", + "caption": "1489", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001489.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001489_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -281,9 +281,9 @@ "num_frames": 81 }, { - "caption": "28: camera left", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000028.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000028_action.npy", + "caption": "1618", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001618.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001618_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -291,9 +291,9 @@ "num_frames": 81 }, { - "caption": "29: W", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000029.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000029_action.npy", + "caption": "1779", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001779.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001779_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -301,9 +301,9 @@ "num_frames": 81 }, { - "caption": "30: S", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000030.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000030_action.npy", + "caption": "1867", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001867.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001867_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -311,9 +311,9 @@ "num_frames": 81 }, { - "caption": "31: A", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000031.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_with_mouse/images/000031_action.npy", + "caption": "1949", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001949.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001949_action.npy", "video_path": null, "num_inference_steps": 4, "height": 352, @@ -321,4 +321,4 @@ "num_frames": 81 } ] -} +} \ No newline at end of file diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json new file mode 100644 index 000000000..612000782 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json @@ -0,0 +1,324 @@ +{ + "data": [ + { + "caption": "51", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000051.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000051_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "229", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000229.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000229_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "250", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000250.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000250_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "380", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000380.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000380_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "382", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000382.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000382_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "387", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000387.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000387_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "418", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000418.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000418_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "505", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000505.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000505_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "515", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000515.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000515_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "534", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000534.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000534_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "599", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000599.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000599_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "613", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000613.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000613_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "745", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000745.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000745_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "861", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000861.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000861_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "940", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000940.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000940_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "946", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000946.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000946_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "996", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000996.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000996_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1011", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001011.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001011_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1037", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001037.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001037_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1057", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001057.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001057_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1195", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001195.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001195_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1236", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001236.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001236_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1276", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001276.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001276_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1368", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001368.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001368_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1403", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001403.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001403_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1417", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001417.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001417_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1481", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001481.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001481_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1489", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001489.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001489_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1618", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001618.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001618_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1779", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001779.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001779_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1867", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001867.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001867_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + }, + { + "caption": "1949", + "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001949.jpg", + "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001949_action.npy", + "video_path": null, + "num_inference_steps": 4, + "height": 352, + "width": 640, + "num_frames": 81 + } + ] +} \ No newline at end of file diff --git a/fastvideo/training/wangame_distillation_pipeline.py b/fastvideo/training/wangame_distillation_pipeline.py new file mode 100644 index 000000000..c72c60250 --- /dev/null +++ b/fastvideo/training/wangame_distillation_pipeline.py @@ -0,0 +1,517 @@ +# SPDX-License-Identifier: Apache-2.0 +import sys +from copy import deepcopy +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F +from einops import rearrange + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.forward_context import set_forward_context +from fastvideo.logger import init_logger +from fastvideo.models.dits.hyworld.pose import process_custom_actions +from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler) +from fastvideo.models.utils import pred_noise_to_pred_video +from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import ( + WanGameActionImageToVideoPipeline) +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch +from fastvideo.training.distillation_pipeline import DistillationPipeline +from fastvideo.training.training_utils import shift_timestep +from fastvideo.utils import is_vsa_available, shallow_asdict + +try: + vsa_available = is_vsa_available() +except Exception: + vsa_available = False + +logger = init_logger(__name__) + + +class WanGameDistillationPipeline(DistillationPipeline): + """ + DMD distillation pipeline for WanGame. + """ + _required_config_modules = ["scheduler", "transformer", "vae"] + + def initialize_pipeline(self, fastvideo_args: FastVideoArgs): + """Initialize WanGame-specific scheduler.""" + self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler( + shift=fastvideo_args.pipeline_config.flow_shift) + + def create_training_stages(self, training_args: TrainingArgs): + """May be used in future refactors.""" + pass + + def set_schemas(self): + self.train_dataset_schema = pyarrow_schema_wangame + + def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: + batch = next(self.train_loader_iter, None) # type: ignore + if batch is None: + self.current_epoch += 1 + logger.info("Starting epoch %s", self.current_epoch) + self.train_loader_iter = iter(self.train_dataloader) + batch = next(self.train_loader_iter) + + device = get_local_torch_device() + dtype = torch.bfloat16 + + clip_feature = batch['clip_feature'] + first_frame_latent = batch['first_frame_latent'] + keyboard_cond = batch.get('keyboard_cond', None) + mouse_cond = batch.get('mouse_cond', None) + infos = batch['info_list'] + + if self.training_args.simulate_generator_forward: + # When simulating, we don't need real VAE latents — just use zeros + batch_size = clip_feature.shape[0] + vae_config = self.training_args.pipeline_config.vae_config.arch_config + num_channels = vae_config.z_dim + spatial_compression_ratio = vae_config.spatial_compression_ratio + + latent_height = self.training_args.num_height // spatial_compression_ratio + latent_width = self.training_args.num_width // spatial_compression_ratio + + latents = torch.zeros( + batch_size, + num_channels, + self.training_args.num_latent_t, + latent_height, + latent_width, + device=device, + dtype=dtype, + ) + else: + if 'vae_latent' not in batch: + raise ValueError( + "vae_latent not found in batch and simulate_generator_forward is False. " + "Either preprocess data with VAE latents or set --simulate_generator_forward." + ) + latents = batch['vae_latent'] + latents = latents[:, :, :self.training_args.num_latent_t] + latents = latents.to(device, dtype=dtype) + + training_batch.latents = latents.to(device, dtype=dtype) + training_batch.encoder_hidden_states = None + training_batch.encoder_attention_mask = None + training_batch.image_embeds = clip_feature.to(device, dtype=dtype) + training_batch.image_latents = first_frame_latent.to(device, dtype=dtype) + + # Action conditioning + if keyboard_cond is not None and keyboard_cond.numel() > 0: + training_batch.keyboard_cond = keyboard_cond.to(device, dtype=dtype) + else: + training_batch.keyboard_cond = None + if mouse_cond is not None and mouse_cond.numel() > 0: + training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) + else: + training_batch.mouse_cond = None + + training_batch.infos = infos + return training_batch + + def _prepare_dit_inputs(self, + training_batch: TrainingBatch) -> TrainingBatch: + """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" + # First, call parent method to prepare noise, timesteps, etc. for video latents + training_batch = super()._prepare_dit_inputs(training_batch) + + training_batch.conditional_dict = { + "encoder_hidden_states": None, + "encoder_attention_mask": None, + } + training_batch.unconditional_dict = None + + assert isinstance(training_batch.image_latents, torch.Tensor) + image_latents = training_batch.image_latents.to( + get_local_torch_device(), dtype=torch.bfloat16) + + # Build mask + image_latent -> cond_concat (20 channels) + temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (self.training_args.num_latent_t - + 1) * temporal_compression_ratio + 1 + batch_size, num_channels, _, latent_height, latent_width = image_latents.shape + mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, + latent_width) + mask_lat_size[:, :, 1:] = 0 + + first_frame_mask = mask_lat_size[:, :, :1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, dim=2, repeats=temporal_compression_ratio) + mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], + dim=2) + mask_lat_size = mask_lat_size.view(batch_size, -1, + temporal_compression_ratio, + latent_height, latent_width) + mask_lat_size = mask_lat_size.transpose(1, 2) + mask_lat_size = mask_lat_size.to( + image_latents.device).to(dtype=torch.bfloat16) + + # cond_concat = [mask(4), image_latent(16)] = 20 channels + image_latents = torch.cat([mask_lat_size, image_latents], dim=1) + + if self.sp_world_size > 1: + total_frames = image_latents.shape[2] + # Split cond latents to local SP shard only when tensor is still global. + if total_frames == self.training_args.num_latent_t: + if total_frames % self.sp_world_size != 0: + raise ValueError( + "image_latents temporal dim is not divisible by SP world size: " + f"frames={total_frames}, sp_world_size={self.sp_world_size}" + ) + image_latents = rearrange(image_latents, + "b c (n t) h w -> b c n t h w", + n=self.sp_world_size).contiguous() + image_latents = image_latents[:, :, self.rank_in_sp_group, :, :, + :] + + training_batch.image_latents = image_latents + + return training_batch + + def _build_distill_input_kwargs( + self, noise_input: torch.Tensor, timestep: torch.Tensor, + text_dict: dict[str, torch.Tensor] | None, + training_batch: TrainingBatch) -> TrainingBatch: + """Build model input with WanGame + """ + # Image embeds (CLIP features) for cross-attention conditioning + image_embeds = training_batch.image_embeds + assert torch.isnan(image_embeds).sum() == 0 + image_embeds = image_embeds.to(get_local_torch_device(), + dtype=torch.bfloat16) + + # already prepared in _prepare_dit_inputs + image_latents = training_batch.image_latents + + # Process action conditioning + keyboard_cond = training_batch.keyboard_cond + mouse_cond = training_batch.mouse_cond + + if keyboard_cond is not None and mouse_cond is not None: + viewmats_list = [] + intrinsics_list = [] + action_labels_list = [] + for b in range(noise_input.shape[0]): + viewmats, intrinsics, action_labels = process_custom_actions( + keyboard_cond[b], mouse_cond[b]) + viewmats_list.append(viewmats) + intrinsics_list.append(intrinsics) + action_labels_list.append(action_labels) + + viewmats = torch.stack(viewmats_list, dim=0).to( + device=get_local_torch_device(), dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, dim=0).to( + device=get_local_torch_device(), dtype=torch.bfloat16) + action_labels = torch.stack(action_labels_list, dim=0).to( + device=get_local_torch_device(), dtype=torch.bfloat16) + else: + viewmats = None + intrinsics = None + action_labels = None + + # I2V concatenation: [noise_input(16ch), image_latents(20ch)] -> 36ch + noisy_model_input = torch.cat( + [noise_input, image_latents.permute(0, 2, 1, 3, 4)], dim=2) + + training_batch.input_kwargs = { + "hidden_states": noisy_model_input.permute(0, 2, 1, 3, 4), + "encoder_hidden_states": None, + "timestep": timestep, + "encoder_hidden_states_image": image_embeds, + "viewmats": viewmats, + "Ks": intrinsics, + "action": action_labels, + "return_dict": False, + } + training_batch.noise_latents = noise_input + + return training_batch + + def _dmd_forward(self, generator_pred_video: torch.Tensor, + training_batch: TrainingBatch) -> torch.Tensor: + """Compute DMD loss for WanGame.""" + original_latent = generator_pred_video + with torch.no_grad(): + timestep = torch.randint(0, + self.num_train_timestep, [1], + device=self.device, + dtype=torch.long) + + timestep = shift_timestep(timestep, self.timestep_shift, + self.num_train_timestep) + + timestep = timestep.clamp(self.min_timestep, self.max_timestep) + + noise = torch.randn(self.video_latent_shape, + device=self.device, + dtype=generator_pred_video.dtype) + + noisy_latent = self.noise_scheduler.add_noise( + generator_pred_video.flatten(0, 1), noise.flatten(0, 1), + timestep).detach().unflatten(0, (1, generator_pred_video.shape[1])) + + # Build input kwargs for critic/teacher + training_batch = self._build_distill_input_kwargs( + noisy_latent, timestep, training_batch.conditional_dict, + training_batch) + + # fake_score_transformer forward + current_fake_score_transformer = self._get_fake_score_transformer( + timestep) + fake_score_pred_noise = current_fake_score_transformer( + **training_batch.input_kwargs).permute(0, 2, 1, 3, 4) + + faker_score_pred_video = pred_noise_to_pred_video( + pred_noise=fake_score_pred_noise.flatten(0, 1), + noise_input_latent=noisy_latent.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler).unflatten( + 0, fake_score_pred_noise.shape[:2]) + + # real_score_transformer forward + current_real_score_transformer = self._get_real_score_transformer( + timestep) + real_score_pred_noise = current_real_score_transformer( + **training_batch.input_kwargs).permute(0, 2, 1, 3, 4) + + real_score_pred_video = pred_noise_to_pred_video( + pred_noise=real_score_pred_noise.flatten(0, 1), + noise_input_latent=noisy_latent.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler).unflatten( + 0, real_score_pred_noise.shape[:2]) + + # No CFG for WanGame - use real_score_pred_video directly + grad = (faker_score_pred_video - real_score_pred_video) / torch.abs( + original_latent - real_score_pred_video).mean() + grad = torch.nan_to_num(grad) + + dmd_loss = 0.5 * F.mse_loss( + original_latent.float(), + (original_latent.float() - grad.float()).detach()) + + training_batch.dmd_latent_vis_dict.update({ + "training_batch_dmd_fwd_clean_latent": + training_batch.latents, + "generator_pred_video": + original_latent.detach(), + "real_score_pred_video": + real_score_pred_video.detach(), + "faker_score_pred_video": + faker_score_pred_video.detach(), + "dmd_timestep": + timestep.detach(), + }) + + return dmd_loss + + def faker_score_forward( + self, training_batch: TrainingBatch + ) -> tuple[TrainingBatch, torch.Tensor]: + """Forward pass for critic training with WanGame action conditioning.""" + with torch.no_grad(), set_forward_context( + current_timestep=training_batch.timesteps, + attn_metadata=training_batch.attn_metadata_vsa): + if self.training_args.simulate_generator_forward: + generator_pred_video = self._generator_multi_step_simulation_forward( + training_batch) + else: + generator_pred_video = self._generator_forward(training_batch) + + fake_score_timestep = torch.randint(0, + self.num_train_timestep, [1], + device=self.device, + dtype=torch.long) + + fake_score_timestep = shift_timestep(fake_score_timestep, + self.timestep_shift, + self.num_train_timestep) + + fake_score_timestep = fake_score_timestep.clamp(self.min_timestep, + self.max_timestep) + + fake_score_noise = torch.randn(self.video_latent_shape, + device=self.device, + dtype=generator_pred_video.dtype) + + noisy_generator_pred_video = self.noise_scheduler.add_noise( + generator_pred_video.flatten(0, 1), + fake_score_noise.flatten(0, 1), fake_score_timestep).unflatten( + 0, (1, generator_pred_video.shape[1])) + + with set_forward_context(current_timestep=training_batch.timesteps, + attn_metadata=training_batch.attn_metadata): + training_batch = self._build_distill_input_kwargs( + noisy_generator_pred_video, fake_score_timestep, + training_batch.conditional_dict, training_batch) + + current_fake_score_transformer = self._get_fake_score_transformer( + fake_score_timestep) + fake_score_pred_noise = current_fake_score_transformer( + **training_batch.input_kwargs).permute(0, 2, 1, 3, 4) + + target = fake_score_noise - generator_pred_video + flow_matching_loss = torch.mean((fake_score_pred_noise - target)**2) + + training_batch.fake_score_latent_vis_dict = { + "training_batch_fakerscore_fwd_clean_latent": + training_batch.latents, + "generator_pred_video": generator_pred_video, + "fake_score_timestep": fake_score_timestep, + } + + return training_batch, flow_matching_loss + + def initialize_validation_pipeline(self, training_args: TrainingArgs): + logger.info("Initializing validation pipeline...") + args_copy = deepcopy(training_args) + args_copy.inference_mode = True + + validation_pipeline = WanGameActionImageToVideoPipeline.from_pretrained( + training_args.model_path, + args=args_copy, # type: ignore + inference_mode=True, + loaded_modules={ + "transformer": self.get_module("transformer"), + "vae": self.get_module("vae"), + }, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + pin_cpu_memory=training_args.pin_cpu_memory, + dit_cpu_offload=True) + + self.validation_pipeline = validation_pipeline + + def _prepare_validation_batch(self, sampling_param: SamplingParam, + training_args: TrainingArgs, + validation_batch: dict[str, Any], + num_inference_steps: int) -> ForwardBatch: + sampling_param.prompt = validation_batch['prompt'] + sampling_param.height = training_args.num_height + sampling_param.width = training_args.num_width + sampling_param.image_path = validation_batch.get( + 'image_path') or validation_batch.get('video_path') + sampling_param.num_inference_steps = num_inference_steps + sampling_param.data_type = "video" + assert self.seed is not None + sampling_param.seed = self.seed + + latents_size = [(sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, sampling_param.width // 8] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (training_args.num_latent_t - + 1) * temporal_compression_factor + 1 + sampling_param.num_frames = num_frames + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=torch.Generator(device="cpu").manual_seed(self.seed), + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=training_args.VSA_sparsity, + ) + if "image" in validation_batch and validation_batch["image"] is not None: + batch.pil_image = validation_batch["image"] + + if "keyboard_cond" in validation_batch and validation_batch[ + "keyboard_cond"] is not None: + keyboard_cond = validation_batch["keyboard_cond"] + if isinstance(keyboard_cond, torch.Tensor): + keyboard_cond = keyboard_cond.detach().clone().to(dtype=torch.bfloat16) + else: + keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) + keyboard_cond = keyboard_cond.unsqueeze(0) + batch.keyboard_cond = keyboard_cond[:num_frames] + + if "mouse_cond" in validation_batch and validation_batch[ + "mouse_cond"] is not None: + mouse_cond = validation_batch["mouse_cond"] + if isinstance(mouse_cond, torch.Tensor): + mouse_cond = mouse_cond.detach().clone().to(dtype=torch.bfloat16) + else: + mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) + mouse_cond = mouse_cond.unsqueeze(0) + batch.mouse_cond = mouse_cond[:num_frames] + + return batch + + def _post_process_validation_frames( + self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: + """Apply action overlay to validation frames for WanGame. + + Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. + """ + # Check if action data is available + keyboard_cond = getattr(batch, 'keyboard_cond', None) + mouse_cond = getattr(batch, 'mouse_cond', None) + + if keyboard_cond is None and mouse_cond is None: + return frames + + # Import overlay functions + from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, + draw_mouse_on_frame) + + # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) + if keyboard_cond is not None: + keyboard_cond = keyboard_cond.squeeze( + 0).cpu().float().numpy() # (T, 6) + if mouse_cond is not None: + mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) + + # WanGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] + key_names = ["W", "S", "A", "D", "left", "right"] + + processed_frames = [] + for frame_idx, frame in enumerate(frames): + frame = np.ascontiguousarray(frame.copy()) + + # Draw keyboard overlay + if keyboard_cond is not None and frame_idx < len(keyboard_cond): + keys = { + key_names[i]: bool(keyboard_cond[frame_idx, i]) + for i in range(min(len(key_names), keyboard_cond.shape[1])) + } + draw_keys_on_frame(frame, keys, mode='universal') + + # Draw mouse overlay + if mouse_cond is not None and frame_idx < len(mouse_cond): + pitch = float(mouse_cond[frame_idx, 0]) + yaw = float(mouse_cond[frame_idx, 1]) + draw_mouse_on_frame(frame, pitch, yaw) + + processed_frames.append(frame) + + return processed_frames + + +def main(args) -> None: + logger.info("Starting WanGame DMD distillation pipeline...") + + pipeline = WanGameDistillationPipeline.from_pretrained( + args.pretrained_model_name_or_path, args=args) + + args = pipeline.training_args + pipeline.train() + logger.info("WanGame DMD distillation pipeline completed") + + +if __name__ == "__main__": + argv = sys.argv + from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.utils import FlexibleArgumentParser + parser = FlexibleArgumentParser() + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args() + main(args) From ef3d6995e92c184ea6c4b05023d0e8135501ce03 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 08:48:56 +0000 Subject: [PATCH 095/214] discussing refactor --- dev/phases/phase_3.md | 5 +- dev/refactor.md | 215 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 dev/refactor.md diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index bd367bab3..d482a910c 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -145,8 +145,9 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop - `predict_noise(handle, ...)`（以及可选 `predict_x0`） - `backward(loss, ctx, ...)`（forward-context/activation ckpt 相关） - [x] configs/examples - - [x] `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` - - [x] `examples/distillation/phase3_3/temp.sh` + - [x] `fastvideo/distillation/outs + ide/fastvideo/configs/distillation/finetun e_wan2.1_t2v_1.3B_phase3.3.yaml` + - [x] `examples/distillation/phase3_3/temp.sh` --- diff --git a/dev/refactor.md b/dev/refactor.md new file mode 100644 index 000000000..89485b075 --- /dev/null +++ b/dev/refactor.md @@ -0,0 +1,215 @@ +# 重构讨论：把复杂度“移到 config” vs 保持“清晰分层” + +这份文档记录 FastVideo distillation 重构（Phase 2 → 3+）过程中，我们对 +架构抽象与配置复杂度之间取舍的讨论，以及对 Phase 4+ 的可能方向。 + +核心矛盾： + +- **方案 A（配置驱动加载 / 更少概念）** + - 在 YAML 里显式写清模型加载细节（VAE、text encoder 等的路径/开关）。 + - `Method` 直接调用共享的 `utils/` 来完成加载与组装。 + - 希望减少“概念数量”（Family / Adapter / Bundle / Validator / …）。 + +- **方案 B（清晰分层 / 显式边界）** + - `Method` 只关注算法与训练接口（algorithm-only）。 + - 模型/管线差异由 `Adapter`（operation-centric）吸收；构建期组装由 + `Family/Builder` 处理。 + - 配置保持结构化并可校验（schema + validation）。 + +本文档不是最终结论，主要记录： +1) 共识是什么； +2) 分歧是什么； +3) 实际 trade-off； +4) 收敛路径（如何减少概念但不破坏边界）。 + +--- + +## 术语表（当前 FastVideo 语义） + +- **Role**：配置里用于索引参与者的字符串 key，例如 `student`、`teacher`、 + `critic`……（不设“高低贵贱”，只是 key）。 +- **RoleHandle**：每个 role 对应的句柄，包含 `modules/optimizers/schedulers` + 以及类似 `trainable` 的标志。 +- **ModelBundle**：持有所有 `RoleHandle` 的容器（未来可能改名）。 +- **Adapter**：训练原语的 operation-centric 接口（如 `prepare_batch`、 + `predict_noise/x0`、`add_noise`、`backward` 等）。它不应包含算法策略。 +- **Family**：构建期工厂：根据 config 组装 bundle + adapter + validator。 +- **Validator**：训练期 validation 的采样/记录层，由 method 通过 + `ValidationRequest` 提供关键参数（steps/sampler/guidance/…）。 + +--- + +## 共同目标（共识） + +1) **消除隐式耦合** + - 避免“validation 的 side-effect 初始化训练状态”这种隐藏依赖。 + - 避免算法名（例如 “dmd”）泄漏进 adapter / validator。 + +2) **可扩展** + - 新增 method 不应导致入口文件/组合爆炸。 + - 新增 role 不应导致“每个 role 一个函数”的 API 爆炸。 + +3) **职责清晰** + - 算法决策归 `Method`。 + - 模型/管线的工程差异归 `Adapter` / 构建期组装。 + +4) **配置表达力强，但要安全** + - 配置需要能表达当前 distillation，也要能表达未来训练语义（例如 + finetune = 只有 student 的一种“特殊 distill”）。 + +--- + +## 核心分歧 + +### A) “把抽象复杂度移到 config” + +直觉是： +- 如果 config 能声明 `vae_path`、`text_encoder_path` 等，那么代码可以更 + 简单：少一些抽象层、少一些类。 +- Loading/组装细节可以抽到 `utils/`，由 method 直接调用。 + +### B) “配置驱动加载仍需要边界” + +反驳点： +- 即使 YAML 写清楚了 *路径*，加载的 *语义* 仍然是 model/pipeline 相关的： + - 有哪些子模块（`transformer` / `vae` / `text_encoder` / …）； + - latent 的归一化/布局； + - attention metadata（VSA/VMoBA）如何构建； + - conditioning 的结构与缓存（text dict 的形状、neg/uncond 的初始化与 + broadcast）； + - dtype/offload/并行切分（tp/sp/hsdp）的要求。 + +如果 `Method` 调用 `utils.load_vae(...)`，而 `utils` 内部其实知道 +“Wan 的 VAE 怎么 load、SDXL 的 VAE 怎么 load”，那么 **method 仍然被污染** +（哪怕是间接污染）。我们只是把耦合从“文件 A”搬到了“文件 utils”，并没有 +真正消除耦合。 + +--- + +## 讨论：能否做一个“完全通用”的 config → load → 调度？ + +先澄清“通用”到底指什么： + +- **通用的模块加载机制**：可以统一（FastVideo 大多数 pipeline 的确会走 + `PipelineComponentLoader.load_module()` 这条路径）。 +- **通用的 pipeline contract（需要哪些模块 + 如何初始化 + 如何采样/训练）**： + **很难做到完全通用**，通常只能做到“框架通用 + 插件/实现分家族”。 + +为什么“contract”无法完全靠 config 自动推导？因为差异不仅在 *加载路径*，还在： + +1) **需要加载哪些模块本身就不同** + - Wan：`text_encoder/tokenizer/vae/transformer/scheduler` + - SD3.5：`text_encoder(1/2/3) + tokenizer(1/2/3)`（三套编码器） + - Hunyuan：`text_encoder_2/tokenizer_2`（两套） + - LTX2：额外的 `audio_vae`、`vocoder` + - Cosmos：额外的 `safety_checker` + +2) **即使模块名相同，初始化/替换策略也不同** + - 有些 pipeline 会在 `initialize_pipeline()` 里“重建/替换” scheduler， + 而不是按权重目录加载（例如 Cosmos 使用 `FlowMatchEulerDiscreteScheduler`， + TurboDiffusion 使用 `RCMScheduler`）。 + - Wan 也会基于 `sampler_kind` 构建 scheduler，并改变 stage graph（ODE/SDE）。 + +3) **部分 pipeline 需要特殊的加载/回退逻辑** + - LTX2 的 tokenizer 目录可能不存在，需要从 `text_encoder/gemma` 回退。 + - StepVideo 的 text encoder/clip 不是走 `model_index.json` 加载，而是在 + `initialize_pipeline()` 里手工构建，并加载自定义 `.so`。 + +4) **stage graph 差异是真实存在的** + - refine pipeline（LongCat）会插入 refine init / timestep override 等 stage； + - audio pipeline（LTX2）会有 audio decode stage； + - 不同 pipeline 对 conditioning/timestep/latent 准备的处理逻辑不一致。 + +因此结论不是“完全无法抽象出通用框架”，而是： + +- 我们可以抽象出 **通用框架**（统一入口、统一 YAML schema、统一 Trainer、统一 + Method 接口、统一 Adapter primitives、统一 Validator request/日志框架）。 +- 但要做到“**只靠一份 dict config + 一个通用 utils** 就能自动适配所有模型家族”， + 很容易被上述 contract 差异击穿；最终要么： + - 在 utils 里堆满 if/else（隐式 family），要么 + - 让 method 变成“懂所有 pipeline 语义的上帝对象”（method 污染）。 + +这也是为什么我们目前更倾向于保留“家族层”的存在（哪怕未来换个名字）： +它是“contract 插件层”，负责把 config 映射到 **正确的 pipeline/adapter/runtime 组装**， +把 method 留在算法层。 + +--- + +## “配置驱动加载”擅长什么、不擅长什么 + +配置擅长： +- **选择声明**：用哪个权重、哪些可选模块要加载、从哪里加载。 +- **策略 knob**：例如 `sampler_kind`、`flow_shift`、validation steps、 + update interval 等。 +- **可复现**：把 run 的 YAML 原样上传到 W&B（后续复盘非常方便）。 + +配置不擅长： +- **跨 role 的一致性约束**（例如共享 VAE、latent 空间兼容、scheduler 类型一致）。 +- **防止“看似能跑但悄悄不一致”**（dtype/offload 不一致、negative embedding + 布局不一致、VSA metadata 不一致等）。 + +这些一致性通常需要： +- 单一组装点（builder/family），和/或 +- 一个共享的 context（由 role-bound view 使用）。 + +--- + +## 为什么现在还需要 Family/Builder（务实角度） + +即使只让 student 初始化 preprocessors，我们仍然需要一个地方来： +- 为所有 roles 加载 `RoleHandle.modules`（student/teacher/critic/…）； +- 决定哪些资源共享、哪些 per-role； +- 初始化共享缓存（neg/uncond conditioning 等）； +- 按 run config 构建 validator（采样 pipeline）并保持一致； +- 只在 rank0 创建 tracker / run 元信息。 + +如果移除 Family，工程上往往会以另外一种形式“复活”同样的东西： +- `SharedContext`， +- `RuntimeFactory`， +- 或“method 构造函数里做 assembly”。 + +因此问题变成：我们到底想要 **显式的组装层**，还是 **隐式的组装层**？ +通常显式层更容易 review、更容易写测试、更容易维护 invariant。 + +--- + +## 建议的收敛方案（减少概念，但不把语义推给 method） + +我们可以通过以下方式减少“概念感”，同时保持边界： + +1) **重命名 / 重塑概念** + - 例如 `ModelBundle` → `RoleManager`（语义更直观）。 + - `Family` → `RuntimeFactory`（如果“Family”这个词更难理解）。 + +2) **让 method “看起来像持有多个 adapter”，但底层共享 context** + - 保留一份共享 adapter + context。 + - 增加轻量 per-role view： + - `adapter.bind(handle)` → `RoleAdapterView` + - method 持有 `student_api` / `teacher_api` / …（view），而不是 raw adapter。 + - 既保留共享资源的一致性，又贴近心智模型： + “method 与 role-specific API 交互”。 + +3) **配置保持结构化，但允许受控扩展** + - 保持顶层 schema（例如 `recipe/models/training/pipeline_config/method_config`）。 + - 允许在 `method_config`（或未来的 `family_config`）里放自由 dict， + 但解析与校验集中在一个入口，避免 dict 在全栈到处传。 + +4) **infra ownership 下沉到 trainer/entrypoint** + - tracker 应由 trainer/runtime 持有（或 entrypoint 创建），而不是由 family + 的实现细节决定。 + +--- + +## 下一轮需要明确的问题（可作为下一次讨论议题） + +1) “load preprocessors” 的 policy 应该放在哪里？ + - Family（assembly） vs Adapter（context） vs Method（algorithm）。 + +2) 如果未来支持跨 family distill（Wan teacher → SDXL student），如何保证 + “共享 VAE/latent 空间一致”的 invariant？ + +3) 当前 adapter API 哪些部分仍然过于 role-centric？ + - 目标：尽量 operation-centric primitives。 + +4) 是否需要一个显式的 `SharedContext` 类型？ + - 还是继续把共享状态作为 adapter 的内部实现细节。 From 5f5e74f30a936dd1c83f1966a7a7a711dfbd4677 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 19:36:20 +0000 Subject: [PATCH 096/214] changing config.md --- dev/config.md | 10 +- dev/phases/phase_3.md | 112 +++++++++++++++++- dev/refactor.md | 3 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 2 +- fastvideo/distillation/builder.py | 2 +- fastvideo/distillation/families/wan.py | 2 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 2 +- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml | 2 +- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml | 2 +- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml | 3 +- .../finetune_wan2.1_t2v_1.3B_phase3.3.yaml | 3 +- ...finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml | 3 +- fastvideo/distillation/registry.py | 2 +- fastvideo/distillation/specs.py | 29 ----- fastvideo/distillation/utils/__init__.py | 4 + .../{runtime.py => utils/config.py} | 25 ++++ fastvideo/distillation/yaml_config.py | 18 +-- 17 files changed, 163 insertions(+), 61 deletions(-) delete mode 100644 fastvideo/distillation/specs.py create mode 100644 fastvideo/distillation/utils/__init__.py rename fastvideo/distillation/{runtime.py => utils/config.py} (66%) diff --git a/dev/config.md b/dev/config.md index 742404ca8..288691c3f 100644 --- a/dev/config.md +++ b/dev/config.md @@ -28,7 +28,7 @@ CLI 仅保留少量 **runtime override**（不属于“实验定义”的内容 ```yaml recipe: # 选择 family + method（只负责“选什么”） -models: # role -> role spec（谁参与） +roles: # role -> role spec（谁参与） training: # infra 参数（直接映射到 TrainingArgs） pipeline_config: # 模型/pipeline 侧 config（可 inline） method_config: # method/algorithm 超参（方法侧 single source of truth） @@ -52,10 +52,10 @@ recipe: - registry dispatch：选择 `families/.py` + `methods/.py` 的组合（N+M，而非 N×M）。 - 语义更通用：未来把 finetuning 也纳入时不会出现 `distill.method=finetune` 的别扭表达。 -## 4) `models`: role-based 参与者 +## 4) `roles`: role-based 参与者 ```yaml -models: +roles: student: path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers trainable: true @@ -69,7 +69,7 @@ models: disable_custom_init_weights: true ``` -字段含义（见 `fastvideo/distillation/specs.py`）： +字段含义（见 `fastvideo/distillation/utils/config.py`）： - `family`：可选；默认继承 `recipe.family` - `path`：模型路径 / hub 名称（由 family 负责加载） - `trainable`：该 role 的参数是否参与训练（影响 `requires_grad`/train/eval） @@ -94,7 +94,7 @@ loader 会注入/补全的 invariants（见 `fastvideo/distillation/yaml_config. - `dit_precision` 默认 `fp32`（master weights） - `dit_cpu_offload = False` - 分布式尺寸默认值（`num_gpus/tp_size/sp_size/hsdp_*`） -- `training.model_path` 若缺失，默认使用 `models.student.path`（供 pipeline_config registry 使用） +- `training.model_path` 若缺失，默认使用 `roles.student.path`（供 pipeline_config registry 使用） ## 6) `pipeline_config` / `pipeline_config_path` diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index d482a910c..0cab28fbc 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -1,4 +1,4 @@ -# Phase 3：3.1 Config schema v2 + 3.2 ODE/SDE sampler + 3.3 Finetuning +# Phase 3：3.1 Config schema v2 + 3.2 ODE/SDE sampler + 3.3 Finetuning + 3.4 命名/结构整理 Phase 2.9 已完成三件关键事情（为 Phase 3 铺路）： - operation-centric adapter（adapter 不看 role string，只收 `RoleHandle`） @@ -10,6 +10,7 @@ Phase 2.9 已完成三件关键事情（为 Phase 3 铺路）： - **Phase 3.1：Config schema v2（`recipe` + `method_config`）** - **Phase 3.2：ODE/SDE sampler 可插拔（淘汰 `Pipeline`）** - **Phase 3.3：Finetuning method 接入（only student + dataset）** +- **Phase 3.4：命名/结构整理（降低概念数量 + 更直觉的目录组织）** 约束（延续前几个 phase）： - 不新增 entry file：继续使用 `fastvideo/training/distillation.py`。 @@ -32,7 +33,7 @@ recipe: family: wan method: dmd2 -models: +roles: student: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers @@ -52,7 +53,7 @@ method_config: {...} # algorithm（方法侧） ``` ### 文件 TODO（实现清单） -- [x] `fastvideo/distillation/specs.py` +- [x] `fastvideo/distillation/utils/config.py` - 新增 `RecipeSpec(family: str, method: str)` - `DistillRunConfig` 增加 `recipe` 与 `method_config` - [x] `fastvideo/distillation/yaml_config.py` @@ -130,7 +131,7 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop ### 目标（DoD） - 新增 `finetune` method，复用同一套 Trainer/Bundle/Adapter/Family/Validator 基础设施。 -- 最小可运行：只需 `models.student` + dataset 即可训练。 +- 最小可运行：只需 `roles.student` + dataset 即可训练。 - finetune 的 method 参数进入 `method_config`（与 Phase 3.1 schema 一致）。 ### 文件 TODO（实现清单） @@ -157,3 +158,106 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop - `WanAdapter.prepare_batch()` 不再读取 `training_args.simulate_generator_forward` - `DMD2Method` 通过 `method_config.rollout_mode` 决定 `latents_source={zeros|data}`， 并把它作为参数传给 adapter（adapter 只处理 batch 形态，不解释 DMD2 语义） + +--- + +## Phase 3.4：命名/结构整理（降低概念数量 + 更直觉） + +### 背景（为什么要做） + +Phase 3.1~3.3 已经把训练端到端跑通；但目前 `fastvideo/distillation/` 的概念命名偏“框架内部术语”，对新 reviewer 不友好： +- `families/` 读起来像“人类家族”，但它实际承担的是 **model/pipeline contract 的集成/装配层**。 +- `bundle.py` 读起来像“打包”，但它本质是 **roles 管理/索引容器**。 +- `registry.py` / `builder.py` /（以及一些纯 dataclass 文件）分散在多个文件，阅读路径长，容易产生“概念过多”的感受。 + +我们希望把这些改成更直觉的命名，并把“infra”从“模型集成层”里抽出来。 + +> 备注：此阶段优先做 **低风险、可 review、行为不变（或可控变更）** 的整理。 +> 若某些重排会牵动较大行为差异（例如数据加载完全抽象成独立 registry），可以拆成 3.4.x 逐步落地。 + +### 目标（DoD） + +1) **更直觉的目录命名** +- `fastvideo/distillation/families/` → `fastvideo/distillation/models/` + - 语义：这里的 “models” 指 **模型家族/管线 contract 的集成插件**（不是 YAML 的 `roles:`）。 + +2) **roles 容器命名统一** +- `fastvideo/distillation/bundle.py` → `fastvideo/distillation/roles.py` +- `ModelBundle` → `RoleManager`（或保持 `ModelBundle` 但在代码内逐步迁移到新名） + +3) **把 infra 从 models(原 families) 中解耦合** +- dataloader 构建逻辑从 `models/*` 抽到 `fastvideo/distillation/utils/`（或 `infra/`） +- tracker 初始化从 `models/*` 抽到 `trainer/entrypoint`（更符合“infra 归 infra”） +- checkpointing 相关（目前 `fastvideo/distillation/checkpoint.py`）移动到 `utils/`（或 `infra/`） + +4) **减少“文件级概念数量”** +- 已将纯 dataclass（原 `specs.py/runtime.py`）合并到 `utils/config.py`，减少“文件级概念数量” +- `registry.py + builder.py` 可以合并/重命名为更直觉的 `dispatch.py`（保留注册表与 build_runtime 的入口） + +5) **迁移策略：保证渐进、可回退** +- 保留兼容 import（re-export shim）一段时间，避免全 repo 级别大范围改动： + - `fastvideo/distillation/families/__init__.py` re-export `fastvideo/distillation/models/*` + - `fastvideo/distillation/bundle.py` re-export `fastvideo/distillation/roles.py` 的类型 +- 更新 `fastvideo/distillation/doc/` 索引与各文件说明 + +### 具体设计：如何“解耦 dataloader/tracker” + +#### Tracker +现状：tracker 在 `models/wan.py`（原 `families/wan.py`）里由 `_build_tracker()` 创建，并传给 validator。 + +Phase 3.4 目标： +- tracker 由 `fastvideo/training/distillation.py`（entrypoint）或 `DistillTrainer` 创建/持有； +- model plugin 只返回“是否需要 tracker config”（例如 raw config dict），validator 也由 method 触发调用； +- validator 构建可以延迟到 tracker 创建之后（factory/closure），避免 plugin 直接依赖 tracker。 + +#### Dataloader +现状：FastVideo 里 “数据 schema/预处理” 的差异主要来自 **任务/数据形态**， +并不严格等价于 model family（同一 family 内也可能有多种 schema）： + +- parquet 族：`fastvideo/training/training_pipeline.py` 统一走 + `build_parquet_map_style_dataloader(..., parquet_schema=..., text_padding_length=...)`。 + - T2V：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_t2v` + - I2V：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_i2v` + （额外字段如 `clip_feature`/`first_frame_latent`/`pil_image`，见 + `fastvideo/training/wan_i2v_training_pipeline.py`） + - MatrixGame：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_matrixgame` + （额外 action cond，且不使用 text embedding，见 + `fastvideo/training/matrixgame_training_pipeline.py`） + - ODE-init：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_ode_trajectory_text_only` + （trajectory latents/timesteps，见 `fastvideo/training/ode_causal_pipeline.py`） +- 非 parquet：例如 LTX2 使用 `.precomputed/*.pt` 的数据形态（见 + `fastvideo/dataset/ltx2_precomputed_dataset.py`）。 + +因此 Phase 3.4 的目标应更准确表述为：**model plugin 不负责 data plumbing**； +dataloader 由通用层基于 `DataSpec`/`dataset_kind` 构建，而 family/adapter 只负责把 +batch 转成 forward primitives 所需输入（若需要额外字段，由 `DataSpec` 显式声明）。 + +Phase 3.4 目标： +- model plugin **不直接构建 dataloader**，而是返回一个 `DataSpec`（或 `dataloader_factory`）描述： + - dataset kind（parquet/webdataset/…） + - schema/text padding length/cfg_rate 等必要参数 +- `distillation/utils/data.py`（或 `infra/data.py`）统一执行 “根据 TrainingArgs + DataSpec 构建 dataloader” + +这样做的收益：models(集成层) 文件更短、更聚焦在“加载模块 + 组装 adapter 需要的 shared context”。 + +### 文件 TODO（实现清单） + +命名/结构（行为尽量不变）： +- [x] YAML schema：顶层 `models:` → `roles:`（与 `DistillRunConfig.roles` 对齐） +- [ ] 新增 `fastvideo/distillation/models/`（拷贝/迁移原 `families/`） +- [ ] 保留 `fastvideo/distillation/families/` 作为兼容 re-export（短期） +- [ ] 新增 `fastvideo/distillation/roles.py` 并迁移 `RoleHandle/ModelBundle` +- [ ] `fastvideo/distillation/bundle.py` 变为兼容层（re-export） +- [x] `fastvideo/distillation/specs.py` + `fastvideo/distillation/runtime.py` 合并到 `fastvideo/distillation/utils/config.py` +- [ ] `fastvideo/distillation/registry.py` + `fastvideo/distillation/builder.py` 收敛为 `dispatch.py`（或最少改名） + +infra 解耦： +- [ ] 新增 `fastvideo/distillation/utils/`（或 `infra/`） + - [ ] `utils/tracking.py`：tracker 初始化（rank0 only）+ W&B run YAML 上传（如果需要） + - [ ] `utils/data.py`：dataloader 构建（基于 `DataSpec`） + - [ ] `utils/checkpoint.py`：checkpoint manager / config（从 `distillation/checkpoint.py` 迁移） +- [ ] `models/*`（原 families）移除 tracker/dataloader/checkpointing 的直接创建逻辑 +- [ ] 更新 `utils/config.py` 的 artifacts 结构（必要时引入 factory/spec 而非直接对象） + +docs： +- [ ] 更新 `fastvideo/distillation/doc/README.md` 与各文件说明（路径/命名变化） diff --git a/dev/refactor.md b/dev/refactor.md index 89485b075..a893afd59 100644 --- a/dev/refactor.md +++ b/dev/refactor.md @@ -71,7 +71,8 @@ ### B) “配置驱动加载仍需要边界” 反驳点： -- 即使 YAML 写清楚了 *路径*，加载的 *语义* 仍然是 model/pipeline 相关的： +- 即使 YAML 写清楚了 *路径*，加载/运行的 *语义* 更准确地说是 **pipeline contract 相关** + （也就近似“模型家族相关”）： - 有哪些子模块（`transformer` / `vae` / `text_encoder` / …）； - latent 的归一化/布局； - attention metadata（VSA/VMoBA）如何构建； diff --git a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index dd273fb6b..b62544d4b 100644 --- a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -9,7 +9,7 @@ distill: model: wan method: dmd2 -models: +roles: student: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index ee886d76f..804460e66 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -3,7 +3,7 @@ from __future__ import annotations from fastvideo.distillation.registry import get_family, get_method -from fastvideo.distillation.runtime import DistillRuntime +from fastvideo.distillation.utils.config import DistillRuntime from fastvideo.distillation.yaml_config import DistillRunConfig diff --git a/fastvideo/distillation/families/wan.py b/fastvideo/distillation/families/wan.py index ad5d91177..6b4cd47aa 100644 --- a/fastvideo/distillation/families/wan.py +++ b/fastvideo/distillation/families/wan.py @@ -11,7 +11,7 @@ from fastvideo.distillation.adapters.wan import WanAdapter from fastvideo.distillation.bundle import ModelBundle, RoleHandle from fastvideo.distillation.registry import register_family -from fastvideo.distillation.runtime import FamilyArtifacts +from fastvideo.distillation.utils.config import FamilyArtifacts from fastvideo.distillation.yaml_config import DistillRunConfig from fastvideo.models.loader.component_loader import PipelineComponentLoader from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index db56dac31..67ba2dbc5 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -2,7 +2,7 @@ recipe: family: wan method: dmd2 -models: +roles: student: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml index 2ecbfa501..c1541eb9b 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml @@ -2,7 +2,7 @@ recipe: family: wan method: dmd2 -models: +roles: student: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml index f3e116031..306406eb8 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml @@ -2,7 +2,7 @@ recipe: family: wan method: dmd2 -models: +roles: student: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml index f89f96ab3..f25ea195a 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml @@ -2,7 +2,7 @@ recipe: family: wan method: dmd2 -models: +roles: student: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers @@ -82,4 +82,3 @@ method_config: real_score_guidance_scale: 3.5 # Few-step schedule (single source of truth for the method). dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] - diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml index b3cab3778..7bc4a1a8b 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml @@ -2,7 +2,7 @@ recipe: family: wan method: finetune -models: +roles: student: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers @@ -62,4 +62,3 @@ pipeline_config: sampler_kind: ode method_config: {} - diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml index 74d436e39..cdede750f 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml +++ b/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml @@ -2,7 +2,7 @@ recipe: family: wan method: finetune -models: +roles: student: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers @@ -74,4 +74,3 @@ pipeline_config: method_config: # Use the VSA attention metadata when FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN. attn_kind: vsa - diff --git a/fastvideo/distillation/registry.py b/fastvideo/distillation/registry.py index 646d51ef5..04588d7b4 100644 --- a/fastvideo/distillation/registry.py +++ b/fastvideo/distillation/registry.py @@ -7,7 +7,7 @@ from fastvideo.distillation.bundle import ModelBundle from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.runtime import FamilyArtifacts +from fastvideo.distillation.utils.config import FamilyArtifacts from fastvideo.distillation.yaml_config import DistillRunConfig diff --git a/fastvideo/distillation/specs.py b/fastvideo/distillation/specs.py deleted file mode 100644 index f971d65b2..000000000 --- a/fastvideo/distillation/specs.py +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from dataclasses import dataclass - -RoleName = str - - -@dataclass(slots=True) -class RecipeSpec: - """Selects the model family + training method. - - This is intentionally small: everything else (roles, training args, and - pipeline config) lives in the run config. - """ - - family: str - method: str - - -@dataclass(slots=True) -class RoleSpec: - """Describes a role's model source and whether it should be trained.""" - - family: str - path: str - trainable: bool = True - disable_custom_init_weights: bool = False diff --git a/fastvideo/distillation/utils/__init__.py b/fastvideo/distillation/utils/__init__.py new file mode 100644 index 000000000..21b2ccb44 --- /dev/null +++ b/fastvideo/distillation/utils/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Distillation utilities shared across families/methods/entrypoints.""" + diff --git a/fastvideo/distillation/runtime.py b/fastvideo/distillation/utils/config.py similarity index 66% rename from fastvideo/distillation/runtime.py rename to fastvideo/distillation/utils/config.py index e0aa2c712..66d94676a 100644 --- a/fastvideo/distillation/runtime.py +++ b/fastvideo/distillation/utils/config.py @@ -10,6 +10,30 @@ from fastvideo.distillation.bundle import ModelBundle from fastvideo.distillation.methods.base import DistillMethod +RoleName = str + + +@dataclass(slots=True) +class RecipeSpec: + """Selects the model family + training method. + + This is intentionally small: everything else (roles, training args, and + pipeline config) lives in the run config. + """ + + family: str + method: str + + +@dataclass(slots=True) +class RoleSpec: + """Describes a role's model source and whether it should be trained.""" + + family: str + path: str + trainable: bool = True + disable_custom_init_weights: bool = False + @dataclass(slots=True) class FamilyArtifacts: @@ -38,3 +62,4 @@ class DistillRuntime: dataloader: Any tracker: Any start_step: int = 0 + diff --git a/fastvideo/distillation/yaml_config.py b/fastvideo/distillation/yaml_config.py index b1174ed29..2452a869b 100644 --- a/fastvideo/distillation/yaml_config.py +++ b/fastvideo/distillation/yaml_config.py @@ -12,7 +12,7 @@ from fastvideo.fastvideo_args import ExecutionMode, TrainingArgs from fastvideo.logger import init_logger -from fastvideo.distillation.specs import RecipeSpec, RoleName, RoleSpec +from fastvideo.distillation.utils.config import RecipeSpec, RoleName, RoleSpec logger = init_logger(__name__) @@ -88,22 +88,22 @@ def load_distill_run_config(path: str) -> DistillRunConfig: recipe_method = _require_str(recipe_raw.get("method"), where="recipe.method") recipe = RecipeSpec(family=recipe_family, method=recipe_method) - roles_raw = _require_mapping(cfg.get("models"), where="models") + roles_raw = _require_mapping(cfg.get("roles"), where="roles") roles: dict[RoleName, RoleSpec] = {} for role, role_cfg_raw in roles_raw.items(): - role_str = _require_str(role, where="models.") - role_cfg = _require_mapping(role_cfg_raw, where=f"models.{role_str}") + role_str = _require_str(role, where="roles.") + role_cfg = _require_mapping(role_cfg_raw, where=f"roles.{role_str}") family = role_cfg.get("family") or recipe_family - family = _require_str(family, where=f"models.{role_str}.family") - model_path = _require_str(role_cfg.get("path"), where=f"models.{role_str}.path") + family = _require_str(family, where=f"roles.{role_str}.family") + model_path = _require_str(role_cfg.get("path"), where=f"roles.{role_str}.path") trainable = _get_bool( role_cfg.get("trainable"), - where=f"models.{role_str}.trainable", + where=f"roles.{role_str}.trainable", default=True, ) disable_custom_init_weights = _get_bool( role_cfg.get("disable_custom_init_weights"), - where=f"models.{role_str}.disable_custom_init_weights", + where=f"roles.{role_str}.disable_custom_init_weights", default=False, ) roles[role_str] = RoleSpec( @@ -151,7 +151,7 @@ def load_distill_run_config(path: str) -> DistillRunConfig: if "model_path" not in training_kwargs: student = roles.get("student") if student is None: - raise ValueError("training.model_path is missing and models.student is not provided") + raise ValueError("training.model_path is missing and roles.student is not provided") training_kwargs["model_path"] = student.path if "pretrained_model_name_or_path" not in training_kwargs: From a7014555cec50f816a092cb469158f04b95658b8 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 19:44:25 +0000 Subject: [PATCH 097/214] better config --- dev/config.md | 2 +- dev/design.md | 24 +++--- fastvideo/distillation/doc/README.md | 4 +- .../doc/methods/fine_tuning/__init__.md | 2 +- .../doc/methods/fine_tuning/finetune.md | 2 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.md | 4 +- .../finetune_wan2.1_t2v_1.3B_phase3.3.md | 2 +- fastvideo/distillation/doc/runtime.md | 27 ------ fastvideo/distillation/doc/specs.md | 22 ----- fastvideo/distillation/doc/utils/__init__.md | 8 ++ fastvideo/distillation/doc/utils/config.md | 35 ++++++++ fastvideo/distillation/doc/yaml_config.md | 4 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 83 ------------------- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml | 83 ------------------- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml | 83 ------------------- fastvideo/distillation/utils/config.py | 11 ++- 16 files changed, 70 insertions(+), 326 deletions(-) delete mode 100644 fastvideo/distillation/doc/runtime.md delete mode 100644 fastvideo/distillation/doc/specs.md create mode 100644 fastvideo/distillation/doc/utils/__init__.md create mode 100644 fastvideo/distillation/doc/utils/config.md delete mode 100644 fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml delete mode 100644 fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml delete mode 100644 fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml diff --git a/dev/config.md b/dev/config.md index 288691c3f..2de8cfc6f 100644 --- a/dev/config.md +++ b/dev/config.md @@ -5,7 +5,7 @@ 相关实现： - YAML loader：`fastvideo/distillation/yaml_config.py` - Entrypoint：`fastvideo/training/distillation.py` -- Schema 定义：`fastvideo/distillation/specs.py` +- Schema/类型定义：`fastvideo/distillation/utils/config.py` - 示例 YAML（outside）：`fastvideo/distillation/outside/fastvideo/configs/distillation/` ## 1) 入口与约束（非常重要） diff --git a/dev/design.md b/dev/design.md index 612b1fc2a..2889abe73 100644 --- a/dev/design.md +++ b/dev/design.md @@ -1,4 +1,4 @@ -# Distill 重构设计（吸收 FastGen 架构）：`models={...}` + Method/Trainer/Adapter 解耦 +# Distill 重构设计（吸收 FastGen 架构）：`roles={...}` + Method/Trainer/Adapter 解耦 本文是基于： @@ -12,12 +12,12 @@ ## 0. TL;DR（推荐最终形态） -把 FastGen 的四层结构迁移到 FastVideo，并显式引入 `models={...}`： +把 FastGen 的四层结构迁移到 FastVideo，并显式引入 `roles={...}`： - `DistillTrainer`：只做训练基础设施（循环、分布式、grad accum、logging、ckpt、validate） - `DistillMethod`：一个“可训练对象”，封装 distill 算法 + 多角色模型 + 多优化器/交替更新 - `DistillAdapter`：把具体 pipeline/network 适配成统一的 noise/forward/CFG/cache 接口 -- `ModelBundle`：`models={student, teacher, critic, ...}` 的统一容器（含 optim/ema/fsdp 策略） +- `ModelBundle`：`roles={student, teacher, critic, ...}` 的统一容器（含 optim/ema/fsdp 策略） - `ConditioningProvider`（或 dataset 常量注入）：显式提供 `neg_condition` 等 conditioning 常量 关键原则：**Trainer 不认识 teacher/critic，也不写 DMD/SF 的 if/else。** @@ -84,7 +84,7 @@ FastGen 把 distillation 的复杂度拆成： ```text CLI/YAML config - -> build ModelBundle(models={student, teacher, critic?, ...}) + -> build ModelBundle(roles={student, teacher, critic?, ...}) -> build DistillAdapter.from_pipelines(bundle) # pipeline/network 适配 -> build DistillMethod(adapter, bundle, method_cfg) -> DistillTrainer(trainer_cfg, callbacks, checkpointer).run(method) @@ -114,7 +114,7 @@ CLI/YAML config ### 4.1 `ModelBundle`：角色显式化（外部输入） -目标：让入口层显式传入 `models={student, teacher, critic, ...}`，并把所有 +目标：让入口层显式传入 `roles={student, teacher, critic, ...}`，并把所有 “训练态（optim/ema/fsdp 策略）”结构化地挂在 role 下。 ```text @@ -253,7 +253,7 @@ FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step` - distill 的“本质复杂度”就是多网络 + 多优化器调度；放在 Method 最自然 - Trainer 只需要稳定地做基础设施，长期维护成本最低 -### 设计 2：`models={...}` 显式输入 + `ModelBundle` 结构化承载训练态 +### 设计 2：`roles={...}` 显式输入 + `ModelBundle` 结构化承载训练态 **设计** @@ -373,7 +373,7 @@ FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step` ### 6.2 复杂配置（建议支持） -- `--models_json path/to/models.json` +- `--roles_json path/to/roles.json` - per-role precision/offload/trainable/fsdp_policy/ckpt_path 等 ### 6.3 YAML schema v2（Phase 3）：`recipe` + `method_config` @@ -390,7 +390,7 @@ recipe: family: wan method: dmd2 -models: +roles: student: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers @@ -428,10 +428,10 @@ method_config: **解析策略（最优雅且低风险）** - 新入口的 parser 只保留 `--config run.yaml`（以及少量 meta flags，如 `--dry-run`）。 -- 训练相关的所有参数（TrainingArgs/FastVideoArgs/pipeline_config/method/models）都来自 YAML。 +- 训练相关的所有参数（TrainingArgs/FastVideoArgs/pipeline_config/method/roles）都来自 YAML。 - 解析流程： 1) `yaml.safe_load` 得到 dict - 2) 规范化/校验 schema（recipe/models/training/pipeline_config/method_config/...） + 2) 规范化/校验 schema（recipe/roles/training/pipeline_config/method_config/...） 3) 将 `training:` 与 `pipeline_config:` 合成 kwargs，调用 `TrainingArgs.from_kwargs(**kwargs)` （由现有 PipelineConfig/PreprocessConfig 负责子配置实例化与校验） @@ -635,7 +635,7 @@ Phase 3.1 计划把 YAML schema 升级为： ```yaml recipe: {family: wan, method: dmd2} # 只负责 “选什么” -models: {student: ..., teacher: ...} # 参与者 +roles: {student: ..., teacher: ...} # 参与者 training: {...} # infra 参数（映射到 TrainingArgs） pipeline_config: {...} # backbone/pipeline config（模型侧） method_config: {...} # algorithm/method 超参（方法侧） @@ -679,7 +679,7 @@ Phase 3.1 的 `recipe/method_config` 对齐。 Finetune 的 config（示意，schema v2）： ```yaml recipe: {family: wan, method: finetune} -models: +roles: student: family: wan path: ... diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index d0b7a938d..88760c318 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -21,10 +21,10 @@ ### Core - `__init__.md` - `yaml_config.md` -- `specs.md` +- `utils/__init__.md` +- `utils/config.md` - `registry.md` - `builder.md` -- `runtime.md` - `bundle.md` - `trainer.md` - `checkpoint.md` diff --git a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md index aaa6fb646..3397ad738 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md @@ -5,7 +5,7 @@ **当前实现** - `finetune.py`：`FineTuneMethod` - - 只要求 `models.student` + - 只要求 `roles.student` - loss/policy 在 method 层 - 复用同一套 trainer/bundle/adapter/family/validator/checkpoint 基础设施 diff --git a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md index 1b6ebe61c..c4a03c16b 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md @@ -13,7 +13,7 @@ finetune 可以被视为一种特殊的 distillation recipe：**只有 student + - 不需要：`teacher / critic / reward / ...` 方法会强制： -- `models.student.trainable=true` +- `roles.student.trainable=true` ## 核心训练逻辑 `FineTuneMethod.single_train_step()`： diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md index 421e2ccaf..797d75022 100644 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md @@ -9,7 +9,7 @@ - `recipe:` - `family: wan` → registry dispatch 到 `families/wan.py` - `method: dmd2` → registry dispatch 到 `methods/distribution_matching/dmd2.py` -- `models:` +- `roles:` - `student / teacher / critic` 三个 roles（role 名称本身由 method 解释语义） - 每个 role 指定： - `family`（默认可省略，继承 `recipe.family`） @@ -26,7 +26,7 @@ ## 关键语义归属（Phase 2.9 视角） **Family（Wan）关心：** -- `models.*.path / trainable` +- `roles.*.path / trainable` - `training.data_path / dataloader_num_workers / train_batch_size / seed / output_dir` - Wan 相关的 shape 信息（`num_latent_t/num_height/num_width/...`） diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md index 569136c33..275ffdbd6 100644 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md @@ -4,6 +4,6 @@ 关键点： - `recipe.method = finetune` -- `models` 里只提供 `student`（no teacher/critic） +- `roles` 里只提供 `student`（no teacher/critic） - 训练 loss 由 `FineTuneMethod` 实现（与 legacy `training_pipeline.py` 的目标对齐） - validation 通过 `ValidationRequest + WanValidator + WanPipeline` 执行（默认走 `ode` sampler） diff --git a/fastvideo/distillation/doc/runtime.md b/fastvideo/distillation/doc/runtime.md deleted file mode 100644 index 6e33ffe80..000000000 --- a/fastvideo/distillation/doc/runtime.md +++ /dev/null @@ -1,27 +0,0 @@ -# `fastvideo/distillation/runtime.py` - -**目的** -- 定义 distillation builder 的结构化输出类型，明确 build-time 与 run-time 的边界。 - -**关键类型** -- `FamilyArtifacts` - - build-time family 插件的产物集合： - - `training_args` - - `bundle` - - `adapter` - - `dataloader` - - `tracker` - - `validator`（可选；family-specific） - - `start_step`（用于 resume / warm-start） -- `DistillRuntime` - - `DistillTrainer.run()` 所需的最小集合： - - `training_args` - - `method`（`DistillMethod`） - - `dataloader` - - `tracker` - - `start_step` - -**设计意图** -- family 负责把 “零件” 装配成 `FamilyArtifacts` -- method 负责把算法绑定到（bundle + adapter）上 -- trainer 只接收 `method` 并开始训练 diff --git a/fastvideo/distillation/doc/specs.md b/fastvideo/distillation/doc/specs.md deleted file mode 100644 index 2c9694322..000000000 --- a/fastvideo/distillation/doc/specs.md +++ /dev/null @@ -1,22 +0,0 @@ -# `fastvideo/distillation/specs.py` - -**目的** -- 把 config 里的“选择项”做成轻量 dataclass，便于： - - YAML 解析（`yaml_config.py`） - - builder/registry dispatch（`builder.py` / `registry.py`） - -**关键类型** -- `RecipeSpec` - - `family`: family 名称（例如 `"wan"`） - - `method`: method 名称（例如 `"dmd2"`） -- `RoleSpec` - - `family`: 该 role 的 family（默认可继承 `recipe.family`） - - `path`: 模型权重路径（HF repo 或本地目录） - - `trainable`: 是否训练该 role（只影响 `requires_grad`/模式；具体 optimizer 由 method 决定） - - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” - - 这是一个 build-time/loader 语义开关（用于 teacher/critic 等 auxiliary roles） - - 角色名不应在 family 内被 hard-code；由 YAML 显式声明更清晰 - -**注意** -- role 名称本身（`student/teacher/critic/...`）是字符串。 - framework 不强行规定“canonical roles”，由 method 决定语义与依赖。 diff --git a/fastvideo/distillation/doc/utils/__init__.md b/fastvideo/distillation/doc/utils/__init__.md new file mode 100644 index 000000000..40fbb80b8 --- /dev/null +++ b/fastvideo/distillation/doc/utils/__init__.md @@ -0,0 +1,8 @@ +# `fastvideo/distillation/utils/` + +**目的** +- 放置 distillation 子系统的中性工具代码（不属于某个 family/method）。 + +当前包含： +- `config.py`：若干轻量 dataclass（schema 选择项 / build-time artifacts / runtime 组装结果）。 + diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md new file mode 100644 index 000000000..2bf506129 --- /dev/null +++ b/fastvideo/distillation/doc/utils/config.md @@ -0,0 +1,35 @@ +# `fastvideo/distillation/utils/config.py` + +**目的** +- 把 distillation 里常用的“结构化类型”集中在一个更直觉的位置，减少文件级概念数量。 + +这里包含两类 dataclass： + +## 1) Schema / run config 相关（轻量选择项） +- `RecipeSpec` + - `family`: family 名称（例如 `"wan"`） + - `method`: method 名称（例如 `"dmd2"`） +- `RoleSpec` + - `family`: 该 role 的 family（默认可继承 `recipe.family`） + - `path`: 模型权重路径（HF repo 或本地目录） + - `trainable`: 是否训练该 role（只影响 `requires_grad`/模式；具体 optimizer 由 method 决定） + - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” + +## 2) Builder 装配相关（build-time / run-time 边界） +- `FamilyArtifacts` + - family 插件 build-time 的产物集合： + - `training_args` + - `bundle` + - `adapter` + - `dataloader` + - `tracker` + - `validator`（可选；family-specific） + - `start_step`（用于 resume / warm-start） +- `DistillRuntime` + - `DistillTrainer.run()` 所需的最小集合： + - `training_args` + - `method`（`DistillMethod`） + - `dataloader` + - `tracker` + - `start_step` + diff --git a/fastvideo/distillation/doc/yaml_config.md b/fastvideo/distillation/doc/yaml_config.md index 679bcb9e6..00444b770 100644 --- a/fastvideo/distillation/doc/yaml_config.md +++ b/fastvideo/distillation/doc/yaml_config.md @@ -6,7 +6,7 @@ **核心产物** - `DistillRunConfig` - `recipe: RecipeSpec`（选择 family + method） - - `roles: dict[str, RoleSpec]`（来自 YAML 的 `models:`） + - `roles: dict[str, RoleSpec]`（来自 YAML 的 `roles:`） - `training_args: TrainingArgs`（来自 YAML 的 `training:`，并注入 entrypoint invariants） - `method_config: dict`（来自 YAML 的 `method_config:`，传给 method 解释） - `raw: dict`（原始 YAML，便于 tracker 记录） @@ -15,7 +15,7 @@ **YAML 结构（schema v2）** - `recipe: {family: ..., method: ...}` -- `models: {: {family?, path, trainable?}, ...}` +- `roles: {: {family?, path, trainable?}, ...}` - `training: {...}`（大部分字段复用 `TrainingArgs.from_kwargs()`） - `pipeline_config` 或 `pipeline_config_path` - `method_config: {...}`（算法/recipe 专属超参） diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml deleted file mode 100644 index 67ba2dbc5..000000000 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ /dev/null @@ -1,83 +0,0 @@ -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase2_wan2.1_dmd2_8steps_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Distillation - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: phase2_wan_dmd2_8steps_wansyn - wandb_run_name: phase2_wan_dmd2_8steps_wansyn - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "8" - validation_guidance_scale: "6.0" - -pipeline_config: - flow_shift: 8 - -method_config: - # Replace legacy `training.simulate_generator_forward`. - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - # Few-step schedule (kept here as the method single source of truth). - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml deleted file mode 100644 index c1541eb9b..000000000 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml +++ /dev/null @@ -1,83 +0,0 @@ -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase2_wan2.1_dmd2_8steps_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Distillation - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: phase2.9_wan_dmd2_8steps_wansyn - wandb_run_name: phase2.9_wan_dmd2_8steps_wansyn - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "8" - validation_guidance_scale: "6.0" - -pipeline_config: - flow_shift: 8 - -method_config: - # Replace legacy `training.simulate_generator_forward`. - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - # Few-step schedule (kept here as the method single source of truth). - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml b/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml deleted file mode 100644 index 306406eb8..000000000 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml +++ /dev/null @@ -1,83 +0,0 @@ -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.1_wan2.1_dmd2_8steps_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Distillation (method-agnostic knobs) - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: phase3.1_wan_dmd2_8steps_wansyn - wandb_run_name: phase3.1_wan_dmd2_8steps_wansyn - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "8" - validation_guidance_scale: "6.0" - -pipeline_config: - flow_shift: 8 - -method_config: - # Replace legacy `training.simulate_generator_forward`. - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - # Few-step schedule (single source of truth for the method). - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index 66d94676a..74b51cb22 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -3,12 +3,12 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any +from typing import Any, TYPE_CHECKING -from fastvideo.fastvideo_args import TrainingArgs - -from fastvideo.distillation.bundle import ModelBundle -from fastvideo.distillation.methods.base import DistillMethod +if TYPE_CHECKING: + from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.distillation.bundle import ModelBundle + from fastvideo.distillation.methods.base import DistillMethod RoleName = str @@ -62,4 +62,3 @@ class DistillRuntime: dataloader: Any tracker: Any start_step: int = 0 - From 63c5985431ad6421472052b1217629b17ad4add8 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 20:06:58 +0000 Subject: [PATCH 098/214] merge yaml_config.py into utils/config.py --- dev/config.md | 4 +- dev/phases/phase_2.md | 4 +- dev/phases/phase_3.md | 4 +- fastvideo/distillation/builder.py | 2 +- fastvideo/distillation/doc/README.md | 3 +- fastvideo/distillation/doc/utils/__init__.md | 3 +- fastvideo/distillation/doc/utils/config.md | 42 +++- fastvideo/distillation/doc/yaml_config.md | 36 ---- fastvideo/distillation/families/wan.py | 2 +- .../methods/distribution_matching/dmd2.py | 2 +- .../methods/fine_tuning/finetune.py | 2 +- fastvideo/distillation/registry.py | 2 +- fastvideo/distillation/utils/config.py | 165 ++++++++++++++++ fastvideo/distillation/yaml_config.py | 179 ------------------ fastvideo/training/distillation.py | 2 +- 15 files changed, 217 insertions(+), 235 deletions(-) delete mode 100644 fastvideo/distillation/doc/yaml_config.md delete mode 100644 fastvideo/distillation/yaml_config.py diff --git a/dev/config.md b/dev/config.md index 2de8cfc6f..286c50381 100644 --- a/dev/config.md +++ b/dev/config.md @@ -3,7 +3,7 @@ 本文档描述当前 distillation 入口所使用的 **YAML schema v2**、字段含义与设计取舍。 相关实现： -- YAML loader：`fastvideo/distillation/yaml_config.py` +- YAML loader：`fastvideo/distillation/utils/config.py::load_distill_run_config` - Entrypoint：`fastvideo/training/distillation.py` - Schema/类型定义：`fastvideo/distillation/utils/config.py` - 示例 YAML（outside）：`fastvideo/distillation/outside/fastvideo/configs/distillation/` @@ -88,7 +88,7 @@ roles: - 优化器默认值：`learning_rate`, `betas`, `lr_scheduler`, ... - tracking/validation：`log_validation`, `validation_*`, `tracker_project_name`, ... -loader 会注入/补全的 invariants（见 `fastvideo/distillation/yaml_config.py`）： +loader 会注入/补全的 invariants（见 `fastvideo/distillation/utils/config.py`）： - `mode = ExecutionMode.DISTILLATION` - `inference_mode = False` - `dit_precision` 默认 `fp32`（master weights） diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md index b0a21160c..d78a0094c 100644 --- a/dev/phases/phase_2.md +++ b/dev/phases/phase_2.md @@ -71,7 +71,7 @@ distillation pipeline**（`fastvideo/training/*distillation_pipeline.py`）。 - [x] 定义结构化 spec（角色驱动）：`DistillSpec / RoleSpec` - 目标：`models={role -> spec}` 成为唯一真相 - method 自己声明需要哪些 roles（缺失则报错） -- [x] 新增 YAML 配置解析：`fastvideo/distillation/yaml_config.py::load_distill_run_config` +- [x] 新增 YAML 配置解析：`fastvideo/distillation/utils/config.py::load_distill_run_config` - `yaml.safe_load` + 最小 schema 校验（不做 legacy CLI merge） - schema：`distill + models + training + (pipeline_config|pipeline_config_path)` - [x] 修改入口：`fastvideo/training/distillation.py` @@ -162,7 +162,7 @@ distillation pipeline**（`fastvideo/training/*distillation_pipeline.py`）。 建议新增： -- `fastvideo/distillation/specs.py` +- `fastvideo/distillation/utils/config.py`（后续实现时收敛到该文件） - `ModelSpec`：`family/path/revision/precision/...` - `RoleSpec`：`role/trainable/optimizer/scheduler/...` - `DistillSpec`：`method + models{role->RoleSpec} + adapter_family(optional)` diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 0cab28fbc..cea9041c8 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -56,7 +56,7 @@ method_config: {...} # algorithm（方法侧） - [x] `fastvideo/distillation/utils/config.py` - 新增 `RecipeSpec(family: str, method: str)` - `DistillRunConfig` 增加 `recipe` 与 `method_config` -- [x] `fastvideo/distillation/yaml_config.py` +- [x] `fastvideo/distillation/utils/config.py`（包含 YAML loader + schema/dataclass） - 解析 `recipe:` 与 `method_config:`（默认 `{}`） - 将 v1 的 `distill:` 视为不再支持（breaking change，直接推进 schema v2） - [x] `fastvideo/distillation/builder.py` @@ -192,6 +192,7 @@ Phase 3.1~3.3 已经把训练端到端跑通；但目前 `fastvideo/distillation 4) **减少“文件级概念数量”** - 已将纯 dataclass（原 `specs.py/runtime.py`）合并到 `utils/config.py`，减少“文件级概念数量” +- 已将 YAML loader（原 `yaml_config.py`）合并到 `utils/config.py`（schema+解析逻辑同处） - `registry.py + builder.py` 可以合并/重命名为更直觉的 `dispatch.py`（保留注册表与 build_runtime 的入口） 5) **迁移策略：保证渐进、可回退** @@ -244,6 +245,7 @@ Phase 3.4 目标： 命名/结构（行为尽量不变）： - [x] YAML schema：顶层 `models:` → `roles:`（与 `DistillRunConfig.roles` 对齐） +- [x] YAML loader：`fastvideo/distillation/yaml_config.py` → `fastvideo/distillation/utils/config.py` - [ ] 新增 `fastvideo/distillation/models/`（拷贝/迁移原 `families/`） - [ ] 保留 `fastvideo/distillation/families/` 作为兼容 re-export（短期） - [ ] 新增 `fastvideo/distillation/roles.py` 并迁移 `RoleHandle/ModelBundle` diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index 804460e66..59b57837e 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -4,7 +4,7 @@ from fastvideo.distillation.registry import get_family, get_method from fastvideo.distillation.utils.config import DistillRuntime -from fastvideo.distillation.yaml_config import DistillRunConfig +from fastvideo.distillation.utils.config import DistillRunConfig def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index 88760c318..85e6857eb 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -10,7 +10,7 @@ API 以 operation 为中心，不以 role 为中心（避免 role 爆炸）。 快速入口（从运行到训练）： -`fastvideo/training/distillation.py` → `yaml_config.load_distill_run_config()` → +`fastvideo/training/distillation.py` → `utils.config.load_distill_run_config()` → `builder.build_runtime_from_config()` → `registry.get_family()/get_method()` → `FamilyArtifacts + DistillMethod` → `DistillTrainer.run()` @@ -20,7 +20,6 @@ ### Core - `__init__.md` -- `yaml_config.md` - `utils/__init__.md` - `utils/config.md` - `registry.md` diff --git a/fastvideo/distillation/doc/utils/__init__.md b/fastvideo/distillation/doc/utils/__init__.md index 40fbb80b8..d768bfe6e 100644 --- a/fastvideo/distillation/doc/utils/__init__.md +++ b/fastvideo/distillation/doc/utils/__init__.md @@ -4,5 +4,4 @@ - 放置 distillation 子系统的中性工具代码（不属于某个 family/method）。 当前包含： -- `config.py`：若干轻量 dataclass（schema 选择项 / build-time artifacts / runtime 组装结果）。 - +- `config.py`：YAML loader + schema/types + build-time artifacts + runtime 组装结果。 diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index 2bf506129..923584804 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -1,11 +1,44 @@ # `fastvideo/distillation/utils/config.py` **目的** -- 把 distillation 里常用的“结构化类型”集中在一个更直觉的位置，减少文件级概念数量。 +- 把 distillation 的 **YAML loader + schema/types + runtime 装配类型** 集中在一个更直觉的位置， + 减少文件级概念数量。 -这里包含两类 dataclass： +这里包含： -## 1) Schema / run config 相关（轻量选择项） +## 1) YAML loader（schema v2；YAML-only） + +**核心 API** +- `load_distill_run_config(path) -> DistillRunConfig` + +**核心产物** +- `DistillRunConfig` + - `recipe: RecipeSpec`（选择 family + method） + - `roles: dict[str, RoleSpec]`（来自 YAML 的 `roles:`） + - `training_args: TrainingArgs`（来自 YAML 的 `training:`，并注入 entrypoint invariants） + - `method_config: dict`（来自 YAML 的 `method_config:`，传给 method 解释） + - `raw: dict`（原始 YAML，便于 tracker 记录/复现） + +**YAML 结构（schema v2）** +- `recipe: {family: ..., method: ...}` +- `roles: {: {family?, path, trainable?}, ...}` +- `training: {...}`（大部分字段复用 `TrainingArgs.from_kwargs()`） +- `pipeline_config` 或 `pipeline_config_path` +- `method_config: {...}`（算法/recipe 专属超参） + +**实现要点** +- `_resolve_existing_file()`：要求传入真实存在的路径（不做 overlay/fallback） +- 默认分布式 size： + - `num_gpus` 默认 1 + - `tp_size` 默认 1 + - `sp_size` 默认 `num_gpus`（保持与现有 pipeline 的期望一致） +- 训练模式 invariants（由入口强制注入）： + - `mode = DISTILLATION` + - `inference_mode = False` + - `dit_precision` 默认 `fp32`（master weights） + - `dit_cpu_offload = False` + +## 2) Schema / run config 相关（轻量选择项） - `RecipeSpec` - `family`: family 名称（例如 `"wan"`） - `method`: method 名称（例如 `"dmd2"`） @@ -15,7 +48,7 @@ - `trainable`: 是否训练该 role（只影响 `requires_grad`/模式；具体 optimizer 由 method 决定） - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” -## 2) Builder 装配相关（build-time / run-time 边界） +## 3) Builder 装配相关（build-time / run-time 边界） - `FamilyArtifacts` - family 插件 build-time 的产物集合： - `training_args` @@ -32,4 +65,3 @@ - `dataloader` - `tracker` - `start_step` - diff --git a/fastvideo/distillation/doc/yaml_config.md b/fastvideo/distillation/doc/yaml_config.md deleted file mode 100644 index 00444b770..000000000 --- a/fastvideo/distillation/doc/yaml_config.md +++ /dev/null @@ -1,36 +0,0 @@ -# `fastvideo/distillation/yaml_config.py` - -**目的** -- distillation 的 YAML-only 配置加载器（schema v2；不兼容/不 merge legacy CLI）。 - -**核心产物** -- `DistillRunConfig` - - `recipe: RecipeSpec`（选择 family + method） - - `roles: dict[str, RoleSpec]`（来自 YAML 的 `roles:`） - - `training_args: TrainingArgs`（来自 YAML 的 `training:`，并注入 entrypoint invariants） - - `method_config: dict`（来自 YAML 的 `method_config:`，传给 method 解释） - - `raw: dict`（原始 YAML，便于 tracker 记录） - - `wan` family 默认会把 `raw` 作为 W&B config 传给 `wandb.init(config=...)` - - 入口还会把 YAML 文件本身以 `run.yaml` 的形式上传到 tracker（如 W&B Files） - -**YAML 结构（schema v2）** -- `recipe: {family: ..., method: ...}` -- `roles: {: {family?, path, trainable?}, ...}` -- `training: {...}`（大部分字段复用 `TrainingArgs.from_kwargs()`） -- `pipeline_config` 或 `pipeline_config_path` -- `method_config: {...}`（算法/recipe 专属超参） - -**实现要点** -- `_resolve_existing_file()`：要求传入真实存在的路径（Phase 2 不做 overlay/fallback） -- 默认分布式 size： - - `num_gpus` 默认 1 - - `tp_size` 默认 1 - - `sp_size` 默认 `num_gpus`（保持与现有 pipeline 的期望一致） -- 训练模式 invariants（由入口强制注入）： - - `mode = DISTILLATION` - - `inference_mode = False` - - `dit_precision` 默认 `fp32`（master weights） - - `dit_cpu_offload = False` - -**兼容性** -- loader 仅接受 schema v2：缺少 `recipe:` 会直接报错。 diff --git a/fastvideo/distillation/families/wan.py b/fastvideo/distillation/families/wan.py index 6b4cd47aa..fbf49d629 100644 --- a/fastvideo/distillation/families/wan.py +++ b/fastvideo/distillation/families/wan.py @@ -12,7 +12,7 @@ from fastvideo.distillation.bundle import ModelBundle, RoleHandle from fastvideo.distillation.registry import register_family from fastvideo.distillation.utils.config import FamilyArtifacts -from fastvideo.distillation.yaml_config import DistillRunConfig +from fastvideo.distillation.utils.config import DistillRunConfig from fastvideo.models.loader.component_loader import PipelineComponentLoader from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( FlowMatchEulerDiscreteScheduler, diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 84cbfbaff..3abc2d119 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -17,7 +17,7 @@ from fastvideo.distillation.methods.base import DistillMethod from fastvideo.distillation.registry import register_method from fastvideo.distillation.validators.base import ValidationRequest -from fastvideo.distillation.yaml_config import DistillRunConfig +from fastvideo.distillation.utils.config import DistillRunConfig class _DMD2Adapter(Protocol): diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 1ec224e0f..e47f4353c 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -16,7 +16,7 @@ from fastvideo.distillation.methods.base import DistillMethod from fastvideo.distillation.registry import register_method from fastvideo.distillation.validators.base import ValidationRequest -from fastvideo.distillation.yaml_config import DistillRunConfig +from fastvideo.distillation.utils.config import DistillRunConfig class _FineTuneAdapter(Protocol): diff --git a/fastvideo/distillation/registry.py b/fastvideo/distillation/registry.py index 04588d7b4..1951dcebd 100644 --- a/fastvideo/distillation/registry.py +++ b/fastvideo/distillation/registry.py @@ -8,7 +8,7 @@ from fastvideo.distillation.bundle import ModelBundle from fastvideo.distillation.methods.base import DistillMethod from fastvideo.distillation.utils.config import FamilyArtifacts -from fastvideo.distillation.yaml_config import DistillRunConfig +from fastvideo.distillation.utils.config import DistillRunConfig class FamilyBuilder(Protocol): diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index 74b51cb22..eddb44bf8 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -2,9 +2,13 @@ from __future__ import annotations +import os from dataclasses import dataclass +from pathlib import Path from typing import Any, TYPE_CHECKING +import yaml + if TYPE_CHECKING: from fastvideo.fastvideo_args import TrainingArgs from fastvideo.distillation.bundle import ModelBundle @@ -35,6 +39,167 @@ class RoleSpec: disable_custom_init_weights: bool = False +@dataclass(slots=True) +class DistillRunConfig: + """Parsed distillation run config loaded from schema-v2 YAML.""" + + recipe: RecipeSpec + roles: dict[RoleName, RoleSpec] + training_args: TrainingArgs + method_config: dict[str, Any] + raw: dict[str, Any] + + +def _resolve_existing_file(path: str) -> str: + """Resolve a user-provided config path and require it exists. + + Distillation intentionally does not perform any "overlay" path rewriting. + The caller must pass a real file path (typically under + ``fastvideo/distillation/outside/``). + """ + + if not path: + return path + + expanded = os.path.expanduser(path) + resolved = Path(expanded).resolve() + if not resolved.exists(): + raise FileNotFoundError(f"Config file not found: {resolved}") + if not resolved.is_file(): + raise ValueError(f"Expected a file path, got: {resolved}") + return str(resolved) + + +def _require_mapping(raw: Any, *, where: str) -> dict[str, Any]: + if not isinstance(raw, dict): + raise ValueError(f"Expected mapping at {where}, got {type(raw).__name__}") + return raw + + +def _require_str(raw: Any, *, where: str) -> str: + if not isinstance(raw, str) or not raw.strip(): + raise ValueError(f"Expected non-empty string at {where}") + return raw + + +def _get_bool(raw: Any, *, where: str, default: bool) -> bool: + if raw is None: + return default + if isinstance(raw, bool): + return raw + raise ValueError(f"Expected bool at {where}, got {type(raw).__name__}") + + +def load_distill_run_config(path: str) -> DistillRunConfig: + """Load a distillation run config from schema-v2 YAML. + + This loader intentionally does **not** merge with legacy CLI args. The YAML + file is the single source of truth for a run. + """ + + from fastvideo.fastvideo_args import ExecutionMode, TrainingArgs + + path = _resolve_existing_file(path) + with open(path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + cfg = _require_mapping(raw, where=path) + + recipe_raw = _require_mapping(cfg.get("recipe"), where="recipe") + recipe_family = _require_str(recipe_raw.get("family"), where="recipe.family") + recipe_method = _require_str(recipe_raw.get("method"), where="recipe.method") + recipe = RecipeSpec(family=recipe_family, method=recipe_method) + + roles_raw = _require_mapping(cfg.get("roles"), where="roles") + roles: dict[RoleName, RoleSpec] = {} + for role, role_cfg_raw in roles_raw.items(): + role_str = _require_str(role, where="roles.") + role_cfg = _require_mapping(role_cfg_raw, where=f"roles.{role_str}") + family = role_cfg.get("family") or recipe_family + family = _require_str(family, where=f"roles.{role_str}.family") + model_path = _require_str(role_cfg.get("path"), where=f"roles.{role_str}.path") + trainable = _get_bool( + role_cfg.get("trainable"), + where=f"roles.{role_str}.trainable", + default=True, + ) + disable_custom_init_weights = _get_bool( + role_cfg.get("disable_custom_init_weights"), + where=f"roles.{role_str}.disable_custom_init_weights", + default=False, + ) + roles[role_str] = RoleSpec( + family=family, + path=model_path, + trainable=trainable, + disable_custom_init_weights=disable_custom_init_weights, + ) + + training_raw = _require_mapping(cfg.get("training"), where="training") + + method_config_raw = cfg.get("method_config", None) + if method_config_raw is None: + method_config: dict[str, Any] = {} + else: + method_config = _require_mapping(method_config_raw, where="method_config") + + pipeline_cfg_raw = cfg.get("pipeline_config", None) + pipeline_cfg_path = cfg.get("pipeline_config_path", None) + if pipeline_cfg_raw is not None and pipeline_cfg_path is not None: + raise ValueError("Provide either pipeline_config or pipeline_config_path, not both") + + training_kwargs: dict[str, Any] = dict(training_raw) + + # Entrypoint invariants. + training_kwargs["mode"] = ExecutionMode.DISTILLATION + training_kwargs["inference_mode"] = False + # Match the training-mode loader behavior in `ComposedPipelineBase`: + # training uses fp32 master weights and should not CPU-offload DiT weights. + training_kwargs.setdefault("dit_precision", "fp32") + training_kwargs["dit_cpu_offload"] = False + + # Default distributed sizes. These must be set *before* TrainingArgs + # construction because `check_fastvideo_args()` asserts they are not -1 in + # training mode. + num_gpus = int(training_kwargs.get("num_gpus", 1) or 1) + training_kwargs.setdefault("num_gpus", num_gpus) + training_kwargs.setdefault("tp_size", 1) + training_kwargs.setdefault("sp_size", num_gpus) + training_kwargs.setdefault("hsdp_replicate_dim", 1) + training_kwargs.setdefault("hsdp_shard_dim", num_gpus) + + # Use student path as the default base model_path. This is needed for + # PipelineConfig registry lookup. + if "model_path" not in training_kwargs: + student = roles.get("student") + if student is None: + raise ValueError("training.model_path is missing and roles.student is not provided") + training_kwargs["model_path"] = student.path + + if "pretrained_model_name_or_path" not in training_kwargs: + training_kwargs["pretrained_model_name_or_path"] = training_kwargs["model_path"] + + if pipeline_cfg_path is not None: + pipeline_cfg_path = _require_str(pipeline_cfg_path, where="pipeline_config_path") + training_kwargs["pipeline_config"] = _resolve_existing_file(pipeline_cfg_path) + elif pipeline_cfg_raw is not None: + if isinstance(pipeline_cfg_raw, str): + training_kwargs["pipeline_config"] = _resolve_existing_file(pipeline_cfg_raw) + elif isinstance(pipeline_cfg_raw, dict): + training_kwargs["pipeline_config"] = pipeline_cfg_raw + else: + raise ValueError("pipeline_config must be a mapping or a path string") + + training_args = TrainingArgs.from_kwargs(**training_kwargs) + + return DistillRunConfig( + recipe=recipe, + roles=roles, + training_args=training_args, + method_config=method_config, + raw=cfg, + ) + + @dataclass(slots=True) class FamilyArtifacts: """Build-time outputs produced by a model family plugin. diff --git a/fastvideo/distillation/yaml_config.py b/fastvideo/distillation/yaml_config.py deleted file mode 100644 index 2452a869b..000000000 --- a/fastvideo/distillation/yaml_config.py +++ /dev/null @@ -1,179 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import os -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import yaml - -from fastvideo.fastvideo_args import ExecutionMode, TrainingArgs -from fastvideo.logger import init_logger - -from fastvideo.distillation.utils.config import RecipeSpec, RoleName, RoleSpec - -logger = init_logger(__name__) - - -@dataclass(slots=True) -class DistillRunConfig: - recipe: RecipeSpec - roles: dict[RoleName, RoleSpec] - training_args: TrainingArgs - method_config: dict[str, Any] - raw: dict[str, Any] - - -def _distillation_root() -> Path: - # .../fastvideo/distillation/yaml_config.py -> .../fastvideo/distillation - return Path(__file__).resolve().parent - - -def _resolve_existing_file(path: str) -> str: - """Resolve a user-provided config path and require it exists. - - Phase 2 intentionally does not perform any "overlay" path rewriting. The - caller must pass the real path (typically under - ``fastvideo/distillation/outside/``). - """ - - if not path: - return path - - expanded = os.path.expanduser(path) - resolved = Path(expanded).resolve() - if not resolved.exists(): - raise FileNotFoundError(f"Config file not found: {resolved}") - if not resolved.is_file(): - raise ValueError(f"Expected a file path, got: {resolved}") - return str(resolved) - - -def _require_mapping(raw: Any, *, where: str) -> dict[str, Any]: - if not isinstance(raw, dict): - raise ValueError(f"Expected mapping at {where}, got {type(raw).__name__}") - return raw - - -def _require_str(raw: Any, *, where: str) -> str: - if not isinstance(raw, str) or not raw.strip(): - raise ValueError(f"Expected non-empty string at {where}") - return raw - - -def _get_bool(raw: Any, *, where: str, default: bool) -> bool: - if raw is None: - return default - if isinstance(raw, bool): - return raw - raise ValueError(f"Expected bool at {where}, got {type(raw).__name__}") - - -def load_distill_run_config(path: str) -> DistillRunConfig: - """Load a distillation run config from YAML (schema v2). - - This loader intentionally does **not** merge with legacy CLI args. The YAML - file is the single source of truth for a run. - """ - - path = _resolve_existing_file(path) - with open(path, encoding="utf-8") as f: - raw = yaml.safe_load(f) - cfg = _require_mapping(raw, where=path) - - recipe_raw = _require_mapping(cfg.get("recipe"), where="recipe") - recipe_family = _require_str(recipe_raw.get("family"), where="recipe.family") - recipe_method = _require_str(recipe_raw.get("method"), where="recipe.method") - recipe = RecipeSpec(family=recipe_family, method=recipe_method) - - roles_raw = _require_mapping(cfg.get("roles"), where="roles") - roles: dict[RoleName, RoleSpec] = {} - for role, role_cfg_raw in roles_raw.items(): - role_str = _require_str(role, where="roles.") - role_cfg = _require_mapping(role_cfg_raw, where=f"roles.{role_str}") - family = role_cfg.get("family") or recipe_family - family = _require_str(family, where=f"roles.{role_str}.family") - model_path = _require_str(role_cfg.get("path"), where=f"roles.{role_str}.path") - trainable = _get_bool( - role_cfg.get("trainable"), - where=f"roles.{role_str}.trainable", - default=True, - ) - disable_custom_init_weights = _get_bool( - role_cfg.get("disable_custom_init_weights"), - where=f"roles.{role_str}.disable_custom_init_weights", - default=False, - ) - roles[role_str] = RoleSpec( - family=family, - path=model_path, - trainable=trainable, - disable_custom_init_weights=disable_custom_init_weights, - ) - - training_raw = _require_mapping(cfg.get("training"), where="training") - - method_config_raw = cfg.get("method_config", None) - if method_config_raw is None: - method_config: dict[str, Any] = {} - else: - method_config = _require_mapping(method_config_raw, where="method_config") - - pipeline_cfg_raw = cfg.get("pipeline_config", None) - pipeline_cfg_path = cfg.get("pipeline_config_path", None) - if pipeline_cfg_raw is not None and pipeline_cfg_path is not None: - raise ValueError("Provide either pipeline_config or pipeline_config_path, not both") - - training_kwargs: dict[str, Any] = dict(training_raw) - - # Entrypoint invariants. - training_kwargs["mode"] = ExecutionMode.DISTILLATION - training_kwargs["inference_mode"] = False - # Match the training-mode loader behavior in `ComposedPipelineBase`: - # training uses fp32 master weights and should not CPU-offload DiT weights. - training_kwargs.setdefault("dit_precision", "fp32") - training_kwargs["dit_cpu_offload"] = False - - # Default distributed sizes. These must be set *before* TrainingArgs - # construction because `check_fastvideo_args()` asserts they are not -1 in - # training mode. - num_gpus = int(training_kwargs.get("num_gpus", 1) or 1) - training_kwargs.setdefault("num_gpus", num_gpus) - training_kwargs.setdefault("tp_size", 1) - training_kwargs.setdefault("sp_size", num_gpus) - training_kwargs.setdefault("hsdp_replicate_dim", 1) - training_kwargs.setdefault("hsdp_shard_dim", num_gpus) - - # Use student path as the default base model_path. This is needed for - # PipelineConfig registry lookup. - if "model_path" not in training_kwargs: - student = roles.get("student") - if student is None: - raise ValueError("training.model_path is missing and roles.student is not provided") - training_kwargs["model_path"] = student.path - - if "pretrained_model_name_or_path" not in training_kwargs: - training_kwargs["pretrained_model_name_or_path"] = training_kwargs["model_path"] - - if pipeline_cfg_path is not None: - pipeline_cfg_path = _require_str(pipeline_cfg_path, where="pipeline_config_path") - training_kwargs["pipeline_config"] = _resolve_existing_file(pipeline_cfg_path) - elif pipeline_cfg_raw is not None: - if isinstance(pipeline_cfg_raw, str): - training_kwargs["pipeline_config"] = _resolve_existing_file(pipeline_cfg_raw) - elif isinstance(pipeline_cfg_raw, dict): - training_kwargs["pipeline_config"] = pipeline_cfg_raw - else: - raise ValueError("pipeline_config must be a mapping or a path string") - - training_args = TrainingArgs.from_kwargs(**training_kwargs) - - return DistillRunConfig( - recipe=recipe, - roles=roles, - training_args=training_args, - method_config=method_config, - raw=cfg, - ) diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index c3f18e197..87a96f02d 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -27,7 +27,7 @@ def run_distillation_from_config( DistillCheckpointManager, ) from fastvideo.distillation.builder import build_runtime_from_config - from fastvideo.distillation.yaml_config import load_distill_run_config + from fastvideo.distillation.utils.config import load_distill_run_config cfg = load_distill_run_config(config_path) training_args = cfg.training_args From 3d8b4823d5c2d677502b4f12e24f6d678fdf2469 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 20:16:11 +0000 Subject: [PATCH 099/214] designing phase 3.4 --- dev/phases/phase_2.md | 4 ++-- dev/phases/phase_3.md | 18 +++++++----------- fastvideo/distillation/builder.py | 12 ------------ fastvideo/distillation/doc/builder.md | 2 -- 4 files changed, 9 insertions(+), 27 deletions(-) diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md index d78a0094c..fabce8196 100644 --- a/dev/phases/phase_2.md +++ b/dev/phases/phase_2.md @@ -44,7 +44,7 @@ distillation pipeline**（`fastvideo/training/*distillation_pipeline.py`）。 当前状态（已达成）： - Phase 2 entrypoint：`fastvideo/training/distillation.py --config ` -- runtime：`build_wan_dmd2_runtime_from_config(...)` +- runtime：`build_runtime_from_config(...)` - validation：`fastvideo/distillation/validators/wan.py::WanValidator` 以上链路不再实例化/调用 legacy `WanDistillationPipeline` / `DistillationPipeline._log_validation(...)`。 @@ -81,7 +81,7 @@ distillation pipeline**（`fastvideo/training/*distillation_pipeline.py`）。 - 新增目录：`fastvideo/distillation/outside/`（视作外部 repo root） - 覆盖路径：`fastvideo/distillation/outside/` - **无自动补全/overlay**：config loader 不做路径重写；运行时传入 outside YAML 的真实路径（无 fallback） -- [x] 实现 standalone runtime builder：`fastvideo/distillation/builder.py::build_wan_dmd2_runtime_from_config` +- [x] 实现 standalone runtime builder：`fastvideo/distillation/builder.py::build_runtime_from_config` - 直接加载 modules（student/teacher/critic）并构建 `ModelBundle` - 构建 per-role optimizers/schedulers（复用 TrainingArgs 超参） - 构建 dataloader（`build_parquet_map_style_dataloader`） diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index cea9041c8..0bfbbaa2d 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -175,6 +175,10 @@ Phase 3.1~3.3 已经把训练端到端跑通；但目前 `fastvideo/distillation > 备注：此阶段优先做 **低风险、可 review、行为不变（或可控变更）** 的整理。 > 若某些重排会牵动较大行为差异（例如数据加载完全抽象成独立 registry），可以拆成 3.4.x 逐步落地。 +**本阶段决策（重要）** +- 不做后向兼容：不保留 re-export shim，不保留旧路径别名。 +- 直接把全 repo 的 import / docs / YAML 示例统一到新语义，允许 breaking change。 + ### 目标（DoD） 1) **更直觉的目录命名** @@ -195,12 +199,6 @@ Phase 3.1~3.3 已经把训练端到端跑通；但目前 `fastvideo/distillation - 已将 YAML loader（原 `yaml_config.py`）合并到 `utils/config.py`（schema+解析逻辑同处） - `registry.py + builder.py` 可以合并/重命名为更直觉的 `dispatch.py`（保留注册表与 build_runtime 的入口） -5) **迁移策略：保证渐进、可回退** -- 保留兼容 import（re-export shim）一段时间，避免全 repo 级别大范围改动： - - `fastvideo/distillation/families/__init__.py` re-export `fastvideo/distillation/models/*` - - `fastvideo/distillation/bundle.py` re-export `fastvideo/distillation/roles.py` 的类型 -- 更新 `fastvideo/distillation/doc/` 索引与各文件说明 - ### 具体设计：如何“解耦 dataloader/tracker” #### Tracker @@ -245,11 +243,9 @@ Phase 3.4 目标： 命名/结构（行为尽量不变）： - [x] YAML schema：顶层 `models:` → `roles:`（与 `DistillRunConfig.roles` 对齐） -- [x] YAML loader：`fastvideo/distillation/yaml_config.py` → `fastvideo/distillation/utils/config.py` -- [ ] 新增 `fastvideo/distillation/models/`（拷贝/迁移原 `families/`） -- [ ] 保留 `fastvideo/distillation/families/` 作为兼容 re-export（短期） -- [ ] 新增 `fastvideo/distillation/roles.py` 并迁移 `RoleHandle/ModelBundle` -- [ ] `fastvideo/distillation/bundle.py` 变为兼容层（re-export） +- [x] YAML loader：`fastvideo/distillation/utils/config.py`（包含 schema + 解析逻辑） +- [ ] `fastvideo/distillation/families/` → `fastvideo/distillation/models/`（直接迁移并更新所有 import） +- [ ] `fastvideo/distillation/bundle.py` → `fastvideo/distillation/roles.py`（直接迁移并更新所有 import） - [x] `fastvideo/distillation/specs.py` + `fastvideo/distillation/runtime.py` 合并到 `fastvideo/distillation/utils/config.py` - [ ] `fastvideo/distillation/registry.py` + `fastvideo/distillation/builder.py` 收敛为 `dispatch.py`（或最少改名） diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index 59b57837e..76efa9dac 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -33,15 +33,3 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: tracker=artifacts.tracker, start_step=int(getattr(artifacts, "start_step", 0) or 0), ) - - -def build_wan_dmd2_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: - """Legacy Phase 2 helper kept for compatibility during Phase 2.9 rollout.""" - - if cfg.recipe.family != "wan" or cfg.recipe.method != "dmd2": - raise ValueError( - "build_wan_dmd2_runtime_from_config expects recipe.family='wan' " - f"and recipe.method='dmd2', got family={cfg.recipe.family!r}, " - f"method={cfg.recipe.method!r}" - ) - return build_runtime_from_config(cfg) diff --git a/fastvideo/distillation/doc/builder.md b/fastvideo/distillation/doc/builder.md index 5a659b41e..e42b9713e 100644 --- a/fastvideo/distillation/doc/builder.md +++ b/fastvideo/distillation/doc/builder.md @@ -10,8 +10,6 @@ - `family_builder = registry.get_family(cfg.recipe.family)` - `method_builder = registry.get_method(cfg.recipe.method)` - `method = method_builder(cfg=cfg, bundle=artifacts.bundle, adapter=artifacts.adapter, validator=artifacts.validator)` -- `build_wan_dmd2_runtime_from_config(cfg)` - - Phase 2.9 期间保留的兼容函数（最终可删除） **边界** - ✅ 这里不写 `if model==... and method==...` 的 N×M 组合逻辑。 From 8ba4271cc8ea55ba2bfb63a4b1471d1834c12caa Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 21:18:33 +0000 Subject: [PATCH 100/214] phase 3.4 --- dev/config.md | 2 +- dev/design.md | 6 +-- dev/phases/phase_0.md | 2 +- dev/phases/phase_2.md | 4 +- dev/phases/phase_2_9.md | 8 ++-- dev/phases/phase_3.md | 18 ++++---- examples/distillation/phase3_3/temp.sh | 3 +- fastvideo/distillation/__init__.py | 3 +- fastvideo/distillation/adapters/wan.py | 2 +- fastvideo/distillation/doc/README.md | 14 +++--- .../doc/{families => models}/__init__.md | 3 +- .../doc/{families => models}/wan.md | 2 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.md | 2 +- fastvideo/distillation/doc/registry.md | 2 +- .../distillation/doc/{bundle.md => roles.md} | 5 +-- fastvideo/distillation/doc/utils/__init__.md | 3 ++ .../doc/{ => utils}/checkpoint.md | 2 +- fastvideo/distillation/doc/utils/data.md | 15 +++++++ fastvideo/distillation/doc/utils/tracking.md | 13 ++++++ fastvideo/distillation/methods/base.py | 2 +- .../methods/distribution_matching/dmd2.py | 18 ++++---- .../methods/fine_tuning/finetune.py | 6 +-- .../{families => models}/__init__.py | 1 - .../distillation/{families => models}/wan.py | 45 +++---------------- fastvideo/distillation/registry.py | 4 +- .../distillation/{bundle.py => roles.py} | 0 .../distillation/{ => utils}/checkpoint.py | 4 +- fastvideo/distillation/utils/config.py | 2 +- fastvideo/distillation/utils/data.py | 29 ++++++++++++ fastvideo/distillation/utils/tracking.py | 42 +++++++++++++++++ fastvideo/distillation/validators/base.py | 2 +- .../test_optimizer_scheduler_alignment.py | 2 +- fastvideo/training/distillation.py | 3 +- 33 files changed, 168 insertions(+), 101 deletions(-) rename fastvideo/distillation/doc/{families => models}/__init__.md (90%) rename fastvideo/distillation/doc/{families => models}/wan.md (97%) rename fastvideo/distillation/doc/{bundle.md => roles.md} (86%) rename fastvideo/distillation/doc/{ => utils}/checkpoint.md (95%) create mode 100644 fastvideo/distillation/doc/utils/data.md create mode 100644 fastvideo/distillation/doc/utils/tracking.md rename fastvideo/distillation/{families => models}/__init__.py (99%) rename fastvideo/distillation/{families => models}/wan.py (80%) rename fastvideo/distillation/{bundle.py => roles.py} (100%) rename fastvideo/distillation/{ => utils}/checkpoint.py (98%) create mode 100644 fastvideo/distillation/utils/data.py create mode 100644 fastvideo/distillation/utils/tracking.py diff --git a/dev/config.md b/dev/config.md index 286c50381..9bf00f252 100644 --- a/dev/config.md +++ b/dev/config.md @@ -49,7 +49,7 @@ recipe: ``` 用途： -- registry dispatch：选择 `families/.py` + `methods/.py` 的组合（N+M，而非 N×M）。 +- registry dispatch：选择 `models/.py` + `methods/.py` 的组合（N+M，而非 N×M）。 - 语义更通用：未来把 finetuning 也纳入时不会出现 `distill.method=finetune` 的别扭表达。 ## 4) `roles`: role-based 参与者 diff --git a/dev/design.md b/dev/design.md index 2889abe73..162064203 100644 --- a/dev/design.md +++ b/dev/design.md @@ -480,13 +480,13 @@ Phase 0 的实践表明：先把新框架以 **additive** 方式落地到一个 建议结构（已部分实现）： -- `fastvideo/distillation/bundle.py`：`ModelBundle/RoleHandle` +- `fastvideo/distillation/roles.py`：`ModelBundle/RoleHandle` - `fastvideo/distillation/adapters/`：`WanAdapter`（Phase 1 已落地；后续新增更多 adapter） - `fastvideo/distillation/methods/`：`base.py`、`distribution_matching/dmd2.py`、（目标）`self_forcing.py` - `fastvideo/distillation/trainer.py`：`DistillTrainer` - `fastvideo/distillation/builder.py`：把 “config -> roles -> bundle/adapter/method” 的胶水集中起来 - `fastvideo/training/distillation.py`：通用入口（YAML-only：`--config path/to/run.yaml`） -- （后续）`fastvideo/distillation/checkpoint.py`：role-based `CheckpointManager`（先兼容旧格式） +- `fastvideo/distillation/utils/checkpoint.py`：role-based `CheckpointManager` - （后续）`fastvideo/distillation/callbacks/`：EMA/clip/log/profiler 等 旧入口（如 `fastvideo/training/*distillation_pipeline.py`）先保留， @@ -573,7 +573,7 @@ Phase 1 的“辉煌”（落地与收益）： - 目标：新框架训练可 save/resume，且协议围绕 role 命名空间（不再绑死 WAN pipeline） - 建议实现： - - `fastvideo/distillation/checkpoint.py`：保存/加载 modules + optimizers + schedulers + RNG states + - `fastvideo/distillation/utils/checkpoint.py`：保存/加载 modules + optimizers + schedulers + RNG states - 明确兼容策略：兼容旧格式（若必要）或提供一次性转换脚本 #### Phase 2.4（Deferred）：收敛与清理（暂不做；完全解耦后手动处理） diff --git a/dev/phases/phase_0.md b/dev/phases/phase_0.md index 8f937a4f5..33da22793 100644 --- a/dev/phases/phase_0.md +++ b/dev/phases/phase_0.md @@ -86,7 +86,7 @@ methods**（normalize/noise/timestep/attention metadata/build_input_kwargs 等 - `fastvideo/distillation/__init__.py` - 导出 Phase 0 需要的核心类（Trainer/Method/Bundle） -- `fastvideo/distillation/bundle.py` +- `fastvideo/distillation/roles.py` - `RoleHandle` / `ModelBundle`：`roles: dict[str, RoleHandle]` - `fastvideo/distillation/trainer.py` - `DistillTrainer`：通用训练循环（grad accum + step/zero_grad），不认识 roles diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md index fabce8196..fc1362059 100644 --- a/dev/phases/phase_2.md +++ b/dev/phases/phase_2.md @@ -94,7 +94,7 @@ distillation pipeline**（`fastvideo/training/*distillation_pipeline.py`）。 ### C. role-based checkpoint/save/resume（Phase 2.3） -- [x] 新增 `fastvideo/distillation/checkpoint.py::DistillCheckpointManager` +- [x] 新增 `fastvideo/distillation/utils/checkpoint.py::DistillCheckpointManager` - 保存内容（Phase 2 路径）： - **trainable roles** 的 `modules/optimizers/lr_schedulers`（teacher 默认 frozen，不保存） - `StatefulDataLoader`（Wan-Syn parquet loader 是 `torchdata.stateful_dataloader.StatefulDataLoader`） @@ -213,7 +213,7 @@ training: 建议新增： -- `fastvideo/distillation/checkpoint.py` +- `fastvideo/distillation/utils/checkpoint.py` - `DistillCheckpointManager` - 内部复用 `fastvideo/training/checkpointing_utils.py` 的 wrappers： - `ModelWrapper/OptimizerWrapper/SchedulerWrapper/RandomStateWrapper` diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md index 16ff29bc6..2b4ce9fac 100644 --- a/dev/phases/phase_2_9.md +++ b/dev/phases/phase_2_9.md @@ -133,9 +133,9 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - `register_family(name)` / `register_method(name)` 装饰器 - `get_family(name)` / `get_method(name)`（错误信息包含可用项） - `ensure_builtin_registrations()`：导入内置 family/method 以完成注册 -- [x] 新增 `fastvideo/distillation/families/` - - `fastvideo/distillation/families/__init__.py` - - `fastvideo/distillation/families/wan.py`：`build_wan_family_artifacts` +- [x] 新增 `fastvideo/distillation/models/` + - `fastvideo/distillation/models/__init__.py` + - `fastvideo/distillation/models/wan.py`：`build_wan_family_artifacts` - 从 Phase 2 builder 迁移 Wan-specific build-time 逻辑： - 加载 role modules（transformer/transformer_2/vae 等） - shared components（scheduler/noise_scheduler） @@ -193,7 +193,7 @@ handle 是为 **forward/select module** 服务的：比如选择哪个 transform - teacher 的 `transformer_2` boundary 逻辑不变 - validation 路径不回退到 legacy - [x] Wan family 不再创建 optimizers/schedulers - - `fastvideo/distillation/families/wan.py` 只负责加载 modules + 构建 `ModelBundle` + - `fastvideo/distillation/models/wan.py` 只负责加载 modules + 构建 `ModelBundle` - `DMD2Method` 在 init 时为 student/critic 创建 optimizers/schedulers（复用 TrainingArgs 字段，未来迁移到 `method_config`） - [x] Wan validator 归属 method（method 决定是否/如何调用 validation） - `fastvideo/distillation/validators/wan.py` 当前使用 `WanDMDPipeline` 进行 validation 采样（对齐 DMD2 的 SDE rollout，便于与 legacy apples-to-apples 对比） diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 0bfbbaa2d..2cf3c2099 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -192,7 +192,7 @@ Phase 3.1~3.3 已经把训练端到端跑通；但目前 `fastvideo/distillation 3) **把 infra 从 models(原 families) 中解耦合** - dataloader 构建逻辑从 `models/*` 抽到 `fastvideo/distillation/utils/`（或 `infra/`） - tracker 初始化从 `models/*` 抽到 `trainer/entrypoint`（更符合“infra 归 infra”） -- checkpointing 相关（目前 `fastvideo/distillation/checkpoint.py`）移动到 `utils/`（或 `infra/`） +- checkpointing 相关（`fastvideo/distillation/utils/checkpoint.py`）统一放在 `utils/`（或 `infra/`） 4) **减少“文件级概念数量”** - 已将纯 dataclass（原 `specs.py/runtime.py`）合并到 `utils/config.py`，减少“文件级概念数量” @@ -244,18 +244,18 @@ Phase 3.4 目标： 命名/结构（行为尽量不变）： - [x] YAML schema：顶层 `models:` → `roles:`（与 `DistillRunConfig.roles` 对齐） - [x] YAML loader：`fastvideo/distillation/utils/config.py`（包含 schema + 解析逻辑） -- [ ] `fastvideo/distillation/families/` → `fastvideo/distillation/models/`（直接迁移并更新所有 import） -- [ ] `fastvideo/distillation/bundle.py` → `fastvideo/distillation/roles.py`（直接迁移并更新所有 import） +- [x] `fastvideo/distillation/families/` → `fastvideo/distillation/models/`（直接迁移并更新所有 import） +- [x] `fastvideo/distillation/bundle.py` → `fastvideo/distillation/roles.py`（直接迁移并更新所有 import） - [x] `fastvideo/distillation/specs.py` + `fastvideo/distillation/runtime.py` 合并到 `fastvideo/distillation/utils/config.py` - [ ] `fastvideo/distillation/registry.py` + `fastvideo/distillation/builder.py` 收敛为 `dispatch.py`（或最少改名） infra 解耦： -- [ ] 新增 `fastvideo/distillation/utils/`（或 `infra/`） - - [ ] `utils/tracking.py`：tracker 初始化（rank0 only）+ W&B run YAML 上传（如果需要） - - [ ] `utils/data.py`：dataloader 构建（基于 `DataSpec`） - - [ ] `utils/checkpoint.py`：checkpoint manager / config（从 `distillation/checkpoint.py` 迁移） -- [ ] `models/*`（原 families）移除 tracker/dataloader/checkpointing 的直接创建逻辑 +- [x] 新增 `fastvideo/distillation/utils/`（或 `infra/`） + - [x] `utils/tracking.py`：tracker 初始化（rank0 only） + - [x] `utils/data.py`：dataloader 构建（当前先覆盖 parquet T2V） + - [x] `utils/checkpoint.py`：checkpoint manager / config（从 `distillation/checkpoint.py` 迁移） +- [x] `models/*`（原 families）移除 tracker/dataloader/checkpointing 的内联实现（迁移到 `utils/`） - [ ] 更新 `utils/config.py` 的 artifacts 结构（必要时引入 factory/spec 而非直接对象） docs： -- [ ] 更新 `fastvideo/distillation/doc/README.md` 与各文件说明（路径/命名变化） +- [x] 更新 `fastvideo/distillation/doc/README.md` 与各文件说明（路径/命名变化） diff --git a/examples/distillation/phase3_3/temp.sh b/examples/distillation/phase3_3/temp.sh index 037e8cd94..329352abd 100644 --- a/examples/distillation/phase3_3/temp.sh +++ b/examples/distillation/phase3_3/temp.sh @@ -7,7 +7,7 @@ fi # One-shot launch script for Phase 3.3 (finetune method on the distillation scaffold). # # - Same trainer/bundle/adapter/family infrastructure as distillation. -# - Only `models.student` is required; the method updates student weights only. +# - Only `roles.student` is required; the method updates student weights only. # - Validation is still supported via `ValidationRequest` + `WanValidator`. export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} @@ -40,4 +40,3 @@ torchrun \ --master_port "$MASTER_PORT" \ fastvideo/training/distillation.py \ --config "$CONFIG" - diff --git a/fastvideo/distillation/__init__.py b/fastvideo/distillation/__init__.py index 2b5d503e8..cf7287653 100644 --- a/fastvideo/distillation/__init__.py +++ b/fastvideo/distillation/__init__.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.roles import ModelBundle, RoleHandle from fastvideo.distillation.trainer import DistillTrainer __all__ = [ @@ -8,4 +8,3 @@ "ModelBundle", "RoleHandle", ] - diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 6716a4cd3..800c79d7b 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -29,7 +29,7 @@ from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed from fastvideo.distillation.adapters.base import DistillAdapter -from fastvideo.distillation.bundle import RoleHandle +from fastvideo.distillation.roles import RoleHandle try: from fastvideo.attention.backends.video_sparse_attn import ( diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index 85e6857eb..f474eaa2b 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -5,7 +5,7 @@ 设计原则（对应 Phase 2.9）： - **Trainer** 只做 infra（loop/accum/日志/ckpt/validate 调用），不包含算法策略。 - **Method** 只做算法（loss + update policy + 需要哪些 roles）。 -- **Family** 只做装配（build-time：加载 modules、构建 bundle/adapter/dataloader/validator/tracker）。 +- **Family** 只做装配（build-time：加载 modules、构建 bundle/adapter/dataloader/validator/tracker；代码在 `models/`）。 - **Adapter** 只做运行时 primitive（step-time：prepare_batch/forward_context/predict/backward 等）， API 以 operation 为中心，不以 role 为中心（避免 role 爆炸）。 @@ -22,20 +22,22 @@ - `__init__.md` - `utils/__init__.md` - `utils/config.md` +- `utils/data.md` +- `utils/tracking.md` +- `utils/checkpoint.md` - `registry.md` - `builder.md` -- `bundle.md` +- `roles.md` - `trainer.md` -- `checkpoint.md` ### adapters/ - `adapters/__init__.md` - `adapters/base.md` - `adapters/wan.md` -### families/ -- `families/__init__.md` -- `families/wan.md` +### models/ +- `models/__init__.md` +- `models/wan.md` ### methods/ - `methods/__init__.md` diff --git a/fastvideo/distillation/doc/families/__init__.md b/fastvideo/distillation/doc/models/__init__.md similarity index 90% rename from fastvideo/distillation/doc/families/__init__.md rename to fastvideo/distillation/doc/models/__init__.md index c04261efa..63b3d60b5 100644 --- a/fastvideo/distillation/doc/families/__init__.md +++ b/fastvideo/distillation/doc/models/__init__.md @@ -1,4 +1,4 @@ -# `fastvideo/distillation/families/__init__.py` +# `fastvideo/distillation/models/__init__.py` **目的** - families 是 build-time 插件层： @@ -12,4 +12,3 @@ - family 专注 build-time 高内聚 - method 专注算法高内聚 - entrypoint/builder 不需要 N×M 组合逻辑 - diff --git a/fastvideo/distillation/doc/families/wan.md b/fastvideo/distillation/doc/models/wan.md similarity index 97% rename from fastvideo/distillation/doc/families/wan.md rename to fastvideo/distillation/doc/models/wan.md index c4413a1cd..2a72730cd 100644 --- a/fastvideo/distillation/doc/families/wan.md +++ b/fastvideo/distillation/doc/models/wan.md @@ -1,4 +1,4 @@ -# `fastvideo/distillation/families/wan.py` +# `fastvideo/distillation/models/wan.py` **定位** - `@register_family("wan")` 的 build-time 插件： diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md index 797d75022..c8cb3c3d8 100644 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md @@ -7,7 +7,7 @@ ## 顶层结构 - `recipe:` - - `family: wan` → registry dispatch 到 `families/wan.py` + - `family: wan` → registry dispatch 到 `models/wan.py` - `method: dmd2` → registry dispatch 到 `methods/distribution_matching/dmd2.py` - `roles:` - `student / teacher / critic` 三个 roles（role 名称本身由 method 解释语义） diff --git a/fastvideo/distillation/doc/registry.md b/fastvideo/distillation/doc/registry.md index 83ebd75a2..5c580220c 100644 --- a/fastvideo/distillation/doc/registry.md +++ b/fastvideo/distillation/doc/registry.md @@ -17,5 +17,5 @@ - `available_families()` / `available_methods()` **扩展方式** -- 新增 family：实现 `fastvideo/distillation/families/.py` 并用 `@register_family("")` +- 新增 family：实现 `fastvideo/distillation/models/.py` 并用 `@register_family("")` - 新增 method：实现 `fastvideo/distillation/methods/...` 并用 `@register_method("")` diff --git a/fastvideo/distillation/doc/bundle.md b/fastvideo/distillation/doc/roles.md similarity index 86% rename from fastvideo/distillation/doc/bundle.md rename to fastvideo/distillation/doc/roles.md index c46416fb0..2273cb7d8 100644 --- a/fastvideo/distillation/doc/bundle.md +++ b/fastvideo/distillation/doc/roles.md @@ -1,4 +1,4 @@ -# `fastvideo/distillation/bundle.py` +# `fastvideo/distillation/roles.py` **目的** - 用统一的数据结构表达 “多角色（roles）参与的训练/蒸馏”： @@ -17,6 +17,5 @@ - `role(name)`：获取 handle **Phase 2.9 约定** -- family 负责 **load modules + 设置 trainable** 并创建 `ModelBundle` +- family(model plugin) 负责 **load modules + 设置 trainable** 并创建 `ModelBundle` - method 负责 **(按算法) 创建 optimizers/schedulers** 并写回对应的 `RoleHandle` - diff --git a/fastvideo/distillation/doc/utils/__init__.md b/fastvideo/distillation/doc/utils/__init__.md index d768bfe6e..8686baaa7 100644 --- a/fastvideo/distillation/doc/utils/__init__.md +++ b/fastvideo/distillation/doc/utils/__init__.md @@ -5,3 +5,6 @@ 当前包含： - `config.py`：YAML loader + schema/types + build-time artifacts + runtime 组装结果。 +- `data.py`：通用 dataloader 构建（按 dataset kind/schema 复用 FastVideo 现有实现）。 +- `tracking.py`：tracker 初始化（wandb / tensorboard 等）。 +- `checkpoint.py`：role-based checkpoint/save-resume（Phase 2 runtime）。 diff --git a/fastvideo/distillation/doc/checkpoint.md b/fastvideo/distillation/doc/utils/checkpoint.md similarity index 95% rename from fastvideo/distillation/doc/checkpoint.md rename to fastvideo/distillation/doc/utils/checkpoint.md index 52612cd57..85fa0a7b9 100644 --- a/fastvideo/distillation/doc/checkpoint.md +++ b/fastvideo/distillation/doc/utils/checkpoint.md @@ -1,4 +1,4 @@ -# `fastvideo/distillation/checkpoint.py` +# `fastvideo/distillation/utils/checkpoint.py` **目的** - Phase 2 的 role-based checkpoint/save-resume 管理： diff --git a/fastvideo/distillation/doc/utils/data.md b/fastvideo/distillation/doc/utils/data.md new file mode 100644 index 000000000..d365e5851 --- /dev/null +++ b/fastvideo/distillation/doc/utils/data.md @@ -0,0 +1,15 @@ +# `fastvideo/distillation/utils/data.py` + +**目的** +- 把 “dataloader 构建” 从 model plugin（原 families/，现 `models/`）中抽离出来， + 让插件更聚焦在加载模块与组装 adapter/bundle。 + +**当前包含** +- `build_parquet_t2v_train_dataloader(training_args, parquet_schema=...)` + - 复用 FastVideo 现有的 `build_parquet_map_style_dataloader(...)` + - 仅做最小封装：从 `training_args` 读取必要参数（data_path/batch/workers/seed/cfg_rate/text_len 等） + +**边界** +- 这里不包含 model/pipeline 语义（例如 Wan 的 forward/backward 细节）。 +- 若未来要支持更多 dataset kind（webdataset / precomputed / i2v / ode-init ...）， + 推荐在本目录新增更通用的 builder（或引入 `DataSpec` 再做统一 dispatch）。 diff --git a/fastvideo/distillation/doc/utils/tracking.md b/fastvideo/distillation/doc/utils/tracking.md new file mode 100644 index 000000000..9283bc28f --- /dev/null +++ b/fastvideo/distillation/doc/utils/tracking.md @@ -0,0 +1,13 @@ +# `fastvideo/distillation/utils/tracking.py` + +**目的** +- 把 tracker 初始化从 model plugin 中抽离出来，避免 “模型集成层” 持有 infra 细节。 + +**当前包含** +- `build_tracker(training_args, config=...)` + - 读取 `training_args.trackers / training_args.tracker_project_name / output_dir / wandb_run_name` + - 只在 global rank0 选择真实 tracker；其余 rank 返回 no-op tracker + - tracker log dir 默认在 `output_dir/tracker/` + +**设计意图** +- tracker 属于 infra：entrypoint/trainer 负责持有；method 只负责产出要 log 的 metrics/artifacts。 diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py index 3c14aced4..3c0a76025 100644 --- a/fastvideo/distillation/methods/base.py +++ b/fastvideo/distillation/methods/base.py @@ -8,7 +8,7 @@ import torch -from fastvideo.distillation.bundle import ModelBundle +from fastvideo.distillation.roles import ModelBundle class DistillMethod(torch.nn.Module, ABC): diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 3abc2d119..2422440fe 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -12,8 +12,8 @@ get_scheduler, ) -from fastvideo.distillation.bundle import ModelBundle -from fastvideo.distillation.bundle import RoleHandle +from fastvideo.distillation.roles import ModelBundle +from fastvideo.distillation.roles import RoleHandle from fastvideo.distillation.methods.base import DistillMethod from fastvideo.distillation.registry import register_method from fastvideo.distillation.validators.base import ValidationRequest @@ -95,7 +95,7 @@ class DMD2Method(DistillMethod): Owns the algorithmic orchestration (loss construction + update policy) and stays independent of any specific model family. It requires a - :class:`~fastvideo.distillation.bundle.ModelBundle` containing at least the + :class:`~fastvideo.distillation.roles.ModelBundle` containing at least the roles ``student``, ``teacher``, and ``critic``. All model-family details (how to run student rollout, teacher CFG @@ -117,12 +117,12 @@ def __init__( self.student = bundle.role("student") self.teacher = bundle.role("teacher") self.critic = bundle.role("critic") - if not getattr(self.student, "trainable", False): - raise ValueError("DMD2Method requires models.student.trainable=true") - if getattr(self.teacher, "trainable", False): - raise ValueError("DMD2Method requires models.teacher.trainable=false") - if not getattr(self.critic, "trainable", False): - raise ValueError("DMD2Method requires models.critic.trainable=true") + if not self.student.trainable: + raise ValueError("DMD2Method requires roles.student.trainable=true") + if self.teacher.trainable: + raise ValueError("DMD2Method requires roles.teacher.trainable=false") + if not self.critic.trainable: + raise ValueError("DMD2Method requires roles.critic.trainable=true") self.adapter = adapter self.validator = validator self.training_args = adapter.training_args diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index e47f4353c..4ff7b251a 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -12,7 +12,7 @@ get_scheduler, ) -from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.roles import ModelBundle, RoleHandle from fastvideo.distillation.methods.base import DistillMethod from fastvideo.distillation.registry import register_method from fastvideo.distillation.validators.base import ValidationRequest @@ -77,8 +77,8 @@ def __init__( super().__init__(bundle) bundle.require_roles(["student"]) self.student = bundle.role("student") - if not getattr(self.student, "trainable", False): - raise ValueError("FineTuneMethod requires models.student.trainable=true") + if not self.student.trainable: + raise ValueError("FineTuneMethod requires roles.student.trainable=true") self.adapter = adapter self.validator = validator diff --git a/fastvideo/distillation/families/__init__.py b/fastvideo/distillation/models/__init__.py similarity index 99% rename from fastvideo/distillation/families/__init__.py rename to fastvideo/distillation/models/__init__.py index 869a647a1..0c301328b 100644 --- a/fastvideo/distillation/families/__init__.py +++ b/fastvideo/distillation/models/__init__.py @@ -1,4 +1,3 @@ # SPDX-License-Identifier: Apache-2.0 """Model-family build plugins for Phase 2/2.9 distillation.""" - diff --git a/fastvideo/distillation/families/wan.py b/fastvideo/distillation/models/wan.py similarity index 80% rename from fastvideo/distillation/families/wan.py rename to fastvideo/distillation/models/wan.py index fbf49d629..f6bf61aab 100644 --- a/fastvideo/distillation/families/wan.py +++ b/fastvideo/distillation/models/wan.py @@ -7,18 +7,18 @@ import torch -from fastvideo.distributed import get_world_group from fastvideo.distillation.adapters.wan import WanAdapter -from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.roles import ModelBundle, RoleHandle from fastvideo.distillation.registry import register_family from fastvideo.distillation.utils.config import FamilyArtifacts from fastvideo.distillation.utils.config import DistillRunConfig +from fastvideo.distillation.utils.data import build_parquet_t2v_train_dataloader +from fastvideo.distillation.utils.tracking import build_tracker from fastvideo.models.loader.component_loader import PipelineComponentLoader from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( FlowMatchEulerDiscreteScheduler, ) from fastvideo.training.activation_checkpoint import apply_activation_checkpointing -from fastvideo.training.trackers import initialize_trackers, Trackers from fastvideo.utils import maybe_download_model, verify_model_config_and_directory @@ -74,30 +74,6 @@ def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Mo return module -def _build_tracker(training_args: Any, *, config: dict[str, Any] | None) -> Any: - world_group = get_world_group() - trackers = list(training_args.trackers) - if not trackers and str(training_args.tracker_project_name): - trackers.append(Trackers.WANDB.value) - if world_group.rank != 0: - trackers = [] - - tracker_log_dir = training_args.output_dir or os.getcwd() - if trackers: - tracker_log_dir = os.path.join(tracker_log_dir, "tracker") - - tracker_config = config if trackers else None - tracker_run_name = training_args.wandb_run_name or None - project = training_args.tracker_project_name or "fastvideo" - return initialize_trackers( - trackers, - experiment_name=project, - config=tracker_config, - log_dir=tracker_log_dir, - run_name=tracker_run_name, - ) - - @register_family("wan") def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: training_args = cfg.training_args @@ -173,7 +149,7 @@ def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: ) bundle = ModelBundle(roles=role_handles) - tracker = _build_tracker(training_args, config=cfg.raw) + tracker = build_tracker(training_args, config=cfg.raw) validator = None if getattr(training_args, "log_validation", False): @@ -195,20 +171,11 @@ def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: noise_scheduler=noise_scheduler, vae=vae, ) - - from fastvideo.dataset import build_parquet_map_style_dataloader from fastvideo.dataset.dataloader.schema import pyarrow_schema_t2v - text_len = training_args.pipeline_config.text_encoder_configs[0].arch_config.text_len # type: ignore[attr-defined] - _dataset, dataloader = build_parquet_map_style_dataloader( - training_args.data_path, - training_args.train_batch_size, - num_data_workers=training_args.dataloader_num_workers, + dataloader = build_parquet_t2v_train_dataloader( + training_args, parquet_schema=pyarrow_schema_t2v, - cfg_rate=training_args.training_cfg_rate, - drop_last=True, - text_padding_length=int(text_len), - seed=int(training_args.seed or 0), ) return FamilyArtifacts( diff --git a/fastvideo/distillation/registry.py b/fastvideo/distillation/registry.py index 1951dcebd..bb39feed3 100644 --- a/fastvideo/distillation/registry.py +++ b/fastvideo/distillation/registry.py @@ -5,7 +5,7 @@ from collections.abc import Callable from typing import Any, Protocol -from fastvideo.distillation.bundle import ModelBundle +from fastvideo.distillation.roles import ModelBundle from fastvideo.distillation.methods.base import DistillMethod from fastvideo.distillation.utils.config import FamilyArtifacts from fastvideo.distillation.utils.config import DistillRunConfig @@ -68,7 +68,7 @@ def ensure_builtin_registrations() -> None: # NOTE: keep these imports explicit (no wildcard scanning) so registration # order is stable and failures are debuggable. - from fastvideo.distillation.families import wan as _wan # noqa: F401 + from fastvideo.distillation.models import wan as _wan # noqa: F401 from fastvideo.distillation.methods.distribution_matching import dmd2 as _dmd2 # noqa: F401 from fastvideo.distillation.methods.fine_tuning import finetune as _finetune # noqa: F401 diff --git a/fastvideo/distillation/bundle.py b/fastvideo/distillation/roles.py similarity index 100% rename from fastvideo/distillation/bundle.py rename to fastvideo/distillation/roles.py diff --git a/fastvideo/distillation/checkpoint.py b/fastvideo/distillation/utils/checkpoint.py similarity index 98% rename from fastvideo/distillation/checkpoint.py rename to fastvideo/distillation/utils/checkpoint.py index 5b09f9eab..7ca637868 100644 --- a/fastvideo/distillation/checkpoint.py +++ b/fastvideo/distillation/utils/checkpoint.py @@ -13,7 +13,7 @@ import torch.distributed as dist import torch.distributed.checkpoint as dcp -from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.roles import ModelBundle, RoleHandle from fastvideo.logger import init_logger from fastvideo.training.checkpointing_utils import ( ModelWrapper, @@ -165,7 +165,7 @@ def _build_role_states(self, role: str, handle: RoleHandle) -> dict[str, Any]: container = _RoleModuleContainer(handle.modules) for module_name, module in handle.modules.items(): - states[f"models.{role}.{module_name}"] = ModelWrapper(module) + states[f"roles.{role}.{module_name}"] = ModelWrapper(module) for name, optimizer in handle.optimizers.items(): states[f"optimizers.{role}.{name}"] = OptimizerWrapper(container, optimizer) diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index eddb44bf8..c0da38b4c 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.distillation.bundle import ModelBundle + from fastvideo.distillation.roles import ModelBundle from fastvideo.distillation.methods.base import DistillMethod RoleName = str diff --git a/fastvideo/distillation/utils/data.py b/fastvideo/distillation/utils/data.py new file mode 100644 index 000000000..c8f7b2f33 --- /dev/null +++ b/fastvideo/distillation/utils/data.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any + + +def build_parquet_t2v_train_dataloader( + training_args: Any, + *, + parquet_schema: Any, +) -> Any: + """Build a parquet map-style dataloader for T2V-style latent datasets.""" + + from fastvideo.dataset import build_parquet_map_style_dataloader + + text_len = training_args.pipeline_config.text_encoder_configs[0].arch_config.text_len # type: ignore[attr-defined] + _dataset, dataloader = build_parquet_map_style_dataloader( + training_args.data_path, + training_args.train_batch_size, + num_data_workers=training_args.dataloader_num_workers, + parquet_schema=parquet_schema, + cfg_rate=training_args.training_cfg_rate, + drop_last=True, + text_padding_length=int(text_len), + seed=int(training_args.seed or 0), + ) + return dataloader + diff --git a/fastvideo/distillation/utils/tracking.py b/fastvideo/distillation/utils/tracking.py new file mode 100644 index 000000000..3a890f3f5 --- /dev/null +++ b/fastvideo/distillation/utils/tracking.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from typing import Any + +from fastvideo.distributed import get_world_group +from fastvideo.training.trackers import initialize_trackers, Trackers + + +def build_tracker(training_args: Any, *, config: dict[str, Any] | None) -> Any: + """Build a tracker instance for a distillation run. + + Tracker selection is rank0-only; other ranks get a no-op tracker from + ``initialize_trackers([])``. + """ + + world_group = get_world_group() + + trackers = list(training_args.trackers) + if not trackers and str(training_args.tracker_project_name): + trackers.append(Trackers.WANDB.value) + if world_group.rank != 0: + trackers = [] + + tracker_log_dir = training_args.output_dir or os.getcwd() + if trackers: + tracker_log_dir = os.path.join(tracker_log_dir, "tracker") + + tracker_config = config if trackers else None + tracker_run_name = training_args.wandb_run_name or None + project = training_args.tracker_project_name or "fastvideo" + + return initialize_trackers( + trackers, + experiment_name=project, + config=tracker_config, + log_dir=tracker_log_dir, + run_name=tracker_run_name, + ) + diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py index 38d094a8f..036fe82f1 100644 --- a/fastvideo/distillation/validators/base.py +++ b/fastvideo/distillation/validators/base.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from typing import Literal -from fastvideo.distillation.bundle import RoleHandle +from fastvideo.distillation.roles import RoleHandle @dataclass(slots=True) diff --git a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py index e37412406..0064f44c1 100644 --- a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py +++ b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py @@ -1,6 +1,6 @@ import torch -from fastvideo.distillation.bundle import ModelBundle, RoleHandle +from fastvideo.distillation.roles import ModelBundle, RoleHandle from fastvideo.distillation.methods.base import DistillMethod diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 87a96f02d..935246971 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -11,6 +11,7 @@ logger = init_logger(__name__) + def run_distillation_from_config( config_path: str, *, @@ -22,7 +23,7 @@ def run_distillation_from_config( from fastvideo.distributed import maybe_init_distributed_environment_and_model_parallel from fastvideo.distillation import DistillTrainer - from fastvideo.distillation.checkpoint import ( + from fastvideo.distillation.utils.checkpoint import ( DistillCheckpointConfig, DistillCheckpointManager, ) From c67eb0d8dcc0806d040847888593ca14a60dae2b Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 21:31:40 +0000 Subject: [PATCH 101/214] family->model --- fastvideo/distillation/builder.py | 24 +++++++------- fastvideo/distillation/doc/README.md | 6 ++-- fastvideo/distillation/doc/builder.md | 6 ++-- fastvideo/distillation/doc/models/wan.md | 10 +++--- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.md | 2 +- fastvideo/distillation/doc/registry.md | 14 ++++---- fastvideo/distillation/doc/utils/config.md | 6 ++-- fastvideo/distillation/models/__init__.py | 5 ++- fastvideo/distillation/models/wan.py | 10 +++--- fastvideo/distillation/registry.py | 32 +++++++++---------- fastvideo/distillation/utils/config.py | 10 +++--- 11 files changed, 64 insertions(+), 61 deletions(-) diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py index 76efa9dac..c5900f1ed 100644 --- a/fastvideo/distillation/builder.py +++ b/fastvideo/distillation/builder.py @@ -2,7 +2,7 @@ from __future__ import annotations -from fastvideo.distillation.registry import get_family, get_method +from fastvideo.distillation.registry import get_model, get_method from fastvideo.distillation.utils.config import DistillRuntime from fastvideo.distillation.utils.config import DistillRunConfig @@ -11,25 +11,25 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: """Build a distillation runtime from a YAML config. This is the Phase 2.9 "elegant dispatch" entry for assembling: - - model family artifacts (bundle/adapter/dataloader/tracker) - - method implementation (algorithm) on top of those artifacts + - model components (bundle/adapter/dataloader/tracker/validator) + - method implementation (algorithm) on top of those components """ - family_builder = get_family(str(cfg.recipe.family)) - artifacts = family_builder(cfg=cfg) + model_builder = get_model(str(cfg.recipe.family)) + components = model_builder(cfg=cfg) method_builder = get_method(str(cfg.recipe.method)) method = method_builder( cfg=cfg, - bundle=artifacts.bundle, - adapter=artifacts.adapter, - validator=artifacts.validator, + bundle=components.bundle, + adapter=components.adapter, + validator=components.validator, ) return DistillRuntime( - training_args=artifacts.training_args, + training_args=components.training_args, method=method, - dataloader=artifacts.dataloader, - tracker=artifacts.tracker, - start_step=int(getattr(artifacts, "start_step", 0) or 0), + dataloader=components.dataloader, + tracker=components.tracker, + start_step=int(getattr(components, "start_step", 0) or 0), ) diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index f474eaa2b..96aac3317 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -5,14 +5,14 @@ 设计原则（对应 Phase 2.9）： - **Trainer** 只做 infra（loop/accum/日志/ckpt/validate 调用），不包含算法策略。 - **Method** 只做算法（loss + update policy + 需要哪些 roles）。 -- **Family** 只做装配（build-time：加载 modules、构建 bundle/adapter/dataloader/validator/tracker；代码在 `models/`）。 +- **Model plugin** 只做装配（build-time：加载 modules、构建 bundle/adapter/dataloader/validator/tracker；代码在 `models/`）。 - **Adapter** 只做运行时 primitive（step-time：prepare_batch/forward_context/predict/backward 等）， API 以 operation 为中心，不以 role 为中心（避免 role 爆炸）。 快速入口（从运行到训练）： `fastvideo/training/distillation.py` → `utils.config.load_distill_run_config()` → -`builder.build_runtime_from_config()` → `registry.get_family()/get_method()` → -`FamilyArtifacts + DistillMethod` → `DistillTrainer.run()` +`builder.build_runtime_from_config()` → `registry.get_model()/get_method()` → +`FamilyComponents + DistillMethod` → `DistillTrainer.run()` --- diff --git a/fastvideo/distillation/doc/builder.md b/fastvideo/distillation/doc/builder.md index e42b9713e..449a826e4 100644 --- a/fastvideo/distillation/doc/builder.md +++ b/fastvideo/distillation/doc/builder.md @@ -2,14 +2,14 @@ **目的** - 把 YAML config（`DistillRunConfig`）装配成一个可运行的 `DistillRuntime`： - - `family` 负责 build-time 产物（`FamilyArtifacts`） + - `model plugin` 负责 build-time 产物（`FamilyComponents`） - `method` 负责算法（`DistillMethod`） **关键 API** - `build_runtime_from_config(cfg) -> DistillRuntime` - - `family_builder = registry.get_family(cfg.recipe.family)` + - `model_builder = registry.get_model(cfg.recipe.family)` - `method_builder = registry.get_method(cfg.recipe.method)` - - `method = method_builder(cfg=cfg, bundle=artifacts.bundle, adapter=artifacts.adapter, validator=artifacts.validator)` + - `method = method_builder(cfg=cfg, bundle=components.bundle, adapter=components.adapter, validator=components.validator)` **边界** - ✅ 这里不写 `if model==... and method==...` 的 N×M 组合逻辑。 diff --git a/fastvideo/distillation/doc/models/wan.md b/fastvideo/distillation/doc/models/wan.md index 2a72730cd..698c1ae9c 100644 --- a/fastvideo/distillation/doc/models/wan.md +++ b/fastvideo/distillation/doc/models/wan.md @@ -1,12 +1,12 @@ # `fastvideo/distillation/models/wan.py` **定位** -- `@register_family("wan")` 的 build-time 插件： - - 负责把 YAML config 装配成 `FamilyArtifacts` +- `@register_model("wan")` 的 build-time 插件（实现：`build_wan_components(...)`）： + - 负责把 YAML config 装配成 `FamilyComponents` - 包含 Wan 特有的模块加载、shared components、dataloader schema 等逻辑 **产物** -- `FamilyArtifacts(training_args, bundle, adapter, dataloader, tracker, validator, start_step)` +- `FamilyComponents(training_args, bundle, adapter, dataloader, tracker, validator, start_step)` **主要职责** 1) **加载 shared components** @@ -23,13 +23,13 @@ 4) **tracker / validator（可选）** - tracker：`initialize_trackers(...)`（rank0 才启用） - validator：`WanValidator`（当 `training_args.log_validation=true`） - - family 只负责构建并返回 `validator` + - model plugin 只负责构建并返回 `validator` - validator 本身不应 hardcode `bundle.role("student")` 等角色语义； method 通过 `ValidationRequest.sample_handle` 指定要采样的模型 - 是否调用、用什么采样配置由 method 决定（method-managed validation） **Phase 2.9 的关键变化** -- ✅ family 不再创建 optimizers/schedulers。 +- ✅ model plugin 不再创建 optimizers/schedulers。 - 这类 update policy（哪些 role 训练、各自超参）属于 method/算法语义。 - 当前由 `DMD2Method` 在初始化时创建并写回 `RoleHandle.optimizers/lr_schedulers`。 diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md index c8cb3c3d8..02614d9bc 100644 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md +++ b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md @@ -25,7 +25,7 @@ ## 关键语义归属（Phase 2.9 视角） -**Family（Wan）关心：** +**Model plugin（Wan）关心：** - `roles.*.path / trainable` - `training.data_path / dataloader_num_workers / train_batch_size / seed / output_dir` - Wan 相关的 shape 信息（`num_latent_t/num_height/num_width/...`） diff --git a/fastvideo/distillation/doc/registry.md b/fastvideo/distillation/doc/registry.md index 5c580220c..e9d24bba5 100644 --- a/fastvideo/distillation/doc/registry.md +++ b/fastvideo/distillation/doc/registry.md @@ -1,21 +1,21 @@ # `fastvideo/distillation/registry.py` **目的** -- 为 distillation 的 “family / method” 提供轻量 registry： - - 新增 family 的成本 ≈ N +- 为 distillation 的 “model / method” 提供轻量 registry： + - 新增 model 的成本 ≈ N - 新增 method 的成本 ≈ M - build 组合不需要写 N×M 的 if/else **关键概念** -- `FamilyBuilder(cfg) -> FamilyArtifacts` +- `ModelBuilder(cfg) -> FamilyComponents` - `MethodBuilder(cfg, bundle, adapter, validator) -> DistillMethod` **关键 API** -- `register_family(name)` / `register_method(name)`：装饰器注册 -- `get_family(name)` / `get_method(name)`：查询（会触发内置注册） +- `register_model(name)` / `register_method(name)`：装饰器注册 +- `get_model(name)` / `get_method(name)`：查询（会触发内置注册） - `ensure_builtin_registrations()`：显式 import 内置实现，避免 import 顺序隐式 bug -- `available_families()` / `available_methods()` +- `available_models()` / `available_methods()` **扩展方式** -- 新增 family：实现 `fastvideo/distillation/models/.py` 并用 `@register_family("")` +- 新增 model：实现 `fastvideo/distillation/models/.py` 并用 `@register_model("")` - 新增 method：实现 `fastvideo/distillation/methods/...` 并用 `@register_method("")` diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index 923584804..e3d12fd16 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -49,14 +49,14 @@ - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” ## 3) Builder 装配相关（build-time / run-time 边界） -- `FamilyArtifacts` - - family 插件 build-time 的产物集合： +- `FamilyComponents` + - model 插件 build-time 的产物集合： - `training_args` - `bundle` - `adapter` - `dataloader` - `tracker` - - `validator`（可选；family-specific） + - `validator`（可选；model-specific） - `start_step`（用于 resume / warm-start） - `DistillRuntime` - `DistillTrainer.run()` 所需的最小集合： diff --git a/fastvideo/distillation/models/__init__.py b/fastvideo/distillation/models/__init__.py index 0c301328b..dd027ea6e 100644 --- a/fastvideo/distillation/models/__init__.py +++ b/fastvideo/distillation/models/__init__.py @@ -1,3 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Model-family build plugins for Phase 2/2.9 distillation.""" +"""Model build plugins for Phase 2/2.9 distillation. + +These are "model plugins" selected by ``recipe.family`` / ``roles..family``. +""" diff --git a/fastvideo/distillation/models/wan.py b/fastvideo/distillation/models/wan.py index f6bf61aab..1a022a6c2 100644 --- a/fastvideo/distillation/models/wan.py +++ b/fastvideo/distillation/models/wan.py @@ -9,8 +9,8 @@ from fastvideo.distillation.adapters.wan import WanAdapter from fastvideo.distillation.roles import ModelBundle, RoleHandle -from fastvideo.distillation.registry import register_family -from fastvideo.distillation.utils.config import FamilyArtifacts +from fastvideo.distillation.registry import register_model +from fastvideo.distillation.utils.config import FamilyComponents from fastvideo.distillation.utils.config import DistillRunConfig from fastvideo.distillation.utils.data import build_parquet_t2v_train_dataloader from fastvideo.distillation.utils.tracking import build_tracker @@ -74,8 +74,8 @@ def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Mo return module -@register_family("wan") -def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: +@register_model("wan") +def build_wan_components(*, cfg: DistillRunConfig) -> FamilyComponents: training_args = cfg.training_args roles_cfg = cfg.roles @@ -178,7 +178,7 @@ def build_wan_family_artifacts(*, cfg: DistillRunConfig) -> FamilyArtifacts: parquet_schema=pyarrow_schema_t2v, ) - return FamilyArtifacts( + return FamilyComponents( training_args=training_args, bundle=bundle, adapter=adapter, diff --git a/fastvideo/distillation/registry.py b/fastvideo/distillation/registry.py index bb39feed3..c6ccdd8b8 100644 --- a/fastvideo/distillation/registry.py +++ b/fastvideo/distillation/registry.py @@ -7,12 +7,12 @@ from fastvideo.distillation.roles import ModelBundle from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.utils.config import FamilyArtifacts +from fastvideo.distillation.utils.config import FamilyComponents from fastvideo.distillation.utils.config import DistillRunConfig -class FamilyBuilder(Protocol): - def __call__(self, *, cfg: DistillRunConfig) -> FamilyArtifacts: +class ModelBuilder(Protocol): + def __call__(self, *, cfg: DistillRunConfig) -> FamilyComponents: ... @@ -28,20 +28,20 @@ def __call__( ... -_FAMILIES: dict[str, FamilyBuilder] = {} +_MODELS: dict[str, ModelBuilder] = {} _METHODS: dict[str, MethodBuilder] = {} _BUILTINS_REGISTERED = False -def register_family(name: str) -> Callable[[FamilyBuilder], FamilyBuilder]: +def register_model(name: str) -> Callable[[ModelBuilder], ModelBuilder]: name = str(name).strip() if not name: - raise ValueError("family name cannot be empty") + raise ValueError("model name cannot be empty") - def decorator(builder: FamilyBuilder) -> FamilyBuilder: - if name in _FAMILIES: - raise KeyError(f"Family already registered: {name!r}") - _FAMILIES[name] = builder + def decorator(builder: ModelBuilder) -> ModelBuilder: + if name in _MODELS: + raise KeyError(f"Model already registered: {name!r}") + _MODELS[name] = builder return builder return decorator @@ -75,19 +75,19 @@ def ensure_builtin_registrations() -> None: _BUILTINS_REGISTERED = True -def available_families() -> list[str]: - return sorted(_FAMILIES.keys()) +def available_models() -> list[str]: + return sorted(_MODELS.keys()) def available_methods() -> list[str]: return sorted(_METHODS.keys()) -def get_family(name: str) -> FamilyBuilder: +def get_model(name: str) -> ModelBuilder: ensure_builtin_registrations() - if name not in _FAMILIES: - raise KeyError(f"Unknown family {name!r}. Available: {available_families()}") - return _FAMILIES[name] + if name not in _MODELS: + raise KeyError(f"Unknown model {name!r}. Available: {available_models()}") + return _MODELS[name] def get_method(name: str) -> MethodBuilder: diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index c0da38b4c..bed1f9db8 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -201,12 +201,12 @@ def load_distill_run_config(path: str) -> DistillRunConfig: @dataclass(slots=True) -class FamilyArtifacts: - """Build-time outputs produced by a model family plugin. +class FamilyComponents: + """Build-time outputs produced by a model plugin. - A family is responsible for loading modules, constructing a `ModelBundle`, - and assembling shared components needed by runtime adapters, dataloaders, - validators, and trackers. + A model plugin is responsible for loading modules, constructing a + `ModelBundle`, and assembling shared components needed by runtime adapters, + dataloaders, validators, and trackers. """ training_args: TrainingArgs From 9febcd7f268fc808f351963a41ffd5c5b779e4c0 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 21:34:47 +0000 Subject: [PATCH 102/214] family->model --- fastvideo/distillation/doc/README.md | 2 +- fastvideo/distillation/doc/builder.md | 2 +- fastvideo/distillation/doc/models/wan.md | 4 ++-- fastvideo/distillation/doc/registry.md | 2 +- fastvideo/distillation/doc/utils/config.md | 2 +- fastvideo/distillation/models/wan.py | 6 +++--- fastvideo/distillation/registry.py | 4 ++-- fastvideo/distillation/utils/config.py | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index 96aac3317..79ddd446d 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -12,7 +12,7 @@ 快速入口（从运行到训练）： `fastvideo/training/distillation.py` → `utils.config.load_distill_run_config()` → `builder.build_runtime_from_config()` → `registry.get_model()/get_method()` → -`FamilyComponents + DistillMethod` → `DistillTrainer.run()` +`ModelComponents + DistillMethod` → `DistillTrainer.run()` --- diff --git a/fastvideo/distillation/doc/builder.md b/fastvideo/distillation/doc/builder.md index 449a826e4..3bf1baa20 100644 --- a/fastvideo/distillation/doc/builder.md +++ b/fastvideo/distillation/doc/builder.md @@ -2,7 +2,7 @@ **目的** - 把 YAML config（`DistillRunConfig`）装配成一个可运行的 `DistillRuntime`： - - `model plugin` 负责 build-time 产物（`FamilyComponents`） + - `model plugin` 负责 build-time 产物（`ModelComponents`） - `method` 负责算法（`DistillMethod`） **关键 API** diff --git a/fastvideo/distillation/doc/models/wan.md b/fastvideo/distillation/doc/models/wan.md index 698c1ae9c..e88c4afbb 100644 --- a/fastvideo/distillation/doc/models/wan.md +++ b/fastvideo/distillation/doc/models/wan.md @@ -2,11 +2,11 @@ **定位** - `@register_model("wan")` 的 build-time 插件（实现：`build_wan_components(...)`）： - - 负责把 YAML config 装配成 `FamilyComponents` + - 负责把 YAML config 装配成 `ModelComponents` - 包含 Wan 特有的模块加载、shared components、dataloader schema 等逻辑 **产物** -- `FamilyComponents(training_args, bundle, adapter, dataloader, tracker, validator, start_step)` +- `ModelComponents(training_args, bundle, adapter, dataloader, tracker, validator, start_step)` **主要职责** 1) **加载 shared components** diff --git a/fastvideo/distillation/doc/registry.md b/fastvideo/distillation/doc/registry.md index e9d24bba5..55b310d32 100644 --- a/fastvideo/distillation/doc/registry.md +++ b/fastvideo/distillation/doc/registry.md @@ -7,7 +7,7 @@ - build 组合不需要写 N×M 的 if/else **关键概念** -- `ModelBuilder(cfg) -> FamilyComponents` +- `ModelBuilder(cfg) -> ModelComponents` - `MethodBuilder(cfg, bundle, adapter, validator) -> DistillMethod` **关键 API** diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index e3d12fd16..e2cc61098 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -49,7 +49,7 @@ - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” ## 3) Builder 装配相关（build-time / run-time 边界） -- `FamilyComponents` +- `ModelComponents` - model 插件 build-time 的产物集合： - `training_args` - `bundle` diff --git a/fastvideo/distillation/models/wan.py b/fastvideo/distillation/models/wan.py index 1a022a6c2..3edd5d536 100644 --- a/fastvideo/distillation/models/wan.py +++ b/fastvideo/distillation/models/wan.py @@ -10,7 +10,7 @@ from fastvideo.distillation.adapters.wan import WanAdapter from fastvideo.distillation.roles import ModelBundle, RoleHandle from fastvideo.distillation.registry import register_model -from fastvideo.distillation.utils.config import FamilyComponents +from fastvideo.distillation.utils.config import ModelComponents from fastvideo.distillation.utils.config import DistillRunConfig from fastvideo.distillation.utils.data import build_parquet_t2v_train_dataloader from fastvideo.distillation.utils.tracking import build_tracker @@ -75,7 +75,7 @@ def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Mo @register_model("wan") -def build_wan_components(*, cfg: DistillRunConfig) -> FamilyComponents: +def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: training_args = cfg.training_args roles_cfg = cfg.roles @@ -178,7 +178,7 @@ def build_wan_components(*, cfg: DistillRunConfig) -> FamilyComponents: parquet_schema=pyarrow_schema_t2v, ) - return FamilyComponents( + return ModelComponents( training_args=training_args, bundle=bundle, adapter=adapter, diff --git a/fastvideo/distillation/registry.py b/fastvideo/distillation/registry.py index c6ccdd8b8..07d499086 100644 --- a/fastvideo/distillation/registry.py +++ b/fastvideo/distillation/registry.py @@ -7,12 +7,12 @@ from fastvideo.distillation.roles import ModelBundle from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.utils.config import FamilyComponents +from fastvideo.distillation.utils.config import ModelComponents from fastvideo.distillation.utils.config import DistillRunConfig class ModelBuilder(Protocol): - def __call__(self, *, cfg: DistillRunConfig) -> FamilyComponents: + def __call__(self, *, cfg: DistillRunConfig) -> ModelComponents: ... diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index bed1f9db8..cea11b064 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -201,7 +201,7 @@ def load_distill_run_config(path: str) -> DistillRunConfig: @dataclass(slots=True) -class FamilyComponents: +class ModelComponents: """Build-time outputs produced by a model plugin. A model plugin is responsible for loading modules, constructing a From 7589720d486587c841fddf3c717c40668dedceb1 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Wed, 25 Feb 2026 21:52:57 +0000 Subject: [PATCH 103/214] a bug of encoder_hidden_states_img --- fastvideo/training/wangame_training_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index 8cb6f491f..4617cb1c4 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -296,7 +296,7 @@ def _build_input_kwargs(self, # "encoder_attention_mask": # training_batch.encoder_attention_mask, "encoder_hidden_states_image": - encoder_hidden_states_image, + [encoder_hidden_states_image], # Action conditioning "viewmats": viewmats, "Ks": intrinsics, From 638d955a35f2cbfbb63c6a4a71b991e5c5dd3c18 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 25 Feb 2026 22:41:14 +0000 Subject: [PATCH 104/214] rolemanager, dispatch. --- dev/config.md | 15 ++-- dev/design.md | 51 +++++------- dev/phases/phase_3.md | 23 ++--- dev/refactor.md | 10 +-- examples/distillation/phase2/README.md | 5 +- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 28 +++---- .../phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh | 7 +- examples/distillation/phase2/temp.sh | 8 +- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml | 83 +++++++++++++++++++ examples/distillation/phase2_9/temp.sh | 4 +- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml | 82 ++++++++++++++++++ examples/distillation/phase3_1/temp.sh | 2 +- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml | 2 +- examples/distillation/phase3_2/temp.sh | 3 +- .../finetune_wan2.1_t2v_1.3B_phase3.3.yaml | 0 ...finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml | 0 examples/distillation/phase3_3/temp-vsa.sh | 3 +- examples/distillation/phase3_3/temp.sh | 2 +- .../distillation/phase3_4/distill-temp.sh | 42 ++++++++++ ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml | 83 +++++++++++++++++++ .../phase3_4/finetune-vsa-temp.sh | 44 ++++++++++ ...finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml | 76 +++++++++++++++++ fastvideo/distillation/__init__.py | 4 +- fastvideo/distillation/builder.py | 35 -------- .../distillation/{registry.py => dispatch.py} | 37 ++++++++- fastvideo/distillation/doc/README.md | 10 +-- fastvideo/distillation/doc/__init__.md | 3 +- fastvideo/distillation/doc/adapters/wan.md | 3 +- fastvideo/distillation/doc/builder.md | 16 ---- fastvideo/distillation/doc/dispatch.md | 24 ++++++ .../distillation/doc/methods/__init__.md | 6 +- fastvideo/distillation/doc/methods/base.md | 3 +- .../doc/methods/consistency_model/__init__.md | 3 +- .../methods/distribution_matching/__init__.md | 3 +- .../doc/methods/distribution_matching/dmd2.md | 8 +- .../doc/methods/fine_tuning/__init__.md | 4 +- .../doc/methods/fine_tuning/finetune.md | 8 +- .../knowledge_distillation/__init__.md | 3 +- fastvideo/distillation/doc/models/__init__.md | 10 +-- fastvideo/distillation/doc/models/wan.md | 2 +- fastvideo/distillation/doc/outside/README.md | 15 ---- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.md | 52 ------------ ...ll_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md | 13 --- ...ll_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md | 14 ---- .../finetune_wan2.1_t2v_1.3B_phase3.3.md | 9 -- fastvideo/distillation/doc/registry.md | 21 ----- fastvideo/distillation/doc/roles.md | 4 +- fastvideo/distillation/doc/utils/__init__.md | 4 +- fastvideo/distillation/doc/utils/config.md | 4 + fastvideo/distillation/doc/utils/tracking.md | 2 +- .../distillation/doc/validators/__init__.md | 2 +- fastvideo/distillation/doc/validators/base.md | 4 +- fastvideo/distillation/doc/validators/wan.md | 4 +- fastvideo/distillation/methods/base.py | 4 +- .../methods/distribution_matching/dmd2.py | 14 ++-- .../methods/fine_tuning/finetune.py | 10 +-- fastvideo/distillation/models/components.py | 29 +++++++ fastvideo/distillation/models/wan.py | 14 ++-- fastvideo/distillation/outside/README.md | 15 ---- fastvideo/distillation/roles.py | 2 +- fastvideo/distillation/utils/checkpoint.py | 4 +- fastvideo/distillation/utils/config.py | 23 +---- fastvideo/distillation/validators/base.py | 2 +- fastvideo/distillation/validators/wan.py | 10 ++- .../test_optimizer_scheduler_alignment.py | 13 ++- fastvideo/training/distillation.py | 2 +- 66 files changed, 654 insertions(+), 391 deletions(-) create mode 100644 examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml create mode 100644 examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml rename {fastvideo/distillation/outside/fastvideo/configs/distillation => examples/distillation/phase3_2}/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml (99%) rename {fastvideo/distillation/outside/fastvideo/configs/distillation => examples/distillation/phase3_3}/finetune_wan2.1_t2v_1.3B_phase3.3.yaml (100%) rename {fastvideo/distillation/outside/fastvideo/configs/distillation => examples/distillation/phase3_3}/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml (100%) create mode 100644 examples/distillation/phase3_4/distill-temp.sh create mode 100644 examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml create mode 100644 examples/distillation/phase3_4/finetune-vsa-temp.sh create mode 100644 examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml delete mode 100644 fastvideo/distillation/builder.py rename fastvideo/distillation/{registry.py => dispatch.py} (72%) delete mode 100644 fastvideo/distillation/doc/builder.md create mode 100644 fastvideo/distillation/doc/dispatch.md delete mode 100644 fastvideo/distillation/doc/outside/README.md delete mode 100644 fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md delete mode 100644 fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md delete mode 100644 fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md delete mode 100644 fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md delete mode 100644 fastvideo/distillation/doc/registry.md create mode 100644 fastvideo/distillation/models/components.py delete mode 100644 fastvideo/distillation/outside/README.md diff --git a/dev/config.md b/dev/config.md index 9bf00f252..a043be69e 100644 --- a/dev/config.md +++ b/dev/config.md @@ -6,17 +6,17 @@ - YAML loader：`fastvideo/distillation/utils/config.py::load_distill_run_config` - Entrypoint：`fastvideo/training/distillation.py` - Schema/类型定义：`fastvideo/distillation/utils/config.py` -- 示例 YAML（outside）：`fastvideo/distillation/outside/fastvideo/configs/distillation/` +- 示例 YAML（examples）：`examples/distillation/` ## 1) 入口与约束（非常重要） distillation 入口 **只接受一个真实存在的 YAML 文件路径**（不 merge legacy CLI/configs， -也不做 outside 路径补全/overlay）。YAML 是 single source of truth。 +也不做路径补全/overlay）。YAML 是 single source of truth。 运行方式（示意）： ```bash python fastvideo/training/distillation.py \ - --config /abs/path/to/fastvideo/distillation/outside/fastvideo/configs/distillation/xxx.yaml + --config /abs/path/to/examples/distillation//xxx.yaml ``` CLI 仅保留少量 **runtime override**（不属于“实验定义”的内容）： @@ -141,7 +141,8 @@ method_config: ## 8) 最小可运行示例（Wan few-step DMD2） -参考 outside 下的三个可运行 YAML： -- `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` -- `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml` -- `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` +参考 `examples/distillation/` 下的可运行 YAML： +- `examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` +- `examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml` +- `examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` +- `examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml` diff --git a/dev/design.md b/dev/design.md index 162064203..9be6b13e4 100644 --- a/dev/design.md +++ b/dev/design.md @@ -17,7 +17,7 @@ - `DistillTrainer`：只做训练基础设施（循环、分布式、grad accum、logging、ckpt、validate） - `DistillMethod`：一个“可训练对象”，封装 distill 算法 + 多角色模型 + 多优化器/交替更新 - `DistillAdapter`：把具体 pipeline/network 适配成统一的 noise/forward/CFG/cache 接口 -- `ModelBundle`：`roles={student, teacher, critic, ...}` 的统一容器（含 optim/ema/fsdp 策略） +- `RoleManager`：`roles={student, teacher, critic, ...}` 的统一容器（含 optim/ema/fsdp 策略） - `ConditioningProvider`（或 dataset 常量注入）：显式提供 `neg_condition` 等 conditioning 常量 关键原则：**Trainer 不认识 teacher/critic，也不写 DMD/SF 的 if/else。** @@ -84,7 +84,7 @@ FastGen 把 distillation 的复杂度拆成： ```text CLI/YAML config - -> build ModelBundle(roles={student, teacher, critic?, ...}) + -> build RoleManager(roles={student, teacher, critic?, ...}) -> build DistillAdapter.from_pipelines(bundle) # pipeline/network 适配 -> build DistillMethod(adapter, bundle, method_cfg) -> DistillTrainer(trainer_cfg, callbacks, checkpointer).run(method) @@ -112,13 +112,13 @@ CLI/YAML config ## 4. 核心对象与接口（建议 API） -### 4.1 `ModelBundle`：角色显式化（外部输入） +### 4.1 `RoleManager`：角色显式化（外部输入） 目标：让入口层显式传入 `roles={student, teacher, critic, ...}`，并把所有 “训练态（optim/ema/fsdp 策略）”结构化地挂在 role 下。 ```text -ModelBundle +RoleManager roles: dict[str, RoleHandle] # key == "student"/"teacher"/"critic"/... RoleHandle @@ -253,12 +253,12 @@ FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step` - distill 的“本质复杂度”就是多网络 + 多优化器调度；放在 Method 最自然 - Trainer 只需要稳定地做基础设施，长期维护成本最低 -### 设计 2：`roles={...}` 显式输入 + `ModelBundle` 结构化承载训练态 +### 设计 2：`roles={...}` 显式输入 + `RoleManager` 结构化承载训练态 **设计** - 配置/CLI 显式给出 `student/teacher/critic?` -- `ModelBundle` 统一挂载冻结策略、precision、FSDP 策略、EMA、optim/sched +- `RoleManager` 统一挂载冻结策略、precision、FSDP 策略、EMA、optim/sched **原因** @@ -437,31 +437,18 @@ method_config: 这样不需要推翻现有 TrainingArgs/FastVideoArgs 体系，但从入口层面彻底摒弃旧式 CLI 传参方式。 -### 6.4 `outside/` overlay（Phase 2 约束下的 workaround） +### 6.4 YAML configs in `examples/`（Phase 3.4+） -我们不能直接修改大项目里的 `fastvideo/configs/`（避免冲突/合并成本）。 -因此 Phase 2 建议在 distillation 侧新增一个 overlay 根目录： +我们不再使用 `outside/` overlay。为了避免修改主仓库的 `fastvideo/configs/` 树，同时让配置更直觉可运行， +distillation 的 runnable YAML 统一放在： -- `fastvideo/distillation/outside/` +- `examples/distillation//*.yaml` 并约定： - -- 把“本应在外部 repo 存在的新增/改版配置”放进： - - `fastvideo/distillation/outside/fastvideo/configs/...` - distillation 入口 **不做任何自动补全/overlay 重写**： - - 用户传入的 `--config` 必须是一个真实存在的文件路径（通常位于 `outside/` 下） + - 用户传入的 `--config` 必须是一个真实存在的文件路径（通常位于 `examples/distillation/` 下） - config 内引用的其它路径（如 `pipeline_config_path`）也必须是 **真实路径** -这让我们可以在不侵入主仓库配置的情况下，迭代 YAML/JSON config、做实验性变更， -同时不影响 legacy 代码路径。 - -**实现注意** - -- 不建议把 `outside/` 直接插入 `sys.path` 去 shadow 整个 `fastvideo` 包（风险太高、调试困难）。 -- 推荐把 `outside/` 仅作为 **外部配置存放目录**（YAML/JSON），避免运行时“魔法寻路”。 -- 如果确实需要覆盖 Python config（`.py`）： - - 用 `importlib` 的“按文件路径加载模块”方式加载为独立 module name，避免影响全局 import。 - ### 6.5 配置系统演进（可选吸收 FastGen 的优点） FastGen 的 python config + instantiate + override 很优秀，但 FastVideo 现阶段可以先： @@ -480,11 +467,11 @@ Phase 0 的实践表明：先把新框架以 **additive** 方式落地到一个 建议结构（已部分实现）： -- `fastvideo/distillation/roles.py`：`ModelBundle/RoleHandle` +- `fastvideo/distillation/roles.py`：`RoleManager/RoleHandle` - `fastvideo/distillation/adapters/`：`WanAdapter`（Phase 1 已落地；后续新增更多 adapter） - `fastvideo/distillation/methods/`：`base.py`、`distribution_matching/dmd2.py`、（目标）`self_forcing.py` - `fastvideo/distillation/trainer.py`：`DistillTrainer` -- `fastvideo/distillation/builder.py`：把 “config -> roles -> bundle/adapter/method” 的胶水集中起来 +- `fastvideo/distillation/dispatch.py`：把 “config -> model components -> method -> runtime” 的胶水集中起来 - `fastvideo/training/distillation.py`：通用入口（YAML-only：`--config path/to/run.yaml`） - `fastvideo/distillation/utils/checkpoint.py`：role-based `CheckpointManager` - （后续）`fastvideo/distillation/callbacks/`：EMA/clip/log/profiler 等 @@ -501,7 +488,7 @@ Phase 0 的实践表明：先把新框架以 **additive** 方式落地到一个 Phase 0 的定位在实践中更明确了：它是“**把旧 Wan distill pipeline 包一层新框架壳**”， 先把训练循环/多 optimizer 调度/validation hook 等基础设施固定下来，再逐步解耦。 -- ✅ 新增 `DistillTrainer/DistillMethod/ModelBundle` 的骨架，并跑通 WAN distill +- ✅ 新增 `DistillTrainer/DistillMethod/RoleManager` 的骨架，并跑通 WAN distill - ✅ 用单测锁定关键语义：scheduler step 与 optimizer step 对齐 - `generator_update_interval > 1` 时不会“空 step scheduler” - ✅ 为后续解耦铺路：把 “roles={student,teacher,critic}” 显式化到 bundle @@ -525,8 +512,8 @@ Phase 1 的“辉煌”（落地与收益）： - ✅ 真正的 WAN 适配层：`fastvideo/distillation/adapters/wan.py::WanAdapter` - `forward_context` 与 backward 重算约束收敛到 adapter（method 只实现算法） - `ensure_negative_conditioning()` 显式化（不再依赖 validation 的隐式副作用） -- ✅ Builder 雏形：`fastvideo/distillation/builder.py` - - 把 “roles -> bundle -> method” 的胶水集中在一处，便于扩展新 method/new model +- ✅ Dispatch 入口：`fastvideo/distillation/dispatch.py` + - 把 “recipe/roles -> model components -> method -> runtime” 的胶水集中在一处，便于扩展新 method/new model - ✅ 通用入口：`fastvideo/training/distillation.py` - Phase 1 仍是 CLI 选择：`--distill-model` + `--distill-method` - Phase 2 起将切换为 **YAML-only**（见第 6 节），并逐步废弃这套 CLI @@ -557,12 +544,12 @@ Phase 1 的“辉煌”（落地与收益）： - 定义结构化 spec：`RoleSpec/ModelSpec`（role -> {family, path, precision, trainable,...}） - 配置形态落地（Phase 2 必做）： - `--config path/to/run.yaml`（YAML 为 single source of truth；CLI 仅指定配置路径） - - `outside/` workaround：把新增/实验性 configs 放在 `outside/`，入口只接受真实路径（不做 overlay 寻路） + - runnable YAML：放在 `examples/distillation//*.yaml`，入口只接受真实路径（不做 overlay 寻路） - （可选）保留 `--models_json` 作为“程序生成配置”的接口 - builder 根据 spec： - 加载 modules（student/teacher/critic） - 构建 role-based optimizers/schedulers - - 组装 `ModelBundle + Adapter + Method` + - 组装 `RoleManager + Adapter + Method` - 构建 dataloader（直接复用 dataset 代码，不经由 legacy pipeline class） - 不新增入口文件：直接增强 `fastvideo/training/distillation.py`，并把它定义为 **YAML-only distill entrypoint** - 仅支持 `--config run.yaml`（以及少量 meta flags），不再兼容旧式 CLI configs @@ -662,7 +649,7 @@ method_config: {...} # algorithm/method 超参（方法侧） #### Phase 3.3：把 finetuning 作为一种 method 接入框架 目标：把 finetune 作为一种 method（only `student` + dataset）接入同一套 -`ModelBundle + Adapter + Method + Trainer + (Validator/Checkpoint)` 基础设施，并让其配置语义与 +`RoleManager + Adapter + Method + Trainer + (Validator/Checkpoint)` 基础设施，并让其配置语义与 Phase 3.1 的 `recipe/method_config` 对齐。 状态：**已完成**（`FineTuneMethod` + Phase 3.3 示例 YAML + one-shot 脚本）。 diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 2cf3c2099..0d22acf26 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -59,7 +59,7 @@ method_config: {...} # algorithm（方法侧） - [x] `fastvideo/distillation/utils/config.py`（包含 YAML loader + schema/dataclass） - 解析 `recipe:` 与 `method_config:`（默认 `{}`） - 将 v1 的 `distill:` 视为不再支持（breaking change，直接推进 schema v2） -- [x] `fastvideo/distillation/builder.py` +- [x] `fastvideo/distillation/dispatch.py` - 从 `cfg.recipe` 取 family/method（不再读 `cfg.distill`） - build method 时传入 `method_config` - [x] `fastvideo/distillation/methods/distribution_matching/dmd2.py` @@ -77,11 +77,11 @@ method_config: {...} # algorithm（方法侧） - `prepare_batch_from_placeholder_latent(raw_batch, ...)`（不依赖 `vae_latent`） - 选项 B：保留单入口但显式参数化：`prepare_batch(..., latents_source=...)`（本阶段采用） - [x] configs / docs - - [x] `fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml` 全部升级到 schema v2 + - [x] `examples/distillation/**.yaml` 全部升级到 schema v2 - [x] 更新 `dev/config.md`（描述 schema v2 与迁移策略） ### 可运行产物 -- Phase 3.1 YAML：`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` +- Phase 3.1 YAML：`examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` - One-shot 脚本：`examples/distillation/phase3_1/temp.sh` --- @@ -122,7 +122,7 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop - validation request 指定 `sampler_kind="sde"` + `sampling_timesteps=` ### 可运行产物 -- Phase 3.2 YAML：`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml` +- Phase 3.2 YAML：`examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml` - One-shot 脚本：`examples/distillation/phase3_2/temp.sh` --- @@ -146,9 +146,10 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop - `predict_noise(handle, ...)`（以及可选 `predict_x0`） - `backward(loss, ctx, ...)`（forward-context/activation ckpt 相关） - [x] configs/examples - - [x] `fastvideo/distillation/outs - ide/fastvideo/configs/distillation/finetun e_wan2.1_t2v_1.3B_phase3.3.yaml` - - [x] `examples/distillation/phase3_3/temp.sh` + - [x] `examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` + - [x] `examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml` + - [x] `examples/distillation/phase3_3/temp.sh` + - [x] `examples/distillation/phase3_3/temp-vsa.sh` --- @@ -168,7 +169,7 @@ Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop Phase 3.1~3.3 已经把训练端到端跑通；但目前 `fastvideo/distillation/` 的概念命名偏“框架内部术语”，对新 reviewer 不友好： - `families/` 读起来像“人类家族”，但它实际承担的是 **model/pipeline contract 的集成/装配层**。 - `bundle.py` 读起来像“打包”，但它本质是 **roles 管理/索引容器**。 -- `registry.py` / `builder.py` /（以及一些纯 dataclass 文件）分散在多个文件，阅读路径长，容易产生“概念过多”的感受。 +- `registry.py` / `builder.py` /（以及一些纯 dataclass 文件）分散在多个文件，阅读路径长，容易产生“概念过多”的感受（本阶段已收敛到 `dispatch.py` 与 `utils/config.py`）。 我们希望把这些改成更直觉的命名，并把“infra”从“模型集成层”里抽出来。 @@ -187,7 +188,7 @@ Phase 3.1~3.3 已经把训练端到端跑通；但目前 `fastvideo/distillation 2) **roles 容器命名统一** - `fastvideo/distillation/bundle.py` → `fastvideo/distillation/roles.py` -- `ModelBundle` → `RoleManager`（或保持 `ModelBundle` 但在代码内逐步迁移到新名） +- `ModelBundle` → `RoleManager` 3) **把 infra 从 models(原 families) 中解耦合** - dataloader 构建逻辑从 `models/*` 抽到 `fastvideo/distillation/utils/`（或 `infra/`） @@ -247,7 +248,7 @@ Phase 3.4 目标： - [x] `fastvideo/distillation/families/` → `fastvideo/distillation/models/`（直接迁移并更新所有 import） - [x] `fastvideo/distillation/bundle.py` → `fastvideo/distillation/roles.py`（直接迁移并更新所有 import） - [x] `fastvideo/distillation/specs.py` + `fastvideo/distillation/runtime.py` 合并到 `fastvideo/distillation/utils/config.py` -- [ ] `fastvideo/distillation/registry.py` + `fastvideo/distillation/builder.py` 收敛为 `dispatch.py`（或最少改名） +- [x] `fastvideo/distillation/dispatch.py`（合并 `registry.py` + `builder.py`） infra 解耦： - [x] 新增 `fastvideo/distillation/utils/`（或 `infra/`） @@ -255,7 +256,7 @@ infra 解耦： - [x] `utils/data.py`：dataloader 构建（当前先覆盖 parquet T2V） - [x] `utils/checkpoint.py`：checkpoint manager / config（从 `distillation/checkpoint.py` 迁移） - [x] `models/*`（原 families）移除 tracker/dataloader/checkpointing 的内联实现（迁移到 `utils/`） -- [ ] 更新 `utils/config.py` 的 artifacts 结构（必要时引入 factory/spec 而非直接对象） +- [x] build-time “装配产物” 统一为 `models/components.py::ModelComponents`（不额外引入 artifacts/factory/spec 抽象） docs： - [x] 更新 `fastvideo/distillation/doc/README.md` 与各文件说明（路径/命名变化） diff --git a/dev/refactor.md b/dev/refactor.md index a893afd59..d90313163 100644 --- a/dev/refactor.md +++ b/dev/refactor.md @@ -30,10 +30,10 @@ `critic`……（不设“高低贵贱”，只是 key）。 - **RoleHandle**：每个 role 对应的句柄，包含 `modules/optimizers/schedulers` 以及类似 `trainable` 的标志。 -- **ModelBundle**：持有所有 `RoleHandle` 的容器（未来可能改名）。 +- **RoleManager**：持有所有 `RoleHandle` 的容器（role 索引与训练态命名空间）。 - **Adapter**：训练原语的 operation-centric 接口（如 `prepare_batch`、 `predict_noise/x0`、`add_noise`、`backward` 等）。它不应包含算法策略。 -- **Family**：构建期工厂：根据 config 组装 bundle + adapter + validator。 +- **Model plugin**：构建期工厂：根据 config 组装 roles + adapter + validator。 - **Validator**：训练期 validation 的采样/记录层，由 method 通过 `ValidationRequest` 提供关键参数（steps/sampler/guidance/…）。 @@ -164,7 +164,7 @@ - 按 run config 构建 validator（采样 pipeline）并保持一致； - 只在 rank0 创建 tracker / run 元信息。 -如果移除 Family，工程上往往会以另外一种形式“复活”同样的东西： +如果移除 model plugin（装配层），工程上往往会以另外一种形式“复活”同样的东西： - `SharedContext`， - `RuntimeFactory`， - 或“method 构造函数里做 assembly”。 @@ -179,8 +179,8 @@ 我们可以通过以下方式减少“概念感”，同时保持边界： 1) **重命名 / 重塑概念** - - 例如 `ModelBundle` → `RoleManager`（语义更直观）。 - - `Family` → `RuntimeFactory`（如果“Family”这个词更难理解）。 + - 例如 `RoleManager`（替代旧的 `ModelBundle`）（语义更直观）。 + - `Model plugin` → `RuntimeFactory`（如果“Model plugin”这个词仍然难理解）。 2) **让 method “看起来像持有多个 adapter”，但底层共享 context** - 保留一份共享 adapter + context。 diff --git a/examples/distillation/phase2/README.md b/examples/distillation/phase2/README.md index 29cf0df62..f44875065 100644 --- a/examples/distillation/phase2/README.md +++ b/examples/distillation/phase2/README.md @@ -8,7 +8,7 @@ entrypoint: Important: - Phase 2 does not rewrite config paths automatically. Pass the explicit YAML - path (we keep runnable YAMLs under `fastvideo/distillation/outside/`). + path (we keep runnable YAMLs next to the scripts under `examples/distillation/`). Start from: @@ -16,8 +16,7 @@ Start from: Recommended: -- Edit the runnable YAML in: - `fastvideo/distillation/outside/fastvideo/configs/distillation/` +- Edit the runnable YAML in this folder. - `temp.sh` (runs the config above; same dataset + validation defaults as Phase0/Phase1). Resume: diff --git a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml index b62544d4b..f3d1fb766 100644 --- a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml @@ -1,12 +1,5 @@ -# NOTE: -# Phase 2 does not rewrite config paths automatically. Put runnable YAML under: -# fastvideo/distillation/outside/fastvideo/configs/distillation/ -# -# The runnable copy for this template lives at: -# fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml -# -distill: - model: wan +recipe: + family: wan method: dmd2 roles: @@ -18,10 +11,12 @@ roles: family: wan path: Wan-AI/Wan2.1-T2V-14B-Diffusers trainable: false + disable_custom_init_weights: true critic: family: wan path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers trainable: true + disable_custom_init_weights: true training: # Distributed @@ -48,7 +43,7 @@ training: seed: 1000 checkpoints_total_limit: 3 - # Optimizer + # Optimizer (student default) learning_rate: 2.0e-6 mixed_precision: bf16 betas: "0.0,0.999" @@ -57,17 +52,15 @@ training: lr_warmup_steps: 0 max_grad_norm: 1.0 + # Critic overrides (DMD2-specific) fake_score_learning_rate: 8.0e-6 fake_score_betas: "0.0,0.999" fake_score_lr_scheduler: constant - # Distillation + # Method-agnostic knobs training_cfg_rate: 0.0 min_timestep_ratio: 0.02 max_timestep_ratio: 0.98 - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - simulate_generator_forward: true enable_gradient_checkpointing_type: full # Tracking / validation @@ -81,4 +74,11 @@ training: pipeline_config: flow_shift: 8 + # DMD2 validation rollout matches SDE-style generation. + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh index 78eb4494a..1f8012d60 100644 --- a/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh +++ b/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh @@ -3,8 +3,9 @@ set -euo pipefail set -x # NOTE: -# Phase 2 expects an explicit YAML path (we keep runnable YAML under outside/): -# fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml +# Phase 2 expects an explicit YAML path (YAML-only entrypoint). +# We keep runnable YAML next to this script under: +# examples/distillation/phase2/*.yaml export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} @@ -13,7 +14,7 @@ export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} export WANDB_MODE=${WANDB_MODE:-offline} export MASTER_PORT=${MASTER_PORT:-29506} -CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} +CONFIG=${CONFIG:-"examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} if [[ ! -f "$CONFIG" ]]; then echo "Missing Phase 2 YAML config at: $CONFIG" >&2 exit 1 diff --git a/examples/distillation/phase2/temp.sh b/examples/distillation/phase2/temp.sh index cc4ed0048..747eba4e1 100644 --- a/examples/distillation/phase2/temp.sh +++ b/examples/distillation/phase2/temp.sh @@ -9,9 +9,9 @@ set -e -x # - validation json: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json # # Notes: -# - Phase 2 expects an explicit YAML path (we keep runnable YAML under outside/). -# Put your YAML under: -# fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml +# - Phase 2 expects an explicit YAML path (YAML-only entrypoint). +# We keep runnable YAML next to this script under: +# examples/distillation/phase2/*.yaml # - By default this runs W&B in offline mode (safer for overnight runs). # If you want online logging: # export WANDB_MODE=online @@ -31,7 +31,7 @@ if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then exit 1 fi -CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} +CONFIG=${CONFIG:-"examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} if [[ ! -f "$CONFIG" ]]; then echo "Missing Phase 2 YAML config at: $CONFIG" >&2 diff --git a/examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml b/examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml new file mode 100644 index 000000000..30bc978b4 --- /dev/null +++ b/examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml @@ -0,0 +1,83 @@ +recipe: + family: wan + method: dmd2 + +roles: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + disable_custom_init_weights: true + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + disable_custom_init_weights: true + +training: + # Distributed + num_gpus: 1 + sp_size: 1 + tp_size: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase2.9_wan2.1_dmd2_8steps_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer (student default) + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Critic overrides (DMD2-specific) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Method-agnostic knobs + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: phase2.9_wan_dmd2_8steps_wansyn + wandb_run_name: phase2.9_wan_dmd2_8steps_wansyn + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "8" + validation_guidance_scale: "6.0" + +pipeline_config: + flow_shift: 8 + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/examples/distillation/phase2_9/temp.sh b/examples/distillation/phase2_9/temp.sh index cc3474ec1..fe5f30fa3 100644 --- a/examples/distillation/phase2_9/temp.sh +++ b/examples/distillation/phase2_9/temp.sh @@ -4,7 +4,7 @@ if [[ "${DEBUG:-0}" == "1" ]]; then set -x fi -# One-shot launch script for Phase 2.9 (Families + registry dispatch + +# One-shot launch script for Phase 2.9 (models + registry dispatch + # operation-centric adapter + method-managed validation). # # Uses the same dataset/validation defaults as phase0/phase1/phase2; the main @@ -24,7 +24,7 @@ if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then exit 1 fi -CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml"} +CONFIG=${CONFIG:-"examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml"} if [[ ! -f "$CONFIG" ]]; then echo "Missing distillation YAML config at: $CONFIG" >&2 diff --git a/examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml b/examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml new file mode 100644 index 000000000..5c42dbce7 --- /dev/null +++ b/examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml @@ -0,0 +1,82 @@ +recipe: + family: wan + method: dmd2 + +roles: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + disable_custom_init_weights: true + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + disable_custom_init_weights: true + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.1_wan2.1_dmd2_8steps_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Method-agnostic knobs + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: phase3.1_wan_dmd2_8steps_wansyn + wandb_run_name: phase3.1_wan_dmd2_8steps + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "8" + validation_guidance_scale: "6.0" + +pipeline_config: + flow_shift: 8 + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/examples/distillation/phase3_1/temp.sh b/examples/distillation/phase3_1/temp.sh index c46aa1e4b..6bff2fc42 100644 --- a/examples/distillation/phase3_1/temp.sh +++ b/examples/distillation/phase3_1/temp.sh @@ -25,7 +25,7 @@ if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then exit 1 fi -CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml"} +CONFIG=${CONFIG:-"examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml"} if [[ ! -f "$CONFIG" ]]; then echo "Missing distillation YAML config at: $CONFIG" >&2 diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml b/examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml similarity index 99% rename from fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml rename to examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml index f25ea195a..4e5d16dbf 100644 --- a/fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml +++ b/examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml @@ -73,7 +73,7 @@ training: pipeline_config: flow_shift: 8 - sampler_kind: ode + sampler_kind: sde method_config: # Replace legacy `training.simulate_generator_forward`. diff --git a/examples/distillation/phase3_2/temp.sh b/examples/distillation/phase3_2/temp.sh index 69aa3d6d2..525d742fe 100644 --- a/examples/distillation/phase3_2/temp.sh +++ b/examples/distillation/phase3_2/temp.sh @@ -24,7 +24,7 @@ if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then exit 1 fi -CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml"} +CONFIG=${CONFIG:-"examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml"} if [[ ! -f "$CONFIG" ]]; then echo "Missing distillation YAML config at: $CONFIG" >&2 @@ -40,4 +40,3 @@ torchrun \ --master_port "$MASTER_PORT" \ fastvideo/training/distillation.py \ --config "$CONFIG" - diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml b/examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml similarity index 100% rename from fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml rename to examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml diff --git a/fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml b/examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml similarity index 100% rename from fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml rename to examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml diff --git a/examples/distillation/phase3_3/temp-vsa.sh b/examples/distillation/phase3_3/temp-vsa.sh index 0055fca08..fac3a526a 100644 --- a/examples/distillation/phase3_3/temp-vsa.sh +++ b/examples/distillation/phase3_3/temp-vsa.sh @@ -25,7 +25,7 @@ if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then exit 1 fi -CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml"} +CONFIG=${CONFIG:-"examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml"} if [[ ! -f "$CONFIG" ]]; then echo "Missing distillation YAML config at: $CONFIG" >&2 @@ -41,4 +41,3 @@ torchrun \ --master_port "$MASTER_PORT" \ fastvideo/training/distillation.py \ --config "$CONFIG" - diff --git a/examples/distillation/phase3_3/temp.sh b/examples/distillation/phase3_3/temp.sh index 329352abd..44323a55e 100644 --- a/examples/distillation/phase3_3/temp.sh +++ b/examples/distillation/phase3_3/temp.sh @@ -24,7 +24,7 @@ if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then exit 1 fi -CONFIG=${CONFIG:-"fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml"} +CONFIG=${CONFIG:-"examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml"} if [[ ! -f "$CONFIG" ]]; then echo "Missing distillation YAML config at: $CONFIG" >&2 diff --git a/examples/distillation/phase3_4/distill-temp.sh b/examples/distillation/phase3_4/distill-temp.sh new file mode 100644 index 000000000..5d92ad5d9 --- /dev/null +++ b/examples/distillation/phase3_4/distill-temp.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -euo pipefail +if [[ "${DEBUG:-0}" == "1" ]]; then + set -x +fi + +# Phase 3.4 distillation (DMD2) launcher. +# +# - YAML config lives next to this script. +# - Run is driven by `fastvideo/training/distillation.py --config `. + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29515} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +CONFIG=${CONFIG:-"examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" + diff --git a/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml b/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml new file mode 100644 index 000000000..859975e82 --- /dev/null +++ b/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml @@ -0,0 +1,83 @@ +recipe: + family: wan + method: dmd2 + +roles: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + family: wan + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + disable_custom_init_weights: true + critic: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + disable_custom_init_weights: true + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.4_wan2.1_dmd2_8steps_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer (student default) + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Critic overrides (DMD2-specific) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Method-agnostic knobs + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: phase3.4_wan_dmd2_8steps_wansyn + wandb_run_name: phase3.4_wan_dmd2_8steps + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "8" + validation_guidance_scale: "6.0" + +pipeline_config: + flow_shift: 8 + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/examples/distillation/phase3_4/finetune-vsa-temp.sh b/examples/distillation/phase3_4/finetune-vsa-temp.sh new file mode 100644 index 000000000..15a941935 --- /dev/null +++ b/examples/distillation/phase3_4/finetune-vsa-temp.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -euo pipefail +if [[ "${DEBUG:-0}" == "1" ]]; then + set -x +fi + +# Phase 3.4 finetune (SFT) with VSA backend launcher. +# +# Mirrors legacy finetune scripts: +# - FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN +# - VSA_* knobs live in the YAML (VSA_sparsity / VSA_decay_rate / VSA_decay_interval_steps) + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-VIDEO_SPARSE_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29516} +export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-"/tmp/triton_cache_${USER}_$$"} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +CONFIG=${CONFIG:-"examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" + diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml new file mode 100644 index 000000000..2c0407c3a --- /dev/null +++ b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml @@ -0,0 +1,76 @@ +recipe: + family: wan + method: finetune + +roles: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # Distributed (mirror legacy Wan VSA finetune scripts) + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.4_wan2.1_finetune_vsa_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer + learning_rate: 1.0e-6 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.1 + enable_gradient_checkpointing_type: full + + # VSA knobs (schedule ramps up sparsity during training) + VSA_sparsity: 0.9 + VSA_decay_rate: 0.03 + VSA_decay_interval_steps: 50 + + # Tracking / validation + tracker_project_name: phase3.4_wan_finetune_vsa_wansyn + wandb_run_name: phase3.4_wan_finetune_vsa + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 200 + validation_sampling_steps: "50" + validation_guidance_scale: "5.0" + +pipeline_config: + # Match legacy Wan VSA finetune scripts. + flow_shift: 1 + sampler_kind: ode + +method_config: + # Use the VSA attention metadata when FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN. + attn_kind: vsa diff --git a/fastvideo/distillation/__init__.py b/fastvideo/distillation/__init__.py index cf7287653..658f32560 100644 --- a/fastvideo/distillation/__init__.py +++ b/fastvideo/distillation/__init__.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 -from fastvideo.distillation.roles import ModelBundle, RoleHandle +from fastvideo.distillation.roles import RoleManager, RoleHandle from fastvideo.distillation.trainer import DistillTrainer __all__ = [ "DistillTrainer", - "ModelBundle", + "RoleManager", "RoleHandle", ] diff --git a/fastvideo/distillation/builder.py b/fastvideo/distillation/builder.py deleted file mode 100644 index c5900f1ed..000000000 --- a/fastvideo/distillation/builder.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from fastvideo.distillation.registry import get_model, get_method -from fastvideo.distillation.utils.config import DistillRuntime -from fastvideo.distillation.utils.config import DistillRunConfig - - -def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: - """Build a distillation runtime from a YAML config. - - This is the Phase 2.9 "elegant dispatch" entry for assembling: - - model components (bundle/adapter/dataloader/tracker/validator) - - method implementation (algorithm) on top of those components - """ - - model_builder = get_model(str(cfg.recipe.family)) - components = model_builder(cfg=cfg) - - method_builder = get_method(str(cfg.recipe.method)) - method = method_builder( - cfg=cfg, - bundle=components.bundle, - adapter=components.adapter, - validator=components.validator, - ) - - return DistillRuntime( - training_args=components.training_args, - method=method, - dataloader=components.dataloader, - tracker=components.tracker, - start_step=int(getattr(components, "start_step", 0) or 0), - ) diff --git a/fastvideo/distillation/registry.py b/fastvideo/distillation/dispatch.py similarity index 72% rename from fastvideo/distillation/registry.py rename to fastvideo/distillation/dispatch.py index 07d499086..be2fe3657 100644 --- a/fastvideo/distillation/registry.py +++ b/fastvideo/distillation/dispatch.py @@ -5,10 +5,10 @@ from collections.abc import Callable from typing import Any, Protocol -from fastvideo.distillation.roles import ModelBundle from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.utils.config import ModelComponents -from fastvideo.distillation.utils.config import DistillRunConfig +from fastvideo.distillation.models.components import ModelComponents +from fastvideo.distillation.roles import RoleManager +from fastvideo.distillation.utils.config import DistillRunConfig, DistillRuntime class ModelBuilder(Protocol): @@ -21,7 +21,7 @@ def __call__( self, *, cfg: DistillRunConfig, - bundle: ModelBundle, + bundle: RoleManager, adapter: Any, validator: Any | None, ) -> DistillMethod: @@ -95,3 +95,32 @@ def get_method(name: str) -> MethodBuilder: if name not in _METHODS: raise KeyError(f"Unknown method {name!r}. Available: {available_methods()}") return _METHODS[name] + + +def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: + """Build a distillation runtime from a YAML config. + + Assembles: + - model components (bundle/adapter/dataloader/tracker/validator) + - method implementation (algorithm) on top of those components + """ + + model_builder = get_model(str(cfg.recipe.family)) + components = model_builder(cfg=cfg) + + method_builder = get_method(str(cfg.recipe.method)) + method = method_builder( + cfg=cfg, + bundle=components.bundle, + adapter=components.adapter, + validator=components.validator, + ) + + return DistillRuntime( + training_args=components.training_args, + method=method, + dataloader=components.dataloader, + tracker=components.tracker, + start_step=int(getattr(components, "start_step", 0) or 0), + ) + diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index 79ddd446d..71177d5e7 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -11,7 +11,7 @@ 快速入口（从运行到训练）： `fastvideo/training/distillation.py` → `utils.config.load_distill_run_config()` → -`builder.build_runtime_from_config()` → `registry.get_model()/get_method()` → +`dispatch.build_runtime_from_config()` → `ModelComponents + DistillMethod` → `DistillTrainer.run()` --- @@ -25,8 +25,7 @@ - `utils/data.md` - `utils/tracking.md` - `utils/checkpoint.md` -- `registry.md` -- `builder.md` +- `dispatch.md` - `roles.md` - `trainer.md` @@ -53,8 +52,3 @@ - `validators/__init__.md` - `validators/base.md` - `validators/wan.md` - -### outside/ -- `outside/README.md` -- `outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md` -- `outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md` diff --git a/fastvideo/distillation/doc/__init__.md b/fastvideo/distillation/doc/__init__.md index 126834da2..506c128a8 100644 --- a/fastvideo/distillation/doc/__init__.md +++ b/fastvideo/distillation/doc/__init__.md @@ -5,10 +5,9 @@ **当前导出** - `DistillTrainer`：训练 loop（infra only） -- `ModelBundle` / `RoleHandle`：multi-role 模型与训练状态容器 +- `RoleManager` / `RoleHandle`：multi-role 模型与训练状态容器 **设计意图** - 让上层（例如 `fastvideo/training/distillation.py`）只依赖稳定 API： - `DistillTrainer.run(method, dataloader, ...)` - `bundle.role("student")` 等 role 访问模式 - diff --git a/fastvideo/distillation/doc/adapters/wan.md b/fastvideo/distillation/doc/adapters/wan.md index cf8d0b264..e6dff910d 100644 --- a/fastvideo/distillation/doc/adapters/wan.md +++ b/fastvideo/distillation/doc/adapters/wan.md @@ -1,7 +1,7 @@ # `fastvideo/distillation/adapters/wan.py` **定位** -- `WanAdapter` 是 Wan family 的 runtime 边界： +- `WanAdapter` 是 Wan model plugin 的 runtime 边界： - 把 FastVideo/Wan 的 batch schema、forward_context、attention metadata 等细节 封装为一组 **operation-centric primitives** - 不实现任何 method 的 rollout policy / step list / loss（这些属于 method） @@ -29,4 +29,3 @@ **边界（Phase 2.9）** - ✅ adapter 不保存/管理 few-step 的 denoising step list，也不决定 rollout 策略。 - ✅ adapter 不区分 `student/teacher/critic` 的专用方法；只提供通用操作，role 语义由 method 管理。 - diff --git a/fastvideo/distillation/doc/builder.md b/fastvideo/distillation/doc/builder.md deleted file mode 100644 index 3bf1baa20..000000000 --- a/fastvideo/distillation/doc/builder.md +++ /dev/null @@ -1,16 +0,0 @@ -# `fastvideo/distillation/builder.py` - -**目的** -- 把 YAML config（`DistillRunConfig`）装配成一个可运行的 `DistillRuntime`： - - `model plugin` 负责 build-time 产物（`ModelComponents`） - - `method` 负责算法（`DistillMethod`） - -**关键 API** -- `build_runtime_from_config(cfg) -> DistillRuntime` - - `model_builder = registry.get_model(cfg.recipe.family)` - - `method_builder = registry.get_method(cfg.recipe.method)` - - `method = method_builder(cfg=cfg, bundle=components.bundle, adapter=components.adapter, validator=components.validator)` - -**边界** -- ✅ 这里不写 `if model==... and method==...` 的 N×M 组合逻辑。 -- ✅ 这里只做“装配”，不包含训练 loop / loss / rollout / optimizer policy。 diff --git a/fastvideo/distillation/doc/dispatch.md b/fastvideo/distillation/doc/dispatch.md new file mode 100644 index 000000000..07f8d63df --- /dev/null +++ b/fastvideo/distillation/doc/dispatch.md @@ -0,0 +1,24 @@ +# `fastvideo/distillation/dispatch.py` + +**目的** +- 把 “可扩展注册（models/methods）” 与 “从 YAML 装配可运行 runtime” 收敛到一个入口文件： + - 新增 model 的成本 ≈ N + - 新增 method 的成本 ≈ M + - 不需要写 N×M 的 if/else 组合逻辑 + +**关键概念** +- `ModelBuilder(cfg) -> ModelComponents` +- `MethodBuilder(cfg, bundle, adapter, validator) -> DistillMethod` + +**关键 API** +- `register_model(name)` / `register_method(name)`：装饰器注册 +- `get_model(name)` / `get_method(name)`：查询（会触发内置注册） +- `available_models()` / `available_methods()` +- `build_runtime_from_config(cfg) -> DistillRuntime` + - 选择 model plugin：`get_model(cfg.recipe.family)` + - 选择 method：`get_method(cfg.recipe.method)` + +**边界** +- ✅ 这里只做“装配 + dispatch”，不包含训练 loop / loss / rollout / optimizer policy。 +- ✅ method 层保持算法高内聚；model plugin 层保持集成高内聚。 + diff --git a/fastvideo/distillation/doc/methods/__init__.md b/fastvideo/distillation/doc/methods/__init__.md index 10126797f..ec9324ab6 100644 --- a/fastvideo/distillation/doc/methods/__init__.md +++ b/fastvideo/distillation/doc/methods/__init__.md @@ -10,8 +10,8 @@ **设计意图** - method 层应当是 **模型无关** 的（不 import 具体 pipeline/模型实现）； - 任何 family 细节都通过 adapter primitives（protocol）注入。 + 任何 model plugin 细节都通过 adapter primitives（protocol）注入。 **实现细节** -- 该模块对 `DMD2Method` 使用 lazy import（`__getattr__`），避免 registry/builder 在 - import 时触发循环依赖（circular import）。 +- 该模块对 `DMD2Method` 使用 lazy import（`__getattr__`），避免 dispatch 在 + import 时触发循环依赖（circular import）；dispatch 侧会在第一次查询时触发内置注册。 diff --git a/fastvideo/distillation/doc/methods/base.md b/fastvideo/distillation/doc/methods/base.md index 199e110f1..efe383cba 100644 --- a/fastvideo/distillation/doc/methods/base.md +++ b/fastvideo/distillation/doc/methods/base.md @@ -7,7 +7,7 @@ - 不负责训练 loop（由 `DistillTrainer` 承担） **关键点** -- `DistillMethod` 持有 `bundle: ModelBundle`，并把所有 role 的 modules 放进 +- `DistillMethod` 持有 `bundle: RoleManager`，并把所有 role 的 modules 放进 `self.role_modules: ModuleDict`，便于 DDP/FSDP/ckpt 系统统一发现参数。 **需要子类实现的抽象方法** @@ -19,4 +19,3 @@ - `backward()`：对 `loss_map["total_loss"]` 做 backward（子类可覆写以处理多 ctx） - `optimizers_schedulers_step()`：按 `get_optimizers/get_lr_schedulers` 的结果 step - `optimizers_zero_grad()`：对当前 iteration 的 optimizers 清梯度 - diff --git a/fastvideo/distillation/doc/methods/consistency_model/__init__.md b/fastvideo/distillation/doc/methods/consistency_model/__init__.md index e4ce40650..da701805f 100644 --- a/fastvideo/distillation/doc/methods/consistency_model/__init__.md +++ b/fastvideo/distillation/doc/methods/consistency_model/__init__.md @@ -5,5 +5,4 @@ **期望的演进方向** - 通过 `@register_method("cm")`（示例）注册具体实现。 -- method 只包含算法与 update policy；family/adapter 提供运行时 primitives。 - +- method 只包含算法与 update policy；model plugin/adapter 提供运行时 primitives。 diff --git a/fastvideo/distillation/doc/methods/distribution_matching/__init__.md b/fastvideo/distillation/doc/methods/distribution_matching/__init__.md index 09dd705db..0215034bf 100644 --- a/fastvideo/distillation/doc/methods/distribution_matching/__init__.md +++ b/fastvideo/distillation/doc/methods/distribution_matching/__init__.md @@ -9,6 +9,5 @@ **扩展** - 新增方法时建议保持： - 算法逻辑在 method - - family/model 细节通过 adapter protocol 注入 + - model plugin 细节通过 adapter protocol 注入 - 注册通过 `@register_method("")` - diff --git a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md index a1fca6415..41deb5da8 100644 --- a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md +++ b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md @@ -2,12 +2,12 @@ **定位** - `DMD2Method`：DMD2 distillation 的算法实现（method layer）。 -- 该文件可以出现 DMD2/critic/fake_score 等算法术语；这些语义不应泄漏到 adapter/family。 +- 该文件可以出现 DMD2/critic/fake_score 等算法术语；这些语义不应泄漏到 adapter/model plugin。 **依赖与边界** - ✅ 不 import 任何具体模型/管线实现（Wan/SDXL/...）。 - ✅ 只依赖： - - `ModelBundle`/`RoleHandle`（获取 student/teacher/critic） + - `RoleManager`/`RoleHandle`（获取 student/teacher/critic） - adapter 提供的 primitives（通过 `_DMD2Adapter` Protocol） **需要的 roles** @@ -39,10 +39,10 @@ - `DMD2Method` 在初始化时创建并写回： - student 的 optimizer/scheduler：使用 `training.learning_rate/betas/lr_scheduler` - critic 的 optimizer/scheduler：优先使用 `training.fake_score_*` 覆盖（否则回退到 student） -- 这样 Wan family 可以完全不“懂” DMD2 的 critic 超参，从 build-time 层面解耦。 +- 这样 Wan model plugin 完全不需要理解 DMD2 的 critic 超参，从 build-time 层面解耦。 **validation 的归属（Phase 2.9）** -- `DMD2Method` 持有 family-specific `validator`（build-time 注入），并在 `log_validation()` 中调用。 +- `DMD2Method` 持有 model-plugin-specific `validator`（build-time 注入），并在 `log_validation()` 中调用。 - method 通过 `ValidationRequest` 明确传入 sampling steps/guidance 等参数； 同时通过 `ValidationRequest.sample_handle` 指定要采样的模型（通常是 student）， validator 负责执行采样与记录，保持 method-agnostic。 diff --git a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md index 3397ad738..9f29f88a6 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md @@ -7,8 +7,8 @@ - `finetune.py`：`FineTuneMethod` - 只要求 `roles.student` - loss/policy 在 method 层 - - 复用同一套 trainer/bundle/adapter/family/validator/checkpoint 基础设施 + - 复用同一套 trainer/roles/adapter/model plugin/validator/checkpoint 基础设施 **设计要点** - adapter 仍保持 operation-centric（`prepare_batch / predict_* / backward`），不内置 finetune 的 loss 语义。 -- family 负责 build-time：加载 student modules、shared components（VAE/scheduler）、dataloader、validator。 +- model plugin 负责 build-time：加载 student modules、shared components（VAE/scheduler）、dataloader、validator。 diff --git a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md index c4a03c16b..0227aab56 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md @@ -3,8 +3,8 @@ ## 目的 - 将 “finetuning / SFT” 以 `DistillMethod` 的方式接入 Phase 2+ 架构： - 复用 `DistillTrainer`（infra loop / accum / step / ckpt / validate 调用） - - 复用 `ModelBundle`（角色容器，finetune 只需要 `student`） - - 复用 family/adapter（加载与 primitives） + - 复用 `RoleManager`（角色容器，finetune 只需要 `student`） + - 复用 model plugin/adapter（加载与 primitives） finetune 可以被视为一种特殊的 distillation recipe：**只有 student + dataset**。 @@ -29,7 +29,7 @@ finetune 可以被视为一种特殊的 distillation recipe：**只有 student + 4. backward 通过 `adapter.backward(loss, ctx, ...)` 执行（确保 forward-context/activation ckpt 兼容） ## Optimizer / Scheduler -- 由 method 创建（而非 family）： +- 由 method 创建（而非 model plugin）： - 使用 `training.learning_rate / training.betas / training.lr_scheduler / ...` - 只为 `student` role 创建 `optimizer + lr_scheduler` @@ -38,4 +38,4 @@ finetune 可以被视为一种特殊的 distillation recipe：**只有 student + - 具体 pipeline 与采样 loop 由 validator + `pipeline_config.sampler_kind` 决定（默认 `ode`） ## 配置示例 -- `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` +- `examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` diff --git a/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md b/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md index 793d3cdef..0c2dce4cd 100644 --- a/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md +++ b/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md @@ -6,6 +6,5 @@ **期望的扩展方式** - 新增 KD method 时： - method 定义需要哪些 roles（student/teacher/aux_teacher/...） - - family 只负责加载这些 roles 的 modules 并构建 bundle/adapter + - model plugin 只负责加载这些 roles 的 modules 并构建 roles/adapter - optimizer/scheduler 由 method 创建并写回 handle - diff --git a/fastvideo/distillation/doc/models/__init__.md b/fastvideo/distillation/doc/models/__init__.md index 63b3d60b5..32b2ef8b2 100644 --- a/fastvideo/distillation/doc/models/__init__.md +++ b/fastvideo/distillation/doc/models/__init__.md @@ -1,14 +1,14 @@ # `fastvideo/distillation/models/__init__.py` **目的** -- families 是 build-time 插件层： +- models 是 build-time 插件层（model plugins）： - 从 YAML config 读取 role spec - 加载模型模块（transformer/vae/...） - - 构建 `ModelBundle` + - 构建 `RoleManager` - 构建 adapter / dataloader / tracker / validator -**为什么需要 families？** +**为什么需要 model plugins？** - 把 “装配/加载/数据/分布式细节” 与 “算法/rollout/loss/update policy” 分离： - - family 专注 build-time 高内聚 + - model plugin 专注 build-time 高内聚 - method 专注算法高内聚 - - entrypoint/builder 不需要 N×M 组合逻辑 + - entrypoint/dispatch 不需要 N×M 组合逻辑 diff --git a/fastvideo/distillation/doc/models/wan.md b/fastvideo/distillation/doc/models/wan.md index e88c4afbb..8a9ceccd7 100644 --- a/fastvideo/distillation/doc/models/wan.md +++ b/fastvideo/distillation/doc/models/wan.md @@ -17,7 +17,7 @@ - 根据 `RoleSpec.trainable` 设置 `requires_grad` + `train()/eval()` - 可选开启 activation checkpoint（仅对 trainable role） 3) **构建 bundle / adapter / dataloader** - - `bundle = ModelBundle(roles=role_handles)` + - `bundle = RoleManager(roles=role_handles)` - `adapter = WanAdapter(prompt_handle=student_handle, ...)` - dataloader：parquet + `pyarrow_schema_t2v` 4) **tracker / validator（可选）** diff --git a/fastvideo/distillation/doc/outside/README.md b/fastvideo/distillation/doc/outside/README.md deleted file mode 100644 index 1bc818db5..000000000 --- a/fastvideo/distillation/doc/outside/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# `fastvideo/distillation/outside/README.md` - -**目的** -- 解释 `fastvideo/distillation/outside/` 的存在理由： - - Phase 2/2.9 期间，我们需要引入新的 YAML config，但又不想直接改动主 repo 的 - `fastvideo/configs/` 树（避免冲突/侵入式修改）。 - -**约定** -- Phase 2 entrypoint（`fastvideo/training/distillation.py`）只接受**真实路径**： - - `--config fastvideo/distillation/outside/fastvideo/configs/distillation/.yaml` -- `outside/` 只放 data/config（不要放可 import 的 Python 代码）。 - -**推荐目录结构** -- `fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml` - diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md deleted file mode 100644 index 02614d9bc..000000000 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.md +++ /dev/null @@ -1,52 +0,0 @@ -# `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` - -这是一个可运行示例（schema v2）：**Wan few-step distillation（8 steps）+ DMD2**。 - ---- - -## 顶层结构 - -- `recipe:` - - `family: wan` → registry dispatch 到 `models/wan.py` - - `method: dmd2` → registry dispatch 到 `methods/distribution_matching/dmd2.py` -- `roles:` - - `student / teacher / critic` 三个 roles（role 名称本身由 method 解释语义） - - 每个 role 指定： - - `family`（默认可省略，继承 `recipe.family`） - - `path`（权重路径） - - `trainable`（是否训练） - - `disable_custom_init_weights`（可选；用于 teacher/critic 等 auxiliary roles 的加载语义） -- `training:` - - 主要复用 `TrainingArgs.from_kwargs()` 的字段集合（batch/shape/steps/logging 等） -- `pipeline_config:` - - Phase 2 允许 inline 提供（也可用 `pipeline_config_path` 指向文件） - ---- - -## 关键语义归属（Phase 2.9 视角） - -**Model plugin（Wan）关心：** -- `roles.*.path / trainable` -- `training.data_path / dataloader_num_workers / train_batch_size / seed / output_dir` -- Wan 相关的 shape 信息（`num_latent_t/num_height/num_width/...`） - -**Method（DMD2）关心：** -- update policy：`generator_update_interval` -- student rollout 相关：`method_config.rollout_mode` -- optimizer/scheduler（Phase 2.9 已迁移到 method 创建）： - - student：`learning_rate / betas / lr_scheduler` - - critic（DMD2 专属覆盖）：`fake_score_learning_rate / fake_score_betas / fake_score_lr_scheduler` -- few-step step list（single source of truth 在 `method_config`）： - - `method_config.dmd_denoising_steps` - -**Adapter（WanAdapter）关心：** -- 把 FastVideo/Wan 的 forward primitives 暴露给 method（不包含 step list/policy） - ---- - -## 备注（Phase 3.2 已完成） - -Phase 3.2 已将 validation sampling 的 ODE/SDE loop 做成可插拔 sampler： -- method 通过 `ValidationRequest(sampler_kind=..., sampling_timesteps=...)` 显式指定采样方式与 timesteps -- validator 使用统一的 `WanPipeline` 执行采样（不再依赖 `WanDMDPipeline`） -- 因此该 YAML 不再需要 `pipeline_config.dmd_denoising_steps` 这类重复字段 diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md deleted file mode 100644 index 63eb3ba88..000000000 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.md +++ /dev/null @@ -1,13 +0,0 @@ -# `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml` - -这是一个 **Phase 2.9** 的可运行示例（schema v2）： -Wan few-step distillation（8 steps）+ DMD2。 - -它与 `distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` 的结构基本一致，主要差异在于： -- `training.output_dir / tracker_project_name / wandb_run_name` 用于区分实验阶段。 - -备注： -- few-step step list 的 single source of truth 在 `method_config.dmd_denoising_steps`。 -- Phase 3.2 已将 validation 采样升级为可插拔的 ODE/SDE sampler： - - method 通过 `ValidationRequest` 显式指定 `sampler_kind` 与 `sampling_timesteps` - - validator 使用统一的 `WanPipeline` 执行采样（不再依赖 `WanDMDPipeline`） diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md deleted file mode 100644 index 832c69595..000000000 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.md +++ /dev/null @@ -1,14 +0,0 @@ -# `fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` - -这是一个 **Phase 3.1** 的可运行示例（schema v2：`recipe` + `method_config`）： -Wan few-step distillation（8 steps）+ DMD2。 - -它与 `distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` 的结构基本一致，主要用于强调 Phase 3.1 的配置语义： -- 顶层选择从 `distill` 升级为 `recipe`。 -- method knobs（例如 `rollout_mode` / `dmd_denoising_steps`）进入 `method_config`。 -- `WanAdapter.prepare_batch()` 不再读取 legacy 的 `training.simulate_generator_forward`。 - -备注： -- Phase 3.2 已将 validation 采样升级为可插拔的 ODE/SDE sampler： - - method 通过 `ValidationRequest` 显式指定 `sampler_kind` 与 `sampling_timesteps` - - validator 使用统一的 `WanPipeline` 执行采样（不再依赖 `WanDMDPipeline`） diff --git a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md b/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md deleted file mode 100644 index 275ffdbd6..000000000 --- a/fastvideo/distillation/doc/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.md +++ /dev/null @@ -1,9 +0,0 @@ -# `fastvideo/distillation/outside/fastvideo/configs/distillation/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` - -这是一个 **Phase 3.3** 的可运行示例：把 finetuning 作为一种 method 接入 Phase 2+ distillation scaffold。 - -关键点： -- `recipe.method = finetune` -- `roles` 里只提供 `student`（no teacher/critic） -- 训练 loss 由 `FineTuneMethod` 实现（与 legacy `training_pipeline.py` 的目标对齐） -- validation 通过 `ValidationRequest + WanValidator + WanPipeline` 执行（默认走 `ode` sampler） diff --git a/fastvideo/distillation/doc/registry.md b/fastvideo/distillation/doc/registry.md deleted file mode 100644 index 55b310d32..000000000 --- a/fastvideo/distillation/doc/registry.md +++ /dev/null @@ -1,21 +0,0 @@ -# `fastvideo/distillation/registry.py` - -**目的** -- 为 distillation 的 “model / method” 提供轻量 registry： - - 新增 model 的成本 ≈ N - - 新增 method 的成本 ≈ M - - build 组合不需要写 N×M 的 if/else - -**关键概念** -- `ModelBuilder(cfg) -> ModelComponents` -- `MethodBuilder(cfg, bundle, adapter, validator) -> DistillMethod` - -**关键 API** -- `register_model(name)` / `register_method(name)`：装饰器注册 -- `get_model(name)` / `get_method(name)`：查询（会触发内置注册） -- `ensure_builtin_registrations()`：显式 import 内置实现，避免 import 顺序隐式 bug -- `available_models()` / `available_methods()` - -**扩展方式** -- 新增 model：实现 `fastvideo/distillation/models/.py` 并用 `@register_model("")` -- 新增 method：实现 `fastvideo/distillation/methods/...` 并用 `@register_method("")` diff --git a/fastvideo/distillation/doc/roles.md b/fastvideo/distillation/doc/roles.md index 2273cb7d8..c9e377d9d 100644 --- a/fastvideo/distillation/doc/roles.md +++ b/fastvideo/distillation/doc/roles.md @@ -11,11 +11,11 @@ - `optimizers: dict[str, Optimizer]` / `lr_schedulers: dict[str, Any]` - `trainable: bool` - `require_module(name)`：强制获取模块（缺失则报错） -- `ModelBundle` +- `RoleManager` - `roles: dict[str, RoleHandle]` - `require_roles([...])`：method 在构造时校验依赖的 role 是否齐全 - `role(name)`：获取 handle **Phase 2.9 约定** -- family(model plugin) 负责 **load modules + 设置 trainable** 并创建 `ModelBundle` +- model plugin 负责 **load modules + 设置 trainable** 并创建 `RoleManager` - method 负责 **(按算法) 创建 optimizers/schedulers** 并写回对应的 `RoleHandle` diff --git a/fastvideo/distillation/doc/utils/__init__.md b/fastvideo/distillation/doc/utils/__init__.md index 8686baaa7..da858c058 100644 --- a/fastvideo/distillation/doc/utils/__init__.md +++ b/fastvideo/distillation/doc/utils/__init__.md @@ -1,10 +1,10 @@ # `fastvideo/distillation/utils/` **目的** -- 放置 distillation 子系统的中性工具代码（不属于某个 family/method）。 +- 放置 distillation 子系统的中性工具代码（不属于某个 model plugin / method）。 当前包含： -- `config.py`：YAML loader + schema/types + build-time artifacts + runtime 组装结果。 +- `config.py`：YAML loader + schema/types（`DistillRunConfig` / `DistillRuntime`）。 - `data.py`：通用 dataloader 构建（按 dataset kind/schema 复用 FastVideo 现有实现）。 - `tracking.py`：tracker 初始化（wandb / tensorboard 等）。 - `checkpoint.py`：role-based checkpoint/save-resume（Phase 2 runtime）。 diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index e2cc61098..b767c4e79 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -4,6 +4,10 @@ - 把 distillation 的 **YAML loader + schema/types + runtime 装配类型** 集中在一个更直觉的位置， 减少文件级概念数量。 +备注： +- model plugin 的 build-time 产物结构体 `ModelComponents` 在 + `fastvideo/distillation/models/components.py`（更贴近语义归属）。 + 这里包含： ## 1) YAML loader（schema v2；YAML-only） diff --git a/fastvideo/distillation/doc/utils/tracking.md b/fastvideo/distillation/doc/utils/tracking.md index 9283bc28f..883f11e02 100644 --- a/fastvideo/distillation/doc/utils/tracking.md +++ b/fastvideo/distillation/doc/utils/tracking.md @@ -10,4 +10,4 @@ - tracker log dir 默认在 `output_dir/tracker/` **设计意图** -- tracker 属于 infra：entrypoint/trainer 负责持有；method 只负责产出要 log 的 metrics/artifacts。 +- tracker 属于 infra：entrypoint/trainer 负责持有；method 只负责产出要 log 的 metrics/媒体（video/image/file 等，tracker API 里常叫 artifacts）。 diff --git a/fastvideo/distillation/doc/validators/__init__.md b/fastvideo/distillation/doc/validators/__init__.md index bc96924b3..e14c687f3 100644 --- a/fastvideo/distillation/doc/validators/__init__.md +++ b/fastvideo/distillation/doc/validators/__init__.md @@ -6,4 +6,4 @@ **当前导出** - `DistillValidator`：最小 validation 接口 - `ValidationRequest`：method 提供的 validation overrides -- `WanValidator`：Wan family 的 validation 采样与记录实现 +- `WanValidator`：Wan model plugin 的 validation 采样与记录实现 diff --git a/fastvideo/distillation/doc/validators/base.md b/fastvideo/distillation/doc/validators/base.md index 55c800391..f69f6af10 100644 --- a/fastvideo/distillation/doc/validators/base.md +++ b/fastvideo/distillation/doc/validators/base.md @@ -7,11 +7,11 @@ - `log_validation(step: int, request: ValidationRequest | None = None) -> None` `ValidationRequest` 用于 method 覆盖关键采样配置（steps/guidance/output_dir 等），让 validator -保持 family-specific、但 method-agnostic。 +保持 model-plugin-specific、但 method-agnostic。 `ValidationRequest.sample_handle` 用于由 method 明确指定“本次 validation 要采样哪个模型/权重” （例如 student / student_ema / refiner / ...）。validator 不应自行 hardcode 角色语义。 **设计意图** - trainer 只调用 `method.log_validation(step)`。 -- method 决定是否做 validation，并把 `ValidationRequest` 传给 family-specific validator。 +- method 决定是否做 validation，并把 `ValidationRequest` 传给 model-plugin-specific validator。 diff --git a/fastvideo/distillation/doc/validators/wan.md b/fastvideo/distillation/doc/validators/wan.md index 4767be116..f115ae05a 100644 --- a/fastvideo/distillation/doc/validators/wan.md +++ b/fastvideo/distillation/doc/validators/wan.md @@ -6,7 +6,7 @@ - 读取 validation dataset（json） - 调用 Wan pipeline 生成视频样本 - 保存 mp4 到 `output_dir` - - 通过 tracker（例如 wandb）记录 artifacts + - 通过 tracker（例如 wandb）记录媒体（video/image/file 等；tracker API 里常叫 artifacts） **关键点** - validator 运行在分布式环境下： @@ -23,4 +23,4 @@ **可演进方向（Phase 3+）** - 将 validation steps/guidance 等采样配置从 `TrainingArgs` 迁移到更明确的配置块（例如 `validation:`）。 -- 进一步抽象 validator API，使其更容易被不同 family/method 复用。 +- 进一步抽象 validator API，使其更容易被不同 model plugin / method 复用。 diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py index 3c0a76025..7b0266c4f 100644 --- a/fastvideo/distillation/methods/base.py +++ b/fastvideo/distillation/methods/base.py @@ -8,11 +8,11 @@ import torch -from fastvideo.distillation.roles import ModelBundle +from fastvideo.distillation.roles import RoleManager class DistillMethod(torch.nn.Module, ABC): - def __init__(self, bundle: ModelBundle) -> None: + def __init__(self, bundle: RoleManager) -> None: super().__init__() self.bundle = bundle self.role_modules = torch.nn.ModuleDict() diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 2422440fe..5ae559248 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -12,10 +12,10 @@ get_scheduler, ) -from fastvideo.distillation.roles import ModelBundle +from fastvideo.distillation.roles import RoleManager from fastvideo.distillation.roles import RoleHandle from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.registry import register_method +from fastvideo.distillation.dispatch import register_method from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.distillation.utils.config import DistillRunConfig @@ -94,11 +94,11 @@ class DMD2Method(DistillMethod): """DMD2 distillation algorithm (method layer). Owns the algorithmic orchestration (loss construction + update policy) and - stays independent of any specific model family. It requires a - :class:`~fastvideo.distillation.roles.ModelBundle` containing at least the + stays independent of any specific model plugin. It requires a + :class:`~fastvideo.distillation.roles.RoleManager` containing at least the roles ``student``, ``teacher``, and ``critic``. - All model-family details (how to run student rollout, teacher CFG + All model-plugin details (how to run student rollout, teacher CFG prediction, critic loss, and how to safely run backward under activation checkpointing/forward-context constraints) are delegated to the adapter passed in at construction time. @@ -107,7 +107,7 @@ class DMD2Method(DistillMethod): def __init__( self, *, - bundle: ModelBundle, + bundle: RoleManager, adapter: _DMD2Adapter, method_config: dict[str, Any] | None = None, validator: Any | None = None, @@ -691,7 +691,7 @@ def optimizers_schedulers_step(self, iteration: int) -> None: def build_dmd2_method( *, cfg: DistillRunConfig, - bundle: ModelBundle, + bundle: RoleManager, adapter: _DMD2Adapter, validator: Any | None, ) -> DistillMethod: diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 4ff7b251a..638cdc371 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -12,9 +12,9 @@ get_scheduler, ) -from fastvideo.distillation.roles import ModelBundle, RoleHandle +from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.registry import register_method +from fastvideo.distillation.dispatch import register_method from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.distillation.utils.config import DistillRunConfig @@ -23,7 +23,7 @@ class _FineTuneAdapter(Protocol): """Adapter contract for :class:`FineTuneMethod`. Finetuning is implemented as a method (algorithm layer) on top of the - family-provided adapter. The method must remain model-family agnostic, so + model-plugin-provided adapter. The method must remain model-plugin agnostic, so it consumes only operation-centric primitives exposed by the adapter. """ @@ -69,7 +69,7 @@ class FineTuneMethod(DistillMethod): def __init__( self, *, - bundle: ModelBundle, + bundle: RoleManager, adapter: _FineTuneAdapter, method_config: dict[str, Any] | None = None, validator: Any | None = None, @@ -337,7 +337,7 @@ def optimizers_schedulers_step(self, iteration: int) -> None: def build_finetune_method( *, cfg: DistillRunConfig, - bundle: ModelBundle, + bundle: RoleManager, adapter: _FineTuneAdapter, validator: Any | None, ) -> DistillMethod: diff --git a/fastvideo/distillation/models/components.py b/fastvideo/distillation/models/components.py new file mode 100644 index 000000000..81b01c467 --- /dev/null +++ b/fastvideo/distillation/models/components.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.distillation.roles import RoleManager + + +@dataclass(slots=True) +class ModelComponents: + """Build-time outputs produced by a model plugin. + + A model plugin is responsible for loading modules, constructing a + role container (`RoleManager`), and assembling shared + components needed by runtime adapters, dataloaders, validators, and + trackers. + """ + + training_args: TrainingArgs + bundle: RoleManager + adapter: Any + dataloader: Any + tracker: Any + validator: Any | None = None + start_step: int = 0 diff --git a/fastvideo/distillation/models/wan.py b/fastvideo/distillation/models/wan.py index 3edd5d536..8b6d7cfd9 100644 --- a/fastvideo/distillation/models/wan.py +++ b/fastvideo/distillation/models/wan.py @@ -8,10 +8,10 @@ import torch from fastvideo.distillation.adapters.wan import WanAdapter -from fastvideo.distillation.roles import ModelBundle, RoleHandle -from fastvideo.distillation.registry import register_model -from fastvideo.distillation.utils.config import ModelComponents +from fastvideo.distillation.roles import RoleHandle, RoleManager +from fastvideo.distillation.dispatch import register_model from fastvideo.distillation.utils.config import DistillRunConfig +from fastvideo.distillation.models.components import ModelComponents from fastvideo.distillation.utils.data import build_parquet_t2v_train_dataloader from fastvideo.distillation.utils.tracking import build_tracker from fastvideo.models.loader.component_loader import PipelineComponentLoader @@ -99,7 +99,7 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: for role, role_spec in roles_cfg.items(): if role_spec.family != "wan": raise ValueError( - "Wan family only supports wan roles; " + "Wan model plugin only supports roles with family='wan'; " f"got {role}={role_spec.family!r}" ) @@ -148,7 +148,7 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: trainable=bool(role_spec.trainable), ) - bundle = ModelBundle(roles=role_handles) + bundle = RoleManager(roles=role_handles) tracker = build_tracker(training_args, config=cfg.raw) validator = None @@ -160,11 +160,11 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: tracker=tracker, ) - # NOTE: adapter is the family runtime boundary; it may implement multiple + # NOTE: adapter is the model runtime boundary; it may implement multiple # method-specific protocols via duck typing. prompt_handle = role_handles.get("student") if prompt_handle is None: - raise ValueError("Wan family requires a 'student' role for prompt encoding") + raise ValueError("Wan model plugin requires a 'student' role for prompt encoding") adapter = WanAdapter( prompt_handle=prompt_handle, training_args=training_args, diff --git a/fastvideo/distillation/outside/README.md b/fastvideo/distillation/outside/README.md deleted file mode 100644 index fbd6b0310..000000000 --- a/fastvideo/distillation/outside/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# `outside/` (Phase 2 config root) - -This directory is a Phase 2 workaround for iterating on new YAML/JSON configs -without modifying the main repository's `fastvideo/configs/` tree. - -Phase 2 does **not** rewrite config paths automatically. Put configs under this -tree and pass the real path to the entrypoint (e.g. `--config -fastvideo/distillation/outside/fastvideo/configs/distillation/foo.yaml`). - -Recommended layout: - -- `fastvideo/distillation/outside/fastvideo/configs/distillation/*.yaml` - -Keep `outside/` for **data/config files only** (do not place importable Python -code here). diff --git a/fastvideo/distillation/roles.py b/fastvideo/distillation/roles.py index a971665bb..50330b376 100644 --- a/fastvideo/distillation/roles.py +++ b/fastvideo/distillation/roles.py @@ -24,7 +24,7 @@ def require_module(self, name: str) -> torch.nn.Module: @dataclass(slots=True) -class ModelBundle: +class RoleManager: roles: dict[RoleName, RoleHandle] def require_roles(self, roles: list[RoleName]) -> None: diff --git a/fastvideo/distillation/utils/checkpoint.py b/fastvideo/distillation/utils/checkpoint.py index 7ca637868..025dbacb8 100644 --- a/fastvideo/distillation/utils/checkpoint.py +++ b/fastvideo/distillation/utils/checkpoint.py @@ -13,7 +13,7 @@ import torch.distributed as dist import torch.distributed.checkpoint as dcp -from fastvideo.distillation.roles import ModelBundle, RoleHandle +from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.logger import init_logger from fastvideo.training.checkpointing_utils import ( ModelWrapper, @@ -144,7 +144,7 @@ class DistillCheckpointManager: def __init__( self, *, - bundle: ModelBundle, + bundle: RoleManager, dataloader: Any, output_dir: str, config: DistillCheckpointConfig, diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index cea11b064..3fb3546f4 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -11,7 +11,6 @@ if TYPE_CHECKING: from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.distillation.roles import ModelBundle from fastvideo.distillation.methods.base import DistillMethod RoleName = str @@ -19,7 +18,7 @@ @dataclass(slots=True) class RecipeSpec: - """Selects the model family + training method. + """Selects the model plugin key (``recipe.family``) + training method. This is intentionally small: everything else (roles, training args, and pipeline config) lives in the run config. @@ -55,7 +54,7 @@ def _resolve_existing_file(path: str) -> str: Distillation intentionally does not perform any "overlay" path rewriting. The caller must pass a real file path (typically under - ``fastvideo/distillation/outside/``). + ``examples/distillation/``). """ if not path: @@ -200,24 +199,6 @@ def load_distill_run_config(path: str) -> DistillRunConfig: ) -@dataclass(slots=True) -class ModelComponents: - """Build-time outputs produced by a model plugin. - - A model plugin is responsible for loading modules, constructing a - `ModelBundle`, and assembling shared components needed by runtime adapters, - dataloaders, validators, and trackers. - """ - - training_args: TrainingArgs - bundle: ModelBundle - adapter: Any - dataloader: Any - tracker: Any - validator: Any | None = None - start_step: int = 0 - - @dataclass(slots=True) class DistillRuntime: """Fully assembled runtime for `DistillTrainer.run()`.""" diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py index 036fe82f1..1adfed12c 100644 --- a/fastvideo/distillation/validators/base.py +++ b/fastvideo/distillation/validators/base.py @@ -13,7 +13,7 @@ class ValidationRequest: """Method-provided validation configuration overrides. - Validators are family-specific (e.g. Wan sampling), but should remain + Validators are model-plugin-specific (e.g. Wan sampling), but should remain method-agnostic. A method may override key sampling parameters by passing a request object here. """ diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 5dcedcb6b..4fac2fda4 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -272,13 +272,15 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) imageio.mimsave(filename, video, fps=sampling_param.fps) video_filenames.append(filename) - artifacts = [] + video_logs = [] for filename, caption in zip(video_filenames, all_captions, strict=True): video_artifact = self.tracker.video(filename, caption=caption) if video_artifact is not None: - artifacts.append(video_artifact) - if artifacts: - logs = {f"validation_videos_{num_inference_steps}_steps": artifacts} + video_logs.append(video_artifact) + if video_logs: + logs = {f"validation_videos_{num_inference_steps}_steps": video_logs} + # Tracker API name uses "artifacts" to mean media/files + # (e.g., videos for W&B). We keep the upstream API name. self.tracker.log_artifacts(logs, step) else: self.world_group.send_object(result.videos, dst=0) diff --git a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py index 0064f44c1..bd024485e 100644 --- a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py +++ b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py @@ -1,6 +1,6 @@ import torch -from fastvideo.distillation.roles import ModelBundle, RoleHandle +from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.distillation.methods.base import DistillMethod @@ -33,19 +33,18 @@ def __init__(self, interval: int) -> None: self.critic_opt = _FakeOptimizer() self.student_sched = _FakeScheduler() self.critic_sched = _FakeScheduler() - bundle = ModelBundle( + bundle = RoleManager( roles={ - "student": - RoleHandle( + "student": RoleHandle( optimizers={"main": self.student_opt}, lr_schedulers={"main": self.student_sched}, ), - "critic": - RoleHandle( + "critic": RoleHandle( optimizers={"main": self.critic_opt}, lr_schedulers={"main": self.critic_sched}, ), - }) + } + ) super().__init__(bundle) self.interval = interval diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 935246971..5bee3ab08 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -27,7 +27,7 @@ def run_distillation_from_config( DistillCheckpointConfig, DistillCheckpointManager, ) - from fastvideo.distillation.builder import build_runtime_from_config + from fastvideo.distillation.dispatch import build_runtime_from_config from fastvideo.distillation.utils.config import load_distill_run_config cfg = load_distill_run_config(config_path) From a866c4a532088b0fd955e1acf9df562ec5147632 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 00:01:07 +0000 Subject: [PATCH 105/214] log metrics --- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml | 4 +- .../finetune_wan2.1_t2v_1.3B_phase3.4.yaml | 70 +++++++++++++++++ ...finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml | 8 +- ...2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml | 76 +++++++++++++++++++ fastvideo/distillation/doc/methods/base.md | 3 + fastvideo/distillation/doc/trainer.md | 4 +- fastvideo/distillation/methods/base.py | 4 +- .../methods/distribution_matching/dmd2.py | 7 +- .../methods/fine_tuning/finetune.py | 7 +- fastvideo/distillation/trainer.py | 43 +++++++++-- .../test_optimizer_scheduler_alignment.py | 2 +- 11 files changed, 208 insertions(+), 20 deletions(-) create mode 100644 examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml create mode 100644 examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml diff --git a/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml b/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml index 859975e82..c84ede635 100644 --- a/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml +++ b/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml @@ -64,8 +64,8 @@ training: enable_gradient_checkpointing_type: full # Tracking / validation - tracker_project_name: phase3.4_wan_dmd2_8steps_wansyn - wandb_run_name: phase3.4_wan_dmd2_8steps + tracker_project_name: distillation_phase3 + wandb_run_name: phase3.4_wan_dmd2_8steps_simulate log_validation: true validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json validation_steps: 50 diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml new file mode 100644 index 000000000..6ba3edc96 --- /dev/null +++ b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml @@ -0,0 +1,70 @@ +recipe: + family: wan + method: finetune + +roles: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # Distributed (mirror legacy Wan VSA finetune scripts) + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.4_wan2.1_finetune_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer + learning_rate: 1.0e-6 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.1 + enable_gradient_checkpointing_type: full + + + # Tracking / validation + tracker_project_name: distillation_phase3 + wandb_run_name: phase3.4_wan_finetune + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "50" + validation_guidance_scale: "5.0" + +pipeline_config: + # Match legacy Wan VSA finetune scripts. + flow_shift: 1 + sampler_kind: ode + +method_config: {} diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml index 2c0407c3a..2932491ea 100644 --- a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml +++ b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml @@ -53,16 +53,16 @@ training: enable_gradient_checkpointing_type: full # VSA knobs (schedule ramps up sparsity during training) - VSA_sparsity: 0.9 + VSA_sparsity: 0.7 VSA_decay_rate: 0.03 VSA_decay_interval_steps: 50 # Tracking / validation - tracker_project_name: phase3.4_wan_finetune_vsa_wansyn - wandb_run_name: phase3.4_wan_finetune_vsa + tracker_project_name: distillation_phase3 + wandb_run_name: phase3.4_wan_finetune_vsa_0.7_0.03_50 log_validation: true validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 200 + validation_steps: 50 validation_sampling_steps: "50" validation_guidance_scale: "5.0" diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml new file mode 100644 index 000000000..ae0b84684 --- /dev/null +++ b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml @@ -0,0 +1,76 @@ +recipe: + family: wan + method: finetune + +roles: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # Distributed (mirror legacy Wan VSA finetune scripts) + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.4_wan2.1_finetune_vsa_0.7_0.03_1_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer + learning_rate: 1.0e-6 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.1 + enable_gradient_checkpointing_type: full + + # VSA knobs (schedule ramps up sparsity during training) + VSA_sparsity: 0.7 + VSA_decay_rate: 0.03 + VSA_decay_interval_steps: 1 + + # Tracking / validation + tracker_project_name: distillation_phase3 + wandb_run_name: phase3.4_wan_finetune_vsa_0.7_0.03_1 + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "50" + validation_guidance_scale: "5.0" + +pipeline_config: + # Match legacy Wan VSA finetune scripts. + flow_shift: 1 + sampler_kind: ode + +method_config: + # Use the VSA attention metadata when FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN. + attn_kind: vsa diff --git a/fastvideo/distillation/doc/methods/base.md b/fastvideo/distillation/doc/methods/base.md index efe383cba..f153a2aa1 100644 --- a/fastvideo/distillation/doc/methods/base.md +++ b/fastvideo/distillation/doc/methods/base.md @@ -12,6 +12,9 @@ **需要子类实现的抽象方法** - `single_train_step(batch, iteration, current_vsa_sparsity=...)` + - 返回：`(loss_map, outputs, metrics)` + - `loss_map: dict[str, Tensor]`：必须包含 `total_loss`（用于 backward） + - `metrics: dict[str, scalar]`：额外要 log 的标量（float/int/0-dim Tensor） - `get_optimizers(iteration)` - `get_lr_schedulers(iteration)` diff --git a/fastvideo/distillation/doc/trainer.md b/fastvideo/distillation/doc/trainer.md index 51f8222aa..591dca6b9 100644 --- a/fastvideo/distillation/doc/trainer.md +++ b/fastvideo/distillation/doc/trainer.md @@ -17,6 +17,9 @@ `run()` 通过 duck-typing 调用（存在则调用）： - `method.on_train_start()` - `method.single_train_step(batch, step, current_vsa_sparsity=...)` + - 返回：`(loss_map, outputs, metrics)` + - `loss_map`：用于 backward 与默认标量 logging（`total_loss` 必需） + - `metrics`：额外的标量指标（非 loss，但希望记录到 tracker） - `method.backward(loss_map, outputs, grad_accum_rounds=...)` - `method.optimizers_schedulers_step(step)` - `method.optimizers_zero_grad(step)` @@ -24,4 +27,3 @@ **重要边界** - trainer 不应知道 roles（student/teacher/critic/...）也不应知道具体算法； optimizer cadence、multi-optimizer 更新策略都应由 method 决定并暴露为 hook。 - diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py index 7b0266c4f..e9d4ef655 100644 --- a/fastvideo/distillation/methods/base.py +++ b/fastvideo/distillation/methods/base.py @@ -10,6 +10,8 @@ from fastvideo.distillation.roles import RoleManager +LogScalar = float | int | torch.Tensor + class DistillMethod(torch.nn.Module, ABC): def __init__(self, bundle: RoleManager) -> None: @@ -27,7 +29,7 @@ def single_train_step( iteration: int, *, current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: raise NotImplementedError @abstractmethod diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 5ae559248..e2caa7334 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -14,7 +14,7 @@ from fastvideo.distillation.roles import RoleManager from fastvideo.distillation.roles import RoleHandle -from fastvideo.distillation.methods.base import DistillMethod +from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.dispatch import register_method from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.distillation.utils.config import DistillRunConfig @@ -589,7 +589,7 @@ def single_train_step( iteration: int, *, current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: latents_source: Literal["data", "zeros"] = "data" if self._rollout_mode == "simulate": latents_source = "zeros" @@ -628,7 +628,8 @@ def single_train_step( "student_ctx": student_ctx, "critic_ctx": critic_ctx, } - return loss_map, outputs + metrics: dict[str, LogScalar] = {"update_student": float(update_student)} + return loss_map, outputs, metrics def backward( self, diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 638cdc371..cd2de6d4d 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -13,7 +13,7 @@ ) from fastvideo.distillation.roles import RoleHandle, RoleManager -from fastvideo.distillation.methods.base import DistillMethod +from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.dispatch import register_method from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.distillation.utils.config import DistillRunConfig @@ -247,7 +247,7 @@ def single_train_step( iteration: int, *, current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any]]: + ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: del iteration training_batch = self.adapter.prepare_batch( batch, @@ -299,7 +299,8 @@ def single_train_step( outputs: dict[str, Any] = { "_fv_backward": (training_batch.timesteps, attn_metadata) } - return loss_map, outputs + metrics: dict[str, LogScalar] = {} + return loss_map, outputs, metrics def backward( self, diff --git a/fastvideo/distillation/trainer.py b/fastvideo/distillation/trainer.py index 7dbaabac8..17f29d5ac 100644 --- a/fastvideo/distillation/trainer.py +++ b/fastvideo/distillation/trainer.py @@ -15,6 +15,20 @@ from fastvideo.training.trackers import BaseTracker, DummyTracker +def _coerce_log_scalar(value: Any, *, where: str) -> float: + if isinstance(value, torch.Tensor): + if value.numel() != 1: + raise ValueError( + f"Expected scalar tensor at {where}, got shape={tuple(value.shape)}" + ) + return float(value.detach().item()) + if isinstance(value, (float, int)): + return float(value) + raise TypeError( + f"Expected a scalar (float/int/Tensor) at {where}, got {type(value).__name__}" + ) + + @dataclass(slots=True) class TrainLoopState: step: int @@ -23,8 +37,12 @@ class TrainLoopState: class DistillTrainer: - def __init__(self, training_args: TrainingArgs, *, tracker: BaseTracker - | None = None) -> None: + def __init__( + self, + training_args: TrainingArgs, + *, + tracker: BaseTracker | None = None, + ) -> None: self.training_args = training_args self.world_group = get_world_group() self.sp_group = get_sp_group() @@ -47,8 +65,10 @@ def _get_current_vsa_sparsity(self, step: int) -> float: vsa_decay_rate = self.training_args.VSA_decay_rate vsa_decay_interval_steps = self.training_args.VSA_decay_interval_steps if vsa_decay_interval_steps > 1: - current_decay_times = min(step // vsa_decay_interval_steps, - int(vsa_sparsity // vsa_decay_rate)) + current_decay_times = min( + step // vsa_decay_interval_steps, + int(vsa_sparsity // vsa_decay_rate), + ) return current_decay_times * vsa_decay_rate return vsa_sparsity @@ -96,10 +116,11 @@ def run( current_vsa_sparsity = self._get_current_vsa_sparsity(step) loss_sums: dict[str, float] = {} + metric_sums: dict[str, float] = {} for accum_iter in range(grad_accum): batch = next(data_stream) if hasattr(method, "single_train_step"): - loss_map, outputs = method.single_train_step( # type: ignore[attr-defined] + loss_map, outputs, step_metrics = method.single_train_step( # type: ignore[attr-defined] batch, step, current_vsa_sparsity=current_vsa_sparsity, @@ -122,6 +143,16 @@ def run( if isinstance(v, torch.Tensor): loss_sums[k] = loss_sums.get(k, 0.0) + float( v.detach().item()) + for k, v in step_metrics.items(): + if k in loss_sums: + raise ValueError( + f"Metric key {k!r} collides with loss key. " + "Use a different name (e.g. prefix with 'train/')." + ) + metric_sums[k] = metric_sums.get(k, 0.0) + _coerce_log_scalar( + v, + where=f"method.single_train_step().metrics[{k!r}]", + ) if hasattr(method, "optimizers_schedulers_step"): method.optimizers_schedulers_step(step) # type: ignore[attr-defined] @@ -129,7 +160,9 @@ def run( method.optimizers_zero_grad(step) # type: ignore[attr-defined] metrics = {k: v / grad_accum for k, v in loss_sums.items()} + metrics.update({k: v / grad_accum for k, v in metric_sums.items()}) metrics["step_time_sec"] = time.perf_counter() - t0 + metrics["vsa_sparsity"] = float(current_vsa_sparsity) if self.global_rank == 0 and metrics: self.tracker.log(metrics, step) diff --git a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py index bd024485e..7daa66b9d 100644 --- a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py +++ b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py @@ -53,7 +53,7 @@ def _update_student(self, iteration: int) -> bool: def single_train_step(self, batch, iteration, *, current_vsa_sparsity=0.0): # noqa: ANN001, ANN201 loss = torch.zeros((), requires_grad=True) - return {"total_loss": loss}, {} + return {"total_loss": loss}, {}, {} def get_optimizers(self, iteration): # noqa: ANN001, ANN201 optimizers = [self.critic_opt] From 64b9feabc28dbea9664139afdab6168e6a92ea29 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 00:43:34 +0000 Subject: [PATCH 106/214] use flowshift=3 validator --- .../phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml | 3 +-- .../phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml | 2 +- .../finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml index 6ba3edc96..24ce82021 100644 --- a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml +++ b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml @@ -63,8 +63,7 @@ training: validation_guidance_scale: "5.0" pipeline_config: - # Match legacy Wan VSA finetune scripts. - flow_shift: 1 + flow_shift: 3 sampler_kind: ode method_config: {} diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml index 2932491ea..1c5b5be49 100644 --- a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml +++ b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml @@ -68,7 +68,7 @@ training: pipeline_config: # Match legacy Wan VSA finetune scripts. - flow_shift: 1 + flow_shift: 3 sampler_kind: ode method_config: diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml index ae0b84684..c8ff8c73a 100644 --- a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml +++ b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml @@ -68,7 +68,7 @@ training: pipeline_config: # Match legacy Wan VSA finetune scripts. - flow_shift: 1 + flow_shift: 3 sampler_kind: ode method_config: From 7399670f0c7313ba919860d489c67118a1bcef3e Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 03:19:42 +0000 Subject: [PATCH 107/214] tracker, utils, loader --- ...2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 76 +++++ fastvideo/distillation/dispatch.py | 39 +-- fastvideo/distillation/doc/README.md | 6 +- fastvideo/distillation/doc/RFC-v1.md | 300 ++++++++++++++++++ fastvideo/distillation/doc/dispatch.md | 5 +- fastvideo/distillation/doc/methods/base.md | 2 + .../doc/methods/distribution_matching/dmd2.md | 4 + .../doc/methods/fine_tuning/finetune.md | 4 + fastvideo/distillation/doc/models/wan.md | 6 +- fastvideo/distillation/doc/trainer.md | 3 +- fastvideo/distillation/doc/utils/__init__.md | 6 +- fastvideo/distillation/doc/utils/config.md | 5 +- .../doc/utils/{data.md => dataloader.md} | 2 +- .../distillation/doc/utils/module_state.md | 14 + .../distillation/doc/utils/moduleloader.md | 17 + fastvideo/distillation/doc/utils/tracking.md | 3 +- fastvideo/distillation/doc/validators/wan.md | 4 + fastvideo/distillation/methods/base.py | 37 ++- .../methods/distribution_matching/dmd2.py | 33 +- .../methods/fine_tuning/finetune.py | 33 +- fastvideo/distillation/models/components.py | 1 - fastvideo/distillation/models/wan.py | 71 +---- fastvideo/distillation/trainer.py | 9 +- fastvideo/distillation/utils/config.py | 1 - .../utils/{data.py => dataloader.py} | 1 - fastvideo/distillation/utils/module_state.py | 17 + fastvideo/distillation/utils/moduleloader.py | 61 ++++ fastvideo/distillation/validators/wan.py | 8 +- fastvideo/training/distillation.py | 5 +- 29 files changed, 628 insertions(+), 145 deletions(-) create mode 100644 examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml create mode 100644 fastvideo/distillation/doc/RFC-v1.md rename fastvideo/distillation/doc/utils/{data.md => dataloader.md} (94%) create mode 100644 fastvideo/distillation/doc/utils/module_state.md create mode 100644 fastvideo/distillation/doc/utils/moduleloader.md rename fastvideo/distillation/utils/{data.py => dataloader.py} (99%) create mode 100644 fastvideo/distillation/utils/module_state.py create mode 100644 fastvideo/distillation/utils/moduleloader.py diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml new file mode 100644 index 000000000..7e2d92a13 --- /dev/null +++ b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml @@ -0,0 +1,76 @@ +recipe: + family: wan + method: finetune + +roles: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # Distributed (mirror legacy Wan VSA finetune scripts) + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder) + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + + # Batch / shape (matches Wan-Syn 77x448x832) + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + # Output / steps + output_dir: outputs/phase3.4_wan2.1_finetune_vsa_0.7_0.03_1_wansyn + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer + learning_rate: 1.0e-6 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.1 + enable_gradient_checkpointing_type: full + + # VSA knobs (schedule ramps up sparsity during training) + VSA_sparsity: 0.9 + VSA_decay_rate: 0.03 + VSA_decay_interval_steps: 1 + + # Tracking / validation + tracker_project_name: distillation_phase3 + wandb_run_name: phase3.4_wan_finetune_vsa_0.9_0.03_1 + log_validation: true + validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + validation_steps: 50 + validation_sampling_steps: "50" + validation_guidance_scale: "5.0" + +pipeline_config: + # Match legacy Wan VSA finetune scripts. + flow_shift: 3 + sampler_kind: ode + +method_config: + # Use the VSA attention metadata when FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN. + attn_kind: vsa diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index be2fe3657..4a856ddc2 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -3,11 +3,10 @@ from __future__ import annotations from collections.abc import Callable -from typing import Any, Protocol +from typing import Protocol from fastvideo.distillation.methods.base import DistillMethod from fastvideo.distillation.models.components import ModelComponents -from fastvideo.distillation.roles import RoleManager from fastvideo.distillation.utils.config import DistillRunConfig, DistillRuntime @@ -16,20 +15,8 @@ def __call__(self, *, cfg: DistillRunConfig) -> ModelComponents: ... -class MethodBuilder(Protocol): - def __call__( - self, - *, - cfg: DistillRunConfig, - bundle: RoleManager, - adapter: Any, - validator: Any | None, - ) -> DistillMethod: - ... - - _MODELS: dict[str, ModelBuilder] = {} -_METHODS: dict[str, MethodBuilder] = {} +_METHODS: dict[str, type[DistillMethod]] = {} _BUILTINS_REGISTERED = False @@ -47,16 +34,20 @@ def decorator(builder: ModelBuilder) -> ModelBuilder: return decorator -def register_method(name: str) -> Callable[[MethodBuilder], MethodBuilder]: +def register_method( + name: str, +) -> Callable[[type[DistillMethod]], type[DistillMethod]]: name = str(name).strip() if not name: raise ValueError("method name cannot be empty") - def decorator(builder: MethodBuilder) -> MethodBuilder: + def decorator(method_cls: type[DistillMethod]) -> type[DistillMethod]: if name in _METHODS: raise KeyError(f"Method already registered: {name!r}") - _METHODS[name] = builder - return builder + if not issubclass(method_cls, DistillMethod): + raise TypeError(f"Registered method must subclass DistillMethod: {method_cls}") + _METHODS[name] = method_cls + return method_cls return decorator @@ -90,7 +81,7 @@ def get_model(name: str) -> ModelBuilder: return _MODELS[name] -def get_method(name: str) -> MethodBuilder: +def get_method(name: str) -> type[DistillMethod]: ensure_builtin_registrations() if name not in _METHODS: raise KeyError(f"Unknown method {name!r}. Available: {available_methods()}") @@ -101,15 +92,15 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: """Build a distillation runtime from a YAML config. Assembles: - - model components (bundle/adapter/dataloader/tracker/validator) + - model components (bundle/adapter/dataloader/validator) - method implementation (algorithm) on top of those components """ model_builder = get_model(str(cfg.recipe.family)) components = model_builder(cfg=cfg) - method_builder = get_method(str(cfg.recipe.method)) - method = method_builder( + method_cls = get_method(str(cfg.recipe.method)) + method = method_cls.build( cfg=cfg, bundle=components.bundle, adapter=components.adapter, @@ -120,7 +111,5 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: training_args=components.training_args, method=method, dataloader=components.dataloader, - tracker=components.tracker, start_step=int(getattr(components, "start_step", 0) or 0), ) - diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index 71177d5e7..ab6fc823b 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -5,7 +5,7 @@ 设计原则（对应 Phase 2.9）： - **Trainer** 只做 infra（loop/accum/日志/ckpt/validate 调用），不包含算法策略。 - **Method** 只做算法（loss + update policy + 需要哪些 roles）。 -- **Model plugin** 只做装配（build-time：加载 modules、构建 bundle/adapter/dataloader/validator/tracker；代码在 `models/`）。 +- **Model plugin** 只做装配（build-time：加载 modules、构建 bundle/adapter/dataloader/validator；代码在 `models/`）。 - **Adapter** 只做运行时 primitive（step-time：prepare_batch/forward_context/predict/backward 等）， API 以 operation 为中心，不以 role 为中心（避免 role 爆炸）。 @@ -22,7 +22,9 @@ - `__init__.md` - `utils/__init__.md` - `utils/config.md` -- `utils/data.md` +- `utils/dataloader.md` +- `utils/moduleloader.md` +- `utils/module_state.md` - `utils/tracking.md` - `utils/checkpoint.md` - `dispatch.md` diff --git a/fastvideo/distillation/doc/RFC-v1.md b/fastvideo/distillation/doc/RFC-v1.md new file mode 100644 index 000000000..cb11bb9cc --- /dev/null +++ b/fastvideo/distillation/doc/RFC-v1.md @@ -0,0 +1,300 @@ +# RFC-v1: FastVideo Distillation（截至 Phase 3） + +本 RFC 记录：截至 Phase 3 结束时，FastVideo 新 distillation/finetuning 框架的**目录结构**、 +**每一层的职责边界**、以及这样设计的原因与当前已达成的效果。 + +> 参考项目：`~/alex/pkgs/FastGen`（FastGen 的 “Trainer ↔ Method ↔ Network” 分层对我们影响很大）。 + +--- + +## 1. 背景与目标 + +### 1.1 旧代码的问题（动机） + +历史上 FastVideo 的 distillation 主要以 `fastvideo/training/*distillation_pipeline*.py` 为核心： +- **model 类型（Wan）与 distill 算法（DMD2 / self-forcing / …）强耦合**； +- training loop、算法策略、validation/sampling、conditioning 初始化混在一起； +- 扩展一个新模型或新方法，往往意味着复制/改造一整条 pipeline； +- reviewer 很难在一个 PR 里把所有 coupling 看清楚。 + +### 1.2 新框架的目标 + +我们希望把 distillation/finetuning 做成可组合的三件事： +- **Model plugin（模型家族差异）**：负责“装配（build-time）”与共享组件。 +- **Method（算法）**：只关心 loss / rollout / update policy / validation policy。 +- **Trainer（基础设施）**：只负责 loop/accum/step/ckpt/log，不包含策略。 + +并额外引入： +- **Adapter（运行时 primitive）**：把 FastVideo 的 pipeline/forward 细节收敛成 method 可复用的操作接口； +- **RoleManager / RoleHandle（多角色容器）**：支持任意角色组合（student/teacher/critic/reward/...），避免“固定三件套”的硬编码。 + +最终目标是：**N 个模型插件 + M 个方法 = N+M 的扩展成本**，而不是 N×M 的构建爆炸。 + +--- + +## 2. 当前目录结构（Phase 3） + +下面是 `fastvideo/distillation/` 的“可读结构图”（省略 `__pycache__/`）： + +```text +fastvideo/distillation/ + __init__.py + trainer.py + dispatch.py + roles.py + + adapters/ + base.py + wan.py + + models/ # (原 families) 现在叫 models = model plugins + components.py + wan.py + + methods/ + __init__.py + base.py + distribution_matching/ + __init__.py + dmd2.py + fine_tuning/ + __init__.py + finetune.py + knowledge_distillation/ # 预留目录（Phase 3 结束时可为空/占位） + __init__.py + consistency_model/ # 预留目录（Phase 3 结束时可为空/占位） + __init__.py + + validators/ + base.py + wan.py + + utils/ + __init__.py + config.py # YAML 解析 + DistillRunConfig/Runtime 数据结构 + data.py # dataloader 构建（当前以 parquet T2V 为主） + tracking.py # tracker 构建（W&B 等） + checkpoint.py # save/resume 管理 + + doc/ + README.md # file-by-file 索引与设计原则 + RFC-v1.md # 本文件 + ... # 其它 file-by-file docs +``` + +此外，新框架的统一入口在： +- `fastvideo/training/distillation.py`：YAML-only entrypoint（不再兼容旧 CLI configs）。 + +--- + +## 3. 为什么这样分层（核心抽象与职责边界） + +### 3.1 `trainer.py`：DistillTrainer（infra only） + +**职责：** +- 迭代 dataloader + gradient accumulation +- 调用 `method.single_train_step()` 得到 loss/metrics +- backward + optimizer/scheduler step/zero_grad +- checkpoint save/resume（通过 utils/checkpoint） +- tracker 记录（统一在 trainer 侧 log） + +**不做的事：** +- 不知道 “Wan / CogVideoX / …” +- 不知道 “DMD2 / finetune / …” +- 不知道 “teacher/student/critic 这些角色语义” + +这样做的原因（FastGen 启发）： +> FastGen 的 Trainer 非常薄，算法更新策略由 method 决定；Trainer 只做 orchestration。 +这能显著降低 reviewer 读 loop 的心智负担，并避免把 update policy 固化在 Trainer 中。 + +### 3.2 `methods/`：DistillMethod（算法层） + +**职责：** +- 定义训练一步：`single_train_step(batch, iteration, current_vsa_sparsity=...)` + - 返回：`loss_map`（tensor）、`outputs`（任意）、`metrics`（可 log 的标量） +- 定义 update policy：`get_optimizers()` / `get_lr_schedulers()` + - multi-optimizer / 不同 update interval 的策略属于算法的一部分 +- 定义 validation policy：method 构造 `ValidationRequest`，告诉 validator： + - 用哪个 role 的 transformer sample（通常是 student） + - sampler_kind（ode/sde）、steps、timesteps list、guidance_scale 等 + +**不做的事：** +- 不去关心具体模型加载（transformer/vae/text_encoder 的来源与细节） +- 不直接依赖 FastVideo pipeline 结构（通过 adapter/validator 间接使用） + +这样做的原因： +- update cadence（比如 generator_update_interval、critic 的 ratio）是算法语义，放 Trainer 会导致 Trainer 越来越“懂算法”。 +- method 作为算法实现者，天然需要决定“哪些 optimizer 该 step”。 + +### 3.3 `roles.py`：RoleManager / RoleHandle（多角色容器） + +我们把训练参与者统一视为 **role**： +- role name 是字符串 key（例如 `"student"`, `"teacher"`, `"critic"`, `"reward"`…） +- `RoleHandle` 内部持有： + - `modules: dict[str, nn.Module]`（例如 transformer / transformer_2 / …） + - `optimizers / lr_schedulers` + - `trainable` 标记 + +**关键点：role 是可扩展的，不存在“canonical role 更高贵”的区分。** +method 决定自己需要哪些 role（通过 `bundle.require_roles([...])`）。 + +这样做的原因： +- distillation 形态差异巨大，硬编码固定角色集合会快速失控； +- 用 role dict 让新方法/新角色可以 additive 地接入，不影响 Trainer。 + +### 3.4 `models/`：Model plugin（build-time 装配层） + +`models/*` 的定位是：**把一个 family 的工程差异高内聚到一个地方**，并输出 method/Trainer 需要的“运行时组件包”： +- load modules（transformer / vae / …） +- 构建 `RoleManager`（把每个 role 的 modules 放进去） +- 构建 adapter / dataloader / validator / tracker +- 产出 `ModelComponents`（`models/components.py`） + +为什么需要 model plugin，而不是让 method 直接 load？ +- method 关心的是算法；如果 method 去处理 “Wan 的 loader、并行切分、offload、module 名称、schema”等，会把模型工程细节污染进算法层。 +- model plugin 把 build-time 的“杂活”集中起来：更高内聚、更易替换/扩展。 + +### 3.5 `adapters/`：Adapter（运行时 primitive 层） + +adapter 是 method 与 FastVideo 运行时之间的“接口层”，应当遵循： +- **operation-centric API（按操作抽象）**，而不是 role-centric API（避免 role 爆炸）： + - 例如：`prepare_batch(...)`、`predict_x0(handle, ...)`、`predict_noise(handle, ...)`… +- adapter 不应该硬编码 DMD2/self-forcing 的 rollout 细节； + - rollout 的 step list/重加噪等属于 method（或 method_config）的策略。 + +### 3.6 `validators/`：Validator（family-specific，method-controlled） + +validator 是 **模型相关** 的（例如 Wan 的采样管线、shape/latents 约定等），因此放在 `validators/wan.py`。 +但 validator 不应包含任何 “DMD2-specific” 逻辑： +- method 通过 `ValidationRequest` 指定 sampler_kind（ode/sde）、timesteps、steps 等； +- validator 只负责执行与记录（生成视频、写 mp4、通过 tracker 上传 artifacts）。 + +这能同时满足： +- validator 不被算法污染（保持复用性） +- 不同方法可以用同一个 validator，但采样策略由方法决定 + +### 3.7 `dispatch.py`：优雅 dispatch（避免 N×M builder） + +`dispatch.py` 提供 registry： +- `@register_model("wan")`：注册 model plugin builder +- `@register_method("dmd2") / @register_method("finetune")`：注册方法 builder + +统一入口只需做一次组合： +- `recipe.family` → 选择模型插件 +- `recipe.method` → 选择算法方法 + +扩展一个新模型或新方法只需要新增一个 plugin 文件并注册，不需要写 25 个 “model×method 组合函数”。 + +--- + +## 4. YAML config 语义（Phase 3 的 schema） + +Phase 3 采用 **YAML-only**（不兼容旧 CLI configs），由 `utils/config.py` 解析为： +- `RecipeSpec`（dataclass）：`recipe.family` + `recipe.method` +- `roles: dict[str, RoleSpec]`（dataclass）：每个 role 的 family/path/trainable… +- `training_args: TrainingArgs`（dataclass）：训练超参（直接映射 FastVideoArgs/TrainingArgs） +- `method_config: dict[str, Any]`：方法私有参数（保持灵活，便于快速迭代） + +一个典型结构： + +```yaml +recipe: + family: wan + method: dmd2 + +roles: + student: + family: wan + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + critic: + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + # 这里基本就是 TrainingArgs 的字段 + max_train_steps: 4000 + learning_rate: 1.0e-6 + VSA_sparsity: 0.7 + ... + +pipeline_config: + flow_shift: 3 + sampler_kind: ode + +method_config: + # 仅 method 关心的内容 + dmd_denoising_steps: [999, 750, 500, 250, 0] + attn_kind: vsa +``` + +为何 `method_config` 是 dict，而不是强类型 dataclass？ +- 方法参数变化频繁（研究迭代快），强类型会导致 schema/迁移成本高； +- 但 `recipe/roles/training_args` 是框架稳定边界，需要结构化来做 invariants 与错误提示。 + +--- + +## 5. 端到端执行路径（从命令到训练） + +``` +fastvideo/training/distillation.py + -> utils.config.load_distill_run_config() + -> dispatch.build_runtime_from_config() + -> models/.py: build_*_components() + -> methods/.py: build_*_method() + -> DistillTrainer.run(method, dataloader, ...) +``` + +工程性改进： +- entrypoint 会把本次运行的 YAML 原封不动通过 tracker 上传（例如 W&B Files），便于复现与审阅。 + +--- + +## 6. Phase 3 已达成的效果（可验证的“产出”） + +截至 Phase 3，框架已经能做到： + +1) **完全摆脱 legacy distillation pipeline 的训练 loop** +- distill/finetune 都走统一入口 `fastvideo/training/distillation.py` +- Trainer/Method/Adapter/Validator 的边界清晰 + +2) **DMD2 few-step distillation 与 SFT/finetune 均可端到端跑通** +- DMD2：method 控制 rollout（SDE-style）、更新策略、validation 的采样步与 timesteps +- Finetune：只需要 student role，同框架内自然表达（“distillation 的特例”） + +3) **VSA（Video Sparse Attention）可作为 backend 使用，并可 schedule/记录** +- `training.VSA_sparsity / VSA_decay_*` 可影响训练时 `current_vsa_sparsity` +- trainer 统一 log `vsa_sparsity` +- validator 确认会使用 `training_args.VSA_sparsity`（forward-time） + +4) **扩展路径更清晰** +- 增加一个新方法：新增 `methods//.py` + `register_method()` +- 增加一个新模型家族：新增 `models/.py` + `adapters/.py` + `validators/.py` + `register_model()` + +--- + +## 7. 已知限制与下一步（面向 Phase 4+） + +Phase 3 结束时仍有一些“刻意未做”的点： +- 多 family 角色组合（例如 student=wan, teacher=sdxl）尚未正式支持； + - 未来可能需要 method 持有多个 adapter/validator，或在 roles 层引入 cross-family 约束检查（例如 VAE/latent space 是否兼容）。 +- 更多方法类别（KD/CM/self-forcing 等）与更多 model plugins 仍需逐步落地。 +- dataloader schema 更系统的抽象（DataSpec）目前仍偏工程化，不影响核心分层但有优化空间。 + +--- + +## 8. 附：FastGen 对我们的直接启发点（总结） + +我们从 FastGen 学到的最关键结构是： +- **Trainer 薄、Method 厚（策略在 method）**； +- config 选择 method/network，训练时 Trainer “只看 method”。 + +在 FastVideo 中，我们保留这条主线，同时增加了： +- **Adapter**（吸收 FastVideo pipeline/forward 的差异，让 method 可复用） +- **Model plugin（build-time 装配层）**（吸收 loading/dataloader/validator/tracker 等工程差异） +- **RoleManager**（支持任意角色组合） + +这套结构让 FastVideo distillation 能以“增量 PR”方式逐步替换 legacy pipeline，而不是一次性大爆炸式重写。 diff --git a/fastvideo/distillation/doc/dispatch.md b/fastvideo/distillation/doc/dispatch.md index 07f8d63df..99135e364 100644 --- a/fastvideo/distillation/doc/dispatch.md +++ b/fastvideo/distillation/doc/dispatch.md @@ -8,7 +8,9 @@ **关键概念** - `ModelBuilder(cfg) -> ModelComponents` -- `MethodBuilder(cfg, bundle, adapter, validator) -> DistillMethod` +- `DistillMethod` class（算法实现） + - `@register_method("...")` 直接注册到 class + - class 需要实现 `DistillMethod.build(cfg, bundle, adapter, validator)` **关键 API** - `register_model(name)` / `register_method(name)`：装饰器注册 @@ -21,4 +23,3 @@ **边界** - ✅ 这里只做“装配 + dispatch”，不包含训练 loop / loss / rollout / optimizer policy。 - ✅ method 层保持算法高内聚；model plugin 层保持集成高内聚。 - diff --git a/fastvideo/distillation/doc/methods/base.md b/fastvideo/distillation/doc/methods/base.md index f153a2aa1..0eb44877e 100644 --- a/fastvideo/distillation/doc/methods/base.md +++ b/fastvideo/distillation/doc/methods/base.md @@ -11,6 +11,7 @@ `self.role_modules: ModuleDict`，便于 DDP/FSDP/ckpt 系统统一发现参数。 **需要子类实现的抽象方法** +- `build(cfg, bundle, adapter, validator)`（classmethod） - `single_train_step(batch, iteration, current_vsa_sparsity=...)` - 返回：`(loss_map, outputs, metrics)` - `loss_map: dict[str, Tensor]`：必须包含 `total_loss`（用于 backward） @@ -19,6 +20,7 @@ - `get_lr_schedulers(iteration)` **默认实现** +- `set_tracker(tracker)`：由 trainer 注入 tracker，并尽力转发到 `self.validator` - `backward()`：对 `loss_map["total_loss"]` 做 backward（子类可覆写以处理多 ctx） - `optimizers_schedulers_step()`：按 `get_optimizers/get_lr_schedulers` 的结果 step - `optimizers_zero_grad()`：对当前 iteration 的 optimizers 清梯度 diff --git a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md index 41deb5da8..3cbd5f6a5 100644 --- a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md +++ b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md @@ -56,3 +56,7 @@ - Phase 3.2 已完成：sampling loop/timesteps 由 method 在 `ValidationRequest` 中显式指定 （`sampler_kind` + `sampling_timesteps`），validator 使用 `WanPipeline` 执行采样； distillation config 不再需要 `pipeline_config.dmd_denoising_steps` 这种重复字段。 + +**注册方式（Phase 3.4）** +- `DMD2Method` 通过 `@register_method("dmd2")` 直接注册到 class。 +- 由 `DMD2Method.build(...)` 负责把 `cfg.method_config` 等注入到实例。 diff --git a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md index 0227aab56..126be6a13 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md @@ -39,3 +39,7 @@ finetune 可以被视为一种特殊的 distillation recipe：**只有 student + ## 配置示例 - `examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` + +## 注册方式（Phase 3.4） +- `FineTuneMethod` 通过 `@register_method("finetune")` 直接注册到 class。 +- 由 `FineTuneMethod.build(...)` 负责把 `cfg.method_config` 等注入到实例。 diff --git a/fastvideo/distillation/doc/models/wan.md b/fastvideo/distillation/doc/models/wan.md index 8a9ceccd7..659cf3afb 100644 --- a/fastvideo/distillation/doc/models/wan.md +++ b/fastvideo/distillation/doc/models/wan.md @@ -6,7 +6,7 @@ - 包含 Wan 特有的模块加载、shared components、dataloader schema 等逻辑 **产物** -- `ModelComponents(training_args, bundle, adapter, dataloader, tracker, validator, start_step)` +- `ModelComponents(training_args, bundle, adapter, dataloader, validator, start_step)` **主要职责** 1) **加载 shared components** @@ -21,9 +21,9 @@ - `adapter = WanAdapter(prompt_handle=student_handle, ...)` - dataloader：parquet + `pyarrow_schema_t2v` 4) **tracker / validator（可选）** - - tracker：`initialize_trackers(...)`（rank0 才启用） - validator：`WanValidator`（当 `training_args.log_validation=true`） - model plugin 只负责构建并返回 `validator` + - tracker 由 trainer 构建并注入到 method/validator（`method.set_tracker(...)`） - validator 本身不应 hardcode `bundle.role("student")` 等角色语义； method 通过 `ValidationRequest.sample_handle` 指定要采样的模型 - 是否调用、用什么采样配置由 method 决定（method-managed validation） @@ -32,6 +32,8 @@ - ✅ model plugin 不再创建 optimizers/schedulers。 - 这类 update policy（哪些 role 训练、各自超参）属于 method/算法语义。 - 当前由 `DMD2Method` 在初始化时创建并写回 `RoleHandle.optimizers/lr_schedulers`。 + - ✅ model plugin 不再构建/持有 tracker。 + - tracker 属于 infra：由 `DistillTrainer` 构建并持有。 **注意 / TODO** - YAML 中目前仍使用 `training.fake_score_*` 这类字段作为 DMD2 的 critic 超参来源； diff --git a/fastvideo/distillation/doc/trainer.md b/fastvideo/distillation/doc/trainer.md index 591dca6b9..26a5a7d0d 100644 --- a/fastvideo/distillation/doc/trainer.md +++ b/fastvideo/distillation/doc/trainer.md @@ -9,12 +9,13 @@ - `run(method, dataloader, max_steps, ...)` - 支持： - grad accumulation - - tracker logging（rank0） + - tracker logging（rank0；tracker 由 trainer 构建并持有） - validation hook（`method.log_validation(step)`） - checkpoint hook（通过 `checkpoint_manager` 注入） **与 Method 的契约** `run()` 通过 duck-typing 调用（存在则调用）： +- `method.set_tracker(tracker)`（注入 tracker；用于 method-managed validation artifacts） - `method.on_train_start()` - `method.single_train_step(batch, step, current_vsa_sparsity=...)` - 返回：`(loss_map, outputs, metrics)` diff --git a/fastvideo/distillation/doc/utils/__init__.md b/fastvideo/distillation/doc/utils/__init__.md index da858c058..6214e97b2 100644 --- a/fastvideo/distillation/doc/utils/__init__.md +++ b/fastvideo/distillation/doc/utils/__init__.md @@ -5,6 +5,8 @@ 当前包含： - `config.py`：YAML loader + schema/types（`DistillRunConfig` / `DistillRuntime`）。 -- `data.py`：通用 dataloader 构建（按 dataset kind/schema 复用 FastVideo 现有实现）。 -- `tracking.py`：tracker 初始化（wandb / tensorboard 等）。 +- `dataloader.py`：通用 dataloader 构建（按 dataset kind/schema 复用 FastVideo 现有实现）。 +- `moduleloader.py`：通用组件加载（`PipelineComponentLoader` 的薄封装）。 +- `module_state.py`：module 的 trainable 状态设置（`requires_grad` + train/eval）。 +- `tracking.py`：tracker 初始化（wandb / tensorboard 等；由 trainer 持有）。 - `checkpoint.py`：role-based checkpoint/save-resume（Phase 2 runtime）。 diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index b767c4e79..2e1ff69ac 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -59,7 +59,6 @@ - `bundle` - `adapter` - `dataloader` - - `tracker` - `validator`（可选；model-specific） - `start_step`（用于 resume / warm-start） - `DistillRuntime` @@ -67,5 +66,7 @@ - `training_args` - `method`（`DistillMethod`） - `dataloader` - - `tracker` - `start_step` + +备注： +- tracker 由 `DistillTrainer` 构建并持有（避免 model plugin 变成 infra owner）。 diff --git a/fastvideo/distillation/doc/utils/data.md b/fastvideo/distillation/doc/utils/dataloader.md similarity index 94% rename from fastvideo/distillation/doc/utils/data.md rename to fastvideo/distillation/doc/utils/dataloader.md index d365e5851..d380ecfa4 100644 --- a/fastvideo/distillation/doc/utils/data.md +++ b/fastvideo/distillation/doc/utils/dataloader.md @@ -1,4 +1,4 @@ -# `fastvideo/distillation/utils/data.py` +# `fastvideo/distillation/utils/dataloader.py` **目的** - 把 “dataloader 构建” 从 model plugin（原 families/，现 `models/`）中抽离出来， diff --git a/fastvideo/distillation/doc/utils/module_state.md b/fastvideo/distillation/doc/utils/module_state.md new file mode 100644 index 000000000..2c4ce9629 --- /dev/null +++ b/fastvideo/distillation/doc/utils/module_state.md @@ -0,0 +1,14 @@ +# `fastvideo/distillation/utils/module_state.py` + +**目的** +- 提供最小且通用的 module 训练状态设置，避免 model plugin 里到处散落： + - `requires_grad_(...)` + - `train()` / `eval()` + +**当前包含** +- `apply_trainable(module, trainable: bool)` + +**边界** +- ✅ 不涉及 optimizer/scheduler（由 method 管理）。 +- ✅ 不涉及激活检查点策略（由 model plugin 在加载后按需启用）。 + diff --git a/fastvideo/distillation/doc/utils/moduleloader.md b/fastvideo/distillation/doc/utils/moduleloader.md new file mode 100644 index 000000000..a0c887fd3 --- /dev/null +++ b/fastvideo/distillation/doc/utils/moduleloader.md @@ -0,0 +1,17 @@ +# `fastvideo/distillation/utils/moduleloader.py` + +**目的** +- 把 “从 FastVideo 模型路径加载某个子模块（transformer/vae/…）” 的通用逻辑收敛成一个 util， + 便于多个 model plugin 复用，避免每个 plugin 都复制一份 loader 细节。 + +**当前包含** +- `load_module_from_path(model_path, module_type, training_args, disable_custom_init_weights=False)` + - 解析/下载 `model_path` + - 读取 FastVideo 的 per-module config entry + - 调用 `PipelineComponentLoader.load_module(...)` + - 可选跳过自定义 init weights（legacy flag：`_loading_teacher_critic_model`） + +**边界** +- ✅ 这里只做 “单模块加载”，不做 role 语义、也不做 optimizer/scheduler。 +- ✅ “哪些模块需要加载/共享/复用” 仍由 model plugin 决定。 + diff --git a/fastvideo/distillation/doc/utils/tracking.md b/fastvideo/distillation/doc/utils/tracking.md index 883f11e02..76f6aa437 100644 --- a/fastvideo/distillation/doc/utils/tracking.md +++ b/fastvideo/distillation/doc/utils/tracking.md @@ -10,4 +10,5 @@ - tracker log dir 默认在 `output_dir/tracker/` **设计意图** -- tracker 属于 infra：entrypoint/trainer 负责持有；method 只负责产出要 log 的 metrics/媒体（video/image/file 等，tracker API 里常叫 artifacts）。 +- tracker 属于 infra：由 `DistillTrainer` 构建并持有；method 只负责产出要 log 的 + metrics/媒体（video/image/file 等，tracker API 里常叫 artifacts）。 diff --git a/fastvideo/distillation/doc/validators/wan.md b/fastvideo/distillation/doc/validators/wan.md index f115ae05a..9c1123f46 100644 --- a/fastvideo/distillation/doc/validators/wan.md +++ b/fastvideo/distillation/doc/validators/wan.md @@ -24,3 +24,7 @@ **可演进方向（Phase 3+）** - 将 validation steps/guidance 等采样配置从 `TrainingArgs` 迁移到更明确的配置块（例如 `validation:`）。 - 进一步抽象 validator API，使其更容易被不同 model plugin / method 复用。 + +**Tracker 注入** +- `WanValidator` 不再要求 build-time 传入 tracker； + trainer 会通过 `method.set_tracker(...)` 把 tracker 注入到 validator。 diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py index e9d4ef655..fc9461147 100644 --- a/fastvideo/distillation/methods/base.py +++ b/fastvideo/distillation/methods/base.py @@ -4,12 +4,15 @@ from abc import ABC, abstractmethod from collections.abc import Sequence -from typing import Any +from typing import Any, TYPE_CHECKING import torch from fastvideo.distillation.roles import RoleManager +if TYPE_CHECKING: + from fastvideo.distillation.utils.config import DistillRunConfig + LogScalar = float | int | torch.Tensor @@ -17,11 +20,43 @@ class DistillMethod(torch.nn.Module, ABC): def __init__(self, bundle: RoleManager) -> None: super().__init__() self.bundle = bundle + self.tracker: Any | None = None self.role_modules = torch.nn.ModuleDict() for role, handle in bundle.roles.items(): if handle.modules: self.role_modules[role] = torch.nn.ModuleDict(handle.modules) + @classmethod + @abstractmethod + def build( + cls, + *, + cfg: DistillRunConfig, + bundle: RoleManager, + adapter: Any, + validator: Any | None, + ) -> "DistillMethod": + raise NotImplementedError + + def set_tracker(self, tracker: Any) -> None: + """Attach a tracker (infra) to this method. + + Trainer constructs/owns the tracker, but method-managed validation may + need it to log artifacts (videos/images/files). This is a best-effort + bridge that keeps model plugins free of tracker ownership. + """ + + self.tracker = tracker + validator = getattr(self, "validator", None) + if validator is None: + return + set_tracker = getattr(validator, "set_tracker", None) + if callable(set_tracker): + set_tracker(tracker) + return + if hasattr(validator, "tracker"): + validator.tracker = tracker # type: ignore[attr-defined] + @abstractmethod def single_train_step( self, diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index e2caa7334..b5c414b40 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -90,6 +90,7 @@ def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> N ... +@register_method("dmd2") class DMD2Method(DistillMethod): """DMD2 distillation algorithm (method layer). @@ -131,6 +132,22 @@ def __init__( self._denoising_step_list: torch.Tensor | None = None self._init_optimizers_and_schedulers() + @classmethod + def build( + cls, + *, + cfg: DistillRunConfig, + bundle: RoleManager, + adapter: Any, + validator: Any | None, + ) -> DistillMethod: + return cls( + bundle=bundle, + adapter=adapter, + method_config=cfg.method_config, + validator=validator, + ) + def _parse_rollout_mode(self) -> Literal["simulate", "data_latent"]: raw = self.method_config.get("rollout_mode", None) if raw is None: @@ -686,19 +703,3 @@ def optimizers_schedulers_step(self, iteration: int) -> None: self._clip_grad_norm(module) super().optimizers_schedulers_step(iteration) - - -@register_method("dmd2") -def build_dmd2_method( - *, - cfg: DistillRunConfig, - bundle: RoleManager, - adapter: _DMD2Adapter, - validator: Any | None, -) -> DistillMethod: - return DMD2Method( - bundle=bundle, - adapter=adapter, - method_config=cfg.method_config, - validator=validator, - ) diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index cd2de6d4d..a0d3c411f 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -57,6 +57,7 @@ def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> N ... +@register_method("finetune") class FineTuneMethod(DistillMethod): """Supervised finetuning as a method: only `student` participates. @@ -90,6 +91,22 @@ def __init__( self._init_optimizers_and_schedulers() + @classmethod + def build( + cls, + *, + cfg: DistillRunConfig, + bundle: RoleManager, + adapter: Any, + validator: Any | None, + ) -> DistillMethod: + return cls( + bundle=bundle, + adapter=adapter, + method_config=cfg.method_config, + validator=validator, + ) + def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: if raw in (None, ""): return "dense" @@ -332,19 +349,3 @@ def optimizers_schedulers_step(self, iteration: int) -> None: for module in self.student.modules.values(): self._clip_grad_norm(module) super().optimizers_schedulers_step(iteration) - - -@register_method("finetune") -def build_finetune_method( - *, - cfg: DistillRunConfig, - bundle: RoleManager, - adapter: _FineTuneAdapter, - validator: Any | None, -) -> DistillMethod: - return FineTuneMethod( - bundle=bundle, - adapter=adapter, - method_config=cfg.method_config, - validator=validator, - ) diff --git a/fastvideo/distillation/models/components.py b/fastvideo/distillation/models/components.py index 81b01c467..033f372fd 100644 --- a/fastvideo/distillation/models/components.py +++ b/fastvideo/distillation/models/components.py @@ -24,6 +24,5 @@ class ModelComponents: bundle: RoleManager adapter: Any dataloader: Any - tracker: Any validator: Any | None = None start_step: int = 0 diff --git a/fastvideo/distillation/models/wan.py b/fastvideo/distillation/models/wan.py index 8b6d7cfd9..eae6fb3f4 100644 --- a/fastvideo/distillation/models/wan.py +++ b/fastvideo/distillation/models/wan.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os from typing import Any import torch @@ -12,66 +11,13 @@ from fastvideo.distillation.dispatch import register_model from fastvideo.distillation.utils.config import DistillRunConfig from fastvideo.distillation.models.components import ModelComponents -from fastvideo.distillation.utils.data import build_parquet_t2v_train_dataloader -from fastvideo.distillation.utils.tracking import build_tracker -from fastvideo.models.loader.component_loader import PipelineComponentLoader +from fastvideo.distillation.utils.dataloader import build_parquet_t2v_train_dataloader +from fastvideo.distillation.utils.module_state import apply_trainable +from fastvideo.distillation.utils.moduleloader import load_module_from_path from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( FlowMatchEulerDiscreteScheduler, ) from fastvideo.training.activation_checkpoint import apply_activation_checkpointing -from fastvideo.utils import maybe_download_model, verify_model_config_and_directory - - -def _load_module_from_path( - *, - model_path: str, - module_type: str, - training_args: Any, - disable_custom_init_weights: bool = False, -) -> torch.nn.Module: - local_model_path = maybe_download_model(model_path) - config = verify_model_config_and_directory(local_model_path) - - if module_type not in config: - raise ValueError(f"Module {module_type!r} not found in config at {local_model_path}") - - module_info = config[module_type] - if module_info is None: - raise ValueError(f"Module {module_type!r} has null value in config at {local_model_path}") - - transformers_or_diffusers, _architecture = module_info - component_path = os.path.join(local_model_path, module_type) - - if disable_custom_init_weights: - # NOTE: This flag is used by PipelineComponentLoader to skip applying - # `init_weights_from_safetensors*` overrides when loading auxiliary - # roles (teacher/critic/etc). The attribute name is legacy. - training_args._loading_teacher_critic_model = True - try: - module = PipelineComponentLoader.load_module( - module_name=module_type, - component_model_path=component_path, - transformers_or_diffusers=transformers_or_diffusers, - fastvideo_args=training_args, - ) - finally: - if disable_custom_init_weights and hasattr( - training_args, "_loading_teacher_critic_model" - ): - del training_args._loading_teacher_critic_model - - if not isinstance(module, torch.nn.Module): - raise TypeError(f"Loaded {module_type!r} is not a torch.nn.Module: {type(module)}") - return module - - -def _apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Module: - module.requires_grad_(trainable) - if trainable: - module.train() - else: - module.eval() - return module @register_model("wan") @@ -86,7 +32,7 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: # Load shared components (student base path). training_args.override_transformer_cls_name = "WanTransformer3DModel" - vae = _load_module_from_path( + vae = load_module_from_path( model_path=str(training_args.model_path), module_type="vae", training_args=training_args, @@ -106,7 +52,7 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: disable_custom_init_weights = bool( getattr(role_spec, "disable_custom_init_weights", False) ) - transformer = _load_module_from_path( + transformer = load_module_from_path( model_path=role_spec.path, module_type="transformer", training_args=training_args, @@ -116,7 +62,7 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: # Optional MoE support: load transformer_2 if present in the model. try: - transformer_2 = _load_module_from_path( + transformer_2 = load_module_from_path( model_path=role_spec.path, module_type="transformer_2", training_args=training_args, @@ -128,7 +74,7 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: modules["transformer_2"] = transformer_2 for name, module in list(modules.items()): - module = _apply_trainable(module, trainable=bool(role_spec.trainable)) + module = apply_trainable(module, trainable=bool(role_spec.trainable)) if role_spec.trainable and getattr( training_args, "enable_gradient_checkpointing_type", None ): @@ -149,7 +95,6 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: ) bundle = RoleManager(roles=role_handles) - tracker = build_tracker(training_args, config=cfg.raw) validator = None if getattr(training_args, "log_validation", False): @@ -157,7 +102,6 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: validator = WanValidator( training_args=training_args, - tracker=tracker, ) # NOTE: adapter is the model runtime boundary; it may implement multiple @@ -183,7 +127,6 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: bundle=bundle, adapter=adapter, dataloader=dataloader, - tracker=tracker, validator=validator, start_step=0, ) diff --git a/fastvideo/distillation/trainer.py b/fastvideo/distillation/trainer.py index 17f29d5ac..3bcfd2e35 100644 --- a/fastvideo/distillation/trainer.py +++ b/fastvideo/distillation/trainer.py @@ -12,7 +12,7 @@ from fastvideo.distributed import get_sp_group, get_world_group from fastvideo.fastvideo_args import TrainingArgs -from fastvideo.training.trackers import BaseTracker, DummyTracker +from fastvideo.distillation.utils.tracking import build_tracker def _coerce_log_scalar(value: Any, *, where: str) -> float: @@ -41,14 +41,14 @@ def __init__( self, training_args: TrainingArgs, *, - tracker: BaseTracker | None = None, + config: dict[str, Any] | None = None, ) -> None: self.training_args = training_args self.world_group = get_world_group() self.sp_group = get_sp_group() self.global_rank = self.world_group.rank self.local_rank = self.world_group.local_rank - self.tracker = tracker or DummyTracker() + self.tracker = build_tracker(training_args, config=config) def _iter_dataloader(self, dataloader: Any) -> Iterator[dict[str, Any]]: data_iter = iter(dataloader) @@ -84,6 +84,9 @@ def run( grad_accum = max(1, int(self.training_args.gradient_accumulation_steps or 1)) + if hasattr(method, "set_tracker"): + method.set_tracker(self.tracker) # type: ignore[attr-defined] + if hasattr(method, "on_train_start"): method.on_train_start() # type: ignore[attr-defined] diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index 3fb3546f4..b8e67694e 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -206,5 +206,4 @@ class DistillRuntime: training_args: TrainingArgs method: DistillMethod dataloader: Any - tracker: Any start_step: int = 0 diff --git a/fastvideo/distillation/utils/data.py b/fastvideo/distillation/utils/dataloader.py similarity index 99% rename from fastvideo/distillation/utils/data.py rename to fastvideo/distillation/utils/dataloader.py index c8f7b2f33..07ace99f2 100644 --- a/fastvideo/distillation/utils/data.py +++ b/fastvideo/distillation/utils/dataloader.py @@ -26,4 +26,3 @@ def build_parquet_t2v_train_dataloader( seed=int(training_args.seed or 0), ) return dataloader - diff --git a/fastvideo/distillation/utils/module_state.py b/fastvideo/distillation/utils/module_state.py new file mode 100644 index 000000000..2b86085bb --- /dev/null +++ b/fastvideo/distillation/utils/module_state.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import torch + + +def apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Module: + """Apply train/eval mode + requires_grad based on a role's trainable flag.""" + + module.requires_grad_(bool(trainable)) + if trainable: + module.train() + else: + module.eval() + return module + diff --git a/fastvideo/distillation/utils/moduleloader.py b/fastvideo/distillation/utils/moduleloader.py new file mode 100644 index 000000000..4072779d3 --- /dev/null +++ b/fastvideo/distillation/utils/moduleloader.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from typing import Any + +import torch + +from fastvideo.models.loader.component_loader import PipelineComponentLoader +from fastvideo.utils import maybe_download_model, verify_model_config_and_directory + + +def load_module_from_path( + *, + model_path: str, + module_type: str, + training_args: Any, + disable_custom_init_weights: bool = False, +) -> torch.nn.Module: + """Load a single pipeline component module from a FastVideo model path. + + This is a thin wrapper over :func:`PipelineComponentLoader.load_module`: + - resolves/downloads ``model_path`` if needed + - reads the per-module config entry to determine transformers/diffusers + - optionally disables custom init weights overrides (legacy flag) + """ + + local_model_path = maybe_download_model(model_path) + config = verify_model_config_and_directory(local_model_path) + + if module_type not in config: + raise ValueError(f"Module {module_type!r} not found in config at {local_model_path}") + + module_info = config[module_type] + if module_info is None: + raise ValueError(f"Module {module_type!r} has null value in config at {local_model_path}") + + transformers_or_diffusers, _architecture = module_info + component_path = os.path.join(local_model_path, module_type) + + if disable_custom_init_weights: + # NOTE: This flag is used by PipelineComponentLoader to skip applying + # `init_weights_from_safetensors*` overrides when loading auxiliary + # roles (teacher/critic/etc). The attribute name is legacy. + training_args._loading_teacher_critic_model = True + try: + module = PipelineComponentLoader.load_module( + module_name=module_type, + component_model_path=component_path, + transformers_or_diffusers=transformers_or_diffusers, + fastvideo_args=training_args, + ) + finally: + if disable_custom_init_weights and hasattr(training_args, "_loading_teacher_critic_model"): + del training_args._loading_teacher_critic_model + + if not isinstance(module, torch.nn.Module): + raise TypeError(f"Loaded {module_type!r} is not a torch.nn.Module: {type(module)}") + return module + diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 4fac2fda4..456384b18 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -20,6 +20,7 @@ from fastvideo.pipelines import ForwardBatch from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline +from fastvideo.training.trackers import DummyTracker from fastvideo.utils import shallow_asdict logger = init_logger(__name__) @@ -38,10 +39,10 @@ def __init__( self, *, training_args: Any, - tracker: Any, + tracker: Any | None = None, ) -> None: self.training_args = training_args - self.tracker = tracker + self.tracker = tracker or DummyTracker() self.world_group = get_world_group() self.sp_group = get_sp_group() @@ -59,6 +60,9 @@ def __init__( self._pipeline_key: tuple[int, str] | None = None self._sampling_param: SamplingParam | None = None + def set_tracker(self, tracker: Any) -> None: + self.tracker = tracker + def _get_sampling_param(self) -> SamplingParam: if self._sampling_param is None: self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 5bee3ab08..2ef6db2ff 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -49,9 +49,11 @@ def run_distillation_from_config( logger.info("Dry-run: config parsed and runtime built successfully.") return + trainer = DistillTrainer(training_args, config=cfg.raw) + # Attach the exact YAML used for this run to the tracker (e.g., W&B Files). # This helps reproducibility and makes runs easy to inspect later. - runtime.tracker.log_file(os.path.abspath(os.path.expanduser(config_path)), name="run.yaml") + trainer.tracker.log_file(os.path.abspath(os.path.expanduser(config_path)), name="run.yaml") ckpt_config = DistillCheckpointConfig( save_steps=int(getattr(training_args, "training_state_checkpointing_steps", 0) or 0), @@ -73,7 +75,6 @@ def run_distillation_from_config( get_rng_generators=get_rng_generators, ) - trainer = DistillTrainer(training_args, tracker=runtime.tracker) trainer.run( runtime.method, dataloader=runtime.dataloader, From 6d25bea74c8e568146151d23fc156c4aec923f4a Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 03:34:52 +0000 Subject: [PATCH 108/214] utils, config. --- fastvideo/distillation/dispatch.py | 17 ++++- fastvideo/distillation/doc/dispatch.md | 1 + fastvideo/distillation/doc/utils/__init__.md | 2 +- fastvideo/distillation/doc/utils/config.md | 14 ++--- .../methods/distribution_matching/dmd2.py | 63 ++++++------------- .../methods/fine_tuning/finetune.py | 16 +---- fastvideo/distillation/utils/config.py | 52 +++++++++++---- 7 files changed, 86 insertions(+), 79 deletions(-) diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index 4a856ddc2..94a3023c7 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -3,11 +3,26 @@ from __future__ import annotations from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, TYPE_CHECKING from typing import Protocol from fastvideo.distillation.methods.base import DistillMethod from fastvideo.distillation.models.components import ModelComponents -from fastvideo.distillation.utils.config import DistillRunConfig, DistillRuntime +from fastvideo.distillation.utils.config import DistillRunConfig + +if TYPE_CHECKING: + from fastvideo.fastvideo_args import TrainingArgs + + +@dataclass(slots=True) +class DistillRuntime: + """Fully assembled runtime for `DistillTrainer.run()`.""" + + training_args: TrainingArgs + method: DistillMethod + dataloader: Any + start_step: int = 0 class ModelBuilder(Protocol): diff --git a/fastvideo/distillation/doc/dispatch.md b/fastvideo/distillation/doc/dispatch.md index 99135e364..a22213231 100644 --- a/fastvideo/distillation/doc/dispatch.md +++ b/fastvideo/distillation/doc/dispatch.md @@ -19,6 +19,7 @@ - `build_runtime_from_config(cfg) -> DistillRuntime` - 选择 model plugin：`get_model(cfg.recipe.family)` - 选择 method：`get_method(cfg.recipe.method)` + - `DistillRuntime` 定义也在本文件中（谁创建谁声明）。 **边界** - ✅ 这里只做“装配 + dispatch”，不包含训练 loop / loss / rollout / optimizer policy。 diff --git a/fastvideo/distillation/doc/utils/__init__.md b/fastvideo/distillation/doc/utils/__init__.md index 6214e97b2..6a6ccb8a9 100644 --- a/fastvideo/distillation/doc/utils/__init__.md +++ b/fastvideo/distillation/doc/utils/__init__.md @@ -4,7 +4,7 @@ - 放置 distillation 子系统的中性工具代码（不属于某个 model plugin / method）。 当前包含： -- `config.py`：YAML loader + schema/types（`DistillRunConfig` / `DistillRuntime`）。 +- `config.py`：YAML loader + schema/types（`DistillRunConfig`）。 - `dataloader.py`：通用 dataloader 构建（按 dataset kind/schema 复用 FastVideo 现有实现）。 - `moduleloader.py`：通用组件加载（`PipelineComponentLoader` 的薄封装）。 - `module_state.py`：module 的 trainable 状态设置（`requires_grad` + train/eval）。 diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index 2e1ff69ac..acb351fcc 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -61,12 +61,12 @@ - `dataloader` - `validator`（可选；model-specific） - `start_step`（用于 resume / warm-start） -- `DistillRuntime` - - `DistillTrainer.run()` 所需的最小集合： - - `training_args` - - `method`（`DistillMethod`） - - `dataloader` - - `start_step` - 备注： +- `DistillRuntime` 由 `dispatch.build_runtime_from_config()` 创建并定义在 + `fastvideo/distillation/dispatch.py`（谁创建谁声明）。 - tracker 由 `DistillTrainer` 构建并持有（避免 model plugin 变成 infra owner）。 + +## 4) 通用解析 helpers（method_config / optimizer 等） +- `get_optional_int(mapping, key, where=...)` +- `get_optional_float(mapping, key, where=...)` +- `parse_betas(raw, where=...)` diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index b5c414b40..5ac676a0c 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -17,7 +17,12 @@ from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.dispatch import register_method from fastvideo.distillation.validators.base import ValidationRequest -from fastvideo.distillation.utils.config import DistillRunConfig +from fastvideo.distillation.utils.config import ( + DistillRunConfig, + get_optional_float, + get_optional_int, + parse_betas, +) class _DMD2Adapter(Protocol): @@ -168,46 +173,6 @@ def _parse_rollout_mode(self) -> Literal["simulate", "data_latent"]: f"{raw!r}" ) - def _get_method_int(self, key: str) -> int | None: - raw = self.method_config.get(key, None) - if raw is None: - return None - if isinstance(raw, bool): - raise ValueError(f"method_config.{key} must be an int, got bool") - if isinstance(raw, int): - return int(raw) - if isinstance(raw, float) and raw.is_integer(): - return int(raw) - if isinstance(raw, str) and raw.strip(): - return int(raw) - raise ValueError(f"method_config.{key} must be an int, got {type(raw).__name__}") - - def _get_method_float(self, key: str) -> float | None: - raw = self.method_config.get(key, None) - if raw is None: - return None - if isinstance(raw, bool): - raise ValueError(f"method_config.{key} must be a float, got bool") - if isinstance(raw, (int, float)): - return float(raw) - if isinstance(raw, str) and raw.strip(): - return float(raw) - raise ValueError( - f"method_config.{key} must be a float, got {type(raw).__name__}" - ) - - def _parse_betas(self, raw: Any, *, where: str) -> tuple[float, float]: - if raw is None: - raise ValueError(f"Missing betas for {where}") - if isinstance(raw, (tuple, list)) and len(raw) == 2: - return float(raw[0]), float(raw[1]) - if isinstance(raw, str): - parts = [p.strip() for p in raw.split(",") if p.strip()] - if len(parts) != 2: - raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {raw!r}") - return float(parts[0]), float(parts[1]) - raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {type(raw).__name__}") - def _build_role_optimizer_and_scheduler( self, *, @@ -251,7 +216,7 @@ def _init_optimizers_and_schedulers(self) -> None: # Student optimizer/scheduler (default training hyperparams). student_lr = float(getattr(training_args, "learning_rate", 0.0) or 0.0) - student_betas = self._parse_betas( + student_betas = parse_betas( getattr(training_args, "betas", None), where="training.betas", ) @@ -272,7 +237,7 @@ def _init_optimizers_and_schedulers(self) -> None: critic_betas_raw = getattr(training_args, "fake_score_betas", None) if critic_betas_raw is None: critic_betas_raw = getattr(training_args, "betas", None) - critic_betas = self._parse_betas(critic_betas_raw, where="training.fake_score_betas") + critic_betas = parse_betas(critic_betas_raw, where="training.fake_score_betas") critic_sched = str(getattr(training_args, "fake_score_lr_scheduler", None) or student_sched) self._build_role_optimizer_and_scheduler( @@ -338,7 +303,11 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: return generators def _should_update_student(self, iteration: int) -> bool: - interval = self._get_method_int("generator_update_interval") + interval = get_optional_int( + self.method_config, + "generator_update_interval", + where="method_config.generator_update_interval", + ) if interval is None: interval = int(getattr(self.training_args, "generator_update_interval", 1) or 1) if interval <= 0: @@ -542,7 +511,11 @@ def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dic return flow_matching_loss, (batch.timesteps, batch.attn_metadata), outputs def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor: - guidance_scale = self._get_method_float("real_score_guidance_scale") + guidance_scale = get_optional_float( + self.method_config, + "real_score_guidance_scale", + where="method_config.real_score_guidance_scale", + ) if guidance_scale is None: guidance_scale = float(getattr(self.training_args, "real_score_guidance_scale", 1.0)) device = generator_pred_x0.device diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index a0d3c411f..f4e347bce 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -16,7 +16,7 @@ from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.dispatch import register_method from fastvideo.distillation.validators.base import ValidationRequest -from fastvideo.distillation.utils.config import DistillRunConfig +from fastvideo.distillation.utils.config import DistillRunConfig, parse_betas class _FineTuneAdapter(Protocol): @@ -118,18 +118,6 @@ def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: ) return cast(Literal["dense", "vsa"], kind) - def _parse_betas(self, raw: Any, *, where: str) -> tuple[float, float]: - if raw is None: - raise ValueError(f"Missing betas for {where}") - if isinstance(raw, (tuple, list)) and len(raw) == 2: - return float(raw[0]), float(raw[1]) - if isinstance(raw, str): - parts = [p.strip() for p in raw.split(",") if p.strip()] - if len(parts) != 2: - raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {raw!r}") - return float(parts[0]), float(parts[1]) - raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {type(raw).__name__}") - def _build_role_optimizer_and_scheduler( self, *, @@ -175,7 +163,7 @@ def _init_optimizers_and_schedulers(self) -> None: if student_lr <= 0.0: raise ValueError("training.learning_rate must be > 0 for finetune") - student_betas = self._parse_betas( + student_betas = parse_betas( getattr(training_args, "betas", None), where="training.betas", ) diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index b8e67694e..eb6d2e679 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -11,7 +11,6 @@ if TYPE_CHECKING: from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.distillation.methods.base import DistillMethod RoleName = str @@ -89,6 +88,47 @@ def _get_bool(raw: Any, *, where: str, default: bool) -> bool: raise ValueError(f"Expected bool at {where}, got {type(raw).__name__}") +def get_optional_int(mapping: dict[str, Any], key: str, *, where: str) -> int | None: + raw = mapping.get(key, None) + if raw is None: + return None + if isinstance(raw, bool): + raise ValueError(f"Expected int at {where}, got bool") + if isinstance(raw, int): + return int(raw) + if isinstance(raw, float) and raw.is_integer(): + return int(raw) + if isinstance(raw, str) and raw.strip(): + return int(raw) + raise ValueError(f"Expected int at {where}, got {type(raw).__name__}") + + +def get_optional_float(mapping: dict[str, Any], key: str, *, where: str) -> float | None: + raw = mapping.get(key, None) + if raw is None: + return None + if isinstance(raw, bool): + raise ValueError(f"Expected float at {where}, got bool") + if isinstance(raw, (int, float)): + return float(raw) + if isinstance(raw, str) and raw.strip(): + return float(raw) + raise ValueError(f"Expected float at {where}, got {type(raw).__name__}") + + +def parse_betas(raw: Any, *, where: str) -> tuple[float, float]: + if raw is None: + raise ValueError(f"Missing betas for {where}") + if isinstance(raw, (tuple, list)) and len(raw) == 2: + return float(raw[0]), float(raw[1]) + if isinstance(raw, str): + parts = [p.strip() for p in raw.split(",") if p.strip()] + if len(parts) != 2: + raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {raw!r}") + return float(parts[0]), float(parts[1]) + raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {type(raw).__name__}") + + def load_distill_run_config(path: str) -> DistillRunConfig: """Load a distillation run config from schema-v2 YAML. @@ -197,13 +237,3 @@ def load_distill_run_config(path: str) -> DistillRunConfig: method_config=method_config, raw=cfg, ) - - -@dataclass(slots=True) -class DistillRuntime: - """Fully assembled runtime for `DistillTrainer.run()`.""" - - training_args: TrainingArgs - method: DistillMethod - dataloader: Any - start_step: int = 0 From 7072dd460718f4e0c11b45590ae5ab2dd7cc156a Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 04:10:19 +0000 Subject: [PATCH 109/214] rfc cn --- fastvideo/distillation/doc/RFC-v1.md | 416 +++++++++++++-------------- 1 file changed, 205 insertions(+), 211 deletions(-) diff --git a/fastvideo/distillation/doc/RFC-v1.md b/fastvideo/distillation/doc/RFC-v1.md index cb11bb9cc..95b0b4b8e 100644 --- a/fastvideo/distillation/doc/RFC-v1.md +++ b/fastvideo/distillation/doc/RFC-v1.md @@ -1,210 +1,196 @@ -# RFC-v1: FastVideo Distillation（截至 Phase 3） -本 RFC 记录：截至 Phase 3 结束时，FastVideo 新 distillation/finetuning 框架的**目录结构**、 -**每一层的职责边界**、以及这样设计的原因与当前已达成的效果。 - -> 参考项目：`~/alex/pkgs/FastGen`（FastGen 的 “Trainer ↔ Method ↔ Network” 分层对我们影响很大）。 - ---- - -## 1. 背景与目标 - -### 1.1 旧代码的问题（动机） - -历史上 FastVideo 的 distillation 主要以 `fastvideo/training/*distillation_pipeline*.py` 为核心： -- **model 类型（Wan）与 distill 算法（DMD2 / self-forcing / …）强耦合**； -- training loop、算法策略、validation/sampling、conditioning 初始化混在一起； -- 扩展一个新模型或新方法，往往意味着复制/改造一整条 pipeline； -- reviewer 很难在一个 PR 里把所有 coupling 看清楚。 - -### 1.2 新框架的目标 - -我们希望把 distillation/finetuning 做成可组合的三件事： -- **Model plugin（模型家族差异）**：负责“装配（build-time）”与共享组件。 -- **Method（算法）**：只关心 loss / rollout / update policy / validation policy。 -- **Trainer（基础设施）**：只负责 loop/accum/step/ckpt/log，不包含策略。 - -并额外引入： -- **Adapter（运行时 primitive）**：把 FastVideo 的 pipeline/forward 细节收敛成 method 可复用的操作接口； -- **RoleManager / RoleHandle（多角色容器）**：支持任意角色组合（student/teacher/critic/reward/...），避免“固定三件套”的硬编码。 - -最终目标是：**N 个模型插件 + M 个方法 = N+M 的扩展成本**，而不是 N×M 的构建爆炸。 - ---- - -## 2. 当前目录结构（Phase 3） - -下面是 `fastvideo/distillation/` 的“可读结构图”（省略 `__pycache__/`）： +## 1) 文件结构（带注释） ```text fastvideo/distillation/ - __init__.py - trainer.py - dispatch.py - roles.py + trainer.py # 构建training loop 调用method提供的train_one_step接口 + dispatch.py # 根据@register_method和@register_model自动识别类型，根据config构建DistillRuntime + roles.py # RoleHandle模型外面包一层role的字段，用于区分teacher/student/critic。 - adapters/ - base.py - wan.py + models/ + components.py # ModelComponent dispatch构建模型过程中的中间查无 记录了模型的各个组成部分 + wan.py # 加载Wan模型。不同模型的加载逻辑有所不同（例如ltx就是double stream）。 + ... - models/ # (原 families) 现在叫 models = model plugins - components.py - wan.py + adapters/ + base.py # Adapter本质是一个把已经load好的模型转变为运行时可用api的框架 把Model转变为predict_x0/add_noise/backward/... + wan.py # 针对Wan的Adapter。 + ... methods/ - __init__.py - base.py + base.py # DistillMethod基类，需要Method提供必要的例如train one step的接口 distribution_matching/ - __init__.py - dmd2.py + dmd2.py # DMD2Method：DMD2 distillation（student/teacher/critic） fine_tuning/ - __init__.py - finetune.py - knowledge_distillation/ # 预留目录（Phase 3 结束时可为空/占位） - __init__.py - consistency_model/ # 预留目录（Phase 3 结束时可为空/占位） - __init__.py + finetune.py # FineTuneMethod：SFT/finetuning（只有 student） + knowledge_distillation/ + consistency_model/ validators/ - base.py - wan.py + base.py # 不同模型推理方式不同，需要根据模型类型写不同的validator。 + wan.py # 调用WanPipeline的validator utils/ - __init__.py - config.py # YAML 解析 + DistillRunConfig/Runtime 数据结构 - data.py # dataloader 构建（当前以 parquet T2V 为主） - tracking.py # tracker 构建（W&B 等） - checkpoint.py # save/resume 管理 - - doc/ - README.md # file-by-file 索引与设计原则 - RFC-v1.md # 本文件 - ... # 其它 file-by-file docs + config.py # yaml parser + dataloader.py + moduleloader.py + module_state.py # apply_trainable(...)：统一 requires_grad + train/eval + tracking.py # wandb tracker，由trainer管理 + checkpoint.py# save/resume ``` -此外，新框架的统一入口在： -- `fastvideo/training/distillation.py`：YAML-only entrypoint（不再兼容旧 CLI configs）。 +统一入口（YAML-only）： +- `fastvideo/training/distillation.py` --- -## 3. 为什么这样分层（核心抽象与职责边界） +## 2) 关键接口（contracts，注释版） -### 3.1 `trainer.py`：DistillTrainer（infra only） +### 2.1 `roles.py`：RoleManager / RoleHandle -**职责：** -- 迭代 dataloader + gradient accumulation -- 调用 `method.single_train_step()` 得到 loss/metrics -- backward + optimizer/scheduler step/zero_grad -- checkpoint save/resume（通过 utils/checkpoint） -- tracker 记录（统一在 trainer 侧 log） +```py +# RoleHandle：一个 role 的“资源包”（method 只通过 RoleHandle 操作 modules/optimizers） +RoleHandle: + modules: dict[str, nn.Module] # e.g. {"transformer": ..., "transformer_2": ...} + optimizers: dict[str, Optimizer] # method 创建并写回 + lr_schedulers: dict[str, Any] # method 创建并写回 + trainable: bool # 来自 roles..trainable（只影响 module 状态） -**不做的事：** -- 不知道 “Wan / CogVideoX / …” -- 不知道 “DMD2 / finetune / …” -- 不知道 “teacher/student/critic 这些角色语义” +# RoleManager：roles 的容器（role key 不限于 student/teacher/critic） +RoleManager: + roles: dict[str, RoleHandle] + require_roles([...]) # method 用它声明依赖（早失败、错误信息清晰） +``` -这样做的原因（FastGen 启发）： -> FastGen 的 Trainer 非常薄，算法更新策略由 method 决定；Trainer 只做 orchestration。 -这能显著降低 reviewer 读 loop 的心智负担，并避免把 update policy 固化在 Trainer 中。 +### 2.2 `dispatch.py`：registry + DistillRuntime -### 3.2 `methods/`：DistillMethod（算法层） +```py +# 目标：新增一个 model plugin 或 method 的成本是 O(1)，而不是写 N×M 组合函数 -**职责：** -- 定义训练一步：`single_train_step(batch, iteration, current_vsa_sparsity=...)` - - 返回：`loss_map`（tensor）、`outputs`（任意）、`metrics`（可 log 的标量） -- 定义 update policy：`get_optimizers()` / `get_lr_schedulers()` - - multi-optimizer / 不同 update interval 的策略属于算法的一部分 -- 定义 validation policy：method 构造 `ValidationRequest`，告诉 validator： - - 用哪个 role 的 transformer sample（通常是 student） - - sampler_kind（ode/sde）、steps、timesteps list、guidance_scale 等 +@register_model("wan") +def build_wan_components(cfg) -> ModelComponents: ... -**不做的事：** -- 不去关心具体模型加载（transformer/vae/text_encoder 的来源与细节） -- 不直接依赖 FastVideo pipeline 结构（通过 adapter/validator 间接使用） +@register_method("dmd2") +class DMD2Method(DistillMethod): ... -这样做的原因： -- update cadence（比如 generator_update_interval、critic 的 ratio）是算法语义，放 Trainer 会导致 Trainer 越来越“懂算法”。 -- method 作为算法实现者，天然需要决定“哪些 optimizer 该 step”。 +build_runtime_from_config(cfg): + components = model_builder(cfg) # -> ModelComponents + method = method_cls.build( # -> DistillMethod instance + cfg=cfg, + bundle=components.bundle, + adapter=components.adapter, + validator=components.validator, + ) + return DistillRuntime(training_args, method, dataloader, start_step) +``` + +### 2.3 `trainer.py`：DistillTrainer（infra only） + +```py +# Trainer 只看见 method（算法对象），不看见 roles 的语义，也不看见模型细节。 + +DistillTrainer(training_args, config=raw_yaml): + tracker = build_tracker(training_args, config=raw_yaml) + +run(method, dataloader, max_steps, start_step, checkpoint_manager?): + method.set_tracker(tracker) # 给 method/validator 注入 tracker（artifact logging） + method.on_train_start()? # 可选：让 method/adapter 做一次性初始化 + for step in steps: + loss_map, outputs, metrics = method.single_train_step(...) + method.backward(loss_map, outputs)? # 可选覆写（forward_context / ctx-aware backward） + method.optimizers_schedulers_step(step)? + method.optimizers_zero_grad(step)? + method.log_validation(step)? # 可选：method-managed validation + checkpoint_manager.maybe_save(step)? + tracker.log(...) +``` -### 3.3 `roles.py`：RoleManager / RoleHandle（多角色容器） +### 2.4 `adapters/`：Adapter 应提供哪些运行时 primitive？ -我们把训练参与者统一视为 **role**： -- role name 是字符串 key（例如 `"student"`, `"teacher"`, `"critic"`, `"reward"`…） -- `RoleHandle` 内部持有： - - `modules: dict[str, nn.Module]`（例如 transformer / transformer_2 / …） - - `optimizers / lr_schedulers` - - `trainable` 标记 +> 说明：`DistillAdapter` 基类只约束最小接口；具体 method 通过自己的 `Protocol` +> 显式声明需要哪些 primitive（duck typing）。这样避免把 DMD2 的需求硬塞进所有 adapter。 -**关键点：role 是可扩展的，不存在“canonical role 更高贵”的区分。** -method 决定自己需要哪些 role（通过 `bundle.require_roles([...])`）。 +当前方法族常用的 primitives（operation-centric）： -这样做的原因： -- distillation 形态差异巨大，硬编码固定角色集合会快速失控； -- 用 role dict 让新方法/新角色可以 additive 地接入，不影响 Trainer。 +```py +# batch/conditioning +prepare_batch(raw_batch, current_vsa_sparsity=..., latents_source={"data"|"zeros"}) -> TrainingBatch +on_train_start()? # seed/RNG/negative conditioning/cache（可选） +get_rng_generators()? # ckpt 时保存 RNG（可选） -### 3.4 `models/`：Model plugin（build-time 装配层） +# timestep/noise +num_train_timesteps -> int +shift_and_clamp_timestep(t) -> t +add_noise(clean_latents, noise, t) -> noisy_latents -`models/*` 的定位是：**把一个 family 的工程差异高内聚到一个地方**，并输出 method/Trainer 需要的“运行时组件包”： -- load modules（transformer / vae / …） -- 构建 `RoleManager`（把每个 role 的 modules 放进去） -- 构建 adapter / dataloader / validator / tracker -- 产出 `ModelComponents`（`models/components.py`） +# forward primitives（不区分 role，只吃 handle） +predict_x0(handle, noisy_latents, t, batch, conditional, attn_kind=...) -> x0 +predict_noise(handle, noisy_latents, t, batch, conditional, attn_kind=...) -> noise_like -为什么需要 model plugin，而不是让 method 直接 load？ -- method 关心的是算法；如果 method 去处理 “Wan 的 loader、并行切分、offload、module 名称、schema”等，会把模型工程细节污染进算法层。 -- model plugin 把 build-time 的“杂活”集中起来：更高内聚、更易替换/扩展。 +# backward（为了适配 forward_context / activation ckpt 等约束） +backward(loss, ctx, grad_accum_rounds=...) -> None +``` -### 3.5 `adapters/`：Adapter（运行时 primitive 层） +### 2.5 `methods/base.py`：一个 Method 应有哪些接口？ -adapter 是 method 与 FastVideo 运行时之间的“接口层”，应当遵循： -- **operation-centric API（按操作抽象）**，而不是 role-centric API（避免 role 爆炸）： - - 例如：`prepare_batch(...)`、`predict_x0(handle, ...)`、`predict_noise(handle, ...)`… -- adapter 不应该硬编码 DMD2/self-forcing 的 rollout 细节； - - rollout 的 step list/重加噪等属于 method（或 method_config）的策略。 +```py +class DistillMethod(nn.Module): + # dispatch 用 build(...) 统一装配实例（避免每个 method 写 build_*_method boilerplate） + @classmethod + def build(cfg, bundle, adapter, validator) -> DistillMethod -### 3.6 `validators/`：Validator（family-specific，method-controlled） + # 必需：训练一步 + def single_train_step(batch, iteration, current_vsa_sparsity=...) -> (loss_map, outputs, metrics) + # - loss_map 必须包含 total_loss（或 method 覆写 backward 自己处理） + # - metrics 用于额外标量日志（trainer 会统一 log） -validator 是 **模型相关** 的（例如 Wan 的采样管线、shape/latents 约定等），因此放在 `validators/wan.py`。 -但 validator 不应包含任何 “DMD2-specific” 逻辑： -- method 通过 `ValidationRequest` 指定 sampler_kind（ode/sde）、timesteps、steps 等； -- validator 只负责执行与记录（生成视频、写 mp4、通过 tracker 上传 artifacts）。 + # 必需：update policy（算法语义） + def get_optimizers(iteration) -> Sequence[Optimizer] + def get_lr_schedulers(iteration) -> Sequence[Any] -这能同时满足： -- validator 不被算法污染（保持复用性） -- 不同方法可以用同一个 validator，但采样策略由方法决定 + # 可选：更复杂的 backward / 多 ctx / forward_context 约束 + def backward(loss_map, outputs, grad_accum_rounds=...) -> None -### 3.7 `dispatch.py`：优雅 dispatch（避免 N×M builder） + # 可选：validation policy（method-managed） + def log_validation(step) -> None +``` -`dispatch.py` 提供 registry： -- `@register_model("wan")`：注册 model plugin builder -- `@register_method("dmd2") / @register_method("finetune")`：注册方法 builder +### 2.6 `validators/`：Validator（family-specific，method-controlled） -统一入口只需做一次组合： -- `recipe.family` → 选择模型插件 -- `recipe.method` → 选择算法方法 +```py +ValidationRequest: + sample_handle: RoleHandle # method 指定要采样哪个 role（通常 student） + sampler_kind: {"ode"|"sde"}? # method 指定采样 loop 类型 + sampling_steps: list[int]? # 展示用：要跑多少步（可能有多个） + sampling_timesteps: list[int]? # few-step schedule（与 sampling_steps 一致时更可控） + guidance_scale: float? + output_dir: str? -扩展一个新模型或新方法只需要新增一个 plugin 文件并注册，不需要写 25 个 “model×method 组合函数”。 +DistillValidator: + log_validation(step, request=ValidationRequest?) -> None +``` --- -## 4. YAML config 语义（Phase 3 的 schema） +## 3) 当前接受的 YAML config 格式（schema v2） -Phase 3 采用 **YAML-only**（不兼容旧 CLI configs），由 `utils/config.py` 解析为： -- `RecipeSpec`（dataclass）：`recipe.family` + `recipe.method` -- `roles: dict[str, RoleSpec]`（dataclass）：每个 role 的 family/path/trainable… -- `training_args: TrainingArgs`（dataclass）：训练超参（直接映射 FastVideoArgs/TrainingArgs） -- `method_config: dict[str, Any]`：方法私有参数（保持灵活，便于快速迭代） - -一个典型结构： +> 入口：`fastvideo/training/distillation.py --config /abs/path/to/run.yaml` +> +> 特性： +> - **YAML-only**：不与 legacy CLI configs merge +> - `training:` 大部分字段直接映射 `TrainingArgs.from_kwargs(...)` +> - `method_config:` 保持 dict（研究迭代快；由 method 自己解释/校验） ```yaml +# ---- 必需：选择 model plugin + method ---- recipe: - family: wan - method: dmd2 + family: wan # dispatch key：models/wan.py + method: dmd2 # dispatch key：methods/**/dmd2.py +# ---- 必需：定义 roles（role key 是任意字符串）---- roles: student: - family: wan + # family 可省略：默认继承 recipe.family path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers trainable: true teacher: @@ -214,87 +200,95 @@ roles: path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers trainable: true +# ---- 必需：训练参数（大部分字段直接走 TrainingArgs）---- training: - # 这里基本就是 TrainingArgs 的字段 + seed: 0 + output_dir: /path/to/out + data_path: /path/to/parquet max_train_steps: 4000 - learning_rate: 1.0e-6 - VSA_sparsity: 0.7 - ... - + train_batch_size: 1 + dataloader_num_workers: 4 + num_gpus: 8 + + # validation（由 method 调用 validator；validator 具体做采样与记录） + log_validation: true + validation_steps: 50 + validation_dataset_file: /path/to/validation.json + validation_sampling_steps: "8" # legacy-style string；method 可选择覆写 + validation_guidance_scale: "5.0" # legacy-style string；method 可选择覆写 + + # tracker（由 trainer 构建并持有；不建议在 model plugin 里构建） + trackers: ["wandb"] + tracker_project_name: my_project + wandb_run_name: my_run + +# ---- 可选：pipeline config（传入 TrainingArgs.pipeline_config）---- +# 只能提供一个：pipeline_config 或 pipeline_config_path pipeline_config: flow_shift: 3 sampler_kind: ode +# pipeline_config_path: /abs/path/to/pipeline_config.yaml +# ---- 可选：method 私有参数（dict，method 自己解析/校验）---- method_config: - # 仅 method 关心的内容 + # DMD2 示例 + rollout_mode: simulate dmd_denoising_steps: [999, 750, 500, 250, 0] - attn_kind: vsa + generator_update_interval: 1 + real_score_guidance_scale: 1.0 + attn_kind: dense ``` -为何 `method_config` 是 dict，而不是强类型 dataclass？ -- 方法参数变化频繁（研究迭代快），强类型会导致 schema/迁移成本高； -- 但 `recipe/roles/training_args` 是框架稳定边界，需要结构化来做 invariants 与错误提示。 - --- -## 5. 端到端执行路径（从命令到训练） - -``` -fastvideo/training/distillation.py - -> utils.config.load_distill_run_config() - -> dispatch.build_runtime_from_config() - -> models/.py: build_*_components() - -> methods/.py: build_*_method() - -> DistillTrainer.run(method, dataloader, ...) -``` - -工程性改进： -- entrypoint 会把本次运行的 YAML 原封不动通过 tracker 上传（例如 W&B Files），便于复现与审阅。 +## 4) 为什么这样设计（取舍逻辑） ---- +### 4.1 “很多东西都能写进 config”——为什么还要 model plugin？ -## 6. Phase 3 已达成的效果（可验证的“产出”） +可以把“模块名/类名/参数”写进 YAML，但最终仍需要一段代码来： +- 解释这些配置（动态 import / 默认值 / 校验 / 失败时给出清晰错误信息）； +- 做 build-time 的工程装配（模块是否存在、可选模块、shared components 复用、role 组合约束等）； +- 把 FastVideo 现实代码的差异（schema、parallel/offload、module packing 约定）收敛起来。 -截至 Phase 3，框架已经能做到： +因此我们把这层解释器命名为 **model plugin（`models/.py`）**： +- config 负责“选择 + 超参” +- model plugin 负责“把选择落地成可运行组件（ModelComponents）” -1) **完全摆脱 legacy distillation pipeline 的训练 loop** -- distill/finetune 都走统一入口 `fastvideo/training/distillation.py` -- Trainer/Method/Adapter/Validator 的边界清晰 +### 4.2 为什么 adapter 基类很薄（`DistillAdapter` 只有 prepare_batch）？ -2) **DMD2 few-step distillation 与 SFT/finetune 均可端到端跑通** -- DMD2：method 控制 rollout（SDE-style）、更新策略、validation 的采样步与 timesteps -- Finetune：只需要 student role，同框架内自然表达（“distillation 的特例”） +如果把 `predict_x0/add_noise/backward/...` 全塞进一个巨大的 adapter 基类： +- 你会把某个算法（例如 DMD2）的需求固化成“全体 adapter 必须实现”的硬约束； +- 未来新增方法会被迫实现一堆不需要的接口（耦合上升、可维护性变差）。 -3) **VSA（Video Sparse Attention）可作为 backend 使用，并可 schedule/记录** -- `training.VSA_sparsity / VSA_decay_*` 可影响训练时 `current_vsa_sparsity` -- trainer 统一 log `vsa_sparsity` -- validator 确认会使用 `training_args.VSA_sparsity`（forward-time） +当前策略： +- adapter 的稳定边界保持最小 +- 每个 method 用 `Protocol` 显式声明自己需要哪些 primitives（代码可读、依赖清晰） -4) **扩展路径更清晰** -- 增加一个新方法：新增 `methods//.py` + `register_method()` -- 增加一个新模型家族：新增 `models/.py` + `adapters/.py` + `validators/.py` + `register_model()` +### 4.3 为什么 optimizer/scheduler 由 method 创建？ ---- +optimizer cadence / 多优化器更新比例 / critic 的超参等都属于算法语义。 +如果放在 model plugin，会导致： +- model plugin 需要理解 DMD2/finetune/... 的算法细节（污染 build-time 层） +- 同一个 family 随着方法增多出现“把算法 if/else 塞进 models/”的风险 -## 7. 已知限制与下一步（面向 Phase 4+） +### 4.4 为什么 tracker 由 trainer 构建并持有？ -Phase 3 结束时仍有一些“刻意未做”的点： -- 多 family 角色组合（例如 student=wan, teacher=sdxl）尚未正式支持； - - 未来可能需要 method 持有多个 adapter/validator，或在 roles 层引入 cross-family 约束检查（例如 VAE/latent space 是否兼容）。 -- 更多方法类别（KD/CM/self-forcing 等）与更多 model plugins 仍需逐步落地。 -- dataloader schema 更系统的抽象（DataSpec）目前仍偏工程化，不影响核心分层但有优化空间。 +tracker 是 infra 资源（日志/文件/媒体记录），生命周期属于训练 loop： +- trainer 负责创建 tracker，并统一 `tracker.log(...)` +- method 只产出 metrics；若 method-managed validation 需要 log 视频，则通过 `method.set_tracker(...)` + 把 tracker 注入到 validator（而不是让 model plugin 构建并传递 tracker） ---- +### 4.5 为什么 `method_config` 是 dict？ -## 8. 附：FastGen 对我们的直接启发点（总结） +研究/工程迭代中，方法参数变化频繁；强类型 schema 会带来迁移成本。 +我们把稳定边界结构化（`recipe/roles/training`），把快速变化的部分留给 dict： +- method 自己解析/校验（并给出明确错误提示） +- 解析 helper（int/float/betas）放在 `utils/config.py` 复用，减少重复代码 -我们从 FastGen 学到的最关键结构是： -- **Trainer 薄、Method 厚（策略在 method）**； -- config 选择 method/network，训练时 Trainer “只看 method”。 +### 4.6 为什么需要 `dispatch.py` 的 registry？ -在 FastVideo 中，我们保留这条主线，同时增加了： -- **Adapter**（吸收 FastVideo pipeline/forward 的差异，让 method 可复用） -- **Model plugin（build-time 装配层）**（吸收 loading/dataloader/validator/tracker 等工程差异） -- **RoleManager**（支持任意角色组合） +目标是避免“模型×方法”的组合爆炸： +- 新增一个 family → 加一个 model plugin 并注册 +- 新增一个 method → 加一个 method 文件并注册 +- 不需要写 25 个 build 函数或 if/else 分支 -这套结构让 FastVideo distillation 能以“增量 PR”方式逐步替换 legacy pipeline，而不是一次性大爆炸式重写。 From cfa4318913ff50b5520840c51db0b47d4c89c924 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 04:59:01 +0000 Subject: [PATCH 110/214] rfc en --- fastvideo/distillation/doc/RFC-v1-en.md | 82 +++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 fastvideo/distillation/doc/RFC-v1-en.md diff --git a/fastvideo/distillation/doc/RFC-v1-en.md b/fastvideo/distillation/doc/RFC-v1-en.md new file mode 100644 index 000000000..e66c13915 --- /dev/null +++ b/fastvideo/distillation/doc/RFC-v1-en.md @@ -0,0 +1,82 @@ +## 1) File Structure + +```text +fastvideo/distillation/ + trainer.py # Builds the training loop; calls method-provided train_one_step + dispatch.py # Auto-dispatch via @register_method/@register_model; builds DistillRuntime from config + roles.py # Wraps model resources in RoleHandle to tag roles (teacher/student/critic/...) + models/ + components.py # intermediate variable during dispatch-time model construction; records all components + wan.py # loads Wan. model-loading logic differs across families + ... + adapters/ + base.py # turns loaded components into runtime primitives: predict_x0/add_noise/backward/... + wan.py # wan-specific adapter. + ... + methods/ + base.py # DistillMethod base; methods must provide e.g. train-one-step for trainer to call + distribution_matching/ + dmd2.py # DMD2 distillation (student/teacher/critic) + fine_tuning/ + finetune.py # SFT finetuning (student only) + knowledge_distillation/ + consistency_model/ + validators/ + base.py # inference differs by model; validators are model-specific. + wan.py # validator backed by WanPipeline. + utils/ + config.py # yaml parser + dataloader.py + moduleloader.py + module_state.py # apply_trainable(...): standardizes requires_grad + train/eval + tracking.py # wandb tracker (owned by trainer) + checkpoint.py # save/resume + +``` + +``` yaml +recipe: + family: wan + method: dmd2 + +roles: + student: + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + path: Wan-AI/Wan2.1-T2V-14B-Diffusers + trainable: false + critic: + path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +training: + seed: 0 + output_dir: /path/to/out + data_path: /path/to/parquet + max_train_steps: 4000 + train_batch_size: 1 + dataloader_num_workers: 4 + num_gpus: 8 + + log_validation: true + validation_steps: 50 + validation_dataset_file: /path/to/validation.json + validation_sampling_steps: "8" # legacy-style string；method 可选择覆写 + validation_guidance_scale: "5.0" # legacy-style string；method 可选择覆写 + + trackers: ["wandb"] + tracker_project_name: my_project + wandb_run_name: my_run + +pipeline_config: + flow_shift: 3 + sampler_kind: ode + +method_config: + rollout_mode: simulate + dmd_denoising_steps: [999, 750, 500, 250, 0] + generator_update_interval: 1 + real_score_guidance_scale: 1.0 + attn_kind: dense +``` \ No newline at end of file From 666ec3625548f5bfe372a6cfbb3928f1e489938d Mon Sep 17 00:00:00 2001 From: mignonjia Date: Thu, 26 Feb 2026 05:56:16 +0000 Subject: [PATCH 111/214] eval --- fastvideo/training/training_pipeline.py | 216 ++++++++++++++---- .../training/wangame_training_pipeline.py | 108 ++++++++- 2 files changed, 273 insertions(+), 51 deletions(-) diff --git a/fastvideo/training/training_pipeline.py b/fastvideo/training/training_pipeline.py index 61cc038e1..467395182 100644 --- a/fastvideo/training/training_pipeline.py +++ b/fastvideo/training/training_pipeline.py @@ -634,6 +634,9 @@ def train(self) -> None: self._log_training_info() + self._best_mf_angle_err_mean = float('inf') + self._last_mf_angle_err_mean = float('inf') + self._log_validation(self.transformer, self.training_args, self.init_steps) @@ -743,6 +746,43 @@ def train(self) -> None: "GPU memory usage after validation: %s MB, trainable params: %sB", gpu_memory_usage, trainable_params) + best_start = self.training_args.best_checkpoint_start_step + if (best_start > 0 + and step >= best_start + and self._last_mf_angle_err_mean + < self._best_mf_angle_err_mean): + self._best_mf_angle_err_mean = ( + self._last_mf_angle_err_mean) + logger.info( + "New best mf_angle_err_mean=%.6f at step %d, " + "saving best checkpoint", + self._best_mf_angle_err_mean, step) + save_checkpoint( + self.transformer, self.global_rank, + self.training_args.output_dir, "best", + self.optimizer, self.train_dataloader, + self.lr_scheduler, + self.noise_random_generator) + if self.global_rank == 0: + import json + meta_path = os.path.join( + self.training_args.output_dir, + "checkpoint-best", + "best_metric.json") + with open(meta_path, "w") as f: + json.dump({ + "step": step, + "mf_angle_err_mean": + self._best_mf_angle_err_mean, + }, f, indent=2) + self.tracker.log({ + "best/mf_angle_err_mean": + self._best_mf_angle_err_mean, + "best/step": step, + }, step) + self.transformer.train() + self.sp_group.barrier() + self.tracker.finish() save_checkpoint(self.transformer, self.global_rank, self.training_args.output_dir, @@ -841,11 +881,25 @@ def _post_process_validation_frames(self, frames: list[np.ndarray], """ return frames + def _evaluate_validation_video( + self, + video_path: str, + caption: str, + action_path: str | None, + global_step: int, + num_inference_steps: int, + ) -> dict[str, float] | None: + """Optionally evaluate a saved validation video and return scalars.""" + del video_path, caption, action_path, global_step + del num_inference_steps + return None + @torch.no_grad() def _log_validation(self, transformer, training_args, global_step) -> None: """ Generate a validation video and log it to the configured tracker to check the quality during training. """ + self._last_mf_angle_err_mean = float('inf') training_args.inference_mode = True training_args.dit_cpu_offload = False if not training_args.log_validation: @@ -889,12 +943,16 @@ def _log_validation(self, transformer, training_args, global_step) -> None: local_main_process_only=False) step_videos: list[np.ndarray] = [] step_captions: list[str] = [] + step_action_paths: list[str | None] = [] for validation_batch in validation_dataloader: batch = self._prepare_validation_batch(sampling_param, training_args, validation_batch, num_inference_steps) + action_path = validation_batch.get("action_path") + if not isinstance(action_path, str): + action_path = None logger.info("rank: %s: rank_in_sp_group: %s, batch.prompt: %s", self.global_rank, self.rank_in_sp_group, @@ -924,64 +982,122 @@ def _log_validation(self, transformer, training_args, global_step) -> None: # Apply optional post-processing (e.g., overlay for action-conditioned models) frames = self._post_process_validation_frames(frames, batch) step_videos.append(frames) + step_action_paths.append(action_path) # Only sp_group leaders (rank_in_sp_group == 0) need to send their # results to global rank 0 - if self.rank_in_sp_group == 0 and self.global_rank == 0: - # Global rank 0 collects results from all sp_group leaders - all_videos = step_videos # Start with own results - all_captions = step_captions - - # Receive from other sp_group leaders - for sp_group_idx in range(1, num_sp_groups): - src_rank = sp_group_idx * self.sp_world_size # Global rank of other sp_group leaders - recv_videos = world_group.recv_object(src=src_rank) - recv_captions = world_group.recv_object(src=src_rank) - all_videos.extend(recv_videos) - all_captions.extend(recv_captions) - - video_filenames = [] - for i, (video, caption) in enumerate( - zip(all_videos, all_captions, strict=True)): + if self.rank_in_sp_group == 0: + local_video_filenames: list[str] = [] + local_validation_metrics: list[dict[str, float]] = [] + local_eval_error: str | None = None + + for i, (video, caption, action_path) in enumerate( + zip(step_videos, + step_captions, + step_action_paths, + strict=True)): os.makedirs(training_args.output_dir, exist_ok=True) filename = os.path.join( training_args.output_dir, - f"validation_step_{global_step}_inference_steps_{num_inference_steps}_video_{i}.mp4" + f"validation_step_{global_step}_inference_steps_{num_inference_steps}_rank_{self.global_rank}_video_{i}.mp4" ) imageio.mimsave(filename, video, fps=sampling_param.fps) - # Mux audio if available - audio = output_batch.extra.get("audio") - audio_sample_rate = output_batch.extra.get( - "audio_sample_rate") - if (audio is not None and audio_sample_rate is not None - and not self._mux_audio( - filename, - audio, - audio_sample_rate, - )): - logger.warning( - "Audio mux failed for validation video %s; saved video without audio.", - filename) - video_filenames.append(filename) - - artifacts = [] - for filename, caption in zip(video_filenames, - all_captions, - strict=True): - video_artifact = self.tracker.video(filename, - caption=caption) - if video_artifact is not None: - artifacts.append(video_artifact) - if artifacts: - logs = { - f"validation_videos_{num_inference_steps}_steps": - artifacts - } - self.tracker.log_artifacts(logs, global_step) - elif self.rank_in_sp_group == 0: - # Other sp_group leaders send their results to global rank 0 - world_group.send_object(step_videos, dst=0) - world_group.send_object(step_captions, dst=0) + local_video_filenames.append(filename) + + try: + sample_metrics = self._evaluate_validation_video( + video_path=filename, + caption=caption, + action_path=action_path, + global_step=global_step, + num_inference_steps=num_inference_steps, + ) + if sample_metrics: + local_validation_metrics.append(sample_metrics) + except Exception as e: + local_eval_error = ( + f"rank {self.global_rank} validation eval failed " + f"for {filename}: {e}") + logger.exception(local_eval_error) + break + + if self.global_rank == 0: + all_video_filenames = local_video_filenames + all_captions = step_captions + validation_metrics = local_validation_metrics + eval_errors: list[str] = [] + if local_eval_error: + eval_errors.append(local_eval_error) + + # Receive from other sp_group leaders + for sp_group_idx in range(1, num_sp_groups): + src_rank = sp_group_idx * self.sp_world_size + recv_video_filenames = world_group.recv_object( + src=src_rank) + recv_captions = world_group.recv_object(src=src_rank) + recv_metrics = world_group.recv_object(src=src_rank) + recv_error = world_group.recv_object(src=src_rank) + + all_video_filenames.extend(recv_video_filenames) + all_captions.extend(recv_captions) + validation_metrics.extend(recv_metrics) + if recv_error: + eval_errors.append(str(recv_error)) + + if eval_errors: + raise RuntimeError( + "Validation flow evaluation failed:\n" + + "\n".join(eval_errors)) + + artifacts = [] + for filename, caption in zip(all_video_filenames, + all_captions, + strict=True): + video_artifact = self.tracker.video(filename, + caption=caption) + if video_artifact is not None: + artifacts.append(video_artifact) + if artifacts: + logs = { + f"validation_videos_{num_inference_steps}_steps": + artifacts + } + self.tracker.log_artifacts(logs, global_step) + + if validation_metrics: + metric_logs: dict[str, float] = {} + metric_keys = sorted( + {k for row in validation_metrics for k in row.keys()}) + for metric_key in metric_keys: + metric_vals = [ + row[metric_key] for row in validation_metrics + if metric_key in row + and np.isfinite(row[metric_key]) + ] + if not metric_vals: + continue + metric_logs[f"metrics/{metric_key}"] = float( + np.mean(metric_vals)) + self.tracker.log(metric_logs, global_step) + + mf_val = metric_logs.get( + "metrics/mf_angle_err_mean") + if mf_val is not None: + self._last_mf_angle_err_mean = mf_val + else: + # Other sp_group leaders send their local results to rank 0 + world_group.send_object(local_video_filenames, dst=0) + world_group.send_object(step_captions, dst=0) + world_group.send_object(local_validation_metrics, dst=0) + world_group.send_object(local_eval_error, dst=0) + if local_eval_error: + raise RuntimeError(local_eval_error) + + # Broadcast the latest mf_angle_err_mean from rank 0 to all ranks + _mf_tensor = torch.tensor( + [self._last_mf_angle_err_mean], device=self.device) + dist.broadcast(_mf_tensor, src=0) + self._last_mf_angle_err_mean = _mf_tensor.item() # Re-enable gradients for training training_args.inference_mode = False diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py index 4617cb1c4..d06ba2480 100644 --- a/fastvideo/training/wangame_training_pipeline.py +++ b/fastvideo/training/wangame_training_pipeline.py @@ -31,6 +31,18 @@ class WanGameTrainingPipeline(TrainingPipeline): """ _required_config_modules = ["scheduler", "transformer", "vae"] + _FLOW_EVAL_SCALAR_KEYS = ( + "mf_epe_mean", + "mf_angle_err_mean", + "mf_cosine_mean", + "mf_mag_ratio_mean", + "pixel_epe_mean_mean", + "px_angle_rmse_mean", + "fl_all_mean", + "foe_dist_mean", + "flow_kl_2d_mean", + ) + def initialize_pipeline(self, fastvideo_args: FastVideoArgs): self.modules["scheduler"] = FlowUniPCMultistepScheduler( shift=fastvideo_args.pipeline_config.flow_shift) @@ -296,7 +308,7 @@ def _build_input_kwargs(self, # "encoder_attention_mask": # training_batch.encoder_attention_mask, "encoder_hidden_states_image": - [encoder_hidden_states_image], + encoder_hidden_states_image, # Action conditioning "viewmats": viewmats, "Ks": intrinsics, @@ -400,6 +412,100 @@ def _post_process_validation_frames(self, frames: list[np.ndarray], return processed_frames + def _init_flow_eval_module(self) -> None: + if getattr(self, "_flow_eval_init_done", False): + return + self._flow_eval_init_done = True + self._flow_eval_ready = False + + ptlflow_dir = Path("/mnt/weka/home/hao.zhang/mhuo/FastVideo/benchmarks/ptlflow") + + try: + ptlflow_dir_str = str(ptlflow_dir.resolve()) + if ptlflow_dir_str not in sys.path: + sys.path.insert(0, ptlflow_dir_str) + + from eval_flow_divergence import evaluate_pair_synthetic # type: ignore + + self._flow_eval_fn = evaluate_pair_synthetic + self._flow_eval_ckpt = str(ptlflow_dir / "dpflow-things-2012b5d6.ckpt") + self._flow_eval_calibration_path = str(ptlflow_dir / + "calibration.json") + self._flow_eval_ready = True + logger.info("Initialized flow divergence evaluator: %s", + ptlflow_dir) + except Exception as e: + logger.warning("Failed to initialize flow divergence evaluator: %s", + e) + + def _evaluate_validation_video( + self, + video_path: str, + caption: str, + action_path: str | None, + global_step: int, + num_inference_steps: int, + ) -> dict[str, float]: + del caption + self._init_flow_eval_module() + if not getattr(self, "_flow_eval_ready", False): + raise RuntimeError( + "ptlflow evaluator is not initialized; cannot compute flow metrics." + ) + + if not isinstance(action_path, str) or not os.path.isfile(action_path): + raise FileNotFoundError( + f"Validation sample is missing a valid action_path: {action_path}" + ) + + eval_output_dir = os.path.join( + self.training_args.output_dir, + "flow_eval", + f"step_{global_step}", + f"inference_steps_{num_inference_steps}", + Path(video_path).stem, + ) + + try: + summary = self._flow_eval_fn( + gen_video=video_path, + action_path=action_path, + calibration_path=self._flow_eval_calibration_path, + output_dir=eval_output_dir, + model_name="dpflow", + ckpt=self._flow_eval_ckpt, + no_viz=True, + use_depth=True, + ) + except Exception as e: + raise RuntimeError( + f"ptlflow synthetic evaluation failed for {video_path}") from e + + if not isinstance(summary, dict): + raise RuntimeError( + f"ptlflow returned invalid summary type: {type(summary)}" + ) + + metrics: dict[str, float] = {} + missing_or_invalid: list[str] = [] + for key in self._FLOW_EVAL_SCALAR_KEYS: + val = summary.get(key) + if not isinstance(val, (float, int, np.floating, np.integer)): + missing_or_invalid.append(key) + continue + val_float = float(val) + if not np.isfinite(val_float): + missing_or_invalid.append(key) + continue + metrics[key] = val_float + + if missing_or_invalid: + raise RuntimeError( + "ptlflow summary missing/invalid metrics: " + f"{', '.join(missing_or_invalid)}") + + return metrics + def main(args) -> None: logger.info("Starting training pipeline...") From e3cd0a370170e686cf15c6312115bdf2518acb7e Mon Sep 17 00:00:00 2001 From: mignonjia Date: Thu, 26 Feb 2026 07:19:41 +0000 Subject: [PATCH 112/214] select best ckpt --- fastvideo/fastvideo_args.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fastvideo/fastvideo_args.py b/fastvideo/fastvideo_args.py index 1e651d293..2ca79b808 100644 --- a/fastvideo/fastvideo_args.py +++ b/fastvideo/fastvideo_args.py @@ -931,6 +931,7 @@ class TrainingArgs(FastVideoArgs): fake_score_betas: str = "0.9,0.999" # betas for fake score optimizer, format: "beta1,beta2" training_state_checkpointing_steps: int = 0 # for resuming training weight_only_checkpointing_steps: int = 0 # for inference + best_checkpoint_start_step: int = 0 # save best checkpoint (by mf_angle_err_mean) after this step; 0 = disabled log_visualization: bool = False visualization_steps: int = 0 # simulate generator forward to match inference @@ -1134,6 +1135,11 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: "--weight-only-checkpointing-steps", type=int, help="Steps between weight-only checkpoints (for inference)") + parser.add_argument( + "--best-checkpoint-start-step", + type=int, + help="Save best checkpoint (by mf_angle_err_mean) after this " + "step; 0 = disabled") parser.add_argument("--resume-from-checkpoint", type=str, help="Path to checkpoint to resume from") From 2fb4655fc00c2a7d55ed4288ac7e5fb80790a0ee Mon Sep 17 00:00:00 2001 From: H1yori233 Date: Thu, 26 Feb 2026 09:31:19 +0000 Subject: [PATCH 113/214] add wangame diffusion forcing --- .../distill/SFWanGame2.1/distill_dmd.slurm | 12 +- .../causal_wangame_ode_init/ar_diff.slurm | 126 +++++ .../training/wangame_ar_diffusion_pipeline.py | 527 ++++++++++++++++++ 3 files changed, 660 insertions(+), 5 deletions(-) create mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/ar_diff.slurm create mode 100644 fastvideo/training/wangame_ar_diffusion_pipeline.py diff --git a/examples/distill/SFWanGame2.1/distill_dmd.slurm b/examples/distill/SFWanGame2.1/distill_dmd.slurm index e2acd5460..b3ec595d3 100644 --- a/examples/distill/SFWanGame2.1/distill_dmd.slurm +++ b/examples/distill/SFWanGame2.1/distill_dmd.slurm @@ -7,8 +7,8 @@ #SBATCH --gres=gpu:8 #SBATCH --cpus-per-task=128 #SBATCH --mem=1440G -#SBATCH --output=sf_train_output/ode_%j.out -#SBATCH --error=sf_train_output/ode_%j.err +#SBATCH --output=log/sf_train_output/ode_%j.out +#SBATCH --error=log/sf_train_output/ode_%j.err #SBATCH --exclusive set -e -x @@ -36,11 +36,13 @@ echo "RUN_NAME: $RUN_NAME" # Model paths for Self-Forcing DMD distillation: # GENERATOR_MODEL_PATH="../wg_models/WanGame-2.1-Student-VizDoom1k-1000steps-Diffusers" -GENERATOR_MODEL_PATH="../wg_models/SFWanGame-2.1-0223-9000steps" +# GENERATOR_MODEL_PATH="../wg_models/SFWanGame-2.1-0223-9000steps" +GENERATOR_MODEL_PATH="../wg_models/SFWanGame-2.1-0224-4k5steps" REAL_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Teacher model FAKE_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Critic model -DATA_DIR="../traindata_0222_0030/ode_init_mc_with_mouse/preprocessed_wangame" +# DATA_DIR="../traindata_0222_0030/ode_init_mc_with_mouse/preprocessed_wangame" +DATA_DIR="../traindata_0222_0030/ode_init_mc_Xonly_3k/preprocessed" VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" # Training arguments @@ -139,7 +141,7 @@ self_forcing_args=( --context_noise 0 # Amount of noise to add during context caching (0 = no noise) ) -mkdir -p sf_train_output +mkdir -p log/sf_train_output srun torchrun \ --nnodes $SLURM_JOB_NUM_NODES \ diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/ar_diff.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/ar_diff.slurm new file mode 100644 index 000000000..1fb83ad48 --- /dev/null +++ b/examples/training/consistency_finetune/causal_wangame_ode_init/ar_diff.slurm @@ -0,0 +1,126 @@ +#!/bin/bash +#SBATCH --job-name=wg-ar-diff +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=log/ar_diff_output/ar_diff_%j.out +#SBATCH --error=log/ar_diff_output/ar_diff_%j.err +#SBATCH --exclusive + +# Environment Setup +source ~/conda/miniconda/bin/activate +conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv +export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" + +export NCCL_P2P_DISABLE=1 +export TORCH_NCCL_ENABLE_MONITORING=0 +export MASTER_PORT=29503 +export TOKENIZERS_PARALLELISM=false +export WANDB_MODE=online +export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN + +# Basic Info +export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" +export WANDB_MODE="online" +export NCCL_P2P_DISABLE=1 +export MASTER_PORT=29500 +export NODE_RANK=$SLURM_PROCID +nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) +export MASTER_ADDR=${nodes[0]} +export TOKENIZERS_PARALLELISM=false + +echo "MASTER_ADDR: $MASTER_ADDR" +echo "NODE_RANK: $NODE_RANK" + +RUN_NAME=$(date +"%m%d_%H%M") +echo "RUN_NAME: $RUN_NAME" + +MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" + +DATA_DIR="../traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed" +VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json" + +training_args=( + --tracker_project_name wangame_ar_diffusion + --output_dir "checkpoints/wangame_ar_diffusion_${RUN_NAME}" + --wandb_run_name "${RUN_NAME}_df_bs32" + --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" + --max_train_steps 5000 + --train_batch_size 1 + --train_sp_batch_size 1 + --gradient_accumulation_steps 1 + --num_latent_t 15 + --num_height 352 + --num_width 640 + --enable_gradient_checkpointing_type "full" + --num_frames 57 + --num_frame_per_block 3 +) + +parallel_args=( + --num_gpus 32 + --sp_size 1 + --tp_size 1 + --hsdp_replicate_dim 1 + --hsdp_shard_dim 32 +) + +model_args=( + --model_path $MODEL_PATH + --pretrained_model_name_or_path $MODEL_PATH +) + +dataset_args=( + --data_path "$DATA_DIR" + --dataloader_num_workers 4 +) + +validation_args=( + --log_validation + --validation_dataset_file "$VALIDATION_DATASET_FILE" + --validation_steps 200 + --validation_sampling_steps "50" + --validation_guidance_scale "1.0" +) + +optimizer_args=( + --learning_rate 2e-5 + --mixed_precision "bf16" + --training_state_checkpointing_steps 500 + --weight_only_checkpointing_steps 500 + --weight_decay 0.01 + --betas '0.9,0.999' + --max_grad_norm 1.0 + --lr_scheduler cosine + --lr_warmup_steps 100 +) + +miscellaneous_args=( + --inference_mode False + --checkpoints_total_limit 3 + --training_cfg_rate 0.0 + --dit_precision "fp32" + --flow_shift 8 + --seed 42 +) + +mkdir -p log/ar_diff_output + +srun torchrun \ +--nnodes $SLURM_JOB_NUM_NODES \ +--nproc_per_node 8 \ +--node_rank $SLURM_PROCID \ +--rdzv_backend=c10d \ +--rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ + fastvideo/training/wangame_ar_diffusion_pipeline.py \ + "${parallel_args[@]}" \ + "${model_args[@]}" \ + "${dataset_args[@]}" \ + "${training_args[@]}" \ + "${optimizer_args[@]}" \ + "${validation_args[@]}" \ + "${miscellaneous_args[@]}" diff --git a/fastvideo/training/wangame_ar_diffusion_pipeline.py b/fastvideo/training/wangame_ar_diffusion_pipeline.py new file mode 100644 index 000000000..7adbbb6c8 --- /dev/null +++ b/fastvideo/training/wangame_ar_diffusion_pipeline.py @@ -0,0 +1,527 @@ +# SPDX-License-Identifier: Apache-2.0 + +import sys +from copy import deepcopy +from typing import Any, cast + +import numpy as np +import torch +import torch.nn.functional as F + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame +from fastvideo.distributed import get_local_torch_device +from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs +from fastvideo.forward_context import set_forward_context +from fastvideo.logger import init_logger +from fastvideo.models.dits.hyworld.pose import process_custom_actions +from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( + SelfForcingFlowMatchScheduler) +from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( + WanGameCausalDMDPipeline) +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch +from fastvideo.training.training_pipeline import TrainingPipeline +from fastvideo.training.training_utils import ( + clip_grad_norm_while_handling_failing_dtensor_cases) +from fastvideo.utils import shallow_asdict + +logger = init_logger(__name__) + + +class WanGameARDiffusionPipeline(TrainingPipeline): + + _required_config_modules = ["scheduler", "transformer", "vae"] + + def initialize_pipeline(self, fastvideo_args: FastVideoArgs): + self.modules["scheduler"] = SelfForcingFlowMatchScheduler( + shift=fastvideo_args.pipeline_config.flow_shift, + sigma_min=0.0, + extra_one_step=True) + self.modules["scheduler"].set_timesteps(num_inference_steps=1000, + training=True) + + def set_schemas(self): + self.train_dataset_schema = pyarrow_schema_wangame + + def initialize_training_pipeline(self, training_args: TrainingArgs): + super().initialize_training_pipeline(training_args) + + self.vae = self.get_module("vae") + self.vae.requires_grad_(False) + + self.num_frame_per_block = getattr(training_args, 'num_frame_per_block', 3) + self.timestep_shift = training_args.pipeline_config.flow_shift + self.ar_noise_scheduler = SelfForcingFlowMatchScheduler( + shift=self.timestep_shift, sigma_min=0.0, extra_one_step=True) + self.ar_noise_scheduler.set_timesteps(num_inference_steps=1000, + training=True) + + logger.info("AR Diffusion pipeline initialized with " + "num_frame_per_block=%d, timestep_shift=%.1f", + self.num_frame_per_block, self.timestep_shift) + + def initialize_validation_pipeline(self, training_args: TrainingArgs): + logger.info("Initializing validation pipeline...") + args_copy = deepcopy(training_args) + args_copy.inference_mode = True + + validation_scheduler = SelfForcingFlowMatchScheduler( + shift=args_copy.pipeline_config.flow_shift, + sigma_min=0.0, + extra_one_step=True) + validation_scheduler.set_timesteps(num_inference_steps=1000, + training=True) + + num_val_steps = int( + training_args.validation_sampling_steps.split(",")[0]) + step_size = 1000 // num_val_steps + args_copy.pipeline_config.dmd_denoising_steps = list( + range(1000, 0, -step_size)) + args_copy.pipeline_config.warp_denoising_step = True + training_args.pipeline_config.dmd_denoising_steps = ( + args_copy.pipeline_config.dmd_denoising_steps) + training_args.pipeline_config.warp_denoising_step = True + + logger.info("Validation: %d-step causal denoising, " + "dmd_denoising_steps has %d entries", + num_val_steps, + len(args_copy.pipeline_config.dmd_denoising_steps)) + + self.validation_pipeline = WanGameCausalDMDPipeline.from_pretrained( + training_args.model_path, + args=args_copy, + inference_mode=True, + loaded_modules={ + "transformer": self.get_module("transformer"), + "vae": self.get_module("vae"), + "scheduler": validation_scheduler, + }, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + pin_cpu_memory=training_args.pin_cpu_memory, + dit_cpu_offload=True) + + def _get_timestep( + self, + min_timestep: int, + max_timestep: int, + batch_size: int, + num_frame: int, + num_frame_per_block: int, + uniform_timestep: bool = False, + ) -> torch.Tensor: + """ + Sample per-block timesteps. + """ + device = get_local_torch_device() + if uniform_timestep: + timestep = torch.randint( + min_timestep, max_timestep, [batch_size, 1], + device=device, dtype=torch.long + ).repeat(1, num_frame) + return timestep + else: + timestep = torch.randint( + min_timestep, max_timestep, [batch_size, num_frame], + device=device, dtype=torch.long + ) + # Make the noise level the same within every block + timestep = timestep.reshape( + timestep.shape[0], -1, num_frame_per_block) + timestep[:, :, 1:] = timestep[:, :, 0:1] + timestep = timestep.reshape(timestep.shape[0], -1) + return timestep + + def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: + batch = next(self.train_loader_iter, None) # type: ignore + if batch is None: + self.current_epoch += 1 + logger.info("Starting epoch %s", self.current_epoch) + self.train_dataset.sampler.set_epoch(self.current_epoch) + self.train_loader_iter = iter(self.train_dataloader) + batch = next(self.train_loader_iter) + + latents = batch['vae_latent'] + latents = latents[:, :, :self.training_args.num_latent_t] + clip_features = batch['clip_feature'] + image_latents = batch['first_frame_latent'] + image_latents = image_latents[:, :, :self.training_args.num_latent_t] + pil_image = batch['pil_image'] + infos = batch['info_list'] + + training_batch.latents = latents.to(get_local_torch_device(), + dtype=torch.bfloat16) + training_batch.encoder_hidden_states = None + training_batch.encoder_attention_mask = None + training_batch.preprocessed_image = pil_image.to( + get_local_torch_device()) + training_batch.image_embeds = clip_features.to(get_local_torch_device()) + training_batch.image_latents = image_latents.to( + get_local_torch_device()) + training_batch.infos = infos + + # Action conditioning + if 'mouse_cond' in batch and batch['mouse_cond'].numel() > 0: + training_batch.mouse_cond = batch['mouse_cond'].to( + get_local_torch_device(), dtype=torch.bfloat16) + else: + training_batch.mouse_cond = None + + if 'keyboard_cond' in batch and batch['keyboard_cond'].numel() > 0: + training_batch.keyboard_cond = batch['keyboard_cond'].to( + get_local_torch_device(), dtype=torch.bfloat16) + else: + training_batch.keyboard_cond = None + + # Validate action temporal dimensions match video num_frames + expected_num_frames = (self.training_args.num_latent_t - 1) * 4 + 1 + if training_batch.keyboard_cond is not None: + assert training_batch.keyboard_cond.shape[1] >= expected_num_frames, ( + f"keyboard_cond has {training_batch.keyboard_cond.shape[1]} frames " + f"but need at least {expected_num_frames}") + training_batch.keyboard_cond = training_batch.keyboard_cond[:, :expected_num_frames] + if training_batch.mouse_cond is not None: + assert training_batch.mouse_cond.shape[1] >= expected_num_frames, ( + f"mouse_cond has {training_batch.mouse_cond.shape[1]} frames " + f"but need at least {expected_num_frames}") + training_batch.mouse_cond = training_batch.mouse_cond[:, :expected_num_frames] + + return training_batch + + def _prepare_dit_inputs(self, + training_batch: TrainingBatch) -> TrainingBatch: + """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" + assert self.training_args is not None + latents = training_batch.latents # [B, C, T, H, W] + batch_size = latents.shape[0] + num_latent_t = latents.shape[2] + + # Reshape latents to [B, T, C, H, W] for per-frame operations + latents_btchw = latents.permute(0, 2, 1, 3, 4) # [B, T, C, H, W] + + # Sample per-block independent timestep indices: [B, T] + timestep_indices = self._get_timestep( + min_timestep=0, + max_timestep=self.ar_noise_scheduler.num_train_timesteps, + batch_size=batch_size, + num_frame=num_latent_t, + num_frame_per_block=self.num_frame_per_block, + uniform_timestep=False) + + # Convert indices to actual timestep values: [B, T] + self.ar_noise_scheduler.timesteps = self.ar_noise_scheduler.timesteps.to( + get_local_torch_device()) + timesteps = self.ar_noise_scheduler.timesteps[timestep_indices] + + # Generate noise: [B, T, C, H, W] + noise = torch.randn_like(latents_btchw) + + # Add noise per-frame: noisy = (1-σ) * clean + σ * noise + noisy_latents = self.ar_noise_scheduler.add_noise( + latents_btchw.flatten(0, 1), # [B*T, C, H, W] + noise.flatten(0, 1), # [B*T, C, H, W] + timesteps.flatten(0, 1) # [B*T] + ).unflatten(0, (batch_size, num_latent_t)) # [B, T, C, H, W] + + # Convert back to [B, C, T, H, W] for transformer input + noisy_model_input = noisy_latents.permute(0, 2, 1, 3, 4) + + # I2V concatenation: [mask(1ch), image_latent(16ch)] → 17+16=33 ch total + assert isinstance(training_batch.image_latents, torch.Tensor) + image_latents = training_batch.image_latents.to( + get_local_torch_device(), dtype=torch.bfloat16) + + temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (num_latent_t - 1) * temporal_compression_ratio + 1 + _, num_channels, _, latent_height, latent_width = image_latents.shape + mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, + latent_width) + mask_lat_size[:, :, 1:] = 0 + + first_frame_mask = mask_lat_size[:, :, :1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, dim=2, repeats=temporal_compression_ratio) + mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], + dim=2) + mask_lat_size = mask_lat_size.view(batch_size, -1, + temporal_compression_ratio, + latent_height, latent_width) + mask_lat_size = mask_lat_size.transpose(1, 2) + mask_lat_size = mask_lat_size.to( + image_latents.device).to(dtype=torch.bfloat16) + + noisy_model_input = torch.cat( + [noisy_model_input, mask_lat_size, image_latents], dim=1) + + # Compute flow-matching training target: target = noise - clean + # Shape: [B, T, C, H, W] + training_target = self.ar_noise_scheduler.training_target( + latents_btchw.flatten(0, 1), + noise.flatten(0, 1), + timesteps.flatten(0, 1) + ).unflatten(0, (batch_size, num_latent_t)) + + # Store everything on training_batch + training_batch.noisy_model_input = noisy_model_input + training_batch.timesteps = timesteps # [B, T] per-frame timesteps + training_batch.noise = noise.permute(0, 2, 1, 3, 4) # [B, C, T, H, W] + training_batch.raw_latent_shape = latents.shape + # Store extra data for the custom loss function + training_batch._ar_training_target = training_target # [B, T, C, H, W] + + return training_batch + + def _build_input_kwargs(self, + training_batch: TrainingBatch) -> TrainingBatch: + """Build transformer kwargs with action conditioning and per-frame timesteps.""" + # Image Embeds for conditioning + image_embeds = training_batch.image_embeds + assert torch.isnan(image_embeds).sum() == 0 + image_embeds = image_embeds.to(get_local_torch_device(), + dtype=torch.bfloat16) + + # Process actions for each batch sample + batch_size = training_batch.noisy_model_input.shape[0] + keyboard_cond = training_batch.keyboard_cond + mouse_cond = training_batch.mouse_cond + + if keyboard_cond is not None and mouse_cond is not None: + viewmats_list, intrinsics_list, action_labels_list = [], [], [] + for b in range(batch_size): + v, i, a = process_custom_actions(keyboard_cond[b], + mouse_cond[b]) + viewmats_list.append(v) + intrinsics_list.append(i) + action_labels_list.append(a) + viewmats = torch.stack(viewmats_list, + dim=0).to(get_local_torch_device(), + dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, + dim=0).to(get_local_torch_device(), + dtype=torch.bfloat16) + action_labels = torch.stack(action_labels_list, + dim=0).to(get_local_torch_device(), + dtype=torch.bfloat16) + else: + viewmats = None + intrinsics = None + action_labels = None + + # Per-frame timesteps: [B, T] + timesteps = training_batch.timesteps + assert timesteps.ndim == 2, ( + f"Expected per-frame timesteps [B, T], got shape {timesteps.shape}") + + training_batch.input_kwargs = { + "hidden_states": training_batch.noisy_model_input, + "encoder_hidden_states": None, # No text conditioning for WanGame + "timestep": timesteps.to(get_local_torch_device(), + dtype=torch.bfloat16), + "encoder_hidden_states_image": image_embeds, + "viewmats": viewmats, + "Ks": intrinsics, + "action": action_labels, + "num_frame_per_block": self.num_frame_per_block, + "return_dict": False, + } + return training_batch + + def _transformer_forward_and_compute_loss( + self, training_batch: TrainingBatch) -> TrainingBatch: + """ + Run transformer forward pass and compute flow-matching loss. + """ + input_kwargs = training_batch.input_kwargs + + # Forward with causal attention via set_forward_context + with set_forward_context(current_timestep=training_batch.timesteps, + attn_metadata=None, + forward_batch=None): + # model_pred: [B, C, T, H, W] (flow prediction) + model_pred = self.transformer(**input_kwargs) + + # model_pred is [B, C, T, H, W], convert to [B, T, C, H, W] + model_pred_btchw = model_pred.permute(0, 2, 1, 3, 4) + + # Training target: [B, T, C, H, W] + training_target = training_batch._ar_training_target.to( + model_pred_btchw.device, dtype=model_pred_btchw.dtype) + + batch_size, num_frame = model_pred_btchw.shape[:2] + + # Per-frame MSE loss with training weight + # loss shape before weight: [B, T] + loss = F.mse_loss( + model_pred_btchw.float(), + training_target.float(), + reduction='none' + ).mean(dim=(2, 3, 4)) # Average over C, H, W → [B, T] + + # Apply per-timestep training weight from scheduler + timesteps = training_batch.timesteps # [B, T] + weights = self.ar_noise_scheduler.training_weight( + timesteps.flatten(0, 1) + ).unflatten(0, (batch_size, num_frame)) + loss = (loss * weights).mean() + + loss = loss / self.training_args.gradient_accumulation_steps + loss.backward() + + avg_loss = loss.detach().clone() + training_batch.total_loss += avg_loss.item() + + return training_batch + + def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: + """Override to use custom AR diffusion training logic.""" + self.transformer.train() + self.optimizer.zero_grad() + training_batch.total_loss = 0.0 + args = cast(TrainingArgs, self.training_args) + + for _ in range(args.gradient_accumulation_steps): + training_batch = self._get_next_batch(training_batch) + + # Prepare noisy inputs with per-block timesteps + I2V concat + training_batch = self._prepare_dit_inputs(training_batch) + + # Build transformer input kwargs (action conditioning etc.) + training_batch = self._build_input_kwargs(training_batch) + + # Forward + loss + training_batch = self._transformer_forward_and_compute_loss( + training_batch) + + # Clip grad and step + grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in self.transformer.parameters() if p.requires_grad], + args.max_grad_norm if args.max_grad_norm is not None else 0.0) + + self.optimizer.step() + self.lr_scheduler.step() + + if grad_norm is None: + grad_value = 0.0 + else: + try: + if isinstance(grad_norm, torch.Tensor): + grad_value = float(grad_norm.detach().float().item()) + else: + grad_value = float(grad_norm) + except Exception: + grad_value = 0.0 + training_batch.grad_norm = grad_value + training_batch.raw_latent_shape = training_batch.latents.shape + return training_batch + + def _prepare_validation_batch(self, sampling_param: SamplingParam, + training_args: TrainingArgs, + validation_batch: dict[str, Any], + num_inference_steps: int) -> ForwardBatch: + sampling_param.prompt = validation_batch['prompt'] + sampling_param.height = training_args.num_height + sampling_param.width = training_args.num_width + sampling_param.image_path = validation_batch.get( + 'image_path') or validation_batch.get('video_path') + sampling_param.num_inference_steps = num_inference_steps + sampling_param.data_type = "video" + assert self.seed is not None + sampling_param.seed = self.seed + + latents_size = [(sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, sampling_param.width // 8] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + num_frames = (training_args.num_latent_t - + 1) * temporal_compression_factor + 1 + sampling_param.num_frames = num_frames + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=torch.Generator(device="cpu").manual_seed(self.seed), + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=training_args.VSA_sparsity, + ) + if "image" in validation_batch and validation_batch["image"] is not None: + batch.pil_image = validation_batch["image"] + + if "keyboard_cond" in validation_batch and validation_batch[ + "keyboard_cond"] is not None: + keyboard_cond = validation_batch["keyboard_cond"] + keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) + keyboard_cond = keyboard_cond.unsqueeze(0) + batch.keyboard_cond = keyboard_cond + + if "mouse_cond" in validation_batch and validation_batch[ + "mouse_cond"] is not None: + mouse_cond = validation_batch["mouse_cond"] + mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) + mouse_cond = mouse_cond.unsqueeze(0) + batch.mouse_cond = mouse_cond + + return batch + + def _post_process_validation_frames( + self, frames: list[np.ndarray], + batch: ForwardBatch) -> list[np.ndarray]: + """Apply action overlay to validation frames.""" + keyboard_cond = getattr(batch, 'keyboard_cond', None) + mouse_cond = getattr(batch, 'mouse_cond', None) + + if keyboard_cond is None and mouse_cond is None: + return frames + + from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, + draw_mouse_on_frame) + + if keyboard_cond is not None: + keyboard_cond = keyboard_cond.squeeze( + 0).cpu().float().numpy() + if mouse_cond is not None: + mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() + + key_names = ["W", "S", "A", "D", "left", "right"] + + processed_frames = [] + for frame_idx, frame in enumerate(frames): + frame = np.ascontiguousarray(frame.copy()) + + if keyboard_cond is not None and frame_idx < len(keyboard_cond): + keys = { + key_names[i]: bool(keyboard_cond[frame_idx, i]) + for i in range(min(len(key_names), keyboard_cond.shape[1])) + } + draw_keys_on_frame(frame, keys, mode='universal') + + if mouse_cond is not None and frame_idx < len(mouse_cond): + pitch = float(mouse_cond[frame_idx, 0]) + yaw = float(mouse_cond[frame_idx, 1]) + draw_mouse_on_frame(frame, pitch, yaw) + + processed_frames.append(frame) + + return processed_frames + + +def main(args) -> None: + logger.info("Starting WanGame AR diffusion training pipeline...") + + pipeline = WanGameARDiffusionPipeline.from_pretrained( + args.pretrained_model_name_or_path, args=args) + args = pipeline.training_args + pipeline.train() + logger.info("WanGame AR diffusion training pipeline done") + + +if __name__ == "__main__": + argv = sys.argv + from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.utils import FlexibleArgumentParser + parser = FlexibleArgumentParser() + parser = TrainingArgs.add_cli_args(parser) + parser = FastVideoArgs.add_cli_args(parser) + args = parser.parse_args() + args.dit_cpu_offload = False + main(args) From dbe17e246ea3489f84f28449d50598aaf4a1ea74 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 19:47:30 +0000 Subject: [PATCH 114/214] no npy --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 657c99fb8..56e5a2d0d 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ env **.pyc **.txt *.log +*.npy weights/ # SSIM test outputs From ceb4de0a966cdc69877bebe273f52f3cbbd773d6 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 20:50:03 +0000 Subject: [PATCH 115/214] designing wangame import --- dev/phase_add_wangame.md | 225 ++++++++++++++++++ ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml | 2 +- 2 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 dev/phase_add_wangame.md diff --git a/dev/phase_add_wangame.md b/dev/phase_add_wangame.md new file mode 100644 index 000000000..cfbfcbe37 --- /dev/null +++ b/dev/phase_add_wangame.md @@ -0,0 +1,225 @@ +# Phase：Add WanGame（把 Wangame 接入新 distillation 框架） + +> 目标：在 **不回到 legacy training pipeline** 的前提下，让 `fastvideo/training/distillation.py` +> 可以通过 YAML（schema v2）跑起 **WanGame** 的 finetune / distill（优先 finetune）。 +> +> 本文件只做“代码层面规划”，不修改代码。 + +--- + +## 0) 我们现在在 FastVideo 里哪里能看到 WanGame？ + +WanGame 不是“想象中的模型”，FastVideo 里已经有一整套（legacy）实现： + +- **模型（DiT）**：`fastvideo/models/dits/wangame/*` + - 关键类：`WanGameActionTransformer3DModel` +- **pipeline config**：`fastvideo/configs/models/dits/wangamevideo.py` + + `fastvideo/configs/pipelines/wan.py:WanGameI2V480PConfig` +- **推理 pipeline（ODE）**：`fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py` + - `WanGameActionImageToVideoPipeline` +- **推理 pipeline（SDE/DMD）**：`fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py` + - `WanGameCausalDMDPipeline` +- **训练 pipeline（legacy）**：`fastvideo/training/wangame_training_pipeline.py` + - 重点参考：`_prepare_dit_inputs()` + `_build_input_kwargs()` +- **数据 schema**：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_wangame` + +结论：要把 WanGame 接到新框架，核心工作是把 legacy pipeline 里 +“raw batch -> model forward primitives”的逻辑迁入 **adapter**。 + +--- + +## 1) 现在我们的新框架需要什么（接口映射） + +新框架（Phase 3.x）的一条 run path 是： + +- `fastvideo/training/distillation.py` + -> parse YAML (`fastvideo/distillation/utils/config.py`) + -> `fastvideo/distillation/dispatch.py:build_runtime_from_config()` + -> `fastvideo/distillation/trainer.py:DistillTrainer.run(method, dataloader, ...)` + +其中 **model plugin** 必须返回： + +- `RoleManager`（roles -> RoleHandle -> modules/optimizers/schedulers） +- `Adapter`（operation-centric primitives） +- `DataLoader` +- `Validator`（可选，method 决定怎么调用） + +因此“加 Wangame”就是补齐一个新的 model plugin + adapter + validator。 + +--- + +## 2) 设计目标（Add WanGame 的 Definition of Done） + +### ✅ 最小可用（建议先交付这一档） + +- 支持 `recipe.family: wangame` +- 支持 `recipe.method: finetune`（优先） +- `log_validation: true` 时能正确走 `WanGameActionImageToVideoPipeline`（ODE） +- 训练 input 与 legacy `fastvideo/training/wangame_training_pipeline.py` 一致： + - I2V concat：`noisy_video_latents + mask + image_latents` + - action conditioning：`viewmats / Ks / action` 来自 `process_custom_actions(...)` + +### ✅ 进阶（第二档） + +- 支持 `recipe.method: dmd2`（distill） +- validation 根据 `ValidationRequest.sampler_kind` 切换： + - `ode` -> `WanGameActionImageToVideoPipeline` + - `sde` -> `WanGameCausalDMDPipeline` + +> 注意：WanGame 的训练通常不依赖 text CFG；DMD2 中的 uncond/cond +> 在 wangame 上可能等价（见风险点）。 + +--- + +## 3) 文件改动规划（建议的最小集合） + +### 3.1 新增：model plugin + +- [ ] `fastvideo/distillation/models/wangame.py` + - `@register_model("wangame")` + - 主要职责： + - 设置 `training_args.override_transformer_cls_name = "WanGameActionTransformer3DModel"` + （必要时增加可配置项，支持 causal transformer） + - 加载 shared：`vae`（用于 `normalize_dit_input("wan", ...)`）+ `noise_scheduler` + - 为每个 role 加载 `transformer`（以及可选 `transformer_2`） + - `apply_trainable(...)` + activation ckpt（沿用 wan plugin 逻辑） + - 构建 `RoleManager` + - 构建 `WanGameAdapter` + - dataloader：使用 `pyarrow_schema_wangame` + - `log_validation` 时构建 `WanGameValidator` + +### 3.2 新增：adapter + +- [ ] `fastvideo/distillation/adapters/wangame.py` + - 复用 `WanAdapter` 的通用 mechanics（timestep/noise/attn_metadata/backward 的模式） + - 重点实现（对齐 legacy `wangame_training_pipeline.py`）： + - `prepare_batch(raw_batch, current_vsa_sparsity, latents_source=...)` + - 从 parquet batch 取： + - `vae_latent`（video x0） + - `clip_feature`（image_embeds） + - `first_frame_latent`（image_latents） + - `keyboard_cond` / `mouse_cond` + - `pil_image`（给 validation 或 debug） + - 计算 timesteps/noise/sigmas，得到 noisy_video_latents + - 构造 I2V 输入： + - 生成 `mask_lat_size`（与 legacy 一致） + - `noisy_model_input = cat([noisy_video_latents, mask, image_latents], dim=1)` + - `process_custom_actions(...)` -> `viewmats / Ks / action` + - 组装 transformer forward 所需的 `input_kwargs` + - `predict_noise(handle, noisy_latents, t, batch, conditional, attn_kind)` + - `predict_x0(handle, noisy_latents, t, batch, conditional, attn_kind)` + - **关键约束**：`noisy_latents` 参数应代表“noisy video latents”； + adapter 内部用 batch 里的 `image_latents/mask/action` 拼出完整 hidden_states。 + - `add_noise(clean_latents, noise, t)`：只对 video latent 做 add_noise + - `backward(loss, ctx, ...)`：延续现有 forward_context 机制 + +> 备注：`conditional` 在 wangame 上可能不区分（先忽略即可），但接口必须收敛一致， +> 以兼容 `DMD2Method` 的调用方式。 + +### 3.3 新增：validator + +- [ ] `fastvideo/distillation/validators/wangame.py` + - API 对齐 `WanValidator`：`log_validation(step, request=ValidationRequest)` + - pipeline 选择： + - `request.sampler_kind == "ode"`：`WanGameActionImageToVideoPipeline` + - `request.sampler_kind == "sde"`：`WanGameCausalDMDPipeline` + - batch 构造参考 legacy： + - `fastvideo/training/wangame_training_pipeline.py:_prepare_validation_batch` + - 只要 method 提供 `sample_handle`（通常 student），validator 就能跑。 + +### 3.4 改动：dispatch builtin registrations + +- [ ] `fastvideo/distillation/dispatch.py:ensure_builtin_registrations()` + - 显式 import 新增的 `fastvideo.distillation.models.wangame` + +### 3.5 可选：dataloader util + +当前 `fastvideo/distillation/utils/dataloader.py` 只有 T2V helper。 +wangame 需要 I2V+action schema，因此建议： + +- [ ] `fastvideo/distillation/utils/dataloader.py` + - 新增 `build_parquet_wangame_train_dataloader(training_args, parquet_schema=pyarrow_schema_wangame)` + - 内部仍调用 `fastvideo.dataset.build_parquet_map_style_dataloader(...)` + +--- + +## 4) 配置（YAML）规划（schema v2） + +建议新增一个最小 finetune 配置（示意）： + +```yaml +recipe: + family: wangame + method: finetune + +roles: + student: + family: wangame + path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + trainable: true + +training: + data_path: /abs/path/to/wangame/preprocessed/combined_parquet_dataset + # shape / dist / lr / validation 同 wan 的 schema 写法即可 + +pipeline_config: + flow_shift: 5 + sampler_kind: ode + +method_config: + attn_kind: dense +``` + +如果要支持 DMD2（第二档），需要： +- roles 增加 teacher/critic（如果暂时没有更大 teacher，可先 teacher==student 跑通 e2e） +- `pipeline_config.sampler_kind: sde`（validation 走 `WanGameCausalDMDPipeline`） +- `method_config` 增加 DMD2 必需字段（`rollout_mode`, `dmd_denoising_steps`, ...） + +--- + +## 5) 关于目录结构：要不要 `models/wan/wan.py + models/wan/wangame.py`？ + +**不需要强制做**。 + +当前 dispatch 只关心注册 key（`@register_model("...")`），文件放哪都行。 +因此建议先保持扁平： + +- `fastvideo/distillation/models/wan.py` +- `fastvideo/distillation/models/wangame.py` + +等到 wan 系变体更多时（wan_t2v / wan_i2v / wangame / lingbot / ...）， +再做结构化重排成 `models/wan/*`，并把共同的 loader 逻辑抽成内部 helper。 + +--- + +## 6) 风险点 / 决策点（需要提前讲清楚） + +1) **DMD2 的 CFG（cond/uncond）语义在 wangame 上可能不存在** + - wangame training pipeline 里 `encoder_hidden_states=None`，主要 conditioning 是 image+action。 + - 我们可以先让 `conditional` flag 在 adapter 里等价处理： + - `conditional=True/False` 都走同一条分支 + - DMD2 的 `real_cond_x0 - real_uncond_x0` 近似为 0，guidance scale 失效但训练可跑通 + - 若未来确实需要“uncond”，需要定义： + - uncond 是否代表“去掉 action”？“去掉 image cond”？还是别的？ + +2) **train_action_only / action_warmup_steps（细粒度 trainable）** + - legacy `wangame_training_pipeline.py` 支持只训练 action 路径参数（pattern-based） + - 目前新框架 roles 只有 `trainable: bool`（整模块开关），不足以表达这一点 + - 建议先把这一点作为可选增强： + - 最小实现：先不做（或要求用户在 wangame 上 train 全模型） + - 完整实现：在 `models/wangame.py` 内做参数 pattern 的 requires_grad 筛选 + +3) **validation pipeline 的 ODE/SDE 差异** + - 需要 validator 根据 `sampler_kind` 切 pipeline，避免出现“看起来像 ODE”但期望 SDE 的错觉。 + +--- + +## 7) 验收（我们如何确认接入是对的） + +- finetune： + - e2e 跑通（2~10 steps）+ 能输出 validation 视频 + - 与 legacy finetune 的 step0/stepN 视觉趋势大体一致（不追求完全 bitwise） +- distill（若做第二档）： + - e2e 跑通（few-step） + - validation 选择 `sde` 时，视觉应接近 legacy DMD pipeline 的 sampling 形态 + diff --git a/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml b/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml index c84ede635..f7f812c9c 100644 --- a/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml +++ b/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml @@ -65,7 +65,7 @@ training: # Tracking / validation tracker_project_name: distillation_phase3 - wandb_run_name: phase3.4_wan_dmd2_8steps_simulate + wandb_run_name: phase3.4_wan_dmd2_8steps_simulate_aftermergewangame log_validation: true validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json validation_steps: 50 From de3b8b36198a4fcebc973f00d31f5098648780e5 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 21:25:25 +0000 Subject: [PATCH 116/214] designing wangame: cfg --- dev/phase_add_wangame.md | 102 ++++++++++++++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 13 deletions(-) diff --git a/dev/phase_add_wangame.md b/dev/phase_add_wangame.md index cfbfcbe37..84d7519f6 100644 --- a/dev/phase_add_wangame.md +++ b/dev/phase_add_wangame.md @@ -170,6 +170,14 @@ method_config: attn_kind: dense ``` +### 4.1 roles 的“自由扩展字段”策略（保留核心字段强校验） + +建议 roles 仍强制解析这三个核心字段（保证错误信息清晰）： +- `family / path / trainable` + +但允许 roles 下出现其它任意 key，并把它们原样保留（例如 `RoleSpec.extra`）， +由 `models/wangame.py` 自行解释（例如 action-only train patterns / cls_name / init 行为）。 + 如果要支持 DMD2（第二档），需要： - roles 增加 teacher/critic（如果暂时没有更大 teacher，可先 teacher==student 跑通 e2e） - `pipeline_config.sampler_kind: sde`（validation 走 `WanGameCausalDMDPipeline`） @@ -195,22 +203,91 @@ method_config: ## 6) 风险点 / 决策点（需要提前讲清楚） 1) **DMD2 的 CFG（cond/uncond）语义在 wangame 上可能不存在** - - wangame training pipeline 里 `encoder_hidden_states=None`，主要 conditioning 是 image+action。 - - 我们可以先让 `conditional` flag 在 adapter 里等价处理： - - `conditional=True/False` 都走同一条分支 - - DMD2 的 `real_cond_x0 - real_uncond_x0` 近似为 0，guidance scale 失效但训练可跑通 - - 若未来确实需要“uncond”，需要定义： - - uncond 是否代表“去掉 action”？“去掉 image cond”？还是别的？ + 先说明：在我们新框架里，`DMD2Method` 的 teacher CFG 语义来自 “文本 CFG”： + - `conditional=True`：用 prompt（正向）作为 conditioning； + - `conditional=False`：用 negative / uncond prompt 作为 conditioning； + - 最终 teacher 的 “real score” 用类似： + `real = uncond + guidance_scale * (cond - uncond)`。 + + 这在 Wan(T2V) 上成立，因为： + - legacy / 我们的 `WanAdapter` 都显式构造了 negative prompt embeds（`ensure_negative_conditioning()`）， + 并把它塞进 `training_batch.unconditional_dict`，从而 `conditional` flag 有真实语义差异。 + + 但在 **WanGame** 上，legacy 训练/推理几乎完全不依赖 text： + - legacy train：`fastvideo/training/wangame_training_pipeline.py:_build_input_kwargs()` + 明确设置： + - `"encoder_hidden_states": training_batch.encoder_hidden_states # None (no text conditioning)` + - `"encoder_hidden_states_image": image_embeds`（I2V conditioning） + - `"viewmats" / "Ks" / "action"`（action conditioning） + - legacy inference pipeline：`fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py` + 没有 `TextEncodingStage`；因此 `ForwardBatch.prompt_embeds` 为空。 + `LatentPreparationStage` 会把空 prompt embeds 变成一个 dummy（形状 `[B, 0, hidden_size]`），并且： + - `batch.do_classifier_free_guidance = False` + （见 `fastvideo/pipelines/stages/latent_preparation.py`） + - legacy scripts/示例也在暗示“没 CFG”： + - `examples/inference/basic/basic_wangame.py` 里 `guidance_scale=1.0` + - `examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm` + 里 `validation_guidance_scale "1.0"` + + 因此：如果我们直接把 DMD2 的 `conditional/unconditional` 套到 wangame 上， + 在 **不重新定义 uncond 的前提下**，它们很可能等价（cond == uncond），导致： + - `real_cond_x0 - real_uncond_x0 ≈ 0` + - guidance_scale 形式上存在，但语义上失效 + - 训练可能仍能跑通，但已经不是“文本 CFG 意义下的 DMD2” + + **建议的落地策略（按风险从低到高）：** + - A（最小可用，推荐先做）：在 `wangame` 的 adapter 里把 `conditional` 当作 no-op + （cond/uncond 同路），并在文档里明确 “wangame 暂不支持文本 CFG”。 + - B（定义 wangame 的 uncond 语义）：由 `method_config` 显式声明 uncond 的定义，例如： + - `uncond_mode: none|zero_action|zero_image|zero_both` + - `zero_action`：把 action/viewmats/Ks 置零或置 None（需要确认 transformer 对 None 的容忍度） + - `zero_image`：把 `encoder_hidden_states_image` 置零（保持 shape） + 这样 `conditional` 才有可解释的差异。 + 归属建议： + - 放在 `method_config`：因为它只在 “需要 CFG 的算法” 中有意义；finetune 等方法不应被迫理解 uncond 语义。 + - 但执行必然落在 adapter（如何 zero_action/zero_image 是模型相关的），因此 adapter 需要提供一个 operation + 来承接该语义（例如“构造 uncond conditioning variant”）。 2) **train_action_only / action_warmup_steps（细粒度 trainable）** - - legacy `wangame_training_pipeline.py` 支持只训练 action 路径参数（pattern-based） - - 目前新框架 roles 只有 `trainable: bool`（整模块开关），不足以表达这一点 - - 建议先把这一点作为可选增强： - - 最小实现：先不做（或要求用户在 wangame 上 train 全模型） - - 完整实现：在 `models/wangame.py` 内做参数 pattern 的 requires_grad 筛选 + legacy `wangame_training_pipeline.py` 支持更细粒度的训练策略： + - `train_action_only`：冻结 base DiT，只训练 action 相关参数（pattern-based） + - `action_warmup_steps`：前 N 步把 action 参数 `requires_grad=False`，之后再打开 + + 重要补充：这两个 knob 在 FastVideo 的 `TrainingArgs` 里已经存在（`fastvideo/fastvideo_args.py`）， + 也就是说 **YAML 的 `training:` 段本身就能表达**： + - `training.train_action_only: true` + - `training.action_train_target: both|action_mlp|prope` + - `training.action_warmup_steps: 1000` + + 我们新框架目前的问题不是 “config 表达不了”，而是： + - roles 只有 `trainable: bool`（整模块开关），无法表达“同一个 transformer 内只训练某些 param” + - warmup 属于 step-time policy，应该由 method（或 method_config）驱动，而不是 loader 一次性决定 + + **关于你提的方案：roles 允许自由 dict（而非完全结构化）** + - 优点：wangame 这类 family 很容易出现 role-specific knobs（比如 per-role train patterns / cls_name / init 行为）， + 不需要每加一个字段就改全局 schema。 + - 缺点：弱化静态校验，typo 会变成 runtime 才爆；文档/IDE 提示也会更差。 + + **折中建议（更可控）：** + - roles 仍解析核心字段（`family/path/trainable`），但把未知字段原样保留到 `RoleSpec.extra`（或 `role_cfg_raw`）， + 由 `models/wangame.py` 自己读取解释。 + - 这样既能满足“roles 自由 dict 扩展”，也不牺牲核心字段的错误提示质量。 3) **validation pipeline 的 ODE/SDE 差异** - - 需要 validator 根据 `sampler_kind` 切 pipeline，避免出现“看起来像 ODE”但期望 SDE 的错觉。 + 现状：FastVideo 已经把 wangame 分成了两个 inference pipeline： + - ODE：`WanGameActionImageToVideoPipeline`（`fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py`） + - SDE/DMD：`WanGameCausalDMDPipeline`（`fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py`） + + 对新框架而言，最小可用的做法是： + - `validators/wangame.py` 根据 `ValidationRequest.sampler_kind` 选择 pipeline class + + 你提的“一个 pipeline 支持多 sampler（ode/sde）”是好方向（我们对 Wan 已经这么做过）： + - 优点：减少 pipeline 分叉/重复逻辑，validation 不容易“走错范式” + - 代价：需要对 wangame pipeline 做更侵入式的重构（引入 `pipelines/samplers/`，并把 denoising loop 抽出来） + + **建议顺序：** + - 接入阶段先做 validator 选 pipeline（改动小、风险低） + - 稳定后再把 wangame pipeline 也升级为 `sampler_kind` 可切换（更优雅，但属于额外工程） --- @@ -222,4 +299,3 @@ method_config: - distill（若做第二档）： - e2e 跑通（few-step） - validation 选择 `sde` 时，视觉应接近 legacy DMD pipeline 的 sampling 形态 - From db37b643f6a69bf5513b9eb82e9df37ee2d1eb6b Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 22:06:10 +0000 Subject: [PATCH 117/214] wangame support distillation --- .../distillation/wangame/finetune-temp.sh | 42 ++ .../wangame/finetune_wangame2.1_i2v_1.3B.yaml | 80 +++ fastvideo/distillation/adapters/wangame.py | 509 ++++++++++++++++++ fastvideo/distillation/dispatch.py | 1 + .../distillation/doc/adapters/wangame.md | 41 ++ fastvideo/distillation/doc/models/wangame.md | 28 + .../distillation/doc/utils/dataloader.md | 3 + .../distillation/doc/validators/wangame.md | 23 + fastvideo/distillation/models/wangame.py | 122 +++++ fastvideo/distillation/utils/dataloader.py | 25 + fastvideo/distillation/validators/wangame.py | 327 +++++++++++ 11 files changed, 1201 insertions(+) create mode 100644 examples/distillation/wangame/finetune-temp.sh create mode 100644 examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml create mode 100644 fastvideo/distillation/adapters/wangame.py create mode 100644 fastvideo/distillation/doc/adapters/wangame.md create mode 100644 fastvideo/distillation/doc/models/wangame.md create mode 100644 fastvideo/distillation/doc/validators/wangame.md create mode 100644 fastvideo/distillation/models/wangame.py create mode 100644 fastvideo/distillation/validators/wangame.py diff --git a/examples/distillation/wangame/finetune-temp.sh b/examples/distillation/wangame/finetune-temp.sh new file mode 100644 index 000000000..ab786423e --- /dev/null +++ b/examples/distillation/wangame/finetune-temp.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -euo pipefail +if [[ "${DEBUG:-0}" == "1" ]]; then + set -x +fi + +# WanGame finetune launcher (new distillation framework). +# +# - YAML config lives next to this script. +# - Run is driven by `fastvideo/training/distillation.py --config `. + +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-offline} +export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} +export MASTER_PORT=${MASTER_PORT:-29515} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +CONFIG=${CONFIG:-"examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") + +torchrun \ + --nnodes 1 \ + --nproc_per_node "$NUM_GPUS" \ + --master_addr "$MASTER_ADDR" \ + --master_port "$MASTER_PORT" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" + diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml new file mode 100644 index 000000000..dd8f3cbe1 --- /dev/null +++ b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml @@ -0,0 +1,80 @@ +recipe: + family: wangame + method: finetune + +roles: + student: + family: wangame + path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + trainable: true + +training: + # Distributed (mirror legacy WanGame finetune scripts) + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder) + # + # Supports comma-separated `path` or `path:N` (repeat count) entries. + # N=0 means skip (handy to toggle datasets without deleting lines). + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_finetune + max_train_steps: 20000 + seed: 1000 + checkpoints_total_limit: 2 + + # Optimizer + learning_rate: 1.0e-5 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 1.0e-4 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.0 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_finetune + log_validation: true + validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json + validation_steps: 100 + validation_sampling_steps: "40" + validation_guidance_scale: "1.0" + +pipeline_config: + flow_shift: 3 + sampler_kind: ode + +method_config: + attn_kind: dense diff --git a/fastvideo/distillation/adapters/wangame.py b/fastvideo/distillation/adapters/wangame.py new file mode 100644 index 000000000..3c0212919 --- /dev/null +++ b/fastvideo/distillation/adapters/wangame.py @@ -0,0 +1,509 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import copy +from typing import Any, Literal + +import torch + +import fastvideo.envs as envs +from fastvideo.distributed import ( + get_local_torch_device, + get_sp_group, + get_world_group, +) +from fastvideo.forward_context import set_forward_context +from fastvideo.models.utils import pred_noise_to_pred_video +from fastvideo.pipelines import TrainingBatch +from fastvideo.training.training_utils import ( + compute_density_for_timestep_sampling, + get_sigmas, + normalize_dit_input, + shift_timestep, +) +from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed + +from fastvideo.distillation.adapters.base import DistillAdapter +from fastvideo.distillation.roles import RoleHandle + +try: + from fastvideo.attention.backends.video_sparse_attn import ( + VideoSparseAttentionMetadataBuilder, + ) + from fastvideo.attention.backends.vmoba import VideoMobaAttentionMetadataBuilder +except Exception: + VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] + VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] + + +class WanGameAdapter(DistillAdapter): + """WanGame adapter: exposes operation-centric primitives for methods. + + This adapter is intentionally *not* method-specific: + - It knows how to turn a wangame parquet batch into forward primitives. + - It knows how to run a model role (handle) for predict_noise / predict_x0. + - It does not encode DMD2/SFT/etc semantics beyond required primitives. + """ + + def __init__( + self, + *, + training_args: Any, + noise_scheduler: Any, + vae: Any, + ) -> None: + self.training_args = training_args + self.noise_scheduler = noise_scheduler + self.vae = vae + + self.world_group = get_world_group() + self.sp_group = get_sp_group() + self.device = get_local_torch_device() + + self.noise_random_generator: torch.Generator | None = None + self.noise_gen_cuda: torch.Generator | None = None + + self._init_timestep_mechanics() + + def _get_training_dtype(self) -> torch.dtype: + return torch.bfloat16 + + def _init_timestep_mechanics(self) -> None: + self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) + self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) + self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) + self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) + + boundary_ratio = getattr(self.training_args, "boundary_ratio", None) + self.boundary_timestep: float | None = ( + float(boundary_ratio) * float(self.num_train_timestep) + if boundary_ratio is not None + else None + ) + + @property + def num_train_timesteps(self) -> int: + return int(self.num_train_timestep) + + def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: + timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) + return timestep.clamp(self.min_timestep, self.max_timestep) + + def on_train_start(self) -> None: + seed = getattr(self.training_args, "seed", None) + if seed is None: + raise ValueError("training_args.seed must be set for distillation") + + global_rank = int(getattr(self.world_group, "rank", 0)) + sp_world_size = int(getattr(self.training_args, "sp_size", 1) or 1) + if sp_world_size > 1: + sp_group_seed = int(seed) + (global_rank // sp_world_size) + set_random_seed(sp_group_seed) + else: + set_random_seed(int(seed) + global_rank) + + self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) + self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) + + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + generators: dict[str, torch.Generator] = {} + if self.noise_random_generator is not None: + generators["noise_cpu"] = self.noise_random_generator + if self.noise_gen_cuda is not None: + generators["noise_cuda"] = self.noise_gen_cuda + return generators + + def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: + if self.noise_random_generator is None: + raise RuntimeError("WanGameAdapter.on_train_start() must be called before prepare_batch()") + + u = compute_density_for_timestep_sampling( + weighting_scheme=self.training_args.weighting_scheme, + batch_size=batch_size, + generator=self.noise_random_generator, + logit_mean=self.training_args.logit_mean, + logit_std=self.training_args.logit_std, + mode_scale=self.training_args.mode_scale, + ) + indices = (u * self.noise_scheduler.config.num_train_timesteps).long() + return self.noise_scheduler.timesteps[indices].to(device=device) + + def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: + latents_shape = training_batch.raw_latent_shape + patch_size = self.training_args.pipeline_config.dit_config.patch_size + current_vsa_sparsity = training_batch.current_vsa_sparsity + assert latents_shape is not None + assert training_batch.timesteps is not None + + if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": + if not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None: + raise ImportError( + "FASTVIDEO_ATTENTION_BACKEND is VIDEO_SPARSE_ATTN, but fastvideo_kernel " + "is not correctly installed or detected." + ) + training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] + raw_latent_shape=latents_shape[2:5], + current_timestep=training_batch.timesteps, + patch_size=patch_size, + VSA_sparsity=current_vsa_sparsity, + device=self.device, + ) + elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": + if not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None: + raise ImportError( + "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, but fastvideo_kernel " + "(or flash_attn>=2.7.4) is not correctly installed." + ) + moba_params = self.training_args.moba_config.copy() + moba_params.update( + { + "current_timestep": training_batch.timesteps, + "raw_latent_shape": training_batch.raw_latent_shape[2:5], + "patch_size": patch_size, + "device": self.device, + } + ) + training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(**moba_params) # type: ignore[misc] + else: + training_batch.attn_metadata = None + + return training_batch + + def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: + latents = training_batch.latents + assert isinstance(latents, torch.Tensor) + batch_size = latents.shape[0] + + if self.noise_gen_cuda is None: + raise RuntimeError("WanGameAdapter.on_train_start() must be called before prepare_batch()") + + noise = torch.randn( + latents.shape, + generator=self.noise_gen_cuda, + device=latents.device, + dtype=latents.dtype, + ) + timesteps = self._sample_timesteps(batch_size, latents.device) + if int(getattr(self.training_args, "sp_size", 1) or 1) > 1: + self.sp_group.broadcast(timesteps, src=0) + + sigmas = get_sigmas( + self.noise_scheduler, + latents.device, + timesteps, + n_dim=latents.ndim, + dtype=latents.dtype, + ) + noisy_model_input = (1.0 - sigmas) * latents + sigmas * noise + + training_batch.noisy_model_input = noisy_model_input + training_batch.timesteps = timesteps + training_batch.sigmas = sigmas + training_batch.noise = noise + training_batch.raw_latent_shape = latents.shape + + training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) + return training_batch + + def _build_i2v_mask_latents(self, image_latents: torch.Tensor) -> torch.Tensor: + temporal_compression_ratio = ( + self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 + + batch_size, _num_channels, _t, latent_height, latent_width = image_latents.shape + mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width) + mask_lat_size[:, :, 1:] = 0 + + first_frame_mask = mask_lat_size[:, :, :1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, + dim=2, + repeats=temporal_compression_ratio, + ) + mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], dim=2) + mask_lat_size = mask_lat_size.view( + batch_size, + -1, + temporal_compression_ratio, + latent_height, + latent_width, + ) + mask_lat_size = mask_lat_size.transpose(1, 2) + return mask_lat_size.to(device=image_latents.device, dtype=image_latents.dtype) + + def _process_actions(self, training_batch: TrainingBatch) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + keyboard_cond = getattr(training_batch, "keyboard_cond", None) + mouse_cond = getattr(training_batch, "mouse_cond", None) + if keyboard_cond is None or mouse_cond is None: + raise ValueError("WanGame batch must provide keyboard_cond and mouse_cond") + + from fastvideo.models.dits.hyworld.pose import process_custom_actions + + batch_size = int(training_batch.noisy_model_input.shape[0]) # type: ignore[union-attr] + viewmats_list: list[torch.Tensor] = [] + intrinsics_list: list[torch.Tensor] = [] + action_labels_list: list[torch.Tensor] = [] + for b in range(batch_size): + v, i, a = process_custom_actions(keyboard_cond[b], mouse_cond[b]) + viewmats_list.append(v) + intrinsics_list.append(i) + action_labels_list.append(a) + + viewmats = torch.stack(viewmats_list, dim=0).to(device=self.device, dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, dim=0).to(device=self.device, dtype=torch.bfloat16) + action_labels = torch.stack(action_labels_list, dim=0).to(device=self.device, dtype=torch.bfloat16) + + num_latent_t = int(training_batch.noisy_model_input.shape[2]) # type: ignore[union-attr] + if int(action_labels.shape[1]) != num_latent_t: + raise ValueError( + "Action conditioning temporal dim mismatch: " + f"action={tuple(action_labels.shape)} vs latent_t={num_latent_t}" + ) + if int(viewmats.shape[1]) != num_latent_t: + raise ValueError( + "Viewmats temporal dim mismatch: " + f"viewmats={tuple(viewmats.shape)} vs latent_t={num_latent_t}" + ) + + return viewmats, intrinsics, action_labels + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", + ) -> TrainingBatch: + dtype = self._get_training_dtype() + device = self.device + + training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + infos = raw_batch.get("info_list") + + if latents_source == "zeros": + clip_feature = raw_batch["clip_feature"] + batch_size = int(clip_feature.shape[0]) + vae_config = self.training_args.pipeline_config.vae_config.arch_config + num_channels = int(vae_config.z_dim) + spatial_compression_ratio = int(vae_config.spatial_compression_ratio) + latent_height = int(self.training_args.num_height) // spatial_compression_ratio + latent_width = int(self.training_args.num_width) // spatial_compression_ratio + latents = torch.zeros( + batch_size, + num_channels, + int(self.training_args.num_latent_t), + latent_height, + latent_width, + device=device, + dtype=dtype, + ) + elif latents_source == "data": + if "vae_latent" not in raw_batch: + raise ValueError("vae_latent not found in batch and latents_source='data'") + latents = raw_batch["vae_latent"] + latents = latents[:, :, : self.training_args.num_latent_t] + latents = latents.to(device, dtype=dtype) + else: + raise ValueError(f"Unknown latents_source: {latents_source!r}") + + if "clip_feature" not in raw_batch: + raise ValueError("clip_feature must be present for WanGame") + image_embeds = raw_batch["clip_feature"].to(device, dtype=dtype) + + if "first_frame_latent" not in raw_batch: + raise ValueError("first_frame_latent must be present for WanGame") + image_latents = raw_batch["first_frame_latent"] + image_latents = image_latents[:, :, : self.training_args.num_latent_t] + image_latents = image_latents.to(device, dtype=dtype) + + pil_image = raw_batch.get("pil_image") + if isinstance(pil_image, torch.Tensor): + training_batch.preprocessed_image = pil_image.to(device=device) + else: + training_batch.preprocessed_image = pil_image + + keyboard_cond = raw_batch.get("keyboard_cond") + if isinstance(keyboard_cond, torch.Tensor) and keyboard_cond.numel() > 0: + training_batch.keyboard_cond = keyboard_cond.to(device, dtype=dtype) + else: + training_batch.keyboard_cond = None + + mouse_cond = raw_batch.get("mouse_cond") + if isinstance(mouse_cond, torch.Tensor) and mouse_cond.numel() > 0: + training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) + else: + training_batch.mouse_cond = None + + temporal_compression_ratio = ( + self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + expected_num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 + if training_batch.keyboard_cond is not None and int(training_batch.keyboard_cond.shape[1]) != int( + expected_num_frames + ): + raise ValueError( + "keyboard_cond temporal dim mismatch: " + f"got {int(training_batch.keyboard_cond.shape[1])}, expected {int(expected_num_frames)}" + ) + if training_batch.mouse_cond is not None and int(training_batch.mouse_cond.shape[1]) != int( + expected_num_frames + ): + raise ValueError( + "mouse_cond temporal dim mismatch: " + f"got {int(training_batch.mouse_cond.shape[1])}, expected {int(expected_num_frames)}" + ) + + training_batch.latents = latents + training_batch.encoder_hidden_states = None + training_batch.encoder_attention_mask = None + training_batch.image_embeds = image_embeds + training_batch.image_latents = image_latents + training_batch.infos = infos + + training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch = self._prepare_dit_inputs(training_batch) + training_batch = self._build_attention_metadata(training_batch) + + training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) + if training_batch.attn_metadata is not None: + training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] + + training_batch.mask_lat_size = self._build_i2v_mask_latents(image_latents) + viewmats, intrinsics, action_labels = self._process_actions(training_batch) + training_batch.viewmats = viewmats + training_batch.Ks = intrinsics + training_batch.action = action_labels + + return training_batch + + def add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + b, t = clean_latents.shape[:2] + noisy = self.noise_scheduler.add_noise( + clean_latents.flatten(0, 1), + noise.flatten(0, 1), + timestep, + ).unflatten(0, (b, t)) + return noisy + + def _build_distill_input_kwargs( + self, + noisy_video_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + ) -> dict[str, Any]: + if batch.image_embeds is None: + raise RuntimeError("WanGameAdapter requires TrainingBatch.image_embeds") + if batch.image_latents is None: + raise RuntimeError("WanGameAdapter requires TrainingBatch.image_latents") + if batch.mask_lat_size is None: + raise RuntimeError("WanGameAdapter requires TrainingBatch.mask_lat_size") + + hidden_states = torch.cat( + [ + noisy_video_latents.permute(0, 2, 1, 3, 4), + batch.mask_lat_size, + batch.image_latents, + ], + dim=1, + ) + return { + "hidden_states": hidden_states, + "encoder_hidden_states": None, + "timestep": timestep.to(device=self.device, dtype=torch.bfloat16), + "encoder_hidden_states_image": batch.image_embeds, + "viewmats": getattr(batch, "viewmats", None), + "Ks": getattr(batch, "Ks", None), + "action": getattr(batch, "action", None), + "return_dict": False, + } + + def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: + transformer = handle.require_module("transformer") + transformer_2 = handle.modules.get("transformer_2") + if ( + transformer_2 is not None + and self.boundary_timestep is not None + and float(timestep.item()) < float(self.boundary_timestep) + ): + return transformer_2 + return transformer + + def predict_x0( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + del conditional + device_type = self.device.type + dtype = noisy_latents.dtype + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, batch) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + def predict_noise( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + del conditional + device_type = self.device.type + dtype = noisy_latents.dtype + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, batch) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + return pred_noise + + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + timesteps, attn_metadata = ctx + with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): + (loss / max(1, int(grad_accum_rounds))).backward() diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index 94a3023c7..266136520 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -75,6 +75,7 @@ def ensure_builtin_registrations() -> None: # NOTE: keep these imports explicit (no wildcard scanning) so registration # order is stable and failures are debuggable. from fastvideo.distillation.models import wan as _wan # noqa: F401 + from fastvideo.distillation.models import wangame as _wangame # noqa: F401 from fastvideo.distillation.methods.distribution_matching import dmd2 as _dmd2 # noqa: F401 from fastvideo.distillation.methods.fine_tuning import finetune as _finetune # noqa: F401 diff --git a/fastvideo/distillation/doc/adapters/wangame.md b/fastvideo/distillation/doc/adapters/wangame.md new file mode 100644 index 000000000..b7bb96ba8 --- /dev/null +++ b/fastvideo/distillation/doc/adapters/wangame.md @@ -0,0 +1,41 @@ +# `fastvideo/distillation/adapters/wangame.py` + +**定位** +- `WanGameAdapter` 是 WanGame model plugin 的 runtime 边界： + - 把 wangame parquet batch（I2V + action）转成 method 可消费的 + **operation-centric primitives** + - 不实现任何 method 的 rollout policy / step list / loss（这些属于 method） + +**和 `WanAdapter` 的关键差异** +- WanGame 是 **I2V + action conditioning**： + - transformer 的 `hidden_states` 不是纯视频 noisy latent，而是： + `cat([noisy_video_latents, mask_lat_size, image_latents], dim=1)` + - 还需要额外输入： + - `encoder_hidden_states_image`（来自 `clip_feature`） + - `viewmats / Ks / action`（由 `process_custom_actions(keyboard, mouse)` 生成） +- 目前 WanGame 的 `conditional/unconditional`（文本 CFG）语义**不成立**： + - adapter 仍保留 `conditional: bool` 形参以匹配 method protocol， + 但当前实现把它当作 no-op（cond/uncond 同路）。 + +**主要 API（被 method 通过 protocol 调用）** +- `prepare_batch(...) -> TrainingBatch` + - 处理 raw batch： + - `vae_latent`（video x0） + - `first_frame_latent`（I2V image latents） + - `clip_feature`（image embeds） + - `keyboard_cond` / `mouse_cond`（action） + - 采样 timesteps + 生成 noise/sigmas + 生成 noisy video latents + - 构建 attention metadata（dense / vsa 两套） + - 生成 `mask_lat_size` 并预处理 action（`viewmats/Ks/action`） +- `predict_noise(handle, noisy_latents, timestep, batch, conditional, attn_kind) -> Tensor` +- `predict_x0(handle, noisy_latents, timestep, batch, conditional, attn_kind) -> Tensor` +- `add_noise(clean_latents, noise, timestep) -> Tensor` +- `shift_and_clamp_timestep(t) -> Tensor` + `num_train_timesteps` +- `backward(loss, ctx, grad_accum_rounds=...)` + +**边界 / TODO** +- ✅ adapter 不保存/管理 few-step denoising step list，也不决定 rollout 策略。 +- ✅ adapter 不引入 DMD2 专属概念（例如 “generator/critic”）。 +- TODO：若未来需要在 wangame 上定义 “uncond” 语义（例如 `zero_action/zero_image`）， + 应通过 `method_config` 声明，并由 adapter 提供可解释的操作入口（而不是硬编码到 adapter 内部逻辑）。 + diff --git a/fastvideo/distillation/doc/models/wangame.md b/fastvideo/distillation/doc/models/wangame.md new file mode 100644 index 000000000..1e1e43c4a --- /dev/null +++ b/fastvideo/distillation/doc/models/wangame.md @@ -0,0 +1,28 @@ +# `fastvideo/distillation/models/wangame.py` + +**定位** +- `@register_model("wangame")` 的 build-time 插件（实现：`build_wangame_components(...)`）： + - 负责把 YAML config 装配成 `ModelComponents` + - 包含 WanGame 特有的模块加载、dataset schema、adapter/validator 选择等逻辑 + +**产物** +- `ModelComponents(training_args, bundle, adapter, dataloader, validator, start_step)` + +**主要职责** +1) **加载 shared components** + - `vae`：从 `training.model_path`（默认 student.path）加载 + - `noise_scheduler`：`FlowMatchEulerDiscreteScheduler(shift=flow_shift)` +2) **按 roles 加载 transformer 模块** + - 对每个 role：加载 `transformer`（可选 `transformer_2`） + - 根据 `RoleSpec.trainable` 设置 `requires_grad` + - 可选开启 activation checkpoint（仅对 trainable role） +3) **构建 bundle / adapter / dataloader / validator** + - `bundle = RoleManager(roles=role_handles)` + - `adapter = WanGameAdapter(...)` + - dataloader：parquet + `pyarrow_schema_wangame` + - validator（可选）：`WanGameValidator`（当 `training.log_validation=true`） + +**关于 roles.family** +- 当前 `wangame` plugin 只支持 `family="wangame"` 的 role。 + 这让 build-time 逻辑保持高内聚：模型加载、batch schema 与 adapter 能保持一致。 + diff --git a/fastvideo/distillation/doc/utils/dataloader.md b/fastvideo/distillation/doc/utils/dataloader.md index d380ecfa4..2c0793738 100644 --- a/fastvideo/distillation/doc/utils/dataloader.md +++ b/fastvideo/distillation/doc/utils/dataloader.md @@ -8,6 +8,9 @@ - `build_parquet_t2v_train_dataloader(training_args, parquet_schema=...)` - 复用 FastVideo 现有的 `build_parquet_map_style_dataloader(...)` - 仅做最小封装：从 `training_args` 读取必要参数（data_path/batch/workers/seed/cfg_rate/text_len 等） +- `build_parquet_wangame_train_dataloader(training_args, parquet_schema=...)` + - 面向 wangame 的 parquet schema（I2V + action） + - 不依赖 `pipeline_config.text_encoder_configs`（schema 不含 text embedding） **边界** - 这里不包含 model/pipeline 语义（例如 Wan 的 forward/backward 细节）。 diff --git a/fastvideo/distillation/doc/validators/wangame.md b/fastvideo/distillation/doc/validators/wangame.md new file mode 100644 index 000000000..103f4d56e --- /dev/null +++ b/fastvideo/distillation/doc/validators/wangame.md @@ -0,0 +1,23 @@ +# `fastvideo/distillation/validators/wangame.py` + +**定位** +- `WanGameValidator` 是 WanGame 的 standalone validator： + - 由 model plugin 构建并返回（当 `training.log_validation=true`） + - 由 method 决定何时调用，并通过 `ValidationRequest` 指定采样细节 + +**pipeline 选择（最小侵入式）** +- `sampler_kind="ode"`： + - 使用 `WanGameActionImageToVideoPipeline`（ODE/UniPC 采样） +- `sampler_kind="sde"`： + - 使用 `WanGameCausalDMDPipeline`（SDE-style rollout / DMD sampling） + +> 说明：当前阶段我们用“切换 pipeline class”的方式区分 ODE/SDE， +> 以降低接入风险。后续若统一到同一个 pipeline 的 `sampler_kind` 分支， +> 可以减少重复逻辑，但改动会更侵入（属于后续 phase）。 + +**调用方式（method-managed validation）** +- method 通过 `ValidationRequest(sample_handle=..., sampler_kind=..., ...)` 指定： + - 用哪个 role 的 transformer 做采样（通常是 student） + - 采样步数（`sampling_steps`） + - 若是 SDE 采样：`sampling_timesteps`（few-step rollout 的 explicit steps） + diff --git a/fastvideo/distillation/models/wangame.py b/fastvideo/distillation/models/wangame.py new file mode 100644 index 000000000..57b3a3436 --- /dev/null +++ b/fastvideo/distillation/models/wangame.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import torch + +from fastvideo.distillation.adapters.wangame import WanGameAdapter +from fastvideo.distillation.dispatch import register_model +from fastvideo.distillation.models.components import ModelComponents +from fastvideo.distillation.roles import RoleHandle, RoleManager +from fastvideo.distillation.utils.config import DistillRunConfig +from fastvideo.distillation.utils.dataloader import build_parquet_wangame_train_dataloader +from fastvideo.distillation.utils.module_state import apply_trainable +from fastvideo.distillation.utils.moduleloader import load_module_from_path +from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from fastvideo.training.activation_checkpoint import apply_activation_checkpointing + + +@register_model("wangame") +def build_wangame_components(*, cfg: DistillRunConfig) -> ModelComponents: + training_args = cfg.training_args + roles_cfg = cfg.roles + + if getattr(training_args, "seed", None) is None: + raise ValueError("training.seed must be set for distillation") + if not getattr(training_args, "data_path", ""): + raise ValueError("training.data_path must be set for distillation") + + # Ensure transformer class name is resolvable by PipelineComponentLoader. + training_args.override_transformer_cls_name = "WanGameActionTransformer3DModel" + + # Load shared components (student base path). + vae = load_module_from_path( + model_path=str(training_args.model_path), + module_type="vae", + training_args=training_args, + ) + noise_scheduler = FlowMatchEulerDiscreteScheduler( + shift=float(training_args.pipeline_config.flow_shift or 0.0) + ) + + role_handles: dict[str, RoleHandle] = {} + for role, role_spec in roles_cfg.items(): + if role_spec.family != "wangame": + raise ValueError( + "Wangame model plugin only supports roles with family='wangame'; " + f"got {role}={role_spec.family!r}" + ) + + disable_custom_init_weights = bool( + getattr(role_spec, "disable_custom_init_weights", False) + ) + transformer = load_module_from_path( + model_path=role_spec.path, + module_type="transformer", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + ) + modules: dict[str, torch.nn.Module] = {"transformer": transformer} + + # Optional MoE support: load transformer_2 if present in the model. + try: + transformer_2 = load_module_from_path( + model_path=role_spec.path, + module_type="transformer_2", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + ) + except ValueError: + transformer_2 = None + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + + for name, module in list(modules.items()): + module = apply_trainable(module, trainable=bool(role_spec.trainable)) + if role_spec.trainable and getattr( + training_args, "enable_gradient_checkpointing_type", None + ): + module = apply_activation_checkpointing( + module, + checkpointing_type=training_args.enable_gradient_checkpointing_type, + ) + modules[name] = module + + role_handles[role] = RoleHandle( + modules=modules, + optimizers={}, + lr_schedulers={}, + trainable=bool(role_spec.trainable), + ) + + bundle = RoleManager(roles=role_handles) + + validator = None + if getattr(training_args, "log_validation", False): + from fastvideo.distillation.validators.wangame import WanGameValidator + + validator = WanGameValidator(training_args=training_args) + + adapter = WanGameAdapter( + training_args=training_args, + noise_scheduler=noise_scheduler, + vae=vae, + ) + + from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame + + dataloader = build_parquet_wangame_train_dataloader( + training_args, + parquet_schema=pyarrow_schema_wangame, + ) + + return ModelComponents( + training_args=training_args, + bundle=bundle, + adapter=adapter, + dataloader=dataloader, + validator=validator, + start_step=0, + ) diff --git a/fastvideo/distillation/utils/dataloader.py b/fastvideo/distillation/utils/dataloader.py index 07ace99f2..b87deb0c9 100644 --- a/fastvideo/distillation/utils/dataloader.py +++ b/fastvideo/distillation/utils/dataloader.py @@ -26,3 +26,28 @@ def build_parquet_t2v_train_dataloader( seed=int(training_args.seed or 0), ) return dataloader + + +def build_parquet_wangame_train_dataloader( + training_args: Any, + *, + parquet_schema: Any, +) -> Any: + """Build a parquet map-style dataloader for WanGame (I2V+action) datasets.""" + + from fastvideo.dataset import build_parquet_map_style_dataloader + + cfg_rate = float(getattr(training_args, "training_cfg_rate", 0.0) or 0.0) + _dataset, dataloader = build_parquet_map_style_dataloader( + training_args.data_path, + training_args.train_batch_size, + num_data_workers=training_args.dataloader_num_workers, + parquet_schema=parquet_schema, + cfg_rate=cfg_rate, + drop_last=True, + # WanGame parquet schema does not include text embeddings, but the + # parquet loader expects a padding length parameter. + text_padding_length=512, + seed=int(training_args.seed or 0), + ) + return dataloader diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py new file mode 100644 index 000000000..633e77416 --- /dev/null +++ b/fastvideo/distillation/validators/wangame.py @@ -0,0 +1,327 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any + +import imageio +import numpy as np +import torch +import torchvision +from einops import rearrange +from torch.utils.data import DataLoader + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.validation_dataset import ValidationDataset +from fastvideo.distributed import get_sp_group, get_world_group +from fastvideo.logger import init_logger +from fastvideo.pipelines import ForwardBatch +from fastvideo.distillation.validators.base import ValidationRequest +from fastvideo.training.trackers import DummyTracker +from fastvideo.utils import shallow_asdict + +logger = init_logger(__name__) + + +@dataclass(slots=True) +class _ValidationStepResult: + videos: list[list[np.ndarray]] + captions: list[str] + + +class WanGameValidator: + """Standalone validator for WanGame distillation/finetuning.""" + + def __init__( + self, + *, + training_args: Any, + tracker: Any | None = None, + ) -> None: + self.training_args = training_args + self.tracker = tracker or DummyTracker() + + self.world_group = get_world_group() + self.sp_group = get_sp_group() + self.global_rank = self.world_group.rank + self.rank_in_sp_group = self.sp_group.rank_in_group + self.sp_world_size = self.sp_group.world_size + + seed = getattr(training_args, "seed", None) + if seed is None: + raise ValueError("training_args.seed must be set for validation") + self.seed = int(seed) + self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) + + self._pipeline: Any | None = None + self._pipeline_key: tuple[int, str] | None = None + self._sampling_param: SamplingParam | None = None + + def set_tracker(self, tracker: Any) -> None: + self.tracker = tracker + + def _get_sampling_param(self) -> SamplingParam: + if self._sampling_param is None: + self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) + return self._sampling_param + + def _get_pipeline(self, *, transformer: torch.nn.Module, sampler_kind: str) -> Any: + key = (id(transformer), str(sampler_kind)) + if self._pipeline is not None and self._pipeline_key == key: + return self._pipeline + + flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) + + if str(sampler_kind).lower() in {"ode"}: + from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import ( + WanGameActionImageToVideoPipeline, + ) + + self._pipeline = WanGameActionImageToVideoPipeline.from_pretrained( + self.training_args.model_path, + inference_mode=True, + flow_shift=float(flow_shift) if flow_shift is not None else None, + loaded_modules={"transformer": transformer}, + tp_size=self.training_args.tp_size, + sp_size=self.training_args.sp_size, + num_gpus=self.training_args.num_gpus, + pin_cpu_memory=self.training_args.pin_cpu_memory, + dit_cpu_offload=True, + ) + elif str(sampler_kind).lower() in {"sde"}: + from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( + WanGameCausalDMDPipeline, + ) + + self._pipeline = WanGameCausalDMDPipeline.from_pretrained( + self.training_args.model_path, + inference_mode=True, + flow_shift=float(flow_shift) if flow_shift is not None else None, + loaded_modules={"transformer": transformer}, + tp_size=self.training_args.tp_size, + sp_size=self.training_args.sp_size, + num_gpus=self.training_args.num_gpus, + pin_cpu_memory=self.training_args.pin_cpu_memory, + dit_cpu_offload=True, + ) + else: + raise ValueError(f"Unknown sampler_kind for WanGame validation: {sampler_kind!r}") + + self._pipeline_key = key + return self._pipeline + + def _parse_validation_steps(self) -> list[int]: + raw = str(getattr(self.training_args, "validation_sampling_steps", "") or "") + steps = [int(s) for s in raw.split(",") if s.strip()] + return [s for s in steps if s > 0] + + def _prepare_validation_batch( + self, + sampling_param: SamplingParam, + validation_batch: dict[str, Any], + num_inference_steps: int, + *, + sampling_timesteps: list[int] | None = None, + guidance_scale: float | None = None, + ) -> ForwardBatch: + training_args = self.training_args + + sampling_param.prompt = validation_batch["prompt"] + sampling_param.height = training_args.num_height + sampling_param.width = training_args.num_width + sampling_param.image_path = validation_batch.get("image_path") or validation_batch.get("video_path") + sampling_param.num_inference_steps = int(num_inference_steps) + sampling_param.data_type = "video" + if guidance_scale is not None: + sampling_param.guidance_scale = float(guidance_scale) + elif getattr(training_args, "validation_guidance_scale", ""): + sampling_param.guidance_scale = float(training_args.validation_guidance_scale) + sampling_param.seed = self.seed + + temporal_compression_factor = ( + training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + num_frames = (training_args.num_latent_t - 1) * temporal_compression_factor + 1 + sampling_param.num_frames = int(num_frames) + + latents_size = [ + (sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, + sampling_param.width // 8, + ] + n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + + sampling_timesteps_tensor = ( + torch.tensor([int(s) for s in sampling_timesteps], dtype=torch.long) + if sampling_timesteps is not None + else None + ) + + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=self.validation_random_generator, + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=training_args.VSA_sparsity, + timesteps=sampling_timesteps_tensor, + sampling_timesteps=sampling_timesteps_tensor, + ) + if "image" in validation_batch and validation_batch["image"] is not None: + batch.pil_image = validation_batch["image"] + + if "keyboard_cond" in validation_batch and validation_batch["keyboard_cond"] is not None: + keyboard_cond = validation_batch["keyboard_cond"] + keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) + keyboard_cond = keyboard_cond.unsqueeze(0) + batch.keyboard_cond = keyboard_cond + + if "mouse_cond" in validation_batch and validation_batch["mouse_cond"] is not None: + mouse_cond = validation_batch["mouse_cond"] + mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) + mouse_cond = mouse_cond.unsqueeze(0) + batch.mouse_cond = mouse_cond + + return batch + + def _run_validation_for_steps( + self, + num_inference_steps: int, + *, + transformer: torch.nn.Module, + sampler_kind: str, + sampling_timesteps: list[int] | None = None, + guidance_scale: float | None = None, + ) -> _ValidationStepResult: + training_args = self.training_args + pipeline = self._get_pipeline(transformer=transformer, sampler_kind=sampler_kind) + sampling_param = self._get_sampling_param() + + dataset = ValidationDataset(training_args.validation_dataset_file) + dataloader = DataLoader(dataset, batch_size=None, num_workers=0) + + videos: list[list[np.ndarray]] = [] + captions: list[str] = [] + + for validation_batch in dataloader: + batch = self._prepare_validation_batch( + sampling_param, + validation_batch, + num_inference_steps, + sampling_timesteps=sampling_timesteps, + guidance_scale=guidance_scale, + ) + + assert batch.prompt is not None and isinstance(batch.prompt, str) + captions.append(batch.prompt) + + with torch.no_grad(): + output_batch = pipeline.forward(batch, training_args) + + samples = output_batch.output.cpu() + if self.rank_in_sp_group != 0: + continue + + video = rearrange(samples, "b c t h w -> t b c h w") + frames: list[np.ndarray] = [] + for x in video: + x = torchvision.utils.make_grid(x, nrow=6) + x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) + frames.append((x * 255).numpy().astype(np.uint8)) + videos.append(frames) + + return _ValidationStepResult(videos=videos, captions=captions) + + def log_validation(self, step: int, *, request: ValidationRequest | None = None) -> None: + training_args = self.training_args + if not getattr(training_args, "log_validation", False): + return + if not getattr(training_args, "validation_dataset_file", ""): + raise ValueError("validation_dataset_file must be set when log_validation is enabled") + + guidance_scale = getattr(request, "guidance_scale", None) + validation_steps = getattr(request, "sampling_steps", None) or self._parse_validation_steps() + if not validation_steps: + return + sampler_kind = getattr(request, "sampler_kind", None) or "ode" + sampling_timesteps = getattr(request, "sampling_timesteps", None) + if sampling_timesteps is not None: + expected = int(len(sampling_timesteps)) + for steps in validation_steps: + if int(steps) != expected: + raise ValueError( + "validation_sampling_steps must match " + f"len(request.sampling_timesteps)={expected} when " + "sampling_timesteps is provided, got " + f"{validation_steps!r}." + ) + + sample_handle = getattr(request, "sample_handle", None) + if sample_handle is None: + raise ValueError("ValidationRequest.sample_handle must be provided by the method") + transformer = sample_handle.require_module("transformer") + was_training = bool(getattr(transformer, "training", False)) + + output_dir = getattr(request, "output_dir", None) or training_args.output_dir + + old_inference_mode = training_args.inference_mode + old_dit_cpu_offload = training_args.dit_cpu_offload + try: + training_args.inference_mode = True + training_args.dit_cpu_offload = True + transformer.eval() + + num_sp_groups = self.world_group.world_size // self.sp_group.world_size + + for num_inference_steps in validation_steps: + result = self._run_validation_for_steps( + num_inference_steps, + transformer=transformer, + sampler_kind=str(sampler_kind), + sampling_timesteps=sampling_timesteps, + guidance_scale=guidance_scale, + ) + + if self.rank_in_sp_group != 0: + continue + + if self.global_rank == 0: + all_videos = list(result.videos) + all_captions = list(result.captions) + for sp_group_idx in range(1, num_sp_groups): + src_rank = sp_group_idx * self.sp_world_size + recv_videos = self.world_group.recv_object(src=src_rank) + recv_captions = self.world_group.recv_object(src=src_rank) + all_videos.extend(recv_videos) + all_captions.extend(recv_captions) + + os.makedirs(output_dir, exist_ok=True) + video_filenames: list[str] = [] + sampling_param = self._get_sampling_param() + for i, video in enumerate(all_videos): + filename = os.path.join( + output_dir, + f"validation_step_{step}_inference_steps_{num_inference_steps}_video_{i}.mp4", + ) + imageio.mimsave(filename, video, fps=sampling_param.fps) + video_filenames.append(filename) + + video_logs = [] + for filename, caption in zip(video_filenames, all_captions, strict=True): + video_artifact = self.tracker.video(filename, caption=caption) + if video_artifact is not None: + video_logs.append(video_artifact) + if video_logs: + logs = {f"validation_videos_{num_inference_steps}_steps": video_logs} + self.tracker.log_artifacts(logs, step) + else: + self.world_group.send_object(result.videos, dst=0) + self.world_group.send_object(result.captions, dst=0) + finally: + training_args.inference_mode = old_inference_mode + training_args.dit_cpu_offload = old_dit_cpu_offload + if was_training: + transformer.train() + From d2387d35fcaa1476a17fd33b1cac951eb1bf6533 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 23:01:55 +0000 Subject: [PATCH 118/214] dmd method cfg_uncond --- dev/phase_add_wangame.md | 76 ++++++- ...ame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml | 114 +++++++++++ .../wangame/finetune_wangame2.1_i2v_1.3B.yaml | 2 +- .../validation_random_8.json | 84 ++++++++ fastvideo/distillation/adapters/wan.py | 114 ++++++++++- fastvideo/distillation/adapters/wangame.py | 187 ++++++++++++++++-- .../methods/distribution_matching/dmd2.py | 64 ++++++ fastvideo/distillation/validators/wan.py | 4 + fastvideo/distillation/validators/wangame.py | 5 +- 9 files changed, 618 insertions(+), 32 deletions(-) create mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml create mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json diff --git a/dev/phase_add_wangame.md b/dev/phase_add_wangame.md index 84d7519f6..39912ce06 100644 --- a/dev/phase_add_wangame.md +++ b/dev/phase_add_wangame.md @@ -3,7 +3,49 @@ > 目标：在 **不回到 legacy training pipeline** 的前提下，让 `fastvideo/training/distillation.py` > 可以通过 YAML（schema v2）跑起 **WanGame** 的 finetune / distill（优先 finetune）。 > -> 本文件只做“代码层面规划”，不修改代码。 +> 本文件最初用于“代码层面规划”，现在也用来记录已落地的实现与遗留 TODO。 + +--- + +## 当前进展（已落地） + +> 下面是“已经写进代码库并通过静态检查（`compileall` + ruff）”的部分。 +> GPU 端到端训练/验证需要你在有 GPU 的机器上跑（我们这边环境可能没有 driver）。 + +### ✅ 已实现（最小可用：finetune） + +- `recipe.family: wangame` + `recipe.method: finetune` +- 新增 model plugin / adapter / validator： + - `fastvideo/distillation/models/wangame.py` + - `fastvideo/distillation/adapters/wangame.py` + - `fastvideo/distillation/validators/wangame.py` +- builtin dispatch 注册： + - `fastvideo/distillation/dispatch.py:ensure_builtin_registrations()` +- dataloader helper（复用 parquet loader，支持 `path:N` + 多路径）： + - `fastvideo/distillation/utils/dataloader.py:build_parquet_wangame_train_dataloader()` +- examples（可直接跑）： + - `examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml` + - 已把 legacy `finetune_wangame.slurm` 的 `DATA_DIR` 语义搬进来 + - 用 YAML folded string（`>-`）让超长 `data_path` 更可读 + - `examples/distillation/wangame/finetune-temp.sh` + +### ✅ 已实现（为 future DMD2 做好 primitives） + +- `WanGameAdapter` 已提供 DMD2 所需的 operation primitives： + - `num_train_timesteps / shift_and_clamp_timestep / add_noise` + - `predict_noise / predict_x0 / backward` + - attention metadata（dense/vsa）+ forward_context +- `WanGameValidator` 支持根据 `ValidationRequest.sampler_kind` 选 pipeline： + - `ode` -> `WanGameActionImageToVideoPipeline` + - `sde` -> `WanGameCausalDMDPipeline` + +### ✅ 关键对齐说明（legacy vs 新框架） + +- **训练 noising scheduler**：新框架 wangame 训练使用 + `FlowMatchEulerDiscreteScheduler`，与 legacy training loop（`TrainingPipeline`） + 一致（训练阶段并不使用 UniPC 做 noising）。 +- **validation sampler**：validator 走 pipeline（ODE/SDE）时，仍由 pipeline + 自己持有对应 scheduler（例如 ODE pipeline 使用 `FlowUniPCMultistepScheduler`）。 --- @@ -75,7 +117,7 @@ WanGame 不是“想象中的模型”，FastVideo 里已经有一整套（legac ### 3.1 新增：model plugin -- [ ] `fastvideo/distillation/models/wangame.py` +- [x] `fastvideo/distillation/models/wangame.py` - `@register_model("wangame")` - 主要职责： - 设置 `training_args.override_transformer_cls_name = "WanGameActionTransformer3DModel"` @@ -90,7 +132,7 @@ WanGame 不是“想象中的模型”，FastVideo 里已经有一整套（legac ### 3.2 新增：adapter -- [ ] `fastvideo/distillation/adapters/wangame.py` +- [x] `fastvideo/distillation/adapters/wangame.py` - 复用 `WanAdapter` 的通用 mechanics（timestep/noise/attn_metadata/backward 的模式） - 重点实现（对齐 legacy `wangame_training_pipeline.py`）： - `prepare_batch(raw_batch, current_vsa_sparsity, latents_source=...)` @@ -118,7 +160,7 @@ WanGame 不是“想象中的模型”，FastVideo 里已经有一整套（legac ### 3.3 新增：validator -- [ ] `fastvideo/distillation/validators/wangame.py` +- [x] `fastvideo/distillation/validators/wangame.py` - API 对齐 `WanValidator`：`log_validation(step, request=ValidationRequest)` - pipeline 选择： - `request.sampler_kind == "ode"`：`WanGameActionImageToVideoPipeline` @@ -129,7 +171,7 @@ WanGame 不是“想象中的模型”，FastVideo 里已经有一整套（legac ### 3.4 改动：dispatch builtin registrations -- [ ] `fastvideo/distillation/dispatch.py:ensure_builtin_registrations()` +- [x] `fastvideo/distillation/dispatch.py:ensure_builtin_registrations()` - 显式 import 新增的 `fastvideo.distillation.models.wangame` ### 3.5 可选：dataloader util @@ -137,10 +179,14 @@ WanGame 不是“想象中的模型”，FastVideo 里已经有一整套（legac 当前 `fastvideo/distillation/utils/dataloader.py` 只有 T2V helper。 wangame 需要 I2V+action schema，因此建议： -- [ ] `fastvideo/distillation/utils/dataloader.py` +- [x] `fastvideo/distillation/utils/dataloader.py` - 新增 `build_parquet_wangame_train_dataloader(training_args, parquet_schema=pyarrow_schema_wangame)` - 内部仍调用 `fastvideo.dataset.build_parquet_map_style_dataloader(...)` +TODO（更通用的方向，暂不做）： +- [ ] 扩展到更多 dataset kind（webdataset / precomputed / ode-init ...）， + 并用更统一的 config/dispatch 管理（例如 `DataSpec`）。 + --- ## 4) 配置（YAML）规划（schema v2） @@ -248,6 +294,14 @@ method_config: - 但执行必然落在 adapter（如何 zero_action/zero_image 是模型相关的），因此 adapter 需要提供一个 operation 来承接该语义（例如“构造 uncond conditioning variant”）。 + **新增 TODO（需要实现）** + - [ ] 为 wangame + DMD2 引入 `method_config.uncond_mode` + - DMD2Method：读取该字段，并在 teacher CFG 时把 `conditional=False` + 映射到对应的 “uncond variant” + - WanGameAdapter：提供可解释的 uncond 变体构造（避免硬编码 DMD2 名词），例如： + - `conditional=False` 时按 `uncond_mode` 将 action/image conditioning 置零 + - 或提供一个更显式的 operation（如 `build_conditioning_variant(...)`） + 2) **train_action_only / action_warmup_steps（细粒度 trainable）** legacy `wangame_training_pipeline.py` 支持更细粒度的训练策略： - `train_action_only`：冻结 base DiT，只训练 action 相关参数（pattern-based） @@ -299,3 +353,13 @@ method_config: - distill（若做第二档）： - e2e 跑通（few-step） - validation 选择 `sde` 时，视觉应接近 legacy DMD pipeline 的 sampling 形态 + +--- + +## 8) 目前遗留 / 下一步（WanGame 接入方向） + +- [ ] DMD2 on wangame 的 `uncond` 语义（`method_config.uncond_mode`） +- [ ] action-only / warmup（把 legacy 的 `train_action_only / action_warmup_steps` + 接到新框架：归属在 method/role policy 而非 model plugin） +- [ ] 若需要减少 ODE/SDE pipeline 分叉：将 wangame inference pipeline 也升级为 + `sampler_kind` 可切换（侵入式更强，建议放更后面的 phase） diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml new file mode 100644 index 000000000..ecbfadbcb --- /dev/null +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml @@ -0,0 +1,114 @@ +# WanGame DMD2 distillation (few-step rollout). +# +# - Fill in `roles.teacher.path` with your teacher checkpoint (must be wangame family). +# - This YAML is the single source of truth for the run. + +recipe: + family: wangame + method: dmd2 + +roles: + student: + family: wangame + path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + trainable: true + teacher: + family: wangame + # TODO: replace with your teacher model path (wangame architecture). + path: YOUR_TEACHER_MODEL_PATH + trainable: false + disable_custom_init_weights: true + critic: + family: wangame + path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + trainable: true + disable_custom_init_weights: true + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder). + # Supports comma-separated `path` or `path:N` (repeat count) entries. + data_path: >- + /mnt/weka/home/hao.zhang/kaiqin/FastVideo/fastvideo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_dmd2_8steps_distill + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer (student) + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Optimizer (critic / fake-score) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation (method-agnostic knobs) + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_dmd2_4steps_distill + log_validation: true + validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + validation_steps: 50 + validation_sampling_steps: "4" + validation_guidance_scale: "1.0" + +pipeline_config: + flow_shift: 3 + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + + # Few-step schedule (single source of truth for the method). + # These are "step indices" and will be warped by the (shifted) scheduler. + warp_denoising_step: true + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + + # Define CFG "uncond" semantics (operation-centric). + # + # If all channels are `keep`, then uncond == cond, so CFG becomes a no-op: + # real_cfg = cond + (cond - uncond) * scale == cond + cfg_uncond: + on_missing: error + # For I2V+action, a practical default is "no-action baseline". + action: zero + # Keep the reference image conditioning for I2V. + image: keep + # Text is unused in current wangame training batches, but kept for schema symmetry. + text: keep diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml index dd8f3cbe1..62077be6a 100644 --- a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml +++ b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml @@ -67,7 +67,7 @@ training: tracker_project_name: distillation_wangame wandb_run_name: wangame_finetune log_validation: true - validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json + validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json validation_steps: 100 validation_sampling_steps: "40" validation_guidance_scale: "1.0" diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json new file mode 100644 index 000000000..0bde93715 --- /dev/null +++ b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json @@ -0,0 +1,84 @@ +{ + "data": [ + { + "caption": "00 Val-00: W", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "01 Val-01: S", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "02 Val-02: A", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "03 Val-03: D", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "04 Val-04: u", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/u.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "05 Val-05: d", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/d.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "06 Val-06: l", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/l.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + }, + { + "caption": "07 Val-07: r", + "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", + "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/r.npy", + "video_path": null, + "num_inference_steps": 40, + "height": 352, + "width": 640, + "num_frames": 77 + } + ] +} \ No newline at end of file diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py index 800c79d7b..bb190bb32 100644 --- a/fastvideo/distillation/adapters/wan.py +++ b/fastvideo/distillation/adapters/wan.py @@ -422,6 +422,100 @@ def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch. return transformer_2 return transformer + def _get_uncond_text_dict( + self, + batch: TrainingBatch, + *, + cfg_uncond: dict[str, Any] | None, + ) -> dict[str, torch.Tensor]: + if cfg_uncond is None: + text_dict = getattr(batch, "unconditional_dict", None) + if text_dict is None: + raise RuntimeError( + "Missing unconditional_dict; ensure_negative_conditioning() may have failed" + ) + return text_dict + + on_missing_raw = cfg_uncond.get("on_missing", "error") + if not isinstance(on_missing_raw, str): + raise ValueError( + "method_config.cfg_uncond.on_missing must be a string, got " + f"{type(on_missing_raw).__name__}" + ) + on_missing = on_missing_raw.strip().lower() + if on_missing not in {"error", "ignore"}: + raise ValueError( + "method_config.cfg_uncond.on_missing must be one of {error, ignore}, got " + f"{on_missing_raw!r}" + ) + + # Wan only supports text CFG. If users configure other channels, fail + # fast (unless explicitly ignored). + for channel, policy_raw in cfg_uncond.items(): + if channel in {"on_missing", "text"}: + continue + if policy_raw is None: + continue + if not isinstance(policy_raw, str): + raise ValueError( + "method_config.cfg_uncond values must be strings, got " + f"{channel}={type(policy_raw).__name__}" + ) + policy = policy_raw.strip().lower() + if policy == "keep": + continue + if on_missing == "ignore": + continue + raise ValueError( + "WanAdapter does not support cfg_uncond channel " + f"{channel!r} (policy={policy!r}). " + "Set cfg_uncond.on_missing=ignore or remove the channel." + ) + + text_policy_raw = cfg_uncond.get("text", None) + if text_policy_raw is None: + text_policy = "negative_prompt" + elif not isinstance(text_policy_raw, str): + raise ValueError( + "method_config.cfg_uncond.text must be a string, got " + f"{type(text_policy_raw).__name__}" + ) + else: + text_policy = text_policy_raw.strip().lower() + + if text_policy in {"negative_prompt"}: + text_dict = getattr(batch, "unconditional_dict", None) + if text_dict is None: + raise RuntimeError( + "Missing unconditional_dict; ensure_negative_conditioning() may have failed" + ) + return text_dict + if text_policy == "keep": + if batch.conditional_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + return batch.conditional_dict + if text_policy == "zero": + if batch.conditional_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + cond = batch.conditional_dict + enc = cond["encoder_hidden_states"] + mask = cond["encoder_attention_mask"] + if not torch.is_tensor(enc) or not torch.is_tensor(mask): + raise TypeError("conditional_dict must contain tensor text inputs") + return { + "encoder_hidden_states": torch.zeros_like(enc), + "encoder_attention_mask": torch.zeros_like(mask), + } + if text_policy == "drop": + raise ValueError( + "cfg_uncond.text=drop is not supported for Wan. " + "Use {negative_prompt, keep, zero}." + ) + raise ValueError( + "cfg_uncond.text must be one of {negative_prompt, keep, zero, drop}, got " + f"{text_policy_raw!r}" + ) + def predict_x0( self, handle: RoleHandle, @@ -430,13 +524,17 @@ def predict_x0( batch: TrainingBatch, *, conditional: bool, + cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: device_type = self.device.type dtype = noisy_latents.dtype - text_dict = batch.conditional_dict if conditional else getattr(batch, "unconditional_dict", None) - if text_dict is None: - raise RuntimeError("Missing unconditional_dict; ensure_negative_conditioning() may have failed") + if conditional: + text_dict = batch.conditional_dict + if text_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + else: + text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) if attn_kind == "dense": attn_metadata = batch.attn_metadata @@ -468,13 +566,17 @@ def predict_noise( batch: TrainingBatch, *, conditional: bool, + cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: device_type = self.device.type dtype = noisy_latents.dtype - text_dict = batch.conditional_dict if conditional else getattr(batch, "unconditional_dict", None) - if text_dict is None: - raise RuntimeError("Missing unconditional_dict; ensure_negative_conditioning() may have failed") + if conditional: + text_dict = batch.conditional_dict + if text_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + else: + text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) if attn_kind == "dense": attn_metadata = batch.attn_metadata diff --git a/fastvideo/distillation/adapters/wangame.py b/fastvideo/distillation/adapters/wangame.py index 3c0212919..b0fc48b79 100644 --- a/fastvideo/distillation/adapters/wangame.py +++ b/fastvideo/distillation/adapters/wangame.py @@ -398,20 +398,19 @@ def _build_distill_input_kwargs( self, noisy_video_latents: torch.Tensor, timestep: torch.Tensor, - batch: TrainingBatch, + *, + image_embeds: torch.Tensor, + image_latents: torch.Tensor, + mask_lat_size: torch.Tensor, + viewmats: torch.Tensor | None, + Ks: torch.Tensor | None, + action: torch.Tensor | None, ) -> dict[str, Any]: - if batch.image_embeds is None: - raise RuntimeError("WanGameAdapter requires TrainingBatch.image_embeds") - if batch.image_latents is None: - raise RuntimeError("WanGameAdapter requires TrainingBatch.image_latents") - if batch.mask_lat_size is None: - raise RuntimeError("WanGameAdapter requires TrainingBatch.mask_lat_size") - hidden_states = torch.cat( [ noisy_video_latents.permute(0, 2, 1, 3, 4), - batch.mask_lat_size, - batch.image_latents, + mask_lat_size, + image_latents, ], dim=1, ) @@ -419,13 +418,137 @@ def _build_distill_input_kwargs( "hidden_states": hidden_states, "encoder_hidden_states": None, "timestep": timestep.to(device=self.device, dtype=torch.bfloat16), - "encoder_hidden_states_image": batch.image_embeds, - "viewmats": getattr(batch, "viewmats", None), - "Ks": getattr(batch, "Ks", None), - "action": getattr(batch, "action", None), + "encoder_hidden_states_image": image_embeds, + "viewmats": viewmats, + "Ks": Ks, + "action": action, "return_dict": False, } + def _select_cfg_condition_inputs( + self, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None, + ) -> dict[str, Any]: + image_embeds = batch.image_embeds + image_latents = batch.image_latents + mask_lat_size = batch.mask_lat_size + if image_embeds is None: + raise RuntimeError("WanGameAdapter requires TrainingBatch.image_embeds") + if image_latents is None: + raise RuntimeError("WanGameAdapter requires TrainingBatch.image_latents") + if mask_lat_size is None: + raise RuntimeError("WanGameAdapter requires TrainingBatch.mask_lat_size") + + viewmats = getattr(batch, "viewmats", None) + Ks = getattr(batch, "Ks", None) + action = getattr(batch, "action", None) + + if conditional or cfg_uncond is None: + return { + "image_embeds": image_embeds, + "image_latents": image_latents, + "mask_lat_size": mask_lat_size, + "viewmats": viewmats, + "Ks": Ks, + "action": action, + } + + on_missing_raw = cfg_uncond.get("on_missing", "error") + if not isinstance(on_missing_raw, str): + raise ValueError( + "method_config.cfg_uncond.on_missing must be a string, got " + f"{type(on_missing_raw).__name__}" + ) + on_missing = on_missing_raw.strip().lower() + if on_missing not in {"error", "ignore"}: + raise ValueError( + "method_config.cfg_uncond.on_missing must be one of {error, ignore}, got " + f"{on_missing_raw!r}" + ) + + supported_channels = {"image", "action"} + for channel, policy_raw in cfg_uncond.items(): + if channel in {"on_missing"}: + continue + if channel in supported_channels: + continue + if policy_raw is None: + continue + if not isinstance(policy_raw, str): + raise ValueError( + "method_config.cfg_uncond values must be strings, got " + f"{channel}={type(policy_raw).__name__}" + ) + policy = policy_raw.strip().lower() + if policy == "keep": + continue + if on_missing == "ignore": + continue + raise ValueError( + "WanGameAdapter does not support cfg_uncond channel " + f"{channel!r} (policy={policy!r}). " + "Set cfg_uncond.on_missing=ignore or remove the channel." + ) + + def _get_policy(channel: str) -> str: + raw = cfg_uncond.get(channel, "keep") + if raw is None: + return "keep" + if not isinstance(raw, str): + raise ValueError( + "method_config.cfg_uncond values must be strings, got " + f"{channel}={type(raw).__name__}" + ) + policy = raw.strip().lower() + if policy not in {"keep", "zero", "drop"}: + raise ValueError( + "method_config.cfg_uncond values must be one of {keep, zero, drop}, got " + f"{channel}={raw!r}" + ) + return policy + + image_policy = _get_policy("image") + if image_policy == "zero": + image_embeds = torch.zeros_like(image_embeds) + image_latents = torch.zeros_like(image_latents) + mask_lat_size = torch.zeros_like(mask_lat_size) + elif image_policy == "drop": + raise ValueError( + "cfg_uncond.image=drop is not supported for WanGame I2V; " + "use cfg_uncond.image=zero or keep." + ) + + action_policy = _get_policy("action") + if action_policy == "zero": + if viewmats is None or Ks is None or action is None: + if on_missing == "ignore": + pass + else: + raise ValueError( + "cfg_uncond.action=zero requires action conditioning tensors, " + "but TrainingBatch is missing {viewmats, Ks, action}." + ) + else: + viewmats = torch.zeros_like(viewmats) + Ks = torch.zeros_like(Ks) + action = torch.zeros_like(action) + elif action_policy == "drop": + viewmats = None + Ks = None + action = None + + return { + "image_embeds": image_embeds, + "image_latents": image_latents, + "mask_lat_size": mask_lat_size, + "viewmats": viewmats, + "Ks": Ks, + "action": action, + } + def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: transformer = handle.require_module("transformer") transformer_2 = handle.modules.get("transformer_2") @@ -445,9 +568,9 @@ def predict_x0( batch: TrainingBatch, *, conditional: bool, + cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: - del conditional device_type = self.device.type dtype = noisy_latents.dtype @@ -462,7 +585,21 @@ def predict_x0( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, batch) + cond_inputs = self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + ) + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs["image_embeds"], + image_latents=cond_inputs["image_latents"], + mask_lat_size=cond_inputs["mask_lat_size"], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + ) transformer = self._get_transformer(handle, timestep) pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) pred_x0 = pred_noise_to_pred_video( @@ -481,9 +618,9 @@ def predict_noise( batch: TrainingBatch, *, conditional: bool, + cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: - del conditional device_type = self.device.type dtype = noisy_latents.dtype @@ -498,7 +635,21 @@ def predict_noise( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, batch) + cond_inputs = self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + ) + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs["image_embeds"], + image_latents=cond_inputs["image_latents"], + mask_lat_size=cond_inputs["mask_lat_size"], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + ) transformer = self._get_transformer(handle, timestep) pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) return pred_noise diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 5ac676a0c..e7b99ed19 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -75,6 +75,7 @@ def predict_x0( batch: Any, *, conditional: bool, + cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: ... @@ -87,6 +88,7 @@ def predict_noise( batch: Any, *, conditional: bool, + cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: ... @@ -133,6 +135,7 @@ def __init__( self.validator = validator self.training_args = adapter.training_args self.method_config: dict[str, Any] = dict(method_config or {}) + self._cfg_uncond = self._parse_cfg_uncond() self._rollout_mode = self._parse_rollout_mode() self._denoising_step_list: torch.Tensor | None = None self._init_optimizers_and_schedulers() @@ -173,6 +176,59 @@ def _parse_rollout_mode(self) -> Literal["simulate", "data_latent"]: f"{raw!r}" ) + def _parse_cfg_uncond(self) -> dict[str, Any] | None: + raw = self.method_config.get("cfg_uncond", None) + if raw is None: + return None + if not isinstance(raw, dict): + raise ValueError( + "method_config.cfg_uncond must be a dict when set, got " + f"{type(raw).__name__}" + ) + + cfg: dict[str, Any] = dict(raw) + + on_missing_raw = cfg.get("on_missing", "error") + if on_missing_raw is None: + on_missing_raw = "error" + if not isinstance(on_missing_raw, str): + raise ValueError( + "method_config.cfg_uncond.on_missing must be a string, got " + f"{type(on_missing_raw).__name__}" + ) + on_missing = on_missing_raw.strip().lower() + if on_missing not in {"error", "ignore"}: + raise ValueError( + "method_config.cfg_uncond.on_missing must be one of " + "{error, ignore}, got " + f"{on_missing_raw!r}" + ) + cfg["on_missing"] = on_missing + + for channel, policy_raw in list(cfg.items()): + if channel == "on_missing": + continue + if policy_raw is None: + continue + if not isinstance(policy_raw, str): + raise ValueError( + "method_config.cfg_uncond values must be strings, got " + f"{channel}={type(policy_raw).__name__}" + ) + policy = policy_raw.strip().lower() + allowed = {"keep", "zero", "drop"} + if channel == "text": + allowed = {*allowed, "negative_prompt"} + if policy not in allowed: + raise ValueError( + "method_config.cfg_uncond values must be one of " + f"{sorted(allowed)}, got " + f"{channel}={policy_raw!r}" + ) + cfg[channel] = policy + + return cfg + def _build_role_optimizer_and_scheduler( self, *, @@ -390,6 +446,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: timestep, batch, conditional=True, + cfg_uncond=self._cfg_uncond, attn_kind="vsa", ) batch.dmd_latent_vis_dict["generator_timestep"] = timestep @@ -426,6 +483,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: current_timestep_tensor, batch, conditional=True, + cfg_uncond=self._cfg_uncond, attn_kind="vsa", ) @@ -455,6 +513,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: target_timestep, batch, conditional=True, + cfg_uncond=self._cfg_uncond, attn_kind="vsa", ) else: @@ -465,6 +524,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: target_timestep, batch, conditional=True, + cfg_uncond=self._cfg_uncond, attn_kind="vsa", ) @@ -498,6 +558,7 @@ def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dic fake_score_timestep, batch, conditional=True, + cfg_uncond=self._cfg_uncond, attn_kind="dense", ) target = noise - generator_pred_x0 @@ -543,6 +604,7 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor timestep, batch, conditional=True, + cfg_uncond=self._cfg_uncond, attn_kind="dense", ) real_cond_x0 = self.adapter.predict_x0( @@ -551,6 +613,7 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor timestep, batch, conditional=True, + cfg_uncond=self._cfg_uncond, attn_kind="dense", ) real_uncond_x0 = self.adapter.predict_x0( @@ -559,6 +622,7 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor timestep, batch, conditional=False, + cfg_uncond=self._cfg_uncond, attn_kind="dense", ) real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 456384b18..2af4165fd 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -16,6 +16,7 @@ from fastvideo.configs.sample import SamplingParam from fastvideo.dataset.validation_dataset import ValidationDataset from fastvideo.distributed import get_sp_group, get_world_group +from fastvideo.fastvideo_args import ExecutionMode from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch from fastvideo.distillation.validators.base import ValidationRequest @@ -236,9 +237,11 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) old_inference_mode = training_args.inference_mode old_dit_cpu_offload = training_args.dit_cpu_offload + old_mode = training_args.mode try: training_args.inference_mode = True training_args.dit_cpu_offload = True + training_args.mode = ExecutionMode.INFERENCE transformer.eval() num_sp_groups = self.world_group.world_size // self.sp_group.world_size @@ -292,5 +295,6 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) finally: training_args.inference_mode = old_inference_mode training_args.dit_cpu_offload = old_dit_cpu_offload + training_args.mode = old_mode if was_training: transformer.train() diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index 633e77416..f71a78097 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -16,6 +16,7 @@ from fastvideo.configs.sample import SamplingParam from fastvideo.dataset.validation_dataset import ValidationDataset from fastvideo.distributed import get_sp_group, get_world_group +from fastvideo.fastvideo_args import ExecutionMode from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch from fastvideo.distillation.validators.base import ValidationRequest @@ -268,9 +269,11 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) old_inference_mode = training_args.inference_mode old_dit_cpu_offload = training_args.dit_cpu_offload + old_mode = training_args.mode try: training_args.inference_mode = True training_args.dit_cpu_offload = True + training_args.mode = ExecutionMode.INFERENCE transformer.eval() num_sp_groups = self.world_group.world_size // self.sp_group.world_size @@ -322,6 +325,6 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) finally: training_args.inference_mode = old_inference_mode training_args.dit_cpu_offload = old_dit_cpu_offload + training_args.mode = old_mode if was_training: transformer.train() - From e556321c010b9231ccd77cd4352d0f15aab80008 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 26 Feb 2026 23:43:51 +0000 Subject: [PATCH 119/214] wangame i2v pipeline support ode/sde --- ...ame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml | 11 +- .../wangame/{finetune-temp.sh => run.sh} | 0 fastvideo/distillation/validators/wangame.py | 115 ++++++++++++------ .../basic/wan/wangame_i2v_pipeline.py | 67 ++++++---- 4 files changed, 132 insertions(+), 61 deletions(-) rename examples/distillation/wangame/{finetune-temp.sh => run.sh} (100%) diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml index ecbfadbcb..0fe4f245f 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml @@ -10,17 +10,16 @@ recipe: roles: student: family: wangame - path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true teacher: family: wangame - # TODO: replace with your teacher model path (wangame architecture). - path: YOUR_TEACHER_MODEL_PATH + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: false disable_custom_init_weights: true critic: family: wangame - path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true disable_custom_init_weights: true @@ -35,7 +34,7 @@ training: # Data (parquet dataset folder). # Supports comma-separated `path` or `path:N` (repeat count) entries. data_path: >- - /mnt/weka/home/hao.zhang/kaiqin/FastVideo/fastvideo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed + /mnt/weka/home/hao.zhang/kaiqin/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed dataloader_num_workers: 4 # Batch / shape @@ -98,7 +97,7 @@ method_config: # Few-step schedule (single source of truth for the method). # These are "step indices" and will be warped by the (shifted) scheduler. warp_denoising_step: true - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + dmd_denoising_steps: [1000,750,500,250] # Define CFG "uncond" semantics (operation-centric). # diff --git a/examples/distillation/wangame/finetune-temp.sh b/examples/distillation/wangame/run.sh similarity index 100% rename from examples/distillation/wangame/finetune-temp.sh rename to examples/distillation/wangame/run.sh diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index f71a78097..3862e7792 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -63,52 +63,98 @@ def __init__( def set_tracker(self, tracker: Any) -> None: self.tracker = tracker + def _post_process_validation_frames( + self, + frames: list[np.ndarray], + batch: ForwardBatch, + ) -> list[np.ndarray]: + """Optionally overlay action indicators on validation frames. + + Mirrors legacy `WanGameTrainingPipeline._post_process_validation_frames()`. + """ + + keyboard_cond = getattr(batch, "keyboard_cond", None) + mouse_cond = getattr(batch, "mouse_cond", None) + if keyboard_cond is None and mouse_cond is None: + return frames + + try: + from fastvideo.models.dits.matrixgame.utils import ( + draw_keys_on_frame, + draw_mouse_on_frame, + ) + except Exception as e: + logger.warning("WanGame action overlay is unavailable: %s", e) + return frames + + if keyboard_cond is not None and torch.is_tensor(keyboard_cond): + keyboard_np = keyboard_cond.squeeze(0).detach().cpu().float().numpy() + else: + keyboard_np = None + + if mouse_cond is not None and torch.is_tensor(mouse_cond): + mouse_np = mouse_cond.squeeze(0).detach().cpu().float().numpy() + else: + mouse_np = None + + # MatrixGame convention: keyboard [W, S, A, D, left, right], + # mouse [Pitch, Yaw]. + key_names = ["W", "S", "A", "D", "left", "right"] + + processed_frames: list[np.ndarray] = [] + for frame_idx, frame in enumerate(frames): + frame = np.ascontiguousarray(frame.copy()) + + if keyboard_np is not None and frame_idx < len(keyboard_np): + keys = { + key_names[i]: bool(keyboard_np[frame_idx, i]) + for i in range(min(len(key_names), int(keyboard_np.shape[1]))) + } + draw_keys_on_frame(frame, keys, mode="universal") + + if mouse_np is not None and frame_idx < len(mouse_np): + pitch = float(mouse_np[frame_idx, 0]) + yaw = float(mouse_np[frame_idx, 1]) + draw_mouse_on_frame(frame, pitch, yaw) + + processed_frames.append(frame) + + return processed_frames + def _get_sampling_param(self) -> SamplingParam: if self._sampling_param is None: self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) return self._sampling_param def _get_pipeline(self, *, transformer: torch.nn.Module, sampler_kind: str) -> Any: - key = (id(transformer), str(sampler_kind)) + sampler_kind = str(sampler_kind).lower() + key = (id(transformer), sampler_kind) if self._pipeline is not None and self._pipeline_key == key: return self._pipeline - flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) - - if str(sampler_kind).lower() in {"ode"}: - from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import ( - WanGameActionImageToVideoPipeline, + if sampler_kind not in {"ode", "sde"}: + raise ValueError( + f"Unknown sampler_kind for WanGame validation: {sampler_kind!r}" ) - self._pipeline = WanGameActionImageToVideoPipeline.from_pretrained( - self.training_args.model_path, - inference_mode=True, - flow_shift=float(flow_shift) if flow_shift is not None else None, - loaded_modules={"transformer": transformer}, - tp_size=self.training_args.tp_size, - sp_size=self.training_args.sp_size, - num_gpus=self.training_args.num_gpus, - pin_cpu_memory=self.training_args.pin_cpu_memory, - dit_cpu_offload=True, - ) - elif str(sampler_kind).lower() in {"sde"}: - from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( - WanGameCausalDMDPipeline, - ) + flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) - self._pipeline = WanGameCausalDMDPipeline.from_pretrained( - self.training_args.model_path, - inference_mode=True, - flow_shift=float(flow_shift) if flow_shift is not None else None, - loaded_modules={"transformer": transformer}, - tp_size=self.training_args.tp_size, - sp_size=self.training_args.sp_size, - num_gpus=self.training_args.num_gpus, - pin_cpu_memory=self.training_args.pin_cpu_memory, - dit_cpu_offload=True, - ) - else: - raise ValueError(f"Unknown sampler_kind for WanGame validation: {sampler_kind!r}") + from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import ( + WanGameActionImageToVideoPipeline, + ) + + self._pipeline = WanGameActionImageToVideoPipeline.from_pretrained( + self.training_args.model_path, + inference_mode=True, + flow_shift=float(flow_shift) if flow_shift is not None else None, + sampler_kind=sampler_kind, + loaded_modules={"transformer": transformer}, + tp_size=self.training_args.tp_size, + sp_size=self.training_args.sp_size, + num_gpus=self.training_args.num_gpus, + pin_cpu_memory=self.training_args.pin_cpu_memory, + dit_cpu_offload=True, + ) self._pipeline_key = key return self._pipeline @@ -231,6 +277,7 @@ def _run_validation_for_steps( x = torchvision.utils.make_grid(x, nrow=6) x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) frames.append((x * 255).numpy().astype(np.uint8)) + frames = self._post_process_validation_frames(frames, batch) videos.append(frames) return _ValidationStepResult(videos=videos, captions=captions) diff --git a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py index cd7e53d4d..fd77540a6 100644 --- a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -""" -Wan video diffusion pipeline implementation. +"""WanGame image-to-video pipeline implementation. -This module contains an implementation of the Wan video diffusion pipeline +This module contains an implementation of the WanGame image-to-video pipeline using the modular pipeline architecture. """ @@ -10,15 +9,26 @@ from fastvideo.logger import init_logger from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase from fastvideo.pipelines.lora_pipeline import LoRAPipeline +from fastvideo.pipelines.samplers.wan import ( + build_wan_scheduler, + get_wan_sampler_kind, + wan_use_btchw_layout, +) # isort: off from fastvideo.pipelines.stages import ( - ImageEncodingStage, ConditioningStage, DecodingStage, DenoisingStage, - ImageVAEEncodingStage, InputValidationStage, LatentPreparationStage, - TimestepPreparationStage) + ConditioningStage, + DecodingStage, + DenoisingStage, + ImageEncodingStage, + ImageVAEEncodingStage, + InputValidationStage, + LatentPreparationStage, + SdeDenoisingStage, + TimestepPreparationStage, +) + # isort: on -from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( - FlowUniPCMultistepScheduler) logger = init_logger(__name__) @@ -26,17 +36,23 @@ class WanGameActionImageToVideoPipeline(LoRAPipeline, ComposedPipelineBase): _required_config_modules = [ - "vae", "transformer", "scheduler", \ - "image_encoder", "image_processor" + "vae", + "transformer", + "scheduler", + "image_encoder", + "image_processor", ] def initialize_pipeline(self, fastvideo_args: FastVideoArgs): - self.modules["scheduler"] = FlowUniPCMultistepScheduler( - shift=fastvideo_args.pipeline_config.flow_shift) + sampler_kind = get_wan_sampler_kind(fastvideo_args) + self.modules["scheduler"] = build_wan_scheduler(fastvideo_args, sampler_kind) def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): """Set up pipeline stages with proper dependency injection.""" + sampler_kind = get_wan_sampler_kind(fastvideo_args) + use_btchw_layout = wan_use_btchw_layout(sampler_kind) + self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) @@ -45,27 +61,36 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): stage=ImageEncodingStage( image_encoder=self.get_module("image_encoder"), image_processor=self.get_module("image_processor"), - )) + ), + ) self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) - self.add_stage(stage_name="timestep_preparation_stage", - stage=TimestepPreparationStage( - scheduler=self.get_module("scheduler"))) + if sampler_kind == "ode": + self.add_stage(stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"))) self.add_stage(stage_name="latent_preparation_stage", stage=LatentPreparationStage( scheduler=self.get_module("scheduler"), - transformer=self.get_module("transformer"))) + transformer=self.get_module("transformer"), + use_btchw_layout=use_btchw_layout)) self.add_stage(stage_name="image_latent_preparation_stage", stage=ImageVAEEncodingStage(vae=self.get_module("vae"))) - self.add_stage(stage_name="denoising_stage", - stage=DenoisingStage( - transformer=self.get_module("transformer"), - scheduler=self.get_module("scheduler"))) + if sampler_kind == "sde": + self.add_stage(stage_name="denoising_stage", + stage=SdeDenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"))) + else: + self.add_stage(stage_name="denoising_stage", + stage=DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"))) self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae"))) From ed4f63659f7ee771954ac8fa4a7d9c0b07deeb8b Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 27 Feb 2026 00:04:57 +0000 Subject: [PATCH 120/214] sde denoising stage --- ...ame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml | 2 +- fastvideo/pipelines/stages/denoising.py | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml index 0fe4f245f..8f633231d 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml @@ -86,7 +86,7 @@ training: validation_guidance_scale: "1.0" pipeline_config: - flow_shift: 3 + flow_shift: 5 sampler_kind: sde method_config: diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index 00ebf1186..52a283196 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -1241,6 +1241,39 @@ def forward( }, ) + if batch.mouse_cond is not None and batch.keyboard_cond is not None: + from fastvideo.models.dits.hyworld.pose import process_custom_actions + + viewmats, intrinsics, action_labels = process_custom_actions( + batch.keyboard_cond, batch.mouse_cond + ) + camera_action_kwargs = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + "viewmats": viewmats.unsqueeze(0).to( + get_local_torch_device(), dtype=target_dtype + ), + "Ks": intrinsics.unsqueeze(0).to( + get_local_torch_device(), dtype=target_dtype + ), + "action": action_labels.unsqueeze(0).to( + get_local_torch_device(), dtype=target_dtype + ), + }, + ) + else: + camera_action_kwargs = {} + + action_kwargs = self.prepare_extra_func_kwargs( + self.transformer.forward, + { + "mouse_cond": batch.mouse_cond, + "keyboard_cond": batch.keyboard_cond, + "c2ws_plucker_emb": batch.c2ws_plucker_emb, + }, + ) + + # Get latents and embeddings assert batch.latents is not None, "latents must be provided" latents = batch.latents @@ -1347,6 +1380,8 @@ def forward( guidance=guidance_expand, **image_kwargs, **pos_cond_kwargs, + **action_kwargs, + **camera_action_kwargs, ).permute(0, 2, 1, 3, 4) pred_video = pred_noise_to_pred_video( From e9942727bc0753dbcb014fd6ef8bb274affe70a9 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 27 Feb 2026 00:26:32 +0000 Subject: [PATCH 121/214] action cfg vs no cfg --- ...ame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml | 2 +- ...wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml | 113 ++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml index 8f633231d..75970c774 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml @@ -78,7 +78,7 @@ training: # Tracking / validation tracker_project_name: distillation_wangame - wandb_run_name: wangame_dmd2_4steps_distill + wandb_run_name: wangame_dmd2_4steps_distill_actioncfg log_validation: true validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json validation_steps: 50 diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml new file mode 100644 index 000000000..a72148759 --- /dev/null +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml @@ -0,0 +1,113 @@ +# WanGame DMD2 distillation (few-step rollout). +# +# - Fill in `roles.teacher.path` with your teacher checkpoint (must be wangame family). +# - This YAML is the single source of truth for the run. + +recipe: + family: wangame + method: dmd2 + +roles: + student: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + teacher: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: false + disable_custom_init_weights: true + critic: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + disable_custom_init_weights: true + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder). + # Supports comma-separated `path` or `path:N` (repeat count) entries. + data_path: >- + /mnt/weka/home/hao.zhang/kaiqin/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_dmd2_8steps_distill + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer (student) + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Optimizer (critic / fake-score) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation (method-agnostic knobs) + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_dmd2_4steps_distill_nocfg + log_validation: true + validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + validation_steps: 50 + validation_sampling_steps: "4" + validation_guidance_scale: "1.0" + +pipeline_config: + flow_shift: 5 + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + + # Few-step schedule (single source of truth for the method). + # These are "step indices" and will be warped by the (shifted) scheduler. + warp_denoising_step: true + dmd_denoising_steps: [1000,750,500,250] + + # Define CFG "uncond" semantics (operation-centric). + # + # If all channels are `keep`, then uncond == cond, so CFG becomes a no-op: + # real_cfg = cond + (cond - uncond) * scale == cond + cfg_uncond: + on_missing: error + # For I2V+action, a practical default is "no-action baseline". + action: keep + # Keep the reference image conditioning for I2V. + image: keep + # Text is unused in current wangame training batches, but kept for schema symmetry. + text: keep From f036db3b2de1ab436a62a99b79b819d62dbcddc1 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 27 Feb 2026 00:43:40 +0000 Subject: [PATCH 122/214] designing causal wangame and dfsft --- dev/phase_add_causal_wangame_dfsft.md | 198 ++++++++++++++++++ .../wangame/finetune_wangame2.1_i2v_1.3B.yaml | 4 +- 2 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 dev/phase_add_causal_wangame_dfsft.md diff --git a/dev/phase_add_causal_wangame_dfsft.md b/dev/phase_add_causal_wangame_dfsft.md new file mode 100644 index 000000000..7dfea369d --- /dev/null +++ b/dev/phase_add_causal_wangame_dfsft.md @@ -0,0 +1,198 @@ +# Phase: Add Causal WanGame + Diffusion-Forcing SFT (DF-SFT) + +> 目标：在现有 distillation framework 上，新增一种 **causal Wangame** 的 +> supervised finetuning 方法（DFSFT），用于把 Wangame 从「双向 / bidirectional」 +> 的训练范式，迁移到「因果 / causal」范式。 +> +> 参考实现：FastGen 的 `CausalSFTModel`（diffusion forcing for SFT）。 + +--- + +## 0. 背景与动机 + +我们已经把 Wangame 的 `finetune` 与 `dmd2` 跑通。 +下一步要做 **bidirectional -> causal**。 + +我建议先落地一个 **Diffusion-Forcing SFT**（DFSFT）baseline： + +- 仅训练 `student`（SFT/DSM loss，和 FastGen 对齐）； +- 使用 **block-wise inhomogeneous timesteps**（`t_inhom`，按 chunk 采样）， + 让 causal student 在训练时就面对“历史上下文不一定是干净的”的分布； +- 不引入 teacher/critic 依赖，降低第一版风险。 + +> 这不是“few-step distill”。它是“训练一个 causal 的基础模型”。 +> 如果后面要把 causal Wangame distill 成 4/8 steps，再做 CausVid/DMD2 +> diffusion-forcing distillation 更合适。 + +--- + +## 1. 本阶段产物（Deliverables） + +- [ ] **Model 侧**：Wangame 支持 `causal` 变体（通过 role 的 `extra` 参数触发） +- [ ] **Method 侧**：新增 `dfsft`（diffusion-forcing SFT）方法 +- [ ] **Examples**：新增一份 DFSFT 的 YAML + temp.sh（端到端可跑） +- [ ] **Validation**：沿用现有 validator，能够用 `validation_sampling_steps=40` + 做验证（ODE 或 SDE 均可，默认用 ODE） + +--- + +## 2. 配置语义（Config） + +### 2.1 Causal variant（Role extra） + +不新增新的 family（避免 `wangame_causal` 这种对外语义膨胀）。 +仍然是： + +```yaml +roles: + student: + family: wangame + path: ... + trainable: true + # extra fields (RoleSpec.extra) + variant: causal + # （可选）更细粒度的 causal invariant：用于表达‘是哪一种因果约束/训练范式’ + # 例如：strict / block / sliding_window / bidirectional_train_causal_eval ... + causal_invariant: block +``` + +- `variant: causal` 由 `models/wangame` 插件解释。 +- 未来如果需要更细粒度，可扩展为： + - `variant: causal|bidirectional` + - `causal: true|false` + - `num_frames_per_block` / `sliding_window_num_frames`（可选） + +### 2.2 DFSFT method config（与 FastGen 对齐） + +推荐：把 DFSFT 的关键 knob 放到 `method_config`： + +```yaml +recipe: + family: wangame + method: dfsft + +method_config: + # diffusion-forcing (SFT) 核心：按 chunk 采样 inhomogeneous t + chunk_size: 3 + + # t sampling（可以复用我们已有的 min/max ratio 语义；最终落到 [0,1000]） + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + + # 可选：更接近“history noisy cache”的效果（第一版可以先不做） + context_noise: 0.0 +``` + +说明： +- `chunk_size`：决定 `t_inhom` 的 block 划分（FastGen 也用 chunk_size）。 + - 对 Wang/Wangame，建议默认 3（与 `num_frames_per_block` 一致）。 +- `context_noise`：未来如果我们实现“prefill cache 前对历史 x0 加噪”， + 这个值将用于控制历史噪声强度。 + +--- + +## 3. 训练逻辑（DFSFT 的算法定义） + +目标：对齐 FastGen `CausalSFTModel`（`fastgen/methods/fine_tuning/sft.py`）。 + +核心步骤（单 step）： + +1) 取真实数据 `x0`（video latents）。 +2) 采样 `eps_inhom ~ N(0, I)`。 +3) 采样 `t_inhom`：形状 **[B, T_lat]**，按 chunk/block-wise 采样，chunk 内 + timestep 相同。 +4) 前向扩散：`x_t = add_noise(x0, eps_inhom, t_inhom)` +5) 学生预测：`pred = student(x_t, t_inhom, cond)`（预测 noise/x0/v，取决于 + adapter/model 的 pred_type） +6) DSM loss：对齐噪声调度器语义（最简单是 MSE(pred_eps, eps_inhom)）。 + +关键点： +- DFSFT 不需要 teacher。 +- “diffusion forcing”体现在 `t_inhom`（按 chunk 的独立噪声水平），而不是 + 直接对 KV tensor 加噪。 + +--- + +## 4. 代码改动清单（按文件） + +### 4.1 models（Wangame causal variant） + +- [ ] `fastvideo/distillation/models/wangame.py` + - 读取 `role_spec.extra.get("variant")`（或 `causal: true`） + - 当 `variant == "causal"`：加载 transformer 时覆盖 cls 为 + `CausalWanGameTransformer3DModel`（FastVideo 已存在该类： + `fastvideo/models/dits/wangame/causal_model.py`） + - 目标：**同一份 ckpt 既可作为 bidirectional student，也可作为 causal + student 初始化**（如果 state_dict 不兼容，需要记录为风险点并加 fallback）。 + +> 备注：如果实现细节需要拆文件，可以内部新增 +> `fastvideo/distillation/models/wangame/causal.py`，但对外 family +> 仍然是 `wangame`。 + +### 4.2 methods（新增 dfsft） + +- [ ] `fastvideo/distillation/methods/fine_tuning/dfsft.py`（新增） + - `@register_method("dfsft")` + - 仅依赖 `roles.student` + - `single_train_step()`：实现第 3 节 DFSFT + - 复用现有 finetune 的 optimizer/lr scheduler wiring + +- [ ] `fastvideo/distillation/methods/__init__.py` + - 暴露/导入新方法（取决于我们当前 registry/dispatch 的约定） + +- [ ] （可能需要）`fastvideo/distillation/adapters/wangame.py` + - 确认 `predict_noise/add_noise` 支持 `timestep` 为 **[B, T_lat]** + - 如果当前只支持 [B]，需要扩展并加形状检查。 + +### 4.3 examples（端到端验证） + +- [ ] `examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_dfsft_causal.yaml` + - `roles.student.variant: causal` + - `recipe.method: dfsft` + - `training.validation_sampling_steps: "40"` + +- [ ] `examples/distillation/wangame/dfsft-temp.sh`（新增） + - 跟现在 `run.sh` 一样只负责 export CONFIG + torchrun + +--- + +## 5. 验收标准（Definition of Done） + +- [ ] DFSFT 端到端可跑（不需要 teacher/critic） +- [ ] step0 validation 能出视频，不 crash +- [ ] 训练若干步后，validation 质量有可见提升 +- [ ] 同一份 wangame checkpoint：bidirectional finetune 和 causal dfsft + 都能启动（若 causal 需要不同 ckpt，要明确写在配置/README） + +--- + +## 6. 风险点 / 需要提前确认的问题 + +1) **权重兼容**：`CausalWanGameTransformer3DModel` 是否能直接 load + bidirectional wangame 的 transformer 权重。 + - 如果不能：需要一个 conversion 逻辑（或要求 user 提供 causal init ckpt）。 + +2) **t_inhom 的 shape 语义**： + - Wangame transformer 是否真正支持 [B, T_lat] timesteps； + - scheduler.add_noise 是否支持 per-frame timesteps（不支持就需要 reshape + 或 per-frame add_noise）。 + +3) **chunk_size 与模型结构对齐**： + - DFSFT 的 chunk_size 是否必须等于模型的 `num_frames_per_block`； + - 如果用户配错，建议直接 error。 + +4) **“40-step causal” 的含义**： + - DFSFT 训练的是基础模型；推理时可以设 `num_inference_steps=40`。 + - 但“few-step（4/8）”仍需要 distillation（DMD2/CM/CausVid）。 + +--- + +## 7. FastGen 对照（便于后续实现 CausVid） + +- DFSFT (SFT + diffusion forcing)： + - `fastgen/methods/fine_tuning/sft.py::CausalSFTModel` + - `fastgen/networks/noise_schedule.py::sample_t_inhom_sft` + +- Diffusion-forcing distillation（未来）： + - `fastgen/methods/distribution_matching/causvid.py::CausVidModel` + diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml index 62077be6a..5cc710360 100644 --- a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml +++ b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml @@ -5,7 +5,7 @@ recipe: roles: student: family: wangame - path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true training: @@ -65,7 +65,7 @@ training: # Tracking / validation tracker_project_name: distillation_wangame - wandb_run_name: wangame_finetune + wandb_run_name: wangame_finetune_from_pretrained log_validation: true validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json validation_steps: 100 From 91f260c5fac95517bcfde8d8e1fc37f3662498bb Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 01:02:31 +0000 Subject: [PATCH 123/214] designing dfsft --- dev/phase_add_causal_wangame_dfsft.md | 133 ++++++++++++++++++++++++-- 1 file changed, 127 insertions(+), 6 deletions(-) diff --git a/dev/phase_add_causal_wangame_dfsft.md b/dev/phase_add_causal_wangame_dfsft.md index 7dfea369d..755f922fb 100644 --- a/dev/phase_add_causal_wangame_dfsft.md +++ b/dev/phase_add_causal_wangame_dfsft.md @@ -31,8 +31,14 @@ - [ ] **Model 侧**：Wangame 支持 `causal` 变体（通过 role 的 `extra` 参数触发） - [ ] **Method 侧**：新增 `dfsft`（diffusion-forcing SFT）方法 - [ ] **Examples**：新增一份 DFSFT 的 YAML + temp.sh（端到端可跑） -- [ ] **Validation**：沿用现有 validator，能够用 `validation_sampling_steps=40` - 做验证（ODE 或 SDE 均可，默认用 ODE） +- [ ] **Validation（关键变更）**：支持“**真正的 causal rollout**”验证，并支持 + **context noise**： + - causal rollout 指 **streaming / chunk-wise** 生成（KV-cache + block processing）， + 而不是“用 causal transformer 但仍一次性 full-video forward”； + - **目标是 ODE-style 的 causal streaming**（用 `scheduler.step(...)` 做数值推进）， + 而不是 DMD/SDE-style 的 `pred_x0 -> add_noise(next_t, eps)` rollout； + - 若要同时保留 SDE-style（用于对照/legacy 对齐），应当由 config 显式选择， + 而不是在 validator 里隐式“偷换语义”。 --- @@ -80,14 +86,68 @@ method_config: max_timestep_ratio: 0.98 # 可选：更接近“history noisy cache”的效果（第一版可以先不做） - context_noise: 0.0 + context_noise: 0 ``` 说明： - `chunk_size`：决定 `t_inhom` 的 block 划分（FastGen 也用 chunk_size）。 - 对 Wang/Wangame，建议默认 3（与 `num_frames_per_block` 一致）。 -- `context_noise`：未来如果我们实现“prefill cache 前对历史 x0 加噪”， - 这个值将用于控制历史噪声强度。 +- `context_noise`：**context noise timestep**（整型）。用于 causal rollout 时 + “更新 KV cache 的上下文噪声水平”（见第 4 节 Validation 设计）。 + +### 2.3 Validation config（真正的 causal rollout + context noise） + +我们需要区分两种 validation 语义： + +1) **full-video rollout**（非 streaming）：`WanGameActionImageToVideoPipeline.forward(...)` +2) **streaming causal rollout**（推荐用于 causal student 的验证）：`WanGameCausalDMDPipeline.streaming_*` + +建议把选择权交给 method（或 method_config），让 validator 只负责执行： + +```yaml +method_config: + validation: + # full | streaming_causal + rollout_mode: streaming_causal + + # 对 full rollout：ode/sde 选择 sampling loop + # 对 streaming_causal：同样允许 ode/sde，但这是一个“明确的语义选择” + sampler_kind: ode + + # ODE: 直接用标准 `num_inference_steps`（与现有 ODE pipeline 对齐） + # （可以复用 training.validation_sampling_steps 的整数；这里单独列出来是为了更清晰） + num_inference_steps: 40 + + # SDE/DMD: 需要显式 step list（few-step schedule） + # sampling_timesteps: [1000,750,500,250] + # warp_denoising_step: true + + # causal cache 的 context noise（timestep） + context_noise: 0 +``` + +> 备注： +> - 现阶段的 `WanGameCausalDMDPipeline / MatrixGameCausalDenoisingStage` 是 DMD/SDE-style； +> 要实现 **ODE-style streaming**，需要新增一个 causal ODE denoising stage（见第 4 节）。 +> - 我们仍可以保留 SDE-style streaming（用于对照/legacy 对齐），但必须由 config 显式选择， +> 避免 “training/validation 语义混用导致 reviewer 困惑”。 + +### 2.4 pipeline_config：ODE solver 选择（可选） + +我们目前的 `pipeline_config.sampler_kind=ode` 默认会选择 `FlowUniPCMultistepScheduler` +（见 `fastvideo/pipelines/samplers/wan.py`）。为了做对照实验/调试，建议增加一个可选字段： + +```yaml +pipeline_config: + sampler_kind: ode + ode_solver: unipc # unipc | euler +``` + +约束（推荐强校验）： +- `sampler_kind=sde` 时不允许 `ode_solver=unipc`（因为 SDE/DMD-style rollout 需要 + `add_noise(next_t, eps)`；UniPC 的 `add_noise` 对 “任意 timestep 值” 不鲁棒）。 +- `ode_solver=euler` 时应强制 deterministic（`stochastic_sampling=false`），否则就变成 + “SDE-like Euler”。 --- @@ -154,6 +214,37 @@ method_config: - [ ] `examples/distillation/wangame/dfsft-temp.sh`（新增） - 跟现在 `run.sh` 一样只负责 export CONFIG + torchrun +### 4.4 validation（真正的 causal rollout） + +> 这是本阶段新增的关键设计点：**不要**默认复用“full-video validator”来验证 causal 模型。 + +- [ ] `fastvideo/distillation/validators/wangame.py`（或新增 `wangame_causal.py`） + - 支持 `rollout_mode: streaming_causal`： + - pipeline：**需要 ODE-style 的 causal streaming pipeline** + - 方案 A（推荐）：扩展/新增 `WanGameCausalPipeline`（同一条 pipeline），内部按 + `sampler_kind={ode|sde}` 选择不同 denoising stage + - 方案 B（过渡）：保留 `WanGameCausalDMDPipeline`（仅 SDE），但这不满足本阶段目标 + - ODE-style streaming 的调用方式（与现有 streaming API 对齐）： + 1) `pipeline.streaming_reset(batch, fastvideo_args)` + 2) 循环 `pipeline.streaming_step(...)` 直到生成完成 + 3) 聚合每个 chunk 的 frames，拼成完整 video 再落盘/上报 tracker + - 支持 `context_noise`（对齐 legacy self-forcing 语义）： + - cache update 前对 context latent 做一次显式 `scheduler.add_noise(x0, eps, t_context)` + 再 forward 更新 KV cache（避免 “只改 timestep embedding，效果像没开”） + +- [ ] `fastvideo/pipelines/stages/`（新增 causal ODE denoising stage） + - `MatrixGameCausalOdeDenoisingStage`（或同等命名） + - block/chunk 框架与 `MatrixGameCausalDenoisingStage` 一致（KV-cache + action） + - block 内 loop 使用 `scheduler.step(...)`（ODE loop） + - **每个 block 必须 reset solver state**（见风险点：UniPC 多步历史不能跨 block 泄漏） + +- [ ] `fastvideo/distillation/validators/base.py` + - 若需要：扩展 `ValidationRequest` 以携带 + - `rollout_mode` + - `context_noise` + - `sampling_timesteps`（已有） + - 目标：把“验证用什么 pipeline/rollout”的决策交给 method。 + --- ## 5. 验收标准（Definition of Done） @@ -163,6 +254,7 @@ method_config: - [ ] 训练若干步后，validation 质量有可见提升 - [ ] 同一份 wangame checkpoint：bidirectional finetune 和 causal dfsft 都能启动（若 causal 需要不同 ckpt，要明确写在配置/README） +- [ ] 支持用 streaming causal rollout 做验证（并能开启/关闭 context noise） --- @@ -185,6 +277,36 @@ method_config: - DFSFT 训练的是基础模型；推理时可以设 `num_inference_steps=40`。 - 但“few-step（4/8）”仍需要 distillation（DMD2/CM/CausVid）。 +5) **“真正 causal rollout” vs “full-video rollout”**（重要决策点）： + - 如果我们只用 `WanGameActionImageToVideoPipeline.forward(...)` 做验证， + 这并不能覆盖 deployment 的 streaming/KV-cache 语义； + - 但 streaming rollout 目前依赖 `WanGameCausalDMDPipeline`（DMD/SDE-style step list）， + 因此需要明确： + - DFSFT baseline 先用 streaming + step list 验证（更贴近 causal 部署）； + - 或者额外实现一个 “streaming ODE” sampler（更大工程，建议后置）。 + +6) **step list / context noise 的配置入口**（重要决策点）： + - 现有 streaming pipeline 读取 `pipeline_config.dmd_denoising_steps / warp_denoising_step / context_noise`； + - 我们的新框架更希望把 few-step schedule/rollout knobs 放到 `method_config`； + - 因此需要一个明确策略： + - 方案 A（最小改动）：validator 在构建 validation pipeline 时，把 + `method_config.validation.*` 写入 `args_copy.pipeline_config.*`（validation-only）； + - 方案 B（更干净）：把 streaming pipeline 改为优先读 `ForwardBatch.sampling_timesteps` + + `ValidationRequest.context_noise`，彻底摆脱 pipeline_config 依赖（工程量更大）。 + +7) **context noise 的“语义一致性”**（潜在坑）： + - legacy self-forcing 里 context noise 是： + `x0 -> add_noise(x0, eps, context_timestep)` 再更新 cache； + - 现有 `MatrixGameCausalDenoisingStage._update_context_cache` 只把 timestep 传给 transformer， + 是否也应显式 add_noise（以匹配语义）需要确认，否则“开了 context noise 但效果像没开”。 + +8) **ODE solver 的 state reset（新增风险点，必须明确）**： + - `FlowUniPCMultistepScheduler` 是 multistep solver，内部有历史状态； + - 在 streaming causal rollout 中，**每个 block/chunk** 都应当像独立的 ODE 求解过程： + - 要么每个 block 调用一次 `scheduler.set_timesteps(...)`（会清空 `_step_index` 和历史） + - 要么为每个 block 构建新的 scheduler 实例 + - 否则会出现 solver 历史跨 block 泄漏，导致质量漂移且很难定位。 + --- ## 7. FastGen 对照（便于后续实现 CausVid） @@ -195,4 +317,3 @@ method_config: - Diffusion-forcing distillation（未来）： - `fastgen/methods/distribution_matching/causvid.py::CausVidModel` - From c7334a28d4af28aaa137ada98068c7450fc9da4a Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 01:17:09 +0000 Subject: [PATCH 124/214] validation config refine --- dev/config.md | 14 +++ dev/phase_add_causal_wangame_dfsft.md | 15 ++- fastvideo/distillation/doc/utils/config.md | 6 ++ .../methods/distribution_matching/dmd2.py | 97 +++++++++++++++++-- .../methods/fine_tuning/finetune.py | 85 ++++++++++++++-- 5 files changed, 194 insertions(+), 23 deletions(-) diff --git a/dev/config.md b/dev/config.md index a043be69e..39a1a1fbe 100644 --- a/dev/config.md +++ b/dev/config.md @@ -96,6 +96,20 @@ loader 会注入/补全的 invariants（见 `fastvideo/distillation/utils/config - 分布式尺寸默认值（`num_gpus/tp_size/sp_size/hsdp_*`） - `training.model_path` 若缺失，默认使用 `roles.student.path`（供 pipeline_config registry 使用） +关于 validation 参数的归属（推荐约定）： +- **run/trainer 级**（“什么时候验证/用什么验证集”）放在 `training:`： + - `training.log_validation` + - `training.validation_dataset_file` + - `training.validation_steps` +- **method 级**（“怎么采样/走什么 rollout 语义”）放在 `method_config.validation:`： + - `sampling_steps`（ODE 下就是 `num_inference_steps`） + - `guidance_scale` + - `sampler_kind` / `sampling_timesteps` / `rollout_mode` / `context_noise`（按 method 需要） + +备注： +- 目前仍允许从 `training.validation_sampling_steps / validation_guidance_scale` 读取默认值， + 但更推荐写到 `method_config.validation`，避免把 method 语义塞到 `TrainingArgs` 里。 + ## 6) `pipeline_config` / `pipeline_config_path` 两种写法（二选一）： diff --git a/dev/phase_add_causal_wangame_dfsft.md b/dev/phase_add_causal_wangame_dfsft.md index 755f922fb..01bf94007 100644 --- a/dev/phase_add_causal_wangame_dfsft.md +++ b/dev/phase_add_causal_wangame_dfsft.md @@ -102,6 +102,13 @@ method_config: 1) **full-video rollout**（非 streaming）：`WanGameActionImageToVideoPipeline.forward(...)` 2) **streaming causal rollout**（推荐用于 causal student 的验证）：`WanGameCausalDMDPipeline.streaming_*` +建议把 validation 的配置拆成两类： + +- **run/trainer 级（什么时候验证、用什么 validation dataset）**：放在 `training:` 下； +- **method 级（怎么采样、走什么 rollout 语义）**：放在 `method_config.validation:` 下。 + +这样 validator 不需要 “从 training_args 猜 method 语义”，而 method 也不会越权去管理 dataloader。 + 建议把选择权交给 method（或 method_config），让 validator 只负责执行： ```yaml @@ -114,9 +121,8 @@ method_config: # 对 streaming_causal：同样允许 ode/sde，但这是一个“明确的语义选择” sampler_kind: ode - # ODE: 直接用标准 `num_inference_steps`（与现有 ODE pipeline 对齐） - # （可以复用 training.validation_sampling_steps 的整数；这里单独列出来是为了更清晰） - num_inference_steps: 40 + # 采样步数（对 ODE pipeline 即 num_inference_steps） + sampling_steps: [40] # SDE/DMD: 需要显式 step list（few-step schedule） # sampling_timesteps: [1000,750,500,250] @@ -209,7 +215,8 @@ pipeline_config: - [ ] `examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_dfsft_causal.yaml` - `roles.student.variant: causal` - `recipe.method: dfsft` - - `training.validation_sampling_steps: "40"` + - `training.validation_dataset_file / training.validation_steps`（run 级） + - `method_config.validation.sampling_steps: [40]`（method 级） - [ ] `examples/distillation/wangame/dfsft-temp.sh`（新增） - 跟现在 `run.sh` 一样只负责 export CONFIG + torchrun diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index acb351fcc..699e13b12 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -30,6 +30,12 @@ - `pipeline_config` 或 `pipeline_config_path` - `method_config: {...}`（算法/recipe 专属超参） +**Validation 参数建议归属** +- run/trainer 级（什么时候验证 / 用什么验证集）放 `training:`： + - `log_validation`, `validation_dataset_file`, `validation_steps` +- method 级（怎么采样 / rollout 语义）放 `method_config.validation:`： + - `sampling_steps`, `guidance_scale`, `sampler_kind`, `sampling_timesteps`, ... + **实现要点** - `_resolve_existing_file()`：要求传入真实存在的路径（不做 overlay/fallback） - 默认分布式 size： diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index e7b99ed19..2638a3371 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any, Literal, Protocol +from typing import Any, cast, Literal, Protocol import torch import torch.nn.functional as F @@ -307,6 +307,61 @@ def _init_optimizers_and_schedulers(self) -> None: def on_train_start(self) -> None: self.adapter.on_train_start() + def _parse_validation_cfg(self) -> dict[str, Any]: + raw = self.method_config.get("validation", None) + if raw is None: + return {} + if not isinstance(raw, dict): + raise ValueError( + "method_config.validation must be a dict when set, got " + f"{type(raw).__name__}" + ) + return dict(raw) + + def _parse_validation_sampling_steps(self, cfg: dict[str, Any]) -> list[int]: + raw = cfg.get("sampling_steps") + if raw is None: + raw = getattr(self.training_args, "validation_sampling_steps", "") or "" + + steps: list[int] = [] + if raw is None or raw == "": + return steps + if isinstance(raw, bool): + raise ValueError( + "validation sampling_steps must be an int/list/str, got bool" + ) + if isinstance(raw, int) or (isinstance(raw, float) and raw.is_integer()): + steps = [int(raw)] + elif isinstance(raw, str): + steps = [int(s) for s in raw.split(",") if str(s).strip()] + elif isinstance(raw, list): + steps = [int(s) for s in raw] + else: + raise ValueError( + "validation sampling_steps must be an int/list/str, got " + f"{type(raw).__name__}" + ) + return [s for s in steps if int(s) > 0] + + def _parse_validation_guidance_scale(self, cfg: dict[str, Any]) -> float | None: + raw = cfg.get("guidance_scale") + if raw is None: + raw = getattr(self.training_args, "validation_guidance_scale", None) + if raw in (None, ""): + return None + if isinstance(raw, bool): + raise ValueError( + "validation guidance_scale must be a number/string, got bool" + ) + if isinstance(raw, (int, float)): + return float(raw) + if isinstance(raw, str) and raw.strip(): + return float(raw) + raise ValueError( + "validation guidance_scale must be a number/string, got " + f"{type(raw).__name__}" + ) + def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: @@ -314,14 +369,16 @@ def log_validation(self, iteration: int) -> None: if not getattr(self.training_args, "log_validation", False): return - raw_steps = str(getattr(self.training_args, "validation_sampling_steps", "") or "") - sampling_steps = [int(s) for s in raw_steps.split(",") if s.strip()] - sampling_steps = [s for s in sampling_steps if s > 0] + validation_cfg = self._parse_validation_cfg() + + sampling_steps = self._parse_validation_sampling_steps(validation_cfg) - raw_rollout = self.method_config.get("dmd_denoising_steps", None) sampling_timesteps: list[int] | None = None - if isinstance(raw_rollout, list) and raw_rollout: - sampling_timesteps = [int(s) for s in raw_rollout] + raw_timesteps = validation_cfg.get("sampling_timesteps", None) + if raw_timesteps is None: + raw_timesteps = self.method_config.get("dmd_denoising_steps", None) + if isinstance(raw_timesteps, list) and raw_timesteps: + sampling_timesteps = [int(s) for s in raw_timesteps] if not sampling_steps: # Default to the few-step student rollout step count for DMD2. @@ -329,13 +386,33 @@ def log_validation(self, iteration: int) -> None: return sampling_steps = [int(len(sampling_timesteps))] - raw_guidance = getattr(self.training_args, "validation_guidance_scale", None) - guidance_scale = float(str(raw_guidance)) if raw_guidance not in (None, "") else None + sampler_kind = validation_cfg.get("sampler_kind", "sde") + if sampler_kind is None: + sampler_kind = "sde" + if not isinstance(sampler_kind, str): + raise ValueError( + "method_config.validation.sampler_kind must be a string, got " + f"{type(sampler_kind).__name__}" + ) + sampler_kind = sampler_kind.strip().lower() + if sampler_kind not in {"ode", "sde"}: + raise ValueError( + "method_config.validation.sampler_kind must be one of {ode, sde}, got " + f"{sampler_kind!r}" + ) + sampler_kind = cast(Literal["ode", "sde"], sampler_kind) + if sampling_timesteps is not None and sampler_kind != "sde": + raise ValueError( + "method_config.validation.sampling_timesteps is only valid when " + "sampler_kind='sde'" + ) + + guidance_scale = self._parse_validation_guidance_scale(validation_cfg) request = ValidationRequest( sample_handle=self.student, sampling_steps=sampling_steps, - sampler_kind="sde", + sampler_kind=sampler_kind, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, ) diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index f4e347bce..84f3bb7b7 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -179,6 +179,61 @@ def _init_optimizers_and_schedulers(self) -> None: def on_train_start(self) -> None: self.adapter.on_train_start() + def _parse_validation_cfg(self) -> dict[str, Any]: + raw = self.method_config.get("validation", None) + if raw is None: + return {} + if not isinstance(raw, dict): + raise ValueError( + "method_config.validation must be a dict when set, got " + f"{type(raw).__name__}" + ) + return dict(raw) + + def _parse_validation_sampling_steps(self, cfg: dict[str, Any]) -> list[int]: + raw = cfg.get("sampling_steps") + if raw is None: + raw = getattr(self.training_args, "validation_sampling_steps", "") or "" + + steps: list[int] = [] + if raw is None or raw == "": + return steps + if isinstance(raw, bool): + raise ValueError( + "validation sampling_steps must be an int/list/str, got bool" + ) + if isinstance(raw, int) or (isinstance(raw, float) and raw.is_integer()): + steps = [int(raw)] + elif isinstance(raw, str): + steps = [int(s) for s in raw.split(",") if str(s).strip()] + elif isinstance(raw, list): + steps = [int(s) for s in raw] + else: + raise ValueError( + "validation sampling_steps must be an int/list/str, got " + f"{type(raw).__name__}" + ) + return [s for s in steps if int(s) > 0] + + def _parse_validation_guidance_scale(self, cfg: dict[str, Any]) -> float | None: + raw = cfg.get("guidance_scale") + if raw is None: + raw = getattr(self.training_args, "validation_guidance_scale", None) + if raw in (None, ""): + return None + if isinstance(raw, bool): + raise ValueError( + "validation guidance_scale must be a number/string, got bool" + ) + if isinstance(raw, (int, float)): + return float(raw) + if isinstance(raw, str) and raw.strip(): + return float(raw) + raise ValueError( + "validation guidance_scale must be a number/string, got " + f"{type(raw).__name__}" + ) + def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: @@ -186,19 +241,31 @@ def log_validation(self, iteration: int) -> None: if not getattr(self.training_args, "log_validation", False): return - raw_steps = str(getattr(self.training_args, "validation_sampling_steps", "") or "") - sampling_steps = [int(s) for s in raw_steps.split(",") if s.strip()] - sampling_steps = [s for s in sampling_steps if s > 0] + validation_cfg = self._parse_validation_cfg() + sampling_steps = self._parse_validation_sampling_steps(validation_cfg) if not sampling_steps: return - raw_guidance = getattr(self.training_args, "validation_guidance_scale", None) - guidance_scale = float(str(raw_guidance)) if raw_guidance not in (None, "") else None + guidance_scale = self._parse_validation_guidance_scale(validation_cfg) - pipeline_sampler_kind = getattr( - getattr(self.training_args, "pipeline_config", None), "sampler_kind", None - ) - sampler_kind = str(pipeline_sampler_kind) if pipeline_sampler_kind else "ode" + sampler_kind_raw = validation_cfg.get("sampler_kind", None) + if sampler_kind_raw is None: + pipeline_sampler_kind = getattr( + getattr(self.training_args, "pipeline_config", None), "sampler_kind", None + ) + sampler_kind_raw = str(pipeline_sampler_kind) if pipeline_sampler_kind else "ode" + if not isinstance(sampler_kind_raw, str): + raise ValueError( + "method_config.validation.sampler_kind must be a string when set, got " + f"{type(sampler_kind_raw).__name__}" + ) + sampler_kind = sampler_kind_raw.strip().lower() + if sampler_kind not in {"ode", "sde"}: + raise ValueError( + "method_config.validation.sampler_kind must be one of {ode, sde}, got " + f"{sampler_kind_raw!r}" + ) + sampler_kind = cast(Literal["ode", "sde"], sampler_kind) request = ValidationRequest( sample_handle=self.student, From a27d4a93be3e58a95bfdce0dd192f62869774337 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 01:47:00 +0000 Subject: [PATCH 125/214] better validation config --- dev/config.md | 42 ++++---- dev/phase_add_causal_wangame_dfsft.md | 16 ++-- ...ame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml | 14 +-- ...wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml | 14 +-- .../wangame/finetune_wangame2.1_i2v_1.3B.yaml | 14 +-- fastvideo/distillation/doc/models/wan.md | 2 +- fastvideo/distillation/doc/models/wangame.md | 3 +- fastvideo/distillation/doc/utils/config.md | 10 +- fastvideo/distillation/doc/validators/base.md | 2 +- fastvideo/distillation/doc/validators/wan.md | 5 +- .../distillation/doc/validators/wangame.md | 19 ++-- .../methods/distribution_matching/dmd2.py | 91 +++++++++++++----- .../methods/fine_tuning/finetune.py | 95 +++++++++++++------ fastvideo/distillation/models/wan.py | 4 +- fastvideo/distillation/models/wangame.py | 4 +- fastvideo/distillation/trainer.py | 10 +- fastvideo/distillation/utils/config.py | 83 ++++++++++++---- fastvideo/distillation/validators/base.py | 1 + fastvideo/distillation/validators/wan.py | 27 +++--- fastvideo/distillation/validators/wangame.py | 27 +++--- 20 files changed, 301 insertions(+), 182 deletions(-) diff --git a/dev/config.md b/dev/config.md index 39a1a1fbe..4089e7f2a 100644 --- a/dev/config.md +++ b/dev/config.md @@ -30,9 +30,9 @@ CLI 仅保留少量 **runtime override**（不属于“实验定义”的内容 recipe: # 选择 family + method（只负责“选什么”） roles: # role -> role spec（谁参与） training: # infra 参数（直接映射到 TrainingArgs） -pipeline_config: # 模型/pipeline 侧 config（可 inline） +default_pipeline_config: # 默认 pipeline config（可 inline） method_config: # method/algorithm 超参（方法侧 single source of truth） -# 或者 pipeline_config_path: /abs/path/to/pipeline_config.json|yaml +# 或者 default_pipeline_config_path: /abs/path/to/pipeline_config.json|yaml ``` loader 规则： @@ -96,33 +96,35 @@ loader 会注入/补全的 invariants（见 `fastvideo/distillation/utils/config - 分布式尺寸默认值（`num_gpus/tp_size/sp_size/hsdp_*`） - `training.model_path` 若缺失，默认使用 `roles.student.path`（供 pipeline_config registry 使用） -关于 validation 参数的归属（推荐约定）： -- **run/trainer 级**（“什么时候验证/用什么验证集”）放在 `training:`： - - `training.log_validation` - - `training.validation_dataset_file` - - `training.validation_steps` -- **method 级**（“怎么采样/走什么 rollout 语义”）放在 `method_config.validation:`： - - `sampling_steps`（ODE 下就是 `num_inference_steps`） - - `guidance_scale` - - `sampler_kind` / `sampling_timesteps` / `rollout_mode` / `context_noise`（按 method 需要） +关于 validation 参数的归属（当前约定）： +- `training.validation`：用于描述 validation（method 也会读取这一段） + - 固定字段（框架层会用到）： + - `enabled`（bool，可省略；有 section 默认启用） + - `dataset_file`（str） + - `every_steps`（int） + - 采样字段（method 按需读取并转成 `ValidationRequest`）： + - `sampling_steps`（list[int] / int / str） + - `guidance_scale`（float，可选） + - `sampler_kind`（ode|sde，可选） + - `sampling_timesteps`（list[int]，可选；DMD2/SDE few-step 才需要） 备注： -- 目前仍允许从 `training.validation_sampling_steps / validation_guidance_scale` 读取默认值， - 但更推荐写到 `method_config.validation`，避免把 method 语义塞到 `TrainingArgs` 里。 +- `DistillTrainer` 不再读取 `training.log_validation/validation_steps/...` 做调度； + trainer 每步调用 `method.log_validation(step)`，method 决定是否执行 validation。 -## 6) `pipeline_config` / `pipeline_config_path` +## 6) `default_pipeline_config` / `default_pipeline_config_path` 两种写法（二选一）： 1) inline（适合少量 override）： ```yaml -pipeline_config: +default_pipeline_config: flow_shift: 8 ``` 2) path（适合复用大型 config 文件）： ```yaml -pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json +default_pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json ``` 常见字段（非穷举）： @@ -130,11 +132,9 @@ pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json - `sampler_kind`：`ode|sde`，选择 sampling loop 语义（`WanPipeline` 内部切换）。 备注（重要）： -- 从语义上讲，`dmd_denoising_steps` 是 algorithm knob，应当只存在于 `method_config`。 -- Phase 3.2 已将 sampling loop 语义显式化： - - method 通过 `ValidationRequest(sampler_kind=..., sampling_timesteps=...)` 指定采样方式与 few-step schedule - - `WanValidator` 将 timesteps 写入 `ForwardBatch.sampling_timesteps`，并使用 `WanPipeline` 执行采样 - - `pipeline_config.dmd_denoising_steps` 不再是 distillation 的必需字段（仅保留为 inference/legacy 兼容） +- `default_pipeline_config` 是 “模型/pipeline 的默认 config”（例如 `flow_shift`、`vae_config`）。 + method/validator 的采样语义不应再依赖它；采样策略应由 method 通过 `ValidationRequest` + 显式传入。 ## 7) `method_config`: method/algorithm 专属超参 diff --git a/dev/phase_add_causal_wangame_dfsft.md b/dev/phase_add_causal_wangame_dfsft.md index 01bf94007..f621b4eeb 100644 --- a/dev/phase_add_causal_wangame_dfsft.md +++ b/dev/phase_add_causal_wangame_dfsft.md @@ -105,14 +105,14 @@ method_config: 建议把 validation 的配置拆成两类： - **run/trainer 级（什么时候验证、用什么 validation dataset）**：放在 `training:` 下； -- **method 级（怎么采样、走什么 rollout 语义）**：放在 `method_config.validation:` 下。 +- **method 级（怎么采样、走什么 rollout 语义）**：放在 `training.validation:` 下（method 读取）。 这样 validator 不需要 “从 training_args 猜 method 语义”，而 method 也不会越权去管理 dataloader。 建议把选择权交给 method（或 method_config），让 validator 只负责执行： ```yaml -method_config: +training: validation: # full | streaming_causal rollout_mode: streaming_causal @@ -138,13 +138,13 @@ method_config: > - 我们仍可以保留 SDE-style streaming（用于对照/legacy 对齐），但必须由 config 显式选择， > 避免 “training/validation 语义混用导致 reviewer 困惑”。 -### 2.4 pipeline_config：ODE solver 选择（可选） +### 2.4 default_pipeline_config：ODE solver 选择（可选） 我们目前的 `pipeline_config.sampler_kind=ode` 默认会选择 `FlowUniPCMultistepScheduler` （见 `fastvideo/pipelines/samplers/wan.py`）。为了做对照实验/调试，建议增加一个可选字段： ```yaml -pipeline_config: +default_pipeline_config: sampler_kind: ode ode_solver: unipc # unipc | euler ``` @@ -215,8 +215,8 @@ pipeline_config: - [ ] `examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_dfsft_causal.yaml` - `roles.student.variant: causal` - `recipe.method: dfsft` - - `training.validation_dataset_file / training.validation_steps`（run 级） - - `method_config.validation.sampling_steps: [40]`（method 级） + - `training.validation.dataset_file / training.validation.every_steps` + - `training.validation.sampling_steps: [40]` - [ ] `examples/distillation/wangame/dfsft-temp.sh`（新增） - 跟现在 `run.sh` 一样只负责 export CONFIG + torchrun @@ -294,10 +294,10 @@ pipeline_config: 6) **step list / context noise 的配置入口**（重要决策点）： - 现有 streaming pipeline 读取 `pipeline_config.dmd_denoising_steps / warp_denoising_step / context_noise`； - - 我们的新框架更希望把 few-step schedule/rollout knobs 放到 `method_config`； + - 我们的新框架更希望把 few-step schedule/rollout knobs 放到 `method_config` 或 `training.validation`； - 因此需要一个明确策略： - 方案 A（最小改动）：validator 在构建 validation pipeline 时，把 - `method_config.validation.*` 写入 `args_copy.pipeline_config.*`（validation-only）； + `training.validation.*` 写入 `args_copy.pipeline_config.*`（validation-only）； - 方案 B（更干净）：把 streaming pipeline 改为优先读 `ForwardBatch.sampling_timesteps` + `ValidationRequest.context_noise`，彻底摆脱 pipeline_config 依赖（工程量更大）。 diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml index 75970c774..56d405a39 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml @@ -79,13 +79,15 @@ training: # Tracking / validation tracker_project_name: distillation_wangame wandb_run_name: wangame_dmd2_4steps_distill_actioncfg - log_validation: true - validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - validation_steps: 50 - validation_sampling_steps: "4" - validation_guidance_scale: "1.0" + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 50 + sampling_steps: [4] + sampler_kind: sde + guidance_scale: 1.0 -pipeline_config: +default_pipeline_config: flow_shift: 5 sampler_kind: sde diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml index a72148759..7adcae97b 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml @@ -79,13 +79,15 @@ training: # Tracking / validation tracker_project_name: distillation_wangame wandb_run_name: wangame_dmd2_4steps_distill_nocfg - log_validation: true - validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - validation_steps: 50 - validation_sampling_steps: "4" - validation_guidance_scale: "1.0" + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 50 + sampling_steps: [4] + sampler_kind: sde + guidance_scale: 1.0 -pipeline_config: +default_pipeline_config: flow_shift: 5 sampler_kind: sde diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml index 5cc710360..a8c800fe7 100644 --- a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml +++ b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml @@ -66,13 +66,15 @@ training: # Tracking / validation tracker_project_name: distillation_wangame wandb_run_name: wangame_finetune_from_pretrained - log_validation: true - validation_dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - validation_steps: 100 - validation_sampling_steps: "40" - validation_guidance_scale: "1.0" + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [40] + sampler_kind: ode + guidance_scale: 1.0 -pipeline_config: +default_pipeline_config: flow_shift: 3 sampler_kind: ode diff --git a/fastvideo/distillation/doc/models/wan.md b/fastvideo/distillation/doc/models/wan.md index 659cf3afb..50dc04897 100644 --- a/fastvideo/distillation/doc/models/wan.md +++ b/fastvideo/distillation/doc/models/wan.md @@ -21,7 +21,7 @@ - `adapter = WanAdapter(prompt_handle=student_handle, ...)` - dataloader：parquet + `pyarrow_schema_t2v` 4) **tracker / validator（可选）** - - validator：`WanValidator`（当 `training_args.log_validation=true`） + - validator：`WanValidator`（当 `training.validation.enabled=true`，或 `training.validation` 非空） - model plugin 只负责构建并返回 `validator` - tracker 由 trainer 构建并注入到 method/validator（`method.set_tracker(...)`） - validator 本身不应 hardcode `bundle.role("student")` 等角色语义； diff --git a/fastvideo/distillation/doc/models/wangame.md b/fastvideo/distillation/doc/models/wangame.md index 1e1e43c4a..f540a5887 100644 --- a/fastvideo/distillation/doc/models/wangame.md +++ b/fastvideo/distillation/doc/models/wangame.md @@ -20,9 +20,8 @@ - `bundle = RoleManager(roles=role_handles)` - `adapter = WanGameAdapter(...)` - dataloader：parquet + `pyarrow_schema_wangame` - - validator（可选）：`WanGameValidator`（当 `training.log_validation=true`） + - validator（可选）：`WanGameValidator`（当 `training.validation.enabled=true`，或 `training.validation` 非空） **关于 roles.family** - 当前 `wangame` plugin 只支持 `family="wangame"` 的 role。 这让 build-time 逻辑保持高内聚：模型加载、batch schema 与 adapter 能保持一致。 - diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index 699e13b12..8914bfb24 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -27,14 +27,14 @@ - `recipe: {family: ..., method: ...}` - `roles: {: {family?, path, trainable?}, ...}` - `training: {...}`（大部分字段复用 `TrainingArgs.from_kwargs()`） -- `pipeline_config` 或 `pipeline_config_path` +- `training.validation: {...}`（validation 配置；method 也会读取） +- `default_pipeline_config` 或 `default_pipeline_config_path`（默认 pipeline config） - `method_config: {...}`（算法/recipe 专属超参） **Validation 参数建议归属** -- run/trainer 级（什么时候验证 / 用什么验证集）放 `training:`： - - `log_validation`, `validation_dataset_file`, `validation_steps` -- method 级（怎么采样 / rollout 语义）放 `method_config.validation:`： - - `sampling_steps`, `guidance_scale`, `sampler_kind`, `sampling_timesteps`, ... +- 统一放在 `training.validation:`（框架层固定字段 + method 按需字段）。 +- trainer 每步调用 `method.log_validation(step)`；是否真正执行由 method 基于 + `training.validation.every_steps` 决定。 **实现要点** - `_resolve_existing_file()`：要求传入真实存在的路径（不做 overlay/fallback） diff --git a/fastvideo/distillation/doc/validators/base.md b/fastvideo/distillation/doc/validators/base.md index f69f6af10..50ae760e1 100644 --- a/fastvideo/distillation/doc/validators/base.md +++ b/fastvideo/distillation/doc/validators/base.md @@ -6,7 +6,7 @@ **接口** - `log_validation(step: int, request: ValidationRequest | None = None) -> None` -`ValidationRequest` 用于 method 覆盖关键采样配置（steps/guidance/output_dir 等），让 validator +`ValidationRequest` 用于 method 覆盖关键采样配置（dataset/steps/guidance/output_dir 等），让 validator 保持 model-plugin-specific、但 method-agnostic。 `ValidationRequest.sample_handle` 用于由 method 明确指定“本次 validation 要采样哪个模型/权重” diff --git a/fastvideo/distillation/doc/validators/wan.md b/fastvideo/distillation/doc/validators/wan.md index 9c1123f46..2700e52c1 100644 --- a/fastvideo/distillation/doc/validators/wan.md +++ b/fastvideo/distillation/doc/validators/wan.md @@ -21,9 +21,8 @@ - `ValidationRequest.sampling_timesteps` 提供 few-step schedule（写入 `ForwardBatch.sampling_timesteps`） - 这样 validator 不再依赖 `Pipeline`（例如 `WanDMDPipeline`），保持 method-agnostic。 -**可演进方向（Phase 3+）** -- 将 validation steps/guidance 等采样配置从 `TrainingArgs` 迁移到更明确的配置块（例如 `validation:`）。 -- 进一步抽象 validator API，使其更容易被不同 model plugin / method 复用。 +**输入约定** +- dataset 由 method 通过 `ValidationRequest.dataset_file` 提供（validator 不再从 `TrainingArgs` 读取）。 **Tracker 注入** - `WanValidator` 不再要求 build-time 传入 tracker； diff --git a/fastvideo/distillation/doc/validators/wangame.md b/fastvideo/distillation/doc/validators/wangame.md index 103f4d56e..2ec6b1378 100644 --- a/fastvideo/distillation/doc/validators/wangame.md +++ b/fastvideo/distillation/doc/validators/wangame.md @@ -2,22 +2,15 @@ **定位** - `WanGameValidator` 是 WanGame 的 standalone validator： - - 由 model plugin 构建并返回（当 `training.log_validation=true`） - - 由 method 决定何时调用，并通过 `ValidationRequest` 指定采样细节 + - 由 model plugin 构建并返回（当 `training.validation.enabled=true`，或 `training.validation` 非空） + - 由 method 决定何时调用，并通过 `ValidationRequest` 指定采样细节（包含 dataset 与采样策略） -**pipeline 选择（最小侵入式）** -- `sampler_kind="ode"`： - - 使用 `WanGameActionImageToVideoPipeline`（ODE/UniPC 采样） -- `sampler_kind="sde"`： - - 使用 `WanGameCausalDMDPipeline`（SDE-style rollout / DMD sampling） - -> 说明：当前阶段我们用“切换 pipeline class”的方式区分 ODE/SDE， -> 以降低接入风险。后续若统一到同一个 pipeline 的 `sampler_kind` 分支， -> 可以减少重复逻辑，但改动会更侵入（属于后续 phase）。 +**pipeline 选择** +- 统一使用 `WanGameActionImageToVideoPipeline`，并通过 `sampler_kind={ode|sde}` 切换采样 loop 语义。 **调用方式（method-managed validation）** -- method 通过 `ValidationRequest(sample_handle=..., sampler_kind=..., ...)` 指定： +- method 通过 `ValidationRequest(sample_handle=..., dataset_file=..., sampler_kind=..., ...)` 指定： - 用哪个 role 的 transformer 做采样（通常是 student） + - validation dataset（`dataset_file`） - 采样步数（`sampling_steps`） - 若是 SDE 采样：`sampling_timesteps`（few-step rollout 的 explicit steps） - diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 2638a3371..2a465cd09 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -118,6 +118,7 @@ def __init__( bundle: RoleManager, adapter: _DMD2Adapter, method_config: dict[str, Any] | None = None, + validation_config: dict[str, Any] | None = None, validator: Any | None = None, ) -> None: super().__init__(bundle) @@ -135,6 +136,7 @@ def __init__( self.validator = validator self.training_args = adapter.training_args self.method_config: dict[str, Any] = dict(method_config or {}) + self.validation_config: dict[str, Any] = dict(validation_config or {}) self._cfg_uncond = self._parse_cfg_uncond() self._rollout_mode = self._parse_rollout_mode() self._denoising_step_list: torch.Tensor | None = None @@ -153,6 +155,7 @@ def build( bundle=bundle, adapter=adapter, method_config=cfg.method_config, + validation_config=cfg.validation, validator=validator, ) @@ -307,25 +310,52 @@ def _init_optimizers_and_schedulers(self) -> None: def on_train_start(self) -> None: self.adapter.on_train_start() - def _parse_validation_cfg(self) -> dict[str, Any]: - raw = self.method_config.get("validation", None) + def _is_validation_enabled(self) -> bool: + cfg = self.validation_config + if not cfg: + return False + enabled = cfg.get("enabled", None) + if enabled is None: + return True + if isinstance(enabled, bool): + return bool(enabled) + raise ValueError( + "training.validation.enabled must be a bool when set, got " + f"{type(enabled).__name__}" + ) + + def _parse_validation_every_steps(self) -> int: + raw = self.validation_config.get("every_steps", None) if raw is None: - return {} - if not isinstance(raw, dict): + raise ValueError("training.validation.every_steps must be set when validation is enabled") + if isinstance(raw, bool): + raise ValueError("training.validation.every_steps must be an int, got bool") + if isinstance(raw, int): + return int(raw) + if isinstance(raw, float) and raw.is_integer(): + return int(raw) + if isinstance(raw, str) and raw.strip(): + return int(raw) + raise ValueError( + "training.validation.every_steps must be an int, got " + f"{type(raw).__name__}" + ) + + def _parse_validation_dataset_file(self) -> str: + raw = self.validation_config.get("dataset_file", None) + if not isinstance(raw, str) or not raw.strip(): raise ValueError( - "method_config.validation must be a dict when set, got " - f"{type(raw).__name__}" + "training.validation.dataset_file must be set when validation is enabled" ) - return dict(raw) - - def _parse_validation_sampling_steps(self, cfg: dict[str, Any]) -> list[int]: - raw = cfg.get("sampling_steps") - if raw is None: - raw = getattr(self.training_args, "validation_sampling_steps", "") or "" + return raw.strip() + def _parse_validation_sampling_steps(self) -> list[int]: + raw = self.validation_config.get("sampling_steps") steps: list[int] = [] if raw is None or raw == "": - return steps + raise ValueError( + "training.validation.sampling_steps must be set for validation" + ) if isinstance(raw, bool): raise ValueError( "validation sampling_steps must be an int/list/str, got bool" @@ -343,10 +373,8 @@ def _parse_validation_sampling_steps(self, cfg: dict[str, Any]) -> list[int]: ) return [s for s in steps if int(s) > 0] - def _parse_validation_guidance_scale(self, cfg: dict[str, Any]) -> float | None: - raw = cfg.get("guidance_scale") - if raw is None: - raw = getattr(self.training_args, "validation_guidance_scale", None) + def _parse_validation_guidance_scale(self) -> float | None: + raw = self.validation_config.get("guidance_scale") if raw in (None, ""): return None if isinstance(raw, bool): @@ -366,15 +394,20 @@ def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: return - if not getattr(self.training_args, "log_validation", False): + if not self._is_validation_enabled(): return - validation_cfg = self._parse_validation_cfg() + every_steps = self._parse_validation_every_steps() + if every_steps <= 0: + return + if iteration % every_steps != 0: + return - sampling_steps = self._parse_validation_sampling_steps(validation_cfg) + dataset_file = self._parse_validation_dataset_file() + sampling_steps = self._parse_validation_sampling_steps() sampling_timesteps: list[int] | None = None - raw_timesteps = validation_cfg.get("sampling_timesteps", None) + raw_timesteps = self.validation_config.get("sampling_timesteps", None) if raw_timesteps is None: raw_timesteps = self.method_config.get("dmd_denoising_steps", None) if isinstance(raw_timesteps, list) and raw_timesteps: @@ -386,18 +419,18 @@ def log_validation(self, iteration: int) -> None: return sampling_steps = [int(len(sampling_timesteps))] - sampler_kind = validation_cfg.get("sampler_kind", "sde") + sampler_kind = self.validation_config.get("sampler_kind", "sde") if sampler_kind is None: sampler_kind = "sde" if not isinstance(sampler_kind, str): raise ValueError( - "method_config.validation.sampler_kind must be a string, got " + "training.validation.sampler_kind must be a string, got " f"{type(sampler_kind).__name__}" ) sampler_kind = sampler_kind.strip().lower() if sampler_kind not in {"ode", "sde"}: raise ValueError( - "method_config.validation.sampler_kind must be one of {ode, sde}, got " + "training.validation.sampler_kind must be one of {ode, sde}, got " f"{sampler_kind!r}" ) sampler_kind = cast(Literal["ode", "sde"], sampler_kind) @@ -407,14 +440,22 @@ def log_validation(self, iteration: int) -> None: "sampler_kind='sde'" ) - guidance_scale = self._parse_validation_guidance_scale(validation_cfg) + guidance_scale = self._parse_validation_guidance_scale() + output_dir = self.validation_config.get("output_dir", None) + if output_dir is not None and not isinstance(output_dir, str): + raise ValueError( + "training.validation.output_dir must be a string when set, got " + f"{type(output_dir).__name__}" + ) request = ValidationRequest( sample_handle=self.student, + dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, + output_dir=output_dir, ) validator.log_validation(iteration, request=request) diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 84f3bb7b7..5a3d781ca 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -73,6 +73,7 @@ def __init__( bundle: RoleManager, adapter: _FineTuneAdapter, method_config: dict[str, Any] | None = None, + validation_config: dict[str, Any] | None = None, validator: Any | None = None, ) -> None: super().__init__(bundle) @@ -85,6 +86,7 @@ def __init__( self.validator = validator self.training_args = adapter.training_args self.method_config: dict[str, Any] = dict(method_config or {}) + self.validation_config: dict[str, Any] = dict(validation_config or {}) self._attn_kind: Literal["dense", "vsa"] = self._parse_attn_kind( self.method_config.get("attn_kind", None) ) @@ -104,6 +106,7 @@ def build( bundle=bundle, adapter=adapter, method_config=cfg.method_config, + validation_config=cfg.validation, validator=validator, ) @@ -179,25 +182,54 @@ def _init_optimizers_and_schedulers(self) -> None: def on_train_start(self) -> None: self.adapter.on_train_start() - def _parse_validation_cfg(self) -> dict[str, Any]: - raw = self.method_config.get("validation", None) + def _is_validation_enabled(self) -> bool: + cfg = self.validation_config + if not cfg: + return False + enabled = cfg.get("enabled", None) + if enabled is None: + return True + if isinstance(enabled, bool): + return bool(enabled) + raise ValueError( + "training.validation.enabled must be a bool when set, got " + f"{type(enabled).__name__}" + ) + + def _parse_validation_every_steps(self) -> int: + raw = self.validation_config.get("every_steps", None) if raw is None: - return {} - if not isinstance(raw, dict): raise ValueError( - "method_config.validation must be a dict when set, got " - f"{type(raw).__name__}" + "training.validation.every_steps must be set when validation is enabled" ) - return dict(raw) + if isinstance(raw, bool): + raise ValueError("training.validation.every_steps must be an int, got bool") + if isinstance(raw, int): + return int(raw) + if isinstance(raw, float) and raw.is_integer(): + return int(raw) + if isinstance(raw, str) and raw.strip(): + return int(raw) + raise ValueError( + "training.validation.every_steps must be an int, got " + f"{type(raw).__name__}" + ) - def _parse_validation_sampling_steps(self, cfg: dict[str, Any]) -> list[int]: - raw = cfg.get("sampling_steps") - if raw is None: - raw = getattr(self.training_args, "validation_sampling_steps", "") or "" + def _parse_validation_dataset_file(self) -> str: + raw = self.validation_config.get("dataset_file", None) + if not isinstance(raw, str) or not raw.strip(): + raise ValueError( + "training.validation.dataset_file must be set when validation is enabled" + ) + return raw.strip() + def _parse_validation_sampling_steps(self) -> list[int]: + raw = self.validation_config.get("sampling_steps") steps: list[int] = [] if raw is None or raw == "": - return steps + raise ValueError( + "training.validation.sampling_steps must be set for validation" + ) if isinstance(raw, bool): raise ValueError( "validation sampling_steps must be an int/list/str, got bool" @@ -215,10 +247,8 @@ def _parse_validation_sampling_steps(self, cfg: dict[str, Any]) -> list[int]: ) return [s for s in steps if int(s) > 0] - def _parse_validation_guidance_scale(self, cfg: dict[str, Any]) -> float | None: - raw = cfg.get("guidance_scale") - if raw is None: - raw = getattr(self.training_args, "validation_guidance_scale", None) + def _parse_validation_guidance_scale(self) -> float | None: + raw = self.validation_config.get("guidance_scale") if raw in (None, ""): return None if isinstance(raw, bool): @@ -238,41 +268,48 @@ def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: return - if not getattr(self.training_args, "log_validation", False): + if not self._is_validation_enabled(): return - validation_cfg = self._parse_validation_cfg() - sampling_steps = self._parse_validation_sampling_steps(validation_cfg) - if not sampling_steps: + every_steps = self._parse_validation_every_steps() + if every_steps <= 0: + return + if iteration % every_steps != 0: return - guidance_scale = self._parse_validation_guidance_scale(validation_cfg) + dataset_file = self._parse_validation_dataset_file() + sampling_steps = self._parse_validation_sampling_steps() + guidance_scale = self._parse_validation_guidance_scale() - sampler_kind_raw = validation_cfg.get("sampler_kind", None) - if sampler_kind_raw is None: - pipeline_sampler_kind = getattr( - getattr(self.training_args, "pipeline_config", None), "sampler_kind", None - ) - sampler_kind_raw = str(pipeline_sampler_kind) if pipeline_sampler_kind else "ode" + sampler_kind_raw = self.validation_config.get("sampler_kind", "ode") if not isinstance(sampler_kind_raw, str): raise ValueError( - "method_config.validation.sampler_kind must be a string when set, got " + "training.validation.sampler_kind must be a string when set, got " f"{type(sampler_kind_raw).__name__}" ) sampler_kind = sampler_kind_raw.strip().lower() if sampler_kind not in {"ode", "sde"}: raise ValueError( - "method_config.validation.sampler_kind must be one of {ode, sde}, got " + "training.validation.sampler_kind must be one of {ode, sde}, got " f"{sampler_kind_raw!r}" ) sampler_kind = cast(Literal["ode", "sde"], sampler_kind) + output_dir = self.validation_config.get("output_dir", None) + if output_dir is not None and not isinstance(output_dir, str): + raise ValueError( + "training.validation.output_dir must be a string when set, got " + f"{type(output_dir).__name__}" + ) + request = ValidationRequest( sample_handle=self.student, + dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, sampling_timesteps=None, guidance_scale=guidance_scale, + output_dir=output_dir, ) validator.log_validation(iteration, request=request) diff --git a/fastvideo/distillation/models/wan.py b/fastvideo/distillation/models/wan.py index eae6fb3f4..7c8e70d4e 100644 --- a/fastvideo/distillation/models/wan.py +++ b/fastvideo/distillation/models/wan.py @@ -97,7 +97,9 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: bundle = RoleManager(roles=role_handles) validator = None - if getattr(training_args, "log_validation", False): + validation_cfg = getattr(cfg, "validation", {}) or {} + validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) + if validation_enabled: from fastvideo.distillation.validators.wan import WanValidator validator = WanValidator( diff --git a/fastvideo/distillation/models/wangame.py b/fastvideo/distillation/models/wangame.py index 57b3a3436..b3701becf 100644 --- a/fastvideo/distillation/models/wangame.py +++ b/fastvideo/distillation/models/wangame.py @@ -94,7 +94,9 @@ def build_wangame_components(*, cfg: DistillRunConfig) -> ModelComponents: bundle = RoleManager(roles=role_handles) validator = None - if getattr(training_args, "log_validation", False): + validation_cfg = getattr(cfg, "validation", {}) or {} + validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) + if validation_enabled: from fastvideo.distillation.validators.wangame import WanGameValidator validator = WanGameValidator(training_args=training_args) diff --git a/fastvideo/distillation/trainer.py b/fastvideo/distillation/trainer.py index 3bcfd2e35..88a8d15b9 100644 --- a/fastvideo/distillation/trainer.py +++ b/fastvideo/distillation/trainer.py @@ -98,10 +98,7 @@ def run( if resumed_step is not None: start_step = int(resumed_step) - validation_interval = int(self.training_args.validation_steps or 0) - if (getattr(self.training_args, "log_validation", False) - and validation_interval > 0 and hasattr(method, - "log_validation")): + if hasattr(method, "log_validation"): method.log_validation(start_step) # type: ignore[attr-defined] if hasattr(method, "optimizers_zero_grad"): @@ -172,10 +169,7 @@ def run( if checkpoint_manager is not None: checkpoint_manager.maybe_save(step) - if (getattr(self.training_args, "log_validation", False) - and validation_interval > 0 - and step % validation_interval == 0 - and hasattr(method, "log_validation")): + if hasattr(method, "log_validation"): method.log_validation(step) # type: ignore[attr-defined] if checkpoint_manager is not None: diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index eb6d2e679..5defd6af6 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -44,6 +44,7 @@ class DistillRunConfig: recipe: RecipeSpec roles: dict[RoleName, RoleSpec] training_args: TrainingArgs + validation: dict[str, Any] method_config: dict[str, Any] raw: dict[str, Any] @@ -89,7 +90,7 @@ def _get_bool(raw: Any, *, where: str, default: bool) -> bool: def get_optional_int(mapping: dict[str, Any], key: str, *, where: str) -> int | None: - raw = mapping.get(key, None) + raw = mapping.get(key) if raw is None: return None if isinstance(raw, bool): @@ -104,7 +105,7 @@ def get_optional_int(mapping: dict[str, Any], key: str, *, where: str) -> int | def get_optional_float(mapping: dict[str, Any], key: str, *, where: str) -> float | None: - raw = mapping.get(key, None) + raw = mapping.get(key) if raw is None: return None if isinstance(raw, bool): @@ -175,18 +176,40 @@ def load_distill_run_config(path: str) -> DistillRunConfig: training_raw = _require_mapping(cfg.get("training"), where="training") + legacy_validation_keys = { + "log_validation", + "validation_dataset_file", + "validation_steps", + "validation_sampling_steps", + "validation_guidance_scale", + } + has_legacy_validation = any(key in training_raw for key in legacy_validation_keys) + + training_validation_raw = training_raw.get("validation", None) + if training_validation_raw is None: + if has_legacy_validation: + raise ValueError( + "Validation config has moved under training.validation " + "(enabled/dataset_file/every_steps/sampling_steps/...). " + "Do not use legacy training.validation_* keys." + ) + validation: dict[str, Any] = {} + else: + if has_legacy_validation: + raise ValueError( + "Do not mix training.validation with legacy training.validation_* keys. " + "Put all validation fields under training.validation." + ) + validation = _require_mapping(training_validation_raw, where="training.validation") + method_config_raw = cfg.get("method_config", None) if method_config_raw is None: method_config: dict[str, Any] = {} else: method_config = _require_mapping(method_config_raw, where="method_config") - pipeline_cfg_raw = cfg.get("pipeline_config", None) - pipeline_cfg_path = cfg.get("pipeline_config_path", None) - if pipeline_cfg_raw is not None and pipeline_cfg_path is not None: - raise ValueError("Provide either pipeline_config or pipeline_config_path, not both") - training_kwargs: dict[str, Any] = dict(training_raw) + training_kwargs.pop("validation", None) # Entrypoint invariants. training_kwargs["mode"] = ExecutionMode.DISTILLATION @@ -217,16 +240,43 @@ def load_distill_run_config(path: str) -> DistillRunConfig: if "pretrained_model_name_or_path" not in training_kwargs: training_kwargs["pretrained_model_name_or_path"] = training_kwargs["model_path"] - if pipeline_cfg_path is not None: - pipeline_cfg_path = _require_str(pipeline_cfg_path, where="pipeline_config_path") - training_kwargs["pipeline_config"] = _resolve_existing_file(pipeline_cfg_path) - elif pipeline_cfg_raw is not None: - if isinstance(pipeline_cfg_raw, str): - training_kwargs["pipeline_config"] = _resolve_existing_file(pipeline_cfg_raw) - elif isinstance(pipeline_cfg_raw, dict): - training_kwargs["pipeline_config"] = pipeline_cfg_raw + default_pipeline_cfg_raw = cfg.get("default_pipeline_config", None) + default_pipeline_cfg_path = cfg.get("default_pipeline_config_path", None) + pipeline_cfg_raw = cfg.get("pipeline_config", None) + pipeline_cfg_path = cfg.get("pipeline_config_path", None) + + if (default_pipeline_cfg_raw is not None or default_pipeline_cfg_path is not None) and ( + pipeline_cfg_raw is not None or pipeline_cfg_path is not None + ): + raise ValueError( + "Provide either default_pipeline_config(_path) or the legacy " + "pipeline_config(_path), not both" + ) + + cfg_raw = default_pipeline_cfg_raw if default_pipeline_cfg_raw is not None else pipeline_cfg_raw + cfg_path = ( + default_pipeline_cfg_path if default_pipeline_cfg_path is not None else pipeline_cfg_path + ) + + if cfg_path is not None: + cfg_path = _require_str( + cfg_path, + where=( + "default_pipeline_config_path" + if default_pipeline_cfg_path is not None + else "pipeline_config_path" + ), + ) + training_kwargs["pipeline_config"] = _resolve_existing_file(cfg_path) + elif cfg_raw is not None: + if isinstance(cfg_raw, str): + training_kwargs["pipeline_config"] = _resolve_existing_file(cfg_raw) + elif isinstance(cfg_raw, dict): + training_kwargs["pipeline_config"] = cfg_raw else: - raise ValueError("pipeline_config must be a mapping or a path string") + raise ValueError( + "default_pipeline_config must be a mapping or a path string" + ) training_args = TrainingArgs.from_kwargs(**training_kwargs) @@ -234,6 +284,7 @@ def load_distill_run_config(path: str) -> DistillRunConfig: recipe=recipe, roles=roles, training_args=training_args, + validation=validation, method_config=method_config, raw=cfg, ) diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py index 1adfed12c..9c6afde40 100644 --- a/fastvideo/distillation/validators/base.py +++ b/fastvideo/distillation/validators/base.py @@ -19,6 +19,7 @@ class ValidationRequest: """ sample_handle: RoleHandle | None = None + dataset_file: str | None = None sampling_steps: list[int] | None = None sampler_kind: Literal["ode", "sde"] | None = None sampling_timesteps: list[int] | None = None diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 2af4165fd..93add4e7d 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -98,11 +98,6 @@ def _get_pipeline( self._pipeline_key = key return self._pipeline - def _parse_validation_steps(self) -> list[int]: - raw = str(getattr(self.training_args, "validation_sampling_steps", "") or "") - steps = [int(s) for s in raw.split(",") if s.strip()] - return [s for s in steps if s > 0] - def _prepare_validation_batch( self, sampling_param: SamplingParam, @@ -119,8 +114,6 @@ def _prepare_validation_batch( sampling_param.data_type = "video" if guidance_scale is not None: sampling_param.guidance_scale = float(guidance_scale) - elif getattr(self.training_args, "validation_guidance_scale", ""): - sampling_param.guidance_scale = float(self.training_args.validation_guidance_scale) sampling_param.seed = self.seed latents_size = [ @@ -159,6 +152,7 @@ def _run_validation_for_steps( self, num_inference_steps: int, *, + dataset_file: str, transformer: torch.nn.Module, sampler_kind: str, sampling_timesteps: list[int] | None = None, @@ -168,7 +162,7 @@ def _run_validation_for_steps( pipeline = self._get_pipeline(transformer=transformer, sampler_kind=sampler_kind) sampling_param = self._get_sampling_param() - dataset = ValidationDataset(training_args.validation_dataset_file) + dataset = ValidationDataset(dataset_file) dataloader = DataLoader(dataset, batch_size=None, num_workers=0) videos: list[list[np.ndarray]] = [] @@ -204,16 +198,17 @@ def _run_validation_for_steps( return _ValidationStepResult(videos=videos, captions=captions) def log_validation(self, step: int, *, request: ValidationRequest | None = None) -> None: - training_args = self.training_args - if not getattr(training_args, "log_validation", False): - return - if not getattr(training_args, "validation_dataset_file", ""): - raise ValueError("validation_dataset_file must be set when log_validation is enabled") + if request is None: + raise ValueError("WanValidator.log_validation requires a ValidationRequest") + + dataset_file = getattr(request, "dataset_file", None) + if not dataset_file: + raise ValueError("ValidationRequest.dataset_file must be provided by the method") guidance_scale = getattr(request, "guidance_scale", None) - validation_steps = getattr(request, "sampling_steps", None) or self._parse_validation_steps() + validation_steps = getattr(request, "sampling_steps", None) if not validation_steps: - return + raise ValueError("ValidationRequest.sampling_steps must be provided by the method") sampler_kind = getattr(request, "sampler_kind", None) or "ode" sampling_timesteps = getattr(request, "sampling_timesteps", None) if sampling_timesteps is not None: @@ -233,6 +228,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) transformer = sample_handle.require_module("transformer") was_training = bool(getattr(transformer, "training", False)) + training_args = self.training_args output_dir = getattr(request, "output_dir", None) or training_args.output_dir old_inference_mode = training_args.inference_mode @@ -249,6 +245,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) for num_inference_steps in validation_steps: result = self._run_validation_for_steps( num_inference_steps, + dataset_file=str(dataset_file), transformer=transformer, sampler_kind=str(sampler_kind), sampling_timesteps=sampling_timesteps, diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index 3862e7792..a6d61da58 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -159,11 +159,6 @@ def _get_pipeline(self, *, transformer: torch.nn.Module, sampler_kind: str) -> A self._pipeline_key = key return self._pipeline - def _parse_validation_steps(self) -> list[int]: - raw = str(getattr(self.training_args, "validation_sampling_steps", "") or "") - steps = [int(s) for s in raw.split(",") if s.strip()] - return [s for s in steps if s > 0] - def _prepare_validation_batch( self, sampling_param: SamplingParam, @@ -183,8 +178,6 @@ def _prepare_validation_batch( sampling_param.data_type = "video" if guidance_scale is not None: sampling_param.guidance_scale = float(guidance_scale) - elif getattr(training_args, "validation_guidance_scale", ""): - sampling_param.guidance_scale = float(training_args.validation_guidance_scale) sampling_param.seed = self.seed temporal_compression_factor = ( @@ -237,6 +230,7 @@ def _run_validation_for_steps( self, num_inference_steps: int, *, + dataset_file: str, transformer: torch.nn.Module, sampler_kind: str, sampling_timesteps: list[int] | None = None, @@ -246,7 +240,7 @@ def _run_validation_for_steps( pipeline = self._get_pipeline(transformer=transformer, sampler_kind=sampler_kind) sampling_param = self._get_sampling_param() - dataset = ValidationDataset(training_args.validation_dataset_file) + dataset = ValidationDataset(dataset_file) dataloader = DataLoader(dataset, batch_size=None, num_workers=0) videos: list[list[np.ndarray]] = [] @@ -283,16 +277,17 @@ def _run_validation_for_steps( return _ValidationStepResult(videos=videos, captions=captions) def log_validation(self, step: int, *, request: ValidationRequest | None = None) -> None: - training_args = self.training_args - if not getattr(training_args, "log_validation", False): - return - if not getattr(training_args, "validation_dataset_file", ""): - raise ValueError("validation_dataset_file must be set when log_validation is enabled") + if request is None: + raise ValueError("WanGameValidator.log_validation requires a ValidationRequest") + + dataset_file = getattr(request, "dataset_file", None) + if not dataset_file: + raise ValueError("ValidationRequest.dataset_file must be provided by the method") guidance_scale = getattr(request, "guidance_scale", None) - validation_steps = getattr(request, "sampling_steps", None) or self._parse_validation_steps() + validation_steps = getattr(request, "sampling_steps", None) if not validation_steps: - return + raise ValueError("ValidationRequest.sampling_steps must be provided by the method") sampler_kind = getattr(request, "sampler_kind", None) or "ode" sampling_timesteps = getattr(request, "sampling_timesteps", None) if sampling_timesteps is not None: @@ -312,6 +307,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) transformer = sample_handle.require_module("transformer") was_training = bool(getattr(transformer, "training", False)) + training_args = self.training_args output_dir = getattr(request, "output_dir", None) or training_args.output_dir old_inference_mode = training_args.inference_mode @@ -328,6 +324,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) for num_inference_steps in validation_steps: result = self._run_validation_for_steps( num_inference_steps, + dataset_file=str(dataset_file), transformer=transformer, sampler_kind=str(sampler_kind), sampling_timesteps=sampling_timesteps, From 82a24ee89186469da0257c0baf0a011cbcc0ab34 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 02:49:29 +0000 Subject: [PATCH 126/214] dfdft and causal wan init impl --- .../dfsft_wangame2.1_i2v_1.3B_causal.yaml | 86 +++ fastvideo/distillation/adapters/wangame.py | 34 +- fastvideo/distillation/dispatch.py | 1 + .../distillation/doc/adapters/wangame.md | 7 +- .../doc/methods/fine_tuning/__init__.md | 4 + .../doc/methods/fine_tuning/dfsft.md | 34 ++ fastvideo/distillation/doc/models/wangame.md | 3 + fastvideo/distillation/doc/utils/config.md | 3 + .../distillation/doc/utils/moduleloader.md | 5 +- .../methods/fine_tuning/__init__.py | 8 + .../distillation/methods/fine_tuning/dfsft.py | 558 ++++++++++++++++++ fastvideo/distillation/models/wangame.py | 19 +- fastvideo/distillation/utils/config.py | 15 +- fastvideo/distillation/utils/moduleloader.py | 15 +- 14 files changed, 777 insertions(+), 15 deletions(-) create mode 100644 examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml create mode 100644 fastvideo/distillation/doc/methods/fine_tuning/dfsft.md create mode 100644 fastvideo/distillation/methods/fine_tuning/dfsft.py diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml new file mode 100644 index 000000000..204e0a6ae --- /dev/null +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml @@ -0,0 +1,86 @@ +recipe: + family: wangame + method: dfsft + +roles: + student: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + variant: causal + +training: + # Distributed (mirror legacy WanGame finetune scripts) + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder) + # + # Supports comma-separated `path` or `path:N` (repeat count) entries. + # N=0 means skip (handy to toggle datasets without deleting lines). + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_dfsft_causal + max_train_steps: 20000 + seed: 1000 + checkpoints_total_limit: 2 + + # Optimizer + learning_rate: 1.0e-5 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 1.0e-4 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.0 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_dfsft_causal + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [40] + sampler_kind: ode + guidance_scale: 1.0 + +default_pipeline_config: + flow_shift: 3 + sampler_kind: ode + +method_config: + attn_kind: dense + chunk_size: 3 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 diff --git a/fastvideo/distillation/adapters/wangame.py b/fastvideo/distillation/adapters/wangame.py index b0fc48b79..bb6881d7a 100644 --- a/fastvideo/distillation/adapters/wangame.py +++ b/fastvideo/distillation/adapters/wangame.py @@ -405,6 +405,8 @@ def _build_distill_input_kwargs( viewmats: torch.Tensor | None, Ks: torch.Tensor | None, action: torch.Tensor | None, + mouse_cond: torch.Tensor | None, + keyboard_cond: torch.Tensor | None, ) -> dict[str, Any]: hidden_states = torch.cat( [ @@ -422,6 +424,8 @@ def _build_distill_input_kwargs( "viewmats": viewmats, "Ks": Ks, "action": action, + "mouse_cond": mouse_cond, + "keyboard_cond": keyboard_cond, "return_dict": False, } @@ -445,6 +449,8 @@ def _select_cfg_condition_inputs( viewmats = getattr(batch, "viewmats", None) Ks = getattr(batch, "Ks", None) action = getattr(batch, "action", None) + mouse_cond = getattr(batch, "mouse_cond", None) + keyboard_cond = getattr(batch, "keyboard_cond", None) if conditional or cfg_uncond is None: return { @@ -454,6 +460,8 @@ def _select_cfg_condition_inputs( "viewmats": viewmats, "Ks": Ks, "action": action, + "mouse_cond": mouse_cond, + "keyboard_cond": keyboard_cond, } on_missing_raw = cfg_uncond.get("on_missing", "error") @@ -535,10 +543,16 @@ def _get_policy(channel: str) -> str: viewmats = torch.zeros_like(viewmats) Ks = torch.zeros_like(Ks) action = torch.zeros_like(action) + if mouse_cond is not None: + mouse_cond = torch.zeros_like(mouse_cond) + if keyboard_cond is not None: + keyboard_cond = torch.zeros_like(keyboard_cond) elif action_policy == "drop": viewmats = None Ks = None action = None + mouse_cond = None + keyboard_cond = None return { "image_embeds": image_embeds, @@ -547,17 +561,21 @@ def _get_policy(channel: str) -> str: "viewmats": viewmats, "Ks": Ks, "action": action, + "mouse_cond": mouse_cond, + "keyboard_cond": keyboard_cond, } def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: transformer = handle.require_module("transformer") transformer_2 = handle.modules.get("transformer_2") - if ( - transformer_2 is not None - and self.boundary_timestep is not None - and float(timestep.item()) < float(self.boundary_timestep) - ): - return transformer_2 + if transformer_2 is not None and self.boundary_timestep is not None: + if timestep.numel() != 1: + raise ValueError( + "MoE boundary selection requires a scalar timestep, got " + f"shape={tuple(timestep.shape)}" + ) + if float(timestep.item()) < float(self.boundary_timestep): + return transformer_2 return transformer def predict_x0( @@ -599,6 +617,8 @@ def predict_x0( viewmats=cond_inputs["viewmats"], Ks=cond_inputs["Ks"], action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs["keyboard_cond"], ) transformer = self._get_transformer(handle, timestep) pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) @@ -649,6 +669,8 @@ def predict_noise( viewmats=cond_inputs["viewmats"], Ks=cond_inputs["Ks"], action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs["keyboard_cond"], ) transformer = self._get_transformer(handle, timestep) pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index 266136520..8a7a0f62a 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -78,6 +78,7 @@ def ensure_builtin_registrations() -> None: from fastvideo.distillation.models import wangame as _wangame # noqa: F401 from fastvideo.distillation.methods.distribution_matching import dmd2 as _dmd2 # noqa: F401 from fastvideo.distillation.methods.fine_tuning import finetune as _finetune # noqa: F401 + from fastvideo.distillation.methods.fine_tuning import dfsft as _dfsft # noqa: F401 _BUILTINS_REGISTERED = True diff --git a/fastvideo/distillation/doc/adapters/wangame.md b/fastvideo/distillation/doc/adapters/wangame.md index b7bb96ba8..04d1e89d7 100644 --- a/fastvideo/distillation/doc/adapters/wangame.md +++ b/fastvideo/distillation/doc/adapters/wangame.md @@ -12,7 +12,9 @@ `cat([noisy_video_latents, mask_lat_size, image_latents], dim=1)` - 还需要额外输入： - `encoder_hidden_states_image`（来自 `clip_feature`） - - `viewmats / Ks / action`（由 `process_custom_actions(keyboard, mouse)` 生成） + - bidirectional transformer：`viewmats / Ks / action` + （由 `process_custom_actions(keyboard, mouse)` 生成） + - causal transformer：`mouse_cond / keyboard_cond`（raw action sequences） - 目前 WanGame 的 `conditional/unconditional`（文本 CFG）语义**不成立**： - adapter 仍保留 `conditional: bool` 形参以匹配 method protocol， 但当前实现把它当作 no-op（cond/uncond 同路）。 @@ -36,6 +38,7 @@ **边界 / TODO** - ✅ adapter 不保存/管理 few-step denoising step list，也不决定 rollout 策略。 - ✅ adapter 不引入 DMD2 专属概念（例如 “generator/critic”）。 +- ✅ adapter 支持 **per-frame timesteps**（例如 DFSFT 的 `t_inhom`），但当启用 MoE + `transformer_2 + boundary_timestep` 时要求 timestep 为标量（否则无法定义“跨帧选择哪个 transformer”）。 - TODO：若未来需要在 wangame 上定义 “uncond” 语义（例如 `zero_action/zero_image`）， 应通过 `method_config` 声明，并由 adapter 提供可解释的操作入口（而不是硬编码到 adapter 内部逻辑）。 - diff --git a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md index 9f29f88a6..f2a2cea1d 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md @@ -8,6 +8,10 @@ - 只要求 `roles.student` - loss/policy 在 method 层 - 复用同一套 trainer/roles/adapter/model plugin/validator/checkpoint 基础设施 +- `dfsft.py`：`DiffusionForcingSFTMethod`（`recipe.method: dfsft`） + - 只要求 `roles.student` + - 训练目标仍是 SFT/flow-matching loss，但使用 **chunk-wise inhomogeneous timesteps** + （`t_inhom`）作为 diffusion forcing baseline **设计要点** - adapter 仍保持 operation-centric（`prepare_batch / predict_* / backward`），不内置 finetune 的 loss 语义。 diff --git a/fastvideo/distillation/doc/methods/fine_tuning/dfsft.md b/fastvideo/distillation/doc/methods/fine_tuning/dfsft.md new file mode 100644 index 000000000..2be08ff9e --- /dev/null +++ b/fastvideo/distillation/doc/methods/fine_tuning/dfsft.md @@ -0,0 +1,34 @@ +# `fastvideo/distillation/methods/fine_tuning/dfsft.py` + +**定位** +- `@register_method("dfsft")`：Diffusion-Forcing SFT（DFSFT）baseline。 +- 只训练 `roles.student`，不依赖 teacher/critic。 +- 目标：在 SFT/flow-matching loss 的基础上，引入 **chunk-wise inhomogeneous timesteps** + （`t_inhom`）来覆盖“历史上下文不总是干净”的噪声分布（为 causal/streaming 训练做铺垫）。 + +**核心训练逻辑（单步）** +1) `adapter.prepare_batch(...)` 产出 `TrainingBatch`（包含 `x0` video latents + conditioning）。 +2) 采样 `t_inhom`： + - 先采样每个 chunk 的 timestep index（`method_config.chunk_size` 控制 chunk 划分） + - 再 repeat 到每帧（`[B, T_lat]`） +3) 采样 `noise ~ N(0, I)`，得到 `x_t = adapter.add_noise(x0, noise, t_inhom_flat)` +4) 学生预测 `pred = adapter.predict_noise(student, x_t, t_inhom, batch, ...)` +5) loss： + - 默认 flow-matching：`MSE(pred, noise - x0)` + - 若 `training.precondition_outputs=true`：precondition 到 `x0` 再回归 `x0` + +**关键 config** +- `method_config`（DFSFT 专属） + - `chunk_size: int`（默认 3）：chunk-wise timestep 的 block size + - `min_timestep_ratio / max_timestep_ratio: float`：采样 index 范围（映射到 scheduler 的 train steps） + - `attn_kind: dense|vsa`：选择 adapter 的 dense/VSA attention metadata 路径 + +**约束** +- 如果 student transformer 暴露 `num_frame_per_block`，DFSFT 会要求 + `method_config.chunk_size == transformer.num_frame_per_block`，否则直接报错（避免配错造成语义不一致）。 + +**Validation** +- DFSFT 依赖 `training.validation`（由 method 驱动 `validator.log_validation(...)`）。 +- 当前 validator 仍是 “full-video pipeline” 语义；真正的 streaming/causal rollout + 仍需要在后续阶段实现（避免把 rollout policy 藏进 validator/adapter）。 + diff --git a/fastvideo/distillation/doc/models/wangame.md b/fastvideo/distillation/doc/models/wangame.md index f540a5887..8b20a86f6 100644 --- a/fastvideo/distillation/doc/models/wangame.md +++ b/fastvideo/distillation/doc/models/wangame.md @@ -14,6 +14,9 @@ - `noise_scheduler`：`FlowMatchEulerDiscreteScheduler(shift=flow_shift)` 2) **按 roles 加载 transformer 模块** - 对每个 role：加载 `transformer`（可选 `transformer_2`） + - 支持 role-level transformer 变体（通过 `RoleSpec.extra`）： + - `roles..variant: bidirectional` → `WanGameActionTransformer3DModel` + - `roles..variant: causal` → `CausalWanGameTransformer3DModel` - 根据 `RoleSpec.trainable` 设置 `requires_grad` - 可选开启 activation checkpoint（仅对 trainable role） 3) **构建 bundle / adapter / dataloader / validator** diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index 8914bfb24..6a3ed5c75 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -20,6 +20,7 @@ - `recipe: RecipeSpec`（选择 family + method） - `roles: dict[str, RoleSpec]`（来自 YAML 的 `roles:`） - `training_args: TrainingArgs`（来自 YAML 的 `training:`，并注入 entrypoint invariants） + - `validation: dict`（来自 `training.validation:`，由 method 解释并驱动验证） - `method_config: dict`（来自 YAML 的 `method_config:`，传给 method 解释） - `raw: dict`（原始 YAML，便于 tracker 记录/复现） @@ -57,6 +58,8 @@ - `path`: 模型权重路径（HF repo 或本地目录） - `trainable`: 是否训练该 role（只影响 `requires_grad`/模式；具体 optimizer 由 method 决定） - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” + - `extra: dict`：保留 `roles.` 下除上述字段外的所有 key/value， + 交给 model plugin 解释（例如 `roles.student.variant: causal`） ## 3) Builder 装配相关（build-time / run-time 边界） - `ModelComponents` diff --git a/fastvideo/distillation/doc/utils/moduleloader.md b/fastvideo/distillation/doc/utils/moduleloader.md index a0c887fd3..423e00a41 100644 --- a/fastvideo/distillation/doc/utils/moduleloader.md +++ b/fastvideo/distillation/doc/utils/moduleloader.md @@ -5,13 +5,14 @@ 便于多个 model plugin 复用，避免每个 plugin 都复制一份 loader 细节。 **当前包含** -- `load_module_from_path(model_path, module_type, training_args, disable_custom_init_weights=False)` +- `load_module_from_path(model_path, module_type, training_args, disable_custom_init_weights=False, override_transformer_cls_name=None)` - 解析/下载 `model_path` - 读取 FastVideo 的 per-module config entry - 调用 `PipelineComponentLoader.load_module(...)` - 可选跳过自定义 init weights（legacy flag：`_loading_teacher_critic_model`） + - 可选临时覆盖 `training_args.override_transformer_cls_name` + - 用于 **同一 family** 内按 role 选择 transformer 变体（例如 `roles.student.variant: causal`） **边界** - ✅ 这里只做 “单模块加载”，不做 role 语义、也不做 optimizer/scheduler。 - ✅ “哪些模块需要加载/共享/复用” 仍由 model plugin 决定。 - diff --git a/fastvideo/distillation/methods/fine_tuning/__init__.py b/fastvideo/distillation/methods/fine_tuning/__init__.py index 730415100..edce4245a 100644 --- a/fastvideo/distillation/methods/fine_tuning/__init__.py +++ b/fastvideo/distillation/methods/fine_tuning/__init__.py @@ -5,15 +5,23 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + from fastvideo.distillation.methods.fine_tuning.dfsft import DiffusionForcingSFTMethod from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod __all__ = [ + "DiffusionForcingSFTMethod", "FineTuneMethod", ] def __getattr__(name: str) -> object: # Lazy import to avoid circular imports during registry bring-up. + if name == "DiffusionForcingSFTMethod": + from fastvideo.distillation.methods.fine_tuning.dfsft import ( + DiffusionForcingSFTMethod, + ) + + return DiffusionForcingSFTMethod if name == "FineTuneMethod": from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py new file mode 100644 index 000000000..e14ecc338 --- /dev/null +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -0,0 +1,558 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any, Literal, Protocol, cast + +import torch +import torch.nn.functional as F + +from fastvideo.training.training_utils import ( + clip_grad_norm_while_handling_failing_dtensor_cases, + get_scheduler, +) + +from fastvideo.distillation.methods.base import DistillMethod, LogScalar +from fastvideo.distillation.dispatch import register_method +from fastvideo.distillation.roles import RoleHandle, RoleManager +from fastvideo.distillation.utils.config import DistillRunConfig, parse_betas +from fastvideo.distillation.validators.base import ValidationRequest + + +class _DFSFTAdapter(Protocol): + """Adapter contract for diffusion-forcing SFT (DFSFT). + + DFSFT is implemented purely at the method (algorithm) layer and relies only + on operation-centric primitives exposed by the model-family adapter. + """ + + training_args: Any + noise_scheduler: Any + + def on_train_start(self) -> None: + ... + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", + ) -> Any: + ... + + def add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + ... + + def predict_noise( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: Any, + *, + conditional: bool, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + ... + + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + ... + + +@register_method("dfsft") +class DiffusionForcingSFTMethod(DistillMethod): + """Diffusion-forcing SFT (DFSFT): train only `student` with inhomogeneous timesteps. + + This is a supervised finetuning objective (flow-matching loss), except that + we sample *block-wise* (chunk-wise) inhomogeneous timesteps `t_inhom` over + the latent time dimension to expose the student to noisy-history regimes. + """ + + def __init__( + self, + *, + bundle: RoleManager, + adapter: _DFSFTAdapter, + method_config: dict[str, Any] | None = None, + validation_config: dict[str, Any] | None = None, + validator: Any | None = None, + ) -> None: + super().__init__(bundle) + bundle.require_roles(["student"]) + self.student = bundle.role("student") + if not self.student.trainable: + raise ValueError("DFSFT requires roles.student.trainable=true") + + self.adapter = adapter + self.validator = validator + self.training_args = adapter.training_args + self.method_config: dict[str, Any] = dict(method_config or {}) + self.validation_config: dict[str, Any] = dict(validation_config or {}) + self._attn_kind: Literal["dense", "vsa"] = self._parse_attn_kind( + self.method_config.get("attn_kind", None) + ) + + self._chunk_size = self._parse_chunk_size(self.method_config.get("chunk_size", None)) + self._timestep_index_range = self._parse_timestep_index_range() + + self._init_optimizers_and_schedulers() + + @classmethod + def build( + cls, + *, + cfg: DistillRunConfig, + bundle: RoleManager, + adapter: Any, + validator: Any | None, + ) -> DistillMethod: + return cls( + bundle=bundle, + adapter=adapter, + method_config=cfg.method_config, + validation_config=cfg.validation, + validator=validator, + ) + + def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: + if raw in (None, ""): + return "dense" + kind = str(raw).strip().lower() + if kind not in {"dense", "vsa"}: + raise ValueError( + "method_config.attn_kind must be one of {'dense', 'vsa'}, " + f"got {raw!r}." + ) + return cast(Literal["dense", "vsa"], kind) + + def _parse_chunk_size(self, raw: Any) -> int: + if raw in (None, ""): + return 3 + if isinstance(raw, bool): + raise ValueError("method_config.chunk_size must be an int, got bool") + if isinstance(raw, float) and not raw.is_integer(): + raise ValueError("method_config.chunk_size must be an int, got float") + if isinstance(raw, str) and not raw.strip(): + raise ValueError("method_config.chunk_size must be an int, got empty string") + try: + value = int(raw) + except (TypeError, ValueError) as e: + raise ValueError( + "method_config.chunk_size must be an int, got " + f"{type(raw).__name__}" + ) from e + if value <= 0: + raise ValueError("method_config.chunk_size must be > 0") + return value + + def _parse_ratio(self, raw: Any, *, where: str, default: float) -> float: + if raw in (None, ""): + return float(default) + if isinstance(raw, bool): + raise ValueError(f"{where} must be a number/string, got bool") + if isinstance(raw, (int, float)): + return float(raw) + if isinstance(raw, str) and raw.strip(): + return float(raw) + raise ValueError(f"{where} must be a number/string, got {type(raw).__name__}") + + def _parse_timestep_index_range(self) -> tuple[int, int]: + scheduler = self.adapter.noise_scheduler + num_steps = int(getattr(scheduler, "config", scheduler).num_train_timesteps) + + min_ratio = self._parse_ratio( + self.method_config.get("min_timestep_ratio", None), + where="method_config.min_timestep_ratio", + default=float(getattr(self.training_args, "min_timestep_ratio", 0.0) or 0.0), + ) + max_ratio = self._parse_ratio( + self.method_config.get("max_timestep_ratio", None), + where="method_config.max_timestep_ratio", + default=float(getattr(self.training_args, "max_timestep_ratio", 1.0) or 1.0), + ) + + if not (0.0 <= min_ratio <= 1.0 and 0.0 <= max_ratio <= 1.0): + raise ValueError( + "DFSFT timestep ratios must be in [0,1], got " + f"min={min_ratio}, max={max_ratio}" + ) + if max_ratio < min_ratio: + raise ValueError( + "method_config.max_timestep_ratio must be >= min_timestep_ratio" + ) + + min_index = int(min_ratio * num_steps) + max_index = int(max_ratio * num_steps) + min_index = max(0, min(min_index, num_steps - 1)) + max_index = max(0, min(max_index, num_steps - 1)) + + if max_index <= min_index: + max_index = min(num_steps - 1, min_index + 1) + + # torch.randint expects [low, high), so make high exclusive. + return min_index, max_index + 1 + + def _build_role_optimizer_and_scheduler( + self, + *, + role: str, + handle: RoleHandle, + learning_rate: float, + betas: tuple[float, float], + scheduler_name: str, + ) -> None: + params: list[torch.nn.Parameter] = [] + for module in handle.modules.values(): + params.extend([p for p in module.parameters() if p.requires_grad]) + if not params: + raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") + + optimizer = torch.optim.AdamW( + params, + lr=float(learning_rate), + betas=betas, + weight_decay=float(getattr(self.training_args, "weight_decay", 0.0) or 0.0), + eps=1e-8, + ) + + scheduler = get_scheduler( + str(scheduler_name), + optimizer=optimizer, + num_warmup_steps=int(getattr(self.training_args, "lr_warmup_steps", 0) or 0), + num_training_steps=int(getattr(self.training_args, "max_train_steps", 0) or 0), + num_cycles=int(getattr(self.training_args, "lr_num_cycles", 0) or 0), + power=float(getattr(self.training_args, "lr_power", 0.0) or 0.0), + min_lr_ratio=float(getattr(self.training_args, "min_lr_ratio", 0.5) or 0.5), + last_epoch=-1, + ) + + handle.optimizers = {"main": optimizer} + handle.lr_schedulers = {"main": scheduler} + + def _init_optimizers_and_schedulers(self) -> None: + student_lr = float(getattr(self.training_args, "learning_rate", 0.0) or 0.0) + if student_lr <= 0.0: + raise ValueError("training.learning_rate must be > 0 for dfsft") + + student_betas = parse_betas( + getattr(self.training_args, "betas", None), + where="training.betas", + ) + student_sched = str(getattr(self.training_args, "lr_scheduler", "constant")) + self._build_role_optimizer_and_scheduler( + role="student", + handle=self.student, + learning_rate=student_lr, + betas=student_betas, + scheduler_name=student_sched, + ) + + def on_train_start(self) -> None: + self.adapter.on_train_start() + + def _is_validation_enabled(self) -> bool: + cfg = self.validation_config + if not cfg: + return False + enabled = cfg.get("enabled", None) + if enabled is None: + return True + if isinstance(enabled, bool): + return bool(enabled) + raise ValueError( + "training.validation.enabled must be a bool when set, got " + f"{type(enabled).__name__}" + ) + + def _parse_validation_every_steps(self) -> int: + raw = self.validation_config.get("every_steps", None) + if raw is None: + raise ValueError( + "training.validation.every_steps must be set when validation is enabled" + ) + if isinstance(raw, bool): + raise ValueError("training.validation.every_steps must be an int, got bool") + if isinstance(raw, int): + return int(raw) + if isinstance(raw, float) and raw.is_integer(): + return int(raw) + if isinstance(raw, str) and raw.strip(): + return int(raw) + raise ValueError( + "training.validation.every_steps must be an int, got " + f"{type(raw).__name__}" + ) + + def _parse_validation_dataset_file(self) -> str: + raw = self.validation_config.get("dataset_file", None) + if not isinstance(raw, str) or not raw.strip(): + raise ValueError( + "training.validation.dataset_file must be set when validation is enabled" + ) + return raw.strip() + + def _parse_validation_sampling_steps(self) -> list[int]: + raw = self.validation_config.get("sampling_steps") + steps: list[int] = [] + if raw is None or raw == "": + raise ValueError("training.validation.sampling_steps must be set for validation") + if isinstance(raw, bool): + raise ValueError("validation sampling_steps must be an int/list/str, got bool") + if isinstance(raw, int) or (isinstance(raw, float) and raw.is_integer()): + steps = [int(raw)] + elif isinstance(raw, str): + steps = [int(s) for s in raw.split(",") if str(s).strip()] + elif isinstance(raw, list): + steps = [int(s) for s in raw] + else: + raise ValueError( + "validation sampling_steps must be an int/list/str, got " + f"{type(raw).__name__}" + ) + return [s for s in steps if int(s) > 0] + + def _parse_validation_guidance_scale(self) -> float | None: + raw = self.validation_config.get("guidance_scale") + if raw in (None, ""): + return None + if isinstance(raw, bool): + raise ValueError("validation guidance_scale must be a number/string, got bool") + if isinstance(raw, (int, float)): + return float(raw) + if isinstance(raw, str) and raw.strip(): + return float(raw) + raise ValueError( + "validation guidance_scale must be a number/string, got " + f"{type(raw).__name__}" + ) + + def log_validation(self, iteration: int) -> None: + validator = getattr(self, "validator", None) + if validator is None: + return + if not self._is_validation_enabled(): + return + + every_steps = self._parse_validation_every_steps() + if every_steps <= 0: + return + if iteration % every_steps != 0: + return + + dataset_file = self._parse_validation_dataset_file() + sampling_steps = self._parse_validation_sampling_steps() + guidance_scale = self._parse_validation_guidance_scale() + + sampler_kind_raw = self.validation_config.get("sampler_kind", "ode") + if not isinstance(sampler_kind_raw, str): + raise ValueError( + "training.validation.sampler_kind must be a string when set, got " + f"{type(sampler_kind_raw).__name__}" + ) + sampler_kind = sampler_kind_raw.strip().lower() + if sampler_kind not in {"ode", "sde"}: + raise ValueError( + "training.validation.sampler_kind must be one of {ode, sde}, got " + f"{sampler_kind_raw!r}" + ) + sampler_kind = cast(Literal["ode", "sde"], sampler_kind) + + output_dir = self.validation_config.get("output_dir", None) + if output_dir is not None and not isinstance(output_dir, str): + raise ValueError( + "training.validation.output_dir must be a string when set, got " + f"{type(output_dir).__name__}" + ) + + request = ValidationRequest( + sample_handle=self.student, + dataset_file=dataset_file, + sampling_steps=sampling_steps, + sampler_kind=sampler_kind, + sampling_timesteps=None, + guidance_scale=guidance_scale, + output_dir=output_dir, + ) + validator.log_validation(iteration, request=request) + + def get_rng_generators(self) -> dict[str, torch.Generator]: + generators: dict[str, torch.Generator] = {} + + adapter = getattr(self, "adapter", None) + get_adapter_generators = getattr(adapter, "get_rng_generators", None) + if callable(get_adapter_generators): + generators.update(get_adapter_generators()) + + validator = getattr(self, "validator", None) + validation_gen = getattr(validator, "validation_random_generator", None) + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen + + return generators + + def _sample_t_inhom_indices(self, *, batch_size: int, num_latents: int, device: torch.device) -> torch.Tensor: + chunk_size = self._chunk_size + num_chunks = (num_latents + chunk_size - 1) // chunk_size + low, high = self._timestep_index_range + chunk_indices = torch.randint( + low=low, + high=high, + size=(batch_size, num_chunks), + device=device, + dtype=torch.long, + ) + expanded = chunk_indices.repeat_interleave(chunk_size, dim=1) + return expanded[:, :num_latents] + + def single_train_step( + self, + batch: dict[str, Any], + iteration: int, + *, + current_vsa_sparsity: float = 0.0, + ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: + del iteration + training_batch = self.adapter.prepare_batch( + batch, + current_vsa_sparsity=current_vsa_sparsity, + latents_source="data", + ) + + if training_batch.latents is None: + raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.latents") + + clean_latents = training_batch.latents + if not torch.is_tensor(clean_latents): + raise TypeError("TrainingBatch.latents must be a torch.Tensor") + if clean_latents.ndim != 5: + raise ValueError( + "TrainingBatch.latents must be [B, T, C, H, W], got " + f"shape={tuple(clean_latents.shape)}" + ) + + batch_size, num_latents = int(clean_latents.shape[0]), int(clean_latents.shape[1]) + + transformer = self.student.require_module("transformer") + expected_chunk = getattr(transformer, "num_frame_per_block", None) + if expected_chunk is not None and int(expected_chunk) != int(self._chunk_size): + raise ValueError( + "DFSFT chunk_size must match transformer.num_frame_per_block for " + f"causal training (got {self._chunk_size}, expected {expected_chunk})." + ) + + timestep_indices = self._sample_t_inhom_indices( + batch_size=batch_size, + num_latents=num_latents, + device=clean_latents.device, + ) + sp_size = int(getattr(self.training_args, "sp_size", 1) or 1) + sp_group = getattr(self.adapter, "sp_group", None) + if sp_size > 1 and sp_group is not None and hasattr(sp_group, "broadcast"): + sp_group.broadcast(timestep_indices, src=0) + + schedule_timesteps = self.adapter.noise_scheduler.timesteps.to( + device=clean_latents.device, dtype=torch.float32 + ) + schedule_sigmas = self.adapter.noise_scheduler.sigmas.to( + device=clean_latents.device, dtype=clean_latents.dtype + ) + t_inhom = schedule_timesteps[timestep_indices] + + noise = getattr(training_batch, "noise", None) + if noise is None: + noise = torch.randn_like(clean_latents) + else: + if not torch.is_tensor(noise): + raise TypeError("TrainingBatch.noise must be a torch.Tensor when set") + noise = noise.permute(0, 2, 1, 3, 4).to(dtype=clean_latents.dtype) + + noisy_latents = self.adapter.add_noise( + clean_latents, + noise, + t_inhom.flatten(), + ) + + pred = self.adapter.predict_noise( + self.student, + noisy_latents, + t_inhom, + training_batch, + conditional=True, + attn_kind=self._attn_kind, + ) + + if bool(getattr(self.training_args, "precondition_outputs", False)): + sigmas = schedule_sigmas[timestep_indices] + sigmas = sigmas.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) + pred_x0 = noisy_latents - pred * sigmas + loss = F.mse_loss(pred_x0.float(), clean_latents.float()) + else: + target = noise - clean_latents + loss = F.mse_loss(pred.float(), target.float()) + + if self._attn_kind == "vsa": + attn_metadata = training_batch.attn_metadata_vsa + else: + attn_metadata = training_batch.attn_metadata + + loss_map = {"total_loss": loss, "dfsft_loss": loss} + outputs: dict[str, Any] = {"_fv_backward": (training_batch.timesteps, attn_metadata)} + metrics: dict[str, LogScalar] = {} + return loss_map, outputs, metrics + + def backward( + self, + loss_map: dict[str, torch.Tensor], + outputs: dict[str, Any], + *, + grad_accum_rounds: int = 1, + ) -> None: + grad_accum_rounds = max(1, int(grad_accum_rounds)) + ctx = outputs.get("_fv_backward") + if ctx is None: + super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + return + self.adapter.backward( + loss_map["total_loss"], + ctx, + grad_accum_rounds=grad_accum_rounds, + ) + + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + del iteration + return list(self.student.optimizers.values()) + + def get_lr_schedulers(self, iteration: int) -> list[Any]: + del iteration + return list(self.student.lr_schedulers.values()) + + def _clip_grad_norm(self, module: torch.nn.Module) -> float: + max_grad_norm_raw = getattr(self.training_args, "max_grad_norm", None) + if max_grad_norm_raw is None: + return 0.0 + try: + max_grad_norm = float(max_grad_norm_raw) + except (TypeError, ValueError) as e: + raise ValueError( + "training.max_grad_norm must be a number when set, got " + f"{max_grad_norm_raw!r}" + ) from e + if max_grad_norm <= 0.0: + return 0.0 + grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in module.parameters()], + max_grad_norm, + foreach=None, + ) + return float(grad_norm.item()) if grad_norm is not None else 0.0 + + def optimizers_schedulers_step(self, iteration: int) -> None: + for module in self.student.modules.values(): + self._clip_grad_norm(module) + super().optimizers_schedulers_step(iteration) diff --git a/fastvideo/distillation/models/wangame.py b/fastvideo/distillation/models/wangame.py index b3701becf..760db736b 100644 --- a/fastvideo/distillation/models/wangame.py +++ b/fastvideo/distillation/models/wangame.py @@ -28,9 +28,6 @@ def build_wangame_components(*, cfg: DistillRunConfig) -> ModelComponents: if not getattr(training_args, "data_path", ""): raise ValueError("training.data_path must be set for distillation") - # Ensure transformer class name is resolvable by PipelineComponentLoader. - training_args.override_transformer_cls_name = "WanGameActionTransformer3DModel" - # Load shared components (student base path). vae = load_module_from_path( model_path=str(training_args.model_path), @@ -49,6 +46,21 @@ def build_wangame_components(*, cfg: DistillRunConfig) -> ModelComponents: f"got {role}={role_spec.family!r}" ) + variant_raw = (role_spec.extra or {}).get("variant", None) + if variant_raw is None or variant_raw == "": + transformer_cls_name = "WanGameActionTransformer3DModel" + else: + variant = str(variant_raw).strip().lower() + if variant in {"bidirectional", "bidi"}: + transformer_cls_name = "WanGameActionTransformer3DModel" + elif variant == "causal": + transformer_cls_name = "CausalWanGameTransformer3DModel" + else: + raise ValueError( + f"Unknown roles.{role}.variant for wangame: " + f"{variant_raw!r}. Expected 'causal' or 'bidirectional'." + ) + disable_custom_init_weights = bool( getattr(role_spec, "disable_custom_init_weights", False) ) @@ -57,6 +69,7 @@ def build_wangame_components(*, cfg: DistillRunConfig) -> ModelComponents: module_type="transformer", training_args=training_args, disable_custom_init_weights=disable_custom_init_weights, + override_transformer_cls_name=transformer_cls_name, ) modules: dict[str, torch.nn.Module] = {"transformer": transformer} diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index 5defd6af6..b1dbe937c 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -3,7 +3,7 @@ from __future__ import annotations import os -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Any, TYPE_CHECKING @@ -35,6 +35,7 @@ class RoleSpec: path: str trainable: bool = True disable_custom_init_weights: bool = False + extra: dict[str, Any] = field(default_factory=dict) @dataclass(slots=True) @@ -167,11 +168,23 @@ def load_distill_run_config(path: str) -> DistillRunConfig: where=f"roles.{role_str}.disable_custom_init_weights", default=False, ) + extra = { + key: value + for key, value in role_cfg.items() + if key + not in { + "family", + "path", + "trainable", + "disable_custom_init_weights", + } + } roles[role_str] = RoleSpec( family=family, path=model_path, trainable=trainable, disable_custom_init_weights=disable_custom_init_weights, + extra=extra, ) training_raw = _require_mapping(cfg.get("training"), where="training") diff --git a/fastvideo/distillation/utils/moduleloader.py b/fastvideo/distillation/utils/moduleloader.py index 4072779d3..673f44a72 100644 --- a/fastvideo/distillation/utils/moduleloader.py +++ b/fastvideo/distillation/utils/moduleloader.py @@ -17,6 +17,7 @@ def load_module_from_path( module_type: str, training_args: Any, disable_custom_init_weights: bool = False, + override_transformer_cls_name: str | None = None, ) -> torch.nn.Module: """Load a single pipeline component module from a FastVideo model path. @@ -39,6 +40,13 @@ def load_module_from_path( transformers_or_diffusers, _architecture = module_info component_path = os.path.join(local_model_path, module_type) + old_override_transformer_cls_name: str | None = None + if override_transformer_cls_name is not None: + old_override_transformer_cls_name = getattr( + training_args, "override_transformer_cls_name", None + ) + training_args.override_transformer_cls_name = str(override_transformer_cls_name) + if disable_custom_init_weights: # NOTE: This flag is used by PipelineComponentLoader to skip applying # `init_weights_from_safetensors*` overrides when loading auxiliary @@ -54,8 +62,13 @@ def load_module_from_path( finally: if disable_custom_init_weights and hasattr(training_args, "_loading_teacher_critic_model"): del training_args._loading_teacher_critic_model + if override_transformer_cls_name is not None: + if old_override_transformer_cls_name is None: + if hasattr(training_args, "override_transformer_cls_name"): + training_args.override_transformer_cls_name = None + else: + training_args.override_transformer_cls_name = old_override_transformer_cls_name if not isinstance(module, torch.nn.Module): raise TypeError(f"Loaded {module_type!r} is not a torch.nn.Module: {type(module)}") return module - From e2747ef3cac3305ab6b0bad1a3cf6235cb0acc6d Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 03:00:53 +0000 Subject: [PATCH 127/214] support other scheduler --- fastvideo/configs/pipelines/base.py | 5 ++++ .../methods/distribution_matching/dmd2.py | 30 +++++++++++++++++++ .../distillation/methods/fine_tuning/dfsft.py | 30 +++++++++++++++++++ .../methods/fine_tuning/finetune.py | 30 +++++++++++++++++++ fastvideo/distillation/validators/base.py | 1 + fastvideo/distillation/validators/wan.py | 15 ++++++++-- fastvideo/distillation/validators/wangame.py | 22 +++++++++++--- fastvideo/pipelines/samplers/wan.py | 14 +++++++-- 8 files changed, 138 insertions(+), 9 deletions(-) diff --git a/fastvideo/configs/pipelines/base.py b/fastvideo/configs/pipelines/base.py index 74bc06d46..f60fb600a 100644 --- a/fastvideo/configs/pipelines/base.py +++ b/fastvideo/configs/pipelines/base.py @@ -74,6 +74,11 @@ class PipelineConfig: # - "sde": stochastic loop with noise injection sampler_kind: str = "ode" + # ODE solver selection when `sampler_kind="ode"`. + # - "unipc": FlowUniPCMultistepScheduler (default) + # - "euler": FlowMatchEulerDiscreteScheduler + ode_solver: str = "unipc" + # Wan2.2 TI2V parameters ti2v_task: bool = False boundary_ratio: float | None = None diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 2a465cd09..bf40a5d25 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -390,6 +390,34 @@ def _parse_validation_guidance_scale(self) -> float | None: f"{type(raw).__name__}" ) + def _parse_validation_ode_solver( + self, + *, + sampler_kind: Literal["ode", "sde"], + ) -> str | None: + raw = self.validation_config.get("ode_solver", None) + if raw in (None, ""): + return None + if sampler_kind != "ode": + raise ValueError( + "training.validation.ode_solver is only valid when " + "training.validation.sampler_kind='ode'" + ) + if not isinstance(raw, str): + raise ValueError( + "training.validation.ode_solver must be a string when set, got " + f"{type(raw).__name__}" + ) + solver = raw.strip().lower() + if solver in {"unipc", "unipc_multistep", "multistep"}: + return "unipc" + if solver in {"euler", "flowmatch", "flowmatch_euler"}: + return "euler" + raise ValueError( + "training.validation.ode_solver must be one of {unipc, euler}, got " + f"{raw!r}" + ) + def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: @@ -434,6 +462,7 @@ def log_validation(self, iteration: int) -> None: f"{sampler_kind!r}" ) sampler_kind = cast(Literal["ode", "sde"], sampler_kind) + ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) if sampling_timesteps is not None and sampler_kind != "sde": raise ValueError( "method_config.validation.sampling_timesteps is only valid when " @@ -453,6 +482,7 @@ def log_validation(self, iteration: int) -> None: dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, + ode_solver=ode_solver, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, output_dir=output_dir, diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index e14ecc338..ab4d68fa8 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -332,6 +332,34 @@ def _parse_validation_guidance_scale(self) -> float | None: f"{type(raw).__name__}" ) + def _parse_validation_ode_solver( + self, + *, + sampler_kind: Literal["ode", "sde"], + ) -> str | None: + raw = self.validation_config.get("ode_solver", None) + if raw in (None, ""): + return None + if sampler_kind != "ode": + raise ValueError( + "training.validation.ode_solver is only valid when " + "training.validation.sampler_kind='ode'" + ) + if not isinstance(raw, str): + raise ValueError( + "training.validation.ode_solver must be a string when set, got " + f"{type(raw).__name__}" + ) + solver = raw.strip().lower() + if solver in {"unipc", "unipc_multistep", "multistep"}: + return "unipc" + if solver in {"euler", "flowmatch", "flowmatch_euler"}: + return "euler" + raise ValueError( + "training.validation.ode_solver must be one of {unipc, euler}, got " + f"{raw!r}" + ) + def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: @@ -362,6 +390,7 @@ def log_validation(self, iteration: int) -> None: f"{sampler_kind_raw!r}" ) sampler_kind = cast(Literal["ode", "sde"], sampler_kind) + ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) output_dir = self.validation_config.get("output_dir", None) if output_dir is not None and not isinstance(output_dir, str): @@ -375,6 +404,7 @@ def log_validation(self, iteration: int) -> None: dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, + ode_solver=ode_solver, sampling_timesteps=None, guidance_scale=guidance_scale, output_dir=output_dir, diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 5a3d781ca..d7bf41247 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -264,6 +264,34 @@ def _parse_validation_guidance_scale(self) -> float | None: f"{type(raw).__name__}" ) + def _parse_validation_ode_solver( + self, + *, + sampler_kind: Literal["ode", "sde"], + ) -> str | None: + raw = self.validation_config.get("ode_solver", None) + if raw in (None, ""): + return None + if sampler_kind != "ode": + raise ValueError( + "training.validation.ode_solver is only valid when " + "training.validation.sampler_kind='ode'" + ) + if not isinstance(raw, str): + raise ValueError( + "training.validation.ode_solver must be a string when set, got " + f"{type(raw).__name__}" + ) + solver = raw.strip().lower() + if solver in {"unipc", "unipc_multistep", "multistep"}: + return "unipc" + if solver in {"euler", "flowmatch", "flowmatch_euler"}: + return "euler" + raise ValueError( + "training.validation.ode_solver must be one of {unipc, euler}, got " + f"{raw!r}" + ) + def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: @@ -294,6 +322,7 @@ def log_validation(self, iteration: int) -> None: f"{sampler_kind_raw!r}" ) sampler_kind = cast(Literal["ode", "sde"], sampler_kind) + ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) output_dir = self.validation_config.get("output_dir", None) if output_dir is not None and not isinstance(output_dir, str): @@ -307,6 +336,7 @@ def log_validation(self, iteration: int) -> None: dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, + ode_solver=ode_solver, sampling_timesteps=None, guidance_scale=guidance_scale, output_dir=output_dir, diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py index 9c6afde40..a049ff96e 100644 --- a/fastvideo/distillation/validators/base.py +++ b/fastvideo/distillation/validators/base.py @@ -22,6 +22,7 @@ class ValidationRequest: dataset_file: str | None = None sampling_steps: list[int] | None = None sampler_kind: Literal["ode", "sde"] | None = None + ode_solver: str | None = None sampling_timesteps: list[int] | None = None guidance_scale: float | None = None output_dir: str | None = None diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 93add4e7d..b59ee8e12 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -58,7 +58,7 @@ def __init__( self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) self._pipeline: WanPipeline | None = None - self._pipeline_key: tuple[int, str] | None = None + self._pipeline_key: tuple[int, str, str] | None = None self._sampling_param: SamplingParam | None = None def set_tracker(self, tracker: Any) -> None: @@ -74,8 +74,9 @@ def _get_pipeline( *, transformer: torch.nn.Module, sampler_kind: str, + ode_solver: str | None, ) -> WanPipeline: - key = (id(transformer), str(sampler_kind)) + key = (id(transformer), str(sampler_kind), str(ode_solver)) if self._pipeline is not None and self._pipeline_key == key: return self._pipeline @@ -88,6 +89,7 @@ def _get_pipeline( inference_mode=True, sampler_kind=str(sampler_kind), flow_shift=float(flow_shift) if flow_shift is not None else None, + ode_solver=str(ode_solver) if ode_solver is not None else None, loaded_modules={"transformer": transformer}, tp_size=self.training_args.tp_size, sp_size=self.training_args.sp_size, @@ -155,11 +157,16 @@ def _run_validation_for_steps( dataset_file: str, transformer: torch.nn.Module, sampler_kind: str, + ode_solver: str | None, sampling_timesteps: list[int] | None = None, guidance_scale: float | None = None, ) -> _ValidationStepResult: training_args = self.training_args - pipeline = self._get_pipeline(transformer=transformer, sampler_kind=sampler_kind) + pipeline = self._get_pipeline( + transformer=transformer, + sampler_kind=sampler_kind, + ode_solver=ode_solver, + ) sampling_param = self._get_sampling_param() dataset = ValidationDataset(dataset_file) @@ -210,6 +217,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) if not validation_steps: raise ValueError("ValidationRequest.sampling_steps must be provided by the method") sampler_kind = getattr(request, "sampler_kind", None) or "ode" + ode_solver = getattr(request, "ode_solver", None) sampling_timesteps = getattr(request, "sampling_timesteps", None) if sampling_timesteps is not None: expected = int(len(sampling_timesteps)) @@ -248,6 +256,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) dataset_file=str(dataset_file), transformer=transformer, sampler_kind=str(sampler_kind), + ode_solver=str(ode_solver) if ode_solver is not None else None, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, ) diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index a6d61da58..f10dfa71c 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -57,7 +57,7 @@ def __init__( self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) self._pipeline: Any | None = None - self._pipeline_key: tuple[int, str] | None = None + self._pipeline_key: tuple[int, str, str] | None = None self._sampling_param: SamplingParam | None = None def set_tracker(self, tracker: Any) -> None: @@ -126,9 +126,15 @@ def _get_sampling_param(self) -> SamplingParam: self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) return self._sampling_param - def _get_pipeline(self, *, transformer: torch.nn.Module, sampler_kind: str) -> Any: + def _get_pipeline( + self, + *, + transformer: torch.nn.Module, + sampler_kind: str, + ode_solver: str | None, + ) -> Any: sampler_kind = str(sampler_kind).lower() - key = (id(transformer), sampler_kind) + key = (id(transformer), sampler_kind, str(ode_solver)) if self._pipeline is not None and self._pipeline_key == key: return self._pipeline @@ -148,6 +154,7 @@ def _get_pipeline(self, *, transformer: torch.nn.Module, sampler_kind: str) -> A inference_mode=True, flow_shift=float(flow_shift) if flow_shift is not None else None, sampler_kind=sampler_kind, + ode_solver=str(ode_solver) if ode_solver is not None else None, loaded_modules={"transformer": transformer}, tp_size=self.training_args.tp_size, sp_size=self.training_args.sp_size, @@ -233,11 +240,16 @@ def _run_validation_for_steps( dataset_file: str, transformer: torch.nn.Module, sampler_kind: str, + ode_solver: str | None, sampling_timesteps: list[int] | None = None, guidance_scale: float | None = None, ) -> _ValidationStepResult: training_args = self.training_args - pipeline = self._get_pipeline(transformer=transformer, sampler_kind=sampler_kind) + pipeline = self._get_pipeline( + transformer=transformer, + sampler_kind=sampler_kind, + ode_solver=ode_solver, + ) sampling_param = self._get_sampling_param() dataset = ValidationDataset(dataset_file) @@ -289,6 +301,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) if not validation_steps: raise ValueError("ValidationRequest.sampling_steps must be provided by the method") sampler_kind = getattr(request, "sampler_kind", None) or "ode" + ode_solver = getattr(request, "ode_solver", None) sampling_timesteps = getattr(request, "sampling_timesteps", None) if sampling_timesteps is not None: expected = int(len(sampling_timesteps)) @@ -327,6 +340,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) dataset_file=str(dataset_file), transformer=transformer, sampler_kind=str(sampler_kind), + ode_solver=str(ode_solver) if ode_solver is not None else None, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, ) diff --git a/fastvideo/pipelines/samplers/wan.py b/fastvideo/pipelines/samplers/wan.py index f749147be..92c868c05 100644 --- a/fastvideo/pipelines/samplers/wan.py +++ b/fastvideo/pipelines/samplers/wan.py @@ -21,9 +21,19 @@ def build_wan_scheduler(fastvideo_args: FastVideoArgs, kind: SamplerKind): shift = fastvideo_args.pipeline_config.flow_shift if kind == "sde": return FlowMatchEulerDiscreteScheduler(shift=shift) - return FlowUniPCMultistepScheduler(shift=shift) + + ode_solver_raw = getattr(fastvideo_args.pipeline_config, "ode_solver", "unipc") + ode_solver = str(ode_solver_raw).strip().lower() if ode_solver_raw is not None else "unipc" + if ode_solver in {"unipc", "unipc_multistep", "multistep"}: + return FlowUniPCMultistepScheduler(shift=shift) + if ode_solver in {"euler", "flowmatch", "flowmatch_euler"}: + return FlowMatchEulerDiscreteScheduler(shift=shift) + + raise ValueError( + "Unknown pipeline_config.ode_solver for wan pipelines: " + f"{ode_solver_raw!r} (expected 'unipc' or 'euler')." + ) def wan_use_btchw_layout(kind: SamplerKind) -> bool: return kind == "sde" - From 27fd58e2013d6f6541cade7e22072b47e831302b Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 07:00:35 +0000 Subject: [PATCH 128/214] not strict loading --- .../dfsft_wangame2.1_i2v_1.3B_causal.yaml | 1 + fastvideo/models/loader/component_loader.py | 24 +++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml index 204e0a6ae..f94aab5e3 100644 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml @@ -73,6 +73,7 @@ training: every_steps: 100 sampling_steps: [40] sampler_kind: ode + ode_solver: euler guidance_scale: 1.0 default_pipeline_config: diff --git a/fastvideo/models/loader/component_loader.py b/fastvideo/models/loader/component_loader.py index d7772453a..39b3aab1a 100644 --- a/fastvideo/models/loader/component_loader.py +++ b/fastvideo/models/loader/component_loader.py @@ -23,7 +23,6 @@ from fastvideo.fastvideo_args import FastVideoArgs from fastvideo.layers.quantization import get_quantization_config from fastvideo.logger import init_logger -from fastvideo.models.encoders.base import TextEncoder from fastvideo.models.hf_transformer_utils import get_diffusers_config from fastvideo.models.loader.fsdp_load import maybe_load_fsdp_model, shard_model from fastvideo.models.loader.utils import set_default_torch_dtype @@ -268,11 +267,12 @@ def load(self, model_path: str, fastvideo_args: FastVideoArgs): gemma_path = candidate gemma_path_from_candidate = True model_config["gemma_model_path"] = gemma_path - if gemma_path and not gemma_path_from_candidate: - if not os.path.isabs(gemma_path): - model_config["gemma_model_path"] = os.path.normpath( - os.path.join(repo_root, gemma_path) - ) + if gemma_path and not gemma_path_from_candidate and not os.path.isabs( + gemma_path + ): + model_config["gemma_model_path"] = os.path.normpath( + os.path.join(repo_root, gemma_path) + ) transformer_config_path = os.path.join( repo_root, "transformer", "config.json" ) @@ -280,12 +280,11 @@ def load(self, model_path: str, fastvideo_args: FastVideoArgs): try: with open(transformer_config_path, encoding="utf-8") as f: transformer_config = json.load(f) - if ( + if (( "connector_double_precision_rope" not in model_config or not model_config["connector_double_precision_rope"] - ): - if transformer_config.get("double_precision_rope") is True: - model_config["connector_double_precision_rope"] = True + ) and transformer_config.get("double_precision_rope") is True): + model_config["connector_double_precision_rope"] = True if "connector_rope_type" not in model_config: rope_type = transformer_config.get("rope_type") if rope_type is not None: @@ -539,7 +538,7 @@ def load(self, model_path: str, fastvideo_args: FastVideoArgs): tokenizer_cfg_path = os.path.join(resolved_model_path, "config.json") if os.path.exists(tokenizer_cfg_path): try: - with open(tokenizer_cfg_path, "r") as f: + with open(tokenizer_cfg_path) as f: tokenizer_cfg = json.load(f) if isinstance(tokenizer_cfg, dict) and ( tokenizer_cfg.get("_class_name") == "AutoProcessor" @@ -847,6 +846,7 @@ def load(self, model_path: str, fastvideo_args: FastVideoArgs): ) and not ( cls_name.startswith("WanGame") or cls_name == "WanGameActionTransformer3DModel" + or cls_name.startswith("CausalWan") or getattr(fastvideo_args.pipeline_config, "prefix", "") == "WanGame" or cls_name.startswith("WanLingBot") or cls_name == "WanLingBotTransformer3DModel" @@ -935,7 +935,7 @@ def load(self, model_path: str, fastvideo_args: FastVideoArgs): try: upsampler_cfg = deepcopy(fastvideo_args.pipeline_config.upsampler_config[0]) upsampler_cfg.update_model_config(config_dict) - except Exception as e: + except Exception: upsampler_cfg = deepcopy(fastvideo_args.pipeline_config.upsampler_config[1]) upsampler_cfg.update_model_config(config_dict) From a225e150090025a765728a4d04b2980cf665cb5a Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 07:16:12 +0000 Subject: [PATCH 129/214] use CausalWanGameActionTransformer3DModel on wangame causal --- fastvideo/distillation/doc/models/wangame.md | 2 +- fastvideo/distillation/models/wangame.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fastvideo/distillation/doc/models/wangame.md b/fastvideo/distillation/doc/models/wangame.md index 8b20a86f6..52403c960 100644 --- a/fastvideo/distillation/doc/models/wangame.md +++ b/fastvideo/distillation/doc/models/wangame.md @@ -16,7 +16,7 @@ - 对每个 role：加载 `transformer`（可选 `transformer_2`） - 支持 role-level transformer 变体（通过 `RoleSpec.extra`）： - `roles..variant: bidirectional` → `WanGameActionTransformer3DModel` - - `roles..variant: causal` → `CausalWanGameTransformer3DModel` + - `roles..variant: causal` → `CausalWanGameActionTransformer3DModel` - 根据 `RoleSpec.trainable` 设置 `requires_grad` - 可选开启 activation checkpoint（仅对 trainable role） 3) **构建 bundle / adapter / dataloader / validator** diff --git a/fastvideo/distillation/models/wangame.py b/fastvideo/distillation/models/wangame.py index 760db736b..d82df33a2 100644 --- a/fastvideo/distillation/models/wangame.py +++ b/fastvideo/distillation/models/wangame.py @@ -54,7 +54,7 @@ def build_wangame_components(*, cfg: DistillRunConfig) -> ModelComponents: if variant in {"bidirectional", "bidi"}: transformer_cls_name = "WanGameActionTransformer3DModel" elif variant == "causal": - transformer_cls_name = "CausalWanGameTransformer3DModel" + transformer_cls_name = "CausalWanGameActionTransformer3DModel" else: raise ValueError( f"Unknown roles.{role}.variant for wangame: " From 339a5511f12681ed27ecb5385833eeaea17d39db Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 18:58:37 +0000 Subject: [PATCH 130/214] validator rollout mode --- .../methods/distribution_matching/dmd2.py | 14 +++ .../distillation/methods/fine_tuning/dfsft.py | 14 +++ .../methods/fine_tuning/finetune.py | 14 +++ fastvideo/distillation/validators/base.py | 1 + fastvideo/distillation/validators/wan.py | 6 + fastvideo/distillation/validators/wangame.py | 108 ++++++++++++++---- fastvideo/pipelines/stages/denoising.py | 17 ++- 7 files changed, 147 insertions(+), 27 deletions(-) diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index bf40a5d25..ca83f9195 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -469,6 +469,19 @@ def log_validation(self, iteration: int) -> None: "sampler_kind='sde'" ) + rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") + if not isinstance(rollout_mode_raw, str): + raise ValueError( + "training.validation.rollout_mode must be a string when set, got " + f"{type(rollout_mode_raw).__name__}" + ) + rollout_mode = rollout_mode_raw.strip().lower() + if rollout_mode not in {"parallel", "streaming"}: + raise ValueError( + "training.validation.rollout_mode must be one of {parallel, streaming}, " + f"got {rollout_mode_raw!r}" + ) + guidance_scale = self._parse_validation_guidance_scale() output_dir = self.validation_config.get("output_dir", None) if output_dir is not None and not isinstance(output_dir, str): @@ -482,6 +495,7 @@ def log_validation(self, iteration: int) -> None: dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, + rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), ode_solver=ode_solver, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index ab4d68fa8..607d37203 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -392,6 +392,19 @@ def log_validation(self, iteration: int) -> None: sampler_kind = cast(Literal["ode", "sde"], sampler_kind) ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) + rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") + if not isinstance(rollout_mode_raw, str): + raise ValueError( + "training.validation.rollout_mode must be a string when set, got " + f"{type(rollout_mode_raw).__name__}" + ) + rollout_mode = rollout_mode_raw.strip().lower() + if rollout_mode not in {"parallel", "streaming"}: + raise ValueError( + "training.validation.rollout_mode must be one of {parallel, streaming}, " + f"got {rollout_mode_raw!r}" + ) + output_dir = self.validation_config.get("output_dir", None) if output_dir is not None and not isinstance(output_dir, str): raise ValueError( @@ -404,6 +417,7 @@ def log_validation(self, iteration: int) -> None: dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, + rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), ode_solver=ode_solver, sampling_timesteps=None, guidance_scale=guidance_scale, diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index d7bf41247..405ef6be4 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -324,6 +324,19 @@ def log_validation(self, iteration: int) -> None: sampler_kind = cast(Literal["ode", "sde"], sampler_kind) ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) + rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") + if not isinstance(rollout_mode_raw, str): + raise ValueError( + "training.validation.rollout_mode must be a string when set, got " + f"{type(rollout_mode_raw).__name__}" + ) + rollout_mode = rollout_mode_raw.strip().lower() + if rollout_mode not in {"parallel", "streaming"}: + raise ValueError( + "training.validation.rollout_mode must be one of {parallel, streaming}, " + f"got {rollout_mode_raw!r}" + ) + output_dir = self.validation_config.get("output_dir", None) if output_dir is not None and not isinstance(output_dir, str): raise ValueError( @@ -336,6 +349,7 @@ def log_validation(self, iteration: int) -> None: dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, + rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), ode_solver=ode_solver, sampling_timesteps=None, guidance_scale=guidance_scale, diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py index a049ff96e..40dbab99d 100644 --- a/fastvideo/distillation/validators/base.py +++ b/fastvideo/distillation/validators/base.py @@ -22,6 +22,7 @@ class ValidationRequest: dataset_file: str | None = None sampling_steps: list[int] | None = None sampler_kind: Literal["ode", "sde"] | None = None + rollout_mode: Literal["parallel", "streaming"] | None = None ode_solver: str | None = None sampling_timesteps: list[int] | None = None guidance_scale: float | None = None diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index b59ee8e12..1b9f89169 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -217,6 +217,12 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) if not validation_steps: raise ValueError("ValidationRequest.sampling_steps must be provided by the method") sampler_kind = getattr(request, "sampler_kind", None) or "ode" + rollout_mode = getattr(request, "rollout_mode", None) + if rollout_mode not in {None, "parallel"}: + raise ValueError( + "WanValidator only supports rollout_mode='parallel'. " + f"Got rollout_mode={rollout_mode!r}." + ) ode_solver = getattr(request, "ode_solver", None) sampling_timesteps = getattr(request, "sampling_timesteps", None) if sampling_timesteps is not None: diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index f10dfa71c..c00e6cf4c 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -57,7 +57,7 @@ def __init__( self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) self._pipeline: Any | None = None - self._pipeline_key: tuple[int, str, str] | None = None + self._pipeline_key: tuple[int, str, str, str] | None = None self._sampling_param: SamplingParam | None = None def set_tracker(self, tracker: Any) -> None: @@ -130,38 +130,61 @@ def _get_pipeline( self, *, transformer: torch.nn.Module, + rollout_mode: str, sampler_kind: str, ode_solver: str | None, ) -> Any: - sampler_kind = str(sampler_kind).lower() - key = (id(transformer), sampler_kind, str(ode_solver)) + rollout_mode = str(rollout_mode).strip().lower() + sampler_kind = str(sampler_kind).strip().lower() + key = (id(transformer), rollout_mode, sampler_kind, str(ode_solver)) if self._pipeline is not None and self._pipeline_key == key: return self._pipeline - if sampler_kind not in {"ode", "sde"}: - raise ValueError( - f"Unknown sampler_kind for WanGame validation: {sampler_kind!r}" - ) + if rollout_mode == "parallel": + if sampler_kind not in {"ode", "sde"}: + raise ValueError( + f"Unknown sampler_kind for WanGame validation: {sampler_kind!r}" + ) - flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) + flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) - from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import ( - WanGameActionImageToVideoPipeline, - ) + from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import ( + WanGameActionImageToVideoPipeline, + ) - self._pipeline = WanGameActionImageToVideoPipeline.from_pretrained( - self.training_args.model_path, - inference_mode=True, - flow_shift=float(flow_shift) if flow_shift is not None else None, - sampler_kind=sampler_kind, - ode_solver=str(ode_solver) if ode_solver is not None else None, - loaded_modules={"transformer": transformer}, - tp_size=self.training_args.tp_size, - sp_size=self.training_args.sp_size, - num_gpus=self.training_args.num_gpus, - pin_cpu_memory=self.training_args.pin_cpu_memory, - dit_cpu_offload=True, - ) + self._pipeline = WanGameActionImageToVideoPipeline.from_pretrained( + self.training_args.model_path, + inference_mode=True, + flow_shift=float(flow_shift) if flow_shift is not None else None, + sampler_kind=sampler_kind, + ode_solver=str(ode_solver) if ode_solver is not None else None, + loaded_modules={"transformer": transformer}, + tp_size=self.training_args.tp_size, + sp_size=self.training_args.sp_size, + num_gpus=self.training_args.num_gpus, + pin_cpu_memory=self.training_args.pin_cpu_memory, + dit_cpu_offload=True, + ) + elif rollout_mode == "streaming": + from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( + WanGameCausalDMDPipeline, + ) + + self._pipeline = WanGameCausalDMDPipeline.from_pretrained( + self.training_args.model_path, + inference_mode=True, + loaded_modules={"transformer": transformer}, + tp_size=self.training_args.tp_size, + sp_size=self.training_args.sp_size, + num_gpus=self.training_args.num_gpus, + pin_cpu_memory=self.training_args.pin_cpu_memory, + dit_cpu_offload=True, + ) + else: + raise ValueError( + "Unknown rollout_mode for WanGame validation: " + f"{rollout_mode!r}. Expected 'parallel' or 'streaming'." + ) self._pipeline_key = key return self._pipeline @@ -239,6 +262,7 @@ def _run_validation_for_steps( *, dataset_file: str, transformer: torch.nn.Module, + rollout_mode: str, sampler_kind: str, ode_solver: str | None, sampling_timesteps: list[int] | None = None, @@ -247,6 +271,7 @@ def _run_validation_for_steps( training_args = self.training_args pipeline = self._get_pipeline( transformer=transformer, + rollout_mode=rollout_mode, sampler_kind=sampler_kind, ode_solver=ode_solver, ) @@ -301,7 +326,30 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) if not validation_steps: raise ValueError("ValidationRequest.sampling_steps must be provided by the method") sampler_kind = getattr(request, "sampler_kind", None) or "ode" + rollout_mode_raw = getattr(request, "rollout_mode", None) or "parallel" + if not isinstance(rollout_mode_raw, str): + raise ValueError( + "ValidationRequest.rollout_mode must be a string when set, got " + f"{type(rollout_mode_raw).__name__}" + ) + rollout_mode = rollout_mode_raw.strip().lower() + if rollout_mode not in {"parallel", "streaming"}: + raise ValueError( + "ValidationRequest.rollout_mode must be one of {parallel, streaming}, got " + f"{rollout_mode_raw!r}" + ) ode_solver = getattr(request, "ode_solver", None) + if rollout_mode == "streaming": + if str(sampler_kind).strip().lower() != "sde": + raise ValueError( + "WanGame validation rollout_mode='streaming' requires " + "sampler_kind='sde' (it uses the causal DMD-style rollout)." + ) + if ode_solver is not None: + raise ValueError( + "WanGame validation rollout_mode='streaming' does not support " + f"ode_solver={ode_solver!r}." + ) sampling_timesteps = getattr(request, "sampling_timesteps", None) if sampling_timesteps is not None: expected = int(len(sampling_timesteps)) @@ -326,6 +374,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) old_inference_mode = training_args.inference_mode old_dit_cpu_offload = training_args.dit_cpu_offload old_mode = training_args.mode + old_dmd_denoising_steps = getattr(training_args.pipeline_config, "dmd_denoising_steps", None) try: training_args.inference_mode = True training_args.dit_cpu_offload = True @@ -335,10 +384,20 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) num_sp_groups = self.world_group.world_size // self.sp_group.world_size for num_inference_steps in validation_steps: + if rollout_mode == "streaming": + if sampling_timesteps is not None: + training_args.pipeline_config.dmd_denoising_steps = list(sampling_timesteps) + else: + timesteps = np.linspace(1000, 0, int(num_inference_steps)) + training_args.pipeline_config.dmd_denoising_steps = [ + int(max(0, min(1000, round(t)))) for t in timesteps + ] + result = self._run_validation_for_steps( num_inference_steps, dataset_file=str(dataset_file), transformer=transformer, + rollout_mode=rollout_mode, sampler_kind=str(sampler_kind), ode_solver=str(ode_solver) if ode_solver is not None else None, sampling_timesteps=sampling_timesteps, @@ -384,5 +443,6 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) training_args.inference_mode = old_inference_mode training_args.dit_cpu_offload = old_dit_cpu_offload training_args.mode = old_mode + training_args.pipeline_config.dmd_denoising_steps = old_dmd_denoising_steps if was_training: transformer.train() diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index acf1d6b87..6ffd7b46c 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -553,10 +553,21 @@ def prepare_extra_func_kwargs(self, func, kwargs) -> dict[str, Any]: Returns: The prepared kwargs. """ - extra_step_kwargs = {} + signature = inspect.signature(func) + if any( + p.kind == inspect.Parameter.VAR_KEYWORD + for p in signature.parameters.values() + ): + # If the callee accepts `**kwargs`, do not filter by signature. + # This is important for models that route parameters internally via + # `forward(*args, **kwargs)` (e.g. causal Wangame), where filtering + # would incorrectly drop conditioning kwargs like `action`. + return dict(kwargs) + + accepted = set(signature.parameters.keys()) + extra_step_kwargs: dict[str, Any] = {} for k, v in kwargs.items(): - accepts = k in set(inspect.signature(func).parameters.keys()) - if accepts: + if k in accepted: extra_step_kwargs[k] = v return extra_step_kwargs From 257495b97638e4eca5623f3b79934cc598fce98c Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 20:31:28 +0000 Subject: [PATCH 131/214] validator streaming causal rollout --- .../dfsft_wangame2.1_i2v_1.3B_causal.yaml | 1 + fastvideo/distillation/validators/wangame.py | 62 +++- .../basic/wan/wangame_causal_dmd_pipeline.py | 36 +- fastvideo/pipelines/stages/__init__.py | 5 +- .../pipelines/stages/matrixgame_denoising.py | 328 +++++++++++++++++- 5 files changed, 414 insertions(+), 18 deletions(-) diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml index f94aab5e3..fb336b686 100644 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml @@ -72,6 +72,7 @@ training: dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json every_steps: 100 sampling_steps: [40] + rollout_mode: streaming sampler_kind: ode ode_solver: euler guidance_scale: 1.0 diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index c00e6cf4c..b570ff739 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -166,6 +166,46 @@ def _get_pipeline( dit_cpu_offload=True, ) elif rollout_mode == "streaming": + if sampler_kind not in {"ode", "sde"}: + raise ValueError( + f"Unknown sampler_kind for WanGame streaming validation: {sampler_kind!r}" + ) + + flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) + if flow_shift is None: + raise ValueError("pipeline_config.flow_shift must be set for WanGame validation") + + if sampler_kind == "sde": + if ode_solver is not None: + raise ValueError( + "ode_solver is only valid when sampler_kind='ode', got " + f"ode_solver={ode_solver!r}." + ) + from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, + ) + + scheduler = FlowMatchEulerDiscreteScheduler(shift=float(flow_shift)) + else: + from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, + ) + from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler, + ) + + ode_solver_norm = (str(ode_solver).strip().lower() + if ode_solver is not None else "unipc") + if ode_solver_norm in {"unipc", "unipc_multistep", "multistep"}: + scheduler = FlowUniPCMultistepScheduler(shift=float(flow_shift)) + elif ode_solver_norm in {"euler", "flowmatch", "flowmatch_euler"}: + scheduler = FlowMatchEulerDiscreteScheduler(shift=float(flow_shift)) + else: + raise ValueError( + "Unknown ode_solver for WanGame streaming validation: " + f"{ode_solver!r} (expected 'unipc' or 'euler')." + ) + from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( WanGameCausalDMDPipeline, ) @@ -173,7 +213,13 @@ def _get_pipeline( self._pipeline = WanGameCausalDMDPipeline.from_pretrained( self.training_args.model_path, inference_mode=True, - loaded_modules={"transformer": transformer}, + flow_shift=float(flow_shift) if flow_shift is not None else None, + sampler_kind=sampler_kind, + ode_solver=str(ode_solver) if ode_solver is not None else None, + loaded_modules={ + "transformer": transformer, + "scheduler": scheduler, + }, tp_size=self.training_args.tp_size, sp_size=self.training_args.sp_size, num_gpus=self.training_args.num_gpus, @@ -340,15 +386,17 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) ) ode_solver = getattr(request, "ode_solver", None) if rollout_mode == "streaming": - if str(sampler_kind).strip().lower() != "sde": + sampler_kind_norm = str(sampler_kind).strip().lower() + if sampler_kind_norm not in {"ode", "sde"}: raise ValueError( "WanGame validation rollout_mode='streaming' requires " - "sampler_kind='sde' (it uses the causal DMD-style rollout)." + "sampler_kind to be one of {'ode', 'sde'}, got " + f"{sampler_kind!r}." ) - if ode_solver is not None: + if sampler_kind_norm == "sde" and ode_solver is not None: raise ValueError( - "WanGame validation rollout_mode='streaming' does not support " - f"ode_solver={ode_solver!r}." + "WanGame validation rollout_mode='streaming' only supports " + f"ode_solver when sampler_kind='ode', got ode_solver={ode_solver!r}." ) sampling_timesteps = getattr(request, "sampling_timesteps", None) if sampling_timesteps is not None: @@ -384,7 +432,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) num_sp_groups = self.world_group.world_size // self.sp_group.world_size for num_inference_steps in validation_steps: - if rollout_mode == "streaming": + if rollout_mode == "streaming" and str(sampler_kind).strip().lower() == "sde": if sampling_timesteps is not None: training_args.pipeline_config.dmd_denoising_steps = list(sampling_timesteps) else: diff --git a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py index 4940b2fd6..ecd05fba8 100644 --- a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py @@ -4,13 +4,16 @@ from fastvideo.fastvideo_args import FastVideoArgs from fastvideo.logger import init_logger from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline +from fastvideo.pipelines.samplers.wan import get_wan_sampler_kind from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, MatrixGameCausalDenoisingStage, + MatrixGameCausalOdeDenoisingStage, MatrixGameImageEncodingStage, InputValidationStage, LatentPreparationStage, - TextEncodingStage) + TextEncodingStage, + TimestepPreparationStage) from fastvideo.pipelines.stages.image_encoding import (MatrixGameImageVAEEncodingStage) logger = init_logger(__name__) @@ -22,6 +25,7 @@ class WanGameCausalDMDPipeline(LoRAPipeline, ComposedPipelineBase): ] def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: + sampler_kind = get_wan_sampler_kind(fastvideo_args) self.add_stage(stage_name="input_validation_stage", stage=InputValidationStage()) @@ -45,6 +49,11 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: self.add_stage(stage_name="conditioning_stage", stage=ConditioningStage()) + if sampler_kind == "ode": + self.add_stage(stage_name="timestep_preparation_stage", + stage=TimestepPreparationStage( + scheduler=self.get_module("scheduler"))) + self.add_stage(stage_name="latent_preparation_stage", stage=LatentPreparationStage( scheduler=self.get_module("scheduler"), @@ -53,13 +62,24 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: self.add_stage(stage_name="image_latent_preparation_stage", stage=MatrixGameImageVAEEncodingStage(vae=self.get_module("vae"))) - self.add_stage(stage_name="denoising_stage", - stage=MatrixGameCausalDenoisingStage( - transformer=self.get_module("transformer"), - transformer_2=self.get_module("transformer_2", None), - scheduler=self.get_module("scheduler"), - pipeline=self, - vae=self.get_module("vae"))) + if sampler_kind == "ode": + denoising_stage = MatrixGameCausalOdeDenoisingStage( + transformer=self.get_module("transformer"), + transformer_2=self.get_module("transformer_2", None), + scheduler=self.get_module("scheduler"), + pipeline=self, + vae=self.get_module("vae"), + ) + else: + denoising_stage = MatrixGameCausalDenoisingStage( + transformer=self.get_module("transformer"), + transformer_2=self.get_module("transformer_2", None), + scheduler=self.get_module("scheduler"), + pipeline=self, + vae=self.get_module("vae"), + ) + + self.add_stage(stage_name="denoising_stage", stage=denoising_stage) self.add_stage(stage_name="decoding_stage", stage=DecodingStage(vae=self.get_module("vae"))) diff --git a/fastvideo/pipelines/stages/__init__.py b/fastvideo/pipelines/stages/__init__.py index 029b16641..e59bd2a4d 100644 --- a/fastvideo/pipelines/stages/__init__.py +++ b/fastvideo/pipelines/stages/__init__.py @@ -33,7 +33,9 @@ LTX2LatentPreparationStage) from fastvideo.pipelines.stages.ltx2_text_encoding import LTX2TextEncodingStage from fastvideo.pipelines.stages.matrixgame_denoising import ( - MatrixGameCausalDenoisingStage) + MatrixGameCausalDenoisingStage, + MatrixGameCausalOdeDenoisingStage, +) from fastvideo.pipelines.stages.hyworld_denoising import HYWorldDenoisingStage from fastvideo.pipelines.stages.gamecraft_denoising import GameCraftDenoisingStage from fastvideo.pipelines.stages.text_encoding import (Cosmos25TextEncodingStage, @@ -65,6 +67,7 @@ "DmdDenoisingStage", "CausalDMDDenosingStage", "MatrixGameCausalDenoisingStage", + "MatrixGameCausalOdeDenoisingStage", "HYWorldDenoisingStage", "GameCraftDenoisingStage", "CosmosDenoisingStage", diff --git a/fastvideo/pipelines/stages/matrixgame_denoising.py b/fastvideo/pipelines/stages/matrixgame_denoising.py index 43a844b27..de0d97e4e 100644 --- a/fastvideo/pipelines/stages/matrixgame_denoising.py +++ b/fastvideo/pipelines/stages/matrixgame_denoising.py @@ -719,8 +719,332 @@ def _update_context_cache( **ctx.pos_cond_kwargs, **context_model_kwargs, ) - if isinstance(cache_update_ret, list) and len(cache_update_ret) > 0: - ctx.kv_cache1 = cache_update_ret + if isinstance(cache_update_ret, list) and len(cache_update_ret) > 0: + ctx.kv_cache1 = cache_update_ret + + +class MatrixGameCausalOdeDenoisingStage(MatrixGameCausalDenoisingStage): + """Causal ODE denoising for WanGame/MatrixGame. + + This is the deterministic counterpart of `MatrixGameCausalDenoisingStage`. + It performs block-by-block causal rollout, but uses the scheduler's ODE-style + `step()` update (no re-noising between steps). + """ + + def forward( + self, + batch: ForwardBatch, + fastvideo_args: FastVideoArgs, + ) -> ForwardBatch: + timesteps = batch.timesteps + if timesteps is None: + raise ValueError( + "MatrixGameCausalOdeDenoisingStage requires batch.timesteps. " + "Make sure TimestepPreparationStage runs before this stage." + ) + + target_dtype = torch.bfloat16 + autocast_enabled = (target_dtype != torch.float32 + ) and not fastvideo_args.disable_autocast + + latent_seq_length = batch.latents.shape[-1] * batch.latents.shape[-2] + if hasattr(self.transformer, "patch_size"): + patch_size = self.transformer.patch_size + else: + patch_size = self.transformer.config.arch_config.patch_size + patch_ratio = patch_size[-1] * patch_size[-2] + self.frame_seq_length = latent_seq_length // patch_ratio + + timesteps = timesteps.to(get_local_torch_device()) + + boundary_ratio = getattr(fastvideo_args.pipeline_config.dit_config, + "boundary_ratio", None) + if boundary_ratio is not None: + boundary_timestep = boundary_ratio * self.scheduler.num_train_timesteps + else: + boundary_timestep = None + + image_embeds = batch.image_embeds + if len(image_embeds) > 0: + assert torch.isnan(image_embeds[0]).sum() == 0 + image_embeds = [ + image_embed.to(target_dtype) for image_embed in image_embeds + ] + + # directly set the kwarg. + image_kwargs = {"encoder_hidden_states_image": image_embeds} + pos_cond_kwargs: dict[str, Any] = {} + + assert batch.latents is not None, "latents must be provided" + latents = batch.latents + b, c, t, h, w = latents.shape + + prompt_embeds = batch.prompt_embeds + assert torch.isnan(prompt_embeds[0]).sum() == 0 + + viewmats_full = None + intrinsics_full = None + action_full = None + if batch.mouse_cond is not None and batch.keyboard_cond is not None: + from fastvideo.models.dits.hyworld.pose import process_custom_actions + + viewmats_list = [] + intrinsics_list = [] + action_list = [] + for bi in range(b): + vm, ks, action = process_custom_actions(batch.keyboard_cond[bi], + batch.mouse_cond[bi]) + viewmats_list.append(vm) + intrinsics_list.append(ks) + action_list.append(action) + viewmats_full = torch.stack(viewmats_list, + dim=0).to(device=latents.device, + dtype=target_dtype) + intrinsics_full = torch.stack(intrinsics_list, + dim=0).to(device=latents.device, + dtype=target_dtype) + action_full = torch.stack(action_list, + dim=0).to(device=latents.device, + dtype=target_dtype) + + kv_cache1 = self._initialize_kv_cache(batch_size=latents.shape[0], + dtype=target_dtype, + device=latents.device) + kv_cache2 = None + if boundary_timestep is not None: + kv_cache2 = self._initialize_kv_cache(batch_size=latents.shape[0], + dtype=target_dtype, + device=latents.device) + + kv_cache_mouse = None + kv_cache_keyboard = None + if self.use_action_module: + kv_cache_mouse, kv_cache_keyboard = self._initialize_action_kv_cache( + batch_size=latents.shape[0], + dtype=target_dtype, + device=latents.device) + + crossattn_cache = self._initialize_crossattn_cache( + batch_size=latents.shape[0], + max_text_len=257, # 1 CLS + 256 patch tokens + dtype=target_dtype, + device=latents.device) + + if t % self.num_frame_per_block != 0: + raise ValueError( + "num_frames must be divisible by num_frame_per_block for causal denoising" + ) + num_blocks = t // self.num_frame_per_block + block_sizes = [self.num_frame_per_block] * num_blocks + start_index = 0 + + if boundary_timestep is not None: + block_sizes[0] = 1 + + ctx = BlockProcessingContext( + batch=batch, + block_idx=0, + start_index=0, + kv_cache1=kv_cache1, + kv_cache2=kv_cache2, + kv_cache_mouse=kv_cache_mouse, + kv_cache_keyboard=kv_cache_keyboard, + crossattn_cache=crossattn_cache, + timesteps=timesteps, + block_sizes=block_sizes, + noise_pool=None, + fastvideo_args=fastvideo_args, + target_dtype=target_dtype, + autocast_enabled=autocast_enabled, + boundary_timestep=boundary_timestep, + high_noise_timesteps=None, + context_noise=getattr(fastvideo_args.pipeline_config, "context_noise", + 0), + image_kwargs=image_kwargs, + pos_cond_kwargs=pos_cond_kwargs, + viewmats_full=viewmats_full, + intrinsics_full=intrinsics_full, + action_full=action_full, + ) + + context_noise = getattr(fastvideo_args.pipeline_config, "context_noise", + 0) + + with self.progress_bar(total=len(block_sizes) * + len(timesteps)) as progress_bar: + for block_idx, current_num_frames in enumerate(block_sizes): + ctx.block_idx = block_idx + ctx.start_index = start_index + current_latents = latents[:, :, start_index:start_index + + current_num_frames, :, :] + + action_kwargs = self._prepare_action_kwargs( + batch, start_index, current_num_frames) + camera_action_kwargs = self._prepare_camera_action_kwargs( + ctx, start_index, current_num_frames) + + current_latents = self._process_single_block_ode( + current_latents=current_latents, + batch=batch, + start_index=start_index, + current_num_frames=current_num_frames, + timesteps=timesteps, + ctx=ctx, + action_kwargs=action_kwargs, + camera_action_kwargs=camera_action_kwargs, + progress_bar=progress_bar, + ) + + latents[:, :, start_index:start_index + + current_num_frames, :, :] = current_latents + + # Update KV caches with clean context + self._update_context_cache( + current_latents=current_latents, + batch=batch, + start_index=start_index, + current_num_frames=current_num_frames, + ctx=ctx, + action_kwargs=action_kwargs, + camera_action_kwargs=camera_action_kwargs, + context_noise=context_noise, + ) + + start_index += current_num_frames + + if boundary_timestep is not None: + num_frames_to_remove = self.num_frame_per_block - 1 + if num_frames_to_remove > 0: + latents = latents[:, :, :-num_frames_to_remove, :, :] + + batch.latents = latents + return batch + + def _process_single_block_ode( + self, + *, + current_latents: torch.Tensor, + batch: ForwardBatch, + start_index: int, + current_num_frames: int, + timesteps: torch.Tensor, + ctx: BlockProcessingContext, + action_kwargs: dict[str, Any], + camera_action_kwargs: dict[str, Any], + progress_bar: Any | None = None, + ) -> torch.Tensor: + prompt_embeds = batch.prompt_embeds + extra_step_kwargs = self.prepare_extra_func_kwargs( + self.scheduler.step, + { + "generator": batch.generator, + "eta": batch.eta, + }, + ) + + for i, t_cur in enumerate(timesteps): + if ctx.boundary_timestep is not None and t_cur < ctx.boundary_timestep: + current_model = self.transformer_2 if self.transformer_2 is not None else self.transformer + else: + current_model = self.transformer + + latent_model_input = current_latents.to(ctx.target_dtype) + + independent_first_frame = getattr(self.transformer, + "independent_first_frame", + False) + if batch.image_latent is not None and not independent_first_frame: + image_latent_chunk = batch.image_latent[:, :, start_index: + start_index + + current_num_frames, :, :] + latent_model_input = torch.cat([ + latent_model_input, + image_latent_chunk.to(ctx.target_dtype) + ], + dim=1) + elif ( + batch.image_latent is not None and independent_first_frame + and start_index == 0 + ): + latent_model_input = torch.cat([ + latent_model_input, + batch.image_latent.to(ctx.target_dtype) + ], + dim=2) + + latent_model_input = self.scheduler.scale_model_input( + latent_model_input, t_cur) + + # Build attention metadata if VSA is available + if vsa_available and self.attn_backend == VideoSparseAttentionBackend: + self.attn_metadata_builder_cls = self.attn_backend.get_builder_cls( + ) + if self.attn_metadata_builder_cls is not None: + self.attn_metadata_builder = self.attn_metadata_builder_cls( + ) + h, w = current_latents.shape[-2:] + attn_metadata = self.attn_metadata_builder.build( + current_timestep=i, + raw_latent_shape=(current_num_frames, h, w), + patch_size=ctx.fastvideo_args.pipeline_config. + dit_config.patch_size, + VSA_sparsity=ctx.fastvideo_args.VSA_sparsity, + device=get_local_torch_device(), + ) + assert attn_metadata is not None, "attn_metadata cannot be None" + else: + attn_metadata = None + else: + attn_metadata = None + + with torch.autocast(device_type="cuda", + dtype=ctx.target_dtype, + enabled=ctx.autocast_enabled), \ + set_forward_context(current_timestep=i, + attn_metadata=attn_metadata, + forward_batch=batch): + t_expanded = t_cur * torch.ones( + (latent_model_input.shape[0], current_num_frames), + device=latent_model_input.device, + dtype=t_cur.dtype) + + model_kwargs = { + "kv_cache": ctx.get_kv_cache(t_cur), + "crossattn_cache": ctx.crossattn_cache, + "current_start": start_index * self.frame_seq_length, + "start_frame": start_index, + "is_cache": False, + } + + if self.use_action_module and current_model == self.transformer: + model_kwargs.update({ + "kv_cache_mouse": ctx.kv_cache_mouse, + "kv_cache_keyboard": ctx.kv_cache_keyboard, + }) + model_kwargs.update(action_kwargs) + + noise_pred = current_model( + latent_model_input, + prompt_embeds, + t_expanded, + **camera_action_kwargs, + **ctx.image_kwargs, + **ctx.pos_cond_kwargs, + **model_kwargs, + ) + + current_latents = self.scheduler.step( + noise_pred, + t_cur, + current_latents, + **extra_step_kwargs, + return_dict=False, + )[0] + + if progress_bar is not None: + progress_bar.update() + + return current_latents def streaming_reset(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs) -> ForwardBatch: From 679a65de80639593f2b22e1657c2ffb654ea0b9b Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 21:18:12 +0000 Subject: [PATCH 132/214] wangame support validator num_frames --- .../dfsft_wangame2.1_i2v_1.3B_causal.yaml | 1 + .../methods/distribution_matching/dmd2.py | 9 +++ .../distillation/methods/fine_tuning/dfsft.py | 15 +++- .../methods/fine_tuning/finetune.py | 15 +++- fastvideo/distillation/validators/base.py | 3 + fastvideo/distillation/validators/wangame.py | 73 ++++++++++++++++--- 6 files changed, 104 insertions(+), 12 deletions(-) diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml index fb336b686..886fd0baf 100644 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml @@ -76,6 +76,7 @@ training: sampler_kind: ode ode_solver: euler guidance_scale: 1.0 + num_frames: 81 default_pipeline_config: flow_shift: 3 diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index ca83f9195..1c2ce4db4 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -490,6 +490,14 @@ def log_validation(self, iteration: int) -> None: f"{type(output_dir).__name__}" ) + num_actions = get_optional_int( + self.validation_config, + "num_frames", + where="training.validation.num_frames", + ) + if num_actions is not None and num_actions <= 0: + raise ValueError("training.validation.num_frames must be > 0 when set") + request = ValidationRequest( sample_handle=self.student, dataset_file=dataset_file, @@ -499,6 +507,7 @@ def log_validation(self, iteration: int) -> None: ode_solver=ode_solver, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, + num_frames=num_actions, output_dir=output_dir, ) validator.log_validation(iteration, request=request) diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index 607d37203..13aa4bf67 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -15,7 +15,11 @@ from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.dispatch import register_method from fastvideo.distillation.roles import RoleHandle, RoleManager -from fastvideo.distillation.utils.config import DistillRunConfig, parse_betas +from fastvideo.distillation.utils.config import ( + DistillRunConfig, + get_optional_int, + parse_betas, +) from fastvideo.distillation.validators.base import ValidationRequest @@ -412,6 +416,14 @@ def log_validation(self, iteration: int) -> None: f"{type(output_dir).__name__}" ) + num_actions = get_optional_int( + self.validation_config, + "num_frames", + where="training.validation.num_frames", + ) + if num_actions is not None and num_actions <= 0: + raise ValueError("training.validation.num_frames must be > 0 when set") + request = ValidationRequest( sample_handle=self.student, dataset_file=dataset_file, @@ -421,6 +433,7 @@ def log_validation(self, iteration: int) -> None: ode_solver=ode_solver, sampling_timesteps=None, guidance_scale=guidance_scale, + num_frames=num_actions, output_dir=output_dir, ) validator.log_validation(iteration, request=request) diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 405ef6be4..87462d36b 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -16,7 +16,11 @@ from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.dispatch import register_method from fastvideo.distillation.validators.base import ValidationRequest -from fastvideo.distillation.utils.config import DistillRunConfig, parse_betas +from fastvideo.distillation.utils.config import ( + DistillRunConfig, + get_optional_int, + parse_betas, +) class _FineTuneAdapter(Protocol): @@ -344,6 +348,14 @@ def log_validation(self, iteration: int) -> None: f"{type(output_dir).__name__}" ) + num_actions = get_optional_int( + self.validation_config, + "num_frames", + where="training.validation.num_frames", + ) + if num_actions is not None and num_actions <= 0: + raise ValueError("training.validation.num_frames must be > 0 when set") + request = ValidationRequest( sample_handle=self.student, dataset_file=dataset_file, @@ -353,6 +365,7 @@ def log_validation(self, iteration: int) -> None: ode_solver=ode_solver, sampling_timesteps=None, guidance_scale=guidance_scale, + num_frames=num_actions, output_dir=output_dir, ) validator.log_validation(iteration, request=request) diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py index 40dbab99d..2aef6b401 100644 --- a/fastvideo/distillation/validators/base.py +++ b/fastvideo/distillation/validators/base.py @@ -26,6 +26,9 @@ class ValidationRequest: ode_solver: str | None = None sampling_timesteps: list[int] | None = None guidance_scale: float | None = None + # Optional override for validation video length. When set, validators may + # truncate/pad auxiliary sequences (e.g. WanGame keyboard/mouse) to match. + num_frames: int | None = None output_dir: str | None = None diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index b570ff739..f425db618 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -243,6 +243,7 @@ def _prepare_validation_batch( *, sampling_timesteps: list[int] | None = None, guidance_scale: float | None = None, + num_frames: int | None = None, ) -> ForwardBatch: training_args = self.training_args @@ -259,8 +260,11 @@ def _prepare_validation_batch( temporal_compression_factor = ( training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio ) - num_frames = (training_args.num_latent_t - 1) * temporal_compression_factor + 1 - sampling_param.num_frames = int(num_frames) + default_num_frames = (training_args.num_latent_t - 1) * temporal_compression_factor + 1 + if num_frames is not None: + sampling_param.num_frames = int(num_frames) + else: + sampling_param.num_frames = int(default_num_frames) latents_size = [ (sampling_param.num_frames - 1) // 4 + 1, @@ -289,16 +293,48 @@ def _prepare_validation_batch( batch.pil_image = validation_batch["image"] if "keyboard_cond" in validation_batch and validation_batch["keyboard_cond"] is not None: - keyboard_cond = validation_batch["keyboard_cond"] - keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) - keyboard_cond = keyboard_cond.unsqueeze(0) - batch.keyboard_cond = keyboard_cond + keyboard_cond = torch.as_tensor(validation_batch["keyboard_cond"]).to( + dtype=torch.bfloat16 + ) + if keyboard_cond.ndim == 3 and keyboard_cond.shape[0] == 1: + keyboard_cond = keyboard_cond.squeeze(0) + if keyboard_cond.ndim != 2: + raise ValueError( + "validation keyboard_cond must have shape (T, K) (or (1, T, K)), " + f"got {tuple(keyboard_cond.shape)}" + ) + target_len = int(sampling_param.num_frames) + if keyboard_cond.shape[0] > target_len: + keyboard_cond = keyboard_cond[:target_len] + elif keyboard_cond.shape[0] < target_len: + pad = torch.zeros( + (target_len - keyboard_cond.shape[0], keyboard_cond.shape[1]), + dtype=keyboard_cond.dtype, + device=keyboard_cond.device, + ) + keyboard_cond = torch.cat([keyboard_cond, pad], dim=0) + batch.keyboard_cond = keyboard_cond.unsqueeze(0) if "mouse_cond" in validation_batch and validation_batch["mouse_cond"] is not None: - mouse_cond = validation_batch["mouse_cond"] - mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) - mouse_cond = mouse_cond.unsqueeze(0) - batch.mouse_cond = mouse_cond + mouse_cond = torch.as_tensor(validation_batch["mouse_cond"]).to(dtype=torch.bfloat16) + if mouse_cond.ndim == 3 and mouse_cond.shape[0] == 1: + mouse_cond = mouse_cond.squeeze(0) + if mouse_cond.ndim != 2: + raise ValueError( + "validation mouse_cond must have shape (T, 2) (or (1, T, 2)), " + f"got {tuple(mouse_cond.shape)}" + ) + target_len = int(sampling_param.num_frames) + if mouse_cond.shape[0] > target_len: + mouse_cond = mouse_cond[:target_len] + elif mouse_cond.shape[0] < target_len: + pad = torch.zeros( + (target_len - mouse_cond.shape[0], mouse_cond.shape[1]), + dtype=mouse_cond.dtype, + device=mouse_cond.device, + ) + mouse_cond = torch.cat([mouse_cond, pad], dim=0) + batch.mouse_cond = mouse_cond.unsqueeze(0) return batch @@ -313,6 +349,7 @@ def _run_validation_for_steps( ode_solver: str | None, sampling_timesteps: list[int] | None = None, guidance_scale: float | None = None, + num_frames: int | None = None, ) -> _ValidationStepResult: training_args = self.training_args pipeline = self._get_pipeline( @@ -336,6 +373,7 @@ def _run_validation_for_steps( num_inference_steps, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, + num_frames=num_frames, ) assert batch.prompt is not None and isinstance(batch.prompt, str) @@ -385,6 +423,20 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) f"{rollout_mode_raw!r}" ) ode_solver = getattr(request, "ode_solver", None) + num_frames_raw = getattr(request, "num_frames", None) + if num_frames_raw is None: + num_frames = None + elif isinstance(num_frames_raw, bool): + raise ValueError("ValidationRequest.num_frames must be an int when set") + elif isinstance(num_frames_raw, int): + num_frames = int(num_frames_raw) + else: + raise ValueError( + "ValidationRequest.num_frames must be an int when set, got " + f"{type(num_frames_raw).__name__}" + ) + if num_frames is not None and num_frames <= 0: + raise ValueError("ValidationRequest.num_frames must be > 0 when set") if rollout_mode == "streaming": sampler_kind_norm = str(sampler_kind).strip().lower() if sampler_kind_norm not in {"ode", "sde"}: @@ -450,6 +502,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) ode_solver=str(ode_solver) if ode_solver is not None else None, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, + num_frames=num_frames, ) if self.rank_in_sp_group != 0: From 8253a9a6fe632d39c39fcc1e541987dc4826648f Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 21:39:39 +0000 Subject: [PATCH 133/214] fix scheduler out of bound --- .../pipelines/stages/matrixgame_denoising.py | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/fastvideo/pipelines/stages/matrixgame_denoising.py b/fastvideo/pipelines/stages/matrixgame_denoising.py index de0d97e4e..4f4b5c89a 100644 --- a/fastvideo/pipelines/stages/matrixgame_denoising.py +++ b/fastvideo/pipelines/stages/matrixgame_denoising.py @@ -364,9 +364,9 @@ def _initialize_kv_cache(self, batch_size: int, dtype: torch.dtype, dtype=dtype, device=device), "global_end_index": - 0, + torch.zeros((), dtype=torch.long, device=device), "local_end_index": - 0, + torch.zeros((), dtype=torch.long, device=device), }) return kv_cache @@ -402,9 +402,9 @@ def _initialize_action_kv_cache(self, batch_size: int, dtype: torch.dtype, dtype=dtype, device=device), "global_end_index": - 0, + torch.zeros((), dtype=torch.long, device=device), "local_end_index": - 0, + torch.zeros((), dtype=torch.long, device=device), }) kv_cache_mouse.append({ "k": @@ -422,9 +422,9 @@ def _initialize_action_kv_cache(self, batch_size: int, dtype: torch.dtype, dtype=dtype, device=device), "global_end_index": - 0, + torch.zeros((), dtype=torch.long, device=device), "local_end_index": - 0, + torch.zeros((), dtype=torch.long, device=device), }) return kv_cache_mouse, kv_cache_keyboard @@ -878,6 +878,12 @@ def forward( current_latents = latents[:, :, start_index:start_index + current_num_frames, :, :] + # The scheduler maintains an internal `step_index` (and potentially + # additional multistep state, e.g. UniPC). Since causal streaming runs + # a full denoising trajectory *per block*, reset that state before + # each block rollout. + self._reset_scheduler_state_for_new_rollout() + action_kwargs = self._prepare_action_kwargs( batch, start_index, current_num_frames) camera_action_kwargs = self._prepare_camera_action_kwargs( @@ -920,6 +926,36 @@ def forward( batch.latents = latents return batch + def _reset_scheduler_state_for_new_rollout(self) -> None: + scheduler = self.scheduler + + # Common diffusers-like state. + if hasattr(scheduler, "_step_index"): + scheduler._step_index = None # type: ignore[attr-defined] + if hasattr(scheduler, "_begin_index"): + scheduler._begin_index = None # type: ignore[attr-defined] + + # UniPC multistep state (FlowUniPCMultistepScheduler) needs additional reset + # between independent trajectories. + if hasattr(scheduler, "model_outputs") and hasattr(scheduler, "config"): + try: + solver_order = int(getattr(scheduler.config, "solver_order", 0) or 0) + except Exception: + solver_order = 0 + if solver_order > 0: + scheduler.model_outputs = [None] * solver_order # type: ignore[attr-defined] + if hasattr(scheduler, "timestep_list") and hasattr(scheduler, "config"): + try: + solver_order = int(getattr(scheduler.config, "solver_order", 0) or 0) + except Exception: + solver_order = 0 + if solver_order > 0: + scheduler.timestep_list = [None] * solver_order # type: ignore[attr-defined] + if hasattr(scheduler, "lower_order_nums"): + scheduler.lower_order_nums = 0 # type: ignore[attr-defined] + if hasattr(scheduler, "last_sample"): + scheduler.last_sample = None # type: ignore[attr-defined] + def _process_single_block_ode( self, *, From 573699f6afbe6ca69918c1fe8d8eba9f14431004 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sat, 28 Feb 2026 23:59:05 +0000 Subject: [PATCH 134/214] adapter -> models. config decleartion. --- fastvideo/distillation/adapters/__init__.py | 8 - fastvideo/distillation/adapters/base.py | 21 - fastvideo/distillation/adapters/wan.py | 600 --------------- fastvideo/distillation/adapters/wangame.py | 682 ----------------- fastvideo/distillation/doc/README.md | 9 +- .../distillation/doc/adapters/__init__.md | 8 - fastvideo/distillation/doc/adapters/base.md | 17 - fastvideo/distillation/doc/adapters/wan.md | 31 - .../distillation/doc/adapters/wangame.md | 44 -- fastvideo/distillation/doc/models/adapter.md | 28 + .../distillation/doc/models/components.md | 21 + .../methods/distribution_matching/dmd2.py | 24 + .../distillation/methods/fine_tuning/dfsft.py | 19 + .../methods/fine_tuning/finetune.py | 17 + fastvideo/distillation/models/adapter.py | 99 +++ fastvideo/distillation/models/wan.py | 633 ++++++++++++++- fastvideo/distillation/models/wangame.py | 719 +++++++++++++++++- fastvideo/distillation/validators/wan.py | 15 + fastvideo/distillation/validators/wangame.py | 16 + 19 files changed, 1573 insertions(+), 1438 deletions(-) delete mode 100644 fastvideo/distillation/adapters/__init__.py delete mode 100644 fastvideo/distillation/adapters/base.py delete mode 100644 fastvideo/distillation/adapters/wan.py delete mode 100644 fastvideo/distillation/adapters/wangame.py delete mode 100644 fastvideo/distillation/doc/adapters/__init__.md delete mode 100644 fastvideo/distillation/doc/adapters/base.md delete mode 100644 fastvideo/distillation/doc/adapters/wan.md delete mode 100644 fastvideo/distillation/doc/adapters/wangame.md create mode 100644 fastvideo/distillation/doc/models/adapter.md create mode 100644 fastvideo/distillation/doc/models/components.md create mode 100644 fastvideo/distillation/models/adapter.py diff --git a/fastvideo/distillation/adapters/__init__.py b/fastvideo/distillation/adapters/__init__.py deleted file mode 100644 index c4c72f746..000000000 --- a/fastvideo/distillation/adapters/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from fastvideo.distillation.adapters.base import DistillAdapter - -__all__ = [ - "DistillAdapter", -] - diff --git a/fastvideo/distillation/adapters/base.py b/fastvideo/distillation/adapters/base.py deleted file mode 100644 index fa40ea280..000000000 --- a/fastvideo/distillation/adapters/base.py +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Literal - -if TYPE_CHECKING: - from fastvideo.pipelines import TrainingBatch - - -class DistillAdapter(ABC): - @abstractmethod - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", - ) -> TrainingBatch: - raise NotImplementedError diff --git a/fastvideo/distillation/adapters/wan.py b/fastvideo/distillation/adapters/wan.py deleted file mode 100644 index bb190bb32..000000000 --- a/fastvideo/distillation/adapters/wan.py +++ /dev/null @@ -1,600 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import copy -import gc -from typing import Any, Literal - -import torch - -import fastvideo.envs as envs -from fastvideo.configs.sample import SamplingParam -from fastvideo.distributed import ( - get_local_torch_device, - get_sp_group, - get_world_group, -) -from fastvideo.forward_context import set_forward_context -from fastvideo.models.utils import pred_noise_to_pred_video -from fastvideo.pipelines import TrainingBatch -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch -from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline -from fastvideo.training.training_utils import ( - compute_density_for_timestep_sampling, - get_sigmas, - normalize_dit_input, - shift_timestep, -) -from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed - -from fastvideo.distillation.adapters.base import DistillAdapter -from fastvideo.distillation.roles import RoleHandle - -try: - from fastvideo.attention.backends.video_sparse_attn import ( - VideoSparseAttentionMetadataBuilder, - ) - from fastvideo.attention.backends.vmoba import VideoMobaAttentionMetadataBuilder -except Exception: - VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] - VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] - - -class WanAdapter(DistillAdapter): - """ - Phase 1 target adapter: provide Wan-specific primitives without calling - legacy distillation pipeline algorithm helpers (e.g. pipeline-private forward wrappers). - """ - - def __init__( - self, - *, - prompt_handle: RoleHandle, - training_args: Any, - noise_scheduler: Any, - vae: Any, - ) -> None: - self.prompt_handle = prompt_handle - self.training_args = training_args - self.noise_scheduler = noise_scheduler - self.vae = vae - - self.world_group = get_world_group() - self.sp_group = get_sp_group() - self.device = get_local_torch_device() - - self.noise_random_generator: torch.Generator | None = None - self.noise_gen_cuda: torch.Generator | None = None - - self.negative_prompt_embeds: torch.Tensor | None = None - self.negative_prompt_attention_mask: torch.Tensor | None = None - - self._init_timestep_mechanics() - - def _get_training_dtype(self) -> torch.dtype: - return torch.bfloat16 - - def _init_timestep_mechanics(self) -> None: - self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) - self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) - self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) - self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) - - boundary_ratio = getattr(self.training_args, "boundary_ratio", None) - self.boundary_timestep: float | None = ( - float(boundary_ratio) * float(self.num_train_timestep) - if boundary_ratio is not None - else None - ) - - @property - def num_train_timesteps(self) -> int: - return int(self.num_train_timestep) - - def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: - timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) - return timestep.clamp(self.min_timestep, self.max_timestep) - - def on_train_start(self) -> None: - seed = self.training_args.seed - if seed is None: - raise ValueError("training_args.seed must be set for distillation") - - global_rank = int(getattr(self.world_group, "rank", 0)) - sp_world_size = int(getattr(self.training_args, "sp_size", 1) or 1) - if sp_world_size > 1: - sp_group_seed = int(seed) + (global_rank // sp_world_size) - set_random_seed(sp_group_seed) - else: - set_random_seed(int(seed) + global_rank) - - self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) - self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) - - self.ensure_negative_conditioning() - - def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - - generators: dict[str, torch.Generator] = {} - if self.noise_random_generator is not None: - generators["noise_cpu"] = self.noise_random_generator - if self.noise_gen_cuda is not None: - generators["noise_cuda"] = self.noise_gen_cuda - - return generators - - def ensure_negative_conditioning(self) -> None: - if self.negative_prompt_embeds is not None: - return - - training_args = self.training_args - world_group = self.world_group - device = self.device - dtype = self._get_training_dtype() - - neg_embeds: torch.Tensor | None = None - neg_mask: torch.Tensor | None = None - - if world_group.rank_in_group == 0: - sampling_param = SamplingParam.from_pretrained(training_args.model_path) - negative_prompt = sampling_param.negative_prompt - - args_copy = copy.deepcopy(training_args) - args_copy.inference_mode = True - - prompt_transformer = self.prompt_handle.require_module("transformer") - prompt_pipeline = WanPipeline.from_pretrained( - training_args.model_path, - args=args_copy, - inference_mode=True, - loaded_modules={"transformer": prompt_transformer}, - tp_size=training_args.tp_size, - sp_size=training_args.sp_size, - num_gpus=training_args.num_gpus, - pin_cpu_memory=training_args.pin_cpu_memory, - dit_cpu_offload=True, - ) - - batch_negative = ForwardBatch( - data_type="video", - prompt=negative_prompt, - prompt_embeds=[], - prompt_attention_mask=[], - ) - result_batch = prompt_pipeline.prompt_encoding_stage( # type: ignore[attr-defined] - batch_negative, - training_args, - ) - - neg_embeds = result_batch.prompt_embeds[0].to(device=device, dtype=dtype) - neg_mask = result_batch.prompt_attention_mask[0].to(device=device, dtype=dtype) - - del prompt_pipeline - gc.collect() - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - meta = torch.zeros((2,), device=device, dtype=torch.int64) - if world_group.rank_in_group == 0: - assert neg_embeds is not None - assert neg_mask is not None - meta[0] = neg_embeds.ndim - meta[1] = neg_mask.ndim - world_group.broadcast(meta, src=0) - embed_ndim, mask_ndim = (int(meta[0].item()), int(meta[1].item())) - - max_ndim = 8 - embed_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) - mask_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) - if world_group.rank_in_group == 0: - assert neg_embeds is not None - assert neg_mask is not None - embed_shape[:embed_ndim] = torch.tensor( - list(neg_embeds.shape), device=device, dtype=torch.int64 - ) - mask_shape[:mask_ndim] = torch.tensor( - list(neg_mask.shape), device=device, dtype=torch.int64 - ) - world_group.broadcast(embed_shape, src=0) - world_group.broadcast(mask_shape, src=0) - - embed_sizes = tuple(int(x) for x in embed_shape[:embed_ndim].tolist()) - mask_sizes = tuple(int(x) for x in mask_shape[:mask_ndim].tolist()) - - if world_group.rank_in_group != 0: - neg_embeds = torch.empty(embed_sizes, device=device, dtype=dtype) - neg_mask = torch.empty(mask_sizes, device=device, dtype=dtype) - assert neg_embeds is not None - assert neg_mask is not None - - world_group.broadcast(neg_embeds, src=0) - world_group.broadcast(neg_mask, src=0) - - self.negative_prompt_embeds = neg_embeds - self.negative_prompt_attention_mask = neg_mask - - def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: - if self.noise_random_generator is None: - raise RuntimeError("WanAdapter.on_train_start() must be called before prepare_batch()") - - u = compute_density_for_timestep_sampling( - weighting_scheme=self.training_args.weighting_scheme, - batch_size=batch_size, - generator=self.noise_random_generator, - logit_mean=self.training_args.logit_mean, - logit_std=self.training_args.logit_std, - mode_scale=self.training_args.mode_scale, - ) - indices = (u * self.noise_scheduler.config.num_train_timesteps).long() - return self.noise_scheduler.timesteps[indices].to(device=device) - - def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: - latents_shape = training_batch.raw_latent_shape - patch_size = self.training_args.pipeline_config.dit_config.patch_size - current_vsa_sparsity = training_batch.current_vsa_sparsity - assert latents_shape is not None - assert training_batch.timesteps is not None - - if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": - if not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None: - raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VIDEO_SPARSE_ATTN, but fastvideo_kernel " - "is not correctly installed or detected." - ) - training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] - raw_latent_shape=latents_shape[2:5], - current_timestep=training_batch.timesteps, - patch_size=patch_size, - VSA_sparsity=current_vsa_sparsity, - device=self.device, - ) - elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": - if not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None: - raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, but fastvideo_kernel " - "(or flash_attn>=2.7.4) is not correctly installed." - ) - moba_params = self.training_args.moba_config.copy() - moba_params.update( - { - "current_timestep": training_batch.timesteps, - "raw_latent_shape": training_batch.raw_latent_shape[2:5], - "patch_size": patch_size, - "device": self.device, - } - ) - training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(**moba_params) # type: ignore[misc] - else: - training_batch.attn_metadata = None - - return training_batch - - def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: - latents = training_batch.latents - batch_size = latents.shape[0] - if self.noise_gen_cuda is None: - raise RuntimeError("WanAdapter.on_train_start() must be called before prepare_batch()") - - noise = torch.randn( - latents.shape, - generator=self.noise_gen_cuda, - device=latents.device, - dtype=latents.dtype, - ) - timesteps = self._sample_timesteps(batch_size, latents.device) - if int(getattr(self.training_args, "sp_size", 1) or 1) > 1: - self.sp_group.broadcast(timesteps, src=0) - - sigmas = get_sigmas( - self.noise_scheduler, - latents.device, - timesteps, - n_dim=latents.ndim, - dtype=latents.dtype, - ) - noisy_model_input = (1.0 - sigmas) * latents + sigmas * noise - - training_batch.noisy_model_input = noisy_model_input - training_batch.timesteps = timesteps - training_batch.sigmas = sigmas - training_batch.noise = noise - training_batch.raw_latent_shape = latents.shape - - training_batch.conditional_dict = { - "encoder_hidden_states": training_batch.encoder_hidden_states, - "encoder_attention_mask": training_batch.encoder_attention_mask, - } - - if self.negative_prompt_embeds is not None and self.negative_prompt_attention_mask is not None: - neg_embeds = self.negative_prompt_embeds - neg_mask = self.negative_prompt_attention_mask - if neg_embeds.shape[0] == 1 and batch_size > 1: - neg_embeds = neg_embeds.expand(batch_size, *neg_embeds.shape[1:]).contiguous() - if neg_mask.shape[0] == 1 and batch_size > 1: - neg_mask = neg_mask.expand(batch_size, *neg_mask.shape[1:]).contiguous() - training_batch.unconditional_dict = { - "encoder_hidden_states": neg_embeds, - "encoder_attention_mask": neg_mask, - } - - training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) - return training_batch - - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", - ) -> TrainingBatch: - self.ensure_negative_conditioning() - - dtype = self._get_training_dtype() - device = self.device - - training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) - encoder_hidden_states = raw_batch["text_embedding"] - encoder_attention_mask = raw_batch["text_attention_mask"] - infos = raw_batch.get("info_list") - - if latents_source == "zeros": - batch_size = encoder_hidden_states.shape[0] - vae_config = self.training_args.pipeline_config.vae_config.arch_config - num_channels = vae_config.z_dim - spatial_compression_ratio = vae_config.spatial_compression_ratio - latent_height = self.training_args.num_height // spatial_compression_ratio - latent_width = self.training_args.num_width // spatial_compression_ratio - latents = torch.zeros( - batch_size, - num_channels, - self.training_args.num_latent_t, - latent_height, - latent_width, - device=device, - dtype=dtype, - ) - elif latents_source == "data": - if "vae_latent" not in raw_batch: - raise ValueError( - "vae_latent not found in batch and latents_source='data'" - ) - latents = raw_batch["vae_latent"] - latents = latents[:, :, : self.training_args.num_latent_t] - latents = latents.to(device, dtype=dtype) - else: - raise ValueError(f"Unknown latents_source: {latents_source!r}") - - training_batch.latents = latents - training_batch.encoder_hidden_states = encoder_hidden_states.to(device, dtype=dtype) - training_batch.encoder_attention_mask = encoder_attention_mask.to(device, dtype=dtype) - training_batch.infos = infos - - training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) - training_batch = self._prepare_dit_inputs(training_batch) - training_batch = self._build_attention_metadata(training_batch) - - training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) - if training_batch.attn_metadata is not None: - training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] - - return training_batch - - def add_noise( - self, - clean_latents: torch.Tensor, - noise: torch.Tensor, - timestep: torch.Tensor, - ) -> torch.Tensor: - b, t = clean_latents.shape[:2] - noisy = self.noise_scheduler.add_noise( - clean_latents.flatten(0, 1), - noise.flatten(0, 1), - timestep, - ).unflatten(0, (b, t)) - return noisy - - def _build_distill_input_kwargs( - self, - noise_input: torch.Tensor, - timestep: torch.Tensor, - text_dict: dict[str, torch.Tensor] | None, - ) -> dict[str, Any]: - if text_dict is None: - raise ValueError("text_dict cannot be None for Wan distillation") - return { - "hidden_states": noise_input.permute(0, 2, 1, 3, 4), - "encoder_hidden_states": text_dict["encoder_hidden_states"], - "encoder_attention_mask": text_dict["encoder_attention_mask"], - "timestep": timestep, - "return_dict": False, - } - - def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: - transformer = handle.require_module("transformer") - transformer_2 = handle.modules.get("transformer_2") - if ( - transformer_2 is not None - and self.boundary_timestep is not None - and float(timestep.item()) < float(self.boundary_timestep) - ): - return transformer_2 - return transformer - - def _get_uncond_text_dict( - self, - batch: TrainingBatch, - *, - cfg_uncond: dict[str, Any] | None, - ) -> dict[str, torch.Tensor]: - if cfg_uncond is None: - text_dict = getattr(batch, "unconditional_dict", None) - if text_dict is None: - raise RuntimeError( - "Missing unconditional_dict; ensure_negative_conditioning() may have failed" - ) - return text_dict - - on_missing_raw = cfg_uncond.get("on_missing", "error") - if not isinstance(on_missing_raw, str): - raise ValueError( - "method_config.cfg_uncond.on_missing must be a string, got " - f"{type(on_missing_raw).__name__}" - ) - on_missing = on_missing_raw.strip().lower() - if on_missing not in {"error", "ignore"}: - raise ValueError( - "method_config.cfg_uncond.on_missing must be one of {error, ignore}, got " - f"{on_missing_raw!r}" - ) - - # Wan only supports text CFG. If users configure other channels, fail - # fast (unless explicitly ignored). - for channel, policy_raw in cfg_uncond.items(): - if channel in {"on_missing", "text"}: - continue - if policy_raw is None: - continue - if not isinstance(policy_raw, str): - raise ValueError( - "method_config.cfg_uncond values must be strings, got " - f"{channel}={type(policy_raw).__name__}" - ) - policy = policy_raw.strip().lower() - if policy == "keep": - continue - if on_missing == "ignore": - continue - raise ValueError( - "WanAdapter does not support cfg_uncond channel " - f"{channel!r} (policy={policy!r}). " - "Set cfg_uncond.on_missing=ignore or remove the channel." - ) - - text_policy_raw = cfg_uncond.get("text", None) - if text_policy_raw is None: - text_policy = "negative_prompt" - elif not isinstance(text_policy_raw, str): - raise ValueError( - "method_config.cfg_uncond.text must be a string, got " - f"{type(text_policy_raw).__name__}" - ) - else: - text_policy = text_policy_raw.strip().lower() - - if text_policy in {"negative_prompt"}: - text_dict = getattr(batch, "unconditional_dict", None) - if text_dict is None: - raise RuntimeError( - "Missing unconditional_dict; ensure_negative_conditioning() may have failed" - ) - return text_dict - if text_policy == "keep": - if batch.conditional_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") - return batch.conditional_dict - if text_policy == "zero": - if batch.conditional_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") - cond = batch.conditional_dict - enc = cond["encoder_hidden_states"] - mask = cond["encoder_attention_mask"] - if not torch.is_tensor(enc) or not torch.is_tensor(mask): - raise TypeError("conditional_dict must contain tensor text inputs") - return { - "encoder_hidden_states": torch.zeros_like(enc), - "encoder_attention_mask": torch.zeros_like(mask), - } - if text_policy == "drop": - raise ValueError( - "cfg_uncond.text=drop is not supported for Wan. " - "Use {negative_prompt, keep, zero}." - ) - raise ValueError( - "cfg_uncond.text must be one of {negative_prompt, keep, zero, drop}, got " - f"{text_policy_raw!r}" - ) - - def predict_x0( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - if conditional: - text_dict = batch.conditional_dict - if text_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") - else: - text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) - return pred_x0 - - def predict_noise( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - if conditional: - text_dict = batch.conditional_dict - if text_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") - else: - text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - return pred_noise - - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: - timesteps, attn_metadata = ctx - with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): - (loss / max(1, int(grad_accum_rounds))).backward() diff --git a/fastvideo/distillation/adapters/wangame.py b/fastvideo/distillation/adapters/wangame.py deleted file mode 100644 index bb6881d7a..000000000 --- a/fastvideo/distillation/adapters/wangame.py +++ /dev/null @@ -1,682 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import copy -from typing import Any, Literal - -import torch - -import fastvideo.envs as envs -from fastvideo.distributed import ( - get_local_torch_device, - get_sp_group, - get_world_group, -) -from fastvideo.forward_context import set_forward_context -from fastvideo.models.utils import pred_noise_to_pred_video -from fastvideo.pipelines import TrainingBatch -from fastvideo.training.training_utils import ( - compute_density_for_timestep_sampling, - get_sigmas, - normalize_dit_input, - shift_timestep, -) -from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed - -from fastvideo.distillation.adapters.base import DistillAdapter -from fastvideo.distillation.roles import RoleHandle - -try: - from fastvideo.attention.backends.video_sparse_attn import ( - VideoSparseAttentionMetadataBuilder, - ) - from fastvideo.attention.backends.vmoba import VideoMobaAttentionMetadataBuilder -except Exception: - VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] - VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] - - -class WanGameAdapter(DistillAdapter): - """WanGame adapter: exposes operation-centric primitives for methods. - - This adapter is intentionally *not* method-specific: - - It knows how to turn a wangame parquet batch into forward primitives. - - It knows how to run a model role (handle) for predict_noise / predict_x0. - - It does not encode DMD2/SFT/etc semantics beyond required primitives. - """ - - def __init__( - self, - *, - training_args: Any, - noise_scheduler: Any, - vae: Any, - ) -> None: - self.training_args = training_args - self.noise_scheduler = noise_scheduler - self.vae = vae - - self.world_group = get_world_group() - self.sp_group = get_sp_group() - self.device = get_local_torch_device() - - self.noise_random_generator: torch.Generator | None = None - self.noise_gen_cuda: torch.Generator | None = None - - self._init_timestep_mechanics() - - def _get_training_dtype(self) -> torch.dtype: - return torch.bfloat16 - - def _init_timestep_mechanics(self) -> None: - self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) - self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) - self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) - self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) - - boundary_ratio = getattr(self.training_args, "boundary_ratio", None) - self.boundary_timestep: float | None = ( - float(boundary_ratio) * float(self.num_train_timestep) - if boundary_ratio is not None - else None - ) - - @property - def num_train_timesteps(self) -> int: - return int(self.num_train_timestep) - - def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: - timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) - return timestep.clamp(self.min_timestep, self.max_timestep) - - def on_train_start(self) -> None: - seed = getattr(self.training_args, "seed", None) - if seed is None: - raise ValueError("training_args.seed must be set for distillation") - - global_rank = int(getattr(self.world_group, "rank", 0)) - sp_world_size = int(getattr(self.training_args, "sp_size", 1) or 1) - if sp_world_size > 1: - sp_group_seed = int(seed) + (global_rank // sp_world_size) - set_random_seed(sp_group_seed) - else: - set_random_seed(int(seed) + global_rank) - - self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) - self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) - - def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - - generators: dict[str, torch.Generator] = {} - if self.noise_random_generator is not None: - generators["noise_cpu"] = self.noise_random_generator - if self.noise_gen_cuda is not None: - generators["noise_cuda"] = self.noise_gen_cuda - return generators - - def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: - if self.noise_random_generator is None: - raise RuntimeError("WanGameAdapter.on_train_start() must be called before prepare_batch()") - - u = compute_density_for_timestep_sampling( - weighting_scheme=self.training_args.weighting_scheme, - batch_size=batch_size, - generator=self.noise_random_generator, - logit_mean=self.training_args.logit_mean, - logit_std=self.training_args.logit_std, - mode_scale=self.training_args.mode_scale, - ) - indices = (u * self.noise_scheduler.config.num_train_timesteps).long() - return self.noise_scheduler.timesteps[indices].to(device=device) - - def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: - latents_shape = training_batch.raw_latent_shape - patch_size = self.training_args.pipeline_config.dit_config.patch_size - current_vsa_sparsity = training_batch.current_vsa_sparsity - assert latents_shape is not None - assert training_batch.timesteps is not None - - if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": - if not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None: - raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VIDEO_SPARSE_ATTN, but fastvideo_kernel " - "is not correctly installed or detected." - ) - training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] - raw_latent_shape=latents_shape[2:5], - current_timestep=training_batch.timesteps, - patch_size=patch_size, - VSA_sparsity=current_vsa_sparsity, - device=self.device, - ) - elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": - if not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None: - raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, but fastvideo_kernel " - "(or flash_attn>=2.7.4) is not correctly installed." - ) - moba_params = self.training_args.moba_config.copy() - moba_params.update( - { - "current_timestep": training_batch.timesteps, - "raw_latent_shape": training_batch.raw_latent_shape[2:5], - "patch_size": patch_size, - "device": self.device, - } - ) - training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(**moba_params) # type: ignore[misc] - else: - training_batch.attn_metadata = None - - return training_batch - - def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: - latents = training_batch.latents - assert isinstance(latents, torch.Tensor) - batch_size = latents.shape[0] - - if self.noise_gen_cuda is None: - raise RuntimeError("WanGameAdapter.on_train_start() must be called before prepare_batch()") - - noise = torch.randn( - latents.shape, - generator=self.noise_gen_cuda, - device=latents.device, - dtype=latents.dtype, - ) - timesteps = self._sample_timesteps(batch_size, latents.device) - if int(getattr(self.training_args, "sp_size", 1) or 1) > 1: - self.sp_group.broadcast(timesteps, src=0) - - sigmas = get_sigmas( - self.noise_scheduler, - latents.device, - timesteps, - n_dim=latents.ndim, - dtype=latents.dtype, - ) - noisy_model_input = (1.0 - sigmas) * latents + sigmas * noise - - training_batch.noisy_model_input = noisy_model_input - training_batch.timesteps = timesteps - training_batch.sigmas = sigmas - training_batch.noise = noise - training_batch.raw_latent_shape = latents.shape - - training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) - return training_batch - - def _build_i2v_mask_latents(self, image_latents: torch.Tensor) -> torch.Tensor: - temporal_compression_ratio = ( - self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - ) - num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 - - batch_size, _num_channels, _t, latent_height, latent_width = image_latents.shape - mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width) - mask_lat_size[:, :, 1:] = 0 - - first_frame_mask = mask_lat_size[:, :, :1] - first_frame_mask = torch.repeat_interleave( - first_frame_mask, - dim=2, - repeats=temporal_compression_ratio, - ) - mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], dim=2) - mask_lat_size = mask_lat_size.view( - batch_size, - -1, - temporal_compression_ratio, - latent_height, - latent_width, - ) - mask_lat_size = mask_lat_size.transpose(1, 2) - return mask_lat_size.to(device=image_latents.device, dtype=image_latents.dtype) - - def _process_actions(self, training_batch: TrainingBatch) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - keyboard_cond = getattr(training_batch, "keyboard_cond", None) - mouse_cond = getattr(training_batch, "mouse_cond", None) - if keyboard_cond is None or mouse_cond is None: - raise ValueError("WanGame batch must provide keyboard_cond and mouse_cond") - - from fastvideo.models.dits.hyworld.pose import process_custom_actions - - batch_size = int(training_batch.noisy_model_input.shape[0]) # type: ignore[union-attr] - viewmats_list: list[torch.Tensor] = [] - intrinsics_list: list[torch.Tensor] = [] - action_labels_list: list[torch.Tensor] = [] - for b in range(batch_size): - v, i, a = process_custom_actions(keyboard_cond[b], mouse_cond[b]) - viewmats_list.append(v) - intrinsics_list.append(i) - action_labels_list.append(a) - - viewmats = torch.stack(viewmats_list, dim=0).to(device=self.device, dtype=torch.bfloat16) - intrinsics = torch.stack(intrinsics_list, dim=0).to(device=self.device, dtype=torch.bfloat16) - action_labels = torch.stack(action_labels_list, dim=0).to(device=self.device, dtype=torch.bfloat16) - - num_latent_t = int(training_batch.noisy_model_input.shape[2]) # type: ignore[union-attr] - if int(action_labels.shape[1]) != num_latent_t: - raise ValueError( - "Action conditioning temporal dim mismatch: " - f"action={tuple(action_labels.shape)} vs latent_t={num_latent_t}" - ) - if int(viewmats.shape[1]) != num_latent_t: - raise ValueError( - "Viewmats temporal dim mismatch: " - f"viewmats={tuple(viewmats.shape)} vs latent_t={num_latent_t}" - ) - - return viewmats, intrinsics, action_labels - - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", - ) -> TrainingBatch: - dtype = self._get_training_dtype() - device = self.device - - training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) - infos = raw_batch.get("info_list") - - if latents_source == "zeros": - clip_feature = raw_batch["clip_feature"] - batch_size = int(clip_feature.shape[0]) - vae_config = self.training_args.pipeline_config.vae_config.arch_config - num_channels = int(vae_config.z_dim) - spatial_compression_ratio = int(vae_config.spatial_compression_ratio) - latent_height = int(self.training_args.num_height) // spatial_compression_ratio - latent_width = int(self.training_args.num_width) // spatial_compression_ratio - latents = torch.zeros( - batch_size, - num_channels, - int(self.training_args.num_latent_t), - latent_height, - latent_width, - device=device, - dtype=dtype, - ) - elif latents_source == "data": - if "vae_latent" not in raw_batch: - raise ValueError("vae_latent not found in batch and latents_source='data'") - latents = raw_batch["vae_latent"] - latents = latents[:, :, : self.training_args.num_latent_t] - latents = latents.to(device, dtype=dtype) - else: - raise ValueError(f"Unknown latents_source: {latents_source!r}") - - if "clip_feature" not in raw_batch: - raise ValueError("clip_feature must be present for WanGame") - image_embeds = raw_batch["clip_feature"].to(device, dtype=dtype) - - if "first_frame_latent" not in raw_batch: - raise ValueError("first_frame_latent must be present for WanGame") - image_latents = raw_batch["first_frame_latent"] - image_latents = image_latents[:, :, : self.training_args.num_latent_t] - image_latents = image_latents.to(device, dtype=dtype) - - pil_image = raw_batch.get("pil_image") - if isinstance(pil_image, torch.Tensor): - training_batch.preprocessed_image = pil_image.to(device=device) - else: - training_batch.preprocessed_image = pil_image - - keyboard_cond = raw_batch.get("keyboard_cond") - if isinstance(keyboard_cond, torch.Tensor) and keyboard_cond.numel() > 0: - training_batch.keyboard_cond = keyboard_cond.to(device, dtype=dtype) - else: - training_batch.keyboard_cond = None - - mouse_cond = raw_batch.get("mouse_cond") - if isinstance(mouse_cond, torch.Tensor) and mouse_cond.numel() > 0: - training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) - else: - training_batch.mouse_cond = None - - temporal_compression_ratio = ( - self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - ) - expected_num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 - if training_batch.keyboard_cond is not None and int(training_batch.keyboard_cond.shape[1]) != int( - expected_num_frames - ): - raise ValueError( - "keyboard_cond temporal dim mismatch: " - f"got {int(training_batch.keyboard_cond.shape[1])}, expected {int(expected_num_frames)}" - ) - if training_batch.mouse_cond is not None and int(training_batch.mouse_cond.shape[1]) != int( - expected_num_frames - ): - raise ValueError( - "mouse_cond temporal dim mismatch: " - f"got {int(training_batch.mouse_cond.shape[1])}, expected {int(expected_num_frames)}" - ) - - training_batch.latents = latents - training_batch.encoder_hidden_states = None - training_batch.encoder_attention_mask = None - training_batch.image_embeds = image_embeds - training_batch.image_latents = image_latents - training_batch.infos = infos - - training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) - training_batch = self._prepare_dit_inputs(training_batch) - training_batch = self._build_attention_metadata(training_batch) - - training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) - if training_batch.attn_metadata is not None: - training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] - - training_batch.mask_lat_size = self._build_i2v_mask_latents(image_latents) - viewmats, intrinsics, action_labels = self._process_actions(training_batch) - training_batch.viewmats = viewmats - training_batch.Ks = intrinsics - training_batch.action = action_labels - - return training_batch - - def add_noise( - self, - clean_latents: torch.Tensor, - noise: torch.Tensor, - timestep: torch.Tensor, - ) -> torch.Tensor: - b, t = clean_latents.shape[:2] - noisy = self.noise_scheduler.add_noise( - clean_latents.flatten(0, 1), - noise.flatten(0, 1), - timestep, - ).unflatten(0, (b, t)) - return noisy - - def _build_distill_input_kwargs( - self, - noisy_video_latents: torch.Tensor, - timestep: torch.Tensor, - *, - image_embeds: torch.Tensor, - image_latents: torch.Tensor, - mask_lat_size: torch.Tensor, - viewmats: torch.Tensor | None, - Ks: torch.Tensor | None, - action: torch.Tensor | None, - mouse_cond: torch.Tensor | None, - keyboard_cond: torch.Tensor | None, - ) -> dict[str, Any]: - hidden_states = torch.cat( - [ - noisy_video_latents.permute(0, 2, 1, 3, 4), - mask_lat_size, - image_latents, - ], - dim=1, - ) - return { - "hidden_states": hidden_states, - "encoder_hidden_states": None, - "timestep": timestep.to(device=self.device, dtype=torch.bfloat16), - "encoder_hidden_states_image": image_embeds, - "viewmats": viewmats, - "Ks": Ks, - "action": action, - "mouse_cond": mouse_cond, - "keyboard_cond": keyboard_cond, - "return_dict": False, - } - - def _select_cfg_condition_inputs( - self, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None, - ) -> dict[str, Any]: - image_embeds = batch.image_embeds - image_latents = batch.image_latents - mask_lat_size = batch.mask_lat_size - if image_embeds is None: - raise RuntimeError("WanGameAdapter requires TrainingBatch.image_embeds") - if image_latents is None: - raise RuntimeError("WanGameAdapter requires TrainingBatch.image_latents") - if mask_lat_size is None: - raise RuntimeError("WanGameAdapter requires TrainingBatch.mask_lat_size") - - viewmats = getattr(batch, "viewmats", None) - Ks = getattr(batch, "Ks", None) - action = getattr(batch, "action", None) - mouse_cond = getattr(batch, "mouse_cond", None) - keyboard_cond = getattr(batch, "keyboard_cond", None) - - if conditional or cfg_uncond is None: - return { - "image_embeds": image_embeds, - "image_latents": image_latents, - "mask_lat_size": mask_lat_size, - "viewmats": viewmats, - "Ks": Ks, - "action": action, - "mouse_cond": mouse_cond, - "keyboard_cond": keyboard_cond, - } - - on_missing_raw = cfg_uncond.get("on_missing", "error") - if not isinstance(on_missing_raw, str): - raise ValueError( - "method_config.cfg_uncond.on_missing must be a string, got " - f"{type(on_missing_raw).__name__}" - ) - on_missing = on_missing_raw.strip().lower() - if on_missing not in {"error", "ignore"}: - raise ValueError( - "method_config.cfg_uncond.on_missing must be one of {error, ignore}, got " - f"{on_missing_raw!r}" - ) - - supported_channels = {"image", "action"} - for channel, policy_raw in cfg_uncond.items(): - if channel in {"on_missing"}: - continue - if channel in supported_channels: - continue - if policy_raw is None: - continue - if not isinstance(policy_raw, str): - raise ValueError( - "method_config.cfg_uncond values must be strings, got " - f"{channel}={type(policy_raw).__name__}" - ) - policy = policy_raw.strip().lower() - if policy == "keep": - continue - if on_missing == "ignore": - continue - raise ValueError( - "WanGameAdapter does not support cfg_uncond channel " - f"{channel!r} (policy={policy!r}). " - "Set cfg_uncond.on_missing=ignore or remove the channel." - ) - - def _get_policy(channel: str) -> str: - raw = cfg_uncond.get(channel, "keep") - if raw is None: - return "keep" - if not isinstance(raw, str): - raise ValueError( - "method_config.cfg_uncond values must be strings, got " - f"{channel}={type(raw).__name__}" - ) - policy = raw.strip().lower() - if policy not in {"keep", "zero", "drop"}: - raise ValueError( - "method_config.cfg_uncond values must be one of {keep, zero, drop}, got " - f"{channel}={raw!r}" - ) - return policy - - image_policy = _get_policy("image") - if image_policy == "zero": - image_embeds = torch.zeros_like(image_embeds) - image_latents = torch.zeros_like(image_latents) - mask_lat_size = torch.zeros_like(mask_lat_size) - elif image_policy == "drop": - raise ValueError( - "cfg_uncond.image=drop is not supported for WanGame I2V; " - "use cfg_uncond.image=zero or keep." - ) - - action_policy = _get_policy("action") - if action_policy == "zero": - if viewmats is None or Ks is None or action is None: - if on_missing == "ignore": - pass - else: - raise ValueError( - "cfg_uncond.action=zero requires action conditioning tensors, " - "but TrainingBatch is missing {viewmats, Ks, action}." - ) - else: - viewmats = torch.zeros_like(viewmats) - Ks = torch.zeros_like(Ks) - action = torch.zeros_like(action) - if mouse_cond is not None: - mouse_cond = torch.zeros_like(mouse_cond) - if keyboard_cond is not None: - keyboard_cond = torch.zeros_like(keyboard_cond) - elif action_policy == "drop": - viewmats = None - Ks = None - action = None - mouse_cond = None - keyboard_cond = None - - return { - "image_embeds": image_embeds, - "image_latents": image_latents, - "mask_lat_size": mask_lat_size, - "viewmats": viewmats, - "Ks": Ks, - "action": action, - "mouse_cond": mouse_cond, - "keyboard_cond": keyboard_cond, - } - - def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: - transformer = handle.require_module("transformer") - transformer_2 = handle.modules.get("transformer_2") - if transformer_2 is not None and self.boundary_timestep is not None: - if timestep.numel() != 1: - raise ValueError( - "MoE boundary selection requires a scalar timestep, got " - f"shape={tuple(timestep.shape)}" - ) - if float(timestep.item()) < float(self.boundary_timestep): - return transformer_2 - return transformer - - def predict_x0( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - cond_inputs = self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - ) - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs["image_embeds"], - image_latents=cond_inputs["image_latents"], - mask_lat_size=cond_inputs["mask_lat_size"], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs["keyboard_cond"], - ) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) - return pred_x0 - - def predict_noise( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - cond_inputs = self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - ) - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs["image_embeds"], - image_latents=cond_inputs["image_latents"], - mask_lat_size=cond_inputs["mask_lat_size"], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs["keyboard_cond"], - ) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - return pred_noise - - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: - timesteps, attn_metadata = ctx - with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): - (loss / max(1, int(grad_accum_rounds))).backward() diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index ab6fc823b..243af47b8 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -7,6 +7,7 @@ - **Method** 只做算法（loss + update policy + 需要哪些 roles）。 - **Model plugin** 只做装配（build-time：加载 modules、构建 bundle/adapter/dataloader/validator；代码在 `models/`）。 - **Adapter** 只做运行时 primitive（step-time：prepare_batch/forward_context/predict/backward 等）， + 并与对应 model plugin **共置于 `models/*.py`**（例如 `models/wan.py:WanAdapter`）。 API 以 operation 为中心，不以 role 为中心（避免 role 爆炸）。 快速入口（从运行到训练）： @@ -31,14 +32,12 @@ - `roles.md` - `trainer.md` -### adapters/ -- `adapters/__init__.md` -- `adapters/base.md` -- `adapters/wan.md` - ### models/ - `models/__init__.md` +- `models/adapter.md` +- `models/components.md` - `models/wan.md` +- `models/wangame.md` ### methods/ - `methods/__init__.md` diff --git a/fastvideo/distillation/doc/adapters/__init__.md b/fastvideo/distillation/doc/adapters/__init__.md deleted file mode 100644 index 6f061234d..000000000 --- a/fastvideo/distillation/doc/adapters/__init__.md +++ /dev/null @@ -1,8 +0,0 @@ -# `fastvideo/distillation/adapters/__init__.py` - -**目的** -- 统一导出 adapter 抽象基类 `DistillAdapter`，避免上层直接 import 具体实现。 - -**当前导出** -- `DistillAdapter`（见 `adapters/base.md`） - diff --git a/fastvideo/distillation/doc/adapters/base.md b/fastvideo/distillation/doc/adapters/base.md deleted file mode 100644 index deee3abb2..000000000 --- a/fastvideo/distillation/doc/adapters/base.md +++ /dev/null @@ -1,17 +0,0 @@ -# `fastvideo/distillation/adapters/base.py` - -**目的** -- 定义 adapter 的最小抽象接口。 - -**当前最小契约** -- `prepare_batch(raw_batch, current_vsa_sparsity=...) -> TrainingBatch` - -**为什么接口这么小？** -- adapter 的“完整能力”通常是 **method-specific protocol**（duck typing），例如： - - `predict_x0(handle, ...)` - - `predict_noise(handle, ...)` - - `add_noise(...)` - - `backward(loss, ctx, ...)` -- 这些能力应由具体 method 在自己的 `Protocol` 里声明（例如 `DMD2Method` 的 `_DMD2Adapter`）， - 从而保持 adapter 的基类稳定、method 的需求显式可读。 - diff --git a/fastvideo/distillation/doc/adapters/wan.md b/fastvideo/distillation/doc/adapters/wan.md deleted file mode 100644 index e6dff910d..000000000 --- a/fastvideo/distillation/doc/adapters/wan.md +++ /dev/null @@ -1,31 +0,0 @@ -# `fastvideo/distillation/adapters/wan.py` - -**定位** -- `WanAdapter` 是 Wan model plugin 的 runtime 边界： - - 把 FastVideo/Wan 的 batch schema、forward_context、attention metadata 等细节 - 封装为一组 **operation-centric primitives** - - 不实现任何 method 的 rollout policy / step list / loss（这些属于 method） - -**关键依赖** -- `RoleHandle`：adapter 不认识 role 字符串，method 传 handle 进来，adapter 只用 handle 拿模块。 -- `fastvideo.forward_context.set_forward_context`：Wan forward/backward 依赖全局上下文。 -- attention metadata builder（VSA / VMOBA）与 `envs.FASTVIDEO_ATTENTION_BACKEND`。 - -**主要 API（被 method 通过 protocol 调用）** -- `prepare_batch(...) -> TrainingBatch` - - 处理 raw_batch → latents/noise/timesteps/sigmas - - 构建 `conditional_dict` / `unconditional_dict`（含 negative prompt embeds） - - 构建 attention metadata（dense / vsa 两套） -- `predict_x0(handle, noisy_latents, timestep, batch, conditional, attn_kind) -> Tensor` -- `predict_noise(handle, noisy_latents, timestep, batch, conditional, attn_kind) -> Tensor` -- `add_noise(clean_latents, noise, timestep) -> Tensor` -- `shift_and_clamp_timestep(t) -> Tensor` + `num_train_timesteps` -- `backward(loss, ctx, grad_accum_rounds=...)` - -**关于 negative/unconditional conditioning** -- `ensure_negative_conditioning()` 只做 prompt encoding（无 denoise）。 -- 为避免算法命名耦合，prompt encoding 使用 `WanPipeline`（而不是带 method 语义的 pipeline 名称）。 - -**边界（Phase 2.9）** -- ✅ adapter 不保存/管理 few-step 的 denoising step list，也不决定 rollout 策略。 -- ✅ adapter 不区分 `student/teacher/critic` 的专用方法；只提供通用操作，role 语义由 method 管理。 diff --git a/fastvideo/distillation/doc/adapters/wangame.md b/fastvideo/distillation/doc/adapters/wangame.md deleted file mode 100644 index 04d1e89d7..000000000 --- a/fastvideo/distillation/doc/adapters/wangame.md +++ /dev/null @@ -1,44 +0,0 @@ -# `fastvideo/distillation/adapters/wangame.py` - -**定位** -- `WanGameAdapter` 是 WanGame model plugin 的 runtime 边界： - - 把 wangame parquet batch（I2V + action）转成 method 可消费的 - **operation-centric primitives** - - 不实现任何 method 的 rollout policy / step list / loss（这些属于 method） - -**和 `WanAdapter` 的关键差异** -- WanGame 是 **I2V + action conditioning**： - - transformer 的 `hidden_states` 不是纯视频 noisy latent，而是： - `cat([noisy_video_latents, mask_lat_size, image_latents], dim=1)` - - 还需要额外输入： - - `encoder_hidden_states_image`（来自 `clip_feature`） - - bidirectional transformer：`viewmats / Ks / action` - （由 `process_custom_actions(keyboard, mouse)` 生成） - - causal transformer：`mouse_cond / keyboard_cond`（raw action sequences） -- 目前 WanGame 的 `conditional/unconditional`（文本 CFG）语义**不成立**： - - adapter 仍保留 `conditional: bool` 形参以匹配 method protocol， - 但当前实现把它当作 no-op（cond/uncond 同路）。 - -**主要 API（被 method 通过 protocol 调用）** -- `prepare_batch(...) -> TrainingBatch` - - 处理 raw batch： - - `vae_latent`（video x0） - - `first_frame_latent`（I2V image latents） - - `clip_feature`（image embeds） - - `keyboard_cond` / `mouse_cond`（action） - - 采样 timesteps + 生成 noise/sigmas + 生成 noisy video latents - - 构建 attention metadata（dense / vsa 两套） - - 生成 `mask_lat_size` 并预处理 action（`viewmats/Ks/action`） -- `predict_noise(handle, noisy_latents, timestep, batch, conditional, attn_kind) -> Tensor` -- `predict_x0(handle, noisy_latents, timestep, batch, conditional, attn_kind) -> Tensor` -- `add_noise(clean_latents, noise, timestep) -> Tensor` -- `shift_and_clamp_timestep(t) -> Tensor` + `num_train_timesteps` -- `backward(loss, ctx, grad_accum_rounds=...)` - -**边界 / TODO** -- ✅ adapter 不保存/管理 few-step denoising step list，也不决定 rollout 策略。 -- ✅ adapter 不引入 DMD2 专属概念（例如 “generator/critic”）。 -- ✅ adapter 支持 **per-frame timesteps**（例如 DFSFT 的 `t_inhom`），但当启用 MoE - `transformer_2 + boundary_timestep` 时要求 timestep 为标量（否则无法定义“跨帧选择哪个 transformer”）。 -- TODO：若未来需要在 wangame 上定义 “uncond” 语义（例如 `zero_action/zero_image`）， - 应通过 `method_config` 声明，并由 adapter 提供可解释的操作入口（而不是硬编码到 adapter 内部逻辑）。 diff --git a/fastvideo/distillation/doc/models/adapter.md b/fastvideo/distillation/doc/models/adapter.md new file mode 100644 index 000000000..297b52f4c --- /dev/null +++ b/fastvideo/distillation/doc/models/adapter.md @@ -0,0 +1,28 @@ +# `fastvideo/distillation/models/adapter.py` + +**定位** +- 定义 `ModelAdapter` 抽象接口：由 model plugin 提供的 *运行时 primitive*（operation-centric）。 +- Method 层不应该 import/依赖任何具体 pipeline/model，只依赖这些 primitive（duck typing / 抽象基类）。 + +**核心思想** +- Adapter **不关心 roles 语义**（student/teacher/critic/... 由 method 定义）。 +- Adapter 提供 “对某个 role handle 执行某个操作” 的 API： + - `predict_noise(handle, ...)` + - `predict_x0(handle, ...)` + - `add_noise(...)` + - `prepare_batch(...)` +- 这样可以避免 “每个 role 一个函数” 的 role 爆炸。 + +**接口概览（必需）** +- `prepare_batch(raw_batch, current_vsa_sparsity, latents_source) -> TrainingBatch` +- `add_noise(clean_latents, noise, timestep) -> Tensor` +- `predict_noise(handle, noisy_latents, timestep, batch, conditional, cfg_uncond?, attn_kind) -> Tensor` +- `predict_x0(handle, noisy_latents, timestep, batch, conditional, cfg_uncond?, attn_kind) -> Tensor` +- `num_train_timesteps` / `shift_and_clamp_timestep(timestep)` +- `on_train_start()` +- `backward(loss, ctx, grad_accum_rounds)` + +**接口概览（可选）** +- `get_rng_generators() -> dict[str, torch.Generator]` + - Trainer/ckpt manager 用于保存 RNG state，实现 “exact resume”。 + diff --git a/fastvideo/distillation/doc/models/components.md b/fastvideo/distillation/doc/models/components.md new file mode 100644 index 000000000..9724eb7e5 --- /dev/null +++ b/fastvideo/distillation/doc/models/components.md @@ -0,0 +1,21 @@ +# `fastvideo/distillation/models/components.py` + +**定位** +- `ModelComponents` 是 model plugin 的 build-time 产物（装配输出）。 +- `dispatch.build_runtime_from_config()` 先构建 `ModelComponents`，再把其中的 + `bundle/adapter/validator` 注入到 method，最终交给 trainer 执行训练。 + +**字段说明** +- `training_args` + - 来自 YAML 的 `training`（通过 `TrainingArgs.from_kwargs` 解析）。 +- `bundle` + - `RoleManager(roles=...)`：role → `RoleHandle(modules/optimizers/schedulers/...)`。 +- `adapter` + - 对应模型家族的运行时 primitive（例如 `WanAdapter` / `WanGameAdapter`）。 +- `dataloader` + - 当前 run 的训练 dataloader（由通用 dataloader builder 构建）。 +- `validator`（可选） + - 模型家族对应的 validator backend；是否启用由 method 的 validation 配置决定。 +- `start_step` + - autoresume/ckpt 恢复后从哪个 step 开始继续。 + diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 1c2ce4db4..cc99e8150 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -1,5 +1,29 @@ # SPDX-License-Identifier: Apache-2.0 +"""DMD2 distillation method (algorithm layer). + +Config keys used (YAML schema-v2): +- `recipe.method`: must be `"dmd2"` for this method. +- `roles`: requires `student`, `teacher`, `critic` (trainable critic). +- `method_config`: + - `rollout_mode` (`simulate` or `data_latent`) + - `dmd_denoising_steps` (list[int]) + - `cfg_uncond` (optional): `on_missing` + channel policies + - optional overrides: `generator_update_interval`, `warp_denoising_step`, + `real_score_guidance_scale` +- `training` (selected fields used for optim/schedule): + - `learning_rate`, `betas`, `lr_scheduler` + - `fake_score_learning_rate`, `fake_score_betas`, `fake_score_lr_scheduler` + - `weight_decay`, `lr_warmup_steps`, `max_train_steps`, `lr_num_cycles`, + `lr_power`, `min_lr_ratio` + - `generator_update_interval`, `warp_denoising_step`, `real_score_guidance_scale`, + `max_grad_norm` +- `training.validation.*` (parsed by method; executed via validator): + - `enabled`, `every_steps`, `dataset_file`, `sampling_steps` + - optional: `sampling_timesteps`, `guidance_scale`, `sampler_kind`, `ode_solver`, + `rollout_mode`, `output_dir`, `num_frames` +""" + from __future__ import annotations from typing import Any, cast, Literal, Protocol diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index 13aa4bf67..bd0639d7a 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -1,5 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 +"""Diffusion-forcing SFT method (DFSFT; algorithm layer). + +Config keys used (YAML schema-v2): +- `recipe.method`: must be `"dfsft"` for this method. +- `roles`: requires `student` (and `roles.student.trainable=true`). +- `method_config`: + - `attn_kind` (optional): `dense` or `vsa` + - `chunk_size` (optional; default=3) + - `min_timestep_ratio` / `max_timestep_ratio` (optional) +- `training` (selected fields used for optim/schedule): + - `learning_rate`, `betas`, `lr_scheduler` + - `weight_decay`, `lr_warmup_steps`, `max_train_steps`, `lr_num_cycles`, + `lr_power`, `min_lr_ratio`, `max_grad_norm` +- `training.validation.*` (parsed by method; executed via validator): + - `enabled`, `every_steps`, `dataset_file`, `sampling_steps` + - optional: `guidance_scale`, `sampler_kind`, `ode_solver`, `rollout_mode`, + `output_dir`, `num_frames` +""" + from __future__ import annotations from typing import Any, Literal, Protocol, cast diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 87462d36b..0b044c957 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -1,5 +1,22 @@ # SPDX-License-Identifier: Apache-2.0 +"""Supervised finetuning method (algorithm layer). + +Config keys used (YAML schema-v2): +- `recipe.method`: must be `"finetune"` for this method. +- `roles`: requires `student` (and `roles.student.trainable=true`). +- `method_config`: + - `attn_kind` (optional): `dense` or `vsa` +- `training` (selected fields used for optim/schedule): + - `learning_rate`, `betas`, `lr_scheduler` + - `weight_decay`, `lr_warmup_steps`, `max_train_steps`, `lr_num_cycles`, + `lr_power`, `min_lr_ratio`, `max_grad_norm` +- `training.validation.*` (parsed by method; executed via validator): + - `enabled`, `every_steps`, `dataset_file`, `sampling_steps` + - optional: `guidance_scale`, `sampler_kind`, `ode_solver`, `rollout_mode`, + `output_dir`, `num_frames` +""" + from __future__ import annotations from typing import Any, Literal, Protocol, cast diff --git a/fastvideo/distillation/models/adapter.py b/fastvideo/distillation/models/adapter.py new file mode 100644 index 000000000..8ad3330a9 --- /dev/null +++ b/fastvideo/distillation/models/adapter.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Literal, TYPE_CHECKING + +import torch + +from fastvideo.distillation.roles import RoleHandle + +if TYPE_CHECKING: + from fastvideo.pipelines import TrainingBatch + + +class ModelAdapter(ABC): + """Operation-centric runtime primitives implemented by a model plugin. + + This interface is intentionally *method-agnostic*: + - A method selects roles (student/teacher/critic/...) and decides how to use + them. + - The adapter implements how to run those roles against FastVideo pipelines, + forward-context requirements, and batch normalization quirks. + + Implementations typically live next to the model plugin (e.g. `models/wan.py`) + rather than in a global adapter registry. + """ + + training_args: Any + + @property + @abstractmethod + def num_train_timesteps(self) -> int: + """Return the scheduler's training timestep horizon (usually 1000).""" + + @abstractmethod + def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: + """Apply model/pipeline timestep shifting and clamp into valid range.""" + + @abstractmethod + def on_train_start(self) -> None: + """Initialize RNG seeds and any cached conditioning needed for training.""" + + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + return {} + + @abstractmethod + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", + ) -> TrainingBatch: + """Convert a dataloader batch into forward primitives for methods.""" + + @abstractmethod + def add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + """Apply forward-process noise at `timestep` for a given scheduler.""" + + @abstractmethod + def predict_noise( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + """Run a role to predict noise/flow for the given noisy latents.""" + + @abstractmethod + def predict_x0( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + """Run a role to predict x0 for the given noisy latents.""" + + @abstractmethod + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + """Backward hook that may restore forward-context for checkpointed modules.""" + diff --git a/fastvideo/distillation/models/wan.py b/fastvideo/distillation/models/wan.py index 7c8e70d4e..1617f0aba 100644 --- a/fastvideo/distillation/models/wan.py +++ b/fastvideo/distillation/models/wan.py @@ -1,23 +1,632 @@ # SPDX-License-Identifier: Apache-2.0 +"""Wan model plugin (components + runtime adapter). + +Config keys used (YAML schema-v2): +- `recipe.family`: must be `"wan"` for this plugin. +- `roles.`: + - `family`, `path`, `trainable`, `disable_custom_init_weights` +- `training` (selected fields): + - `seed`, `data_path`, `model_path` + - `num_height`, `num_width`, `num_latent_t` + - `min_timestep_ratio`, `max_timestep_ratio`, `boundary_ratio` (optional) + - `weighting_scheme`, `logit_mean`, `logit_std`, `mode_scale` + - `sp_size`, `tp_size`, `num_gpus`, `pin_cpu_memory` + - `pipeline_config.flow_shift`, `pipeline_config.dit_config.patch_size` + - `moba_config` (if `FASTVIDEO_ATTENTION_BACKEND=VMOBA_ATTN`) + - `enable_gradient_checkpointing_type` (optional) +- `training.validation.*` (consumed by `WanValidator` when enabled) +- `method_config.cfg_uncond.*` (consumed by `WanAdapter` for CFG-uncond policy) +""" + from __future__ import annotations -from typing import Any +import copy +import gc +from typing import Any, Literal import torch -from fastvideo.distillation.adapters.wan import WanAdapter -from fastvideo.distillation.roles import RoleHandle, RoleManager +import fastvideo.envs as envs +from fastvideo.configs.sample import SamplingParam +from fastvideo.distributed import ( + get_local_torch_device, + get_sp_group, + get_world_group, +) +from fastvideo.forward_context import set_forward_context +from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from fastvideo.models.utils import pred_noise_to_pred_video +from fastvideo.pipelines import TrainingBatch +from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline +from fastvideo.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.training.activation_checkpoint import apply_activation_checkpointing +from fastvideo.training.training_utils import ( + compute_density_for_timestep_sampling, + get_sigmas, + normalize_dit_input, + shift_timestep, +) +from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed + from fastvideo.distillation.dispatch import register_model -from fastvideo.distillation.utils.config import DistillRunConfig +from fastvideo.distillation.models.adapter import ModelAdapter from fastvideo.distillation.models.components import ModelComponents +from fastvideo.distillation.roles import RoleHandle, RoleManager +from fastvideo.distillation.utils.config import DistillRunConfig from fastvideo.distillation.utils.dataloader import build_parquet_t2v_train_dataloader from fastvideo.distillation.utils.module_state import apply_trainable from fastvideo.distillation.utils.moduleloader import load_module_from_path -from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, -) -from fastvideo.training.activation_checkpoint import apply_activation_checkpointing + +try: + from fastvideo.attention.backends.video_sparse_attn import ( + VideoSparseAttentionMetadataBuilder, + ) + from fastvideo.attention.backends.vmoba import VideoMobaAttentionMetadataBuilder +except Exception: + VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] + VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] + + +class WanAdapter(ModelAdapter): + """Wan runtime adapter: exposes operation-centric primitives to methods.""" + + def __init__( + self, + *, + prompt_handle: RoleHandle, + training_args: Any, + noise_scheduler: Any, + vae: Any, + ) -> None: + self.prompt_handle = prompt_handle + self.training_args = training_args + self.noise_scheduler = noise_scheduler + self.vae = vae + + self.world_group = get_world_group() + self.sp_group = get_sp_group() + self.device = get_local_torch_device() + + self.noise_random_generator: torch.Generator | None = None + self.noise_gen_cuda: torch.Generator | None = None + + self.negative_prompt_embeds: torch.Tensor | None = None + self.negative_prompt_attention_mask: torch.Tensor | None = None + + self._init_timestep_mechanics() + + def _get_training_dtype(self) -> torch.dtype: + return torch.bfloat16 + + def _init_timestep_mechanics(self) -> None: + self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) + self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) + self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) + self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) + + boundary_ratio = getattr(self.training_args, "boundary_ratio", None) + self.boundary_timestep: float | None = ( + float(boundary_ratio) * float(self.num_train_timestep) + if boundary_ratio is not None + else None + ) + + @property + def num_train_timesteps(self) -> int: + return int(self.num_train_timestep) + + def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: + timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) + return timestep.clamp(self.min_timestep, self.max_timestep) + + def on_train_start(self) -> None: + seed = self.training_args.seed + if seed is None: + raise ValueError("training_args.seed must be set for distillation") + + global_rank = int(getattr(self.world_group, "rank", 0)) + sp_world_size = int(getattr(self.training_args, "sp_size", 1) or 1) + if sp_world_size > 1: + sp_group_seed = int(seed) + (global_rank // sp_world_size) + set_random_seed(sp_group_seed) + else: + set_random_seed(int(seed) + global_rank) + + self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) + self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) + + self.ensure_negative_conditioning() + + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + generators: dict[str, torch.Generator] = {} + if self.noise_random_generator is not None: + generators["noise_cpu"] = self.noise_random_generator + if self.noise_gen_cuda is not None: + generators["noise_cuda"] = self.noise_gen_cuda + + return generators + + def ensure_negative_conditioning(self) -> None: + if self.negative_prompt_embeds is not None: + return + + training_args = self.training_args + world_group = self.world_group + device = self.device + dtype = self._get_training_dtype() + + neg_embeds: torch.Tensor | None = None + neg_mask: torch.Tensor | None = None + + if world_group.rank_in_group == 0: + sampling_param = SamplingParam.from_pretrained(training_args.model_path) + negative_prompt = sampling_param.negative_prompt + + args_copy = copy.deepcopy(training_args) + args_copy.inference_mode = True + + prompt_transformer = self.prompt_handle.require_module("transformer") + prompt_pipeline = WanPipeline.from_pretrained( + training_args.model_path, + args=args_copy, + inference_mode=True, + loaded_modules={"transformer": prompt_transformer}, + tp_size=training_args.tp_size, + sp_size=training_args.sp_size, + num_gpus=training_args.num_gpus, + pin_cpu_memory=training_args.pin_cpu_memory, + dit_cpu_offload=True, + ) + + batch_negative = ForwardBatch( + data_type="video", + prompt=negative_prompt, + prompt_embeds=[], + prompt_attention_mask=[], + ) + result_batch = prompt_pipeline.prompt_encoding_stage( # type: ignore[attr-defined] + batch_negative, + training_args, + ) + + neg_embeds = result_batch.prompt_embeds[0].to(device=device, dtype=dtype) + neg_mask = result_batch.prompt_attention_mask[0].to(device=device, dtype=dtype) + + del prompt_pipeline + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + meta = torch.zeros((2,), device=device, dtype=torch.int64) + if world_group.rank_in_group == 0: + assert neg_embeds is not None + assert neg_mask is not None + meta[0] = neg_embeds.ndim + meta[1] = neg_mask.ndim + world_group.broadcast(meta, src=0) + embed_ndim, mask_ndim = (int(meta[0].item()), int(meta[1].item())) + + max_ndim = 8 + embed_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) + mask_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) + if world_group.rank_in_group == 0: + assert neg_embeds is not None + assert neg_mask is not None + embed_shape[:embed_ndim] = torch.tensor( + list(neg_embeds.shape), device=device, dtype=torch.int64 + ) + mask_shape[:mask_ndim] = torch.tensor( + list(neg_mask.shape), device=device, dtype=torch.int64 + ) + world_group.broadcast(embed_shape, src=0) + world_group.broadcast(mask_shape, src=0) + + embed_sizes = tuple(int(x) for x in embed_shape[:embed_ndim].tolist()) + mask_sizes = tuple(int(x) for x in mask_shape[:mask_ndim].tolist()) + + if world_group.rank_in_group != 0: + neg_embeds = torch.empty(embed_sizes, device=device, dtype=dtype) + neg_mask = torch.empty(mask_sizes, device=device, dtype=dtype) + assert neg_embeds is not None + assert neg_mask is not None + + world_group.broadcast(neg_embeds, src=0) + world_group.broadcast(neg_mask, src=0) + + self.negative_prompt_embeds = neg_embeds + self.negative_prompt_attention_mask = neg_mask + + def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: + if self.noise_random_generator is None: + raise RuntimeError("WanAdapter.on_train_start() must be called before prepare_batch()") + + u = compute_density_for_timestep_sampling( + weighting_scheme=self.training_args.weighting_scheme, + batch_size=batch_size, + generator=self.noise_random_generator, + logit_mean=self.training_args.logit_mean, + logit_std=self.training_args.logit_std, + mode_scale=self.training_args.mode_scale, + ) + indices = (u * self.noise_scheduler.config.num_train_timesteps).long() + return self.noise_scheduler.timesteps[indices].to(device=device) + + def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: + latents_shape = training_batch.raw_latent_shape + patch_size = self.training_args.pipeline_config.dit_config.patch_size + current_vsa_sparsity = training_batch.current_vsa_sparsity + assert latents_shape is not None + assert training_batch.timesteps is not None + + if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": + if not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None: + raise ImportError( + "FASTVIDEO_ATTENTION_BACKEND is VIDEO_SPARSE_ATTN, but fastvideo_kernel " + "is not correctly installed or detected." + ) + training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] + raw_latent_shape=latents_shape[2:5], + current_timestep=training_batch.timesteps, + patch_size=patch_size, + VSA_sparsity=current_vsa_sparsity, + device=self.device, + ) + elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": + if not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None: + raise ImportError( + "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, but fastvideo_kernel " + "(or flash_attn>=2.7.4) is not correctly installed." + ) + moba_params = self.training_args.moba_config.copy() + moba_params.update( + { + "current_timestep": training_batch.timesteps, + "raw_latent_shape": training_batch.raw_latent_shape[2:5], + "patch_size": patch_size, + "device": self.device, + } + ) + training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(**moba_params) # type: ignore[misc] + else: + training_batch.attn_metadata = None + + return training_batch + + def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: + latents = training_batch.latents + assert isinstance(latents, torch.Tensor) + batch_size = latents.shape[0] + + if self.noise_gen_cuda is None: + raise RuntimeError("WanAdapter.on_train_start() must be called before prepare_batch()") + + noise = torch.randn( + latents.shape, + generator=self.noise_gen_cuda, + device=latents.device, + dtype=latents.dtype, + ) + timesteps = self._sample_timesteps(batch_size, latents.device) + if int(getattr(self.training_args, "sp_size", 1) or 1) > 1: + self.sp_group.broadcast(timesteps, src=0) + + sigmas = get_sigmas( + self.noise_scheduler, + latents.device, + timesteps, + n_dim=latents.ndim, + dtype=latents.dtype, + ) + noisy_model_input = (1.0 - sigmas) * latents + sigmas * noise + + training_batch.noisy_model_input = noisy_model_input + training_batch.timesteps = timesteps + training_batch.sigmas = sigmas + training_batch.noise = noise + training_batch.raw_latent_shape = latents.shape + + training_batch.conditional_dict = { + "encoder_hidden_states": training_batch.encoder_hidden_states, + "encoder_attention_mask": training_batch.encoder_attention_mask, + } + + if ( + self.negative_prompt_embeds is not None + and self.negative_prompt_attention_mask is not None + ): + neg_embeds = self.negative_prompt_embeds + neg_mask = self.negative_prompt_attention_mask + if neg_embeds.shape[0] == 1 and batch_size > 1: + neg_embeds = neg_embeds.expand(batch_size, *neg_embeds.shape[1:]).contiguous() + if neg_mask.shape[0] == 1 and batch_size > 1: + neg_mask = neg_mask.expand(batch_size, *neg_mask.shape[1:]).contiguous() + training_batch.unconditional_dict = { + "encoder_hidden_states": neg_embeds, + "encoder_attention_mask": neg_mask, + } + + training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) + return training_batch + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", + ) -> TrainingBatch: + self.ensure_negative_conditioning() + + dtype = self._get_training_dtype() + device = self.device + + training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + encoder_hidden_states = raw_batch["text_embedding"] + encoder_attention_mask = raw_batch["text_attention_mask"] + infos = raw_batch.get("info_list") + + if latents_source == "zeros": + batch_size = encoder_hidden_states.shape[0] + vae_config = self.training_args.pipeline_config.vae_config.arch_config + num_channels = vae_config.z_dim + spatial_compression_ratio = vae_config.spatial_compression_ratio + latent_height = self.training_args.num_height // spatial_compression_ratio + latent_width = self.training_args.num_width // spatial_compression_ratio + latents = torch.zeros( + batch_size, + num_channels, + self.training_args.num_latent_t, + latent_height, + latent_width, + device=device, + dtype=dtype, + ) + elif latents_source == "data": + if "vae_latent" not in raw_batch: + raise ValueError("vae_latent not found in batch and latents_source='data'") + latents = raw_batch["vae_latent"] + latents = latents[:, :, : self.training_args.num_latent_t] + latents = latents.to(device, dtype=dtype) + else: + raise ValueError(f"Unknown latents_source: {latents_source!r}") + + training_batch.latents = latents + training_batch.encoder_hidden_states = encoder_hidden_states.to(device, dtype=dtype) + training_batch.encoder_attention_mask = encoder_attention_mask.to(device, dtype=dtype) + training_batch.infos = infos + + training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch = self._prepare_dit_inputs(training_batch) + training_batch = self._build_attention_metadata(training_batch) + + training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) + if training_batch.attn_metadata is not None: + training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] + + return training_batch + + def add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + b, t = clean_latents.shape[:2] + noisy = self.noise_scheduler.add_noise( + clean_latents.flatten(0, 1), + noise.flatten(0, 1), + timestep, + ).unflatten(0, (b, t)) + return noisy + + def _build_distill_input_kwargs( + self, + noise_input: torch.Tensor, + timestep: torch.Tensor, + text_dict: dict[str, torch.Tensor] | None, + ) -> dict[str, Any]: + if text_dict is None: + raise ValueError("text_dict cannot be None for Wan distillation") + return { + "hidden_states": noise_input.permute(0, 2, 1, 3, 4), + "encoder_hidden_states": text_dict["encoder_hidden_states"], + "encoder_attention_mask": text_dict["encoder_attention_mask"], + "timestep": timestep, + "return_dict": False, + } + + def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: + transformer = handle.require_module("transformer") + transformer_2 = handle.modules.get("transformer_2") + if ( + transformer_2 is not None + and self.boundary_timestep is not None + and float(timestep.item()) < float(self.boundary_timestep) + ): + return transformer_2 + return transformer + + def _get_uncond_text_dict( + self, + batch: TrainingBatch, + *, + cfg_uncond: dict[str, Any] | None, + ) -> dict[str, torch.Tensor]: + if cfg_uncond is None: + text_dict = getattr(batch, "unconditional_dict", None) + if text_dict is None: + raise RuntimeError( + "Missing unconditional_dict; ensure_negative_conditioning() may have failed" + ) + return text_dict + + on_missing_raw = cfg_uncond.get("on_missing", "error") + if not isinstance(on_missing_raw, str): + raise ValueError( + "method_config.cfg_uncond.on_missing must be a string, got " + f"{type(on_missing_raw).__name__}" + ) + on_missing = on_missing_raw.strip().lower() + if on_missing not in {"error", "ignore"}: + raise ValueError( + "method_config.cfg_uncond.on_missing must be one of {error, ignore}, got " + f"{on_missing_raw!r}" + ) + + # Wan only supports text CFG. If users configure other channels, fail + # fast (unless explicitly ignored). + for channel, policy_raw in cfg_uncond.items(): + if channel in {"on_missing", "text"}: + continue + if policy_raw is None: + continue + if not isinstance(policy_raw, str): + raise ValueError( + "method_config.cfg_uncond values must be strings, got " + f"{channel}={type(policy_raw).__name__}" + ) + policy = policy_raw.strip().lower() + if policy == "keep": + continue + if on_missing == "ignore": + continue + raise ValueError( + "WanAdapter does not support cfg_uncond channel " + f"{channel!r} (policy={policy!r}). " + "Set cfg_uncond.on_missing=ignore or remove the channel." + ) + + text_policy_raw = cfg_uncond.get("text", None) + if text_policy_raw is None: + text_policy = "negative_prompt" + elif not isinstance(text_policy_raw, str): + raise ValueError( + "method_config.cfg_uncond.text must be a string, got " + f"{type(text_policy_raw).__name__}" + ) + else: + text_policy = text_policy_raw.strip().lower() + + if text_policy in {"negative_prompt"}: + text_dict = getattr(batch, "unconditional_dict", None) + if text_dict is None: + raise RuntimeError( + "Missing unconditional_dict; ensure_negative_conditioning() may have failed" + ) + return text_dict + if text_policy == "keep": + if batch.conditional_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + return batch.conditional_dict + if text_policy == "zero": + if batch.conditional_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + cond = batch.conditional_dict + enc = cond["encoder_hidden_states"] + mask = cond["encoder_attention_mask"] + if not torch.is_tensor(enc) or not torch.is_tensor(mask): + raise TypeError("conditional_dict must contain tensor text inputs") + return { + "encoder_hidden_states": torch.zeros_like(enc), + "encoder_attention_mask": torch.zeros_like(mask), + } + if text_policy == "drop": + raise ValueError( + "cfg_uncond.text=drop is not supported for Wan. " + "Use {negative_prompt, keep, zero}." + ) + raise ValueError( + "cfg_uncond.text must be one of {negative_prompt, keep, zero, drop}, got " + f"{text_policy_raw!r}" + ) + + def predict_x0( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + if conditional: + text_dict = batch.conditional_dict + if text_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + else: + text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + def predict_noise( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + if conditional: + text_dict = batch.conditional_dict + if text_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + else: + text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + return pred_noise + + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + timesteps, attn_metadata = ctx + with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): + (loss / max(1, int(grad_accum_rounds))).backward() @register_model("wan") @@ -49,9 +658,7 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: f"got {role}={role_spec.family!r}" ) - disable_custom_init_weights = bool( - getattr(role_spec, "disable_custom_init_weights", False) - ) + disable_custom_init_weights = bool(getattr(role_spec, "disable_custom_init_weights", False)) transformer = load_module_from_path( model_path=role_spec.path, module_type="transformer", @@ -102,9 +709,7 @@ def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: if validation_enabled: from fastvideo.distillation.validators.wan import WanValidator - validator = WanValidator( - training_args=training_args, - ) + validator = WanValidator(training_args=training_args) # NOTE: adapter is the model runtime boundary; it may implement multiple # method-specific protocols via duck typing. diff --git a/fastvideo/distillation/models/wangame.py b/fastvideo/distillation/models/wangame.py index d82df33a2..41b1b9456 100644 --- a/fastvideo/distillation/models/wangame.py +++ b/fastvideo/distillation/models/wangame.py @@ -1,21 +1,725 @@ # SPDX-License-Identifier: Apache-2.0 +"""WanGame model plugin (components + runtime adapter). + +Config keys used (YAML schema-v2): +- `recipe.family`: must be `"wangame"` for this plugin. +- `roles.`: + - `family`, `path`, `trainable`, `disable_custom_init_weights` + - extra: `variant` (optional; `"bidirectional"`/`"causal"`) +- `training` (selected fields): + - `seed`, `data_path`, `model_path` + - `num_height`, `num_width`, `num_latent_t` + - `min_timestep_ratio`, `max_timestep_ratio`, `boundary_ratio` (optional) + - `weighting_scheme`, `logit_mean`, `logit_std`, `mode_scale` + - `sp_size`, `tp_size`, `num_gpus` + - `pipeline_config.flow_shift`, `pipeline_config.dit_config.patch_size` + - `moba_config` (if `FASTVIDEO_ATTENTION_BACKEND=VMOBA_ATTN`) + - `enable_gradient_checkpointing_type` (optional) +- `training.validation.*` (consumed by `WanGameValidator` when enabled) +- `method_config.cfg_uncond.*` (consumed by `WanGameAdapter` for CFG-uncond policy) +""" + from __future__ import annotations +import copy +from typing import Any, Literal + import torch -from fastvideo.distillation.adapters.wangame import WanGameAdapter +import fastvideo.envs as envs +from fastvideo.distributed import ( + get_local_torch_device, + get_sp_group, + get_world_group, +) +from fastvideo.forward_context import set_forward_context +from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from fastvideo.models.utils import pred_noise_to_pred_video +from fastvideo.pipelines import TrainingBatch +from fastvideo.training.activation_checkpoint import apply_activation_checkpointing +from fastvideo.training.training_utils import ( + compute_density_for_timestep_sampling, + get_sigmas, + normalize_dit_input, + shift_timestep, +) +from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed + from fastvideo.distillation.dispatch import register_model +from fastvideo.distillation.models.adapter import ModelAdapter from fastvideo.distillation.models.components import ModelComponents from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.distillation.utils.config import DistillRunConfig from fastvideo.distillation.utils.dataloader import build_parquet_wangame_train_dataloader from fastvideo.distillation.utils.module_state import apply_trainable from fastvideo.distillation.utils.moduleloader import load_module_from_path -from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, -) -from fastvideo.training.activation_checkpoint import apply_activation_checkpointing + +try: + from fastvideo.attention.backends.video_sparse_attn import ( + VideoSparseAttentionMetadataBuilder, + ) + from fastvideo.attention.backends.vmoba import VideoMobaAttentionMetadataBuilder +except Exception: + VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] + VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] + + +class WanGameAdapter(ModelAdapter): + """WanGame adapter: exposes operation-centric primitives for methods. + + This adapter is intentionally *not* method-specific: + - It knows how to turn a wangame parquet batch into forward primitives. + - It knows how to run a model role (handle) for predict_noise / predict_x0. + - It does not encode DMD2/SFT/etc semantics beyond required primitives. + """ + + def __init__( + self, + *, + training_args: Any, + noise_scheduler: Any, + vae: Any, + ) -> None: + self.training_args = training_args + self.noise_scheduler = noise_scheduler + self.vae = vae + + self.world_group = get_world_group() + self.sp_group = get_sp_group() + self.device = get_local_torch_device() + + self.noise_random_generator: torch.Generator | None = None + self.noise_gen_cuda: torch.Generator | None = None + + self._init_timestep_mechanics() + + def _get_training_dtype(self) -> torch.dtype: + return torch.bfloat16 + + def _init_timestep_mechanics(self) -> None: + self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) + self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) + self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) + self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) + + boundary_ratio = getattr(self.training_args, "boundary_ratio", None) + self.boundary_timestep: float | None = ( + float(boundary_ratio) * float(self.num_train_timestep) + if boundary_ratio is not None + else None + ) + + @property + def num_train_timesteps(self) -> int: + return int(self.num_train_timestep) + + def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: + timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) + return timestep.clamp(self.min_timestep, self.max_timestep) + + def on_train_start(self) -> None: + seed = getattr(self.training_args, "seed", None) + if seed is None: + raise ValueError("training_args.seed must be set for distillation") + + global_rank = int(getattr(self.world_group, "rank", 0)) + sp_world_size = int(getattr(self.training_args, "sp_size", 1) or 1) + if sp_world_size > 1: + sp_group_seed = int(seed) + (global_rank // sp_world_size) + set_random_seed(sp_group_seed) + else: + set_random_seed(int(seed) + global_rank) + + self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) + self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) + + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + generators: dict[str, torch.Generator] = {} + if self.noise_random_generator is not None: + generators["noise_cpu"] = self.noise_random_generator + if self.noise_gen_cuda is not None: + generators["noise_cuda"] = self.noise_gen_cuda + return generators + + def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: + if self.noise_random_generator is None: + raise RuntimeError( + "WanGameAdapter.on_train_start() must be called before prepare_batch()" + ) + + u = compute_density_for_timestep_sampling( + weighting_scheme=self.training_args.weighting_scheme, + batch_size=batch_size, + generator=self.noise_random_generator, + logit_mean=self.training_args.logit_mean, + logit_std=self.training_args.logit_std, + mode_scale=self.training_args.mode_scale, + ) + indices = (u * self.noise_scheduler.config.num_train_timesteps).long() + return self.noise_scheduler.timesteps[indices].to(device=device) + + def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: + latents_shape = training_batch.raw_latent_shape + patch_size = self.training_args.pipeline_config.dit_config.patch_size + current_vsa_sparsity = training_batch.current_vsa_sparsity + assert latents_shape is not None + assert training_batch.timesteps is not None + + if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": + if not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None: + raise ImportError( + "FASTVIDEO_ATTENTION_BACKEND is VIDEO_SPARSE_ATTN, but fastvideo_kernel " + "is not correctly installed or detected." + ) + training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] + raw_latent_shape=latents_shape[2:5], + current_timestep=training_batch.timesteps, + patch_size=patch_size, + VSA_sparsity=current_vsa_sparsity, + device=self.device, + ) + elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": + if not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None: + raise ImportError( + "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, but fastvideo_kernel " + "(or flash_attn>=2.7.4) is not correctly installed." + ) + moba_params = self.training_args.moba_config.copy() + moba_params.update( + { + "current_timestep": training_batch.timesteps, + "raw_latent_shape": training_batch.raw_latent_shape[2:5], + "patch_size": patch_size, + "device": self.device, + } + ) + training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(**moba_params) # type: ignore[misc] + else: + training_batch.attn_metadata = None + + return training_batch + + def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: + latents = training_batch.latents + assert isinstance(latents, torch.Tensor) + batch_size = latents.shape[0] + + if self.noise_gen_cuda is None: + raise RuntimeError( + "WanGameAdapter.on_train_start() must be called before prepare_batch()" + ) + + noise = torch.randn( + latents.shape, + generator=self.noise_gen_cuda, + device=latents.device, + dtype=latents.dtype, + ) + timesteps = self._sample_timesteps(batch_size, latents.device) + if int(getattr(self.training_args, "sp_size", 1) or 1) > 1: + self.sp_group.broadcast(timesteps, src=0) + + sigmas = get_sigmas( + self.noise_scheduler, + latents.device, + timesteps, + n_dim=latents.ndim, + dtype=latents.dtype, + ) + noisy_model_input = (1.0 - sigmas) * latents + sigmas * noise + + training_batch.noisy_model_input = noisy_model_input + training_batch.timesteps = timesteps + training_batch.sigmas = sigmas + training_batch.noise = noise + training_batch.raw_latent_shape = latents.shape + + training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) + return training_batch + + def _build_i2v_mask_latents(self, image_latents: torch.Tensor) -> torch.Tensor: + temporal_compression_ratio = ( + self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 + + batch_size, _num_channels, _t, latent_height, latent_width = image_latents.shape + mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width) + mask_lat_size[:, :, 1:] = 0 + + first_frame_mask = mask_lat_size[:, :, :1] + first_frame_mask = torch.repeat_interleave( + first_frame_mask, + dim=2, + repeats=temporal_compression_ratio, + ) + mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], dim=2) + mask_lat_size = mask_lat_size.view( + batch_size, + -1, + temporal_compression_ratio, + latent_height, + latent_width, + ) + mask_lat_size = mask_lat_size.transpose(1, 2) + return mask_lat_size.to(device=image_latents.device, dtype=image_latents.dtype) + + def _process_actions( + self, training_batch: TrainingBatch + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + keyboard_cond = getattr(training_batch, "keyboard_cond", None) + mouse_cond = getattr(training_batch, "mouse_cond", None) + if keyboard_cond is None or mouse_cond is None: + raise ValueError("WanGame batch must provide keyboard_cond and mouse_cond") + + from fastvideo.models.dits.hyworld.pose import process_custom_actions + + batch_size = int(training_batch.noisy_model_input.shape[0]) # type: ignore[union-attr] + viewmats_list: list[torch.Tensor] = [] + intrinsics_list: list[torch.Tensor] = [] + action_labels_list: list[torch.Tensor] = [] + for b in range(batch_size): + v, i, a = process_custom_actions(keyboard_cond[b], mouse_cond[b]) + viewmats_list.append(v) + intrinsics_list.append(i) + action_labels_list.append(a) + + viewmats = torch.stack(viewmats_list, dim=0).to(device=self.device, dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, dim=0).to( + device=self.device, dtype=torch.bfloat16 + ) + action_labels = torch.stack(action_labels_list, dim=0).to( + device=self.device, dtype=torch.bfloat16 + ) + + num_latent_t = int(training_batch.noisy_model_input.shape[2]) # type: ignore[union-attr] + if int(action_labels.shape[1]) != num_latent_t: + raise ValueError( + "Action conditioning temporal dim mismatch: " + f"action={tuple(action_labels.shape)} vs latent_t={num_latent_t}" + ) + if int(viewmats.shape[1]) != num_latent_t: + raise ValueError( + "Viewmats temporal dim mismatch: " + f"viewmats={tuple(viewmats.shape)} vs latent_t={num_latent_t}" + ) + + return viewmats, intrinsics, action_labels + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", + ) -> TrainingBatch: + dtype = self._get_training_dtype() + device = self.device + + training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + infos = raw_batch.get("info_list") + + if latents_source == "zeros": + clip_feature = raw_batch["clip_feature"] + batch_size = int(clip_feature.shape[0]) + vae_config = self.training_args.pipeline_config.vae_config.arch_config + num_channels = int(vae_config.z_dim) + spatial_compression_ratio = int(vae_config.spatial_compression_ratio) + latent_height = int(self.training_args.num_height) // spatial_compression_ratio + latent_width = int(self.training_args.num_width) // spatial_compression_ratio + latents = torch.zeros( + batch_size, + num_channels, + int(self.training_args.num_latent_t), + latent_height, + latent_width, + device=device, + dtype=dtype, + ) + elif latents_source == "data": + if "vae_latent" not in raw_batch: + raise ValueError("vae_latent not found in batch and latents_source='data'") + latents = raw_batch["vae_latent"] + latents = latents[:, :, : self.training_args.num_latent_t] + latents = latents.to(device, dtype=dtype) + else: + raise ValueError(f"Unknown latents_source: {latents_source!r}") + + if "clip_feature" not in raw_batch: + raise ValueError("clip_feature must be present for WanGame") + image_embeds = raw_batch["clip_feature"].to(device, dtype=dtype) + + if "first_frame_latent" not in raw_batch: + raise ValueError("first_frame_latent must be present for WanGame") + image_latents = raw_batch["first_frame_latent"] + image_latents = image_latents[:, :, : self.training_args.num_latent_t] + image_latents = image_latents.to(device, dtype=dtype) + + pil_image = raw_batch.get("pil_image") + if isinstance(pil_image, torch.Tensor): + training_batch.preprocessed_image = pil_image.to(device=device) + else: + training_batch.preprocessed_image = pil_image + + keyboard_cond = raw_batch.get("keyboard_cond") + if isinstance(keyboard_cond, torch.Tensor) and keyboard_cond.numel() > 0: + training_batch.keyboard_cond = keyboard_cond.to(device, dtype=dtype) + else: + training_batch.keyboard_cond = None + + mouse_cond = raw_batch.get("mouse_cond") + if isinstance(mouse_cond, torch.Tensor) and mouse_cond.numel() > 0: + training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) + else: + training_batch.mouse_cond = None + + temporal_compression_ratio = ( + self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + expected_num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 + if training_batch.keyboard_cond is not None and int( + training_batch.keyboard_cond.shape[1] + ) != int(expected_num_frames): + raise ValueError( + "keyboard_cond temporal dim mismatch: " + f"got {int(training_batch.keyboard_cond.shape[1])}, expected {int(expected_num_frames)}" + ) + if training_batch.mouse_cond is not None and int(training_batch.mouse_cond.shape[1]) != int( + expected_num_frames + ): + raise ValueError( + "mouse_cond temporal dim mismatch: " + f"got {int(training_batch.mouse_cond.shape[1])}, expected {int(expected_num_frames)}" + ) + + training_batch.latents = latents + training_batch.encoder_hidden_states = None + training_batch.encoder_attention_mask = None + training_batch.image_embeds = image_embeds + training_batch.image_latents = image_latents + training_batch.infos = infos + + training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch = self._prepare_dit_inputs(training_batch) + training_batch = self._build_attention_metadata(training_batch) + + training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) + if training_batch.attn_metadata is not None: + training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] + + training_batch.mask_lat_size = self._build_i2v_mask_latents(image_latents) + viewmats, intrinsics, action_labels = self._process_actions(training_batch) + training_batch.viewmats = viewmats + training_batch.Ks = intrinsics + training_batch.action = action_labels + + return training_batch + + def add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + b, t = clean_latents.shape[:2] + noisy = self.noise_scheduler.add_noise( + clean_latents.flatten(0, 1), + noise.flatten(0, 1), + timestep, + ).unflatten(0, (b, t)) + return noisy + + def _build_distill_input_kwargs( + self, + noisy_video_latents: torch.Tensor, + timestep: torch.Tensor, + *, + image_embeds: torch.Tensor, + image_latents: torch.Tensor, + mask_lat_size: torch.Tensor, + viewmats: torch.Tensor | None, + Ks: torch.Tensor | None, + action: torch.Tensor | None, + mouse_cond: torch.Tensor | None, + keyboard_cond: torch.Tensor | None, + ) -> dict[str, Any]: + hidden_states = torch.cat( + [ + noisy_video_latents.permute(0, 2, 1, 3, 4), + mask_lat_size, + image_latents, + ], + dim=1, + ) + return { + "hidden_states": hidden_states, + "encoder_hidden_states": None, + "timestep": timestep.to(device=self.device, dtype=torch.bfloat16), + "encoder_hidden_states_image": image_embeds, + "viewmats": viewmats, + "Ks": Ks, + "action": action, + "mouse_cond": mouse_cond, + "keyboard_cond": keyboard_cond, + "return_dict": False, + } + + def _select_cfg_condition_inputs( + self, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None, + ) -> dict[str, Any]: + image_embeds = batch.image_embeds + image_latents = batch.image_latents + mask_lat_size = batch.mask_lat_size + if image_embeds is None: + raise RuntimeError("WanGameAdapter requires TrainingBatch.image_embeds") + if image_latents is None: + raise RuntimeError("WanGameAdapter requires TrainingBatch.image_latents") + if mask_lat_size is None: + raise RuntimeError("WanGameAdapter requires TrainingBatch.mask_lat_size") + + viewmats = getattr(batch, "viewmats", None) + Ks = getattr(batch, "Ks", None) + action = getattr(batch, "action", None) + mouse_cond = getattr(batch, "mouse_cond", None) + keyboard_cond = getattr(batch, "keyboard_cond", None) + + if conditional or cfg_uncond is None: + return { + "image_embeds": image_embeds, + "image_latents": image_latents, + "mask_lat_size": mask_lat_size, + "viewmats": viewmats, + "Ks": Ks, + "action": action, + "mouse_cond": mouse_cond, + "keyboard_cond": keyboard_cond, + } + + on_missing_raw = cfg_uncond.get("on_missing", "error") + if not isinstance(on_missing_raw, str): + raise ValueError( + "method_config.cfg_uncond.on_missing must be a string, got " + f"{type(on_missing_raw).__name__}" + ) + on_missing = on_missing_raw.strip().lower() + if on_missing not in {"error", "ignore"}: + raise ValueError( + "method_config.cfg_uncond.on_missing must be one of {error, ignore}, got " + f"{on_missing_raw!r}" + ) + + supported_channels = {"image", "action"} + for channel, policy_raw in cfg_uncond.items(): + if channel in {"on_missing"}: + continue + if channel in supported_channels: + continue + if policy_raw is None: + continue + if not isinstance(policy_raw, str): + raise ValueError( + "method_config.cfg_uncond values must be strings, got " + f"{channel}={type(policy_raw).__name__}" + ) + policy = policy_raw.strip().lower() + if policy == "keep": + continue + if on_missing == "ignore": + continue + raise ValueError( + "WanGameAdapter does not support cfg_uncond channel " + f"{channel!r} (policy={policy!r}). " + "Set cfg_uncond.on_missing=ignore or remove the channel." + ) + + def _get_policy(channel: str) -> str: + raw = cfg_uncond.get(channel, "keep") + if raw is None: + return "keep" + if not isinstance(raw, str): + raise ValueError( + "method_config.cfg_uncond values must be strings, got " + f"{channel}={type(raw).__name__}" + ) + policy = raw.strip().lower() + if policy not in {"keep", "zero", "drop"}: + raise ValueError( + "method_config.cfg_uncond values must be one of {keep, zero, drop}, got " + f"{channel}={raw!r}" + ) + return policy + + image_policy = _get_policy("image") + if image_policy == "zero": + image_embeds = torch.zeros_like(image_embeds) + image_latents = torch.zeros_like(image_latents) + mask_lat_size = torch.zeros_like(mask_lat_size) + elif image_policy == "drop": + raise ValueError( + "cfg_uncond.image=drop is not supported for WanGame I2V; " + "use cfg_uncond.image=zero or keep." + ) + + action_policy = _get_policy("action") + if action_policy == "zero": + if viewmats is None or Ks is None or action is None: + if on_missing == "ignore": + pass + else: + raise ValueError( + "cfg_uncond.action=zero requires action conditioning tensors, " + "but TrainingBatch is missing {viewmats, Ks, action}." + ) + else: + viewmats = torch.zeros_like(viewmats) + Ks = torch.zeros_like(Ks) + action = torch.zeros_like(action) + if mouse_cond is not None: + mouse_cond = torch.zeros_like(mouse_cond) + if keyboard_cond is not None: + keyboard_cond = torch.zeros_like(keyboard_cond) + elif action_policy == "drop": + viewmats = None + Ks = None + action = None + mouse_cond = None + keyboard_cond = None + + return { + "image_embeds": image_embeds, + "image_latents": image_latents, + "mask_lat_size": mask_lat_size, + "viewmats": viewmats, + "Ks": Ks, + "action": action, + "mouse_cond": mouse_cond, + "keyboard_cond": keyboard_cond, + } + + def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: + transformer = handle.require_module("transformer") + transformer_2 = handle.modules.get("transformer_2") + if transformer_2 is not None and self.boundary_timestep is not None: + if timestep.numel() != 1: + raise ValueError( + "MoE boundary selection requires a scalar timestep, got " + f"shape={tuple(timestep.shape)}" + ) + if float(timestep.item()) < float(self.boundary_timestep): + return transformer_2 + return transformer + + def predict_x0( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + cond_inputs = self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + ) + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs["image_embeds"], + image_latents=cond_inputs["image_latents"], + mask_lat_size=cond_inputs["mask_lat_size"], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs["keyboard_cond"], + ) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + def predict_noise( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + cond_inputs = self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + ) + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs["image_embeds"], + image_latents=cond_inputs["image_latents"], + mask_lat_size=cond_inputs["mask_lat_size"], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs["keyboard_cond"], + ) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + return pred_noise + + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + timesteps, attn_metadata = ctx + with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): + (loss / max(1, int(grad_accum_rounds))).backward() @register_model("wangame") @@ -61,9 +765,7 @@ def build_wangame_components(*, cfg: DistillRunConfig) -> ModelComponents: f"{variant_raw!r}. Expected 'causal' or 'bidirectional'." ) - disable_custom_init_weights = bool( - getattr(role_spec, "disable_custom_init_weights", False) - ) + disable_custom_init_weights = bool(getattr(role_spec, "disable_custom_init_weights", False)) transformer = load_module_from_path( model_path=role_spec.path, module_type="transformer", @@ -135,3 +837,4 @@ def build_wangame_components(*, cfg: DistillRunConfig) -> ModelComponents: validator=validator, start_step=0, ) + diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 1b9f89169..5e98cbf5d 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -1,5 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 +"""Wan validator (model-family validation backend). + +Config keys used: +- `training` (selected fields): + - `seed`, `model_path` + - `num_height`, `num_width`, `num_latent_t` + - `tp_size`, `sp_size`, `num_gpus`, `pin_cpu_memory` + - `pipeline_config.flow_shift` + - `pipeline_config.vae_config.arch_config.temporal_compression_ratio` + - `VSA_sparsity` +- `training.validation.*` (typically parsed by a method into `ValidationRequest`): + - `dataset_file`, `sampling_steps`, `guidance_scale` + - `sampler_kind` (`ode`/`sde`), `ode_solver` (`euler`/`unipc`), `rollout_mode` +""" + from __future__ import annotations import os diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index f425db618..4754e72a3 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -1,5 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 +"""WanGame validator (model-family validation backend). + +Config keys used: +- `training` (selected fields): + - `seed`, `model_path` + - `num_height`, `num_width`, `num_latent_t` + - `tp_size`, `sp_size`, `num_gpus`, `pin_cpu_memory` + - `pipeline_config.flow_shift` + - `pipeline_config.vae_config.arch_config.temporal_compression_ratio` + - `VSA_sparsity`, `VSA_decay_rate`, `VSA_decay_interval_steps` (when applicable) +- `training.validation.*` (typically parsed by a method into `ValidationRequest`): + - `dataset_file`, `sampling_steps`, `guidance_scale` + - `sampler_kind` (`ode`/`sde`), `ode_solver` (`euler`/`unipc`) + - `rollout_mode` (`parallel`/`streaming`), `num_frames` (action length) +""" + from __future__ import annotations import os From 9144a43f7fbf7052b676ef7ebf21225bc5cf355b Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Sun, 1 Mar 2026 18:35:30 +0000 Subject: [PATCH 135/214] 32 gpu training slurm --- .../dfsft_wangame2.1_i2v_1.3B_causal.yaml | 4 +- ...fsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm | 64 +++++++++++++ ...dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml | 90 +++++++++++++++++++ 3 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm create mode 100644 examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml index 886fd0baf..7c1d4aa33 100644 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml @@ -69,14 +69,14 @@ training: wandb_run_name: wangame_dfsft_causal validation: enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json every_steps: 100 sampling_steps: [40] rollout_mode: streaming sampler_kind: ode ode_solver: euler guidance_scale: 1.0 - num_frames: 81 + num_frames: 75 default_pipeline_config: flow_shift: 3 diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm new file mode 100644 index 000000000..95b6171e5 --- /dev/null +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm @@ -0,0 +1,64 @@ +#!/bin/bash +#SBATCH --job-name=wangame_dfsft_causal +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=slurm_outputs/wangame_dfsft_causal_4n8g/wangame_dfsft_causal_%j.out +#SBATCH --error=slurm_outputs/wangame_dfsft_causal_4n8g/wangame_dfsft_causal_%j.err +#SBATCH --exclusive + +set -euo pipefail + +# ---- Env (mirror legacy WanGame slurm scripts) ---- +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-INIT,NET} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} + +# Triton cache (avoid cross-job collisions; per-node task is OK in practice) +export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/tmp/triton_cache_${SLURM_JOB_ID}_${SLURM_PROCID}} + +# Rendezvous (torchrun) +export MASTER_PORT=${MASTER_PORT:-29501} +nodes=( $(scontrol show hostnames "$SLURM_JOB_NODELIST") ) +export MASTER_ADDR=${MASTER_ADDR:-${nodes[0]}} + +# W&B (recommended to pass WANDB_API_KEY via environment / secret manager) +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-online} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +source ~/conda/miniconda/bin/activate +conda activate alexfv + +CONFIG=${CONFIG:-"examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml"} +OUTPUT_DIR=${OUTPUT_DIR:-"outputs/wangame_dfsft_causal_4n8g/${SLURM_JOB_ID}"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +# 4 nodes × 8 GPUs +NUM_NODES=${SLURM_NNODES} +GPUS_PER_NODE=8 + +srun torchrun \ + --nnodes "$NUM_NODES" \ + --nproc_per_node "$GPUS_PER_NODE" \ + --rdzv_backend c10d \ + --rdzv_endpoint "$MASTER_ADDR:$MASTER_PORT" \ + --node_rank "$SLURM_PROCID" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" \ + --override-output-dir "$OUTPUT_DIR" + diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml new file mode 100644 index 000000000..5df3e17a5 --- /dev/null +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml @@ -0,0 +1,90 @@ +recipe: + family: wangame + method: dfsft + +roles: + student: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + variant: causal + +training: + # Distributed (4 nodes × 8 GPUs = 32 ranks) + num_gpus: 32 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 32 + + # Data (parquet dataset folder) + # + # Supports comma-separated `path` or `path:N` (repeat count) entries. + # N=0 means skip (handy to toggle datasets without deleting lines). + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_dfsft_causal_4n8g + max_train_steps: 20000 + seed: 1000 + checkpoints_total_limit: 2 + + # Optimizer + learning_rate: 1.0e-5 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 1.0e-4 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.0 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_dfsft_causal_4n8g + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [40] + rollout_mode: streaming + sampler_kind: ode + ode_solver: euler + guidance_scale: 1.0 + num_frames: 75 + +default_pipeline_config: + flow_shift: 3 + sampler_kind: ode + +method_config: + attn_kind: dense + chunk_size: 3 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + From 7e5ed86b3da843db952cf8c5a901b9b53a00a0df Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 2 Mar 2026 01:42:10 +0000 Subject: [PATCH 136/214] config --- .gitignore | 1 + .../wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml | 2 +- .../dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml | 11 +++++------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 82fce83d6..d2217678b 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ env *.log *.npy weights/ +slurm_outputs/ # SSIM test outputs fastvideo/tests/ssim/generated_videos/ diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml index 7c1d4aa33..42b568503 100644 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml @@ -69,7 +69,7 @@ training: wandb_run_name: wangame_dfsft_causal validation: enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json every_steps: 100 sampling_steps: [40] rollout_mode: streaming diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml index 5df3e17a5..1bdace7a0 100644 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml @@ -43,15 +43,15 @@ training: # Output / steps output_dir: outputs/wangame_dfsft_causal_4n8g - max_train_steps: 20000 + max_train_steps: 200000 seed: 1000 checkpoints_total_limit: 2 # Optimizer learning_rate: 1.0e-5 mixed_precision: bf16 - betas: "0.9,0.999" - weight_decay: 1.0e-4 + betas: "0.9,0.95" + weight_decay: 1.0e-5 lr_scheduler: constant lr_warmup_steps: 0 max_grad_norm: 1.0 @@ -69,14 +69,14 @@ training: wandb_run_name: wangame_dfsft_causal_4n8g validation: enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json every_steps: 100 sampling_steps: [40] rollout_mode: streaming sampler_kind: ode ode_solver: euler guidance_scale: 1.0 - num_frames: 75 + num_frames: 69 default_pipeline_config: flow_shift: 3 @@ -87,4 +87,3 @@ method_config: chunk_size: 3 min_timestep_ratio: 0.02 max_timestep_ratio: 0.98 - From e85b78664fdc44080b699189e3426c87b4893e90 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 2 Mar 2026 06:10:49 +0000 Subject: [PATCH 137/214] designing new model class --- dev/phase_mode.md | 239 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 dev/phase_mode.md diff --git a/dev/phase_mode.md b/dev/phase_mode.md new file mode 100644 index 000000000..b4750ecd1 --- /dev/null +++ b/dev/phase_mode.md @@ -0,0 +1,239 @@ +# Phase：Model Plugin “去 Adapter 化”（命名收敛 + 去掉 ModelComponents 中间层） + +> 本阶段只做 **结构与命名** 的收敛，目标是让代码读起来更直观、更少概念。 +> 训练/验证行为应保持不变（同 YAML、同 loss、同 pipeline 选择）。 +> +> 你提出的方向：`ModelBase / WanModel / WanGameModel`，模型构建由 `__init__`（或 `from_config`）自行管理， +> 从而不再需要 `ModelComponents` 这种“中间件”——我认为是合理且优雅的。 + +--- + +## 0) 背景：为什么现在 “Adapter” 这个词让人困惑 + +当前 `fastvideo/distillation/models/{wan,wangame}.py` 里所谓的 `*Adapter`，实际上已经承担了： + +- family 相关的 runtime primitives（`prepare_batch/add_noise/predict_noise/predict_x0/backward/...`） +- shared components 的持有（`vae/noise_scheduler`）与生命周期管理 +- 训练期状态（RNG、negative conditioning cache 等） + +这更像“模型运行时封装（model plugin / runtime model）”，而不是一个“薄 adapter”。 + +同时 `ModelComponents` 只是把这些东西再打包一次，本质信息冗余。 + +--- + +## 1) 目标 / 非目标 + +### ✅ 目标（本阶段要做到） + +1. **命名收敛**：彻底去掉 `Adapter` 这个词。 + - `ModelAdapter` → `ModelBase` + - `WanAdapter` → `WanModel` + - `WanGameAdapter` → `WanGameModel` +2. **去掉 build-time 中间件**：删除 `ModelComponents`（以及对应的 `models/components.py`）。 + - “构建产物”不再是 dataclass，而是 **模型对象本身**（它天然持有 bundle/dataloader/validator）。 +3. **dispatch 更直觉**： + - `@register_model("wan")` 直接注册 `WanModel` 类（或其 factory），而不是注册 `build_wan_components()` 这种函数。 + - `dispatch.build_runtime_from_config()` 变成：`model = WanModel(cfg)` → `method = ...build(..., model=model)`。 +4. **保持 method 仍是 role-centric**： + - method 仍通过 role（student/teacher/critic/…）决定算法逻辑； + - model 只提供 operation-centric primitives，不因为 role 增多而出现“role 爆炸式 API”。 + +### ❌ 非目标（本阶段不做） + +- 不改 YAML schema（保持现有 schema v2）。 +- 不改算法行为（DMD2/finetune/dfsft 的 loss、rollout、optimizer cadence 不变）。 +- 不改 pipeline/validator 选择逻辑（仍由 method/validator 按 config 决定）。 +- 不引入 multi-family roles（先保持 `recipe.family` 主导一个 family）。 + +--- + +## 2) 新的核心抽象：ModelBase / WanModel / WanGameModel + +### 2.1 `ModelBase`（替代 `ModelAdapter` + `ModelComponents`） + +建议文件：`fastvideo/distillation/models/base.py` + +`ModelBase` 是一个 **“模型插件对象”**，它既是构建产物，也是 runtime boundary。 + +它必须持有（构建期产物）： + +- `training_args` +- `bundle: RoleManager` +- `dataloader` +- `validator`（可选） +- `start_step`（用于 resume） + +同时提供（runtime primitives）： + +- `num_train_timesteps` +- `shift_and_clamp_timestep(t)` +- `on_train_start()` / `get_rng_generators()` +- `prepare_batch(...)` +- `add_noise(...)` +- `predict_noise(handle, ...)` +- `predict_x0(handle, ...)` +- `backward(loss, ctx, grad_accum_rounds)` + +> 说明：这基本就是现有 `ModelAdapter` 的抽象 + `ModelComponents` 的字段合并。 + +--- + +## 3) 代码改动设计（按文件列 TODO） + +### 3.1 `fastvideo/distillation/models/adapter.py` → `.../models/base.py` + +- [ ] 文件改名：`adapter.py` → `base.py` +- [ ] 类改名：`ModelAdapter` → `ModelBase` +- [ ] 文档/注释同步：强调这是 “model plugin object”，而非薄 adapter + +### 3.2 `fastvideo/distillation/models/components.py` 删除 + +- [ ] 删除 `ModelComponents` dataclass +- [ ] 删除其在 dispatch / models 插件中的引用 + +### 3.3 `fastvideo/distillation/models/wan.py` + +将结构从： + +- `class WanAdapter(ModelAdapter): ...` +- `@register_model("wan") def build_wan_components(...) -> ModelComponents` + +改成： + +- `@register_model("wan") class WanModel(ModelBase): ...` + +#### 建议实现形态 + +```py +@register_model("wan") +class WanModel(ModelBase): + def __init__(self, *, cfg: DistillRunConfig) -> None: + # 1) parse + validate cfg + # 2) build shared components (vae/noise_scheduler) + # 3) build roles -> RoleManager + # 4) build validator (optional) + # 5) build dataloader + # 6) init runtime caches (rng / negative conditioning state) +``` + +把原本 `WanAdapter` 的方法体原封不动迁到 `WanModel` 上即可（第一版只做搬迁/改名）。 + +> 注意：`ensure_negative_conditioning()` 目前依赖 `prompt_handle`（student transformer + prompt encoding pipeline）。 +> `WanModel` 仍可用 `self.student_handle = self.bundle.role("student")` 解决。 + +### 3.4 `fastvideo/distillation/models/wangame.py` + +同 `wan.py`： + +- `WanGameAdapter` → `WanGameModel` +- `build_wangame_components(...) -> ModelComponents` → `WanGameModel(cfg)` + +需要保持： + +- streaming validation 的 `num_frames` 约束（`1 + 4k` 且 latent 可被 `num_frame_per_block` 整除） +- validator pipeline 选择逻辑不变（parallel vs streaming / ode vs sde / ode_solver） + +### 3.5 `fastvideo/distillation/dispatch.py` + +当前： + +- `_MODELS: dict[str, ModelBuilder]`（builder 返回 `ModelComponents`） + +改为： + +- `_MODELS: dict[str, type[ModelBase]]`（或 `Callable[[DistillRunConfig], ModelBase]`） + +并把 `build_runtime_from_config()` 改为： + +```py +model_cls = get_model(cfg.recipe.family) +model = model_cls(cfg=cfg) + +method_cls = get_method(cfg.recipe.method) +method = method_cls.build(cfg=cfg, bundle=model.bundle, model=model, validator=model.validator) + +return DistillRuntime( + training_args=model.training_args, + method=method, + dataloader=model.dataloader, + start_step=model.start_step, +) +``` + +> 这里建议把传参从 `adapter=` 改名为 `model=`，让含义更直观。 + +### 3.6 `fastvideo/distillation/methods/base.py`（以及所有 methods） + +目标：把 method 里对 `self.adapter` 的依赖改成对 `self.model` 的依赖。 + +- [ ] `DistillMethod.build(...)` 签名建议改为： + - `build(cfg, bundle, model, validator)`（或更简化：`build(cfg, model)`） +- [ ] methods 内部字段： + - `self.model` 替代 `self.adapter` +- [ ] `on_train_start()` 里调用 `self.model.on_train_start()` +- [ ] `get_rng_generators()` 读取 `self.model.get_rng_generators()` + +> 这一改动对行为应是 0 diff（只是字段名变化）。 + +### 3.7 文档与 “Config keys used” 头部更新 + +- [ ] `models/*.py` 的 “Config keys used” 头部同步改成 “consumed by WanModel/WanGameModel” +- [ ] 方法文件、validator 文件不强制改（但建议把 “adapter” 的文字替换为 “model”） + +--- + +## 4) 兼容性与风险点（需要提前明确） + +### 4.1 导入/注册顺序（避免循环 import） + +要求： + +- `dispatch.ensure_builtin_registrations()` 仍然显式 import `fastvideo.distillation.models.wan` 等模块， + 让 `@register_model` 在 import 时完成注册。 +- `models/*.py` 只 import `register_model`（不要反向 import dispatch 里的 heavy objects）。 + +### 4.2 Checkpoint/RNG 断点续训 + +目前 checkpoint manager 通过： + +- `runtime.method.get_rng_generators()`（优先） +- fallback 到 `runtime.method.adapter.get_rng_generators()` + +重构后建议： + +- `runtime.method.get_rng_generators()` 永远存在并返回 `self.model.get_rng_generators()`， + 不再做 adapter fallback。 + +### 4.3 行为不变（Definition of Done） + +必须满足： + +- `fastvideo/training/distillation.py --config ` 能跑通： + - wan: DMD2 / finetune + - wangame: finetune / dmd2 / dfsft（尤其 streaming validation） +- 静态检查至少通过： + - `python -m py_compile`（相关文件） + - `ruff check`（相关文件） + +--- + +## 5) 实施顺序（推荐最小风险落地） + +1) **纯改名 + 0 行为改动** +- 先在 models 内部把 `WanAdapter` 改名 `WanModel`（类名、注释、引用） +- `ModelAdapter` 改名 `ModelBase` + +2) **去掉 ModelComponents** +- model plugin 不再返回 dataclass，而是返回模型对象本身 +- dispatch 改为实例化模型对象 + +3) **methods 参数名收敛** +- `adapter` 全部改为 `model` + +做到这一步就能得到一个更直觉的结构： + +- `models/` 里就是“模型插件对象” +- `methods/` 只看 `model`（operation-centric）+ `bundle`（role-centric） +- dispatch 就是 “cfg -> model -> method -> trainer” + From e63e214e2c288953a73341fdb11b84f0d5d63485 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 2 Mar 2026 06:40:02 +0000 Subject: [PATCH 138/214] deprecate adapter design --- fastvideo/distillation/dispatch.py | 18 +- fastvideo/distillation/doc/README.md | 10 +- fastvideo/distillation/doc/dispatch.md | 4 +- .../distillation/doc/methods/__init__.md | 2 +- fastvideo/distillation/doc/methods/base.md | 2 +- .../doc/methods/consistency_model/__init__.md | 2 +- .../methods/distribution_matching/__init__.md | 2 +- .../doc/methods/distribution_matching/dmd2.md | 14 +- .../doc/methods/fine_tuning/__init__.md | 5 +- .../doc/methods/fine_tuning/dfsft.md | 11 +- .../doc/methods/fine_tuning/finetune.md | 8 +- .../knowledge_distillation/__init__.md | 2 +- fastvideo/distillation/doc/models/__init__.md | 5 +- .../doc/models/{adapter.md => base.md} | 11 +- .../distillation/doc/models/components.md | 21 -- fastvideo/distillation/doc/models/wan.md | 12 +- fastvideo/distillation/doc/models/wangame.md | 16 +- .../distillation/doc/utils/checkpoint.md | 2 +- fastvideo/distillation/doc/utils/config.md | 24 +- .../distillation/doc/utils/dataloader.md | 2 +- fastvideo/distillation/methods/base.py | 4 +- .../methods/distribution_matching/dmd2.py | 70 ++--- .../distillation/methods/fine_tuning/dfsft.py | 50 ++-- .../methods/fine_tuning/finetune.py | 45 ++-- .../models/{adapter.py => base.py} | 11 +- fastvideo/distillation/models/components.py | 28 -- fastvideo/distillation/models/wan.py | 222 +++++++--------- fastvideo/distillation/models/wangame.py | 249 ++++++++---------- fastvideo/training/distillation.py | 4 +- 29 files changed, 387 insertions(+), 469 deletions(-) rename fastvideo/distillation/doc/models/{adapter.md => base.md} (70%) delete mode 100644 fastvideo/distillation/doc/models/components.md rename fastvideo/distillation/models/{adapter.py => base.py} (91%) delete mode 100644 fastvideo/distillation/models/components.py diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index 8a7a0f62a..e17549703 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -8,11 +8,11 @@ from typing import Protocol from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.models.components import ModelComponents from fastvideo.distillation.utils.config import DistillRunConfig if TYPE_CHECKING: from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.distillation.models.base import ModelBase @dataclass(slots=True) @@ -26,7 +26,7 @@ class DistillRuntime: class ModelBuilder(Protocol): - def __call__(self, *, cfg: DistillRunConfig) -> ModelComponents: + def __call__(self, *, cfg: DistillRunConfig) -> ModelBase: ... @@ -114,19 +114,19 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: """ model_builder = get_model(str(cfg.recipe.family)) - components = model_builder(cfg=cfg) + model = model_builder(cfg=cfg) method_cls = get_method(str(cfg.recipe.method)) method = method_cls.build( cfg=cfg, - bundle=components.bundle, - adapter=components.adapter, - validator=components.validator, + bundle=model.bundle, + model=model, + validator=model.validator, ) return DistillRuntime( - training_args=components.training_args, + training_args=model.training_args, method=method, - dataloader=components.dataloader, - start_step=int(getattr(components, "start_step", 0) or 0), + dataloader=model.dataloader, + start_step=int(getattr(model, "start_step", 0) or 0), ) diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index 243af47b8..ed5a41850 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -5,15 +5,14 @@ 设计原则（对应 Phase 2.9）： - **Trainer** 只做 infra（loop/accum/日志/ckpt/validate 调用），不包含算法策略。 - **Method** 只做算法（loss + update policy + 需要哪些 roles）。 -- **Model plugin** 只做装配（build-time：加载 modules、构建 bundle/adapter/dataloader/validator；代码在 `models/`）。 -- **Adapter** 只做运行时 primitive（step-time：prepare_batch/forward_context/predict/backward 等）， - 并与对应 model plugin **共置于 `models/*.py`**（例如 `models/wan.py:WanAdapter`）。 +- **Model plugin** 负责装配 + primitives（build-time：加载 modules、构建 bundle/dataloader/validator； + step-time：实现 `ModelBase` 的 primitives，例如 `prepare_batch/predict_*/backward/...`）。 API 以 operation 为中心，不以 role 为中心（避免 role 爆炸）。 快速入口（从运行到训练）： `fastvideo/training/distillation.py` → `utils.config.load_distill_run_config()` → `dispatch.build_runtime_from_config()` → -`ModelComponents + DistillMethod` → `DistillTrainer.run()` +`ModelBase(model) + DistillMethod` → `DistillTrainer.run()` --- @@ -34,8 +33,7 @@ ### models/ - `models/__init__.md` -- `models/adapter.md` -- `models/components.md` +- `models/base.md` - `models/wan.md` - `models/wangame.md` diff --git a/fastvideo/distillation/doc/dispatch.md b/fastvideo/distillation/doc/dispatch.md index a22213231..70c1ee2c9 100644 --- a/fastvideo/distillation/doc/dispatch.md +++ b/fastvideo/distillation/doc/dispatch.md @@ -7,10 +7,10 @@ - 不需要写 N×M 的 if/else 组合逻辑 **关键概念** -- `ModelBuilder(cfg) -> ModelComponents` +- `ModelBuilder(cfg) -> ModelBase`（一个可运行的 model plugin 实例） - `DistillMethod` class（算法实现） - `@register_method("...")` 直接注册到 class - - class 需要实现 `DistillMethod.build(cfg, bundle, adapter, validator)` + - class 需要实现 `DistillMethod.build(cfg, bundle, model, validator)` **关键 API** - `register_model(name)` / `register_method(name)`：装饰器注册 diff --git a/fastvideo/distillation/doc/methods/__init__.md b/fastvideo/distillation/doc/methods/__init__.md index ec9324ab6..38764da0c 100644 --- a/fastvideo/distillation/doc/methods/__init__.md +++ b/fastvideo/distillation/doc/methods/__init__.md @@ -10,7 +10,7 @@ **设计意图** - method 层应当是 **模型无关** 的（不 import 具体 pipeline/模型实现）； - 任何 model plugin 细节都通过 adapter primitives（protocol）注入。 + 任何 model plugin 细节都通过 `ModelBase` primitives（protocol/抽象接口）注入。 **实现细节** - 该模块对 `DMD2Method` 使用 lazy import（`__getattr__`），避免 dispatch 在 diff --git a/fastvideo/distillation/doc/methods/base.md b/fastvideo/distillation/doc/methods/base.md index 0eb44877e..0fc795246 100644 --- a/fastvideo/distillation/doc/methods/base.md +++ b/fastvideo/distillation/doc/methods/base.md @@ -11,7 +11,7 @@ `self.role_modules: ModuleDict`，便于 DDP/FSDP/ckpt 系统统一发现参数。 **需要子类实现的抽象方法** -- `build(cfg, bundle, adapter, validator)`（classmethod） +- `build(cfg, bundle, model, validator)`（classmethod） - `single_train_step(batch, iteration, current_vsa_sparsity=...)` - 返回：`(loss_map, outputs, metrics)` - `loss_map: dict[str, Tensor]`：必须包含 `total_loss`（用于 backward） diff --git a/fastvideo/distillation/doc/methods/consistency_model/__init__.md b/fastvideo/distillation/doc/methods/consistency_model/__init__.md index da701805f..fe5838270 100644 --- a/fastvideo/distillation/doc/methods/consistency_model/__init__.md +++ b/fastvideo/distillation/doc/methods/consistency_model/__init__.md @@ -5,4 +5,4 @@ **期望的演进方向** - 通过 `@register_method("cm")`（示例）注册具体实现。 -- method 只包含算法与 update policy；model plugin/adapter 提供运行时 primitives。 +- method 只包含算法与 update policy；model plugin（`ModelBase`）提供运行时 primitives。 diff --git a/fastvideo/distillation/doc/methods/distribution_matching/__init__.md b/fastvideo/distillation/doc/methods/distribution_matching/__init__.md index 0215034bf..8109b2587 100644 --- a/fastvideo/distillation/doc/methods/distribution_matching/__init__.md +++ b/fastvideo/distillation/doc/methods/distribution_matching/__init__.md @@ -9,5 +9,5 @@ **扩展** - 新增方法时建议保持： - 算法逻辑在 method - - model plugin 细节通过 adapter protocol 注入 + - model plugin 细节通过 `ModelBase` primitives 注入 - 注册通过 `@register_method("")` diff --git a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md index 3cbd5f6a5..c2dd32a77 100644 --- a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md +++ b/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md @@ -2,13 +2,13 @@ **定位** - `DMD2Method`：DMD2 distillation 的算法实现（method layer）。 -- 该文件可以出现 DMD2/critic/fake_score 等算法术语；这些语义不应泄漏到 adapter/model plugin。 +- 该文件可以出现 DMD2/critic/fake_score 等算法术语；这些语义不应泄漏到 model plugin/validator。 **依赖与边界** - ✅ 不 import 任何具体模型/管线实现（Wan/SDXL/...）。 - ✅ 只依赖： - `RoleManager`/`RoleHandle`（获取 student/teacher/critic） - - adapter 提供的 primitives（通过 `_DMD2Adapter` Protocol） + - model plugin 提供的 primitives（通过 `_DMD2Model` Protocol / `ModelBase`） **需要的 roles** - `student`：可训练（trainable=true） @@ -16,23 +16,23 @@ - `critic`：可训练（trainable=true） **算法结构（高层）** -1) `prepare_batch()`：交给 adapter +1) `prepare_batch()`：交给 model plugin 2) student 更新（按 `generator_update_interval` 控制频率） - 先做 student rollout 得到 `generator_pred_x0` - 再计算 DMD loss（teacher CFG 引导 + critic 输出构造梯度） 3) critic 更新（每 step） - 使用 student rollout（no-grad）构造 flow matching loss 4) backward - - 由于 Wan 的 forward_context 约束，需要 adapter.backward(loss, ctx) + - 由于 Wan 的 forward_context 约束，需要 `model.backward(loss, ctx)` **few-step rollout policy（Phase 2.9）** - rollout 的 step list / simulate 逻辑由 method 管理： - `method_config.dmd_denoising_steps` → `_get_denoising_step_list()` - `method_config.rollout_mode={simulate|data_latent}` - - `simulate`：batch 不要求 `vae_latent`（adapter 用零 latents 构造形状） + - `simulate`：batch 不要求 `vae_latent`（model 用零 latents 构造形状） - `data_latent`：batch 必须带 `vae_latent` - - 可选 `warp_denoising_step`（通过 adapter.noise_scheduler.timesteps duck-typing） -- adapter 只提供单步 primitives： + - 可选 `warp_denoising_step`（通过 model.noise_scheduler.timesteps duck-typing） +- model plugin 只提供单步 primitives： - `predict_x0()` / `predict_noise()` / `add_noise()` **optimizer/scheduler 的归属（Phase 2.9）** diff --git a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md index f2a2cea1d..474e8fdbb 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/__init__.md @@ -7,12 +7,13 @@ - `finetune.py`：`FineTuneMethod` - 只要求 `roles.student` - loss/policy 在 method 层 - - 复用同一套 trainer/roles/adapter/model plugin/validator/checkpoint 基础设施 + - 复用同一套 trainer/roles/model plugin/validator/checkpoint 基础设施 - `dfsft.py`：`DiffusionForcingSFTMethod`（`recipe.method: dfsft`） - 只要求 `roles.student` - 训练目标仍是 SFT/flow-matching loss，但使用 **chunk-wise inhomogeneous timesteps** （`t_inhom`）作为 diffusion forcing baseline **设计要点** -- adapter 仍保持 operation-centric（`prepare_batch / predict_* / backward`），不内置 finetune 的 loss 语义。 +- model plugin 通过 `ModelBase` 提供 operation-centric primitives（`prepare_batch / predict_* / backward`）， + 不内置 finetune 的 loss 语义（由 method 管理）。 - model plugin 负责 build-time：加载 student modules、shared components（VAE/scheduler）、dataloader、validator。 diff --git a/fastvideo/distillation/doc/methods/fine_tuning/dfsft.md b/fastvideo/distillation/doc/methods/fine_tuning/dfsft.md index 2be08ff9e..382ea5cb2 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/dfsft.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/dfsft.md @@ -7,12 +7,12 @@ （`t_inhom`）来覆盖“历史上下文不总是干净”的噪声分布（为 causal/streaming 训练做铺垫）。 **核心训练逻辑（单步）** -1) `adapter.prepare_batch(...)` 产出 `TrainingBatch`（包含 `x0` video latents + conditioning）。 +1) `model.prepare_batch(...)` 产出 `TrainingBatch`（包含 `x0` video latents + conditioning）。 2) 采样 `t_inhom`： - 先采样每个 chunk 的 timestep index（`method_config.chunk_size` 控制 chunk 划分） - 再 repeat 到每帧（`[B, T_lat]`） -3) 采样 `noise ~ N(0, I)`，得到 `x_t = adapter.add_noise(x0, noise, t_inhom_flat)` -4) 学生预测 `pred = adapter.predict_noise(student, x_t, t_inhom, batch, ...)` +3) 采样 `noise ~ N(0, I)`，得到 `x_t = model.add_noise(x0, noise, t_inhom_flat)` +4) 学生预测 `pred = model.predict_noise(student, x_t, t_inhom, batch, ...)` 5) loss： - 默认 flow-matching：`MSE(pred, noise - x0)` - 若 `training.precondition_outputs=true`：precondition 到 `x0` 再回归 `x0` @@ -21,7 +21,7 @@ - `method_config`（DFSFT 专属） - `chunk_size: int`（默认 3）：chunk-wise timestep 的 block size - `min_timestep_ratio / max_timestep_ratio: float`：采样 index 范围（映射到 scheduler 的 train steps） - - `attn_kind: dense|vsa`：选择 adapter 的 dense/VSA attention metadata 路径 + - `attn_kind: dense|vsa`：选择 model 的 dense/VSA attention metadata 路径 **约束** - 如果 student transformer 暴露 `num_frame_per_block`，DFSFT 会要求 @@ -30,5 +30,4 @@ **Validation** - DFSFT 依赖 `training.validation`（由 method 驱动 `validator.log_validation(...)`）。 - 当前 validator 仍是 “full-video pipeline” 语义；真正的 streaming/causal rollout - 仍需要在后续阶段实现（避免把 rollout policy 藏进 validator/adapter）。 - + 仍需要在后续阶段实现（避免把 rollout policy 藏进 validator/model plugin）。 diff --git a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md index 126be6a13..57c199fd8 100644 --- a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md +++ b/fastvideo/distillation/doc/methods/fine_tuning/finetune.md @@ -4,7 +4,7 @@ - 将 “finetuning / SFT” 以 `DistillMethod` 的方式接入 Phase 2+ 架构： - 复用 `DistillTrainer`（infra loop / accum / step / ckpt / validate 调用） - 复用 `RoleManager`（角色容器，finetune 只需要 `student`） - - 复用 model plugin/adapter（加载与 primitives） + - 复用 model plugin（加载与 primitives） finetune 可以被视为一种特殊的 distillation recipe：**只有 student + dataset**。 @@ -17,8 +17,8 @@ finetune 可以被视为一种特殊的 distillation recipe：**只有 student + ## 核心训练逻辑 `FineTuneMethod.single_train_step()`： -1. `adapter.prepare_batch(..., latents_source="data")` -2. 用 student 做 `adapter.predict_noise(student, noisy_latents, timesteps, batch, conditional=True)` +1. `model.prepare_batch(..., latents_source="data")` +2. 用 student 做 `model.predict_noise(student, noisy_latents, timesteps, batch, conditional=True)` 3. 计算 loss（与 legacy `training_pipeline.py` 对齐）： - 默认（`training.precondition_outputs=false`）： - target = `noise - x0` @@ -26,7 +26,7 @@ finetune 可以被视为一种特殊的 distillation recipe：**只有 student + - 若 `training.precondition_outputs=true`： - 先 precondition 到 `x0`：`pred_x0 = x_t - sigma * pred` - loss = `mse(pred_x0, x0)` -4. backward 通过 `adapter.backward(loss, ctx, ...)` 执行（确保 forward-context/activation ckpt 兼容） +4. backward 通过 `model.backward(loss, ctx, ...)` 执行（确保 forward-context/activation ckpt 兼容） ## Optimizer / Scheduler - 由 method 创建（而非 model plugin）： diff --git a/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md b/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md index 0c2dce4cd..2c224f718 100644 --- a/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md +++ b/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md @@ -6,5 +6,5 @@ **期望的扩展方式** - 新增 KD method 时： - method 定义需要哪些 roles（student/teacher/aux_teacher/...） - - model plugin 只负责加载这些 roles 的 modules 并构建 roles/adapter + - model plugin 只负责加载这些 roles 的 modules 并构建 `RoleManager`/dataloader/validator（可选） - optimizer/scheduler 由 method 创建并写回 handle diff --git a/fastvideo/distillation/doc/models/__init__.md b/fastvideo/distillation/doc/models/__init__.md index 32b2ef8b2..6de0a486f 100644 --- a/fastvideo/distillation/doc/models/__init__.md +++ b/fastvideo/distillation/doc/models/__init__.md @@ -1,11 +1,12 @@ # `fastvideo/distillation/models/__init__.py` **目的** -- models 是 build-time 插件层（model plugins）： +- models 是 “model plugins”： - 从 YAML config 读取 role spec - 加载模型模块（transformer/vae/...） - 构建 `RoleManager` - - 构建 adapter / dataloader / tracker / validator + - 构建 dataloader / validator（可选） + - **同时实现 `ModelBase` 的运行时 primitives** **为什么需要 model plugins？** - 把 “装配/加载/数据/分布式细节” 与 “算法/rollout/loss/update policy” 分离： diff --git a/fastvideo/distillation/doc/models/adapter.md b/fastvideo/distillation/doc/models/base.md similarity index 70% rename from fastvideo/distillation/doc/models/adapter.md rename to fastvideo/distillation/doc/models/base.md index 297b52f4c..b6b765729 100644 --- a/fastvideo/distillation/doc/models/adapter.md +++ b/fastvideo/distillation/doc/models/base.md @@ -1,12 +1,12 @@ -# `fastvideo/distillation/models/adapter.py` +# `fastvideo/distillation/models/base.py` **定位** -- 定义 `ModelAdapter` 抽象接口：由 model plugin 提供的 *运行时 primitive*（operation-centric）。 -- Method 层不应该 import/依赖任何具体 pipeline/model，只依赖这些 primitive（duck typing / 抽象基类）。 +- 定义 `ModelBase` 抽象接口：由 model plugin 提供的 *运行时 primitives*（operation-centric）。 +- Method 层不应该 import/依赖任何具体 pipeline/model，只依赖这些 primitives（duck typing / 抽象基类）。 **核心思想** -- Adapter **不关心 roles 语义**（student/teacher/critic/... 由 method 定义）。 -- Adapter 提供 “对某个 role handle 执行某个操作” 的 API： +- `ModelBase` **不关心 roles 语义**（student/teacher/critic/... 由 method 定义）。 +- `ModelBase` 提供 “对某个 role handle 执行某个操作” 的 API： - `predict_noise(handle, ...)` - `predict_x0(handle, ...)` - `add_noise(...)` @@ -25,4 +25,3 @@ **接口概览（可选）** - `get_rng_generators() -> dict[str, torch.Generator]` - Trainer/ckpt manager 用于保存 RNG state，实现 “exact resume”。 - diff --git a/fastvideo/distillation/doc/models/components.md b/fastvideo/distillation/doc/models/components.md deleted file mode 100644 index 9724eb7e5..000000000 --- a/fastvideo/distillation/doc/models/components.md +++ /dev/null @@ -1,21 +0,0 @@ -# `fastvideo/distillation/models/components.py` - -**定位** -- `ModelComponents` 是 model plugin 的 build-time 产物（装配输出）。 -- `dispatch.build_runtime_from_config()` 先构建 `ModelComponents`，再把其中的 - `bundle/adapter/validator` 注入到 method，最终交给 trainer 执行训练。 - -**字段说明** -- `training_args` - - 来自 YAML 的 `training`（通过 `TrainingArgs.from_kwargs` 解析）。 -- `bundle` - - `RoleManager(roles=...)`：role → `RoleHandle(modules/optimizers/schedulers/...)`。 -- `adapter` - - 对应模型家族的运行时 primitive（例如 `WanAdapter` / `WanGameAdapter`）。 -- `dataloader` - - 当前 run 的训练 dataloader（由通用 dataloader builder 构建）。 -- `validator`（可选） - - 模型家族对应的 validator backend；是否启用由 method 的 validation 配置决定。 -- `start_step` - - autoresume/ckpt 恢复后从哪个 step 开始继续。 - diff --git a/fastvideo/distillation/doc/models/wan.md b/fastvideo/distillation/doc/models/wan.md index 50dc04897..c6222ebb4 100644 --- a/fastvideo/distillation/doc/models/wan.md +++ b/fastvideo/distillation/doc/models/wan.md @@ -1,12 +1,14 @@ # `fastvideo/distillation/models/wan.py` **定位** -- `@register_model("wan")` 的 build-time 插件（实现：`build_wan_components(...)`）： - - 负责把 YAML config 装配成 `ModelComponents` +- `@register_model("wan")` 的 model plugin（实现：`WanModel(cfg)`）： + - 负责把 YAML config 装配成一个可运行的 `WanModel`（同时实现 `ModelBase` primitives） - 包含 Wan 特有的模块加载、shared components、dataloader schema 等逻辑 **产物** -- `ModelComponents(training_args, bundle, adapter, dataloader, validator, start_step)` +- `WanModel` 实例（关键字段）： + - `training_args`, `bundle`, `dataloader`, `validator`, `start_step` + - 以及 `ModelBase` 的 primitives（`prepare_batch/add_noise/predict_*/backward/...`） **主要职责** 1) **加载 shared components** @@ -16,10 +18,10 @@ - 对每个 role：加载 `transformer`（teacher 可选 `transformer_2`） - 根据 `RoleSpec.trainable` 设置 `requires_grad` + `train()/eval()` - 可选开启 activation checkpoint（仅对 trainable role） -3) **构建 bundle / adapter / dataloader** +3) **构建 bundle / dataloader** - `bundle = RoleManager(roles=role_handles)` - - `adapter = WanAdapter(prompt_handle=student_handle, ...)` - dataloader：parquet + `pyarrow_schema_t2v` + - runtime primitives 由 `WanModel` 直接实现（不再额外分一层 `*Adapter` 类/文件） 4) **tracker / validator（可选）** - validator：`WanValidator`（当 `training.validation.enabled=true`，或 `training.validation` 非空） - model plugin 只负责构建并返回 `validator` diff --git a/fastvideo/distillation/doc/models/wangame.md b/fastvideo/distillation/doc/models/wangame.md index 52403c960..bc675ed0b 100644 --- a/fastvideo/distillation/doc/models/wangame.md +++ b/fastvideo/distillation/doc/models/wangame.md @@ -1,12 +1,14 @@ # `fastvideo/distillation/models/wangame.py` **定位** -- `@register_model("wangame")` 的 build-time 插件（实现：`build_wangame_components(...)`）： - - 负责把 YAML config 装配成 `ModelComponents` - - 包含 WanGame 特有的模块加载、dataset schema、adapter/validator 选择等逻辑 +- `@register_model("wangame")` 的 model plugin（实现：`WanGameModel(cfg)`）： + - 负责把 YAML config 装配成一个可运行的 `WanGameModel`（同时实现 `ModelBase` primitives） + - 包含 WanGame 特有的模块加载、dataset schema、validator 选择等逻辑 **产物** -- `ModelComponents(training_args, bundle, adapter, dataloader, validator, start_step)` +- `WanGameModel` 实例（关键字段）： + - `training_args`, `bundle`, `dataloader`, `validator`, `start_step` + - 以及 `ModelBase` 的 primitives（`prepare_batch/add_noise/predict_*/backward/...`） **主要职责** 1) **加载 shared components** @@ -19,12 +21,12 @@ - `roles..variant: causal` → `CausalWanGameActionTransformer3DModel` - 根据 `RoleSpec.trainable` 设置 `requires_grad` - 可选开启 activation checkpoint（仅对 trainable role） -3) **构建 bundle / adapter / dataloader / validator** +3) **构建 bundle / dataloader / validator** - `bundle = RoleManager(roles=role_handles)` - - `adapter = WanGameAdapter(...)` - dataloader：parquet + `pyarrow_schema_wangame` - validator（可选）：`WanGameValidator`（当 `training.validation.enabled=true`，或 `training.validation` 非空） + - runtime primitives 由 `WanGameModel` 直接实现（不再额外分一层 `*Adapter` 类/文件） **关于 roles.family** - 当前 `wangame` plugin 只支持 `family="wangame"` 的 role。 - 这让 build-time 逻辑保持高内聚：模型加载、batch schema 与 adapter 能保持一致。 + 这让 build-time 逻辑保持高内聚：模型加载、batch schema 与 primitives 能保持一致。 diff --git a/fastvideo/distillation/doc/utils/checkpoint.md b/fastvideo/distillation/doc/utils/checkpoint.md index 85fa0a7b9..a9e88f7d8 100644 --- a/fastvideo/distillation/doc/utils/checkpoint.md +++ b/fastvideo/distillation/doc/utils/checkpoint.md @@ -4,7 +4,7 @@ - Phase 2 的 role-based checkpoint/save-resume 管理： - 按 role 保存/恢复 modules、optimizers、schedulers - 可选保存 dataloader 状态（如果 dataloader 是 stateful） - - 保存 RNG（全局 RNG + method 暴露的额外 generators，例如 adapter/validator 的 RNG） + - 保存 RNG（全局 RNG + method 暴露的额外 generators，例如 model/validator 的 RNG） **关键类型** - `DistillCheckpointConfig` diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index 6a3ed5c75..8efa5da42 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -5,8 +5,9 @@ 减少文件级概念数量。 备注： -- model plugin 的 build-time 产物结构体 `ModelComponents` 在 - `fastvideo/distillation/models/components.py`（更贴近语义归属）。 +- `DistillRuntime` 由 `dispatch.build_runtime_from_config()` 创建并定义在 + `fastvideo/distillation/dispatch.py`（谁创建谁声明）。 +- tracker 由 `DistillTrainer` 构建并持有（避免 model plugin 变成 infra owner）。 这里包含： @@ -62,18 +63,13 @@ 交给 model plugin 解释（例如 `roles.student.variant: causal`） ## 3) Builder 装配相关（build-time / run-time 边界） -- `ModelComponents` - - model 插件 build-time 的产物集合： - - `training_args` - - `bundle` - - `adapter` - - `dataloader` - - `validator`（可选；model-specific） - - `start_step`（用于 resume / warm-start） -备注： -- `DistillRuntime` 由 `dispatch.build_runtime_from_config()` 创建并定义在 - `fastvideo/distillation/dispatch.py`（谁创建谁声明）。 -- tracker 由 `DistillTrainer` 构建并持有（避免 model plugin 变成 infra owner）。 +- model plugin（`@register_model`）直接构建并返回一个 `ModelBase` 实例： + - `training_args` + - `bundle` + - `dataloader` + - `validator`（可选；model-specific） + - `start_step`（用于 resume / warm-start） +- `dispatch.build_runtime_from_config()` 选择 model/method 并返回 `DistillRuntime`。 ## 4) 通用解析 helpers（method_config / optimizer 等） - `get_optional_int(mapping, key, where=...)` diff --git a/fastvideo/distillation/doc/utils/dataloader.md b/fastvideo/distillation/doc/utils/dataloader.md index 2c0793738..ec9d21f35 100644 --- a/fastvideo/distillation/doc/utils/dataloader.md +++ b/fastvideo/distillation/doc/utils/dataloader.md @@ -2,7 +2,7 @@ **目的** - 把 “dataloader 构建” 从 model plugin（原 families/，现 `models/`）中抽离出来， - 让插件更聚焦在加载模块与组装 adapter/bundle。 + 让插件更聚焦在加载模块与组装 `RoleManager` + `ModelBase` primitives。 **当前包含** - `build_parquet_t2v_train_dataloader(training_args, parquet_schema=...)` diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py index fc9461147..c544ec83c 100644 --- a/fastvideo/distillation/methods/base.py +++ b/fastvideo/distillation/methods/base.py @@ -33,9 +33,9 @@ def build( *, cfg: DistillRunConfig, bundle: RoleManager, - adapter: Any, + model: Any, validator: Any | None, - ) -> "DistillMethod": + ) -> DistillMethod: raise NotImplementedError def set_tracker(self, tracker: Any) -> None: diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index cc99e8150..1de8cab24 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -49,14 +49,14 @@ ) -class _DMD2Adapter(Protocol): - """Algorithm-specific adapter contract for :class:`DMD2Method`. +class _DMD2Model(Protocol): + """Algorithm-specific model contract for :class:`DMD2Method`. The method layer is intentionally model-agnostic: it should not import or depend on any concrete pipeline/model implementation. Instead, all model-specific primitives (batch preparation, noise schedule helpers, forward-context management, and role-specific backward behavior) are - provided by an adapter (e.g. ``WanAdapter``). + provided by a model plugin (e.g. ``WanModel``). This ``Protocol`` documents the required surface area and helps static type checkers/IDE tooling; it is not enforced at runtime (duck typing). @@ -132,7 +132,7 @@ class DMD2Method(DistillMethod): All model-plugin details (how to run student rollout, teacher CFG prediction, critic loss, and how to safely run backward under activation - checkpointing/forward-context constraints) are delegated to the adapter + checkpointing/forward-context constraints) are delegated to the model plugin passed in at construction time. """ @@ -140,7 +140,7 @@ def __init__( self, *, bundle: RoleManager, - adapter: _DMD2Adapter, + model: _DMD2Model, method_config: dict[str, Any] | None = None, validation_config: dict[str, Any] | None = None, validator: Any | None = None, @@ -156,9 +156,9 @@ def __init__( raise ValueError("DMD2Method requires roles.teacher.trainable=false") if not self.critic.trainable: raise ValueError("DMD2Method requires roles.critic.trainable=true") - self.adapter = adapter + self.model = model self.validator = validator - self.training_args = adapter.training_args + self.training_args = model.training_args self.method_config: dict[str, Any] = dict(method_config or {}) self.validation_config: dict[str, Any] = dict(validation_config or {}) self._cfg_uncond = self._parse_cfg_uncond() @@ -172,12 +172,12 @@ def build( *, cfg: DistillRunConfig, bundle: RoleManager, - adapter: Any, + model: Any, validator: Any | None, ) -> DistillMethod: return cls( bundle=bundle, - adapter=adapter, + model=model, method_config=cfg.method_config, validation_config=cfg.validation, validator=validator, @@ -332,7 +332,7 @@ def _init_optimizers_and_schedulers(self) -> None: ) def on_train_start(self) -> None: - self.adapter.on_train_start() + self.model.on_train_start() def _is_validation_enabled(self) -> bool: cfg = self.validation_config @@ -541,10 +541,10 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: generators: dict[str, torch.Generator] = {} - adapter = getattr(self, "adapter", None) - get_adapter_generators = getattr(adapter, "get_rng_generators", None) - if callable(get_adapter_generators): - generators.update(get_adapter_generators()) + model = getattr(self, "model", None) + get_model_generators = getattr(model, "get_rng_generators", None) + if callable(get_model_generators): + generators.update(get_model_generators()) validator = getattr(self, "validator", None) validation_gen = getattr(validator, "validation_random_generator", None) @@ -599,9 +599,9 @@ def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: if warp is None: warp = getattr(self.training_args, "warp_denoising_step", False) if bool(warp): - noise_scheduler = getattr(self.adapter, "noise_scheduler", None) + noise_scheduler = getattr(self.model, "noise_scheduler", None) if noise_scheduler is None: - raise ValueError("warp_denoising_step requires adapter.noise_scheduler.timesteps") + raise ValueError("warp_denoising_step requires model.noise_scheduler.timesteps") timesteps = torch.cat( ( @@ -634,8 +634,8 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: if self._rollout_mode != "simulate": timestep = self._sample_rollout_timestep(device) noise = torch.randn(latents.shape, device=device, dtype=dtype) - noisy_latents = self.adapter.add_noise(latents, noise, timestep) - pred_x0 = self.adapter.predict_x0( + noisy_latents = self.model.add_noise(latents, noise, timestep) + pred_x0 = self.model.predict_x0( self.student, noisy_latents, timestep, @@ -672,7 +672,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: 1, device=device, dtype=torch.long ) - pred_clean = self.adapter.predict_x0( + pred_clean = self.model.predict_x0( self.student, current_noise_latents, current_timestep_tensor, @@ -687,7 +687,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: 1, device=device, dtype=torch.long ) noise = torch.randn(latents.shape, device=device, dtype=pred_clean.dtype) - current_noise_latents = self.adapter.add_noise( + current_noise_latents = self.model.add_noise( pred_clean, noise, next_timestep_tensor, @@ -702,7 +702,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: noisy_input = current_noise_latents_copy if with_grad: - pred_x0 = self.adapter.predict_x0( + pred_x0 = self.model.predict_x0( self.student, noisy_input, target_timestep, @@ -713,7 +713,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: ) else: with torch.no_grad(): - pred_x0 = self.adapter.predict_x0( + pred_x0 = self.model.predict_x0( self.student, noisy_input, target_timestep, @@ -733,21 +733,21 @@ def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dic device = generator_pred_x0.device fake_score_timestep = torch.randint( 0, - int(self.adapter.num_train_timesteps), + int(self.model.num_train_timesteps), [1], device=device, dtype=torch.long, ) - fake_score_timestep = self.adapter.shift_and_clamp_timestep(fake_score_timestep) + fake_score_timestep = self.model.shift_and_clamp_timestep(fake_score_timestep) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_x0 = self.adapter.add_noise(generator_pred_x0, noise, fake_score_timestep) + noisy_x0 = self.model.add_noise(generator_pred_x0, noise, fake_score_timestep) - pred_noise = self.adapter.predict_noise( + pred_noise = self.model.predict_noise( self.critic, noisy_x0, fake_score_timestep, @@ -779,21 +779,21 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor with torch.no_grad(): timestep = torch.randint( 0, - int(self.adapter.num_train_timesteps), + int(self.model.num_train_timesteps), [1], device=device, dtype=torch.long, ) - timestep = self.adapter.shift_and_clamp_timestep(timestep) + timestep = self.model.shift_and_clamp_timestep(timestep) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_latents = self.adapter.add_noise(generator_pred_x0, noise, timestep) + noisy_latents = self.model.add_noise(generator_pred_x0, noise, timestep) - faker_x0 = self.adapter.predict_x0( + faker_x0 = self.model.predict_x0( self.critic, noisy_latents, timestep, @@ -802,7 +802,7 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_cond_x0 = self.adapter.predict_x0( + real_cond_x0 = self.model.predict_x0( self.teacher, noisy_latents, timestep, @@ -811,7 +811,7 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_uncond_x0 = self.adapter.predict_x0( + real_uncond_x0 = self.model.predict_x0( self.teacher, noisy_latents, timestep, @@ -843,7 +843,7 @@ def single_train_step( if self._rollout_mode == "simulate": latents_source = "zeros" - training_batch = self.adapter.prepare_batch( + training_batch = self.model.prepare_batch( batch, current_vsa_sparsity=current_vsa_sparsity, latents_source=latents_source, @@ -898,7 +898,7 @@ def backward( student_ctx = backward_ctx.get("student_ctx") if student_ctx is None: raise RuntimeError("Missing student backward context") - self.adapter.backward( + self.model.backward( loss_map["generator_loss"], student_ctx, grad_accum_rounds=grad_accum_rounds, @@ -907,7 +907,7 @@ def backward( critic_ctx = backward_ctx.get("critic_ctx") if critic_ctx is None: raise RuntimeError("Missing critic backward context") - self.adapter.backward( + self.model.backward( loss_map["fake_score_loss"], critic_ctx, grad_accum_rounds=grad_accum_rounds, diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index bd0639d7a..111d3ca70 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -42,11 +42,11 @@ from fastvideo.distillation.validators.base import ValidationRequest -class _DFSFTAdapter(Protocol): - """Adapter contract for diffusion-forcing SFT (DFSFT). +class _DFSFTModel(Protocol): + """Model contract for diffusion-forcing SFT (DFSFT). DFSFT is implemented purely at the method (algorithm) layer and relies only - on operation-centric primitives exposed by the model-family adapter. + on operation-centric primitives exposed by the model plugin. """ training_args: Any @@ -101,7 +101,7 @@ def __init__( self, *, bundle: RoleManager, - adapter: _DFSFTAdapter, + model: _DFSFTModel, method_config: dict[str, Any] | None = None, validation_config: dict[str, Any] | None = None, validator: Any | None = None, @@ -112,9 +112,9 @@ def __init__( if not self.student.trainable: raise ValueError("DFSFT requires roles.student.trainable=true") - self.adapter = adapter + self.model = model self.validator = validator - self.training_args = adapter.training_args + self.training_args = model.training_args self.method_config: dict[str, Any] = dict(method_config or {}) self.validation_config: dict[str, Any] = dict(validation_config or {}) self._attn_kind: Literal["dense", "vsa"] = self._parse_attn_kind( @@ -132,12 +132,12 @@ def build( *, cfg: DistillRunConfig, bundle: RoleManager, - adapter: Any, + model: Any, validator: Any | None, ) -> DistillMethod: return cls( bundle=bundle, - adapter=adapter, + model=model, method_config=cfg.method_config, validation_config=cfg.validation, validator=validator, @@ -186,7 +186,9 @@ def _parse_ratio(self, raw: Any, *, where: str, default: float) -> float: raise ValueError(f"{where} must be a number/string, got {type(raw).__name__}") def _parse_timestep_index_range(self) -> tuple[int, int]: - scheduler = self.adapter.noise_scheduler + scheduler = getattr(self.model, "noise_scheduler", None) + if scheduler is None: + raise ValueError("DFSFT requires model.noise_scheduler") num_steps = int(getattr(scheduler, "config", scheduler).num_train_timesteps) min_ratio = self._parse_ratio( @@ -277,7 +279,7 @@ def _init_optimizers_and_schedulers(self) -> None: ) def on_train_start(self) -> None: - self.adapter.on_train_start() + self.model.on_train_start() def _is_validation_enabled(self) -> bool: cfg = self.validation_config @@ -460,10 +462,10 @@ def log_validation(self, iteration: int) -> None: def get_rng_generators(self) -> dict[str, torch.Generator]: generators: dict[str, torch.Generator] = {} - adapter = getattr(self, "adapter", None) - get_adapter_generators = getattr(adapter, "get_rng_generators", None) - if callable(get_adapter_generators): - generators.update(get_adapter_generators()) + model = getattr(self, "model", None) + get_model_generators = getattr(model, "get_rng_generators", None) + if callable(get_model_generators): + generators.update(get_model_generators()) validator = getattr(self, "validator", None) validation_gen = getattr(validator, "validation_random_generator", None) @@ -494,14 +496,14 @@ def single_train_step( current_vsa_sparsity: float = 0.0, ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: del iteration - training_batch = self.adapter.prepare_batch( + training_batch = self.model.prepare_batch( batch, current_vsa_sparsity=current_vsa_sparsity, latents_source="data", ) if training_batch.latents is None: - raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.latents") + raise RuntimeError("model.prepare_batch() must set TrainingBatch.latents") clean_latents = training_batch.latents if not torch.is_tensor(clean_latents): @@ -528,14 +530,18 @@ def single_train_step( device=clean_latents.device, ) sp_size = int(getattr(self.training_args, "sp_size", 1) or 1) - sp_group = getattr(self.adapter, "sp_group", None) + sp_group = getattr(self.model, "sp_group", None) if sp_size > 1 and sp_group is not None and hasattr(sp_group, "broadcast"): sp_group.broadcast(timestep_indices, src=0) - schedule_timesteps = self.adapter.noise_scheduler.timesteps.to( + scheduler = getattr(self.model, "noise_scheduler", None) + if scheduler is None: + raise ValueError("DFSFT requires model.noise_scheduler") + + schedule_timesteps = scheduler.timesteps.to( device=clean_latents.device, dtype=torch.float32 ) - schedule_sigmas = self.adapter.noise_scheduler.sigmas.to( + schedule_sigmas = scheduler.sigmas.to( device=clean_latents.device, dtype=clean_latents.dtype ) t_inhom = schedule_timesteps[timestep_indices] @@ -548,13 +554,13 @@ def single_train_step( raise TypeError("TrainingBatch.noise must be a torch.Tensor when set") noise = noise.permute(0, 2, 1, 3, 4).to(dtype=clean_latents.dtype) - noisy_latents = self.adapter.add_noise( + noisy_latents = self.model.add_noise( clean_latents, noise, t_inhom.flatten(), ) - pred = self.adapter.predict_noise( + pred = self.model.predict_noise( self.student, noisy_latents, t_inhom, @@ -594,7 +600,7 @@ def backward( if ctx is None: super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) return - self.adapter.backward( + self.model.backward( loss_map["total_loss"], ctx, grad_accum_rounds=grad_accum_rounds, diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 0b044c957..849a0f0da 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -40,12 +40,13 @@ ) -class _FineTuneAdapter(Protocol): - """Adapter contract for :class:`FineTuneMethod`. +class _FineTuneModel(Protocol): + """Model contract for :class:`FineTuneMethod`. Finetuning is implemented as a method (algorithm layer) on top of the - model-plugin-provided adapter. The method must remain model-plugin agnostic, so - it consumes only operation-centric primitives exposed by the adapter. + model-plugin-provided model plugin. The method must remain model-plugin + agnostic, so it consumes only operation-centric primitives exposed by the + model. """ training_args: Any @@ -92,7 +93,7 @@ def __init__( self, *, bundle: RoleManager, - adapter: _FineTuneAdapter, + model: _FineTuneModel, method_config: dict[str, Any] | None = None, validation_config: dict[str, Any] | None = None, validator: Any | None = None, @@ -103,9 +104,9 @@ def __init__( if not self.student.trainable: raise ValueError("FineTuneMethod requires roles.student.trainable=true") - self.adapter = adapter + self.model = model self.validator = validator - self.training_args = adapter.training_args + self.training_args = model.training_args self.method_config: dict[str, Any] = dict(method_config or {}) self.validation_config: dict[str, Any] = dict(validation_config or {}) self._attn_kind: Literal["dense", "vsa"] = self._parse_attn_kind( @@ -120,12 +121,12 @@ def build( *, cfg: DistillRunConfig, bundle: RoleManager, - adapter: Any, + model: Any, validator: Any | None, ) -> DistillMethod: return cls( bundle=bundle, - adapter=adapter, + model=model, method_config=cfg.method_config, validation_config=cfg.validation, validator=validator, @@ -201,7 +202,7 @@ def _init_optimizers_and_schedulers(self) -> None: ) def on_train_start(self) -> None: - self.adapter.on_train_start() + self.model.on_train_start() def _is_validation_enabled(self) -> bool: cfg = self.validation_config @@ -392,10 +393,10 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: generators: dict[str, torch.Generator] = {} - adapter = getattr(self, "adapter", None) - get_adapter_generators = getattr(adapter, "get_rng_generators", None) - if callable(get_adapter_generators): - generators.update(get_adapter_generators()) + model = getattr(self, "model", None) + get_model_generators = getattr(model, "get_rng_generators", None) + if callable(get_model_generators): + generators.update(get_model_generators()) validator = getattr(self, "validator", None) validation_gen = getattr(validator, "validation_random_generator", None) @@ -432,24 +433,24 @@ def single_train_step( current_vsa_sparsity: float = 0.0, ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: del iteration - training_batch = self.adapter.prepare_batch( + training_batch = self.model.prepare_batch( batch, current_vsa_sparsity=current_vsa_sparsity, latents_source="data", ) if training_batch.latents is None: - raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.latents") + raise RuntimeError("model.prepare_batch() must set TrainingBatch.latents") if training_batch.noisy_model_input is None: raise RuntimeError( - "adapter.prepare_batch() must set TrainingBatch.noisy_model_input" + "model.prepare_batch() must set TrainingBatch.noisy_model_input" ) if training_batch.noise is None: - raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.noise") + raise RuntimeError("model.prepare_batch() must set TrainingBatch.noise") if training_batch.sigmas is None: - raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.sigmas") + raise RuntimeError("model.prepare_batch() must set TrainingBatch.sigmas") if training_batch.timesteps is None: - raise RuntimeError("adapter.prepare_batch() must set TrainingBatch.timesteps") + raise RuntimeError("model.prepare_batch() must set TrainingBatch.timesteps") clean_latents = training_batch.latents noisy_latents = training_batch.noisy_model_input.permute(0, 2, 1, 3, 4) @@ -457,7 +458,7 @@ def single_train_step( sigmas = training_batch.sigmas timesteps = training_batch.timesteps - pred = self.adapter.predict_noise( + pred = self.model.predict_noise( self.student, noisy_latents, timesteps, @@ -497,7 +498,7 @@ def backward( if ctx is None: super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) return - self.adapter.backward( + self.model.backward( loss_map["total_loss"], ctx, grad_accum_rounds=grad_accum_rounds, diff --git a/fastvideo/distillation/models/adapter.py b/fastvideo/distillation/models/base.py similarity index 91% rename from fastvideo/distillation/models/adapter.py rename to fastvideo/distillation/models/base.py index 8ad3330a9..a31e2d94c 100644 --- a/fastvideo/distillation/models/adapter.py +++ b/fastvideo/distillation/models/base.py @@ -13,20 +13,24 @@ from fastvideo.pipelines import TrainingBatch -class ModelAdapter(ABC): +class ModelBase(ABC): """Operation-centric runtime primitives implemented by a model plugin. This interface is intentionally *method-agnostic*: - A method selects roles (student/teacher/critic/...) and decides how to use them. - - The adapter implements how to run those roles against FastVideo pipelines, - forward-context requirements, and batch normalization quirks. + - The model plugin implements how to run those roles against FastVideo + pipelines, forward-context requirements, and batch normalization quirks. Implementations typically live next to the model plugin (e.g. `models/wan.py`) rather than in a global adapter registry. """ training_args: Any + bundle: Any + dataloader: Any + validator: Any | None + start_step: int @property @abstractmethod @@ -96,4 +100,3 @@ def predict_x0( @abstractmethod def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: """Backward hook that may restore forward-context for checkpointed modules.""" - diff --git a/fastvideo/distillation/models/components.py b/fastvideo/distillation/models/components.py deleted file mode 100644 index 033f372fd..000000000 --- a/fastvideo/distillation/models/components.py +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.distillation.roles import RoleManager - - -@dataclass(slots=True) -class ModelComponents: - """Build-time outputs produced by a model plugin. - - A model plugin is responsible for loading modules, constructing a - role container (`RoleManager`), and assembling shared - components needed by runtime adapters, dataloaders, validators, and - trackers. - """ - - training_args: TrainingArgs - bundle: RoleManager - adapter: Any - dataloader: Any - validator: Any | None = None - start_step: int = 0 diff --git a/fastvideo/distillation/models/wan.py b/fastvideo/distillation/models/wan.py index 1617f0aba..1ec4fd099 100644 --- a/fastvideo/distillation/models/wan.py +++ b/fastvideo/distillation/models/wan.py @@ -16,7 +16,7 @@ - `moba_config` (if `FASTVIDEO_ATTENTION_BACKEND=VMOBA_ATTN`) - `enable_gradient_checkpointing_type` (optional) - `training.validation.*` (consumed by `WanValidator` when enabled) -- `method_config.cfg_uncond.*` (consumed by `WanAdapter` for CFG-uncond policy) +- `method_config.cfg_uncond.*` (consumed by `WanModel` for CFG-uncond policy) """ from __future__ import annotations @@ -52,8 +52,7 @@ from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed from fastvideo.distillation.dispatch import register_model -from fastvideo.distillation.models.adapter import ModelAdapter -from fastvideo.distillation.models.components import ModelComponents +from fastvideo.distillation.models.base import ModelBase from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.distillation.utils.config import DistillRunConfig from fastvideo.distillation.utils.dataloader import build_parquet_t2v_train_dataloader @@ -70,17 +69,96 @@ VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] -class WanAdapter(ModelAdapter): - """Wan runtime adapter: exposes operation-centric primitives to methods.""" +@register_model("wan") +class WanModel(ModelBase): + """Wan model plugin: loads roles + shared components and exposes runtime primitives.""" + + def __init__(self, *, cfg: DistillRunConfig) -> None: + training_args = cfg.training_args + roles_cfg = cfg.roles + + if getattr(training_args, "seed", None) is None: + raise ValueError("training.seed must be set for distillation") + if not getattr(training_args, "data_path", ""): + raise ValueError("training.data_path must be set for distillation") + + # Load shared components (student base path). + training_args.override_transformer_cls_name = "WanTransformer3DModel" + vae = load_module_from_path( + model_path=str(training_args.model_path), + module_type="vae", + training_args=training_args, + ) + noise_scheduler = FlowMatchEulerDiscreteScheduler( + shift=float(training_args.pipeline_config.flow_shift or 0.0) + ) + + role_handles: dict[str, RoleHandle] = {} + for role, role_spec in roles_cfg.items(): + if role_spec.family != "wan": + raise ValueError( + "Wan model plugin only supports roles with family='wan'; " + f"got {role}={role_spec.family!r}" + ) + + disable_custom_init_weights = bool( + getattr(role_spec, "disable_custom_init_weights", False) + ) + transformer = load_module_from_path( + model_path=role_spec.path, + module_type="transformer", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + ) + modules: dict[str, torch.nn.Module] = {"transformer": transformer} + + # Optional MoE support: load transformer_2 if present in the model. + try: + transformer_2 = load_module_from_path( + model_path=role_spec.path, + module_type="transformer_2", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + ) + except ValueError: + transformer_2 = None + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + + for name, module in list(modules.items()): + module = apply_trainable(module, trainable=bool(role_spec.trainable)) + if role_spec.trainable and getattr( + training_args, "enable_gradient_checkpointing_type", None + ): + module = apply_activation_checkpointing( + module, + checkpointing_type=training_args.enable_gradient_checkpointing_type, + ) + modules[name] = module + + role_handles[role] = RoleHandle( + modules=modules, + optimizers={}, + lr_schedulers={}, + trainable=bool(role_spec.trainable), + ) + + self.bundle = RoleManager(roles=role_handles) + + # Optional validator. + self.validator = None + validation_cfg = getattr(cfg, "validation", {}) or {} + validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) + if validation_enabled: + from fastvideo.distillation.validators.wan import WanValidator + + self.validator = WanValidator(training_args=training_args) + + # NOTE: runtime primitives need a prompt-encoding capable handle. + prompt_handle = role_handles.get("student") + if prompt_handle is None: + raise ValueError("Wan model plugin requires a 'student' role for prompt encoding") - def __init__( - self, - *, - prompt_handle: RoleHandle, - training_args: Any, - noise_scheduler: Any, - vae: Any, - ) -> None: self.prompt_handle = prompt_handle self.training_args = training_args self.noise_scheduler = noise_scheduler @@ -98,6 +176,14 @@ def __init__( self._init_timestep_mechanics() + from fastvideo.dataset.dataloader.schema import pyarrow_schema_t2v + + self.dataloader = build_parquet_t2v_train_dataloader( + training_args, + parquet_schema=pyarrow_schema_t2v, + ) + self.start_step = 0 + def _get_training_dtype(self) -> torch.dtype: return torch.bfloat16 @@ -627,113 +713,3 @@ def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> N timesteps, attn_metadata = ctx with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): (loss / max(1, int(grad_accum_rounds))).backward() - - -@register_model("wan") -def build_wan_components(*, cfg: DistillRunConfig) -> ModelComponents: - training_args = cfg.training_args - roles_cfg = cfg.roles - - if getattr(training_args, "seed", None) is None: - raise ValueError("training.seed must be set for distillation") - if not getattr(training_args, "data_path", ""): - raise ValueError("training.data_path must be set for distillation") - - # Load shared components (student base path). - training_args.override_transformer_cls_name = "WanTransformer3DModel" - vae = load_module_from_path( - model_path=str(training_args.model_path), - module_type="vae", - training_args=training_args, - ) - noise_scheduler = FlowMatchEulerDiscreteScheduler( - shift=float(training_args.pipeline_config.flow_shift or 0.0) - ) - - role_handles: dict[str, RoleHandle] = {} - for role, role_spec in roles_cfg.items(): - if role_spec.family != "wan": - raise ValueError( - "Wan model plugin only supports roles with family='wan'; " - f"got {role}={role_spec.family!r}" - ) - - disable_custom_init_weights = bool(getattr(role_spec, "disable_custom_init_weights", False)) - transformer = load_module_from_path( - model_path=role_spec.path, - module_type="transformer", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - ) - modules: dict[str, torch.nn.Module] = {"transformer": transformer} - - # Optional MoE support: load transformer_2 if present in the model. - try: - transformer_2 = load_module_from_path( - model_path=role_spec.path, - module_type="transformer_2", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - ) - except ValueError: - transformer_2 = None - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 - - for name, module in list(modules.items()): - module = apply_trainable(module, trainable=bool(role_spec.trainable)) - if role_spec.trainable and getattr( - training_args, "enable_gradient_checkpointing_type", None - ): - module = apply_activation_checkpointing( - module, - checkpointing_type=training_args.enable_gradient_checkpointing_type, - ) - modules[name] = module - - optimizers: dict[str, torch.optim.Optimizer] = {} - lr_schedulers: dict[str, Any] = {} - - role_handles[role] = RoleHandle( - modules=modules, - optimizers=optimizers, - lr_schedulers=lr_schedulers, - trainable=bool(role_spec.trainable), - ) - - bundle = RoleManager(roles=role_handles) - - validator = None - validation_cfg = getattr(cfg, "validation", {}) or {} - validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) - if validation_enabled: - from fastvideo.distillation.validators.wan import WanValidator - - validator = WanValidator(training_args=training_args) - - # NOTE: adapter is the model runtime boundary; it may implement multiple - # method-specific protocols via duck typing. - prompt_handle = role_handles.get("student") - if prompt_handle is None: - raise ValueError("Wan model plugin requires a 'student' role for prompt encoding") - adapter = WanAdapter( - prompt_handle=prompt_handle, - training_args=training_args, - noise_scheduler=noise_scheduler, - vae=vae, - ) - from fastvideo.dataset.dataloader.schema import pyarrow_schema_t2v - - dataloader = build_parquet_t2v_train_dataloader( - training_args, - parquet_schema=pyarrow_schema_t2v, - ) - - return ModelComponents( - training_args=training_args, - bundle=bundle, - adapter=adapter, - dataloader=dataloader, - validator=validator, - start_step=0, - ) diff --git a/fastvideo/distillation/models/wangame.py b/fastvideo/distillation/models/wangame.py index 41b1b9456..98b62b13b 100644 --- a/fastvideo/distillation/models/wangame.py +++ b/fastvideo/distillation/models/wangame.py @@ -17,7 +17,7 @@ - `moba_config` (if `FASTVIDEO_ATTENTION_BACKEND=VMOBA_ATTN`) - `enable_gradient_checkpointing_type` (optional) - `training.validation.*` (consumed by `WanGameValidator` when enabled) -- `method_config.cfg_uncond.*` (consumed by `WanGameAdapter` for CFG-uncond policy) +- `method_config.cfg_uncond.*` (consumed by `WanGameModel` for CFG-uncond policy) """ from __future__ import annotations @@ -49,8 +49,7 @@ from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed from fastvideo.distillation.dispatch import register_model -from fastvideo.distillation.models.adapter import ModelAdapter -from fastvideo.distillation.models.components import ModelComponents +from fastvideo.distillation.models.base import ModelBase from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.distillation.utils.config import DistillRunConfig from fastvideo.distillation.utils.dataloader import build_parquet_wangame_train_dataloader @@ -67,10 +66,11 @@ VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] -class WanGameAdapter(ModelAdapter): - """WanGame adapter: exposes operation-centric primitives for methods. +@register_model("wangame") +class WanGameModel(ModelBase): + """WanGame model plugin: loads roles + shared components and exposes runtime primitives. - This adapter is intentionally *not* method-specific: + This model plugin is intentionally *not* method-specific: - It knows how to turn a wangame parquet batch into forward primitives. - It knows how to run a model role (handle) for predict_noise / predict_x0. - It does not encode DMD2/SFT/etc semantics beyond required primitives. @@ -79,10 +79,103 @@ class WanGameAdapter(ModelAdapter): def __init__( self, *, - training_args: Any, - noise_scheduler: Any, - vae: Any, + cfg: DistillRunConfig, ) -> None: + training_args = cfg.training_args + roles_cfg = cfg.roles + + if getattr(training_args, "seed", None) is None: + raise ValueError("training.seed must be set for distillation") + if not getattr(training_args, "data_path", ""): + raise ValueError("training.data_path must be set for distillation") + + # Load shared components (student base path). + vae = load_module_from_path( + model_path=str(training_args.model_path), + module_type="vae", + training_args=training_args, + ) + noise_scheduler = FlowMatchEulerDiscreteScheduler( + shift=float(training_args.pipeline_config.flow_shift or 0.0) + ) + + role_handles: dict[str, RoleHandle] = {} + for role, role_spec in roles_cfg.items(): + if role_spec.family != "wangame": + raise ValueError( + "Wangame model plugin only supports roles with family='wangame'; " + f"got {role}={role_spec.family!r}" + ) + + variant_raw = (role_spec.extra or {}).get("variant", None) + if variant_raw is None or variant_raw == "": + transformer_cls_name = "WanGameActionTransformer3DModel" + else: + variant = str(variant_raw).strip().lower() + if variant in {"bidirectional", "bidi"}: + transformer_cls_name = "WanGameActionTransformer3DModel" + elif variant == "causal": + transformer_cls_name = "CausalWanGameActionTransformer3DModel" + else: + raise ValueError( + f"Unknown roles.{role}.variant for wangame: " + f"{variant_raw!r}. Expected 'causal' or 'bidirectional'." + ) + + disable_custom_init_weights = bool( + getattr(role_spec, "disable_custom_init_weights", False) + ) + transformer = load_module_from_path( + model_path=role_spec.path, + module_type="transformer", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + override_transformer_cls_name=transformer_cls_name, + ) + modules: dict[str, torch.nn.Module] = {"transformer": transformer} + + # Optional MoE support: load transformer_2 if present in the model. + try: + transformer_2 = load_module_from_path( + model_path=role_spec.path, + module_type="transformer_2", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + ) + except ValueError: + transformer_2 = None + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + + for name, module in list(modules.items()): + module = apply_trainable(module, trainable=bool(role_spec.trainable)) + if role_spec.trainable and getattr( + training_args, "enable_gradient_checkpointing_type", None + ): + module = apply_activation_checkpointing( + module, + checkpointing_type=training_args.enable_gradient_checkpointing_type, + ) + modules[name] = module + + role_handles[role] = RoleHandle( + modules=modules, + optimizers={}, + lr_schedulers={}, + trainable=bool(role_spec.trainable), + ) + + self.bundle = RoleManager(roles=role_handles) + + # Optional validator. + self.validator = None + validation_cfg = getattr(cfg, "validation", {}) or {} + validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) + if validation_enabled: + from fastvideo.distillation.validators.wangame import WanGameValidator + + self.validator = WanGameValidator(training_args=training_args) + self.training_args = training_args self.noise_scheduler = noise_scheduler self.vae = vae @@ -96,6 +189,14 @@ def __init__( self._init_timestep_mechanics() + from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame + + self.dataloader = build_parquet_wangame_train_dataloader( + training_args, + parquet_schema=pyarrow_schema_wangame, + ) + self.start_step = 0 + def _get_training_dtype(self) -> torch.dtype: return torch.bfloat16 @@ -149,7 +250,7 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: if self.noise_random_generator is None: raise RuntimeError( - "WanGameAdapter.on_train_start() must be called before prepare_batch()" + "WanGameModel.on_train_start() must be called before prepare_batch()" ) u = compute_density_for_timestep_sampling( @@ -211,7 +312,7 @@ def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: if self.noise_gen_cuda is None: raise RuntimeError( - "WanGameAdapter.on_train_start() must be called before prepare_batch()" + "WanGameModel.on_train_start() must be called before prepare_batch()" ) noise = torch.randn( @@ -480,11 +581,11 @@ def _select_cfg_condition_inputs( image_latents = batch.image_latents mask_lat_size = batch.mask_lat_size if image_embeds is None: - raise RuntimeError("WanGameAdapter requires TrainingBatch.image_embeds") + raise RuntimeError("WanGameModel requires TrainingBatch.image_embeds") if image_latents is None: - raise RuntimeError("WanGameAdapter requires TrainingBatch.image_latents") + raise RuntimeError("WanGameModel requires TrainingBatch.image_latents") if mask_lat_size is None: - raise RuntimeError("WanGameAdapter requires TrainingBatch.mask_lat_size") + raise RuntimeError("WanGameModel requires TrainingBatch.mask_lat_size") viewmats = getattr(batch, "viewmats", None) Ks = getattr(batch, "Ks", None) @@ -536,7 +637,7 @@ def _select_cfg_condition_inputs( if on_missing == "ignore": continue raise ValueError( - "WanGameAdapter does not support cfg_uncond channel " + "WanGameModel does not support cfg_uncond channel " f"{channel!r} (policy={policy!r}). " "Set cfg_uncond.on_missing=ignore or remove the channel." ) @@ -720,121 +821,3 @@ def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> N timesteps, attn_metadata = ctx with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): (loss / max(1, int(grad_accum_rounds))).backward() - - -@register_model("wangame") -def build_wangame_components(*, cfg: DistillRunConfig) -> ModelComponents: - training_args = cfg.training_args - roles_cfg = cfg.roles - - if getattr(training_args, "seed", None) is None: - raise ValueError("training.seed must be set for distillation") - if not getattr(training_args, "data_path", ""): - raise ValueError("training.data_path must be set for distillation") - - # Load shared components (student base path). - vae = load_module_from_path( - model_path=str(training_args.model_path), - module_type="vae", - training_args=training_args, - ) - noise_scheduler = FlowMatchEulerDiscreteScheduler( - shift=float(training_args.pipeline_config.flow_shift or 0.0) - ) - - role_handles: dict[str, RoleHandle] = {} - for role, role_spec in roles_cfg.items(): - if role_spec.family != "wangame": - raise ValueError( - "Wangame model plugin only supports roles with family='wangame'; " - f"got {role}={role_spec.family!r}" - ) - - variant_raw = (role_spec.extra or {}).get("variant", None) - if variant_raw is None or variant_raw == "": - transformer_cls_name = "WanGameActionTransformer3DModel" - else: - variant = str(variant_raw).strip().lower() - if variant in {"bidirectional", "bidi"}: - transformer_cls_name = "WanGameActionTransformer3DModel" - elif variant == "causal": - transformer_cls_name = "CausalWanGameActionTransformer3DModel" - else: - raise ValueError( - f"Unknown roles.{role}.variant for wangame: " - f"{variant_raw!r}. Expected 'causal' or 'bidirectional'." - ) - - disable_custom_init_weights = bool(getattr(role_spec, "disable_custom_init_weights", False)) - transformer = load_module_from_path( - model_path=role_spec.path, - module_type="transformer", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - override_transformer_cls_name=transformer_cls_name, - ) - modules: dict[str, torch.nn.Module] = {"transformer": transformer} - - # Optional MoE support: load transformer_2 if present in the model. - try: - transformer_2 = load_module_from_path( - model_path=role_spec.path, - module_type="transformer_2", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - ) - except ValueError: - transformer_2 = None - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 - - for name, module in list(modules.items()): - module = apply_trainable(module, trainable=bool(role_spec.trainable)) - if role_spec.trainable and getattr( - training_args, "enable_gradient_checkpointing_type", None - ): - module = apply_activation_checkpointing( - module, - checkpointing_type=training_args.enable_gradient_checkpointing_type, - ) - modules[name] = module - - role_handles[role] = RoleHandle( - modules=modules, - optimizers={}, - lr_schedulers={}, - trainable=bool(role_spec.trainable), - ) - - bundle = RoleManager(roles=role_handles) - - validator = None - validation_cfg = getattr(cfg, "validation", {}) or {} - validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) - if validation_enabled: - from fastvideo.distillation.validators.wangame import WanGameValidator - - validator = WanGameValidator(training_args=training_args) - - adapter = WanGameAdapter( - training_args=training_args, - noise_scheduler=noise_scheduler, - vae=vae, - ) - - from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame - - dataloader = build_parquet_wangame_train_dataloader( - training_args, - parquet_schema=pyarrow_schema_wangame, - ) - - return ModelComponents( - training_args=training_args, - bundle=bundle, - adapter=adapter, - dataloader=dataloader, - validator=validator, - start_step=0, - ) - diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 2ef6db2ff..45689d6f7 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -62,8 +62,8 @@ def run_distillation_from_config( get_rng_generators = getattr(runtime.method, "get_rng_generators", None) if not callable(get_rng_generators): - adapter = getattr(runtime.method, "adapter", None) - get_rng_generators = getattr(adapter, "get_rng_generators", None) + model = getattr(runtime.method, "model", None) + get_rng_generators = getattr(model, "get_rng_generators", None) if not callable(get_rng_generators): get_rng_generators = None From 7c1442c6b9cfd78172ea5ff1233a45bd0efc8068 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 2 Mar 2026 08:55:31 +0000 Subject: [PATCH 139/214] reorder and structure inherit hierarchy --- .../methods/distribution_matching/dmd2.py | 447 ++++++++-------- .../distillation/methods/fine_tuning/dfsft.py | 457 ++++++++-------- .../methods/fine_tuning/finetune.py | 379 ++++++------- fastvideo/distillation/models/wan.py | 353 +++++++------ fastvideo/distillation/models/wangame.py | 497 +++++++++--------- 5 files changed, 1094 insertions(+), 1039 deletions(-) diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 1de8cab24..5419298f0 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -166,6 +166,7 @@ def __init__( self._denoising_step_list: torch.Tensor | None = None self._init_optimizers_and_schedulers() + # DistillMethod override: build @classmethod def build( cls, @@ -183,6 +184,234 @@ def build( validator=validator, ) + # DistillMethod override: single_train_step + def single_train_step( + self, + batch: dict[str, Any], + iteration: int, + *, + current_vsa_sparsity: float = 0.0, + ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: + latents_source: Literal["data", "zeros"] = "data" + if self._rollout_mode == "simulate": + latents_source = "zeros" + + training_batch = self.model.prepare_batch( + batch, + current_vsa_sparsity=current_vsa_sparsity, + latents_source=latents_source, + ) + + update_student = self._should_update_student(iteration) + + generator_loss = torch.zeros( + (), + device=training_batch.latents.device, + dtype=training_batch.latents.dtype, + ) + student_ctx = None + if update_student: + generator_pred_x0 = self._student_rollout(training_batch, with_grad=True) + student_ctx = (training_batch.timesteps, training_batch.attn_metadata_vsa) + generator_loss = self._dmd_loss(generator_pred_x0, training_batch) + + fake_score_loss, critic_ctx, critic_outputs = self._critic_flow_matching_loss( + training_batch + ) + + total_loss = generator_loss + fake_score_loss + loss_map = { + "total_loss": total_loss, + "generator_loss": generator_loss, + "fake_score_loss": fake_score_loss, + } + + outputs: dict[str, Any] = dict(critic_outputs) + outputs["_fv_backward"] = { + "update_student": update_student, + "student_ctx": student_ctx, + "critic_ctx": critic_ctx, + } + metrics: dict[str, LogScalar] = {"update_student": float(update_student)} + return loss_map, outputs, metrics + + # DistillMethod override: backward + def backward( + self, + loss_map: dict[str, torch.Tensor], + outputs: dict[str, Any], + *, + grad_accum_rounds: int = 1, + ) -> None: + grad_accum_rounds = max(1, int(grad_accum_rounds)) + backward_ctx = outputs.get("_fv_backward") + if not isinstance(backward_ctx, dict): + super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + return + + update_student = bool(backward_ctx.get("update_student", False)) + if update_student: + student_ctx = backward_ctx.get("student_ctx") + if student_ctx is None: + raise RuntimeError("Missing student backward context") + self.model.backward( + loss_map["generator_loss"], + student_ctx, + grad_accum_rounds=grad_accum_rounds, + ) + + critic_ctx = backward_ctx.get("critic_ctx") + if critic_ctx is None: + raise RuntimeError("Missing critic backward context") + self.model.backward( + loss_map["fake_score_loss"], + critic_ctx, + grad_accum_rounds=grad_accum_rounds, + ) + + # DistillMethod override: get_optimizers + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + optimizers: list[torch.optim.Optimizer] = [] + optimizers.extend(self.critic.optimizers.values()) + if self._should_update_student(iteration): + optimizers.extend(self.student.optimizers.values()) + return optimizers + + # DistillMethod override: get_lr_schedulers + def get_lr_schedulers(self, iteration: int) -> list[Any]: + schedulers: list[Any] = [] + schedulers.extend(self.bundle.role("critic").lr_schedulers.values()) + if self._should_update_student(iteration): + schedulers.extend(self.bundle.role("student").lr_schedulers.values()) + return schedulers + + # DistillMethod override: optimizers_schedulers_step + def optimizers_schedulers_step(self, iteration: int) -> None: + if self._should_update_student(iteration): + for module in self.bundle.role("student").modules.values(): + self._clip_grad_norm(module) + for module in self.bundle.role("critic").modules.values(): + self._clip_grad_norm(module) + + super().optimizers_schedulers_step(iteration) + + # DistillTrainer hook: on_train_start + def on_train_start(self) -> None: + self.model.on_train_start() + + # DistillTrainer hook: log_validation + def log_validation(self, iteration: int) -> None: + validator = getattr(self, "validator", None) + if validator is None: + return + if not self._is_validation_enabled(): + return + + every_steps = self._parse_validation_every_steps() + if every_steps <= 0: + return + if iteration % every_steps != 0: + return + + dataset_file = self._parse_validation_dataset_file() + sampling_steps = self._parse_validation_sampling_steps() + + sampling_timesteps: list[int] | None = None + raw_timesteps = self.validation_config.get("sampling_timesteps", None) + if raw_timesteps is None: + raw_timesteps = self.method_config.get("dmd_denoising_steps", None) + if isinstance(raw_timesteps, list) and raw_timesteps: + sampling_timesteps = [int(s) for s in raw_timesteps] + + if not sampling_steps: + # Default to the few-step student rollout step count for DMD2. + if sampling_timesteps is None: + return + sampling_steps = [int(len(sampling_timesteps))] + + sampler_kind = self.validation_config.get("sampler_kind", "sde") + if sampler_kind is None: + sampler_kind = "sde" + if not isinstance(sampler_kind, str): + raise ValueError( + "training.validation.sampler_kind must be a string, got " + f"{type(sampler_kind).__name__}" + ) + sampler_kind = sampler_kind.strip().lower() + if sampler_kind not in {"ode", "sde"}: + raise ValueError( + "training.validation.sampler_kind must be one of {ode, sde}, got " + f"{sampler_kind!r}" + ) + sampler_kind = cast(Literal["ode", "sde"], sampler_kind) + ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) + if sampling_timesteps is not None and sampler_kind != "sde": + raise ValueError( + "method_config.validation.sampling_timesteps is only valid when " + "sampler_kind='sde'" + ) + + rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") + if not isinstance(rollout_mode_raw, str): + raise ValueError( + "training.validation.rollout_mode must be a string when set, got " + f"{type(rollout_mode_raw).__name__}" + ) + rollout_mode = rollout_mode_raw.strip().lower() + if rollout_mode not in {"parallel", "streaming"}: + raise ValueError( + "training.validation.rollout_mode must be one of {parallel, streaming}, " + f"got {rollout_mode_raw!r}" + ) + + guidance_scale = self._parse_validation_guidance_scale() + output_dir = self.validation_config.get("output_dir", None) + if output_dir is not None and not isinstance(output_dir, str): + raise ValueError( + "training.validation.output_dir must be a string when set, got " + f"{type(output_dir).__name__}" + ) + + num_actions = get_optional_int( + self.validation_config, + "num_frames", + where="training.validation.num_frames", + ) + if num_actions is not None and num_actions <= 0: + raise ValueError("training.validation.num_frames must be > 0 when set") + + request = ValidationRequest( + sample_handle=self.student, + dataset_file=dataset_file, + sampling_steps=sampling_steps, + sampler_kind=sampler_kind, + rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), + ode_solver=ode_solver, + sampling_timesteps=sampling_timesteps, + guidance_scale=guidance_scale, + num_frames=num_actions, + output_dir=output_dir, + ) + validator.log_validation(iteration, request=request) + + # Checkpoint hook: get_rng_generators + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + generators: dict[str, torch.Generator] = {} + + model = getattr(self, "model", None) + get_model_generators = getattr(model, "get_rng_generators", None) + if callable(get_model_generators): + generators.update(get_model_generators()) + + validator = getattr(self, "validator", None) + validation_gen = getattr(validator, "validation_random_generator", None) + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen + + return generators + def _parse_rollout_mode(self) -> Literal["simulate", "data_latent"]: raw = self.method_config.get("rollout_mode", None) if raw is None: @@ -331,9 +560,6 @@ def _init_optimizers_and_schedulers(self) -> None: scheduler_name=critic_sched, ) - def on_train_start(self) -> None: - self.model.on_train_start() - def _is_validation_enabled(self) -> bool: cfg = self.validation_config if not cfg: @@ -442,117 +668,6 @@ def _parse_validation_ode_solver( f"{raw!r}" ) - def log_validation(self, iteration: int) -> None: - validator = getattr(self, "validator", None) - if validator is None: - return - if not self._is_validation_enabled(): - return - - every_steps = self._parse_validation_every_steps() - if every_steps <= 0: - return - if iteration % every_steps != 0: - return - - dataset_file = self._parse_validation_dataset_file() - sampling_steps = self._parse_validation_sampling_steps() - - sampling_timesteps: list[int] | None = None - raw_timesteps = self.validation_config.get("sampling_timesteps", None) - if raw_timesteps is None: - raw_timesteps = self.method_config.get("dmd_denoising_steps", None) - if isinstance(raw_timesteps, list) and raw_timesteps: - sampling_timesteps = [int(s) for s in raw_timesteps] - - if not sampling_steps: - # Default to the few-step student rollout step count for DMD2. - if sampling_timesteps is None: - return - sampling_steps = [int(len(sampling_timesteps))] - - sampler_kind = self.validation_config.get("sampler_kind", "sde") - if sampler_kind is None: - sampler_kind = "sde" - if not isinstance(sampler_kind, str): - raise ValueError( - "training.validation.sampler_kind must be a string, got " - f"{type(sampler_kind).__name__}" - ) - sampler_kind = sampler_kind.strip().lower() - if sampler_kind not in {"ode", "sde"}: - raise ValueError( - "training.validation.sampler_kind must be one of {ode, sde}, got " - f"{sampler_kind!r}" - ) - sampler_kind = cast(Literal["ode", "sde"], sampler_kind) - ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) - if sampling_timesteps is not None and sampler_kind != "sde": - raise ValueError( - "method_config.validation.sampling_timesteps is only valid when " - "sampler_kind='sde'" - ) - - rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") - if not isinstance(rollout_mode_raw, str): - raise ValueError( - "training.validation.rollout_mode must be a string when set, got " - f"{type(rollout_mode_raw).__name__}" - ) - rollout_mode = rollout_mode_raw.strip().lower() - if rollout_mode not in {"parallel", "streaming"}: - raise ValueError( - "training.validation.rollout_mode must be one of {parallel, streaming}, " - f"got {rollout_mode_raw!r}" - ) - - guidance_scale = self._parse_validation_guidance_scale() - output_dir = self.validation_config.get("output_dir", None) - if output_dir is not None and not isinstance(output_dir, str): - raise ValueError( - "training.validation.output_dir must be a string when set, got " - f"{type(output_dir).__name__}" - ) - - num_actions = get_optional_int( - self.validation_config, - "num_frames", - where="training.validation.num_frames", - ) - if num_actions is not None and num_actions <= 0: - raise ValueError("training.validation.num_frames must be > 0 when set") - - request = ValidationRequest( - sample_handle=self.student, - dataset_file=dataset_file, - sampling_steps=sampling_steps, - sampler_kind=sampler_kind, - rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), - ode_solver=ode_solver, - sampling_timesteps=sampling_timesteps, - guidance_scale=guidance_scale, - num_frames=num_actions, - output_dir=output_dir, - ) - validator.log_validation(iteration, request=request) - - def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - - generators: dict[str, torch.Generator] = {} - - model = getattr(self, "model", None) - get_model_generators = getattr(model, "get_rng_generators", None) - if callable(get_model_generators): - generators.update(get_model_generators()) - - validator = getattr(self, "validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen - - return generators - def _should_update_student(self, iteration: int) -> bool: interval = get_optional_int( self.method_config, @@ -831,107 +946,3 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor (generator_pred_x0.float() - grad.float()).detach(), ) return loss - - def single_train_step( - self, - batch: dict[str, Any], - iteration: int, - *, - current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: - latents_source: Literal["data", "zeros"] = "data" - if self._rollout_mode == "simulate": - latents_source = "zeros" - - training_batch = self.model.prepare_batch( - batch, - current_vsa_sparsity=current_vsa_sparsity, - latents_source=latents_source, - ) - - update_student = self._should_update_student(iteration) - - generator_loss = torch.zeros( - (), - device=training_batch.latents.device, - dtype=training_batch.latents.dtype, - ) - student_ctx = None - if update_student: - generator_pred_x0 = self._student_rollout(training_batch, with_grad=True) - student_ctx = (training_batch.timesteps, training_batch.attn_metadata_vsa) - generator_loss = self._dmd_loss(generator_pred_x0, training_batch) - - fake_score_loss, critic_ctx, critic_outputs = self._critic_flow_matching_loss(training_batch) - - total_loss = generator_loss + fake_score_loss - loss_map = { - "total_loss": total_loss, - "generator_loss": generator_loss, - "fake_score_loss": fake_score_loss, - } - - outputs: dict[str, Any] = dict(critic_outputs) - outputs["_fv_backward"] = { - "update_student": update_student, - "student_ctx": student_ctx, - "critic_ctx": critic_ctx, - } - metrics: dict[str, LogScalar] = {"update_student": float(update_student)} - return loss_map, outputs, metrics - - def backward( - self, - loss_map: dict[str, torch.Tensor], - outputs: dict[str, Any], - *, - grad_accum_rounds: int = 1, - ) -> None: - grad_accum_rounds = max(1, int(grad_accum_rounds)) - backward_ctx = outputs.get("_fv_backward") - if not isinstance(backward_ctx, dict): - super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) - return - - update_student = bool(backward_ctx.get("update_student", False)) - if update_student: - student_ctx = backward_ctx.get("student_ctx") - if student_ctx is None: - raise RuntimeError("Missing student backward context") - self.model.backward( - loss_map["generator_loss"], - student_ctx, - grad_accum_rounds=grad_accum_rounds, - ) - - critic_ctx = backward_ctx.get("critic_ctx") - if critic_ctx is None: - raise RuntimeError("Missing critic backward context") - self.model.backward( - loss_map["fake_score_loss"], - critic_ctx, - grad_accum_rounds=grad_accum_rounds, - ) - - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: - optimizers: list[torch.optim.Optimizer] = [] - optimizers.extend(self.critic.optimizers.values()) - if self._should_update_student(iteration): - optimizers.extend(self.student.optimizers.values()) - return optimizers - - def get_lr_schedulers(self, iteration: int) -> list[Any]: - schedulers: list[Any] = [] - schedulers.extend(self.bundle.role("critic").lr_schedulers.values()) - if self._should_update_student(iteration): - schedulers.extend(self.bundle.role("student").lr_schedulers.values()) - return schedulers - - def optimizers_schedulers_step(self, iteration: int) -> None: - if self._should_update_student(iteration): - for module in self.bundle.role("student").modules.values(): - self._clip_grad_norm(module) - for module in self.bundle.role("critic").modules.values(): - self._clip_grad_norm(module) - - super().optimizers_schedulers_step(iteration) diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index 111d3ca70..833524c5a 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -126,6 +126,7 @@ def __init__( self._init_optimizers_and_schedulers() + # DistillMethod override: build @classmethod def build( cls, @@ -143,6 +144,239 @@ def build( validator=validator, ) + # DistillMethod override: single_train_step + def single_train_step( + self, + batch: dict[str, Any], + iteration: int, + *, + current_vsa_sparsity: float = 0.0, + ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: + del iteration + training_batch = self.model.prepare_batch( + batch, + current_vsa_sparsity=current_vsa_sparsity, + latents_source="data", + ) + + if training_batch.latents is None: + raise RuntimeError("model.prepare_batch() must set TrainingBatch.latents") + + clean_latents = training_batch.latents + if not torch.is_tensor(clean_latents): + raise TypeError("TrainingBatch.latents must be a torch.Tensor") + if clean_latents.ndim != 5: + raise ValueError( + "TrainingBatch.latents must be [B, T, C, H, W], got " + f"shape={tuple(clean_latents.shape)}" + ) + + batch_size, num_latents = int(clean_latents.shape[0]), int(clean_latents.shape[1]) + + transformer = self.student.require_module("transformer") + expected_chunk = getattr(transformer, "num_frame_per_block", None) + if expected_chunk is not None and int(expected_chunk) != int(self._chunk_size): + raise ValueError( + "DFSFT chunk_size must match transformer.num_frame_per_block for " + f"causal training (got {self._chunk_size}, expected {expected_chunk})." + ) + + timestep_indices = self._sample_t_inhom_indices( + batch_size=batch_size, + num_latents=num_latents, + device=clean_latents.device, + ) + sp_size = int(getattr(self.training_args, "sp_size", 1) or 1) + sp_group = getattr(self.model, "sp_group", None) + if sp_size > 1 and sp_group is not None and hasattr(sp_group, "broadcast"): + sp_group.broadcast(timestep_indices, src=0) + + scheduler = getattr(self.model, "noise_scheduler", None) + if scheduler is None: + raise ValueError("DFSFT requires model.noise_scheduler") + + schedule_timesteps = scheduler.timesteps.to( + device=clean_latents.device, dtype=torch.float32 + ) + schedule_sigmas = scheduler.sigmas.to( + device=clean_latents.device, dtype=clean_latents.dtype + ) + t_inhom = schedule_timesteps[timestep_indices] + + noise = getattr(training_batch, "noise", None) + if noise is None: + noise = torch.randn_like(clean_latents) + else: + if not torch.is_tensor(noise): + raise TypeError("TrainingBatch.noise must be a torch.Tensor when set") + noise = noise.permute(0, 2, 1, 3, 4).to(dtype=clean_latents.dtype) + + noisy_latents = self.model.add_noise( + clean_latents, + noise, + t_inhom.flatten(), + ) + + pred = self.model.predict_noise( + self.student, + noisy_latents, + t_inhom, + training_batch, + conditional=True, + attn_kind=self._attn_kind, + ) + + if bool(getattr(self.training_args, "precondition_outputs", False)): + sigmas = schedule_sigmas[timestep_indices] + sigmas = sigmas.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) + pred_x0 = noisy_latents - pred * sigmas + loss = F.mse_loss(pred_x0.float(), clean_latents.float()) + else: + target = noise - clean_latents + loss = F.mse_loss(pred.float(), target.float()) + + if self._attn_kind == "vsa": + attn_metadata = training_batch.attn_metadata_vsa + else: + attn_metadata = training_batch.attn_metadata + + loss_map = {"total_loss": loss, "dfsft_loss": loss} + outputs: dict[str, Any] = {"_fv_backward": (training_batch.timesteps, attn_metadata)} + metrics: dict[str, LogScalar] = {} + return loss_map, outputs, metrics + + # DistillMethod override: backward + def backward( + self, + loss_map: dict[str, torch.Tensor], + outputs: dict[str, Any], + *, + grad_accum_rounds: int = 1, + ) -> None: + grad_accum_rounds = max(1, int(grad_accum_rounds)) + ctx = outputs.get("_fv_backward") + if ctx is None: + super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + return + self.model.backward( + loss_map["total_loss"], + ctx, + grad_accum_rounds=grad_accum_rounds, + ) + + # DistillMethod override: get_optimizers + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + del iteration + return list(self.student.optimizers.values()) + + # DistillMethod override: get_lr_schedulers + def get_lr_schedulers(self, iteration: int) -> list[Any]: + del iteration + return list(self.student.lr_schedulers.values()) + + # DistillMethod override: optimizers_schedulers_step + def optimizers_schedulers_step(self, iteration: int) -> None: + for module in self.student.modules.values(): + self._clip_grad_norm(module) + super().optimizers_schedulers_step(iteration) + + # DistillTrainer hook: on_train_start + def on_train_start(self) -> None: + self.model.on_train_start() + + # DistillTrainer hook: log_validation + def log_validation(self, iteration: int) -> None: + validator = getattr(self, "validator", None) + if validator is None: + return + if not self._is_validation_enabled(): + return + + every_steps = self._parse_validation_every_steps() + if every_steps <= 0: + return + if iteration % every_steps != 0: + return + + dataset_file = self._parse_validation_dataset_file() + sampling_steps = self._parse_validation_sampling_steps() + guidance_scale = self._parse_validation_guidance_scale() + + sampler_kind_raw = self.validation_config.get("sampler_kind", "ode") + if not isinstance(sampler_kind_raw, str): + raise ValueError( + "training.validation.sampler_kind must be a string when set, got " + f"{type(sampler_kind_raw).__name__}" + ) + sampler_kind = sampler_kind_raw.strip().lower() + if sampler_kind not in {"ode", "sde"}: + raise ValueError( + "training.validation.sampler_kind must be one of {ode, sde}, got " + f"{sampler_kind_raw!r}" + ) + sampler_kind = cast(Literal["ode", "sde"], sampler_kind) + ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) + + rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") + if not isinstance(rollout_mode_raw, str): + raise ValueError( + "training.validation.rollout_mode must be a string when set, got " + f"{type(rollout_mode_raw).__name__}" + ) + rollout_mode = rollout_mode_raw.strip().lower() + if rollout_mode not in {"parallel", "streaming"}: + raise ValueError( + "training.validation.rollout_mode must be one of {parallel, streaming}, " + f"got {rollout_mode_raw!r}" + ) + + output_dir = self.validation_config.get("output_dir", None) + if output_dir is not None and not isinstance(output_dir, str): + raise ValueError( + "training.validation.output_dir must be a string when set, got " + f"{type(output_dir).__name__}" + ) + + num_actions = get_optional_int( + self.validation_config, + "num_frames", + where="training.validation.num_frames", + ) + if num_actions is not None and num_actions <= 0: + raise ValueError("training.validation.num_frames must be > 0 when set") + + request = ValidationRequest( + sample_handle=self.student, + dataset_file=dataset_file, + sampling_steps=sampling_steps, + sampler_kind=sampler_kind, + rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), + ode_solver=ode_solver, + sampling_timesteps=None, + guidance_scale=guidance_scale, + num_frames=num_actions, + output_dir=output_dir, + ) + validator.log_validation(iteration, request=request) + + # Checkpoint hook: get_rng_generators + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + generators: dict[str, torch.Generator] = {} + + model = getattr(self, "model", None) + get_model_generators = getattr(model, "get_rng_generators", None) + if callable(get_model_generators): + generators.update(get_model_generators()) + + validator = getattr(self, "validator", None) + validation_gen = getattr(validator, "validation_random_generator", None) + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen + + return generators + def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: if raw in (None, ""): return "dense" @@ -278,9 +512,6 @@ def _init_optimizers_and_schedulers(self) -> None: scheduler_name=student_sched, ) - def on_train_start(self) -> None: - self.model.on_train_start() - def _is_validation_enabled(self) -> bool: cfg = self.validation_config if not cfg: @@ -385,95 +616,6 @@ def _parse_validation_ode_solver( f"{raw!r}" ) - def log_validation(self, iteration: int) -> None: - validator = getattr(self, "validator", None) - if validator is None: - return - if not self._is_validation_enabled(): - return - - every_steps = self._parse_validation_every_steps() - if every_steps <= 0: - return - if iteration % every_steps != 0: - return - - dataset_file = self._parse_validation_dataset_file() - sampling_steps = self._parse_validation_sampling_steps() - guidance_scale = self._parse_validation_guidance_scale() - - sampler_kind_raw = self.validation_config.get("sampler_kind", "ode") - if not isinstance(sampler_kind_raw, str): - raise ValueError( - "training.validation.sampler_kind must be a string when set, got " - f"{type(sampler_kind_raw).__name__}" - ) - sampler_kind = sampler_kind_raw.strip().lower() - if sampler_kind not in {"ode", "sde"}: - raise ValueError( - "training.validation.sampler_kind must be one of {ode, sde}, got " - f"{sampler_kind_raw!r}" - ) - sampler_kind = cast(Literal["ode", "sde"], sampler_kind) - ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) - - rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") - if not isinstance(rollout_mode_raw, str): - raise ValueError( - "training.validation.rollout_mode must be a string when set, got " - f"{type(rollout_mode_raw).__name__}" - ) - rollout_mode = rollout_mode_raw.strip().lower() - if rollout_mode not in {"parallel", "streaming"}: - raise ValueError( - "training.validation.rollout_mode must be one of {parallel, streaming}, " - f"got {rollout_mode_raw!r}" - ) - - output_dir = self.validation_config.get("output_dir", None) - if output_dir is not None and not isinstance(output_dir, str): - raise ValueError( - "training.validation.output_dir must be a string when set, got " - f"{type(output_dir).__name__}" - ) - - num_actions = get_optional_int( - self.validation_config, - "num_frames", - where="training.validation.num_frames", - ) - if num_actions is not None and num_actions <= 0: - raise ValueError("training.validation.num_frames must be > 0 when set") - - request = ValidationRequest( - sample_handle=self.student, - dataset_file=dataset_file, - sampling_steps=sampling_steps, - sampler_kind=sampler_kind, - rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), - ode_solver=ode_solver, - sampling_timesteps=None, - guidance_scale=guidance_scale, - num_frames=num_actions, - output_dir=output_dir, - ) - validator.log_validation(iteration, request=request) - - def get_rng_generators(self) -> dict[str, torch.Generator]: - generators: dict[str, torch.Generator] = {} - - model = getattr(self, "model", None) - get_model_generators = getattr(model, "get_rng_generators", None) - if callable(get_model_generators): - generators.update(get_model_generators()) - - validator = getattr(self, "validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen - - return generators - def _sample_t_inhom_indices(self, *, batch_size: int, num_latents: int, device: torch.device) -> torch.Tensor: chunk_size = self._chunk_size num_chunks = (num_latents + chunk_size - 1) // chunk_size @@ -488,132 +630,6 @@ def _sample_t_inhom_indices(self, *, batch_size: int, num_latents: int, device: expanded = chunk_indices.repeat_interleave(chunk_size, dim=1) return expanded[:, :num_latents] - def single_train_step( - self, - batch: dict[str, Any], - iteration: int, - *, - current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: - del iteration - training_batch = self.model.prepare_batch( - batch, - current_vsa_sparsity=current_vsa_sparsity, - latents_source="data", - ) - - if training_batch.latents is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.latents") - - clean_latents = training_batch.latents - if not torch.is_tensor(clean_latents): - raise TypeError("TrainingBatch.latents must be a torch.Tensor") - if clean_latents.ndim != 5: - raise ValueError( - "TrainingBatch.latents must be [B, T, C, H, W], got " - f"shape={tuple(clean_latents.shape)}" - ) - - batch_size, num_latents = int(clean_latents.shape[0]), int(clean_latents.shape[1]) - - transformer = self.student.require_module("transformer") - expected_chunk = getattr(transformer, "num_frame_per_block", None) - if expected_chunk is not None and int(expected_chunk) != int(self._chunk_size): - raise ValueError( - "DFSFT chunk_size must match transformer.num_frame_per_block for " - f"causal training (got {self._chunk_size}, expected {expected_chunk})." - ) - - timestep_indices = self._sample_t_inhom_indices( - batch_size=batch_size, - num_latents=num_latents, - device=clean_latents.device, - ) - sp_size = int(getattr(self.training_args, "sp_size", 1) or 1) - sp_group = getattr(self.model, "sp_group", None) - if sp_size > 1 and sp_group is not None and hasattr(sp_group, "broadcast"): - sp_group.broadcast(timestep_indices, src=0) - - scheduler = getattr(self.model, "noise_scheduler", None) - if scheduler is None: - raise ValueError("DFSFT requires model.noise_scheduler") - - schedule_timesteps = scheduler.timesteps.to( - device=clean_latents.device, dtype=torch.float32 - ) - schedule_sigmas = scheduler.sigmas.to( - device=clean_latents.device, dtype=clean_latents.dtype - ) - t_inhom = schedule_timesteps[timestep_indices] - - noise = getattr(training_batch, "noise", None) - if noise is None: - noise = torch.randn_like(clean_latents) - else: - if not torch.is_tensor(noise): - raise TypeError("TrainingBatch.noise must be a torch.Tensor when set") - noise = noise.permute(0, 2, 1, 3, 4).to(dtype=clean_latents.dtype) - - noisy_latents = self.model.add_noise( - clean_latents, - noise, - t_inhom.flatten(), - ) - - pred = self.model.predict_noise( - self.student, - noisy_latents, - t_inhom, - training_batch, - conditional=True, - attn_kind=self._attn_kind, - ) - - if bool(getattr(self.training_args, "precondition_outputs", False)): - sigmas = schedule_sigmas[timestep_indices] - sigmas = sigmas.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) - pred_x0 = noisy_latents - pred * sigmas - loss = F.mse_loss(pred_x0.float(), clean_latents.float()) - else: - target = noise - clean_latents - loss = F.mse_loss(pred.float(), target.float()) - - if self._attn_kind == "vsa": - attn_metadata = training_batch.attn_metadata_vsa - else: - attn_metadata = training_batch.attn_metadata - - loss_map = {"total_loss": loss, "dfsft_loss": loss} - outputs: dict[str, Any] = {"_fv_backward": (training_batch.timesteps, attn_metadata)} - metrics: dict[str, LogScalar] = {} - return loss_map, outputs, metrics - - def backward( - self, - loss_map: dict[str, torch.Tensor], - outputs: dict[str, Any], - *, - grad_accum_rounds: int = 1, - ) -> None: - grad_accum_rounds = max(1, int(grad_accum_rounds)) - ctx = outputs.get("_fv_backward") - if ctx is None: - super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) - return - self.model.backward( - loss_map["total_loss"], - ctx, - grad_accum_rounds=grad_accum_rounds, - ) - - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: - del iteration - return list(self.student.optimizers.values()) - - def get_lr_schedulers(self, iteration: int) -> list[Any]: - del iteration - return list(self.student.lr_schedulers.values()) - def _clip_grad_norm(self, module: torch.nn.Module) -> float: max_grad_norm_raw = getattr(self.training_args, "max_grad_norm", None) if max_grad_norm_raw is None: @@ -633,8 +649,3 @@ def _clip_grad_norm(self, module: torch.nn.Module) -> float: foreach=None, ) return float(grad_norm.item()) if grad_norm is not None else 0.0 - - def optimizers_schedulers_step(self, iteration: int) -> None: - for module in self.student.modules.values(): - self._clip_grad_norm(module) - super().optimizers_schedulers_step(iteration) diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 849a0f0da..897fff7b4 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -115,6 +115,7 @@ def __init__( self._init_optimizers_and_schedulers() + # DistillMethod override: build @classmethod def build( cls, @@ -132,6 +133,198 @@ def build( validator=validator, ) + # DistillMethod override: single_train_step + def single_train_step( + self, + batch: dict[str, Any], + iteration: int, + *, + current_vsa_sparsity: float = 0.0, + ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: + del iteration + training_batch = self.model.prepare_batch( + batch, + current_vsa_sparsity=current_vsa_sparsity, + latents_source="data", + ) + + if training_batch.latents is None: + raise RuntimeError("model.prepare_batch() must set TrainingBatch.latents") + if training_batch.noisy_model_input is None: + raise RuntimeError( + "model.prepare_batch() must set TrainingBatch.noisy_model_input" + ) + if training_batch.noise is None: + raise RuntimeError("model.prepare_batch() must set TrainingBatch.noise") + if training_batch.sigmas is None: + raise RuntimeError("model.prepare_batch() must set TrainingBatch.sigmas") + if training_batch.timesteps is None: + raise RuntimeError("model.prepare_batch() must set TrainingBatch.timesteps") + + clean_latents = training_batch.latents + noisy_latents = training_batch.noisy_model_input.permute(0, 2, 1, 3, 4) + noise = training_batch.noise.permute(0, 2, 1, 3, 4) + sigmas = training_batch.sigmas + timesteps = training_batch.timesteps + + pred = self.model.predict_noise( + self.student, + noisy_latents, + timesteps, + training_batch, + conditional=True, + attn_kind=self._attn_kind, + ) + + if bool(getattr(self.training_args, "precondition_outputs", False)): + pred_x0 = noisy_latents - pred * sigmas + loss = F.mse_loss(pred_x0.float(), clean_latents.float()) + else: + target = noise - clean_latents + loss = F.mse_loss(pred.float(), target.float()) + + if self._attn_kind == "vsa": + attn_metadata = training_batch.attn_metadata_vsa + else: + attn_metadata = training_batch.attn_metadata + + loss_map = {"total_loss": loss, "finetune_loss": loss} + outputs: dict[str, Any] = {"_fv_backward": (training_batch.timesteps, attn_metadata)} + metrics: dict[str, LogScalar] = {} + return loss_map, outputs, metrics + + # DistillMethod override: backward + def backward( + self, + loss_map: dict[str, torch.Tensor], + outputs: dict[str, Any], + *, + grad_accum_rounds: int = 1, + ) -> None: + grad_accum_rounds = max(1, int(grad_accum_rounds)) + ctx = outputs.get("_fv_backward") + if ctx is None: + super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + return + self.model.backward( + loss_map["total_loss"], + ctx, + grad_accum_rounds=grad_accum_rounds, + ) + + # DistillMethod override: get_optimizers + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + del iteration + return list(self.student.optimizers.values()) + + # DistillMethod override: get_lr_schedulers + def get_lr_schedulers(self, iteration: int) -> list[Any]: + del iteration + return list(self.student.lr_schedulers.values()) + + # DistillMethod override: optimizers_schedulers_step + def optimizers_schedulers_step(self, iteration: int) -> None: + for module in self.student.modules.values(): + self._clip_grad_norm(module) + super().optimizers_schedulers_step(iteration) + + # DistillTrainer hook: on_train_start + def on_train_start(self) -> None: + self.model.on_train_start() + + # DistillTrainer hook: log_validation + def log_validation(self, iteration: int) -> None: + validator = getattr(self, "validator", None) + if validator is None: + return + if not self._is_validation_enabled(): + return + + every_steps = self._parse_validation_every_steps() + if every_steps <= 0: + return + if iteration % every_steps != 0: + return + + dataset_file = self._parse_validation_dataset_file() + sampling_steps = self._parse_validation_sampling_steps() + guidance_scale = self._parse_validation_guidance_scale() + + sampler_kind_raw = self.validation_config.get("sampler_kind", "ode") + if not isinstance(sampler_kind_raw, str): + raise ValueError( + "training.validation.sampler_kind must be a string when set, got " + f"{type(sampler_kind_raw).__name__}" + ) + sampler_kind = sampler_kind_raw.strip().lower() + if sampler_kind not in {"ode", "sde"}: + raise ValueError( + "training.validation.sampler_kind must be one of {ode, sde}, got " + f"{sampler_kind_raw!r}" + ) + sampler_kind = cast(Literal["ode", "sde"], sampler_kind) + ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) + + rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") + if not isinstance(rollout_mode_raw, str): + raise ValueError( + "training.validation.rollout_mode must be a string when set, got " + f"{type(rollout_mode_raw).__name__}" + ) + rollout_mode = rollout_mode_raw.strip().lower() + if rollout_mode not in {"parallel", "streaming"}: + raise ValueError( + "training.validation.rollout_mode must be one of {parallel, streaming}, " + f"got {rollout_mode_raw!r}" + ) + + output_dir = self.validation_config.get("output_dir", None) + if output_dir is not None and not isinstance(output_dir, str): + raise ValueError( + "training.validation.output_dir must be a string when set, got " + f"{type(output_dir).__name__}" + ) + + num_actions = get_optional_int( + self.validation_config, + "num_frames", + where="training.validation.num_frames", + ) + if num_actions is not None and num_actions <= 0: + raise ValueError("training.validation.num_frames must be > 0 when set") + + request = ValidationRequest( + sample_handle=self.student, + dataset_file=dataset_file, + sampling_steps=sampling_steps, + sampler_kind=sampler_kind, + rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), + ode_solver=ode_solver, + sampling_timesteps=None, + guidance_scale=guidance_scale, + num_frames=num_actions, + output_dir=output_dir, + ) + validator.log_validation(iteration, request=request) + + # Checkpoint hook: get_rng_generators + def get_rng_generators(self) -> dict[str, torch.Generator]: + """Return RNG generators that should be checkpointed for exact resume.""" + + generators: dict[str, torch.Generator] = {} + + model = getattr(self, "model", None) + get_model_generators = getattr(model, "get_rng_generators", None) + if callable(get_model_generators): + generators.update(get_model_generators()) + + validator = getattr(self, "validator", None) + validation_gen = getattr(validator, "validation_random_generator", None) + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen + + return generators + def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: if raw in (None, ""): return "dense" @@ -201,9 +394,6 @@ def _init_optimizers_and_schedulers(self) -> None: scheduler_name=student_sched, ) - def on_train_start(self) -> None: - self.model.on_train_start() - def _is_validation_enabled(self) -> bool: cfg = self.validation_config if not cfg: @@ -314,97 +504,6 @@ def _parse_validation_ode_solver( f"{raw!r}" ) - def log_validation(self, iteration: int) -> None: - validator = getattr(self, "validator", None) - if validator is None: - return - if not self._is_validation_enabled(): - return - - every_steps = self._parse_validation_every_steps() - if every_steps <= 0: - return - if iteration % every_steps != 0: - return - - dataset_file = self._parse_validation_dataset_file() - sampling_steps = self._parse_validation_sampling_steps() - guidance_scale = self._parse_validation_guidance_scale() - - sampler_kind_raw = self.validation_config.get("sampler_kind", "ode") - if not isinstance(sampler_kind_raw, str): - raise ValueError( - "training.validation.sampler_kind must be a string when set, got " - f"{type(sampler_kind_raw).__name__}" - ) - sampler_kind = sampler_kind_raw.strip().lower() - if sampler_kind not in {"ode", "sde"}: - raise ValueError( - "training.validation.sampler_kind must be one of {ode, sde}, got " - f"{sampler_kind_raw!r}" - ) - sampler_kind = cast(Literal["ode", "sde"], sampler_kind) - ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) - - rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") - if not isinstance(rollout_mode_raw, str): - raise ValueError( - "training.validation.rollout_mode must be a string when set, got " - f"{type(rollout_mode_raw).__name__}" - ) - rollout_mode = rollout_mode_raw.strip().lower() - if rollout_mode not in {"parallel", "streaming"}: - raise ValueError( - "training.validation.rollout_mode must be one of {parallel, streaming}, " - f"got {rollout_mode_raw!r}" - ) - - output_dir = self.validation_config.get("output_dir", None) - if output_dir is not None and not isinstance(output_dir, str): - raise ValueError( - "training.validation.output_dir must be a string when set, got " - f"{type(output_dir).__name__}" - ) - - num_actions = get_optional_int( - self.validation_config, - "num_frames", - where="training.validation.num_frames", - ) - if num_actions is not None and num_actions <= 0: - raise ValueError("training.validation.num_frames must be > 0 when set") - - request = ValidationRequest( - sample_handle=self.student, - dataset_file=dataset_file, - sampling_steps=sampling_steps, - sampler_kind=sampler_kind, - rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), - ode_solver=ode_solver, - sampling_timesteps=None, - guidance_scale=guidance_scale, - num_frames=num_actions, - output_dir=output_dir, - ) - validator.log_validation(iteration, request=request) - - def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - - generators: dict[str, torch.Generator] = {} - - model = getattr(self, "model", None) - get_model_generators = getattr(model, "get_rng_generators", None) - if callable(get_model_generators): - generators.update(get_model_generators()) - - validator = getattr(self, "validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen - - return generators - def _clip_grad_norm(self, module: torch.nn.Module) -> float: max_grad_norm_raw = getattr(self.training_args, "max_grad_norm", None) if max_grad_norm_raw is None: @@ -424,95 +523,3 @@ def _clip_grad_norm(self, module: torch.nn.Module) -> float: foreach=None, ) return float(grad_norm.item()) if grad_norm is not None else 0.0 - - def single_train_step( - self, - batch: dict[str, Any], - iteration: int, - *, - current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: - del iteration - training_batch = self.model.prepare_batch( - batch, - current_vsa_sparsity=current_vsa_sparsity, - latents_source="data", - ) - - if training_batch.latents is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.latents") - if training_batch.noisy_model_input is None: - raise RuntimeError( - "model.prepare_batch() must set TrainingBatch.noisy_model_input" - ) - if training_batch.noise is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.noise") - if training_batch.sigmas is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.sigmas") - if training_batch.timesteps is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.timesteps") - - clean_latents = training_batch.latents - noisy_latents = training_batch.noisy_model_input.permute(0, 2, 1, 3, 4) - noise = training_batch.noise.permute(0, 2, 1, 3, 4) - sigmas = training_batch.sigmas - timesteps = training_batch.timesteps - - pred = self.model.predict_noise( - self.student, - noisy_latents, - timesteps, - training_batch, - conditional=True, - attn_kind=self._attn_kind, - ) - - if bool(getattr(self.training_args, "precondition_outputs", False)): - pred_x0 = noisy_latents - pred * sigmas - loss = F.mse_loss(pred_x0.float(), clean_latents.float()) - else: - target = noise - clean_latents - loss = F.mse_loss(pred.float(), target.float()) - - if self._attn_kind == "vsa": - attn_metadata = training_batch.attn_metadata_vsa - else: - attn_metadata = training_batch.attn_metadata - - loss_map = {"total_loss": loss, "finetune_loss": loss} - outputs: dict[str, Any] = { - "_fv_backward": (training_batch.timesteps, attn_metadata) - } - metrics: dict[str, LogScalar] = {} - return loss_map, outputs, metrics - - def backward( - self, - loss_map: dict[str, torch.Tensor], - outputs: dict[str, Any], - *, - grad_accum_rounds: int = 1, - ) -> None: - grad_accum_rounds = max(1, int(grad_accum_rounds)) - ctx = outputs.get("_fv_backward") - if ctx is None: - super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) - return - self.model.backward( - loss_map["total_loss"], - ctx, - grad_accum_rounds=grad_accum_rounds, - ) - - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: - del iteration - return list(self.student.optimizers.values()) - - def get_lr_schedulers(self, iteration: int) -> list[Any]: - del iteration - return list(self.student.lr_schedulers.values()) - - def optimizers_schedulers_step(self, iteration: int) -> None: - for module in self.student.modules.values(): - self._clip_grad_norm(module) - super().optimizers_schedulers_step(iteration) diff --git a/fastvideo/distillation/models/wan.py b/fastvideo/distillation/models/wan.py index 1ec4fd099..fb106a211 100644 --- a/fastvideo/distillation/models/wan.py +++ b/fastvideo/distillation/models/wan.py @@ -184,30 +184,17 @@ def __init__(self, *, cfg: DistillRunConfig) -> None: ) self.start_step = 0 - def _get_training_dtype(self) -> torch.dtype: - return torch.bfloat16 - - def _init_timestep_mechanics(self) -> None: - self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) - self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) - self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) - self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) - - boundary_ratio = getattr(self.training_args, "boundary_ratio", None) - self.boundary_timestep: float | None = ( - float(boundary_ratio) * float(self.num_train_timestep) - if boundary_ratio is not None - else None - ) - + # ModelBase override: num_train_timesteps @property def num_train_timesteps(self) -> int: return int(self.num_train_timestep) + # ModelBase override: shift_and_clamp_timestep def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) return timestep.clamp(self.min_timestep, self.max_timestep) + # ModelBase override: on_train_start def on_train_start(self) -> None: seed = self.training_args.seed if seed is None: @@ -226,6 +213,7 @@ def on_train_start(self) -> None: self.ensure_negative_conditioning() + # ModelBase override: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: """Return RNG generators that should be checkpointed for exact resume.""" @@ -237,6 +225,185 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: return generators + # ModelBase override: prepare_batch + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", + ) -> TrainingBatch: + """Convert a dataloader batch into forward primitives for methods.""" + + self.ensure_negative_conditioning() + + dtype = self._get_training_dtype() + device = self.device + + training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + encoder_hidden_states = raw_batch["text_embedding"] + encoder_attention_mask = raw_batch["text_attention_mask"] + infos = raw_batch.get("info_list") + + if latents_source == "zeros": + batch_size = encoder_hidden_states.shape[0] + vae_config = self.training_args.pipeline_config.vae_config.arch_config + num_channels = vae_config.z_dim + spatial_compression_ratio = vae_config.spatial_compression_ratio + latent_height = self.training_args.num_height // spatial_compression_ratio + latent_width = self.training_args.num_width // spatial_compression_ratio + latents = torch.zeros( + batch_size, + num_channels, + self.training_args.num_latent_t, + latent_height, + latent_width, + device=device, + dtype=dtype, + ) + elif latents_source == "data": + if "vae_latent" not in raw_batch: + raise ValueError("vae_latent not found in batch and latents_source='data'") + latents = raw_batch["vae_latent"] + latents = latents[:, :, : self.training_args.num_latent_t] + latents = latents.to(device, dtype=dtype) + else: + raise ValueError(f"Unknown latents_source: {latents_source!r}") + + training_batch.latents = latents + training_batch.encoder_hidden_states = encoder_hidden_states.to(device, dtype=dtype) + training_batch.encoder_attention_mask = encoder_attention_mask.to(device, dtype=dtype) + training_batch.infos = infos + + training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch = self._prepare_dit_inputs(training_batch) + training_batch = self._build_attention_metadata(training_batch) + + training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) + if training_batch.attn_metadata is not None: + training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] + + return training_batch + + # ModelBase override: add_noise + def add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + b, t = clean_latents.shape[:2] + noisy = self.noise_scheduler.add_noise( + clean_latents.flatten(0, 1), + noise.flatten(0, 1), + timestep, + ).unflatten(0, (b, t)) + return noisy + + # ModelBase override: predict_x0 + def predict_x0( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + if conditional: + text_dict = batch.conditional_dict + if text_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + else: + text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + # ModelBase override: predict_noise + def predict_noise( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + if conditional: + text_dict = batch.conditional_dict + if text_dict is None: + raise RuntimeError("Missing conditional_dict in TrainingBatch") + else: + text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + return pred_noise + + # ModelBase override: backward + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + timesteps, attn_metadata = ctx + with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): + (loss / max(1, int(grad_accum_rounds))).backward() + + # --- Wan-specific helpers below --- + + def _get_training_dtype(self) -> torch.dtype: + return torch.bfloat16 + + def _init_timestep_mechanics(self) -> None: + self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) + self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) + self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) + self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) + + boundary_ratio = getattr(self.training_args, "boundary_ratio", None) + self.boundary_timestep: float | None = ( + float(boundary_ratio) * float(self.num_train_timestep) + if boundary_ratio is not None + else None + ) + def ensure_negative_conditioning(self) -> None: if self.negative_prompt_embeds is not None: return @@ -439,77 +606,6 @@ def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) return training_batch - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", - ) -> TrainingBatch: - self.ensure_negative_conditioning() - - dtype = self._get_training_dtype() - device = self.device - - training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) - encoder_hidden_states = raw_batch["text_embedding"] - encoder_attention_mask = raw_batch["text_attention_mask"] - infos = raw_batch.get("info_list") - - if latents_source == "zeros": - batch_size = encoder_hidden_states.shape[0] - vae_config = self.training_args.pipeline_config.vae_config.arch_config - num_channels = vae_config.z_dim - spatial_compression_ratio = vae_config.spatial_compression_ratio - latent_height = self.training_args.num_height // spatial_compression_ratio - latent_width = self.training_args.num_width // spatial_compression_ratio - latents = torch.zeros( - batch_size, - num_channels, - self.training_args.num_latent_t, - latent_height, - latent_width, - device=device, - dtype=dtype, - ) - elif latents_source == "data": - if "vae_latent" not in raw_batch: - raise ValueError("vae_latent not found in batch and latents_source='data'") - latents = raw_batch["vae_latent"] - latents = latents[:, :, : self.training_args.num_latent_t] - latents = latents.to(device, dtype=dtype) - else: - raise ValueError(f"Unknown latents_source: {latents_source!r}") - - training_batch.latents = latents - training_batch.encoder_hidden_states = encoder_hidden_states.to(device, dtype=dtype) - training_batch.encoder_attention_mask = encoder_attention_mask.to(device, dtype=dtype) - training_batch.infos = infos - - training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) - training_batch = self._prepare_dit_inputs(training_batch) - training_batch = self._build_attention_metadata(training_batch) - - training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) - if training_batch.attn_metadata is not None: - training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] - - return training_batch - - def add_noise( - self, - clean_latents: torch.Tensor, - noise: torch.Tensor, - timestep: torch.Tensor, - ) -> torch.Tensor: - b, t = clean_latents.shape[:2] - noisy = self.noise_scheduler.add_noise( - clean_latents.flatten(0, 1), - noise.flatten(0, 1), - timestep, - ).unflatten(0, (b, t)) - return noisy - def _build_distill_input_kwargs( self, noise_input: torch.Tensor, @@ -630,86 +726,3 @@ def _get_uncond_text_dict( "cfg_uncond.text must be one of {negative_prompt, keep, zero, drop}, got " f"{text_policy_raw!r}" ) - - def predict_x0( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - if conditional: - text_dict = batch.conditional_dict - if text_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") - else: - text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) - return pred_x0 - - def predict_noise( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - if conditional: - text_dict = batch.conditional_dict - if text_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") - else: - text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - return pred_noise - - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: - timesteps, attn_metadata = ctx - with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): - (loss / max(1, int(grad_accum_rounds))).backward() diff --git a/fastvideo/distillation/models/wangame.py b/fastvideo/distillation/models/wangame.py index 98b62b13b..5d155c075 100644 --- a/fastvideo/distillation/models/wangame.py +++ b/fastvideo/distillation/models/wangame.py @@ -197,30 +197,17 @@ def __init__( ) self.start_step = 0 - def _get_training_dtype(self) -> torch.dtype: - return torch.bfloat16 - - def _init_timestep_mechanics(self) -> None: - self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) - self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) - self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) - self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) - - boundary_ratio = getattr(self.training_args, "boundary_ratio", None) - self.boundary_timestep: float | None = ( - float(boundary_ratio) * float(self.num_train_timestep) - if boundary_ratio is not None - else None - ) - + # ModelBase override: num_train_timesteps @property def num_train_timesteps(self) -> int: return int(self.num_train_timestep) + # ModelBase override: shift_and_clamp_timestep def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) return timestep.clamp(self.min_timestep, self.max_timestep) + # ModelBase override: on_train_start def on_train_start(self) -> None: seed = getattr(self.training_args, "seed", None) if seed is None: @@ -237,6 +224,7 @@ def on_train_start(self) -> None: self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) + # ModelBase override: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: """Return RNG generators that should be checkpointed for exact resume.""" @@ -247,6 +235,257 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: generators["noise_cuda"] = self.noise_gen_cuda return generators + # ModelBase override: prepare_batch + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", + ) -> TrainingBatch: + """Convert a dataloader batch into forward primitives for methods.""" + + dtype = self._get_training_dtype() + device = self.device + + training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + infos = raw_batch.get("info_list") + + if latents_source == "zeros": + clip_feature = raw_batch["clip_feature"] + batch_size = int(clip_feature.shape[0]) + vae_config = self.training_args.pipeline_config.vae_config.arch_config + num_channels = int(vae_config.z_dim) + spatial_compression_ratio = int(vae_config.spatial_compression_ratio) + latent_height = int(self.training_args.num_height) // spatial_compression_ratio + latent_width = int(self.training_args.num_width) // spatial_compression_ratio + latents = torch.zeros( + batch_size, + num_channels, + int(self.training_args.num_latent_t), + latent_height, + latent_width, + device=device, + dtype=dtype, + ) + elif latents_source == "data": + if "vae_latent" not in raw_batch: + raise ValueError("vae_latent not found in batch and latents_source='data'") + latents = raw_batch["vae_latent"] + latents = latents[:, :, : self.training_args.num_latent_t] + latents = latents.to(device, dtype=dtype) + else: + raise ValueError(f"Unknown latents_source: {latents_source!r}") + + if "clip_feature" not in raw_batch: + raise ValueError("clip_feature must be present for WanGame") + image_embeds = raw_batch["clip_feature"].to(device, dtype=dtype) + + if "first_frame_latent" not in raw_batch: + raise ValueError("first_frame_latent must be present for WanGame") + image_latents = raw_batch["first_frame_latent"] + image_latents = image_latents[:, :, : self.training_args.num_latent_t] + image_latents = image_latents.to(device, dtype=dtype) + + pil_image = raw_batch.get("pil_image") + if isinstance(pil_image, torch.Tensor): + training_batch.preprocessed_image = pil_image.to(device=device) + else: + training_batch.preprocessed_image = pil_image + + keyboard_cond = raw_batch.get("keyboard_cond") + if isinstance(keyboard_cond, torch.Tensor) and keyboard_cond.numel() > 0: + training_batch.keyboard_cond = keyboard_cond.to(device, dtype=dtype) + else: + training_batch.keyboard_cond = None + + mouse_cond = raw_batch.get("mouse_cond") + if isinstance(mouse_cond, torch.Tensor) and mouse_cond.numel() > 0: + training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) + else: + training_batch.mouse_cond = None + + temporal_compression_ratio = ( + self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + expected_num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 + if training_batch.keyboard_cond is not None and int( + training_batch.keyboard_cond.shape[1] + ) != int(expected_num_frames): + raise ValueError( + "keyboard_cond temporal dim mismatch: " + f"got {int(training_batch.keyboard_cond.shape[1])}, expected {int(expected_num_frames)}" + ) + if training_batch.mouse_cond is not None and int(training_batch.mouse_cond.shape[1]) != int( + expected_num_frames + ): + raise ValueError( + "mouse_cond temporal dim mismatch: " + f"got {int(training_batch.mouse_cond.shape[1])}, expected {int(expected_num_frames)}" + ) + + training_batch.latents = latents + training_batch.encoder_hidden_states = None + training_batch.encoder_attention_mask = None + training_batch.image_embeds = image_embeds + training_batch.image_latents = image_latents + training_batch.infos = infos + + training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch = self._prepare_dit_inputs(training_batch) + training_batch = self._build_attention_metadata(training_batch) + + training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) + if training_batch.attn_metadata is not None: + training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] + + training_batch.mask_lat_size = self._build_i2v_mask_latents(image_latents) + viewmats, intrinsics, action_labels = self._process_actions(training_batch) + training_batch.viewmats = viewmats + training_batch.Ks = intrinsics + training_batch.action = action_labels + + return training_batch + + # ModelBase override: add_noise + def add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + b, t = clean_latents.shape[:2] + noisy = self.noise_scheduler.add_noise( + clean_latents.flatten(0, 1), + noise.flatten(0, 1), + timestep, + ).unflatten(0, (b, t)) + return noisy + + # ModelBase override: predict_x0 + def predict_x0( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + cond_inputs = self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + ) + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs["image_embeds"], + image_latents=cond_inputs["image_latents"], + mask_lat_size=cond_inputs["mask_lat_size"], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs["keyboard_cond"], + ) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + # ModelBase override: predict_noise + def predict_noise( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor: + device_type = self.device.type + dtype = noisy_latents.dtype + + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + cond_inputs = self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + ) + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs["image_embeds"], + image_latents=cond_inputs["image_latents"], + mask_lat_size=cond_inputs["mask_lat_size"], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs["keyboard_cond"], + ) + transformer = self._get_transformer(handle, timestep) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + return pred_noise + + # ModelBase override: backward + def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + timesteps, attn_metadata = ctx + with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): + (loss / max(1, int(grad_accum_rounds))).backward() + + # --- WanGame-specific helpers below --- + + def _get_training_dtype(self) -> torch.dtype: + return torch.bfloat16 + + def _init_timestep_mechanics(self) -> None: + self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) + self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) + self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) + self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) + + boundary_ratio = getattr(self.training_args, "boundary_ratio", None) + self.boundary_timestep: float | None = ( + float(boundary_ratio) * float(self.num_train_timestep) + if boundary_ratio is not None + else None + ) + def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: if self.noise_random_generator is None: raise RuntimeError( @@ -412,129 +651,6 @@ def _process_actions( return viewmats, intrinsics, action_labels - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", - ) -> TrainingBatch: - dtype = self._get_training_dtype() - device = self.device - - training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) - infos = raw_batch.get("info_list") - - if latents_source == "zeros": - clip_feature = raw_batch["clip_feature"] - batch_size = int(clip_feature.shape[0]) - vae_config = self.training_args.pipeline_config.vae_config.arch_config - num_channels = int(vae_config.z_dim) - spatial_compression_ratio = int(vae_config.spatial_compression_ratio) - latent_height = int(self.training_args.num_height) // spatial_compression_ratio - latent_width = int(self.training_args.num_width) // spatial_compression_ratio - latents = torch.zeros( - batch_size, - num_channels, - int(self.training_args.num_latent_t), - latent_height, - latent_width, - device=device, - dtype=dtype, - ) - elif latents_source == "data": - if "vae_latent" not in raw_batch: - raise ValueError("vae_latent not found in batch and latents_source='data'") - latents = raw_batch["vae_latent"] - latents = latents[:, :, : self.training_args.num_latent_t] - latents = latents.to(device, dtype=dtype) - else: - raise ValueError(f"Unknown latents_source: {latents_source!r}") - - if "clip_feature" not in raw_batch: - raise ValueError("clip_feature must be present for WanGame") - image_embeds = raw_batch["clip_feature"].to(device, dtype=dtype) - - if "first_frame_latent" not in raw_batch: - raise ValueError("first_frame_latent must be present for WanGame") - image_latents = raw_batch["first_frame_latent"] - image_latents = image_latents[:, :, : self.training_args.num_latent_t] - image_latents = image_latents.to(device, dtype=dtype) - - pil_image = raw_batch.get("pil_image") - if isinstance(pil_image, torch.Tensor): - training_batch.preprocessed_image = pil_image.to(device=device) - else: - training_batch.preprocessed_image = pil_image - - keyboard_cond = raw_batch.get("keyboard_cond") - if isinstance(keyboard_cond, torch.Tensor) and keyboard_cond.numel() > 0: - training_batch.keyboard_cond = keyboard_cond.to(device, dtype=dtype) - else: - training_batch.keyboard_cond = None - - mouse_cond = raw_batch.get("mouse_cond") - if isinstance(mouse_cond, torch.Tensor) and mouse_cond.numel() > 0: - training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) - else: - training_batch.mouse_cond = None - - temporal_compression_ratio = ( - self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - ) - expected_num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 - if training_batch.keyboard_cond is not None and int( - training_batch.keyboard_cond.shape[1] - ) != int(expected_num_frames): - raise ValueError( - "keyboard_cond temporal dim mismatch: " - f"got {int(training_batch.keyboard_cond.shape[1])}, expected {int(expected_num_frames)}" - ) - if training_batch.mouse_cond is not None and int(training_batch.mouse_cond.shape[1]) != int( - expected_num_frames - ): - raise ValueError( - "mouse_cond temporal dim mismatch: " - f"got {int(training_batch.mouse_cond.shape[1])}, expected {int(expected_num_frames)}" - ) - - training_batch.latents = latents - training_batch.encoder_hidden_states = None - training_batch.encoder_attention_mask = None - training_batch.image_embeds = image_embeds - training_batch.image_latents = image_latents - training_batch.infos = infos - - training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) - training_batch = self._prepare_dit_inputs(training_batch) - training_batch = self._build_attention_metadata(training_batch) - - training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) - if training_batch.attn_metadata is not None: - training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] - - training_batch.mask_lat_size = self._build_i2v_mask_latents(image_latents) - viewmats, intrinsics, action_labels = self._process_actions(training_batch) - training_batch.viewmats = viewmats - training_batch.Ks = intrinsics - training_batch.action = action_labels - - return training_batch - - def add_noise( - self, - clean_latents: torch.Tensor, - noise: torch.Tensor, - timestep: torch.Tensor, - ) -> torch.Tensor: - b, t = clean_latents.shape[:2] - noisy = self.noise_scheduler.add_noise( - clean_latents.flatten(0, 1), - noise.flatten(0, 1), - timestep, - ).unflatten(0, (b, t)) - return noisy - def _build_distill_input_kwargs( self, noisy_video_latents: torch.Tensor, @@ -718,106 +834,3 @@ def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch. if float(timestep.item()) < float(self.boundary_timestep): return transformer_2 return transformer - - def predict_x0( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - cond_inputs = self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - ) - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs["image_embeds"], - image_latents=cond_inputs["image_latents"], - mask_lat_size=cond_inputs["mask_lat_size"], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs["keyboard_cond"], - ) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) - return pred_x0 - - def predict_noise( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - cond_inputs = self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - ) - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs["image_embeds"], - image_latents=cond_inputs["image_latents"], - mask_lat_size=cond_inputs["mask_lat_size"], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs["keyboard_cond"], - ) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - return pred_noise - - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: - timesteps, attn_metadata = ctx - with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): - (loss / max(1, int(grad_accum_rounds))).backward() From 628765b7a454a679d0f1e27160af061eaa131c01 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 2 Mar 2026 21:43:07 +0000 Subject: [PATCH 140/214] support init from ckpt --- fastvideo/distillation/dispatch.py | 16 ++ fastvideo/distillation/utils/checkpoint.py | 189 ++++++++++++++++++++- 2 files changed, 203 insertions(+), 2 deletions(-) diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index e17549703..f1c6ae4f6 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -116,6 +116,22 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: model_builder = get_model(str(cfg.recipe.family)) model = model_builder(cfg=cfg) + student_cfg = cfg.roles.get("student") + init_from_checkpoint = None + if student_cfg is not None: + init_from_checkpoint = (student_cfg.extra or {}).get( + "init_from_checkpoint", + None, + ) + if init_from_checkpoint: + from fastvideo.distillation.utils.checkpoint import maybe_warmstart_role_modules + + maybe_warmstart_role_modules( + bundle=model.bundle, + role="student", + init_from_checkpoint=str(init_from_checkpoint), + ) + method_cls = get_method(str(cfg.recipe.method)) method = method_cls.build( cfg=cfg, diff --git a/fastvideo/distillation/utils/checkpoint.py b/fastvideo/distillation/utils/checkpoint.py index 025dbacb8..8bbbe2131 100644 --- a/fastvideo/distillation/utils/checkpoint.py +++ b/fastvideo/distillation/utils/checkpoint.py @@ -5,13 +5,18 @@ import os import re import shutil +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path -from typing import Any, Callable +from typing import Any import torch import torch.distributed as dist import torch.distributed.checkpoint as dcp +from torch.distributed.checkpoint.state_dict import ( + StateDictOptions, + get_model_state_dict, +) from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.logger import init_logger @@ -115,6 +120,183 @@ def _resolve_resume_checkpoint(resume_from_checkpoint: str, *, output_dir: str) ) +def _get_dcp_role_module_names(dcp_dir: Path, role: str) -> set[str]: + """Inspect a DCP checkpoint and return module names present for `roles..*`.""" + + if _rank() == 0: + reader = dcp.FileSystemReader(str(dcp_dir)) + metadata = reader.read_metadata() + module_names: set[str] = set() + prefix = f"roles.{role}." + for key in metadata.state_dict_metadata: + if not key.startswith(prefix): + continue + parts = key.split(".") + if len(parts) >= 3 and parts[2]: + module_names.add(parts[2]) + packed: list[str] = sorted(module_names) + else: + packed = [] + + if dist.is_available() and dist.is_initialized(): + obj_list: list[Any] = [packed] + dist.broadcast_object_list(obj_list, src=0) + packed = obj_list[0] + + return set(str(name) for name in packed) + + +def maybe_warmstart_role_modules( + *, + bundle: RoleManager, + role: str, + init_from_checkpoint: str | None, +) -> None: + """Warmstart model modules for `role` from a Phase 2/3 DCP checkpoint. + + This is **not** a training resume: + - only loads role modules (no optimizer/scheduler/dataloader/RNG state) + - does not advance `start_step` + + The checkpoint directory is expected to be `checkpoint-/dcp/*.distcp`. + """ + + if not init_from_checkpoint: + return + + resolved = _resolve_resume_checkpoint( + str(init_from_checkpoint), + output_dir=str(init_from_checkpoint), + ) + dcp_dir = resolved / "dcp" + if not dcp_dir.is_dir(): + raise FileNotFoundError(f"Missing dcp dir under checkpoint: {dcp_dir}") + + handle = bundle.role(str(role)) + available_modules = _get_dcp_role_module_names(dcp_dir, role=str(role)) + + states: dict[str, Any] = {} + for module_name, module in handle.modules.items(): + if module_name not in available_modules: + continue + states[f"roles.{role}.{module_name}"] = ModelWrapper(module) + + if not states: + raise ValueError( + f"init_from_checkpoint={resolved} does not contain any saved modules for " + f"role={role!r}. Available modules in checkpoint: {sorted(available_modules)}" + ) + + if _rank() == 0: + logger.info( + "Warmstarting role=%s from checkpoint=%s (modules=%s)", + role, + resolved, + sorted(states.keys()), + ) + dcp.load(states, checkpoint_id=str(dcp_dir)) + _barrier() + + +def save_role_pretrained( + *, + bundle: RoleManager, + role: str, + base_model_path: str, + output_dir: str, + module_names: list[str] | None = None, + overwrite: bool = False, +) -> str: + """Export a role's modules into a diffusers-style model directory. + + This is intended to produce a `model_path` that can be loaded by + `PipelineComponentLoader` (i.e., has `model_index.json`, `transformer/`, + `vae/`, and other pipeline components copied from `base_model_path`). + + Notes: + - Only role modules are exported (e.g. `transformer`, optionally `transformer_2`). + - The output directory is based on `base_model_path`, then overwritten with + current in-memory module weights. + """ + + # Resolve HF IDs to local directories (same behavior as module loader). + from fastvideo.utils import maybe_download_model + + local_base = Path(maybe_download_model(str(base_model_path))).resolve() + dst = Path(os.path.expanduser(str(output_dir))).resolve() + + if _rank() == 0: + if dst.exists(): + if overwrite: + shutil.rmtree(dst, ignore_errors=True) + else: + raise FileExistsError( + f"Refusing to overwrite existing directory: {dst}. " + "Pass overwrite=True to replace it." + ) + + def _copy_or_link(src: str, dest: str) -> None: + try: + os.link(src, dest) + except OSError: + shutil.copy2(src, dest) + + logger.info("Creating pretrained export dir at %s (base=%s)", dst, local_base) + shutil.copytree(local_base, dst, symlinks=True, copy_function=_copy_or_link) + + _barrier() + + handle = bundle.role(str(role)) + modules = dict(handle.modules) + if module_names is None: + module_names = sorted(modules.keys()) + + for module_name in module_names: + if module_name not in modules: + raise KeyError( + f"Role {role!r} does not have module {module_name!r}. " + f"Available: {sorted(modules.keys())}" + ) + + module_dir = dst / module_name + if not module_dir.is_dir(): + raise FileNotFoundError( + f"Export directory missing component dir {module_name!r}: {module_dir}" + ) + + options = StateDictOptions(full_state_dict=True, cpu_offload=True) + state_dict = get_model_state_dict(modules[module_name], options=options) + + if _rank() == 0: + # Remove existing *.safetensors to avoid loading duplicate weights. + for path in module_dir.glob("*.safetensors"): + path.unlink(missing_ok=True) + + tensor_state: dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + if not isinstance(value, torch.Tensor): + raise TypeError( + f"Expected tensor in state_dict for {module_name}.{key}, " + f"got {type(value).__name__}" + ) + tensor_state[key] = value.detach().cpu() + + from safetensors.torch import save_file + + out_path = module_dir / "model.safetensors" + logger.info( + "Saving %s weights to %s (%s tensors)", + module_name, + out_path, + len(tensor_state), + ) + save_file(tensor_state, str(out_path)) + + _barrier() + + return str(dst) + + class _RoleModuleContainer(torch.nn.Module): """Ephemeral container to expose multiple role modules as a single module. @@ -236,7 +418,10 @@ def maybe_resume(self, *, resume_from_checkpoint: str | None) -> int | None: if not resume_from_checkpoint: return None - resolved = _resolve_resume_checkpoint(resume_from_checkpoint, output_dir=self.output_dir) + resolved = _resolve_resume_checkpoint( + resume_from_checkpoint, + output_dir=self.output_dir, + ) step = _parse_step_from_dir(resolved) states = self._build_states() From 9d08b03c256494fd7c445b6a0e0f78a3e2ec4b8c Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Mon, 2 Mar 2026 22:00:17 +0000 Subject: [PATCH 141/214] no repetitive model protocal --- .../methods/distribution_matching/dmd2.py | 77 +------------------ .../distillation/methods/fine_tuning/dfsft.py | 51 +----------- .../methods/fine_tuning/finetune.py | 44 +---------- 3 files changed, 12 insertions(+), 160 deletions(-) diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 5419298f0..3c55b3374 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -26,7 +26,7 @@ from __future__ import annotations -from typing import Any, cast, Literal, Protocol +from typing import Any, cast, Literal, TYPE_CHECKING import torch import torch.nn.functional as F @@ -48,77 +48,8 @@ parse_betas, ) - -class _DMD2Model(Protocol): - """Algorithm-specific model contract for :class:`DMD2Method`. - - The method layer is intentionally model-agnostic: it should not import or - depend on any concrete pipeline/model implementation. Instead, all - model-specific primitives (batch preparation, noise schedule helpers, - forward-context management, and role-specific backward behavior) are - provided by a model plugin (e.g. ``WanModel``). - - This ``Protocol`` documents the required surface area and helps static type - checkers/IDE tooling; it is not enforced at runtime (duck typing). - """ - - training_args: Any - - def on_train_start(self) -> None: - ... - - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", - ) -> Any: - ... - - @property - def num_train_timesteps(self) -> int: - ... - - def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: - ... - - def add_noise( - self, - clean_latents: torch.Tensor, - noise: torch.Tensor, - timestep: torch.Tensor, - ) -> torch.Tensor: - ... - - def predict_x0( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: Any, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - ... - - def predict_noise( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: Any, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - ... - - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: - ... +if TYPE_CHECKING: + from fastvideo.distillation.models.base import ModelBase @register_method("dmd2") @@ -140,7 +71,7 @@ def __init__( self, *, bundle: RoleManager, - model: _DMD2Model, + model: ModelBase, method_config: dict[str, Any] | None = None, validation_config: dict[str, Any] | None = None, validator: Any | None = None, diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index 833524c5a..3bccb4df2 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -21,7 +21,7 @@ from __future__ import annotations -from typing import Any, Literal, Protocol, cast +from typing import Any, Literal, TYPE_CHECKING, cast import torch import torch.nn.functional as F @@ -41,51 +41,8 @@ ) from fastvideo.distillation.validators.base import ValidationRequest - -class _DFSFTModel(Protocol): - """Model contract for diffusion-forcing SFT (DFSFT). - - DFSFT is implemented purely at the method (algorithm) layer and relies only - on operation-centric primitives exposed by the model plugin. - """ - - training_args: Any - noise_scheduler: Any - - def on_train_start(self) -> None: - ... - - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", - ) -> Any: - ... - - def add_noise( - self, - clean_latents: torch.Tensor, - noise: torch.Tensor, - timestep: torch.Tensor, - ) -> torch.Tensor: - ... - - def predict_noise( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: Any, - *, - conditional: bool, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - ... - - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: - ... +if TYPE_CHECKING: + from fastvideo.distillation.models.base import ModelBase @register_method("dfsft") @@ -101,7 +58,7 @@ def __init__( self, *, bundle: RoleManager, - model: _DFSFTModel, + model: ModelBase, method_config: dict[str, Any] | None = None, validation_config: dict[str, Any] | None = None, validator: Any | None = None, diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 897fff7b4..78be0b6fd 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -19,7 +19,7 @@ from __future__ import annotations -from typing import Any, Literal, Protocol, cast +from typing import Any, Literal, TYPE_CHECKING, cast import torch import torch.nn.functional as F @@ -39,44 +39,8 @@ parse_betas, ) - -class _FineTuneModel(Protocol): - """Model contract for :class:`FineTuneMethod`. - - Finetuning is implemented as a method (algorithm layer) on top of the - model-plugin-provided model plugin. The method must remain model-plugin - agnostic, so it consumes only operation-centric primitives exposed by the - model. - """ - - training_args: Any - - def on_train_start(self) -> None: - ... - - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", - ) -> Any: - ... - - def predict_noise( - self, - handle: RoleHandle, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: Any, - *, - conditional: bool, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - ... - - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: - ... +if TYPE_CHECKING: + from fastvideo.distillation.models.base import ModelBase @register_method("finetune") @@ -93,7 +57,7 @@ def __init__( self, *, bundle: RoleManager, - model: _FineTuneModel, + model: ModelBase, method_config: dict[str, Any] | None = None, validation_config: dict[str, Any] | None = None, validator: Any | None = None, From 9050496283ae491c084a7e870de2f94cd99b1244 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 00:08:13 +0000 Subject: [PATCH 142/214] reduce concept: distillruntime --- fastvideo/distillation/dispatch.py | 23 ++++------------------ fastvideo/distillation/doc/README.md | 2 +- fastvideo/distillation/doc/RFC-v1-en.md | 4 ++-- fastvideo/distillation/doc/RFC-v1.md | 9 ++++----- fastvideo/distillation/doc/dispatch.md | 4 ++-- fastvideo/distillation/doc/utils/config.md | 5 ++--- fastvideo/training/distillation.py | 20 +++++++++---------- 7 files changed, 25 insertions(+), 42 deletions(-) diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index f1c6ae4f6..d14ed6429 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -3,7 +3,6 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass from typing import Any, TYPE_CHECKING from typing import Protocol @@ -15,16 +14,6 @@ from fastvideo.distillation.models.base import ModelBase -@dataclass(slots=True) -class DistillRuntime: - """Fully assembled runtime for `DistillTrainer.run()`.""" - - training_args: TrainingArgs - method: DistillMethod - dataloader: Any - start_step: int = 0 - - class ModelBuilder(Protocol): def __call__(self, *, cfg: DistillRunConfig) -> ModelBase: ... @@ -105,8 +94,8 @@ def get_method(name: str) -> type[DistillMethod]: return _METHODS[name] -def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: - """Build a distillation runtime from a YAML config. +def build_from_config(cfg: DistillRunConfig) -> tuple[TrainingArgs, DistillMethod, Any, int]: + """Build method+dataloader from a YAML config. Assembles: - model components (bundle/adapter/dataloader/validator) @@ -140,9 +129,5 @@ def build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime: validator=model.validator, ) - return DistillRuntime( - training_args=model.training_args, - method=method, - dataloader=model.dataloader, - start_step=int(getattr(model, "start_step", 0) or 0), - ) + start_step = int(getattr(model, "start_step", 0) or 0) + return model.training_args, method, model.dataloader, start_step diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/distillation/doc/README.md index ed5a41850..a7f857147 100644 --- a/fastvideo/distillation/doc/README.md +++ b/fastvideo/distillation/doc/README.md @@ -11,7 +11,7 @@ 快速入口（从运行到训练）： `fastvideo/training/distillation.py` → `utils.config.load_distill_run_config()` → -`dispatch.build_runtime_from_config()` → +`dispatch.build_from_config()` → `ModelBase(model) + DistillMethod` → `DistillTrainer.run()` --- diff --git a/fastvideo/distillation/doc/RFC-v1-en.md b/fastvideo/distillation/doc/RFC-v1-en.md index e66c13915..4bb892152 100644 --- a/fastvideo/distillation/doc/RFC-v1-en.md +++ b/fastvideo/distillation/doc/RFC-v1-en.md @@ -3,7 +3,7 @@ ```text fastvideo/distillation/ trainer.py # Builds the training loop; calls method-provided train_one_step - dispatch.py # Auto-dispatch via @register_method/@register_model; builds DistillRuntime from config + dispatch.py # Auto-dispatch via @register_method/@register_model; builds (training_args, method, dataloader, start_step) from config roles.py # Wraps model resources in RoleHandle to tag roles (teacher/student/critic/...) models/ components.py # intermediate variable during dispatch-time model construction; records all components @@ -79,4 +79,4 @@ method_config: generator_update_interval: 1 real_score_guidance_scale: 1.0 attn_kind: dense -``` \ No newline at end of file +``` diff --git a/fastvideo/distillation/doc/RFC-v1.md b/fastvideo/distillation/doc/RFC-v1.md index 95b0b4b8e..d341c9696 100644 --- a/fastvideo/distillation/doc/RFC-v1.md +++ b/fastvideo/distillation/doc/RFC-v1.md @@ -4,7 +4,7 @@ ```text fastvideo/distillation/ trainer.py # 构建training loop 调用method提供的train_one_step接口 - dispatch.py # 根据@register_method和@register_model自动识别类型，根据config构建DistillRuntime + dispatch.py # 根据@register_method和@register_model自动识别类型，根据config构建 (training_args, method, dataloader, start_step) roles.py # RoleHandle模型外面包一层role的字段，用于区分teacher/student/critic。 models/ @@ -62,7 +62,7 @@ RoleManager: require_roles([...]) # method 用它声明依赖（早失败、错误信息清晰） ``` -### 2.2 `dispatch.py`：registry + DistillRuntime +### 2.2 `dispatch.py`：registry + build_from_config() ```py # 目标：新增一个 model plugin 或 method 的成本是 O(1)，而不是写 N×M 组合函数 @@ -73,7 +73,7 @@ def build_wan_components(cfg) -> ModelComponents: ... @register_method("dmd2") class DMD2Method(DistillMethod): ... -build_runtime_from_config(cfg): +build_from_config(cfg): components = model_builder(cfg) # -> ModelComponents method = method_cls.build( # -> DistillMethod instance cfg=cfg, @@ -81,7 +81,7 @@ build_runtime_from_config(cfg): adapter=components.adapter, validator=components.validator, ) - return DistillRuntime(training_args, method, dataloader, start_step) + return (training_args, method, dataloader, start_step) ``` ### 2.3 `trainer.py`：DistillTrainer（infra only） @@ -291,4 +291,3 @@ tracker 是 infra 资源（日志/文件/媒体记录），生命周期属于训 - 新增一个 family → 加一个 model plugin 并注册 - 新增一个 method → 加一个 method 文件并注册 - 不需要写 25 个 build 函数或 if/else 分支 - diff --git a/fastvideo/distillation/doc/dispatch.md b/fastvideo/distillation/doc/dispatch.md index 70c1ee2c9..dbf7210ae 100644 --- a/fastvideo/distillation/doc/dispatch.md +++ b/fastvideo/distillation/doc/dispatch.md @@ -16,10 +16,10 @@ - `register_model(name)` / `register_method(name)`：装饰器注册 - `get_model(name)` / `get_method(name)`：查询（会触发内置注册） - `available_models()` / `available_methods()` -- `build_runtime_from_config(cfg) -> DistillRuntime` +- `build_from_config(cfg) -> (training_args, method, dataloader, start_step)` - 选择 model plugin：`get_model(cfg.recipe.family)` - 选择 method：`get_method(cfg.recipe.method)` - - `DistillRuntime` 定义也在本文件中（谁创建谁声明）。 + - 返回值是一个 tuple，避免额外引入 `Runtime` 概念。 **边界** - ✅ 这里只做“装配 + dispatch”，不包含训练 loop / loss / rollout / optimizer policy。 diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index 8efa5da42..e9edff7a6 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -5,8 +5,7 @@ 减少文件级概念数量。 备注： -- `DistillRuntime` 由 `dispatch.build_runtime_from_config()` 创建并定义在 - `fastvideo/distillation/dispatch.py`（谁创建谁声明）。 +- `dispatch.build_from_config()` 负责把 YAML 装配成 `(training_args, method, dataloader, start_step)`。 - tracker 由 `DistillTrainer` 构建并持有（避免 model plugin 变成 infra owner）。 这里包含： @@ -69,7 +68,7 @@ - `dataloader` - `validator`（可选；model-specific） - `start_step`（用于 resume / warm-start） -- `dispatch.build_runtime_from_config()` 选择 model/method 并返回 `DistillRuntime`。 +- `dispatch.build_from_config()` 选择 model/method 并返回 `(training_args, method, dataloader, start_step)`。 ## 4) 通用解析 helpers（method_config / optimizer 等） - `get_optional_int(mapping, key, where=...)` diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 45689d6f7..72c8e24e8 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -27,7 +27,7 @@ def run_distillation_from_config( DistillCheckpointConfig, DistillCheckpointManager, ) - from fastvideo.distillation.dispatch import build_runtime_from_config + from fastvideo.distillation.dispatch import build_from_config from fastvideo.distillation.utils.config import load_distill_run_config cfg = load_distill_run_config(config_path) @@ -43,10 +43,10 @@ def run_distillation_from_config( training_args.sp_size, ) - runtime = build_runtime_from_config(cfg) + _, method, dataloader, start_step = build_from_config(cfg) if dry_run: - logger.info("Dry-run: config parsed and runtime built successfully.") + logger.info("Dry-run: config parsed and build_from_config succeeded.") return trainer = DistillTrainer(training_args, config=cfg.raw) @@ -60,26 +60,26 @@ def run_distillation_from_config( keep_last=int(getattr(training_args, "checkpoints_total_limit", 0) or 0), ) - get_rng_generators = getattr(runtime.method, "get_rng_generators", None) + get_rng_generators = getattr(method, "get_rng_generators", None) if not callable(get_rng_generators): - model = getattr(runtime.method, "model", None) + model = getattr(method, "model", None) get_rng_generators = getattr(model, "get_rng_generators", None) if not callable(get_rng_generators): get_rng_generators = None checkpoint_manager = DistillCheckpointManager( - bundle=runtime.method.bundle, - dataloader=runtime.dataloader, + bundle=method.bundle, + dataloader=dataloader, output_dir=training_args.output_dir, config=ckpt_config, get_rng_generators=get_rng_generators, ) trainer.run( - runtime.method, - dataloader=runtime.dataloader, + method, + dataloader=dataloader, max_steps=training_args.max_train_steps, - start_step=runtime.start_step, + start_step=start_step, checkpoint_manager=checkpoint_manager, ) From ba9686be6f8217fb197df9d7db4faa0cc8fc3793 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 00:47:31 +0000 Subject: [PATCH 143/214] utils/optimizer, utils/validation --- .../methods/distribution_matching/dmd2.py | 264 +++--------------- .../distillation/methods/fine_tuning/dfsft.py | 250 +++-------------- .../methods/fine_tuning/finetune.py | 259 +++-------------- fastvideo/distillation/utils/optimizer.py | 78 ++++++ fastvideo/distillation/utils/validation.py | 175 ++++++++++++ 5 files changed, 355 insertions(+), 671 deletions(-) create mode 100644 fastvideo/distillation/utils/optimizer.py create mode 100644 fastvideo/distillation/utils/validation.py diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 3c55b3374..56f042bc3 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -26,20 +26,30 @@ from __future__ import annotations -from typing import Any, cast, Literal, TYPE_CHECKING +from typing import Any, Literal, TYPE_CHECKING import torch import torch.nn.functional as F -from fastvideo.training.training_utils import ( - clip_grad_norm_while_handling_failing_dtensor_cases, - get_scheduler, -) - -from fastvideo.distillation.roles import RoleManager -from fastvideo.distillation.roles import RoleHandle from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.dispatch import register_method +from fastvideo.distillation.roles import RoleManager +from fastvideo.distillation.utils.optimizer import ( + build_role_optimizer_and_scheduler, + clip_grad_norm_if_needed, +) +from fastvideo.distillation.utils.validation import ( + is_validation_enabled, + parse_validation_dataset_file, + parse_validation_every_steps, + parse_validation_guidance_scale, + parse_validation_num_frames, + parse_validation_ode_solver, + parse_validation_output_dir, + parse_validation_rollout_mode, + parse_validation_sampler_kind, + parse_validation_sampling_steps, +) from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.distillation.utils.config import ( DistillRunConfig, @@ -220,9 +230,9 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: def optimizers_schedulers_step(self, iteration: int) -> None: if self._should_update_student(iteration): for module in self.bundle.role("student").modules.values(): - self._clip_grad_norm(module) + clip_grad_norm_if_needed(module, self.training_args) for module in self.bundle.role("critic").modules.values(): - self._clip_grad_norm(module) + clip_grad_norm_if_needed(module, self.training_args) super().optimizers_schedulers_step(iteration) @@ -235,17 +245,17 @@ def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: return - if not self._is_validation_enabled(): + if not is_validation_enabled(self.validation_config): return - every_steps = self._parse_validation_every_steps() + every_steps = parse_validation_every_steps(self.validation_config) if every_steps <= 0: return if iteration % every_steps != 0: return - dataset_file = self._parse_validation_dataset_file() - sampling_steps = self._parse_validation_sampling_steps() + dataset_file = parse_validation_dataset_file(self.validation_config) + sampling_steps = parse_validation_sampling_steps(self.validation_config) sampling_timesteps: list[int] | None = None raw_timesteps = self.validation_config.get("sampling_timesteps", None) @@ -260,63 +270,27 @@ def log_validation(self, iteration: int) -> None: return sampling_steps = [int(len(sampling_timesteps))] - sampler_kind = self.validation_config.get("sampler_kind", "sde") - if sampler_kind is None: - sampler_kind = "sde" - if not isinstance(sampler_kind, str): - raise ValueError( - "training.validation.sampler_kind must be a string, got " - f"{type(sampler_kind).__name__}" - ) - sampler_kind = sampler_kind.strip().lower() - if sampler_kind not in {"ode", "sde"}: - raise ValueError( - "training.validation.sampler_kind must be one of {ode, sde}, got " - f"{sampler_kind!r}" - ) - sampler_kind = cast(Literal["ode", "sde"], sampler_kind) - ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) + sampler_kind = parse_validation_sampler_kind(self.validation_config, default="sde") + ode_solver = parse_validation_ode_solver( + self.validation_config, sampler_kind=sampler_kind + ) if sampling_timesteps is not None and sampler_kind != "sde": raise ValueError( "method_config.validation.sampling_timesteps is only valid when " "sampler_kind='sde'" ) - rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") - if not isinstance(rollout_mode_raw, str): - raise ValueError( - "training.validation.rollout_mode must be a string when set, got " - f"{type(rollout_mode_raw).__name__}" - ) - rollout_mode = rollout_mode_raw.strip().lower() - if rollout_mode not in {"parallel", "streaming"}: - raise ValueError( - "training.validation.rollout_mode must be one of {parallel, streaming}, " - f"got {rollout_mode_raw!r}" - ) - - guidance_scale = self._parse_validation_guidance_scale() - output_dir = self.validation_config.get("output_dir", None) - if output_dir is not None and not isinstance(output_dir, str): - raise ValueError( - "training.validation.output_dir must be a string when set, got " - f"{type(output_dir).__name__}" - ) - - num_actions = get_optional_int( - self.validation_config, - "num_frames", - where="training.validation.num_frames", - ) - if num_actions is not None and num_actions <= 0: - raise ValueError("training.validation.num_frames must be > 0 when set") + rollout_mode = parse_validation_rollout_mode(self.validation_config) + guidance_scale = parse_validation_guidance_scale(self.validation_config) + output_dir = parse_validation_output_dir(self.validation_config) + num_actions = parse_validation_num_frames(self.validation_config) request = ValidationRequest( sample_handle=self.student, dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, - rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), + rollout_mode=rollout_mode, ode_solver=ode_solver, sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, @@ -416,44 +390,6 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: return cfg - def _build_role_optimizer_and_scheduler( - self, - *, - role: str, - handle: RoleHandle, - learning_rate: float, - betas: tuple[float, float], - scheduler_name: str, - ) -> None: - modules = handle.modules - params: list[torch.nn.Parameter] = [] - for module in modules.values(): - params.extend([p for p in module.parameters() if p.requires_grad]) - if not params: - raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") - - optimizer = torch.optim.AdamW( - params, - lr=float(learning_rate), - betas=betas, - weight_decay=float(getattr(self.training_args, "weight_decay", 0.0) or 0.0), - eps=1e-8, - ) - - scheduler = get_scheduler( - str(scheduler_name), - optimizer=optimizer, - num_warmup_steps=int(getattr(self.training_args, "lr_warmup_steps", 0) or 0), - num_training_steps=int(getattr(self.training_args, "max_train_steps", 0) or 0), - num_cycles=int(getattr(self.training_args, "lr_num_cycles", 0) or 0), - power=float(getattr(self.training_args, "lr_power", 0.0) or 0.0), - min_lr_ratio=float(getattr(self.training_args, "min_lr_ratio", 0.5) or 0.5), - last_epoch=-1, - ) - - handle.optimizers = {"main": optimizer} - handle.lr_schedulers = {"main": scheduler} - def _init_optimizers_and_schedulers(self) -> None: training_args = self.training_args @@ -464,9 +400,10 @@ def _init_optimizers_and_schedulers(self) -> None: where="training.betas", ) student_sched = str(getattr(training_args, "lr_scheduler", "constant")) - self._build_role_optimizer_and_scheduler( + build_role_optimizer_and_scheduler( role="student", handle=self.student, + training_args=self.training_args, learning_rate=student_lr, betas=student_betas, scheduler_name=student_sched, @@ -483,122 +420,15 @@ def _init_optimizers_and_schedulers(self) -> None: critic_betas = parse_betas(critic_betas_raw, where="training.fake_score_betas") critic_sched = str(getattr(training_args, "fake_score_lr_scheduler", None) or student_sched) - self._build_role_optimizer_and_scheduler( + build_role_optimizer_and_scheduler( role="critic", handle=self.critic, + training_args=self.training_args, learning_rate=critic_lr, betas=critic_betas, scheduler_name=critic_sched, ) - def _is_validation_enabled(self) -> bool: - cfg = self.validation_config - if not cfg: - return False - enabled = cfg.get("enabled", None) - if enabled is None: - return True - if isinstance(enabled, bool): - return bool(enabled) - raise ValueError( - "training.validation.enabled must be a bool when set, got " - f"{type(enabled).__name__}" - ) - - def _parse_validation_every_steps(self) -> int: - raw = self.validation_config.get("every_steps", None) - if raw is None: - raise ValueError("training.validation.every_steps must be set when validation is enabled") - if isinstance(raw, bool): - raise ValueError("training.validation.every_steps must be an int, got bool") - if isinstance(raw, int): - return int(raw) - if isinstance(raw, float) and raw.is_integer(): - return int(raw) - if isinstance(raw, str) and raw.strip(): - return int(raw) - raise ValueError( - "training.validation.every_steps must be an int, got " - f"{type(raw).__name__}" - ) - - def _parse_validation_dataset_file(self) -> str: - raw = self.validation_config.get("dataset_file", None) - if not isinstance(raw, str) or not raw.strip(): - raise ValueError( - "training.validation.dataset_file must be set when validation is enabled" - ) - return raw.strip() - - def _parse_validation_sampling_steps(self) -> list[int]: - raw = self.validation_config.get("sampling_steps") - steps: list[int] = [] - if raw is None or raw == "": - raise ValueError( - "training.validation.sampling_steps must be set for validation" - ) - if isinstance(raw, bool): - raise ValueError( - "validation sampling_steps must be an int/list/str, got bool" - ) - if isinstance(raw, int) or (isinstance(raw, float) and raw.is_integer()): - steps = [int(raw)] - elif isinstance(raw, str): - steps = [int(s) for s in raw.split(",") if str(s).strip()] - elif isinstance(raw, list): - steps = [int(s) for s in raw] - else: - raise ValueError( - "validation sampling_steps must be an int/list/str, got " - f"{type(raw).__name__}" - ) - return [s for s in steps if int(s) > 0] - - def _parse_validation_guidance_scale(self) -> float | None: - raw = self.validation_config.get("guidance_scale") - if raw in (None, ""): - return None - if isinstance(raw, bool): - raise ValueError( - "validation guidance_scale must be a number/string, got bool" - ) - if isinstance(raw, (int, float)): - return float(raw) - if isinstance(raw, str) and raw.strip(): - return float(raw) - raise ValueError( - "validation guidance_scale must be a number/string, got " - f"{type(raw).__name__}" - ) - - def _parse_validation_ode_solver( - self, - *, - sampler_kind: Literal["ode", "sde"], - ) -> str | None: - raw = self.validation_config.get("ode_solver", None) - if raw in (None, ""): - return None - if sampler_kind != "ode": - raise ValueError( - "training.validation.ode_solver is only valid when " - "training.validation.sampler_kind='ode'" - ) - if not isinstance(raw, str): - raise ValueError( - "training.validation.ode_solver must be a string when set, got " - f"{type(raw).__name__}" - ) - solver = raw.strip().lower() - if solver in {"unipc", "unipc_multistep", "multistep"}: - return "unipc" - if solver in {"euler", "flowmatch", "flowmatch_euler"}: - return "euler" - raise ValueError( - "training.validation.ode_solver must be one of {unipc, euler}, got " - f"{raw!r}" - ) - def _should_update_student(self, iteration: int) -> bool: interval = get_optional_int( self.method_config, @@ -611,26 +441,6 @@ def _should_update_student(self, iteration: int) -> bool: return True return iteration % interval == 0 - def _clip_grad_norm(self, module: torch.nn.Module) -> float: - max_grad_norm_raw = getattr(self.training_args, "max_grad_norm", None) - if max_grad_norm_raw is None: - return 0.0 - try: - max_grad_norm = float(max_grad_norm_raw) - except (TypeError, ValueError) as e: - raise ValueError( - "training.max_grad_norm must be a number when set, got " - f"{max_grad_norm_raw!r}" - ) from e - if max_grad_norm <= 0.0: - return 0.0 - grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( - [p for p in module.parameters()], - max_grad_norm, - foreach=None, - ) - return float(grad_norm.item()) if grad_norm is not None else 0.0 - def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: if self._denoising_step_list is not None and self._denoising_step_list.device == device: return self._denoising_step_list diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index 3bccb4df2..a81138f49 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -26,19 +26,29 @@ import torch import torch.nn.functional as F -from fastvideo.training.training_utils import ( - clip_grad_norm_while_handling_failing_dtensor_cases, - get_scheduler, -) - from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.dispatch import register_method -from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.distillation.utils.config import ( DistillRunConfig, - get_optional_int, parse_betas, ) +from fastvideo.distillation.roles import RoleManager +from fastvideo.distillation.utils.optimizer import ( + build_role_optimizer_and_scheduler, + clip_grad_norm_if_needed, +) +from fastvideo.distillation.utils.validation import ( + is_validation_enabled, + parse_validation_dataset_file, + parse_validation_every_steps, + parse_validation_guidance_scale, + parse_validation_num_frames, + parse_validation_ode_solver, + parse_validation_output_dir, + parse_validation_rollout_mode, + parse_validation_sampler_kind, + parse_validation_sampling_steps, +) from fastvideo.distillation.validators.base import ValidationRequest if TYPE_CHECKING: @@ -234,7 +244,7 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: for module in self.student.modules.values(): - self._clip_grad_norm(module) + clip_grad_norm_if_needed(module, self.training_args) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start @@ -246,68 +256,32 @@ def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: return - if not self._is_validation_enabled(): + if not is_validation_enabled(self.validation_config): return - every_steps = self._parse_validation_every_steps() + every_steps = parse_validation_every_steps(self.validation_config) if every_steps <= 0: return if iteration % every_steps != 0: return - dataset_file = self._parse_validation_dataset_file() - sampling_steps = self._parse_validation_sampling_steps() - guidance_scale = self._parse_validation_guidance_scale() - - sampler_kind_raw = self.validation_config.get("sampler_kind", "ode") - if not isinstance(sampler_kind_raw, str): - raise ValueError( - "training.validation.sampler_kind must be a string when set, got " - f"{type(sampler_kind_raw).__name__}" - ) - sampler_kind = sampler_kind_raw.strip().lower() - if sampler_kind not in {"ode", "sde"}: - raise ValueError( - "training.validation.sampler_kind must be one of {ode, sde}, got " - f"{sampler_kind_raw!r}" - ) - sampler_kind = cast(Literal["ode", "sde"], sampler_kind) - ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) - - rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") - if not isinstance(rollout_mode_raw, str): - raise ValueError( - "training.validation.rollout_mode must be a string when set, got " - f"{type(rollout_mode_raw).__name__}" - ) - rollout_mode = rollout_mode_raw.strip().lower() - if rollout_mode not in {"parallel", "streaming"}: - raise ValueError( - "training.validation.rollout_mode must be one of {parallel, streaming}, " - f"got {rollout_mode_raw!r}" - ) - - output_dir = self.validation_config.get("output_dir", None) - if output_dir is not None and not isinstance(output_dir, str): - raise ValueError( - "training.validation.output_dir must be a string when set, got " - f"{type(output_dir).__name__}" - ) - - num_actions = get_optional_int( - self.validation_config, - "num_frames", - where="training.validation.num_frames", + dataset_file = parse_validation_dataset_file(self.validation_config) + sampling_steps = parse_validation_sampling_steps(self.validation_config) + guidance_scale = parse_validation_guidance_scale(self.validation_config) + sampler_kind = parse_validation_sampler_kind(self.validation_config, default="ode") + rollout_mode = parse_validation_rollout_mode(self.validation_config) + output_dir = parse_validation_output_dir(self.validation_config) + num_actions = parse_validation_num_frames(self.validation_config) + ode_solver = parse_validation_ode_solver( + self.validation_config, sampler_kind=sampler_kind ) - if num_actions is not None and num_actions <= 0: - raise ValueError("training.validation.num_frames must be > 0 when set") request = ValidationRequest( sample_handle=self.student, dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, - rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), + rollout_mode=rollout_mode, ode_solver=ode_solver, sampling_timesteps=None, guidance_scale=guidance_scale, @@ -414,43 +388,6 @@ def _parse_timestep_index_range(self) -> tuple[int, int]: # torch.randint expects [low, high), so make high exclusive. return min_index, max_index + 1 - def _build_role_optimizer_and_scheduler( - self, - *, - role: str, - handle: RoleHandle, - learning_rate: float, - betas: tuple[float, float], - scheduler_name: str, - ) -> None: - params: list[torch.nn.Parameter] = [] - for module in handle.modules.values(): - params.extend([p for p in module.parameters() if p.requires_grad]) - if not params: - raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") - - optimizer = torch.optim.AdamW( - params, - lr=float(learning_rate), - betas=betas, - weight_decay=float(getattr(self.training_args, "weight_decay", 0.0) or 0.0), - eps=1e-8, - ) - - scheduler = get_scheduler( - str(scheduler_name), - optimizer=optimizer, - num_warmup_steps=int(getattr(self.training_args, "lr_warmup_steps", 0) or 0), - num_training_steps=int(getattr(self.training_args, "max_train_steps", 0) or 0), - num_cycles=int(getattr(self.training_args, "lr_num_cycles", 0) or 0), - power=float(getattr(self.training_args, "lr_power", 0.0) or 0.0), - min_lr_ratio=float(getattr(self.training_args, "min_lr_ratio", 0.5) or 0.5), - last_epoch=-1, - ) - - handle.optimizers = {"main": optimizer} - handle.lr_schedulers = {"main": scheduler} - def _init_optimizers_and_schedulers(self) -> None: student_lr = float(getattr(self.training_args, "learning_rate", 0.0) or 0.0) if student_lr <= 0.0: @@ -461,118 +398,15 @@ def _init_optimizers_and_schedulers(self) -> None: where="training.betas", ) student_sched = str(getattr(self.training_args, "lr_scheduler", "constant")) - self._build_role_optimizer_and_scheduler( + build_role_optimizer_and_scheduler( role="student", handle=self.student, + training_args=self.training_args, learning_rate=student_lr, betas=student_betas, scheduler_name=student_sched, ) - def _is_validation_enabled(self) -> bool: - cfg = self.validation_config - if not cfg: - return False - enabled = cfg.get("enabled", None) - if enabled is None: - return True - if isinstance(enabled, bool): - return bool(enabled) - raise ValueError( - "training.validation.enabled must be a bool when set, got " - f"{type(enabled).__name__}" - ) - - def _parse_validation_every_steps(self) -> int: - raw = self.validation_config.get("every_steps", None) - if raw is None: - raise ValueError( - "training.validation.every_steps must be set when validation is enabled" - ) - if isinstance(raw, bool): - raise ValueError("training.validation.every_steps must be an int, got bool") - if isinstance(raw, int): - return int(raw) - if isinstance(raw, float) and raw.is_integer(): - return int(raw) - if isinstance(raw, str) and raw.strip(): - return int(raw) - raise ValueError( - "training.validation.every_steps must be an int, got " - f"{type(raw).__name__}" - ) - - def _parse_validation_dataset_file(self) -> str: - raw = self.validation_config.get("dataset_file", None) - if not isinstance(raw, str) or not raw.strip(): - raise ValueError( - "training.validation.dataset_file must be set when validation is enabled" - ) - return raw.strip() - - def _parse_validation_sampling_steps(self) -> list[int]: - raw = self.validation_config.get("sampling_steps") - steps: list[int] = [] - if raw is None or raw == "": - raise ValueError("training.validation.sampling_steps must be set for validation") - if isinstance(raw, bool): - raise ValueError("validation sampling_steps must be an int/list/str, got bool") - if isinstance(raw, int) or (isinstance(raw, float) and raw.is_integer()): - steps = [int(raw)] - elif isinstance(raw, str): - steps = [int(s) for s in raw.split(",") if str(s).strip()] - elif isinstance(raw, list): - steps = [int(s) for s in raw] - else: - raise ValueError( - "validation sampling_steps must be an int/list/str, got " - f"{type(raw).__name__}" - ) - return [s for s in steps if int(s) > 0] - - def _parse_validation_guidance_scale(self) -> float | None: - raw = self.validation_config.get("guidance_scale") - if raw in (None, ""): - return None - if isinstance(raw, bool): - raise ValueError("validation guidance_scale must be a number/string, got bool") - if isinstance(raw, (int, float)): - return float(raw) - if isinstance(raw, str) and raw.strip(): - return float(raw) - raise ValueError( - "validation guidance_scale must be a number/string, got " - f"{type(raw).__name__}" - ) - - def _parse_validation_ode_solver( - self, - *, - sampler_kind: Literal["ode", "sde"], - ) -> str | None: - raw = self.validation_config.get("ode_solver", None) - if raw in (None, ""): - return None - if sampler_kind != "ode": - raise ValueError( - "training.validation.ode_solver is only valid when " - "training.validation.sampler_kind='ode'" - ) - if not isinstance(raw, str): - raise ValueError( - "training.validation.ode_solver must be a string when set, got " - f"{type(raw).__name__}" - ) - solver = raw.strip().lower() - if solver in {"unipc", "unipc_multistep", "multistep"}: - return "unipc" - if solver in {"euler", "flowmatch", "flowmatch_euler"}: - return "euler" - raise ValueError( - "training.validation.ode_solver must be one of {unipc, euler}, got " - f"{raw!r}" - ) - def _sample_t_inhom_indices(self, *, batch_size: int, num_latents: int, device: torch.device) -> torch.Tensor: chunk_size = self._chunk_size num_chunks = (num_latents + chunk_size - 1) // chunk_size @@ -586,23 +420,3 @@ def _sample_t_inhom_indices(self, *, batch_size: int, num_latents: int, device: ) expanded = chunk_indices.repeat_interleave(chunk_size, dim=1) return expanded[:, :num_latents] - - def _clip_grad_norm(self, module: torch.nn.Module) -> float: - max_grad_norm_raw = getattr(self.training_args, "max_grad_norm", None) - if max_grad_norm_raw is None: - return 0.0 - try: - max_grad_norm = float(max_grad_norm_raw) - except (TypeError, ValueError) as e: - raise ValueError( - "training.max_grad_norm must be a number when set, got " - f"{max_grad_norm_raw!r}" - ) from e - if max_grad_norm <= 0.0: - return 0.0 - grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( - [p for p in module.parameters()], - max_grad_norm, - foreach=None, - ) - return float(grad_norm.item()) if grad_norm is not None else 0.0 diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 78be0b6fd..ab8f0614a 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -24,18 +24,28 @@ import torch import torch.nn.functional as F -from fastvideo.training.training_utils import ( - clip_grad_norm_while_handling_failing_dtensor_cases, - get_scheduler, -) - -from fastvideo.distillation.roles import RoleHandle, RoleManager -from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.dispatch import register_method +from fastvideo.distillation.methods.base import DistillMethod, LogScalar +from fastvideo.distillation.roles import RoleManager +from fastvideo.distillation.utils.optimizer import ( + build_role_optimizer_and_scheduler, + clip_grad_norm_if_needed, +) +from fastvideo.distillation.utils.validation import ( + is_validation_enabled, + parse_validation_dataset_file, + parse_validation_every_steps, + parse_validation_guidance_scale, + parse_validation_num_frames, + parse_validation_ode_solver, + parse_validation_output_dir, + parse_validation_rollout_mode, + parse_validation_sampler_kind, + parse_validation_sampling_steps, +) from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.distillation.utils.config import ( DistillRunConfig, - get_optional_int, parse_betas, ) @@ -189,7 +199,7 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: for module in self.student.modules.values(): - self._clip_grad_norm(module) + clip_grad_norm_if_needed(module, self.training_args) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start @@ -201,68 +211,32 @@ def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: return - if not self._is_validation_enabled(): + if not is_validation_enabled(self.validation_config): return - every_steps = self._parse_validation_every_steps() + every_steps = parse_validation_every_steps(self.validation_config) if every_steps <= 0: return if iteration % every_steps != 0: return - dataset_file = self._parse_validation_dataset_file() - sampling_steps = self._parse_validation_sampling_steps() - guidance_scale = self._parse_validation_guidance_scale() - - sampler_kind_raw = self.validation_config.get("sampler_kind", "ode") - if not isinstance(sampler_kind_raw, str): - raise ValueError( - "training.validation.sampler_kind must be a string when set, got " - f"{type(sampler_kind_raw).__name__}" - ) - sampler_kind = sampler_kind_raw.strip().lower() - if sampler_kind not in {"ode", "sde"}: - raise ValueError( - "training.validation.sampler_kind must be one of {ode, sde}, got " - f"{sampler_kind_raw!r}" - ) - sampler_kind = cast(Literal["ode", "sde"], sampler_kind) - ode_solver = self._parse_validation_ode_solver(sampler_kind=sampler_kind) - - rollout_mode_raw = self.validation_config.get("rollout_mode", "parallel") - if not isinstance(rollout_mode_raw, str): - raise ValueError( - "training.validation.rollout_mode must be a string when set, got " - f"{type(rollout_mode_raw).__name__}" - ) - rollout_mode = rollout_mode_raw.strip().lower() - if rollout_mode not in {"parallel", "streaming"}: - raise ValueError( - "training.validation.rollout_mode must be one of {parallel, streaming}, " - f"got {rollout_mode_raw!r}" - ) - - output_dir = self.validation_config.get("output_dir", None) - if output_dir is not None and not isinstance(output_dir, str): - raise ValueError( - "training.validation.output_dir must be a string when set, got " - f"{type(output_dir).__name__}" - ) - - num_actions = get_optional_int( - self.validation_config, - "num_frames", - where="training.validation.num_frames", + dataset_file = parse_validation_dataset_file(self.validation_config) + sampling_steps = parse_validation_sampling_steps(self.validation_config) + guidance_scale = parse_validation_guidance_scale(self.validation_config) + sampler_kind = parse_validation_sampler_kind(self.validation_config, default="ode") + rollout_mode = parse_validation_rollout_mode(self.validation_config) + output_dir = parse_validation_output_dir(self.validation_config) + num_actions = parse_validation_num_frames(self.validation_config) + ode_solver = parse_validation_ode_solver( + self.validation_config, sampler_kind=sampler_kind ) - if num_actions is not None and num_actions <= 0: - raise ValueError("training.validation.num_frames must be > 0 when set") request = ValidationRequest( sample_handle=self.student, dataset_file=dataset_file, sampling_steps=sampling_steps, sampler_kind=sampler_kind, - rollout_mode=cast(Literal["parallel", "streaming"], rollout_mode), + rollout_mode=rollout_mode, ode_solver=ode_solver, sampling_timesteps=None, guidance_scale=guidance_scale, @@ -300,44 +274,6 @@ def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: ) return cast(Literal["dense", "vsa"], kind) - def _build_role_optimizer_and_scheduler( - self, - *, - role: str, - handle: RoleHandle, - learning_rate: float, - betas: tuple[float, float], - scheduler_name: str, - ) -> None: - modules = handle.modules - params: list[torch.nn.Parameter] = [] - for module in modules.values(): - params.extend([p for p in module.parameters() if p.requires_grad]) - if not params: - raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") - - optimizer = torch.optim.AdamW( - params, - lr=float(learning_rate), - betas=betas, - weight_decay=float(getattr(self.training_args, "weight_decay", 0.0) or 0.0), - eps=1e-8, - ) - - scheduler = get_scheduler( - str(scheduler_name), - optimizer=optimizer, - num_warmup_steps=int(getattr(self.training_args, "lr_warmup_steps", 0) or 0), - num_training_steps=int(getattr(self.training_args, "max_train_steps", 0) or 0), - num_cycles=int(getattr(self.training_args, "lr_num_cycles", 0) or 0), - power=float(getattr(self.training_args, "lr_power", 0.0) or 0.0), - min_lr_ratio=float(getattr(self.training_args, "min_lr_ratio", 0.5) or 0.5), - last_epoch=-1, - ) - - handle.optimizers = {"main": optimizer} - handle.lr_schedulers = {"main": scheduler} - def _init_optimizers_and_schedulers(self) -> None: training_args = self.training_args @@ -350,140 +286,11 @@ def _init_optimizers_and_schedulers(self) -> None: where="training.betas", ) student_sched = str(getattr(training_args, "lr_scheduler", "constant")) - self._build_role_optimizer_and_scheduler( + build_role_optimizer_and_scheduler( role="student", handle=self.student, + training_args=self.training_args, learning_rate=student_lr, betas=student_betas, scheduler_name=student_sched, ) - - def _is_validation_enabled(self) -> bool: - cfg = self.validation_config - if not cfg: - return False - enabled = cfg.get("enabled", None) - if enabled is None: - return True - if isinstance(enabled, bool): - return bool(enabled) - raise ValueError( - "training.validation.enabled must be a bool when set, got " - f"{type(enabled).__name__}" - ) - - def _parse_validation_every_steps(self) -> int: - raw = self.validation_config.get("every_steps", None) - if raw is None: - raise ValueError( - "training.validation.every_steps must be set when validation is enabled" - ) - if isinstance(raw, bool): - raise ValueError("training.validation.every_steps must be an int, got bool") - if isinstance(raw, int): - return int(raw) - if isinstance(raw, float) and raw.is_integer(): - return int(raw) - if isinstance(raw, str) and raw.strip(): - return int(raw) - raise ValueError( - "training.validation.every_steps must be an int, got " - f"{type(raw).__name__}" - ) - - def _parse_validation_dataset_file(self) -> str: - raw = self.validation_config.get("dataset_file", None) - if not isinstance(raw, str) or not raw.strip(): - raise ValueError( - "training.validation.dataset_file must be set when validation is enabled" - ) - return raw.strip() - - def _parse_validation_sampling_steps(self) -> list[int]: - raw = self.validation_config.get("sampling_steps") - steps: list[int] = [] - if raw is None or raw == "": - raise ValueError( - "training.validation.sampling_steps must be set for validation" - ) - if isinstance(raw, bool): - raise ValueError( - "validation sampling_steps must be an int/list/str, got bool" - ) - if isinstance(raw, int) or (isinstance(raw, float) and raw.is_integer()): - steps = [int(raw)] - elif isinstance(raw, str): - steps = [int(s) for s in raw.split(",") if str(s).strip()] - elif isinstance(raw, list): - steps = [int(s) for s in raw] - else: - raise ValueError( - "validation sampling_steps must be an int/list/str, got " - f"{type(raw).__name__}" - ) - return [s for s in steps if int(s) > 0] - - def _parse_validation_guidance_scale(self) -> float | None: - raw = self.validation_config.get("guidance_scale") - if raw in (None, ""): - return None - if isinstance(raw, bool): - raise ValueError( - "validation guidance_scale must be a number/string, got bool" - ) - if isinstance(raw, (int, float)): - return float(raw) - if isinstance(raw, str) and raw.strip(): - return float(raw) - raise ValueError( - "validation guidance_scale must be a number/string, got " - f"{type(raw).__name__}" - ) - - def _parse_validation_ode_solver( - self, - *, - sampler_kind: Literal["ode", "sde"], - ) -> str | None: - raw = self.validation_config.get("ode_solver", None) - if raw in (None, ""): - return None - if sampler_kind != "ode": - raise ValueError( - "training.validation.ode_solver is only valid when " - "training.validation.sampler_kind='ode'" - ) - if not isinstance(raw, str): - raise ValueError( - "training.validation.ode_solver must be a string when set, got " - f"{type(raw).__name__}" - ) - solver = raw.strip().lower() - if solver in {"unipc", "unipc_multistep", "multistep"}: - return "unipc" - if solver in {"euler", "flowmatch", "flowmatch_euler"}: - return "euler" - raise ValueError( - "training.validation.ode_solver must be one of {unipc, euler}, got " - f"{raw!r}" - ) - - def _clip_grad_norm(self, module: torch.nn.Module) -> float: - max_grad_norm_raw = getattr(self.training_args, "max_grad_norm", None) - if max_grad_norm_raw is None: - return 0.0 - try: - max_grad_norm = float(max_grad_norm_raw) - except (TypeError, ValueError) as e: - raise ValueError( - "training.max_grad_norm must be a number when set, got " - f"{max_grad_norm_raw!r}" - ) from e - if max_grad_norm <= 0.0: - return 0.0 - grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( - [p for p in module.parameters()], - max_grad_norm, - foreach=None, - ) - return float(grad_norm.item()) if grad_norm is not None else 0.0 diff --git a/fastvideo/distillation/utils/optimizer.py b/fastvideo/distillation/utils/optimizer.py new file mode 100644 index 000000000..66913329d --- /dev/null +++ b/fastvideo/distillation/utils/optimizer.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from fastvideo.distillation.roles import RoleHandle + +from fastvideo.training.training_utils import ( + clip_grad_norm_while_handling_failing_dtensor_cases, + get_scheduler, +) + +if TYPE_CHECKING: + from fastvideo.fastvideo_args import TrainingArgs + + +def build_role_optimizer_and_scheduler( + *, + role: str, + handle: RoleHandle, + training_args: TrainingArgs, + learning_rate: float, + betas: tuple[float, float], + scheduler_name: str, +) -> None: + modules = handle.modules + params: list[torch.nn.Parameter] = [] + for module in modules.values(): + params.extend([p for p in module.parameters() if p.requires_grad]) + if not params: + raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") + + optimizer = torch.optim.AdamW( + params, + lr=float(learning_rate), + betas=betas, + weight_decay=float(getattr(training_args, "weight_decay", 0.0) or 0.0), + eps=1e-8, + ) + + scheduler = get_scheduler( + str(scheduler_name), + optimizer=optimizer, + num_warmup_steps=int(getattr(training_args, "lr_warmup_steps", 0) or 0), + num_training_steps=int(getattr(training_args, "max_train_steps", 0) or 0), + num_cycles=int(getattr(training_args, "lr_num_cycles", 0) or 0), + power=float(getattr(training_args, "lr_power", 0.0) or 0.0), + min_lr_ratio=float(getattr(training_args, "min_lr_ratio", 0.5) or 0.5), + last_epoch=-1, + ) + + handle.optimizers = {"main": optimizer} + handle.lr_schedulers = {"main": scheduler} + + +def clip_grad_norm_if_needed(module: torch.nn.Module, training_args: TrainingArgs) -> float: + max_grad_norm_raw = getattr(training_args, "max_grad_norm", None) + if max_grad_norm_raw is None: + return 0.0 + try: + max_grad_norm = float(max_grad_norm_raw) + except (TypeError, ValueError) as e: + raise ValueError( + "training.max_grad_norm must be a number when set, got " + f"{max_grad_norm_raw!r}" + ) from e + if max_grad_norm <= 0.0: + return 0.0 + grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in module.parameters()], + max_grad_norm, + foreach=None, + ) + return float(grad_norm.item()) if grad_norm is not None else 0.0 + diff --git a/fastvideo/distillation/utils/validation.py b/fastvideo/distillation/utils/validation.py new file mode 100644 index 000000000..6f7cc4f0d --- /dev/null +++ b/fastvideo/distillation/utils/validation.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any, Literal, cast + +from fastvideo.distillation.utils.config import get_optional_int + + +def is_validation_enabled(cfg: dict[str, Any]) -> bool: + if not cfg: + return False + enabled = cfg.get("enabled") + if enabled is None: + return True + if isinstance(enabled, bool): + return bool(enabled) + raise ValueError( + "training.validation.enabled must be a bool when set, got " + f"{type(enabled).__name__}" + ) + + +def parse_validation_every_steps(cfg: dict[str, Any]) -> int: + raw = cfg.get("every_steps") + if raw is None: + raise ValueError("training.validation.every_steps must be set when validation is enabled") + if isinstance(raw, bool): + raise ValueError("training.validation.every_steps must be an int, got bool") + if isinstance(raw, int): + return int(raw) + if isinstance(raw, float) and raw.is_integer(): + return int(raw) + if isinstance(raw, str) and raw.strip(): + return int(raw) + raise ValueError( + "training.validation.every_steps must be an int, got " + f"{type(raw).__name__}" + ) + + +def parse_validation_dataset_file(cfg: dict[str, Any]) -> str: + raw = cfg.get("dataset_file") + if not isinstance(raw, str) or not raw.strip(): + raise ValueError("training.validation.dataset_file must be set when validation is enabled") + return raw.strip() + + +def parse_validation_sampling_steps(cfg: dict[str, Any]) -> list[int]: + raw = cfg.get("sampling_steps") + steps: list[int] = [] + if raw is None or raw == "": + raise ValueError("training.validation.sampling_steps must be set for validation") + if isinstance(raw, bool): + raise ValueError("validation sampling_steps must be an int/list/str, got bool") + if isinstance(raw, int) or (isinstance(raw, float) and raw.is_integer()): + steps = [int(raw)] + elif isinstance(raw, str): + steps = [int(s) for s in raw.split(",") if str(s).strip()] + elif isinstance(raw, list): + steps = [int(s) for s in raw] + else: + raise ValueError( + "validation sampling_steps must be an int/list/str, got " + f"{type(raw).__name__}" + ) + return [s for s in steps if int(s) > 0] + + +def parse_validation_guidance_scale(cfg: dict[str, Any]) -> float | None: + raw = cfg.get("guidance_scale") + if raw in (None, ""): + return None + if isinstance(raw, bool): + raise ValueError("validation guidance_scale must be a number/string, got bool") + if isinstance(raw, (int, float)): + return float(raw) + if isinstance(raw, str) and raw.strip(): + return float(raw) + raise ValueError( + "validation guidance_scale must be a number/string, got " + f"{type(raw).__name__}" + ) + + +def parse_validation_sampler_kind( + cfg: dict[str, Any], + *, + default: Literal["ode", "sde"], +) -> Literal["ode", "sde"]: + raw = cfg.get("sampler_kind", default) + if raw is None: + raw = default + if not isinstance(raw, str): + raise ValueError( + "training.validation.sampler_kind must be a string when set, got " + f"{type(raw).__name__}" + ) + kind = raw.strip().lower() + if kind not in {"ode", "sde"}: + raise ValueError( + "training.validation.sampler_kind must be one of {ode, sde}, got " + f"{raw!r}" + ) + return cast(Literal["ode", "sde"], kind) + + +def parse_validation_rollout_mode( + cfg: dict[str, Any], + *, + default: Literal["parallel", "streaming"] = "parallel", +) -> Literal["parallel", "streaming"]: + raw = cfg.get("rollout_mode", default) + if raw is None: + raw = default + if not isinstance(raw, str): + raise ValueError( + "training.validation.rollout_mode must be a string when set, got " + f"{type(raw).__name__}" + ) + mode = raw.strip().lower() + if mode not in {"parallel", "streaming"}: + raise ValueError( + "training.validation.rollout_mode must be one of {parallel, streaming}, " + f"got {raw!r}" + ) + return cast(Literal["parallel", "streaming"], mode) + + +def parse_validation_ode_solver( + cfg: dict[str, Any], + *, + sampler_kind: Literal["ode", "sde"], +) -> str | None: + raw = cfg.get("ode_solver") + if raw in (None, ""): + return None + if sampler_kind != "ode": + raise ValueError( + "training.validation.ode_solver is only valid when " + "training.validation.sampler_kind='ode'" + ) + if not isinstance(raw, str): + raise ValueError( + "training.validation.ode_solver must be a string when set, got " + f"{type(raw).__name__}" + ) + solver = raw.strip().lower() + if solver in {"unipc", "unipc_multistep", "multistep"}: + return "unipc" + if solver in {"euler", "flowmatch", "flowmatch_euler"}: + return "euler" + raise ValueError( + "training.validation.ode_solver must be one of {unipc, euler}, got " + f"{raw!r}" + ) + + +def parse_validation_output_dir(cfg: dict[str, Any]) -> str | None: + raw = cfg.get("output_dir") + if raw is None: + return None + if not isinstance(raw, str): + raise ValueError( + "training.validation.output_dir must be a string when set, got " + f"{type(raw).__name__}" + ) + return raw + + +def parse_validation_num_frames(cfg: dict[str, Any]) -> int | None: + num_frames = get_optional_int(cfg, "num_frames", where="training.validation.num_frames") + if num_frames is not None and num_frames <= 0: + raise ValueError("training.validation.num_frames must be > 0 when set") + return num_frames From f4eb1e6b89965ef9f81a3335b21dff4bf766649e Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 01:00:46 +0000 Subject: [PATCH 144/214] causal dmd config --- ...4steps_causal_teacher_ckpt22000_nocfg.yaml | 121 ++++++++++++++++++ fastvideo/distillation/dispatch.py | 25 ++-- fastvideo/distillation/utils/checkpoint.py | 14 +- 3 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml new file mode 100644 index 000000000..a08e1e61b --- /dev/null +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml @@ -0,0 +1,121 @@ +# WanGame causal DMD2 distillation (40-step teacher -> 4-step student). +# +# - Teacher weights come from a DFSFT causal checkpoint (DCP format). +# - Student/Critic are warmstarted from the same checkpoint (copied from +# checkpoint role `student`). +# - Role `path` still points at a *base* wangame model directory (needed to +# load configs/architectures); `init_from_checkpoint` overwrites weights. + +recipe: + family: wangame + method: dmd2 + +roles: + student: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + variant: causal + init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + teacher: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: false + disable_custom_init_weights: true + variant: causal + init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + init_from_checkpoint_role: student + critic: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + disable_custom_init_weights: true + variant: causal + init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + init_from_checkpoint_role: student + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder). + # Supports comma-separated `path` or `path:N` (repeat count) entries. + data_path: >- + /mnt/weka/home/hao.zhang/kaiqin/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_dmd2_4steps_distill_causal_teacher22000 + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer (student) + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Optimizer (critic / fake-score) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation (method-agnostic knobs) + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_dmd2_4steps_distill_causal_teacher22000 + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 50 + sampling_steps: [4] + sampler_kind: sde + guidance_scale: 1.0 + +default_pipeline_config: + flow_shift: 3 + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + + # Few-step schedule (single source of truth for the method). + # These are "step indices" and will be warped by the (shifted) scheduler. + warp_denoising_step: true + dmd_denoising_steps: [1000,750,500,250] + + # Define CFG "uncond" semantics (operation-centric). + cfg_uncond: + on_missing: error + action: keep + image: keep + text: keep + diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index d14ed6429..7def44d09 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -105,20 +105,25 @@ def build_from_config(cfg: DistillRunConfig) -> tuple[TrainingArgs, DistillMetho model_builder = get_model(str(cfg.recipe.family)) model = model_builder(cfg=cfg) - student_cfg = cfg.roles.get("student") - init_from_checkpoint = None - if student_cfg is not None: - init_from_checkpoint = (student_cfg.extra or {}).get( - "init_from_checkpoint", - None, - ) - if init_from_checkpoint: - from fastvideo.distillation.utils.checkpoint import maybe_warmstart_role_modules + from fastvideo.distillation.utils.checkpoint import maybe_warmstart_role_modules + + for role, role_cfg in cfg.roles.items(): + init_from_checkpoint = (role_cfg.extra or {}).get("init_from_checkpoint", None) + if not init_from_checkpoint: + continue + + checkpoint_role = (role_cfg.extra or {}).get("init_from_checkpoint_role", None) + if checkpoint_role is not None and not isinstance(checkpoint_role, str): + raise ValueError( + f"roles.{role}.init_from_checkpoint_role must be a string when set, " + f"got {type(checkpoint_role).__name__}" + ) maybe_warmstart_role_modules( bundle=model.bundle, - role="student", + role=str(role), init_from_checkpoint=str(init_from_checkpoint), + checkpoint_role=str(checkpoint_role) if checkpoint_role else None, ) method_cls = get_method(str(cfg.recipe.method)) diff --git a/fastvideo/distillation/utils/checkpoint.py b/fastvideo/distillation/utils/checkpoint.py index 8bbbe2131..b73656588 100644 --- a/fastvideo/distillation/utils/checkpoint.py +++ b/fastvideo/distillation/utils/checkpoint.py @@ -151,6 +151,7 @@ def maybe_warmstart_role_modules( bundle: RoleManager, role: str, init_from_checkpoint: str | None, + checkpoint_role: str | None = None, ) -> None: """Warmstart model modules for `role` from a Phase 2/3 DCP checkpoint. @@ -164,6 +165,9 @@ def maybe_warmstart_role_modules( if not init_from_checkpoint: return + if checkpoint_role is None: + checkpoint_role = role + resolved = _resolve_resume_checkpoint( str(init_from_checkpoint), output_dir=str(init_from_checkpoint), @@ -173,25 +177,27 @@ def maybe_warmstart_role_modules( raise FileNotFoundError(f"Missing dcp dir under checkpoint: {dcp_dir}") handle = bundle.role(str(role)) - available_modules = _get_dcp_role_module_names(dcp_dir, role=str(role)) + available_modules = _get_dcp_role_module_names(dcp_dir, role=str(checkpoint_role)) states: dict[str, Any] = {} for module_name, module in handle.modules.items(): if module_name not in available_modules: continue - states[f"roles.{role}.{module_name}"] = ModelWrapper(module) + states[f"roles.{checkpoint_role}.{module_name}"] = ModelWrapper(module) if not states: raise ValueError( f"init_from_checkpoint={resolved} does not contain any saved modules for " - f"role={role!r}. Available modules in checkpoint: {sorted(available_modules)}" + f"checkpoint_role={checkpoint_role!r}. Available modules in checkpoint: " + f"{sorted(available_modules)}" ) if _rank() == 0: logger.info( - "Warmstarting role=%s from checkpoint=%s (modules=%s)", + "Warmstarting role=%s from checkpoint=%s (checkpoint_role=%s, modules=%s)", role, resolved, + checkpoint_role, sorted(states.keys()), ) dcp.load(states, checkpoint_id=str(dcp_dir)) From 07d8a57420e9607c96d123dc7ebbcdcb0f8f06fc Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 01:30:17 +0000 Subject: [PATCH 145/214] 4n8g finetuning --- ...4steps_causal_teacher_ckpt22000_nocfg.yaml | 2 +- ...ngame2.1_i2v_1.3B_bidirectional_4n8g.slurm | 64 +++++++++++++ ...angame2.1_i2v_1.3B_bidirectional_4n8g.yaml | 90 +++++++++++++++++++ fastvideo/distillation/validators/wan.py | 29 +++--- fastvideo/distillation/validators/wangame.py | 57 +++++++----- 5 files changed, 206 insertions(+), 36 deletions(-) create mode 100644 examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.slurm create mode 100644 examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml index a08e1e61b..a752f3eef 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml @@ -59,7 +59,7 @@ training: # Output / steps output_dir: outputs/wangame_dmd2_4steps_distill_causal_teacher22000 - max_train_steps: 4000 + max_train_steps: 400000 seed: 1000 checkpoints_total_limit: 3 diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.slurm b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.slurm new file mode 100644 index 000000000..c1219a171 --- /dev/null +++ b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.slurm @@ -0,0 +1,64 @@ +#!/bin/bash +#SBATCH --job-name=wangame_finetune_bidi +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=slurm_outputs/wangame_finetune_bidi_4n8g/wangame_finetune_bidi_%j.out +#SBATCH --error=slurm_outputs/wangame_finetune_bidi_4n8g/wangame_finetune_bidi_%j.err +#SBATCH --exclusive + +set -euo pipefail + +# ---- Env (mirror legacy WanGame slurm scripts) ---- +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-INIT,NET} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} + +# Triton cache (avoid cross-job collisions; per-node task is OK in practice) +export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/tmp/triton_cache_${SLURM_JOB_ID}_${SLURM_PROCID}} + +# Rendezvous (torchrun) +export MASTER_PORT=${MASTER_PORT:-29501} +nodes=( $(scontrol show hostnames "$SLURM_JOB_NODELIST") ) +export MASTER_ADDR=${MASTER_ADDR:-${nodes[0]}} + +# W&B (recommended to pass WANDB_API_KEY via environment / secret manager) +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-online} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +source ~/conda/miniconda/bin/activate +conda activate alexfv + +CONFIG=${CONFIG:-"examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml"} +OUTPUT_DIR=${OUTPUT_DIR:-"outputs/wangame_finetune_bidi_4n8g/${SLURM_JOB_ID}"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +# 4 nodes × 8 GPUs +NUM_NODES=${SLURM_NNODES} +GPUS_PER_NODE=8 + +srun torchrun \ + --nnodes "$NUM_NODES" \ + --nproc_per_node "$GPUS_PER_NODE" \ + --rdzv_backend c10d \ + --rdzv_endpoint "$MASTER_ADDR:$MASTER_PORT" \ + --node_rank "$SLURM_PROCID" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" \ + --override-output-dir "$OUTPUT_DIR" + diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml new file mode 100644 index 000000000..ffa03048c --- /dev/null +++ b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml @@ -0,0 +1,90 @@ +# WanGame bidirectional finetune (4 nodes × 8 GPUs). +# +# This is a "from scratch" run in the sense that it does not warmstart from any +# DCP checkpoint (no `init_from_checkpoint`). It starts from the base pretrained +# `roles.student.path` weights. + +recipe: + family: wangame + method: finetune + +roles: + student: + family: wangame + path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + trainable: true + variant: bidirectional + +training: + # Distributed (4 nodes × 8 GPUs = 32 ranks) + num_gpus: 32 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 32 + + # Data (parquet dataset folder) + # + # Supports comma-separated `path` or `path:N` (repeat count) entries. + # N=0 means skip (handy to toggle datasets without deleting lines). + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_finetune_bidi_4n8g + max_train_steps: 200000 + seed: 1000 + checkpoints_total_limit: 2 + + # Optimizer + learning_rate: 1.0e-5 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 1.0e-4 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.0 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_finetune_bidirectional_4n8g_from_pretrained + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json + every_steps: 100 + sampling_steps: [40] + sampler_kind: ode + guidance_scale: 1.0 + +default_pipeline_config: + flow_shift: 3 + sampler_kind: ode + +method_config: + attn_kind: dense + diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 5e98cbf5d..8aa09a657 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -99,19 +99,22 @@ def _get_pipeline( # `inference_mode=True`, so we must pass pipeline knobs via kwargs. flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) - self._pipeline = WanPipeline.from_pretrained( - self.training_args.model_path, - inference_mode=True, - sampler_kind=str(sampler_kind), - flow_shift=float(flow_shift) if flow_shift is not None else None, - ode_solver=str(ode_solver) if ode_solver is not None else None, - loaded_modules={"transformer": transformer}, - tp_size=self.training_args.tp_size, - sp_size=self.training_args.sp_size, - num_gpus=self.training_args.num_gpus, - pin_cpu_memory=self.training_args.pin_cpu_memory, - dit_cpu_offload=True, - ) + kwargs: dict[str, Any] = { + "inference_mode": True, + "sampler_kind": str(sampler_kind), + "loaded_modules": {"transformer": transformer}, + "tp_size": self.training_args.tp_size, + "sp_size": self.training_args.sp_size, + "num_gpus": self.training_args.num_gpus, + "pin_cpu_memory": self.training_args.pin_cpu_memory, + "dit_cpu_offload": True, + } + if flow_shift is not None: + kwargs["flow_shift"] = float(flow_shift) + if ode_solver is not None: + kwargs["ode_solver"] = str(ode_solver) + + self._pipeline = WanPipeline.from_pretrained(self.training_args.model_path, **kwargs) self._pipeline_key = key return self._pipeline diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index 4754e72a3..4e9363a81 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -168,18 +168,24 @@ def _get_pipeline( WanGameActionImageToVideoPipeline, ) + kwargs: dict[str, Any] = { + "inference_mode": True, + "sampler_kind": sampler_kind, + "loaded_modules": {"transformer": transformer}, + "tp_size": self.training_args.tp_size, + "sp_size": self.training_args.sp_size, + "num_gpus": self.training_args.num_gpus, + "pin_cpu_memory": self.training_args.pin_cpu_memory, + "dit_cpu_offload": True, + } + if flow_shift is not None: + kwargs["flow_shift"] = float(flow_shift) + if ode_solver is not None: + kwargs["ode_solver"] = str(ode_solver) + self._pipeline = WanGameActionImageToVideoPipeline.from_pretrained( self.training_args.model_path, - inference_mode=True, - flow_shift=float(flow_shift) if flow_shift is not None else None, - sampler_kind=sampler_kind, - ode_solver=str(ode_solver) if ode_solver is not None else None, - loaded_modules={"transformer": transformer}, - tp_size=self.training_args.tp_size, - sp_size=self.training_args.sp_size, - num_gpus=self.training_args.num_gpus, - pin_cpu_memory=self.training_args.pin_cpu_memory, - dit_cpu_offload=True, + **kwargs, ) elif rollout_mode == "streaming": if sampler_kind not in {"ode", "sde"}: @@ -226,21 +232,28 @@ def _get_pipeline( WanGameCausalDMDPipeline, ) - self._pipeline = WanGameCausalDMDPipeline.from_pretrained( - self.training_args.model_path, - inference_mode=True, - flow_shift=float(flow_shift) if flow_shift is not None else None, - sampler_kind=sampler_kind, - ode_solver=str(ode_solver) if ode_solver is not None else None, - loaded_modules={ + kwargs = { + "inference_mode": True, + "flow_shift": float(flow_shift) if flow_shift is not None else None, + "sampler_kind": sampler_kind, + "loaded_modules": { "transformer": transformer, "scheduler": scheduler, }, - tp_size=self.training_args.tp_size, - sp_size=self.training_args.sp_size, - num_gpus=self.training_args.num_gpus, - pin_cpu_memory=self.training_args.pin_cpu_memory, - dit_cpu_offload=True, + "tp_size": self.training_args.tp_size, + "sp_size": self.training_args.sp_size, + "num_gpus": self.training_args.num_gpus, + "pin_cpu_memory": self.training_args.pin_cpu_memory, + "dit_cpu_offload": True, + } + if kwargs["flow_shift"] is None: + kwargs.pop("flow_shift") + if ode_solver is not None: + kwargs["ode_solver"] = str(ode_solver) + + self._pipeline = WanGameCausalDMDPipeline.from_pretrained( + self.training_args.model_path, + **kwargs, ) else: raise ValueError( From ced0ae5bae8834a5803cffbb5c63373e3a74a325 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 01:33:53 +0000 Subject: [PATCH 146/214] wangame causal dmd 4n8g --- ..._causal_teacher_ckpt22000_nocfg_4n8g.slurm | 64 +++++++++ ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 124 ++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm create mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm new file mode 100644 index 000000000..ebde340d0 --- /dev/null +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm @@ -0,0 +1,64 @@ +#!/bin/bash +#SBATCH --job-name=wangame_dmd2_causal +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=slurm_outputs/wangame_dmd2_causal_teacher22000_4n8g/wangame_dmd2_causal_%j.out +#SBATCH --error=slurm_outputs/wangame_dmd2_causal_teacher22000_4n8g/wangame_dmd2_causal_%j.err +#SBATCH --exclusive + +set -euo pipefail + +# ---- Env (mirror legacy WanGame slurm scripts) ---- +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-INIT,NET} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} + +# Triton cache (avoid cross-job collisions; per-node task is OK in practice) +export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/tmp/triton_cache_${SLURM_JOB_ID}_${SLURM_PROCID}} + +# Rendezvous (torchrun) +export MASTER_PORT=${MASTER_PORT:-29501} +nodes=( $(scontrol show hostnames "$SLURM_JOB_NODELIST") ) +export MASTER_ADDR=${MASTER_ADDR:-${nodes[0]}} + +# W&B (recommended to pass WANDB_API_KEY via environment / secret manager) +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-online} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +source ~/conda/miniconda/bin/activate +conda activate alexfv + +CONFIG=${CONFIG:-"examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml"} +OUTPUT_DIR=${OUTPUT_DIR:-"outputs/wangame_dmd2_causal_teacher22000_4n8g/${SLURM_JOB_ID}"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +# 4 nodes × 8 GPUs +NUM_NODES=${SLURM_NNODES} +GPUS_PER_NODE=8 + +srun torchrun \ + --nnodes "$NUM_NODES" \ + --nproc_per_node "$GPUS_PER_NODE" \ + --rdzv_backend c10d \ + --rdzv_endpoint "$MASTER_ADDR:$MASTER_PORT" \ + --node_rank "$SLURM_PROCID" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" \ + --override-output-dir "$OUTPUT_DIR" + diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml new file mode 100644 index 000000000..e268b744f --- /dev/null +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml @@ -0,0 +1,124 @@ +# WanGame causal DMD2 distillation (40-step teacher -> 4-step student). +# +# 4 nodes × 8 GPUs config (32 ranks). +# +# - Teacher weights come from a DFSFT causal checkpoint (DCP format). +# - Student/Critic are warmstarted from the same checkpoint (copied from +# checkpoint role `student`). +# - Role `path` still points at a *base* wangame model directory (needed to +# load configs/architectures); `init_from_checkpoint` overwrites weights. + +recipe: + family: wangame + method: dmd2 + +roles: + student: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + variant: causal + init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + teacher: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: false + disable_custom_init_weights: true + variant: causal + init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + init_from_checkpoint_role: student + critic: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + disable_custom_init_weights: true + variant: causal + init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + init_from_checkpoint_role: student + +training: + # Distributed (4 nodes × 8 GPUs = 32 ranks) + num_gpus: 32 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 32 + + # Data (parquet dataset folder). + # Supports comma-separated `path` or `path:N` (repeat count) entries. + data_path: >- + /mnt/weka/home/hao.zhang/kaiqin/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_dmd2_4steps_distill_causal_teacher22000_4n8g + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer (student) + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Optimizer (critic / fake-score) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation (method-agnostic knobs) + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_dmd2_4steps_distill_causal_teacher22000_4n8g + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 50 + sampling_steps: [4] + sampler_kind: sde + guidance_scale: 1.0 + +default_pipeline_config: + flow_shift: 3 + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + + # Few-step schedule (single source of truth for the method). + # These are "step indices" and will be warped by the (shifted) scheduler. + warp_denoising_step: true + dmd_denoising_steps: [1000,750,500,250] + + # Define CFG "uncond" semantics (operation-centric). + # Here `keep` means uncond == cond (CFG becomes a no-op). + cfg_uncond: + on_missing: error + action: keep + image: keep + text: keep + From c461e6bc32929824d8e6d08e850600efb18bc13b Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 02:45:49 +0000 Subject: [PATCH 147/214] sf init impl --- fastvideo/distillation/dispatch.py | 3 + .../methods/distribution_matching/__init__.py | 5 +- .../distribution_matching/self_forcing.py | 614 ++++++++++++++++++ 3 files changed, 621 insertions(+), 1 deletion(-) create mode 100644 fastvideo/distillation/methods/distribution_matching/self_forcing.py diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index 7def44d09..febf888b3 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -66,6 +66,9 @@ def ensure_builtin_registrations() -> None: from fastvideo.distillation.models import wan as _wan # noqa: F401 from fastvideo.distillation.models import wangame as _wangame # noqa: F401 from fastvideo.distillation.methods.distribution_matching import dmd2 as _dmd2 # noqa: F401 + from fastvideo.distillation.methods.distribution_matching import ( + self_forcing as _self_forcing, # noqa: F401 + ) from fastvideo.distillation.methods.fine_tuning import finetune as _finetune # noqa: F401 from fastvideo.distillation.methods.fine_tuning import dfsft as _dfsft # noqa: F401 diff --git a/fastvideo/distillation/methods/distribution_matching/__init__.py b/fastvideo/distillation/methods/distribution_matching/__init__.py index 061614127..1f5c53781 100644 --- a/fastvideo/distillation/methods/distribution_matching/__init__.py +++ b/fastvideo/distillation/methods/distribution_matching/__init__.py @@ -1,8 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method +from fastvideo.distillation.methods.distribution_matching.self_forcing import ( + SelfForcingMethod, +) __all__ = [ "DMD2Method", + "SelfForcingMethod", ] - diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/distillation/methods/distribution_matching/self_forcing.py new file mode 100644 index 000000000..b34e8cfde --- /dev/null +++ b/fastvideo/distillation/methods/distribution_matching/self_forcing.py @@ -0,0 +1,614 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Self-Forcing distillation method (algorithm layer). + +Self-Forcing is a distribution-matching variant that changes *how the student +sample (gen_data) is obtained*: instead of a single-step rollout, we perform a +multi-step simulate rollout and only keep gradients at a stochastic "exit" +denoising step per temporal block (chunk). + +This implementation follows the structure of: +- FastGen: `fastgen/methods/distribution_matching/self_forcing.py` +- FastVideo legacy: `fastvideo/training/self_forcing_distillation_pipeline.py` + +Config keys used (YAML schema-v2): +- `recipe.method`: must be `"self_forcing"` for this method. +- `roles`: requires `student`, `teacher`, `critic` (trainable critic). +- `method_config`: + - `rollout_mode`: must be `"simulate"` for self-forcing. + - `dmd_denoising_steps` (list[int]): the candidate denoising steps. + - `cfg_uncond` (optional): same as DMD2. + - self-forcing rollout knobs: + - `chunk_size` (int, optional; default=3) + - `student_sample_type` (`"sde"` or `"ode"`, optional; default="sde") + - `same_step_across_blocks` (bool, optional; default=false) + - `last_step_only` (bool, optional; default=false) + - `context_noise` (float, optional; default=0.0) + - `enable_gradient_in_rollout` (bool, optional; default=true) + - `start_gradient_frame` (int, optional; default=0) +- `training` (selected fields used for optim/schedule): same as DMD2. +- `training.validation.*`: parsed by base DMD2 method; executed via validator. + +Notes / limitations: +- This method uses `SelfForcingFlowMatchScheduler` for simulate-rollout + (matching legacy behavior). Teacher/critic losses also interpret noise under + the same scheduler for consistency. +- Context/no-grad KV cache updates are model-family specific in legacy. The new + framework keeps method/model decoupled, so this method implements a cache-free + rollout using the model plugin's `predict_noise` primitive. +""" + +from __future__ import annotations + +from typing import Any, Literal, TYPE_CHECKING + +import torch +import torch.distributed as dist + +from fastvideo.distillation.dispatch import register_method +from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method +from fastvideo.distillation.utils.config import get_optional_float, get_optional_int +from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( + SelfForcingFlowMatchScheduler, +) +from fastvideo.models.utils import pred_noise_to_pred_video + +if TYPE_CHECKING: + from fastvideo.pipelines import TrainingBatch + + +def _require_bool(raw: Any, *, where: str) -> bool: + if isinstance(raw, bool): + return raw + raise ValueError(f"Expected bool at {where}, got {type(raw).__name__}") + + +def _require_str(raw: Any, *, where: str) -> str: + if not isinstance(raw, str) or not raw.strip(): + raise ValueError(f"Expected non-empty string at {where}") + return raw + + +@register_method("self_forcing") +class SelfForcingMethod(DMD2Method): + """Self-Forcing DMD2 (distribution matching) method. + + Inherits DMD2 losses/update policy, but replaces the student rollout + procedure with a self-forcing rollout. + """ + + def __init__( + self, + *, + bundle: Any, + model: Any, + method_config: dict[str, Any] | None = None, + validation_config: dict[str, Any] | None = None, + validator: Any | None = None, + ) -> None: + super().__init__( + bundle=bundle, + model=model, + method_config=method_config, + validation_config=validation_config, + validator=validator, + ) + + if self._rollout_mode != "simulate": + raise ValueError("SelfForcingMethod only supports method_config.rollout_mode='simulate'") + + cfg = self.method_config + + chunk_size = get_optional_int( + cfg, + "chunk_size", + where="method_config.chunk_size", + ) + if chunk_size is None: + chunk_size = 3 + if chunk_size <= 0: + raise ValueError( + "method_config.chunk_size must be a positive integer, got " + f"{chunk_size}" + ) + self._chunk_size = int(chunk_size) + + sample_type_raw = cfg.get("student_sample_type", "sde") + sample_type = _require_str(sample_type_raw, where="method_config.student_sample_type") + sample_type = sample_type.strip().lower() + if sample_type not in {"sde", "ode"}: + raise ValueError( + "method_config.student_sample_type must be one of {sde, ode}, got " + f"{sample_type_raw!r}" + ) + self._student_sample_type: Literal["sde", "ode"] = sample_type # type: ignore[assignment] + + same_step_raw = cfg.get("same_step_across_blocks", False) + if same_step_raw is None: + same_step_raw = False + self._same_step_across_blocks = _require_bool( + same_step_raw, where="method_config.same_step_across_blocks" + ) + + last_step_raw = cfg.get("last_step_only", False) + if last_step_raw is None: + last_step_raw = False + self._last_step_only = _require_bool( + last_step_raw, where="method_config.last_step_only" + ) + + context_noise = get_optional_float( + cfg, + "context_noise", + where="method_config.context_noise", + ) + if context_noise is None: + context_noise = 0.0 + if context_noise < 0.0: + raise ValueError( + "method_config.context_noise must be >= 0, got " + f"{context_noise}" + ) + self._context_noise = float(context_noise) + + enable_grad_raw = cfg.get("enable_gradient_in_rollout", True) + if enable_grad_raw is None: + enable_grad_raw = True + self._enable_gradient_in_rollout = _require_bool( + enable_grad_raw, where="method_config.enable_gradient_in_rollout" + ) + + start_grad_frame = get_optional_int( + cfg, + "start_gradient_frame", + where="method_config.start_gradient_frame", + ) + if start_grad_frame is None: + start_grad_frame = 0 + if start_grad_frame < 0: + raise ValueError( + "method_config.start_gradient_frame must be >= 0, got " + f"{start_grad_frame}" + ) + self._start_gradient_frame = int(start_grad_frame) + + # Legacy Self-Forcing uses a dedicated scheduler (different sigma_min). + shift = float(getattr(self.training_args.pipeline_config, "flow_shift", 0.0) or 0.0) + self._sf_scheduler = SelfForcingFlowMatchScheduler( + num_inference_steps=1000, + num_train_timesteps=int(self.model.num_train_timesteps), + shift=shift, + sigma_min=0.0, + extra_one_step=True, + training=True, + ) + + # Cache for warped denoising step list (device specific). + self._sf_denoising_step_list: torch.Tensor | None = None + + def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: + if ( + self._sf_denoising_step_list is not None + and self._sf_denoising_step_list.device == device + ): + return self._sf_denoising_step_list + + raw = self.method_config.get("dmd_denoising_steps", None) + if not isinstance(raw, list) or not raw: + raise ValueError("method_config.dmd_denoising_steps must be set for self_forcing") + steps = torch.tensor([int(s) for s in raw], dtype=torch.long, device=device) + + warp = self.method_config.get("warp_denoising_step", None) + if warp is None: + warp = getattr(self.training_args, "warp_denoising_step", False) + if bool(warp): + timesteps = torch.cat( + ( + self._sf_scheduler.timesteps.to("cpu"), + torch.tensor([0], dtype=torch.float32), + ) + ).to(device) + steps = timesteps[int(self.model.num_train_timesteps) - steps] + + self._sf_denoising_step_list = steps + return steps + + def _predict_x0_with_scheduler( + self, + handle: Any, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + attn_kind: Literal["dense", "vsa"], + ) -> torch.Tensor: + pred_noise = self.model.predict_noise( + handle, + noisy_latents, + timestep, + batch, + conditional=conditional, + cfg_uncond=self._cfg_uncond, + attn_kind=attn_kind, + ) + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=timestep, + scheduler=self._sf_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + def _sf_add_noise( + self, + clean_latents: torch.Tensor, + noise: torch.Tensor, + timestep: torch.Tensor, + ) -> torch.Tensor: + b, t = clean_latents.shape[:2] + noisy = self._sf_scheduler.add_noise( + clean_latents.flatten(0, 1), + noise.flatten(0, 1), + timestep, + ).unflatten(0, (b, t)) + return noisy + + def _timestep_to_sigma(self, timestep: torch.Tensor) -> torch.Tensor: + sigmas = self._sf_scheduler.sigmas.to(device=timestep.device, dtype=torch.float32) + timesteps = self._sf_scheduler.timesteps.to(device=timestep.device, dtype=torch.float32) + t = timestep.to(device=timestep.device, dtype=torch.float32) + if t.ndim == 2: + t = t.flatten(0, 1) + elif t.ndim == 1 and t.numel() == 1: + t = t.expand(1) + elif t.ndim != 1: + raise ValueError(f"Invalid timestep shape: {tuple(timestep.shape)}") + idx = torch.argmin((timesteps.unsqueeze(0) - t.unsqueeze(1)).abs(), dim=1) + return sigmas[idx] + + def _sample_exit_indices( + self, + *, + num_blocks: int, + num_steps: int, + device: torch.device, + ) -> list[int]: + if num_blocks <= 0: + return [] + if num_steps <= 0: + raise ValueError("num_steps must be positive") + + shape = (1,) if self._same_step_across_blocks else (num_blocks,) + + if not dist.is_initialized() or dist.get_rank() == 0: + if self._last_step_only: + indices = torch.full( + shape, + num_steps - 1, + dtype=torch.long, + device=device, + ) + else: + indices = torch.randint( + low=0, + high=num_steps, + size=shape, + device=device, + ) + else: + indices = torch.empty(shape, dtype=torch.long, device=device) + + if dist.is_initialized(): + dist.broadcast(indices, src=0) + + if self._same_step_across_blocks: + return [int(indices.item()) for _ in range(num_blocks)] + return [int(i) for i in indices.tolist()] + + def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: + latents = batch.latents + if latents is None: + raise RuntimeError("TrainingBatch.latents is required for self-forcing rollout") + if latents.ndim != 5: + raise ValueError( + "TrainingBatch.latents must be [B, T, C, H, W], got " + f"shape={tuple(latents.shape)}" + ) + + device = latents.device + dtype = latents.dtype + batch_size = int(latents.shape[0]) + num_frames = int(latents.shape[1]) + + denoising_steps = self._get_denoising_step_list(device) + num_steps = int(denoising_steps.numel()) + + noise_full = torch.randn_like(latents, device=device, dtype=dtype) + + chunk = int(self._chunk_size) + if chunk <= 0: + raise ValueError("chunk_size must be positive") + + remaining = num_frames % chunk + num_blocks = num_frames // chunk + if num_blocks == 0: + num_blocks = 1 + remaining = num_frames + + exit_indices = self._sample_exit_indices( + num_blocks=num_blocks, + num_steps=num_steps, + device=device, + ) + + context_latents: list[torch.Tensor] = [] + context_timesteps: list[torch.Tensor] = [] + denoised_blocks: list[torch.Tensor] = [] + + for block_idx in range(num_blocks): + if block_idx == 0: + start = 0 + end = remaining + chunk if remaining else chunk + else: + start = remaining + block_idx * chunk + end = remaining + (block_idx + 1) * chunk + start = int(start) + end = int(min(end, num_frames)) + if start >= end: + break + + noisy_block = noise_full[:, start:end] + exit_idx = int(exit_indices[block_idx]) + + context_len = start + future_latents = noise_full[:, end:] + future_len = int(future_latents.shape[1]) + + for step_idx, current_timestep in enumerate(denoising_steps): + exit_flag = step_idx == exit_idx + + timestep_block = current_timestep * torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ) + + if context_len > 0: + context_lat = torch.cat(context_latents, dim=1) + context_t = torch.cat(context_timesteps, dim=1) + if int(context_lat.shape[1]) != context_len: + raise RuntimeError("context_latents length mismatch") + else: + context_lat = None + context_t = None + + if context_lat is None: + model_latents = torch.cat([noisy_block, future_latents], dim=1) + timestep_full = torch.cat( + [ + timestep_block, + current_timestep + * torch.ones( + (batch_size, future_len), + device=device, + dtype=torch.float32, + ), + ], + dim=1, + ) + else: + model_latents = torch.cat([context_lat, noisy_block, future_latents], dim=1) + timestep_full = torch.cat( + [ + context_t, + timestep_block, + current_timestep + * torch.ones( + (batch_size, future_len), + device=device, + dtype=torch.float32, + ), + ], + dim=1, + ) + + enable_grad = ( + bool(with_grad) + and bool(self._enable_gradient_in_rollout) + and torch.is_grad_enabled() + and start >= int(self._start_gradient_frame) + ) + + if not exit_flag: + with torch.no_grad(): + pred_x0_full = self._predict_x0_with_scheduler( + self.student, + model_latents, + timestep_full, + batch, + conditional=True, + attn_kind="vsa", + ) + pred_x0_chunk = pred_x0_full[:, context_len:context_len + (end - start)] + if step_idx + 1 >= num_steps: + break + next_timestep = denoising_steps[step_idx + 1] + if self._student_sample_type == "sde": + noisy_block = self._sf_add_noise( + pred_x0_chunk, + torch.randn_like(pred_x0_chunk), + next_timestep + * torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ), + ) + else: + sigma_cur = self._timestep_to_sigma(timestep_block).view( + batch_size, end - start, 1, 1, 1 + ) + sigma_next = self._timestep_to_sigma( + next_timestep + * torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ) + ).view(batch_size, end - start, 1, 1, 1) + eps = (noisy_block - (1 - sigma_cur) * pred_x0_chunk) / sigma_cur.clamp_min( + 1e-8 + ) + noisy_block = (1 - sigma_next) * pred_x0_chunk + sigma_next * eps + continue + + with torch.set_grad_enabled(enable_grad): + pred_x0_full = self._predict_x0_with_scheduler( + self.student, + model_latents, + timestep_full, + batch, + conditional=True, + attn_kind="vsa", + ) + pred_x0_chunk = pred_x0_full[:, context_len:context_len + (end - start)] + break + + denoised_blocks.append(pred_x0_chunk) + + # Update context frames (cache-free): store context-noised frames + timesteps. + if self._context_noise > 0.0: + context_timestep = torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ) * float(self._context_noise) + with torch.no_grad(): + context_latents.append( + self._sf_add_noise( + pred_x0_chunk.detach(), + torch.randn_like(pred_x0_chunk), + context_timestep, + ) + ) + context_timesteps.append(context_timestep) + else: + context_latents.append(pred_x0_chunk.detach()) + context_timesteps.append( + torch.zeros( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ) + ) + + if not denoised_blocks: + raise RuntimeError("Self-forcing rollout produced no blocks") + + return torch.cat(denoised_blocks, dim=1) + + def _critic_flow_matching_loss( + self, batch: Any + ) -> tuple[torch.Tensor, Any, dict[str, Any]]: + with torch.no_grad(): + generator_pred_x0 = self._student_rollout(batch, with_grad=False) + + device = generator_pred_x0.device + fake_score_timestep = torch.randint( + 0, + int(self.model.num_train_timesteps), + [1], + device=device, + dtype=torch.long, + ) + fake_score_timestep = self.model.shift_and_clamp_timestep(fake_score_timestep) + + noise = torch.randn( + generator_pred_x0.shape, + device=device, + dtype=generator_pred_x0.dtype, + ) + noisy_x0 = self._sf_add_noise(generator_pred_x0, noise, fake_score_timestep) + + pred_noise = self.model.predict_noise( + self.critic, + noisy_x0, + fake_score_timestep, + batch, + conditional=True, + cfg_uncond=self._cfg_uncond, + attn_kind="dense", + ) + target = noise - generator_pred_x0 + flow_matching_loss = torch.mean((pred_noise - target) ** 2) + + batch.fake_score_latent_vis_dict = { + "generator_pred_video": generator_pred_x0, + "fake_score_timestep": fake_score_timestep, + } + outputs = {"fake_score_latent_vis_dict": batch.fake_score_latent_vis_dict} + return flow_matching_loss, (batch.timesteps, batch.attn_metadata), outputs + + def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor: + guidance_scale = get_optional_float( + self.method_config, + "real_score_guidance_scale", + where="method_config.real_score_guidance_scale", + ) + if guidance_scale is None: + guidance_scale = float(getattr(self.training_args, "real_score_guidance_scale", 1.0)) + device = generator_pred_x0.device + + with torch.no_grad(): + timestep = torch.randint( + 0, + int(self.model.num_train_timesteps), + [1], + device=device, + dtype=torch.long, + ) + timestep = self.model.shift_and_clamp_timestep(timestep) + + noise = torch.randn( + generator_pred_x0.shape, + device=device, + dtype=generator_pred_x0.dtype, + ) + noisy_latents = self._sf_add_noise(generator_pred_x0, noise, timestep) + + faker_x0 = self._predict_x0_with_scheduler( + self.critic, + noisy_latents, + timestep, + batch, + conditional=True, + attn_kind="dense", + ) + real_cond_x0 = self._predict_x0_with_scheduler( + self.teacher, + noisy_latents, + timestep, + batch, + conditional=True, + attn_kind="dense", + ) + real_uncond_x0 = self._predict_x0_with_scheduler( + self.teacher, + noisy_latents, + timestep, + batch, + conditional=False, + attn_kind="dense", + ) + real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale + + denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() + grad = (faker_x0 - real_cfg_x0) / denom + grad = torch.nan_to_num(grad) + + loss = 0.5 * torch.mean( + (generator_pred_x0.float() - (generator_pred_x0.float() - grad.float()).detach()) + ** 2 + ) + return loss From 27196501c3cf5426096476217d8d27c1c097ce0c Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 03:36:51 +0000 Subject: [PATCH 148/214] designing causal rollout stuff --- dev/phase_causal.md | 144 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 dev/phase_causal.md diff --git a/dev/phase_causal.md b/dev/phase_causal.md new file mode 100644 index 000000000..cd5547bd6 --- /dev/null +++ b/dev/phase_causal.md @@ -0,0 +1,144 @@ +# Phase: Causal（对齐 FastGen 的 causal / cache 设计） + +> 目标：让 **causal/streaming** 能力像 FastGen 一样成为“可选增强”，只落在 causal 变体上； +> 同时把 **预处理器（preprocessors：VAE / text encoder / image encoder / …）** 的管理收敛到 +> **student**，避免 teacher/critic 重复加载与显存浪费，并减少跨 role 的耦合。 + +## 1. 背景与动机 + +在 FastVideo 的 distillation 框架里，我们会遇到典型组合： + +- teacher：bidirectional（质量高、但不适合 streaming rollout） +- student：causal（需要 KV-cache / streaming rollout） + +如果把 KV-cache 相关的接口（`kv_cache` / `crossattn_cache` / `current_start` / …）直接塞进 +通用的 `predict_noise/predict_x0`，会导致： + +- **所有模型**都被迫兼容 causal 的 cache 语义（污染接口）； +- method 代码不得不理解具体 cache 结构（污染算法层）； +- 加一个新模型/新任务时，很容易“接口爆炸”。 + +FastGen 的经验是： + +- **只有 causal network** 需要并实现 cache（`store_kv=True` + internal caches）； +- **预处理器只在 student 侧管理/按需初始化**（VAE/text encoder 主要用于数据编码与可视化 decode）， + teacher forward 通常直接吃 latent，不需要再加载一份 VAE。 + +本 phase 将这两条经验收敛到我们的 distillation 框架中。 + +## 2. 关键原则 + +### 2.1 只有 student 管 preprocessors + +定义： + +- preprocessors = `vae` / `text_encoder` / `image_encoder` / （未来可能的）action encoder 等。 + +策略： + +- **只在 student 对应的 model plugin 中初始化/持有** preprocessors（必要时 lazy init）。 +- teacher/critic role **不再重复加载** preprocessors。 +- 训练/蒸馏计算以 latent 为主，validation/wandb 可视化 decode 时再用 student 的 VAE。 + +收益： + +- 显存更友好（尤其是多 role / 大模型 teacher）。 +- 语义更清晰：teacher 的职责是提供 score/target，而不是承担数据编解码。 + +### 2.2 causal 能力只落在 *_causal 变体 + +将“streaming rollout / KV-cache / blockwise causal mask / context noise”等能力视为 +**causal 变体的 runtime primitive**，而不是所有模型的通用能力。 + +对应的 method（例如 self-forcing/DFSFT/causal validator）如果需要 cache， +应当只依赖 causal 变体提供的可选接口（或不透明 `cache_state`），而非强制所有模型支持。 + +#### CausalModelBase：不污染 ModelBase 的接口 + +为了避免把 `kv_cache/crossattn_cache/...` 这类实现细节塞进通用的 +`ModelBase.predict_noise/predict_x0`，我们引入一个 causal 专属基类： + +- `ModelBase`：保持通用 primitives（`predict_noise/predict_x0/add_noise/...`），**不出现 cache**。 +- `CausalModelBase(ModelBase)`：仅定义 causal/streaming 所需的最小契约，形态参考 FastGen。 + +FastGen 的经验是：cache 是 **causal network 内部可变状态**，method 不传递 cache 张量本体。 +method 只通过 “清空缓存 + 运行 forward（可选 store_kv）” 来驱动 streaming rollout。 + +因此在我们的 `CausalModelBase` 中更优雅的 API 组合是： + +- `clear_caches(handle, cache_tag=...)`：开始新 rollout / 新 validation request 前清空。 +- `predict_noise_streaming(..., cache_tag=..., store_kv=..., cur_start_frame=...)`： + 在 causal 路径中通过 `store_kv` 控制是否写入历史 cache。 + +而不是设计成： + +- `predict_noise_kvcache(kv_cache=...)`：这会迫使 method 管理 cache 生命周期与结构（污染算法层）， + 且最终仍需要额外的 reset/init API。 + +## 3. 目录层级拆分（models） + +将 `fastvideo/distillation/models/*` 拆成更清晰的层级结构，按 “family + variant” 组织： + +```text +fastvideo/distillation/models/ + base.py # ModelBase：方法无关的 operation-centric primitives + wan/ + __init__.py + wan.py # Wan（T2V）bidirectional primitives + wangame/ + __init__.py + wangame.py # WanGame bidirectional primitives（不含 cache） + wangame_causal.py # WanGame causal primitives（包含 streaming/cache） +``` + +注： + +- `wangame_causal.py` 的职责是提供 “causal 专属的 predict/rollout primitive”，例如： + - `clear_caches()`（或 `clear_*`）等 cache 生命周期管理； + - `store_kv` / cache reset / cache update 的封装（cache **由 causal 变体内部持有**）； + - streaming rollout 所需的 block/chunk 约束。 +- `wangame.py` 保持纯 bidirectional（不引入 cache 语义）。 + +## 4. 运行时组织（role → 独立网络实例） + +对齐 FastGen：**每个 role 持有自己的 transformer 实例**（student/teacher/critic 可以是不同类/不同权重）， +但 preprocessors 只由 student 管理。 + +落地方式（保持我们现有 RoleManager/RoleHandle 架构）： + +- `RoleHandle.modules["transformer"]`：每个 role 独立的 denoiser/transformer +- `ModelBase`（或 family-specific model plugin）： + - 统一持有 student preprocessors（VAE/text encoder…） + - 统一实现 batch→forward primitives 的规范化 + - 根据 role 的 variant 选择对应的 bidi/causal 实现路径（必要时由 `roles..extra.variant` 决定） + +## 5. 风险点与验证 + +### 5.1 latent 语义一致性 + +该设计隐含前提是：teacher/student/critic 共享同一 latent 语义（通常意味着同一 VAE 语义）。 +如果未来要支持跨 family（例如 teacher=SDXL, student=Wan），需要额外的对齐层（暂不在本 phase 处理）。 + +### 5.2 cache 生命周期与 no-grad 语义 + +causal cache 的更新通常应在 `torch.no_grad()` 下进行，以避免历史 cache 引入梯度/爆显存。 +需要在实现阶段明确： + +- cache 初始化/重置的时机（每个 rollout / 每个 validation request） +- cache 的 dtype/device 与 FSDP/activation checkpoint 的交互 + +### 5.3 验证策略 + +最小验证集合： + +- bidirectional Wan/WanGame：原有 DMD2/finetune 训练与 validator 不回归； +- causal WanGame：streaming rollout 能跑通（block mask 正确、cache 正确更新）； +- 多 role：teacher=bidi + student=causal 能正确构建并 forward。 + +## 6. TODO（实施时的文件清单） + +- [ ] `fastvideo/distillation/models/` 目录结构调整（新增子目录、移动文件、更新 imports） +- [ ] preprocessors 收敛到 student：移除 teacher/critic 侧的 preprocessor 初始化与依赖 +- [ ] `wangame_causal.py`：封装 cache/streaming primitives（仅 causal 需要） +- [ ] 更新 `dispatch.py` / `register_model` 的 import 路径（保持注册行为不变） +- [ ] 更新必要的 doc（只更新本 phase 相关文档） From 0b924e7b2e920b887fda442ba52ec0c2a7cbd7cb Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 05:29:42 +0000 Subject: [PATCH 149/214] causal for self forcing --- fastvideo/distillation/models/base.py | 53 +- fastvideo/distillation/models/wan/__init__.py | 9 + .../distillation/models/{ => wan}/wan.py | 0 .../distillation/models/wangame/__init__.py | 40 ++ .../models/{ => wangame}/wangame.py | 134 +++-- .../models/wangame/wangame_causal.py | 497 ++++++++++++++++++ 6 files changed, 663 insertions(+), 70 deletions(-) create mode 100644 fastvideo/distillation/models/wan/__init__.py rename fastvideo/distillation/models/{ => wan}/wan.py (100%) create mode 100644 fastvideo/distillation/models/wangame/__init__.py rename fastvideo/distillation/models/{ => wangame}/wangame.py (91%) create mode 100644 fastvideo/distillation/models/wangame/wangame_causal.py diff --git a/fastvideo/distillation/models/base.py b/fastvideo/distillation/models/base.py index a31e2d94c..fec1e589f 100644 --- a/fastvideo/distillation/models/base.py +++ b/fastvideo/distillation/models/base.py @@ -22,7 +22,8 @@ class ModelBase(ABC): - The model plugin implements how to run those roles against FastVideo pipelines, forward-context requirements, and batch normalization quirks. - Implementations typically live next to the model plugin (e.g. `models/wan.py`) + Implementations typically live next to the model plugin + (e.g. `models/wan/wan.py`) rather than in a global adapter registry. """ @@ -100,3 +101,53 @@ def predict_x0( @abstractmethod def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: """Backward hook that may restore forward-context for checkpointed modules.""" + + +class CausalModelBase(ModelBase): + """Optional extension for causal / streaming model plugins. + + This mirrors the FastGen design choice that *only* causal networks expose a + cache lifecycle API, while non-causal models stay on a clean `ModelBase` + interface. + + Important: methods should not pass KV-cache tensors around. Cache state is + internal to the causal model plugin and keyed by `(role handle, cache_tag)`. + """ + + @abstractmethod + def clear_caches(self, handle: RoleHandle, *, cache_tag: str = "pos") -> None: + """Clear internal caches for a role before starting a new rollout.""" + + @abstractmethod + def predict_noise_streaming( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cache_tag: str = "pos", + store_kv: bool = False, + cur_start_frame: int = 0, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor | None: + """Streaming predict-noise primitive that may update internal caches.""" + + @abstractmethod + def predict_x0_streaming( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cache_tag: str = "pos", + store_kv: bool = False, + cur_start_frame: int = 0, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor | None: + """Streaming predict-x0 primitive that may update internal caches.""" diff --git a/fastvideo/distillation/models/wan/__init__.py b/fastvideo/distillation/models/wan/__init__.py new file mode 100644 index 000000000..c9cc3e811 --- /dev/null +++ b/fastvideo/distillation/models/wan/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Wan model plugin package. + +Importing this package registers the `recipe.family: wan` model builder. +""" + +from fastvideo.distillation.models.wan.wan import WanModel as WanModel # noqa: F401 + diff --git a/fastvideo/distillation/models/wan.py b/fastvideo/distillation/models/wan/wan.py similarity index 100% rename from fastvideo/distillation/models/wan.py rename to fastvideo/distillation/models/wan/wan.py diff --git a/fastvideo/distillation/models/wangame/__init__.py b/fastvideo/distillation/models/wangame/__init__.py new file mode 100644 index 000000000..242628dca --- /dev/null +++ b/fastvideo/distillation/models/wangame/__init__.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""WanGame model plugin package. + +This package registers the `recipe.family: wangame` model builder. + +We support both bidirectional and causal transformers: +- If any role declares `roles..variant: causal`, we build the causal-capable + model plugin (which implements `CausalModelBase`). +- Otherwise we build the bidirectional-only model plugin. +""" + +from __future__ import annotations + +from fastvideo.distillation.dispatch import register_model +from fastvideo.distillation.utils.config import DistillRunConfig + + +@register_model("wangame") +def _build_wangame_model(*, cfg: DistillRunConfig): + wants_causal = False + for role, role_spec in cfg.roles.items(): + variant = (role_spec.extra or {}).get("variant", None) + if variant is None: + continue + if str(variant).strip().lower() == "causal": + wants_causal = True + break + + if wants_causal: + from fastvideo.distillation.models.wangame.wangame_causal import ( + WanGameCausalModel, + ) + + return WanGameCausalModel(cfg=cfg) + + from fastvideo.distillation.models.wangame.wangame import WanGameModel + + return WanGameModel(cfg=cfg) + diff --git a/fastvideo/distillation/models/wangame.py b/fastvideo/distillation/models/wangame/wangame.py similarity index 91% rename from fastvideo/distillation/models/wangame.py rename to fastvideo/distillation/models/wangame/wangame.py index 5d155c075..5c10f1057 100644 --- a/fastvideo/distillation/models/wangame.py +++ b/fastvideo/distillation/models/wangame/wangame.py @@ -1,12 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 -"""WanGame model plugin (components + runtime adapter). +"""WanGame bidirectional model plugin (components + runtime adapter). Config keys used (YAML schema-v2): - `recipe.family`: must be `"wangame"` for this plugin. - `roles.`: - `family`, `path`, `trainable`, `disable_custom_init_weights` - - extra: `variant` (optional; `"bidirectional"`/`"causal"`) + - extra: `variant` is **not used** by this bidirectional-only plugin. - `training` (selected fields): - `seed`, `data_path`, `model_path` - `num_height`, `num_width`, `num_latent_t` @@ -23,6 +23,7 @@ from __future__ import annotations import copy +from collections.abc import Callable from typing import Any, Literal import torch @@ -48,7 +49,6 @@ ) from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed -from fastvideo.distillation.dispatch import register_model from fastvideo.distillation.models.base import ModelBase from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.distillation.utils.config import DistillRunConfig @@ -66,7 +66,63 @@ VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] -@register_model("wangame") +def _build_wangame_role_handles( + *, + roles_cfg: dict[str, Any], + training_args: Any, + transformer_cls_name_for_role: Callable[[str, Any], str], +) -> dict[str, RoleHandle]: + role_handles: dict[str, RoleHandle] = {} + for role, role_spec in roles_cfg.items(): + if role_spec.family != "wangame": + raise ValueError( + "Wangame model plugin only supports roles with family='wangame'; " + f"got {role}={role_spec.family!r}" + ) + + transformer_cls_name = transformer_cls_name_for_role(str(role), role_spec) + disable_custom_init_weights = bool(getattr(role_spec, "disable_custom_init_weights", False)) + transformer = load_module_from_path( + model_path=role_spec.path, + module_type="transformer", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + override_transformer_cls_name=transformer_cls_name, + ) + modules: dict[str, torch.nn.Module] = {"transformer": transformer} + + # Optional MoE support: load transformer_2 if present in the model. + try: + transformer_2 = load_module_from_path( + model_path=role_spec.path, + module_type="transformer_2", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + ) + except ValueError: + transformer_2 = None + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + + for name, module in list(modules.items()): + module = apply_trainable(module, trainable=bool(role_spec.trainable)) + if role_spec.trainable and getattr(training_args, "enable_gradient_checkpointing_type", None): + module = apply_activation_checkpointing( + module, + checkpointing_type=training_args.enable_gradient_checkpointing_type, + ) + modules[name] = module + + role_handles[str(role)] = RoleHandle( + modules=modules, + optimizers={}, + lr_schedulers={}, + trainable=bool(role_spec.trainable), + ) + + return role_handles + + class WanGameModel(ModelBase): """WanGame model plugin: loads roles + shared components and exposes runtime primitives. @@ -99,71 +155,11 @@ def __init__( shift=float(training_args.pipeline_config.flow_shift or 0.0) ) - role_handles: dict[str, RoleHandle] = {} - for role, role_spec in roles_cfg.items(): - if role_spec.family != "wangame": - raise ValueError( - "Wangame model plugin only supports roles with family='wangame'; " - f"got {role}={role_spec.family!r}" - ) - - variant_raw = (role_spec.extra or {}).get("variant", None) - if variant_raw is None or variant_raw == "": - transformer_cls_name = "WanGameActionTransformer3DModel" - else: - variant = str(variant_raw).strip().lower() - if variant in {"bidirectional", "bidi"}: - transformer_cls_name = "WanGameActionTransformer3DModel" - elif variant == "causal": - transformer_cls_name = "CausalWanGameActionTransformer3DModel" - else: - raise ValueError( - f"Unknown roles.{role}.variant for wangame: " - f"{variant_raw!r}. Expected 'causal' or 'bidirectional'." - ) - - disable_custom_init_weights = bool( - getattr(role_spec, "disable_custom_init_weights", False) - ) - transformer = load_module_from_path( - model_path=role_spec.path, - module_type="transformer", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - override_transformer_cls_name=transformer_cls_name, - ) - modules: dict[str, torch.nn.Module] = {"transformer": transformer} - - # Optional MoE support: load transformer_2 if present in the model. - try: - transformer_2 = load_module_from_path( - model_path=role_spec.path, - module_type="transformer_2", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - ) - except ValueError: - transformer_2 = None - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 - - for name, module in list(modules.items()): - module = apply_trainable(module, trainable=bool(role_spec.trainable)) - if role_spec.trainable and getattr( - training_args, "enable_gradient_checkpointing_type", None - ): - module = apply_activation_checkpointing( - module, - checkpointing_type=training_args.enable_gradient_checkpointing_type, - ) - modules[name] = module - - role_handles[role] = RoleHandle( - modules=modules, - optimizers={}, - lr_schedulers={}, - trainable=bool(role_spec.trainable), - ) + role_handles = _build_wangame_role_handles( + roles_cfg=roles_cfg, + training_args=training_args, + transformer_cls_name_for_role=lambda _role, _spec: "WanGameActionTransformer3DModel", + ) self.bundle = RoleManager(roles=role_handles) diff --git a/fastvideo/distillation/models/wangame/wangame_causal.py b/fastvideo/distillation/models/wangame/wangame_causal.py new file mode 100644 index 000000000..dd189250f --- /dev/null +++ b/fastvideo/distillation/models/wangame/wangame_causal.py @@ -0,0 +1,497 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""WanGame causal model plugin (streaming/cache primitives). + +This module provides the *causal* extension for the WanGame model family. + +Key differences vs. `models/wangame/wangame.py`: +- Supports `roles..variant: causal` by loading a causal transformer class. +- Implements `CausalModelBase` APIs (`clear_caches`, `predict_*_streaming`) so + methods can drive streaming rollouts without passing KV-cache tensors around. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Literal + +import torch + +from fastvideo.forward_context import set_forward_context +from fastvideo.models.utils import pred_noise_to_pred_video + +from fastvideo.distillation.models.base import CausalModelBase +from fastvideo.distillation.roles import RoleHandle +from fastvideo.distillation.utils.config import DistillRunConfig + +from fastvideo.distillation.models.wangame.wangame import ( + WanGameModel, + _build_wangame_role_handles, +) + + +@dataclass(slots=True) +class _StreamingCaches: + kv_cache: list[dict[str, Any]] + crossattn_cache: list[dict[str, Any]] | None + frame_seq_length: int + local_attn_size: int + sliding_window_num_frames: int + batch_size: int + dtype: torch.dtype + device: torch.device + + +class WanGameCausalModel(WanGameModel, CausalModelBase): + """WanGame model plugin with optional causal/streaming primitives.""" + + def __init__(self, *, cfg: DistillRunConfig) -> None: + training_args = cfg.training_args + roles_cfg = cfg.roles + + if getattr(training_args, "seed", None) is None: + raise ValueError("training.seed must be set for distillation") + if not getattr(training_args, "data_path", ""): + raise ValueError("training.data_path must be set for distillation") + + # Load shared components (student base path). + vae = self._load_shared_vae(training_args) + noise_scheduler = self._build_noise_scheduler(training_args) + + def _transformer_cls_name_for_role(role: str, role_spec: Any) -> str: + variant_raw = (role_spec.extra or {}).get("variant", None) + if variant_raw is None or str(variant_raw).strip() == "": + return "WanGameActionTransformer3DModel" + + variant = str(variant_raw).strip().lower() + if variant in {"bidirectional", "bidi"}: + return "WanGameActionTransformer3DModel" + if variant == "causal": + return "CausalWanGameActionTransformer3DModel" + raise ValueError( + f"Unknown roles.{role}.variant for wangame: {variant_raw!r}. " + "Expected 'causal' or 'bidirectional'." + ) + + role_handles = _build_wangame_role_handles( + roles_cfg=roles_cfg, + training_args=training_args, + transformer_cls_name_for_role=_transformer_cls_name_for_role, + ) + + # NOTE: re-run the rest of WanGameModel init without rebuilding roles. + self._init_from_built_roles( + cfg=cfg, + role_handles=role_handles, + vae=vae, + noise_scheduler=noise_scheduler, + ) + + self._streaming_caches: dict[tuple[int, str], _StreamingCaches] = {} + + # --- CausalModelBase override: clear_caches --- + def clear_caches(self, handle: RoleHandle, *, cache_tag: str = "pos") -> None: + self._streaming_caches.pop((id(handle), str(cache_tag)), None) + + # --- CausalModelBase override: predict_noise_streaming --- + def predict_noise_streaming( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: Any, + *, + conditional: bool, + cache_tag: str = "pos", + store_kv: bool = False, + cur_start_frame: int = 0, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor | None: + if attn_kind == "dense": + attn_metadata = batch.attn_metadata + elif attn_kind == "vsa": + attn_metadata = batch.attn_metadata_vsa + else: + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + + cache_tag = str(cache_tag) + cur_start_frame = int(cur_start_frame) + if cur_start_frame < 0: + raise ValueError("cur_start_frame must be >= 0") + + # Ensure per-frame timestep shape [B, T] (mirrors pipeline causal inference). + batch_size = int(noisy_latents.shape[0]) + num_frames = int(noisy_latents.shape[1]) + timestep_full = self._ensure_per_frame_timestep( + timestep=timestep, + batch_size=batch_size, + num_frames=num_frames, + device=noisy_latents.device, + ) + + transformer = self._get_transformer(handle, timestep_full) + caches = self._get_or_init_streaming_caches( + handle=handle, + cache_tag=cache_tag, + transformer=transformer, + noisy_latents=noisy_latents, + ) + + frame_seq_length = int(caches.frame_seq_length) + model_kwargs: dict[str, Any] = { + "kv_cache": caches.kv_cache, + "crossattn_cache": caches.crossattn_cache, + "current_start": cur_start_frame * frame_seq_length, + "start_frame": cur_start_frame, + "is_cache": bool(store_kv), + } + + device_type = self.device.type + dtype = noisy_latents.dtype + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, + ): + cond_inputs = self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + ) + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, + timestep_full, + image_embeds=cond_inputs["image_embeds"], + image_latents=cond_inputs["image_latents"], + mask_lat_size=cond_inputs["mask_lat_size"], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs["keyboard_cond"], + ) + + # Override timestep dtype: causal inference expects integer timesteps. + input_kwargs["timestep"] = timestep_full.to( + device=self.device, dtype=torch.long + ) + input_kwargs.update(model_kwargs) + + if store_kv: + with torch.no_grad(): + _ = transformer(**input_kwargs) + return None + + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + return pred_noise + + # --- CausalModelBase override: predict_x0_streaming --- + def predict_x0_streaming( + self, + handle: RoleHandle, + noisy_latents: torch.Tensor, + timestep: torch.Tensor, + batch: Any, + *, + conditional: bool, + cache_tag: str = "pos", + store_kv: bool = False, + cur_start_frame: int = 0, + cfg_uncond: dict[str, Any] | None = None, + attn_kind: Literal["dense", "vsa"] = "dense", + ) -> torch.Tensor | None: + pred_noise = self.predict_noise_streaming( + handle, + noisy_latents, + timestep, + batch, + conditional=conditional, + cache_tag=cache_tag, + store_kv=store_kv, + cur_start_frame=cur_start_frame, + cfg_uncond=cfg_uncond, + attn_kind=attn_kind, + ) + if pred_noise is None: + return None + + pred_x0 = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_latents.flatten(0, 1), + timestep=self.shift_and_clamp_timestep( + self._ensure_per_frame_timestep( + timestep=timestep, + batch_size=int(noisy_latents.shape[0]), + num_frames=int(noisy_latents.shape[1]), + device=noisy_latents.device, + ).flatten() + ), + scheduler=self.noise_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + return pred_x0 + + # --- internal helpers --- + def _load_shared_vae(self, training_args: Any) -> torch.nn.Module: + from fastvideo.distillation.utils.moduleloader import load_module_from_path + + return load_module_from_path( + model_path=str(training_args.model_path), + module_type="vae", + training_args=training_args, + ) + + def _build_noise_scheduler(self, training_args: Any): + from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, + ) + + return FlowMatchEulerDiscreteScheduler( + shift=float(training_args.pipeline_config.flow_shift or 0.0) + ) + + def _init_from_built_roles( + self, + *, + cfg: DistillRunConfig, + role_handles: dict[str, RoleHandle], + vae: torch.nn.Module, + noise_scheduler: Any, + ) -> None: + # This is a small, explicit extraction of `WanGameModel.__init__` so the + # causal model can reuse all non-causal primitives without duplicating + # the full class body. + training_args = cfg.training_args + + self.bundle = self._build_bundle(role_handles) + + self.validator = None + validation_cfg = getattr(cfg, "validation", {}) or {} + validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) + if validation_enabled: + from fastvideo.distillation.validators.wangame import WanGameValidator + + self.validator = WanGameValidator(training_args=training_args) + + self.training_args = training_args + self.noise_scheduler = noise_scheduler + self.vae = vae + + from fastvideo.distributed import ( + get_local_torch_device, + get_sp_group, + get_world_group, + ) + + self.world_group = get_world_group() + self.sp_group = get_sp_group() + self.device = get_local_torch_device() + + self.noise_random_generator = None + self.noise_gen_cuda = None + + self._init_timestep_mechanics() + + from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame + from fastvideo.distillation.utils.dataloader import ( + build_parquet_wangame_train_dataloader, + ) + + self.dataloader = build_parquet_wangame_train_dataloader( + training_args, + parquet_schema=pyarrow_schema_wangame, + ) + self.start_step = 0 + + def _build_bundle(self, role_handles: dict[str, RoleHandle]): + from fastvideo.distillation.roles import RoleManager + + return RoleManager(roles=role_handles) + + def _ensure_per_frame_timestep( + self, + *, + timestep: torch.Tensor, + batch_size: int, + num_frames: int, + device: torch.device, + ) -> torch.Tensor: + if timestep.ndim == 0: + return timestep.view(1, 1).expand(batch_size, num_frames).to(device=device) + if timestep.ndim == 1: + if int(timestep.shape[0]) == batch_size: + return timestep.view(batch_size, 1).expand(batch_size, num_frames).to(device=device) + raise ValueError( + "streaming timestep must be scalar, [B], or [B, T]; got " + f"shape={tuple(timestep.shape)}" + ) + if timestep.ndim == 2: + return timestep.to(device=device) + raise ValueError( + "streaming timestep must be scalar, [B], or [B, T]; got " + f"ndim={int(timestep.ndim)}" + ) + + def _get_or_init_streaming_caches( + self, + *, + handle: RoleHandle, + cache_tag: str, + transformer: torch.nn.Module, + noisy_latents: torch.Tensor, + ) -> _StreamingCaches: + key = (id(handle), cache_tag) + cached = self._streaming_caches.get(key) + + batch_size = int(noisy_latents.shape[0]) + dtype = noisy_latents.dtype + device = noisy_latents.device + + frame_seq_length = self._compute_frame_seq_length(transformer, noisy_latents) + local_attn_size = self._get_local_attn_size(transformer) + sliding_window_num_frames = self._get_sliding_window_num_frames(transformer) + + meta = ( + frame_seq_length, + local_attn_size, + sliding_window_num_frames, + batch_size, + dtype, + device, + ) + + if cached is not None: + cached_meta = ( + cached.frame_seq_length, + cached.local_attn_size, + cached.sliding_window_num_frames, + cached.batch_size, + cached.dtype, + cached.device, + ) + if cached_meta == meta: + return cached + + kv_cache = self._initialize_kv_cache( + transformer=transformer, + batch_size=batch_size, + dtype=dtype, + device=device, + frame_seq_length=frame_seq_length, + local_attn_size=local_attn_size, + sliding_window_num_frames=sliding_window_num_frames, + ) + crossattn_cache = self._initialize_crossattn_cache(transformer=transformer, device=device) + + caches = _StreamingCaches( + kv_cache=kv_cache, + crossattn_cache=crossattn_cache, + frame_seq_length=frame_seq_length, + local_attn_size=local_attn_size, + sliding_window_num_frames=sliding_window_num_frames, + batch_size=batch_size, + dtype=dtype, + device=device, + ) + self._streaming_caches[key] = caches + return caches + + def _compute_frame_seq_length(self, transformer: torch.nn.Module, noisy_latents: torch.Tensor) -> int: + latent_seq_length = int(noisy_latents.shape[-1]) * int(noisy_latents.shape[-2]) + patch_size = getattr(transformer, "patch_size", None) + if patch_size is None: + patch_size = getattr(getattr(transformer, "config", None), "arch_config", None) + patch_size = getattr(patch_size, "patch_size", None) + if patch_size is None: + raise ValueError("Unable to determine transformer.patch_size for causal streaming") + patch_ratio = int(patch_size[-1]) * int(patch_size[-2]) + if patch_ratio <= 0: + raise ValueError("Invalid patch_size for causal streaming") + return latent_seq_length // patch_ratio + + def _get_sliding_window_num_frames(self, transformer: torch.nn.Module) -> int: + cfg = getattr(transformer, "config", None) + arch_cfg = getattr(cfg, "arch_config", None) + value = getattr(arch_cfg, "sliding_window_num_frames", None) if arch_cfg is not None else None + if value is None: + return 15 + return int(value) + + def _get_local_attn_size(self, transformer: torch.nn.Module) -> int: + try: + value = getattr(transformer, "local_attn_size", -1) + except Exception: + value = -1 + if value is None: + return -1 + return int(value) + + def _initialize_kv_cache( + self, + *, + transformer: torch.nn.Module, + batch_size: int, + dtype: torch.dtype, + device: torch.device, + frame_seq_length: int, + local_attn_size: int, + sliding_window_num_frames: int, + ) -> list[dict[str, Any]]: + num_blocks = len(getattr(transformer, "blocks", [])) + if num_blocks <= 0: + raise ValueError("Unexpected transformer.blocks for causal streaming") + + try: + num_attention_heads = int(transformer.num_attention_heads) # type: ignore[attr-defined] + except AttributeError as e: + raise ValueError("Transformer is missing num_attention_heads") from e + + try: + attention_head_dim = int(transformer.attention_head_dim) # type: ignore[attr-defined] + except AttributeError: + try: + hidden_size = int(transformer.hidden_size) # type: ignore[attr-defined] + except AttributeError as e: + raise ValueError("Transformer is missing attention_head_dim and hidden_size") from e + attention_head_dim = hidden_size // max(1, num_attention_heads) + + if local_attn_size != -1: + kv_cache_size = int(local_attn_size) * int(frame_seq_length) + else: + kv_cache_size = int(frame_seq_length) * int(sliding_window_num_frames) + + kv_cache: list[dict[str, Any]] = [] + for _ in range(num_blocks): + kv_cache.append( + { + "k": torch.zeros( + [batch_size, kv_cache_size, num_attention_heads, attention_head_dim], + dtype=dtype, + device=device, + ), + "v": torch.zeros( + [batch_size, kv_cache_size, num_attention_heads, attention_head_dim], + dtype=dtype, + device=device, + ), + "global_end_index": torch.zeros((), dtype=torch.long, device=device), + "local_end_index": torch.zeros((), dtype=torch.long, device=device), + } + ) + + return kv_cache + + def _initialize_crossattn_cache( + self, + *, + transformer: torch.nn.Module, + device: torch.device, + ) -> list[dict[str, Any]] | None: + # WanGame uses image conditioning; caching the image K/V is optional but + # helps avoid repeated projections across timesteps in a rollout. + num_blocks = len(getattr(transformer, "blocks", [])) + if num_blocks <= 0: + return None + return [ + {"is_init": False, "k": torch.empty(0, device=device), "v": torch.empty(0, device=device)} + for _ in range(num_blocks) + ] From d63f6734947e1e4e1aaaf07b6cf062e14cd40706 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 06:34:14 +0000 Subject: [PATCH 150/214] self forcing only allows student to be causalmodel --- .../distribution_matching/self_forcing.py | 245 +++++++++++++++++- .../models/wangame/wangame_causal.py | 57 ++++ 2 files changed, 299 insertions(+), 3 deletions(-) diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/distillation/methods/distribution_matching/self_forcing.py index b34e8cfde..73ad940d9 100644 --- a/fastvideo/distillation/methods/distribution_matching/self_forcing.py +++ b/fastvideo/distillation/methods/distribution_matching/self_forcing.py @@ -33,9 +33,8 @@ - This method uses `SelfForcingFlowMatchScheduler` for simulate-rollout (matching legacy behavior). Teacher/critic losses also interpret noise under the same scheduler for consistency. -- Context/no-grad KV cache updates are model-family specific in legacy. The new - framework keeps method/model decoupled, so this method implements a cache-free - rollout using the model plugin's `predict_noise` primitive. +- This method requires a causal model plugin implementing `CausalModelBase` and + a causal student role. Cache-free rollout is intentionally unsupported. """ from __future__ import annotations @@ -46,6 +45,7 @@ import torch.distributed as dist from fastvideo.distillation.dispatch import register_method +from fastvideo.distillation.models.base import CausalModelBase from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method from fastvideo.distillation.utils.config import get_optional_float, get_optional_int from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( @@ -55,6 +55,8 @@ if TYPE_CHECKING: from fastvideo.pipelines import TrainingBatch + from fastvideo.distillation.roles import RoleManager + from fastvideo.distillation.utils.config import DistillRunConfig def _require_bool(raw: Any, *, where: str) -> bool: @@ -186,6 +188,39 @@ def __init__( # Cache for warped denoising step list (device specific). self._sf_denoising_step_list: torch.Tensor | None = None + # DistillMethod override: build + @classmethod + def build( + cls, + *, + cfg: DistillRunConfig, + bundle: RoleManager, + model: Any, + validator: Any | None, + ) -> DMD2Method: + student_spec = cfg.roles.get("student") + if student_spec is None: + raise ValueError("SelfForcingMethod requires roles.student") + student_variant = (student_spec.extra or {}).get("variant", None) + if str(student_variant).strip().lower() != "causal": + raise ValueError( + "SelfForcingMethod requires a causal student. " + "Set roles.student.variant: causal in the YAML config." + ) + if not isinstance(model, CausalModelBase): + raise ValueError( + "SelfForcingMethod requires a causal model plugin implementing CausalModelBase. " + "Make sure you are using a causal-capable family/model plugin." + ) + + return cls( + bundle=bundle, + model=model, + method_config=cfg.method_config, + validation_config=cfg.validation, + validator=validator, + ) + def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: if ( self._sf_denoising_step_list is not None @@ -307,6 +342,210 @@ def _sample_exit_indices( return [int(i) for i in indices.tolist()] def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: + if not isinstance(self.model, CausalModelBase): + raise ValueError( + "SelfForcingMethod requires a causal model plugin implementing CausalModelBase." + ) + return self._student_rollout_streaming(batch, with_grad=with_grad) + + def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Tensor: + latents = batch.latents + if latents is None: + raise RuntimeError("TrainingBatch.latents is required for self-forcing rollout") + if latents.ndim != 5: + raise ValueError( + "TrainingBatch.latents must be [B, T, C, H, W], got " + f"shape={tuple(latents.shape)}" + ) + + device = latents.device + dtype = latents.dtype + batch_size = int(latents.shape[0]) + num_frames = int(latents.shape[1]) + + denoising_steps = self._get_denoising_step_list(device) + num_steps = int(denoising_steps.numel()) + + noise_full = torch.randn_like(latents, device=device, dtype=dtype) + + chunk = int(self._chunk_size) + if chunk <= 0: + raise ValueError("chunk_size must be positive") + + remaining = num_frames % chunk + num_blocks = num_frames // chunk + if num_blocks == 0: + num_blocks = 1 + remaining = num_frames + + exit_indices = self._sample_exit_indices( + num_blocks=num_blocks, + num_steps=num_steps, + device=device, + ) + + denoised_blocks: list[torch.Tensor] = [] + + cache_tag = "pos" + self.model.clear_caches(self.student, cache_tag=cache_tag) + + for block_idx in range(num_blocks): + if block_idx == 0: + start = 0 + end = remaining + chunk if remaining else chunk + else: + start = remaining + block_idx * chunk + end = remaining + (block_idx + 1) * chunk + start = int(start) + end = int(min(end, num_frames)) + if start >= end: + break + + noisy_block = noise_full[:, start:end] + exit_idx = int(exit_indices[block_idx]) + + for step_idx, current_timestep in enumerate(denoising_steps): + exit_flag = step_idx == exit_idx + + timestep_block = current_timestep * torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ) + + enable_grad = ( + bool(with_grad) + and bool(self._enable_gradient_in_rollout) + and torch.is_grad_enabled() + and start >= int(self._start_gradient_frame) + ) + + if not exit_flag: + with torch.no_grad(): + pred_noise = self.model.predict_noise_streaming( + self.student, + noisy_block, + timestep_block, + batch, + conditional=True, + cache_tag=cache_tag, + store_kv=False, + cur_start_frame=start, + cfg_uncond=self._cfg_uncond, + attn_kind="vsa", + ) + if pred_noise is None: + raise RuntimeError( + "predict_noise_streaming returned None " + "(store_kv=False)" + ) + pred_x0_chunk = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_block.flatten(0, 1), + timestep=timestep_block, + scheduler=self._sf_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + + if step_idx + 1 >= num_steps: + break + next_timestep = denoising_steps[step_idx + 1] + if self._student_sample_type == "sde": + noisy_block = self._sf_add_noise( + pred_x0_chunk, + torch.randn_like(pred_x0_chunk), + next_timestep + * torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ), + ) + else: + sigma_cur = self._timestep_to_sigma(timestep_block).view( + batch_size, end - start, 1, 1, 1 + ) + sigma_next = self._timestep_to_sigma( + next_timestep + * torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ) + ).view(batch_size, end - start, 1, 1, 1) + eps = (noisy_block - (1 - sigma_cur) * pred_x0_chunk) / sigma_cur.clamp_min( + 1e-8 + ) + noisy_block = (1 - sigma_next) * pred_x0_chunk + sigma_next * eps + continue + + with torch.set_grad_enabled(enable_grad): + pred_noise = self.model.predict_noise_streaming( + self.student, + noisy_block, + timestep_block, + batch, + conditional=True, + cache_tag=cache_tag, + store_kv=False, + cur_start_frame=start, + cfg_uncond=self._cfg_uncond, + attn_kind="vsa", + ) + if pred_noise is None: + raise RuntimeError( + "predict_noise_streaming returned None " + "(store_kv=False)" + ) + pred_x0_chunk = pred_noise_to_pred_video( + pred_noise=pred_noise.flatten(0, 1), + noise_input_latent=noisy_block.flatten(0, 1), + timestep=timestep_block, + scheduler=self._sf_scheduler, + ).unflatten(0, pred_noise.shape[:2]) + break + + denoised_blocks.append(pred_x0_chunk) + + with torch.no_grad(): + if self._context_noise > 0.0: + context_timestep = torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ) * float(self._context_noise) + context_latents = self._sf_add_noise( + pred_x0_chunk.detach(), + torch.randn_like(pred_x0_chunk), + context_timestep, + ) + else: + context_timestep = torch.zeros( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ) + context_latents = pred_x0_chunk.detach() + + _ = self.model.predict_noise_streaming( + self.student, + context_latents, + context_timestep, + batch, + conditional=True, + cache_tag=cache_tag, + store_kv=True, + cur_start_frame=start, + cfg_uncond=self._cfg_uncond, + attn_kind="vsa", + ) + + if not denoised_blocks: + raise RuntimeError("Self-forcing rollout produced no blocks") + + self.model.clear_caches(self.student, cache_tag=cache_tag) + return torch.cat(denoised_blocks, dim=1) + + def _student_rollout_cache_free(self, batch: Any, *, with_grad: bool) -> torch.Tensor: latents = batch.latents if latents is None: raise RuntimeError("TrainingBatch.latents is required for self-forcing rollout") diff --git a/fastvideo/distillation/models/wangame/wangame_causal.py b/fastvideo/distillation/models/wangame/wangame_causal.py index dd189250f..e94f14c0c 100644 --- a/fastvideo/distillation/models/wangame/wangame_causal.py +++ b/fastvideo/distillation/models/wangame/wangame_causal.py @@ -158,6 +158,11 @@ def predict_noise_streaming( conditional=conditional, cfg_uncond=cfg_uncond, ) + cond_inputs = self._slice_cond_inputs_for_streaming( + cond_inputs=cond_inputs, + cur_start_frame=cur_start_frame, + num_frames=num_frames, + ) input_kwargs = self._build_distill_input_kwargs( noisy_latents, timestep_full, @@ -331,6 +336,58 @@ def _ensure_per_frame_timestep( f"ndim={int(timestep.ndim)}" ) + def _slice_cond_inputs_for_streaming( + self, + *, + cond_inputs: dict[str, Any], + cur_start_frame: int, + num_frames: int, + ) -> dict[str, Any]: + start = int(cur_start_frame) + num_frames = int(num_frames) + if num_frames <= 0: + raise ValueError("num_frames must be positive for streaming") + if start < 0: + raise ValueError("cur_start_frame must be >= 0 for streaming") + end = start + num_frames + + sliced: dict[str, Any] = dict(cond_inputs) + + image_latents = cond_inputs.get("image_latents") + if isinstance(image_latents, torch.Tensor): + sliced["image_latents"] = image_latents[:, :, start:end] + + mask_lat_size = cond_inputs.get("mask_lat_size") + if isinstance(mask_lat_size, torch.Tensor): + sliced["mask_lat_size"] = mask_lat_size[:, :, start:end] + + viewmats = cond_inputs.get("viewmats") + if isinstance(viewmats, torch.Tensor): + sliced["viewmats"] = viewmats[:, start:end] + + Ks = cond_inputs.get("Ks") + if isinstance(Ks, torch.Tensor): + sliced["Ks"] = Ks[:, start:end] + + action = cond_inputs.get("action") + if isinstance(action, torch.Tensor): + sliced["action"] = action[:, start:end] + + temporal_compression_ratio = int( + self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + ) + raw_end_frame_idx = 1 + temporal_compression_ratio * max(0, end - 1) + + mouse_cond = cond_inputs.get("mouse_cond") + if isinstance(mouse_cond, torch.Tensor): + sliced["mouse_cond"] = mouse_cond[:, :raw_end_frame_idx] + + keyboard_cond = cond_inputs.get("keyboard_cond") + if isinstance(keyboard_cond, torch.Tensor): + sliced["keyboard_cond"] = keyboard_cond[:, :raw_end_frame_idx] + + return sliced + def _get_or_init_streaming_caches( self, *, From 07a673c01581981b4919d9d5755a6a413f99cdaf Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 07:51:03 +0000 Subject: [PATCH 151/214] self forcing config --- ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 9 +- ..._causal_teacher_ckpt22000_nocfg_4n8g.slurm | 63 ++++++++ ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 139 ++++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm create mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml index e268b744f..dec35e288 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml @@ -47,7 +47,14 @@ training: # Data (parquet dataset folder). # Supports comma-separated `path` or `path:N` (repeat count) entries. data_path: >- - /mnt/weka/home/hao.zhang/kaiqin/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 dataloader_num_workers: 4 # Batch / shape diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm new file mode 100644 index 000000000..c7216fb4f --- /dev/null +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm @@ -0,0 +1,63 @@ +#!/bin/bash +#SBATCH --job-name=wangame_self_forcing_causal +#SBATCH --partition=main +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=128 +#SBATCH --mem=1440G +#SBATCH --output=slurm_outputs/wangame_self_forcing_causal_teacher22000_4n8g/wangame_self_forcing_causal_%j.out +#SBATCH --error=slurm_outputs/wangame_self_forcing_causal_teacher22000_4n8g/wangame_self_forcing_causal_%j.err +#SBATCH --exclusive + +set -euo pipefail + +# ---- Env (mirror legacy WanGame slurm scripts) ---- +export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} +export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} +export NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-INIT,NET} +export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} +export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} + +# Triton cache (avoid cross-job collisions; per-node task is OK in practice) +export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/tmp/triton_cache_${SLURM_JOB_ID}_${SLURM_PROCID}} + +# Rendezvous (torchrun) +export MASTER_PORT=${MASTER_PORT:-29501} +nodes=( $(scontrol show hostnames "$SLURM_JOB_NODELIST") ) +export MASTER_ADDR=${MASTER_ADDR:-${nodes[0]}} + +# W&B (recommended to pass WANDB_API_KEY via environment / secret manager) +export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} +export WANDB_MODE=${WANDB_MODE:-online} + +if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 + exit 1 +fi + +source ~/conda/miniconda/bin/activate +conda activate alexfv + +CONFIG=${CONFIG:-"examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml"} +OUTPUT_DIR=${OUTPUT_DIR:-"outputs/wangame_self_forcing_causal_teacher22000_4n8g/${SLURM_JOB_ID}"} + +if [[ ! -f "$CONFIG" ]]; then + echo "Missing distillation YAML config at: $CONFIG" >&2 + exit 1 +fi + +# 4 nodes × 8 GPUs +NUM_NODES=${SLURM_NNODES} +GPUS_PER_NODE=8 + +srun torchrun \ + --nnodes "$NUM_NODES" \ + --nproc_per_node "$GPUS_PER_NODE" \ + --rdzv_backend c10d \ + --rdzv_endpoint "$MASTER_ADDR:$MASTER_PORT" \ + --node_rank "$SLURM_PROCID" \ + fastvideo/training/distillation.py \ + --config "$CONFIG" \ + --override-output-dir "$OUTPUT_DIR" diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml new file mode 100644 index 000000000..531ebed29 --- /dev/null +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml @@ -0,0 +1,139 @@ +# WanGame causal Self-Forcing distillation (40-step teacher -> 4-step student). +# +# 4 nodes × 8 GPUs config (32 ranks). +# +# - Teacher weights come from a DFSFT causal checkpoint (DCP format). +# - Student/Critic are warmstarted from the same checkpoint (copied from +# checkpoint role `student`). +# - Role `path` still points at a *base* wangame model directory (needed to +# load configs/architectures); `init_from_checkpoint` overwrites weights. + +recipe: + family: wangame + method: self_forcing + +roles: + student: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + variant: causal + init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + teacher: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: false + disable_custom_init_weights: true + variant: causal + init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + init_from_checkpoint_role: student + critic: + family: wangame + path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + disable_custom_init_weights: true + variant: causal + init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + init_from_checkpoint_role: student + +training: + # Distributed (4 nodes × 8 GPUs = 32 ranks) + num_gpus: 32 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 32 + + # Data (parquet dataset folder). + # Supports comma-separated `path` or `path:N` (repeat count) entries. + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_self_forcing_4steps_distill_causal_teacher22000_4n8g + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer (student) + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Optimizer (critic / fake-score) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation (method-agnostic knobs) + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: full + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_self_forcing_4steps_distill_causal_teacher22000_4n8g + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [4] + sampler_kind: sde + guidance_scale: 1.0 + +default_pipeline_config: + flow_shift: 3 + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + + # Few-step schedule (single source of truth for the method). + # These are "step indices" and will be warped by the (shifted) scheduler. + warp_denoising_step: true + dmd_denoising_steps: [1000,750,500,250] + + # Self-Forcing rollout knobs + chunk_size: 3 + student_sample_type: sde + same_step_across_blocks: false + last_step_only: false + context_noise: 0.0 + enable_gradient_in_rollout: true + start_gradient_frame: 0 + + # Define CFG "uncond" semantics (operation-centric). + # Here `keep` means uncond == cond (CFG becomes a no-op). + cfg_uncond: + on_missing: error + action: keep + image: keep + text: keep From 6b3ff77853a62356c98f99cab8fec92f7c8a0068 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 08:05:04 +0000 Subject: [PATCH 152/214] better yaml tracker --- ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 6 +++- ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 5 +++ fastvideo/training/trackers.py | 34 ++++++++++++------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml index dec35e288..9909ecc77 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml @@ -105,7 +105,12 @@ training: every_steps: 50 sampling_steps: [4] sampler_kind: sde + rollout_mode: streaming guidance_scale: 1.0 + # Streaming causal denoising requires: + # - num_frames divisible by num_frame_per_block (default=3) + # - (num_frames - 1) divisible by 4 (action tensor convention) + num_frames: 69 default_pipeline_config: flow_shift: 3 @@ -128,4 +133,3 @@ method_config: action: keep image: keep text: keep - diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml index 531ebed29..8f66c73a0 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml @@ -105,7 +105,12 @@ training: every_steps: 100 sampling_steps: [4] sampler_kind: sde + rollout_mode: streaming guidance_scale: 1.0 + # Streaming causal denoising requires: + # - num_frames divisible by num_frame_per_block (default=3) + # - (num_frames - 1) divisible by 4 (action tensor convention) + num_frames: 69 default_pipeline_config: flow_shift: 3 diff --git a/fastvideo/training/trackers.py b/fastvideo/training/trackers.py index 93f48ddd2..02578c58a 100644 --- a/fastvideo/training/trackers.py +++ b/fastvideo/training/trackers.py @@ -177,19 +177,29 @@ def log_file(self, path: str, *, name: str | None = None) -> None: logger.warning("W&B log_file skipped; file not found: %s", resolved) return + target_name = str(name).strip() if name is not None and str(name).strip() else None + if target_name is None: + target_name = os.path.basename(resolved) + + # Prefer placing files directly under the W&B run directory to avoid + # symlink-based saves (which may not sync reliably on some clusters). + run_dir = getattr(self._run, "dir", None) + dest_root = self._log_dir if not isinstance(run_dir, str) else run_dir + dest_root = os.path.abspath(str(dest_root)) + save_path = resolved - if name is not None and str(name).strip(): - dest_path = os.path.join(self._log_dir, str(name).strip()) - try: - shutil.copyfile(resolved, dest_path) - except Exception: - logger.exception( - "Failed to copy file for W&B upload: %s -> %s", - resolved, - dest_path, - ) - else: - save_path = dest_path + dest_path = os.path.join(dest_root, target_name) + try: + pathlib.Path(dest_root).mkdir(parents=True, exist_ok=True) + shutil.copyfile(resolved, dest_path) + except Exception: + logger.exception( + "Failed to copy file for W&B upload: %s -> %s", + resolved, + dest_path, + ) + else: + save_path = dest_path try: self._run.save( From 2d82e59e142717d221f869dfd5adae154579016d Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 19:43:45 +0000 Subject: [PATCH 153/214] safe checkpointing for wangame causal for self forcing --- dev/phase_causal.md | 36 +++++++++-- ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 2 +- .../models/wangame/wangame_causal.py | 61 ++++++++++++++++++- 3 files changed, 92 insertions(+), 7 deletions(-) diff --git a/dev/phase_causal.md b/dev/phase_causal.md index cd5547bd6..41c26436a 100644 --- a/dev/phase_causal.md +++ b/dev/phase_causal.md @@ -4,6 +4,22 @@ > 同时把 **预处理器（preprocessors：VAE / text encoder / image encoder / …）** 的管理收敛到 > **student**，避免 teacher/critic 重复加载与显存浪费，并减少跨 role 的耦合。 +## 0. 当前进度（Snapshot） + +截至目前（实现/验证以 WangGame causal 为主）： + +- ✅ `ModelBase` / `CausalModelBase`：cache 语义不进入 `ModelBase`，只存在于 causal 扩展接口中。 +- ✅ `models/` 层级结构：按 family + variant 拆分（`wan/`、`wangame/`、`wangame_causal.py`）。 +- ✅ preprocessors 收敛：VAE 等 shared components 只在 model plugin 内加载一次； + teacher/critic 仅加载各自 transformer，不重复加载 preprocessors。 +- ✅ WangGame causal streaming primitives：cache 由 causal model 内部持有，method 不传 KV 张量。 +- ✅ Self-Forcing 要求 student 为 causal（不允许 cache-free rollout）。 +- ✅ 方案 A（Activation Checkpointing × Streaming KV-cache 的一致性修复）： + - grad-enabled streaming forward 会对 `kv_cache[*].{global_end_index, local_end_index}` 做 snapshot， + 避免 backward recompute 读到被后续 `store_kv=True` 更新后的 index 导致 `CheckpointError`； + - checkpoint-safe 模式会把 KV-cache capacity 扩到 `training.num_frames * frame_seq_length`， + 避免 forward→backward 生命周期内发生 rolling/eviction 破坏重算一致性。 + ## 1. 背景与动机 在 FastVideo 的 distillation 框架里，我们会遇到典型组合： @@ -127,6 +143,16 @@ causal cache 的更新通常应在 `torch.no_grad()` 下进行，以避免历史 - cache 初始化/重置的时机（每个 rollout / 每个 validation request） - cache 的 dtype/device 与 FSDP/activation checkpoint 的交互 +补充（已落地）： + +- 在 self-forcing 的 streaming rollout 中，`store_kv=True` 会在 forward 过程中更新 cache； + 但当 `training.enable_gradient_checkpointing_type: full` 开启时，torch 会在 backward 对 + checkpointed blocks 进行 recompute，这要求 forward→backward 生命周期内“cache 可观测状态” + 必须稳定（至少对同一 forward 调用而言）。 +- 我们采用“方案 A”在 `models/wangame/wangame_causal.py` 内部做了两点保证： + - **index snapshot**：对每次 grad-enabled 的 streaming forward，快照 cache 的 end-index 张量； + - **capacity 扩容**：checkpoint-safe 时扩大 cache，保证 cache append-only（不触发 rolling/eviction）。 + ### 5.3 验证策略 最小验证集合： @@ -137,8 +163,10 @@ causal cache 的更新通常应在 `torch.no_grad()` 下进行，以避免历史 ## 6. TODO（实施时的文件清单） -- [ ] `fastvideo/distillation/models/` 目录结构调整（新增子目录、移动文件、更新 imports） -- [ ] preprocessors 收敛到 student：移除 teacher/critic 侧的 preprocessor 初始化与依赖 -- [ ] `wangame_causal.py`：封装 cache/streaming primitives（仅 causal 需要） -- [ ] 更新 `dispatch.py` / `register_model` 的 import 路径（保持注册行为不变） +- [x] `fastvideo/distillation/models/` 目录结构调整（新增子目录、移动文件、更新 imports） +- [x] preprocessors 收敛到 student：移除 teacher/critic 侧的 preprocessor 初始化与依赖 +- [x] `wangame_causal.py`：封装 cache/streaming primitives（仅 causal 需要） +- [x] 更新 `dispatch.py` / `register_model` 的 import 路径（保持注册行为不变） +- [x] 方案 A：支持 activation checkpointing 下的 streaming rollout（index snapshot + capacity 扩容） +- [ ] 进一步工程化：把“checkpoint-safe cache”做成可显式开关（必要时打印内存提示/风险） - [ ] 更新必要的 doc（只更新本 phase 相关文档） diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml index 9909ecc77..0dcb00ed0 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml @@ -68,7 +68,7 @@ training: # Output / steps output_dir: outputs/wangame_dmd2_4steps_distill_causal_teacher22000_4n8g - max_train_steps: 4000 + max_train_steps: 400000 seed: 1000 checkpoints_total_limit: 3 diff --git a/fastvideo/distillation/models/wangame/wangame_causal.py b/fastvideo/distillation/models/wangame/wangame_causal.py index e94f14c0c..d4f650970 100644 --- a/fastvideo/distillation/models/wangame/wangame_causal.py +++ b/fastvideo/distillation/models/wangame/wangame_causal.py @@ -139,9 +139,22 @@ def predict_noise_streaming( ) frame_seq_length = int(caches.frame_seq_length) + kv_cache = caches.kv_cache + crossattn_cache = caches.crossattn_cache + + # When activation checkpointing is enabled, torch will recompute + # transformer blocks during backward. Self-forcing streaming rollout + # mutates KV-cache *between* forward and backward (store_kv=True), which + # can break checkpoint recompute with metadata mismatches. + # + # Scheme A: snapshot the cache end-index tensors per call, so backward + # recompute sees the same "effective cache length" as forward. + if self._should_snapshot_streaming_cache(handle) and torch.is_grad_enabled(): + kv_cache = self._snapshot_kv_cache_indices(kv_cache) + model_kwargs: dict[str, Any] = { - "kv_cache": caches.kv_cache, - "crossattn_cache": caches.crossattn_cache, + "kv_cache": kv_cache, + "crossattn_cache": crossattn_cache, "current_start": cur_start_frame * frame_seq_length, "start_frame": cur_start_frame, "is_cache": bool(store_kv), @@ -436,6 +449,7 @@ def _get_or_init_streaming_caches( frame_seq_length=frame_seq_length, local_attn_size=local_attn_size, sliding_window_num_frames=sliding_window_num_frames, + checkpoint_safe=self._should_use_checkpoint_safe_kv_cache(handle), ) crossattn_cache = self._initialize_crossattn_cache(transformer=transformer, device=device) @@ -492,6 +506,7 @@ def _initialize_kv_cache( frame_seq_length: int, local_attn_size: int, sliding_window_num_frames: int, + checkpoint_safe: bool, ) -> list[dict[str, Any]]: num_blocks = len(getattr(transformer, "blocks", [])) if num_blocks <= 0: @@ -516,6 +531,19 @@ def _initialize_kv_cache( else: kv_cache_size = int(frame_seq_length) * int(sliding_window_num_frames) + # Checkpoint-safe mode: allocate enough cache capacity to avoid rolling + # / eviction during a forward->backward lifetime. This keeps the KV + # cache append-only, which is required for correct activation + # checkpoint recompute in streaming methods (e.g., self-forcing). + if checkpoint_safe: + total_frames = int(getattr(self.training_args, "num_frames", 0) or 0) + if total_frames <= 0: + raise ValueError( + "training.num_frames must be set to enable checkpoint-safe " + f"streaming KV cache; got {total_frames}" + ) + kv_cache_size = max(kv_cache_size, int(frame_seq_length) * total_frames) + kv_cache: list[dict[str, Any]] = [] for _ in range(num_blocks): kv_cache.append( @@ -537,6 +565,35 @@ def _initialize_kv_cache( return kv_cache + def _should_use_checkpoint_safe_kv_cache(self, handle: RoleHandle) -> bool: + checkpointing_type = getattr(self.training_args, "enable_gradient_checkpointing_type", None) + return bool(checkpointing_type) and bool(getattr(handle, "trainable", False)) + + def _should_snapshot_streaming_cache(self, handle: RoleHandle) -> bool: + # Snapshotting indices is only needed when recompute may happen. + return self._should_use_checkpoint_safe_kv_cache(handle) + + def _snapshot_kv_cache_indices( + self, kv_cache: list[dict[str, Any]] + ) -> list[dict[str, Any]]: + snapshot: list[dict[str, Any]] = [] + for block_cache in kv_cache: + global_end_index = block_cache.get("global_end_index") + local_end_index = block_cache.get("local_end_index") + if not isinstance(global_end_index, torch.Tensor) or not isinstance( + local_end_index, torch.Tensor + ): + raise ValueError( + "Unexpected kv_cache index tensors; expected tensors at " + "kv_cache[*].{global_end_index, local_end_index}" + ) + + copied = dict(block_cache) + copied["global_end_index"] = global_end_index.detach().clone() + copied["local_end_index"] = local_end_index.detach().clone() + snapshot.append(copied) + return snapshot + def _initialize_crossattn_cache( self, *, From 4e77535674b2215913e8a97feeabe3816294455b Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 22:02:56 +0000 Subject: [PATCH 154/214] common part of wangame --- dev/phase_causal2.md | 117 ++++++++++++++++++ .../distillation/models/wangame/common.py | 89 +++++++++++++ .../distillation/models/wangame/wangame.py | 61 +-------- .../models/wangame/wangame_causal.py | 6 +- 4 files changed, 209 insertions(+), 64 deletions(-) create mode 100644 dev/phase_causal2.md create mode 100644 fastvideo/distillation/models/wangame/common.py diff --git a/dev/phase_causal2.md b/dev/phase_causal2.md new file mode 100644 index 000000000..c5183581e --- /dev/null +++ b/dev/phase_causal2.md @@ -0,0 +1,117 @@ +# Phase: Causal2（更贴近 FastGen 的“student 管 shared components”语义） + +> 目标：进一步把 “shared / expensive components（VAE / text encoder / image encoder / …）” +> 的生命周期与加载语义收敛到 **student**，并让一个 run 内部自然支持 +> “student/teacher/critic 多模型（transformers）并存”，而不引入额外的中间概念。 + +## 1. 背景：FastGen 的语义是什么？ + +FastGen 在配置层面并不使用 `variant: causal` 这种字段，而是通过“选择不同的 net 配置类” +来表达“我要跑 causal 还是 bidirectional”： + +- `config.model.net = CausalWan_1_3B_Config` → student 是 causal +- `config.model.teacher = Wan_1_3B_Config` → teacher 仍然可以是 bidirectional + +它的核心思路可以概括成两点： + +1) **student/net 是主模型（main network）**：很多“数据侧/推理侧的 shared 组件” + 会天然围绕它组织（例如 preprocessors、一些全局 shape 约束等）。 +2) **teacher 只是额外的网络实例**：提供 target/score，不应该因为 teacher 存在就把 + preprocessors 再加载一份、再维护一份生命周期。 + +## 2. 我们想对齐的“语义”是什么？ + +### 2.1 只让 student 负责 shared components（preprocessors） + +定义 shared components（示例）： + +- VAE（latent encode/decode + latent norm） +- text encoder / tokenizer（如果训练/验证需要 text conditioning） +- image encoder（I2V 可能需要） +- 其它可能跨 role 复用、且体积/显存开销大的组件 + +原则： + +- **一个 run 只加载一份 shared components**，默认由 `roles.student` 持有/提供。 +- teacher/critic role **只加载它们各自的 transformer**（以及必要的轻量配置），不再重复加载 VAE 等。 + +收益： + +- 显存与初始化成本显著降低（尤其 teacher 大模型时）。 +- 更清晰的职责划分：teacher 是“提供 target 的网络”，不是“数据编解码提供者”。 + +### 2.2 一个 run 直接持有多个模型（多 role transformers） + +对齐 FastGen：“一个训练 job 同时持有 student + teacher（+ critic/…）”。 + +在我们的框架里，最自然的落地是： + +- `RoleManager` 仍然是“多 role modules”的容器（student/teacher/critic/...）。 +- model plugin 负责： + - 构建每个 role 的 transformer（以及 trainable 开关/activation checkpointing 等）； + - 加载 shared components（只加载一次）； + - 把 dataloader batch 规范化为 methods 可用的 forward primitives； + - 提供 operation-centric primitives（`add_noise / predict_noise / predict_x0 / backward / ...`）。 + +也就是说：**“多模型并存”是运行时事实，不需要额外引入 ModelComponents 之类中间层**。 + +## 3. 配置（YAML）层面的约定 + +### 3.1 shared components 的来源：固定使用 student（从简） + +从简规则（不做 fallback / 不做复杂合并）： + +- shared components 的加载路径 = `roles.student.path` + - 这也是 `training.model_path` 的默认来源（用于 pipeline registry / 组件加载等）。 +- 如果没有 `roles.student`，则： + - 对于需要 shared components 的 recipe：直接报错（强约束，避免 silent 行为）。 + +未来如果需要支持 “teacher/student family 不同”，再引入更复杂的机制（例如多套 shared components）。 + +### 3.2 causal / bidirectional 的选择（FastGen 风格） + +我们可以同时支持两种表达方式（但优先推荐 FastGen 风格）： + +- FastGen 风格（推荐）：通过 `recipe.family` 选择模型变体 + - `recipe.family: wangame` / `recipe.family: wangame_causal` + - `wangame_causal` 默认所有 role 走 causal transformer（除非 role 显式声明 bidi） +- 兼容表达（可选）：`roles..variant: causal|bidirectional` + - 用于 “student causal + teacher bidirectional” 等混合场景 + +注：即使使用 `recipe.family: wangame_causal`，仍建议保留 per-role override 的能力， +以覆盖 FastGen 常见组合（student causal + teacher bidi）。 + +## 4. 代码落地点（文件与职责） + +以 wangame 为例： + +```text +fastvideo/distillation/models/wangame/ + common.py # role transformer 构建的共享逻辑（不涉及 preprocessors） + wangame.py # bidi 版本：加载 shared components + bidi primitives + wangame_causal.py # causal 版本：在 wangame.py 基础上增加 cache/streaming primitives + __init__.py # register_model("wangame") (and maybe "wangame_causal") +``` + +关键点： + +- `common.py` 只负责 “按 role 构建 transformer handle”，不加载 VAE/text encoder 等。 +- `wangame.py / wangame_causal.py` 只在 **一个地方**加载 shared components（来自 student path），并复用给所有 role。 +- methods 永远通过 `model.predict_* (handle=RoleHandle, ...)` 这类 operation-centric API 调用网络； + methods 不直接“知道/管理”VAE 等加载细节。 + +## 5. 风险与边界（明确不做） + +- **跨 family shared components**：例如 teacher=SDXL, student=Wan。 + - 这会带来 latent 语义不一致、conditioning schema 不一致等问题。 + - 本 phase 不解决；遇到则应当在构建期直接 error，避免 silent mismatch。 +- **让 config 变成无结构 dict 并把语义搬进 utils**： + - “路径可配置”不等于“语义可抽象”；加载/调度的语义仍然高度 family 相关。 + - 仍坚持 model plugin 层吸收 family 差异，methods 保持算法纯净。 + +## 6. TODO（实施清单） + +- [ ] 新增 `recipe.family: wangame_causal`（更 FastGen 风格），默认所有 role 为 causal +- [ ] 明确并 enforce：shared components 仅从 `roles.student.path` 加载 +- [ ] 文档化：每个 model/method 文件开头声明本文件会读取的 config keys（降低阅读成本） + diff --git a/fastvideo/distillation/models/wangame/common.py b/fastvideo/distillation/models/wangame/common.py new file mode 100644 index 000000000..bc3f72446 --- /dev/null +++ b/fastvideo/distillation/models/wangame/common.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""WanGame shared helpers. + +This module hosts "family-level" helpers used by both bidirectional and causal +WanGame model plugins (e.g. role-handle assembly). + +The goal is to avoid duplicated logic across: +- `fastvideo/distillation/models/wangame/wangame.py` +- `fastvideo/distillation/models/wangame/wangame_causal.py` + +Keep this file free of imports of the variant-specific model classes to prevent +circular dependencies. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import torch + +from fastvideo.training.activation_checkpoint import apply_activation_checkpointing + +from fastvideo.distillation.roles import RoleHandle +from fastvideo.distillation.utils.module_state import apply_trainable +from fastvideo.distillation.utils.moduleloader import load_module_from_path + + +def _build_wangame_role_handles( + *, + roles_cfg: dict[str, Any], + training_args: Any, + transformer_cls_name_for_role: Callable[[str, Any], str], +) -> dict[str, RoleHandle]: + role_handles: dict[str, RoleHandle] = {} + for role, role_spec in roles_cfg.items(): + if role_spec.family != "wangame": + raise ValueError( + "Wangame model plugin only supports roles with family='wangame'; " + f"got {role}={role_spec.family!r}" + ) + + transformer_cls_name = transformer_cls_name_for_role(str(role), role_spec) + disable_custom_init_weights = bool( + getattr(role_spec, "disable_custom_init_weights", False) + ) + transformer = load_module_from_path( + model_path=role_spec.path, + module_type="transformer", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + override_transformer_cls_name=transformer_cls_name, + ) + modules: dict[str, torch.nn.Module] = {"transformer": transformer} + + # Optional MoE support: load transformer_2 if present in the model. + try: + transformer_2 = load_module_from_path( + model_path=role_spec.path, + module_type="transformer_2", + training_args=training_args, + disable_custom_init_weights=disable_custom_init_weights, + ) + except ValueError: + transformer_2 = None + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + + for name, module in list(modules.items()): + module = apply_trainable(module, trainable=bool(role_spec.trainable)) + if bool(role_spec.trainable) and bool( + getattr(training_args, "enable_gradient_checkpointing_type", None) + ): + module = apply_activation_checkpointing( + module, + checkpointing_type=training_args.enable_gradient_checkpointing_type, + ) + modules[name] = module + + role_handles[str(role)] = RoleHandle( + modules=modules, + optimizers={}, + lr_schedulers={}, + trainable=bool(role_spec.trainable), + ) + + return role_handles + diff --git a/fastvideo/distillation/models/wangame/wangame.py b/fastvideo/distillation/models/wangame/wangame.py index 5c10f1057..d5b86b59a 100644 --- a/fastvideo/distillation/models/wangame/wangame.py +++ b/fastvideo/distillation/models/wangame/wangame.py @@ -23,7 +23,6 @@ from __future__ import annotations import copy -from collections.abc import Callable from typing import Any, Literal import torch @@ -40,7 +39,6 @@ ) from fastvideo.models.utils import pred_noise_to_pred_video from fastvideo.pipelines import TrainingBatch -from fastvideo.training.activation_checkpoint import apply_activation_checkpointing from fastvideo.training.training_utils import ( compute_density_for_timestep_sampling, get_sigmas, @@ -49,11 +47,11 @@ ) from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed +from fastvideo.distillation.models.wangame.common import _build_wangame_role_handles from fastvideo.distillation.models.base import ModelBase from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.distillation.utils.config import DistillRunConfig from fastvideo.distillation.utils.dataloader import build_parquet_wangame_train_dataloader -from fastvideo.distillation.utils.module_state import apply_trainable from fastvideo.distillation.utils.moduleloader import load_module_from_path try: @@ -66,63 +64,6 @@ VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] -def _build_wangame_role_handles( - *, - roles_cfg: dict[str, Any], - training_args: Any, - transformer_cls_name_for_role: Callable[[str, Any], str], -) -> dict[str, RoleHandle]: - role_handles: dict[str, RoleHandle] = {} - for role, role_spec in roles_cfg.items(): - if role_spec.family != "wangame": - raise ValueError( - "Wangame model plugin only supports roles with family='wangame'; " - f"got {role}={role_spec.family!r}" - ) - - transformer_cls_name = transformer_cls_name_for_role(str(role), role_spec) - disable_custom_init_weights = bool(getattr(role_spec, "disable_custom_init_weights", False)) - transformer = load_module_from_path( - model_path=role_spec.path, - module_type="transformer", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - override_transformer_cls_name=transformer_cls_name, - ) - modules: dict[str, torch.nn.Module] = {"transformer": transformer} - - # Optional MoE support: load transformer_2 if present in the model. - try: - transformer_2 = load_module_from_path( - model_path=role_spec.path, - module_type="transformer_2", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - ) - except ValueError: - transformer_2 = None - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 - - for name, module in list(modules.items()): - module = apply_trainable(module, trainable=bool(role_spec.trainable)) - if role_spec.trainable and getattr(training_args, "enable_gradient_checkpointing_type", None): - module = apply_activation_checkpointing( - module, - checkpointing_type=training_args.enable_gradient_checkpointing_type, - ) - modules[name] = module - - role_handles[str(role)] = RoleHandle( - modules=modules, - optimizers={}, - lr_schedulers={}, - trainable=bool(role_spec.trainable), - ) - - return role_handles - - class WanGameModel(ModelBase): """WanGame model plugin: loads roles + shared components and exposes runtime primitives. diff --git a/fastvideo/distillation/models/wangame/wangame_causal.py b/fastvideo/distillation/models/wangame/wangame_causal.py index d4f650970..4ecd56cf2 100644 --- a/fastvideo/distillation/models/wangame/wangame_causal.py +++ b/fastvideo/distillation/models/wangame/wangame_causal.py @@ -24,10 +24,8 @@ from fastvideo.distillation.roles import RoleHandle from fastvideo.distillation.utils.config import DistillRunConfig -from fastvideo.distillation.models.wangame.wangame import ( - WanGameModel, - _build_wangame_role_handles, -) +from fastvideo.distillation.models.wangame.common import _build_wangame_role_handles +from fastvideo.distillation.models.wangame.wangame import WanGameModel @dataclass(slots=True) From 15c644a07c1b7283ad93b38f740d3590c3085167 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 22:05:57 +0000 Subject: [PATCH 155/214] remove manual branch in selfforcing --- ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 2 +- .../distribution_matching/self_forcing.py | 202 ------------------ 2 files changed, 1 insertion(+), 203 deletions(-) diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml index 8f66c73a0..5d9a47210 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml @@ -90,7 +90,7 @@ training: training_cfg_rate: 0.0 min_timestep_ratio: 0.02 max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full + enable_gradient_checkpointing_type: null # Checkpointing training_state_checkpointing_steps: 1000 diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/distillation/methods/distribution_matching/self_forcing.py index 73ad940d9..f74b0a2fc 100644 --- a/fastvideo/distillation/methods/distribution_matching/self_forcing.py +++ b/fastvideo/distillation/methods/distribution_matching/self_forcing.py @@ -545,208 +545,6 @@ def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Te self.model.clear_caches(self.student, cache_tag=cache_tag) return torch.cat(denoised_blocks, dim=1) - def _student_rollout_cache_free(self, batch: Any, *, with_grad: bool) -> torch.Tensor: - latents = batch.latents - if latents is None: - raise RuntimeError("TrainingBatch.latents is required for self-forcing rollout") - if latents.ndim != 5: - raise ValueError( - "TrainingBatch.latents must be [B, T, C, H, W], got " - f"shape={tuple(latents.shape)}" - ) - - device = latents.device - dtype = latents.dtype - batch_size = int(latents.shape[0]) - num_frames = int(latents.shape[1]) - - denoising_steps = self._get_denoising_step_list(device) - num_steps = int(denoising_steps.numel()) - - noise_full = torch.randn_like(latents, device=device, dtype=dtype) - - chunk = int(self._chunk_size) - if chunk <= 0: - raise ValueError("chunk_size must be positive") - - remaining = num_frames % chunk - num_blocks = num_frames // chunk - if num_blocks == 0: - num_blocks = 1 - remaining = num_frames - - exit_indices = self._sample_exit_indices( - num_blocks=num_blocks, - num_steps=num_steps, - device=device, - ) - - context_latents: list[torch.Tensor] = [] - context_timesteps: list[torch.Tensor] = [] - denoised_blocks: list[torch.Tensor] = [] - - for block_idx in range(num_blocks): - if block_idx == 0: - start = 0 - end = remaining + chunk if remaining else chunk - else: - start = remaining + block_idx * chunk - end = remaining + (block_idx + 1) * chunk - start = int(start) - end = int(min(end, num_frames)) - if start >= end: - break - - noisy_block = noise_full[:, start:end] - exit_idx = int(exit_indices[block_idx]) - - context_len = start - future_latents = noise_full[:, end:] - future_len = int(future_latents.shape[1]) - - for step_idx, current_timestep in enumerate(denoising_steps): - exit_flag = step_idx == exit_idx - - timestep_block = current_timestep * torch.ones( - (batch_size, end - start), - device=device, - dtype=torch.float32, - ) - - if context_len > 0: - context_lat = torch.cat(context_latents, dim=1) - context_t = torch.cat(context_timesteps, dim=1) - if int(context_lat.shape[1]) != context_len: - raise RuntimeError("context_latents length mismatch") - else: - context_lat = None - context_t = None - - if context_lat is None: - model_latents = torch.cat([noisy_block, future_latents], dim=1) - timestep_full = torch.cat( - [ - timestep_block, - current_timestep - * torch.ones( - (batch_size, future_len), - device=device, - dtype=torch.float32, - ), - ], - dim=1, - ) - else: - model_latents = torch.cat([context_lat, noisy_block, future_latents], dim=1) - timestep_full = torch.cat( - [ - context_t, - timestep_block, - current_timestep - * torch.ones( - (batch_size, future_len), - device=device, - dtype=torch.float32, - ), - ], - dim=1, - ) - - enable_grad = ( - bool(with_grad) - and bool(self._enable_gradient_in_rollout) - and torch.is_grad_enabled() - and start >= int(self._start_gradient_frame) - ) - - if not exit_flag: - with torch.no_grad(): - pred_x0_full = self._predict_x0_with_scheduler( - self.student, - model_latents, - timestep_full, - batch, - conditional=True, - attn_kind="vsa", - ) - pred_x0_chunk = pred_x0_full[:, context_len:context_len + (end - start)] - if step_idx + 1 >= num_steps: - break - next_timestep = denoising_steps[step_idx + 1] - if self._student_sample_type == "sde": - noisy_block = self._sf_add_noise( - pred_x0_chunk, - torch.randn_like(pred_x0_chunk), - next_timestep - * torch.ones( - (batch_size, end - start), - device=device, - dtype=torch.float32, - ), - ) - else: - sigma_cur = self._timestep_to_sigma(timestep_block).view( - batch_size, end - start, 1, 1, 1 - ) - sigma_next = self._timestep_to_sigma( - next_timestep - * torch.ones( - (batch_size, end - start), - device=device, - dtype=torch.float32, - ) - ).view(batch_size, end - start, 1, 1, 1) - eps = (noisy_block - (1 - sigma_cur) * pred_x0_chunk) / sigma_cur.clamp_min( - 1e-8 - ) - noisy_block = (1 - sigma_next) * pred_x0_chunk + sigma_next * eps - continue - - with torch.set_grad_enabled(enable_grad): - pred_x0_full = self._predict_x0_with_scheduler( - self.student, - model_latents, - timestep_full, - batch, - conditional=True, - attn_kind="vsa", - ) - pred_x0_chunk = pred_x0_full[:, context_len:context_len + (end - start)] - break - - denoised_blocks.append(pred_x0_chunk) - - # Update context frames (cache-free): store context-noised frames + timesteps. - if self._context_noise > 0.0: - context_timestep = torch.ones( - (batch_size, end - start), - device=device, - dtype=torch.float32, - ) * float(self._context_noise) - with torch.no_grad(): - context_latents.append( - self._sf_add_noise( - pred_x0_chunk.detach(), - torch.randn_like(pred_x0_chunk), - context_timestep, - ) - ) - context_timesteps.append(context_timestep) - else: - context_latents.append(pred_x0_chunk.detach()) - context_timesteps.append( - torch.zeros( - (batch_size, end - start), - device=device, - dtype=torch.float32, - ) - ) - - if not denoised_blocks: - raise RuntimeError("Self-forcing rollout produced no blocks") - - return torch.cat(denoised_blocks, dim=1) - def _critic_flow_matching_loss( self, batch: Any ) -> tuple[torch.Tensor, Any, dict[str, Any]]: From 24bc5dc4dfeda6ccf2e117cb26af2fbb10a14ca9 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 22:31:32 +0000 Subject: [PATCH 156/214] fix gradient ckpt missing keys --- dev/phase_causal2.md | 65 +++++++++++++++++-- fastvideo/distillation/utils/checkpoint.py | 75 +++++++++++++++++++++- fastvideo/training/checkpointing_utils.py | 23 +++++-- 3 files changed, 151 insertions(+), 12 deletions(-) diff --git a/dev/phase_causal2.md b/dev/phase_causal2.md index c5183581e..bda621811 100644 --- a/dev/phase_causal2.md +++ b/dev/phase_causal2.md @@ -4,6 +4,17 @@ > 的生命周期与加载语义收敛到 **student**，并让一个 run 内部自然支持 > “student/teacher/critic 多模型（transformers）并存”，而不引入额外的中间概念。 +## 0. 当前进度（Snapshot） + +截至目前： + +- ✅ 修复 DCP checkpoint 在开启 activation checkpointing（`enable_gradient_checkpointing_type: full/ops/...`） + 时漏存大量 transformer 参数的问题（根因是 wrapper 导致 `named_parameters()` 与 + `get_model_state_dict()` key 不一致，见 5. 风险与边界）。 +- ✅ warmstart（`roles.*.init_from_checkpoint`）改为 best-effort：仅加载 checkpoint 里实际存在的 keys， + 避免 DCP planner 因 “Missing key” 直接失败（对于历史不完整 ckpt 也能继续跑，但缺失的权重当然无法凭空恢复）。 +- ✅ 文档化了 `models.shared_component_role` 的语义（默认 student），用于显式指定 shared components owner。 + ## 1. 背景：FastGen 的语义是什么？ FastGen 在配置层面并不使用 `variant: causal` 这种字段，而是通过“选择不同的 net 配置类” @@ -57,14 +68,32 @@ FastGen 在配置层面并不使用 `variant: causal` 这种字段，而是通 ## 3. 配置（YAML）层面的约定 -### 3.1 shared components 的来源：固定使用 student（从简） +### 3.1 shared components 的来源：`models.shared_component_role`（默认 student） + +为了把 “谁负责 shared components” 变成**显式语义**（而不是硬编码只认 student）， +引入一个模型层面的配置段： + +```yaml +models: + # 哪个 role 负责加载/持有 shared components（VAE / text encoder / image encoder / …） + # 默认：student + shared_component_role: student +``` + +约束（从简）： + +- 该字段必须是一个 role name（string），且 `roles.` 必须存在；否则直接报错。 +- shared components 的加载路径 = `roles[shared_component_role].path` +- `training.model_path` 的默认来源也应当改为这个 role（用于 pipeline registry / 组件加载等）。 + +动机： + +- 对齐 FastGen 的 “main network” 语义：它是一个概念，不一定永远叫 `student`。 +- 未来如果出现 “只有 teacher 但没有 student 的 recipe”（例如纯评估/蒸馏变体），也能工作。 从简规则（不做 fallback / 不做复杂合并）： -- shared components 的加载路径 = `roles.student.path` - - 这也是 `training.model_path` 的默认来源（用于 pipeline registry / 组件加载等）。 -- 如果没有 `roles.student`，则： - - 对于需要 shared components 的 recipe：直接报错（强约束，避免 silent 行为）。 +- 如果不写 `models.shared_component_role`，默认等价于 `student`。 未来如果需要支持 “teacher/student family 不同”，再引入更复杂的机制（例如多套 shared components）。 @@ -109,9 +138,31 @@ fastvideo/distillation/models/wangame/ - “路径可配置”不等于“语义可抽象”；加载/调度的语义仍然高度 family 相关。 - 仍坚持 model plugin 层吸收 family 差异，methods 保持算法纯净。 +### 5.1 DCP checkpoint × activation checkpointing 的 key 对齐（已修复） + +问题现象： + +- 开启 activation checkpointing 时，部分模块会被 `checkpoint_wrapper(...)` 包裹； +- 此时 `named_parameters()` 看到的 key 形如： + - `blocks.0._checkpoint_wrapped_module.weight` +- 但 `torch.distributed.checkpoint.state_dict.get_model_state_dict()` 返回的 key 仍是： + - `blocks.0.weight` +- 如果用 “named_parameters 的 key 集合” 去过滤 `get_model_state_dict()`，就会把 blocks.* 全部过滤掉， + 导致 DCP checkpoint **漏存大部分 transformer 权重**（训练能跑，但 ckpt 不可用 / warmstart 会 Missing key）。 + +修复策略： + +- 在 `ModelWrapper.state_dict()` 里对 `._checkpoint_wrapped_module.` 做 canonicalize： + - 既保留原始 key，也加入 unwrapped key（把 `._checkpoint_wrapped_module.` 替换成 `.`） + - 从而保证 `get_model_state_dict()` 的 key 能被正确匹配并保存。 + +对历史 checkpoint 的影响： + +- 旧 checkpoint 里缺失的权重无法恢复（除非另有完整的 pretrained/export）；但 warmstart 现在会 best-effort 加载已有 keys， + 不再因为 DCP planner strict missing key 直接 crash。 + ## 6. TODO（实施清单） - [ ] 新增 `recipe.family: wangame_causal`（更 FastGen 风格），默认所有 role 为 causal -- [ ] 明确并 enforce：shared components 仅从 `roles.student.path` 加载 +- [ ] 落地 `models.shared_component_role`（默认 student）用于 shared components 的 owner（含：`training.model_path` 默认来源切换） - [ ] 文档化：每个 model/method 文件开头声明本文件会读取的 config keys（降低阅读成本） - diff --git a/fastvideo/distillation/utils/checkpoint.py b/fastvideo/distillation/utils/checkpoint.py index b73656588..61244d85e 100644 --- a/fastvideo/distillation/utils/checkpoint.py +++ b/fastvideo/distillation/utils/checkpoint.py @@ -16,6 +16,7 @@ from torch.distributed.checkpoint.state_dict import ( StateDictOptions, get_model_state_dict, + set_model_state_dict, ) from fastvideo.distillation.roles import RoleHandle, RoleManager @@ -146,6 +147,68 @@ def _get_dcp_role_module_names(dcp_dir: Path, role: str) -> set[str]: return set(str(name) for name in packed) +def _get_dcp_role_module_param_keys( + dcp_dir: Path, *, role: str, module_name: str +) -> set[str]: + """Return inner param keys present under `roles...*` in a DCP checkpoint. + + Example checkpoint key: + roles.student.transformer.blocks.0.to_q.weight + Returned inner key: + blocks.0.to_q.weight + + This is used for warmstart to avoid DCP planner failures when the target + module expects parameters that are not present in the checkpoint (e.g., + older checkpoints produced under different wrapping/filtering). + """ + + if _rank() == 0: + reader = dcp.FileSystemReader(str(dcp_dir)) + metadata = reader.read_metadata() + prefix = f"roles.{role}.{module_name}." + keys: set[str] = set() + for key in metadata.state_dict_metadata: + if not key.startswith(prefix): + continue + inner = key[len(prefix) :] + if inner: + keys.add(inner) + packed: list[str] = sorted(keys) + else: + packed = [] + + if dist.is_available() and dist.is_initialized(): + obj_list: list[Any] = [packed] + dist.broadcast_object_list(obj_list, src=0) + packed = obj_list[0] + + return set(str(name) for name in packed) + + +class _WarmstartModelWrapper(torch.distributed.checkpoint.stateful.Stateful): + """A StateDict wrapper used for warmstart (best-effort load). + + Unlike training checkpoints, warmstart is not required to be an exact + resume. We therefore restrict the *expected* parameter keys to those + actually present in the checkpoint to prevent DCP load planning failures. + """ + + def __init__(self, model: torch.nn.Module, *, allowed_keys: set[str]) -> None: + self.model = model + self.allowed_keys = allowed_keys + + def state_dict(self) -> dict[str, Any]: + state_dict = get_model_state_dict(self.model) # type: ignore[no-any-return] + return {k: v for k, v in state_dict.items() if k in self.allowed_keys} + + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + set_model_state_dict( + self.model, + model_state_dict=state_dict, + options=StateDictOptions(strict=False), + ) + + def maybe_warmstart_role_modules( *, bundle: RoleManager, @@ -183,7 +246,17 @@ def maybe_warmstart_role_modules( for module_name, module in handle.modules.items(): if module_name not in available_modules: continue - states[f"roles.{checkpoint_role}.{module_name}"] = ModelWrapper(module) + allowed_keys = _get_dcp_role_module_param_keys( + dcp_dir, + role=str(checkpoint_role), + module_name=str(module_name), + ) + if not allowed_keys: + continue + states[f"roles.{checkpoint_role}.{module_name}"] = _WarmstartModelWrapper( + module, + allowed_keys=allowed_keys, + ) if not states: raise ValueError( diff --git a/fastvideo/training/checkpointing_utils.py b/fastvideo/training/checkpointing_utils.py index bc6aeed55..a3d4e84e2 100644 --- a/fastvideo/training/checkpointing_utils.py +++ b/fastvideo/training/checkpointing_utils.py @@ -21,10 +21,25 @@ def state_dict(self) -> dict[str, Any]: state_dict = get_model_state_dict( self.model) # type: ignore[no-any-return] # filter out non-trainable parameters - param_requires_grad = set([ - k for k, v in dict(self.model.named_parameters()).items() - if v.requires_grad - ]) + param_requires_grad: set[str] = set() + for name, param in self.model.named_parameters(): + if not bool(param.requires_grad): + continue + param_requires_grad.add(name) + + # Activation checkpointing wraps modules with an internal attribute + # `_checkpoint_wrapped_module`, which changes the *parameter name* + # observed via `named_parameters()`: + # + # named_parameters: blocks.0._checkpoint_wrapped_module.weight + # state_dict: blocks.0.weight + # + # `get_model_state_dict()` returns the unwrapped key names, so we + # also add the unwrapped form for filtering. + if "._checkpoint_wrapped_module." in name: + param_requires_grad.add( + name.replace("._checkpoint_wrapped_module.", ".") + ) state_dict = { k: v for k, v in state_dict.items() if k in param_requires_grad From acae34e85ae8119f1c764a3e1e571aa8aef5e26d Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 22:57:55 +0000 Subject: [PATCH 157/214] causal refactor 2 --- dev/phase_causal2.md | 17 +++--- .../dfsft_wangame2.1_i2v_1.3B_causal.yaml | 5 +- ...dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml | 5 +- ...4steps_causal_teacher_ckpt22000_nocfg.yaml | 10 +--- ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 9 +--- ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 9 +--- .../distribution_matching/self_forcing.py | 11 +++- fastvideo/distillation/models/wan/wan.py | 4 +- .../distillation/models/wangame/__init__.py | 10 +++- .../distillation/models/wangame/common.py | 6 +-- .../distillation/models/wangame/wangame.py | 4 +- .../models/wangame/wangame_causal.py | 31 ++++++++--- fastvideo/distillation/utils/config.py | 54 ++++++++++++++++--- 13 files changed, 119 insertions(+), 56 deletions(-) diff --git a/dev/phase_causal2.md b/dev/phase_causal2.md index bda621811..e7abb6a9b 100644 --- a/dev/phase_causal2.md +++ b/dev/phase_causal2.md @@ -13,7 +13,10 @@ `get_model_state_dict()` key 不一致，见 5. 风险与边界）。 - ✅ warmstart（`roles.*.init_from_checkpoint`）改为 best-effort：仅加载 checkpoint 里实际存在的 keys， 避免 DCP planner 因 “Missing key” 直接失败（对于历史不完整 ckpt 也能继续跑，但缺失的权重当然无法凭空恢复）。 -- ✅ 文档化了 `models.shared_component_role` 的语义（默认 student），用于显式指定 shared components owner。 +- ✅ 新增 `recipe.family: wangame_causal`（更 FastGen 风格）：causal by default。 +- ✅ 落地 `roles.shared_component_role`（默认 student），用于显式指定 shared components owner + （并把 `training.model_path` 的默认来源切换到该 role）。 +- ✅ 文档化：补齐 causal model/method 文件头的 config keys 声明（降低阅读成本）。 ## 1. 背景：FastGen 的语义是什么？ @@ -68,13 +71,13 @@ FastGen 在配置层面并不使用 `variant: causal` 这种字段，而是通 ## 3. 配置（YAML）层面的约定 -### 3.1 shared components 的来源：`models.shared_component_role`（默认 student） +### 3.1 shared components 的来源：`roles.shared_component_role`（默认 student） 为了把 “谁负责 shared components” 变成**显式语义**（而不是硬编码只认 student）， 引入一个模型层面的配置段： ```yaml -models: +roles: # 哪个 role 负责加载/持有 shared components（VAE / text encoder / image encoder / …） # 默认：student shared_component_role: student @@ -93,7 +96,7 @@ models: 从简规则（不做 fallback / 不做复杂合并）： -- 如果不写 `models.shared_component_role`，默认等价于 `student`。 +- 如果不写 `roles.shared_component_role`，默认等价于 `student`。 未来如果需要支持 “teacher/student family 不同”，再引入更复杂的机制（例如多套 shared components）。 @@ -163,6 +166,6 @@ fastvideo/distillation/models/wangame/ ## 6. TODO（实施清单） -- [ ] 新增 `recipe.family: wangame_causal`（更 FastGen 风格），默认所有 role 为 causal -- [ ] 落地 `models.shared_component_role`（默认 student）用于 shared components 的 owner（含：`training.model_path` 默认来源切换） -- [ ] 文档化：每个 model/method 文件开头声明本文件会读取的 config keys（降低阅读成本） +- [x] 新增 `recipe.family: wangame_causal`（更 FastGen 风格），默认所有 role 为 causal +- [x] 落地 `roles.shared_component_role`（默认 student）用于 shared components 的 owner（含：`training.model_path` 默认来源切换） +- [x] 文档化：每个 model/method 文件开头声明本文件会读取的 config keys（降低阅读成本） diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml index 42b568503..75c61ac26 100644 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml @@ -1,13 +1,12 @@ recipe: - family: wangame + family: wangame_causal method: dfsft roles: + shared_component_role: student student: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true - variant: causal training: # Distributed (mirror legacy WanGame finetune scripts) diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml index 1bdace7a0..73e8b7254 100644 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml +++ b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml @@ -1,13 +1,12 @@ recipe: - family: wangame + family: wangame_causal method: dfsft roles: + shared_component_role: student student: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true - variant: causal training: # Distributed (4 nodes × 8 GPUs = 32 ranks) diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml index a752f3eef..add07c027 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml @@ -7,30 +7,25 @@ # load configs/architectures); `init_from_checkpoint` overwrites weights. recipe: - family: wangame + family: wangame_causal method: dmd2 roles: + shared_component_role: student student: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true - variant: causal init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 teacher: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: false disable_custom_init_weights: true - variant: causal init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 init_from_checkpoint_role: student critic: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true disable_custom_init_weights: true - variant: causal init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 init_from_checkpoint_role: student @@ -118,4 +113,3 @@ method_config: action: keep image: keep text: keep - diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml index 0dcb00ed0..359d34544 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml @@ -9,30 +9,25 @@ # load configs/architectures); `init_from_checkpoint` overwrites weights. recipe: - family: wangame + family: wangame_causal method: dmd2 roles: + shared_component_role: student student: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true - variant: causal init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 teacher: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: false disable_custom_init_weights: true - variant: causal init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 init_from_checkpoint_role: student critic: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true disable_custom_init_weights: true - variant: causal init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 init_from_checkpoint_role: student diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml index 5d9a47210..ece7f0303 100644 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml +++ b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml @@ -9,30 +9,25 @@ # load configs/architectures); `init_from_checkpoint` overwrites weights. recipe: - family: wangame + family: wangame_causal method: self_forcing roles: + shared_component_role: student student: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true - variant: causal init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 teacher: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: false disable_custom_init_weights: true - variant: causal init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 init_from_checkpoint_role: student critic: - family: wangame path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true disable_custom_init_weights: true - variant: causal init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 init_from_checkpoint_role: student diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/distillation/methods/distribution_matching/self_forcing.py index f74b0a2fc..9fd351c66 100644 --- a/fastvideo/distillation/methods/distribution_matching/self_forcing.py +++ b/fastvideo/distillation/methods/distribution_matching/self_forcing.py @@ -202,7 +202,16 @@ def build( if student_spec is None: raise ValueError("SelfForcingMethod requires roles.student") student_variant = (student_spec.extra or {}).get("variant", None) - if str(student_variant).strip().lower() != "causal": + if student_variant is None or str(student_variant).strip() == "": + recipe_family = str(getattr(cfg.recipe, "family", "")).strip().lower() + student_family = str(getattr(student_spec, "family", "")).strip().lower() + if not (recipe_family.endswith("_causal") or student_family.endswith("_causal")): + raise ValueError( + "SelfForcingMethod requires a causal student. " + "Set roles.student.variant: causal, or use a causal family " + "(e.g. recipe.family: wangame_causal)." + ) + elif str(student_variant).strip().lower() != "causal": raise ValueError( "SelfForcingMethod requires a causal student. " "Set roles.student.variant: causal in the YAML config." diff --git a/fastvideo/distillation/models/wan/wan.py b/fastvideo/distillation/models/wan/wan.py index fb106a211..d4f8d3d0e 100644 --- a/fastvideo/distillation/models/wan/wan.py +++ b/fastvideo/distillation/models/wan/wan.py @@ -4,6 +4,7 @@ Config keys used (YAML schema-v2): - `recipe.family`: must be `"wan"` for this plugin. +- `roles.shared_component_role` (affects default `training.model_path`) - `roles.`: - `family`, `path`, `trainable`, `disable_custom_init_weights` - `training` (selected fields): @@ -82,7 +83,8 @@ def __init__(self, *, cfg: DistillRunConfig) -> None: if not getattr(training_args, "data_path", ""): raise ValueError("training.data_path must be set for distillation") - # Load shared components (student base path). + # Load shared components (training.model_path; defaults to + # roles.shared_component_role's path). training_args.override_transformer_cls_name = "WanTransformer3DModel" vae = load_module_from_path( model_path=str(training_args.model_path), diff --git a/fastvideo/distillation/models/wangame/__init__.py b/fastvideo/distillation/models/wangame/__init__.py index 242628dca..b5dec4de4 100644 --- a/fastvideo/distillation/models/wangame/__init__.py +++ b/fastvideo/distillation/models/wangame/__init__.py @@ -2,7 +2,9 @@ """WanGame model plugin package. -This package registers the `recipe.family: wangame` model builder. +This package registers the WanGame model builders: +- `recipe.family: wangame` (bidirectional by default; can opt into causal per-role) +- `recipe.family: wangame_causal` (causal by default; can opt into bidirectional per-role) We support both bidirectional and causal transformers: - If any role declares `roles..variant: causal`, we build the causal-capable @@ -38,3 +40,9 @@ def _build_wangame_model(*, cfg: DistillRunConfig): return WanGameModel(cfg=cfg) + +@register_model("wangame_causal") +def _build_wangame_causal_model(*, cfg: DistillRunConfig): + from fastvideo.distillation.models.wangame.wangame_causal import WanGameCausalModel + + return WanGameCausalModel(cfg=cfg) diff --git a/fastvideo/distillation/models/wangame/common.py b/fastvideo/distillation/models/wangame/common.py index bc3f72446..6aea5d31d 100644 --- a/fastvideo/distillation/models/wangame/common.py +++ b/fastvideo/distillation/models/wangame/common.py @@ -35,9 +35,10 @@ def _build_wangame_role_handles( ) -> dict[str, RoleHandle]: role_handles: dict[str, RoleHandle] = {} for role, role_spec in roles_cfg.items(): - if role_spec.family != "wangame": + if role_spec.family not in {"wangame", "wangame_causal"}: raise ValueError( - "Wangame model plugin only supports roles with family='wangame'; " + "Wangame model plugin only supports roles with family in " + "{'wangame', 'wangame_causal'}; " f"got {role}={role_spec.family!r}" ) @@ -86,4 +87,3 @@ def _build_wangame_role_handles( ) return role_handles - diff --git a/fastvideo/distillation/models/wangame/wangame.py b/fastvideo/distillation/models/wangame/wangame.py index d5b86b59a..37659aa23 100644 --- a/fastvideo/distillation/models/wangame/wangame.py +++ b/fastvideo/distillation/models/wangame/wangame.py @@ -4,6 +4,7 @@ Config keys used (YAML schema-v2): - `recipe.family`: must be `"wangame"` for this plugin. +- `roles.shared_component_role` (affects default `training.model_path`) - `roles.`: - `family`, `path`, `trainable`, `disable_custom_init_weights` - extra: `variant` is **not used** by this bidirectional-only plugin. @@ -86,7 +87,8 @@ def __init__( if not getattr(training_args, "data_path", ""): raise ValueError("training.data_path must be set for distillation") - # Load shared components (student base path). + # Load shared components (training.model_path; defaults to + # roles.shared_component_role's path). vae = load_module_from_path( model_path=str(training_args.model_path), module_type="vae", diff --git a/fastvideo/distillation/models/wangame/wangame_causal.py b/fastvideo/distillation/models/wangame/wangame_causal.py index 4ecd56cf2..77545214e 100644 --- a/fastvideo/distillation/models/wangame/wangame_causal.py +++ b/fastvideo/distillation/models/wangame/wangame_causal.py @@ -2,12 +2,23 @@ """WanGame causal model plugin (streaming/cache primitives). -This module provides the *causal* extension for the WanGame model family. +Config keys used (YAML schema-v2): +- `recipe.family`: `"wangame_causal"` (causal by default) or `"wangame"` (when + at least one role requests `variant: causal`). +- `roles.shared_component_role` (affects default `training.model_path`). +- `roles.`: + - `family`, `path`, `trainable`, `disable_custom_init_weights` + - extra: `variant` (`causal`/`bidirectional`); defaults depend on + `recipe.family`. +- `training` (selected fields): same as `WanGameModel` (see + `models/wangame/wangame.py`). +- `training.validation.*` (consumed by `WanGameValidator` when enabled) +- `method_config.cfg_uncond.*` (consumed by base WanGame primitives for CFG) Key differences vs. `models/wangame/wangame.py`: -- Supports `roles..variant: causal` by loading a causal transformer class. -- Implements `CausalModelBase` APIs (`clear_caches`, `predict_*_streaming`) so - methods can drive streaming rollouts without passing KV-cache tensors around. +- Supports causal transformers and streaming rollouts via `CausalModelBase`. +- Exposes cache lifecycle + streaming primitives (`predict_*_streaming`), so + methods can drive causal rollouts without passing KV-cache tensors around. """ from __future__ import annotations @@ -52,16 +63,20 @@ def __init__(self, *, cfg: DistillRunConfig) -> None: if not getattr(training_args, "data_path", ""): raise ValueError("training.data_path must be set for distillation") - # Load shared components (student base path). + # Load shared components (training.model_path; defaults to + # roles.shared_component_role's path). vae = self._load_shared_vae(training_args) noise_scheduler = self._build_noise_scheduler(training_args) + recipe_family = str(getattr(cfg.recipe, "family", "")).strip().lower() + default_variant = "causal" if recipe_family == "wangame_causal" else "bidirectional" + def _transformer_cls_name_for_role(role: str, role_spec: Any) -> str: variant_raw = (role_spec.extra or {}).get("variant", None) if variant_raw is None or str(variant_raw).strip() == "": - return "WanGameActionTransformer3DModel" - - variant = str(variant_raw).strip().lower() + variant = default_variant + else: + variant = str(variant_raw).strip().lower() if variant in {"bidirectional", "bidi"}: return "WanGameActionTransformer3DModel" if variant == "causal": diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index b1dbe937c..0486f9fdd 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -43,6 +43,7 @@ class DistillRunConfig: """Parsed distillation run config loaded from schema-v2 YAML.""" recipe: RecipeSpec + shared_component_role: RoleName roles: dict[RoleName, RoleSpec] training_args: TrainingArgs validation: dict[str, Any] @@ -150,9 +151,26 @@ def load_distill_run_config(path: str) -> DistillRunConfig: recipe_method = _require_str(recipe_raw.get("method"), where="recipe.method") recipe = RecipeSpec(family=recipe_family, method=recipe_method) + models_raw = cfg.get("models", None) + if models_raw is not None: + raise ValueError( + "Top-level `models` is not supported in schema-v2. " + "Use `roles.shared_component_role` and `roles..*` instead." + ) + roles_raw = _require_mapping(cfg.get("roles"), where="roles") + shared_component_role_raw = roles_raw.get("shared_component_role", None) + if shared_component_role_raw is None: + shared_component_role = "student" + else: + shared_component_role = _require_str( + shared_component_role_raw, + where="roles.shared_component_role", + ) roles: dict[RoleName, RoleSpec] = {} for role, role_cfg_raw in roles_raw.items(): + if role == "shared_component_role": + continue role_str = _require_str(role, where="roles.") role_cfg = _require_mapping(role_cfg_raw, where=f"roles.{role_str}") family = role_cfg.get("family") or recipe_family @@ -187,6 +205,15 @@ def load_distill_run_config(path: str) -> DistillRunConfig: extra=extra, ) + shared_component_role = str(shared_component_role).strip() + if not shared_component_role: + raise ValueError("roles.shared_component_role cannot be empty") + if shared_component_role not in roles: + raise ValueError( + "roles.shared_component_role must be a role name under roles.*, got " + f"{shared_component_role!r}" + ) + training_raw = _require_mapping(cfg.get("training"), where="training") legacy_validation_keys = { @@ -242,13 +269,27 @@ def load_distill_run_config(path: str) -> DistillRunConfig: training_kwargs.setdefault("hsdp_replicate_dim", 1) training_kwargs.setdefault("hsdp_shard_dim", num_gpus) - # Use student path as the default base model_path. This is needed for - # PipelineConfig registry lookup. + # Use the shared-component role path as the default base model_path. This + # is needed for PipelineConfig registry lookup and shared component loading. + shared_role_spec = roles.get(shared_component_role) + if shared_role_spec is None: + raise ValueError( + "roles.shared_component_role must reference an existing role under roles.*, got " + f"{shared_component_role!r}" + ) + if "model_path" not in training_kwargs: - student = roles.get("student") - if student is None: - raise ValueError("training.model_path is missing and roles.student is not provided") - training_kwargs["model_path"] = student.path + training_kwargs["model_path"] = shared_role_spec.path + else: + model_path_raw = training_kwargs.get("model_path") + model_path = _require_str(model_path_raw, where="training.model_path").rstrip("/") + expected = str(shared_role_spec.path).rstrip("/") + if model_path != expected: + raise ValueError( + "training.model_path must match roles..path. " + f"Got training.model_path={model_path_raw!r}, " + f"roles.{shared_component_role}.path={shared_role_spec.path!r}" + ) if "pretrained_model_name_or_path" not in training_kwargs: training_kwargs["pretrained_model_name_or_path"] = training_kwargs["model_path"] @@ -295,6 +336,7 @@ def load_distill_run_config(path: str) -> DistillRunConfig: return DistillRunConfig( recipe=recipe, + shared_component_role=shared_component_role, roles=roles, training_args=training_args, validation=validation, From fcfae1286e12b43a62a5c6e8750916666a41ef5d Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Tue, 3 Mar 2026 23:41:40 +0000 Subject: [PATCH 158/214] remove redundent varient --- ...angame2.1_i2v_1.3B_bidirectional_4n8g.yaml | 2 -- fastvideo/distillation/doc/models/wangame.md | 13 ++++++---- fastvideo/distillation/doc/utils/config.md | 2 +- .../distillation/doc/utils/moduleloader.md | 3 ++- .../distribution_matching/self_forcing.py | 21 +++++++--------- .../distillation/models/wangame/__init__.py | 25 ++----------------- .../distillation/models/wangame/common.py | 2 +- .../distillation/models/wangame/wangame.py | 13 +++++++++- .../models/wangame/wangame_causal.py | 22 +++++----------- fastvideo/distillation/utils/config.py | 6 +++++ 10 files changed, 47 insertions(+), 62 deletions(-) diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml index ffa03048c..a6935afb6 100644 --- a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml +++ b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml @@ -13,7 +13,6 @@ roles: family: wangame path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers trainable: true - variant: bidirectional training: # Distributed (4 nodes × 8 GPUs = 32 ranks) @@ -87,4 +86,3 @@ default_pipeline_config: method_config: attn_kind: dense - diff --git a/fastvideo/distillation/doc/models/wangame.md b/fastvideo/distillation/doc/models/wangame.md index bc675ed0b..fccf60380 100644 --- a/fastvideo/distillation/doc/models/wangame.md +++ b/fastvideo/distillation/doc/models/wangame.md @@ -16,9 +16,9 @@ - `noise_scheduler`：`FlowMatchEulerDiscreteScheduler(shift=flow_shift)` 2) **按 roles 加载 transformer 模块** - 对每个 role：加载 `transformer`（可选 `transformer_2`） - - 支持 role-level transformer 变体（通过 `RoleSpec.extra`）： - - `roles..variant: bidirectional` → `WanGameActionTransformer3DModel` - - `roles..variant: causal` → `CausalWanGameActionTransformer3DModel` + - role-level transformer 类型由 `roles..family` 决定（而不是 `variant`）： + - `roles..family: wangame` → `WanGameActionTransformer3DModel` + - `roles..family: wangame_causal` → `CausalWanGameActionTransformer3DModel` - 根据 `RoleSpec.trainable` 设置 `requires_grad` - 可选开启 activation checkpoint（仅对 trainable role） 3) **构建 bundle / dataloader / validator** @@ -28,5 +28,8 @@ - runtime primitives 由 `WanGameModel` 直接实现（不再额外分一层 `*Adapter` 类/文件） **关于 roles.family** -- 当前 `wangame` plugin 只支持 `family="wangame"` 的 role。 - 这让 build-time 逻辑保持高内聚：模型加载、batch schema 与 primitives 能保持一致。 +- `recipe.family: wangame`（bidirectional）： + - 只支持 `roles..family="wangame"`，否则直接报错。 +- `recipe.family: wangame_causal`（causal-capable）： + - 支持 `roles..family in {"wangame","wangame_causal"}`，用于 self-forcing + 等场景下的 “student causal + teacher/critic bidirectional” 组合。 diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/distillation/doc/utils/config.md index e9edff7a6..24cac22d9 100644 --- a/fastvideo/distillation/doc/utils/config.md +++ b/fastvideo/distillation/doc/utils/config.md @@ -59,7 +59,7 @@ - `trainable`: 是否训练该 role（只影响 `requires_grad`/模式；具体 optimizer 由 method 决定） - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” - `extra: dict`：保留 `roles.` 下除上述字段外的所有 key/value， - 交给 model plugin 解释（例如 `roles.student.variant: causal`） + 交给 model plugin / method 解释（例如 `roles.student.init_from_checkpoint: ...`） ## 3) Builder 装配相关（build-time / run-time 边界） - model plugin（`@register_model`）直接构建并返回一个 `ModelBase` 实例： diff --git a/fastvideo/distillation/doc/utils/moduleloader.md b/fastvideo/distillation/doc/utils/moduleloader.md index 423e00a41..ea298836b 100644 --- a/fastvideo/distillation/doc/utils/moduleloader.md +++ b/fastvideo/distillation/doc/utils/moduleloader.md @@ -11,7 +11,8 @@ - 调用 `PipelineComponentLoader.load_module(...)` - 可选跳过自定义 init weights（legacy flag：`_loading_teacher_critic_model`） - 可选临时覆盖 `training_args.override_transformer_cls_name` - - 用于 **同一 family** 内按 role 选择 transformer 变体（例如 `roles.student.variant: causal`） + - 用于 model plugin 按 role 选择 transformer 类（例如根据 `roles..family` + 选择 `WanGameActionTransformer3DModel` / `CausalWanGameActionTransformer3DModel`） **边界** - ✅ 这里只做 “单模块加载”，不做 role 语义、也不做 optimizer/scheduler。 diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/distillation/methods/distribution_matching/self_forcing.py index 9fd351c66..9ae64e529 100644 --- a/fastvideo/distillation/methods/distribution_matching/self_forcing.py +++ b/fastvideo/distillation/methods/distribution_matching/self_forcing.py @@ -201,20 +201,17 @@ def build( student_spec = cfg.roles.get("student") if student_spec is None: raise ValueError("SelfForcingMethod requires roles.student") - student_variant = (student_spec.extra or {}).get("variant", None) - if student_variant is None or str(student_variant).strip() == "": - recipe_family = str(getattr(cfg.recipe, "family", "")).strip().lower() - student_family = str(getattr(student_spec, "family", "")).strip().lower() - if not (recipe_family.endswith("_causal") or student_family.endswith("_causal")): - raise ValueError( - "SelfForcingMethod requires a causal student. " - "Set roles.student.variant: causal, or use a causal family " - "(e.g. recipe.family: wangame_causal)." - ) - elif str(student_variant).strip().lower() != "causal": + recipe_family = str(getattr(cfg.recipe, "family", "")).strip().lower() + if not recipe_family.endswith("_causal"): raise ValueError( "SelfForcingMethod requires a causal student. " - "Set roles.student.variant: causal in the YAML config." + f"Got recipe.family={recipe_family!r}." + ) + student_family = str(getattr(student_spec, "family", "")).strip().lower() + if not student_family.endswith("_causal"): + raise ValueError( + "SelfForcingMethod requires roles.student.family to be a causal " + f"family (suffix '_causal'). Got roles.student.family={student_family!r}." ) if not isinstance(model, CausalModelBase): raise ValueError( diff --git a/fastvideo/distillation/models/wangame/__init__.py b/fastvideo/distillation/models/wangame/__init__.py index b5dec4de4..e05c01e0f 100644 --- a/fastvideo/distillation/models/wangame/__init__.py +++ b/fastvideo/distillation/models/wangame/__init__.py @@ -3,13 +3,8 @@ """WanGame model plugin package. This package registers the WanGame model builders: -- `recipe.family: wangame` (bidirectional by default; can opt into causal per-role) -- `recipe.family: wangame_causal` (causal by default; can opt into bidirectional per-role) - -We support both bidirectional and causal transformers: -- If any role declares `roles..variant: causal`, we build the causal-capable - model plugin (which implements `CausalModelBase`). -- Otherwise we build the bidirectional-only model plugin. +- `recipe.family: wangame` (bidirectional) +- `recipe.family: wangame_causal` (causal-capable; supports streaming primitives) """ from __future__ import annotations @@ -20,22 +15,6 @@ @register_model("wangame") def _build_wangame_model(*, cfg: DistillRunConfig): - wants_causal = False - for role, role_spec in cfg.roles.items(): - variant = (role_spec.extra or {}).get("variant", None) - if variant is None: - continue - if str(variant).strip().lower() == "causal": - wants_causal = True - break - - if wants_causal: - from fastvideo.distillation.models.wangame.wangame_causal import ( - WanGameCausalModel, - ) - - return WanGameCausalModel(cfg=cfg) - from fastvideo.distillation.models.wangame.wangame import WanGameModel return WanGameModel(cfg=cfg) diff --git a/fastvideo/distillation/models/wangame/common.py b/fastvideo/distillation/models/wangame/common.py index 6aea5d31d..2d482fb4e 100644 --- a/fastvideo/distillation/models/wangame/common.py +++ b/fastvideo/distillation/models/wangame/common.py @@ -9,7 +9,7 @@ - `fastvideo/distillation/models/wangame/wangame.py` - `fastvideo/distillation/models/wangame/wangame_causal.py` -Keep this file free of imports of the variant-specific model classes to prevent +Keep this file free of imports of the family-specific model classes to prevent circular dependencies. """ diff --git a/fastvideo/distillation/models/wangame/wangame.py b/fastvideo/distillation/models/wangame/wangame.py index 37659aa23..54198dc61 100644 --- a/fastvideo/distillation/models/wangame/wangame.py +++ b/fastvideo/distillation/models/wangame/wangame.py @@ -7,7 +7,6 @@ - `roles.shared_component_role` (affects default `training.model_path`) - `roles.`: - `family`, `path`, `trainable`, `disable_custom_init_weights` - - extra: `variant` is **not used** by this bidirectional-only plugin. - `training` (selected fields): - `seed`, `data_path`, `model_path` - `num_height`, `num_width`, `num_latent_t` @@ -82,6 +81,18 @@ def __init__( training_args = cfg.training_args roles_cfg = cfg.roles + non_wangame_roles = [ + role + for role, spec in roles_cfg.items() + if str(spec.family).strip().lower() != "wangame" + ] + if non_wangame_roles: + raise ValueError( + "recipe.family=wangame only supports roles..family=wangame " + f"(got non-wangame roles: {non_wangame_roles}). " + "If you need causal roles, use recipe.family: wangame_causal." + ) + if getattr(training_args, "seed", None) is None: raise ValueError("training.seed must be set for distillation") if not getattr(training_args, "data_path", ""): diff --git a/fastvideo/distillation/models/wangame/wangame_causal.py b/fastvideo/distillation/models/wangame/wangame_causal.py index 77545214e..0f2a6e533 100644 --- a/fastvideo/distillation/models/wangame/wangame_causal.py +++ b/fastvideo/distillation/models/wangame/wangame_causal.py @@ -3,13 +3,10 @@ """WanGame causal model plugin (streaming/cache primitives). Config keys used (YAML schema-v2): -- `recipe.family`: `"wangame_causal"` (causal by default) or `"wangame"` (when - at least one role requests `variant: causal`). +- `recipe.family`: must be `"wangame_causal"` for this plugin. - `roles.shared_component_role` (affects default `training.model_path`). - `roles.`: - `family`, `path`, `trainable`, `disable_custom_init_weights` - - extra: `variant` (`causal`/`bidirectional`); defaults depend on - `recipe.family`. - `training` (selected fields): same as `WanGameModel` (see `models/wangame/wangame.py`). - `training.validation.*` (consumed by `WanGameValidator` when enabled) @@ -68,22 +65,15 @@ def __init__(self, *, cfg: DistillRunConfig) -> None: vae = self._load_shared_vae(training_args) noise_scheduler = self._build_noise_scheduler(training_args) - recipe_family = str(getattr(cfg.recipe, "family", "")).strip().lower() - default_variant = "causal" if recipe_family == "wangame_causal" else "bidirectional" - def _transformer_cls_name_for_role(role: str, role_spec: Any) -> str: - variant_raw = (role_spec.extra or {}).get("variant", None) - if variant_raw is None or str(variant_raw).strip() == "": - variant = default_variant - else: - variant = str(variant_raw).strip().lower() - if variant in {"bidirectional", "bidi"}: + family = str(getattr(role_spec, "family", "")).strip().lower() + if family == "wangame": return "WanGameActionTransformer3DModel" - if variant == "causal": + if family == "wangame_causal": return "CausalWanGameActionTransformer3DModel" raise ValueError( - f"Unknown roles.{role}.variant for wangame: {variant_raw!r}. " - "Expected 'causal' or 'bidirectional'." + f"Unknown roles.{role}.family for wangame_causal: {family!r}. " + "Expected 'wangame' or 'wangame_causal'." ) role_handles = _build_wangame_role_handles( diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index 0486f9fdd..9a8c4e33f 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -173,6 +173,12 @@ def load_distill_run_config(path: str) -> DistillRunConfig: continue role_str = _require_str(role, where="roles.") role_cfg = _require_mapping(role_cfg_raw, where=f"roles.{role_str}") + if "variant" in role_cfg: + raise ValueError( + f"roles.{role_str}.variant is not supported in schema-v2. " + "Use roles..family to select the model family instead " + "(e.g. family: wangame_causal)." + ) family = role_cfg.get("family") or recipe_family family = _require_str(family, where=f"roles.{role_str}.family") model_path = _require_str(role_cfg.get("path"), where=f"roles.{role_str}.path") From 6d57a0aae3a6f549b50dd266cb3826874de47a0f Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 00:27:25 +0000 Subject: [PATCH 159/214] designing refactor --- dev/phase_refactor.md | 280 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 dev/phase_refactor.md diff --git a/dev/phase_refactor.md b/dev/phase_refactor.md new file mode 100644 index 000000000..2b09a41a1 --- /dev/null +++ b/dev/phase_refactor.md @@ -0,0 +1,280 @@ +# Phase：Refactor（对齐 FastGen：Method 管“有哪些网络实例”，Student 持有 shared parts） + +> 本阶段是一个 **较大重构**，目标是把目前 distillation 框架里“概念过多、职责漂移、读起来像一盘散沙”的问题， +> 收敛成一条更直观、FastGen 风格的主线： +> +> - **Method/Algorithm 决定自己需要哪些网络实例**（student/teacher/critic/fake_score/...），而不是框架层试图泛化所有 role。 +> - **shared / expensive components**（VAE、encoder、scheduler、dataloader、validator 等）只构建一份， +> 默认由 `roles.shared_component_role`（通常是 student）负责加载并提供。 +> - 每个 role 变成一个 **独立的模型实例（RoleModel）**：加载自己的 transformer（bidi 或 causal），并提供 forward primitives。 +> +> 这一思路对齐 FastGen： +> - FastGen 里 `self.net`（student）初始化 preprocessors（`init_preprocessors()`），teacher 只是另一个 network 实例； +> - method/model class（如 `DMD2Model`/`SelfForcingModel`）硬编码构建 teacher/fake_score 等，而不是“传入 roles 列表自动生成”。 + +--- + +## 0) 目标 / 非目标 + +### ✅ 目标 + +1) **去掉 run-level “大 model plugin 内部 per-role if/else 分发 transformer”** + - 例如现在 `wangame_causal` 里出现 `def _transformer_cls_name_for_role(...)` 这种逻辑。 + - 重构后：每个 role 自己对应一个独立 RoleModel（bidi / causal 各是一个 class），不在一个大类里分发。 + +2) **Method 明确声明“我需要哪些网络”** + - `finetune`：只需要 student + - `dmd2`：需要 student + teacher + critic（或 future：fake_score/discriminator） + - `self_forcing`：需要 student（必须 causal）+ teacher/critic（可 bidi） + +3) **shared parts 只加载一份（默认 student 持有）** + - VAE / text encoder / image encoder / scheduler / dataloader / validator + - 由 `roles.shared_component_role` 指定加载来源（默认 student），mismatch 就直接报错（从简、无 fallback）。 + +### ❌ 非目标（本阶段不做） + +- 不追求兼容旧 YAML / legacy entrypoints（从简、只支持新语义）。 +- 不做跨 family 的 teacher/student（例如 teacher=SDXL、student=Wan），遇到直接 error（后续再讨论）。 +- 不在本阶段引入复杂的“shared parts 多套并存”（例如 teacher 也要自己的 VAE）。 + +--- + +## 1) 新的核心对象：`shared_context` + `RoleModel` + +### 1.1 `shared_context`（按 run 只构建一次） + +> 叫 `shared_context` 是为了避免和 backward 的 ctx 混淆。 + +它解决的问题：把“shared / expensive / 与 role 无关，但与 family/数据形态强相关”的东西集中管理。 + +以 **WanGame** 为例，`shared_context` 需要持有： + +- `training_args` +- `dataloader`（parquet schema + workers 等） +- `validator`（可选；具体何时调用由 method 决定） +- shared modules： + - `vae` + - （可选）`text_encoder` / `image_encoder` / tokenizer / image_processor + - `noise_scheduler`（以及 `num_train_timesteps`） +- batch 规范化： + - `prepare_batch(raw_batch) -> TrainingBatch` + - attention metadata builder（VSA/VMoBA）初始化与构建（如果属于 batch schema） +- RNG： + - 训练噪声 RNG（用于 exact resume） + +**代码落点（按你要求）**：直接放在 family 目录里，例如： + +- `fastvideo/distillation/models/wangame/shared_context.py` + +> FastGen 不需要显式 `shared_context.py` 的原因：它把 shared parts 隐式挂在 `self.net`（student）里； +> 我们这里显式拆出来，是因为 FastVideo 的 pipeline/validator/dataloader 语义更重、更适合集中持有（也更利于 checkpoint/resume）。 + +### 1.2 `RoleModel`（每个 role 一个独立实例） + +RoleModel 只负责该 role 的 transformer 以及 forward primitives： + +- `role: str`（用于日志/报错） +- `spec: RoleSpec`（family/path/trainable/extra…） +- `modules`（至少 `transformer`，可选 `transformer_2`） +- `trainable` 标记 + activation checkpointing（仅对 trainable role） + +核心接口（operation-centric）： + +- `predict_noise(shared_context, noisy_latents, timestep, batch, conditional, cfg_uncond, attn_kind)` +- `predict_x0(...)` +- （可选）`backward(loss, bwd_ctx, grad_accum_rounds)`：仅当 backward 需要恢复 forward_context 或处理 checkpoint wrapper 的特殊情况。 + +对 causal role（需要 KV cache / streaming rollout 的角色）： + +- `CausalRoleModelBase` 扩展接口： + - `clear_caches(cache_tag="pos")` + - `predict_noise_streaming(shared_context, noisy_latents, timestep, batch, store_kv, cur_start_frame, ...)` + - `predict_x0_streaming(...)` + +> 对齐 FastGen：cache state 是 RoleModel 的 internal state，method 不传 KV tensor。 + +**代码落点建议**：仍放在 `models//` 里（和 shared_context 同一目录，读起来更高内聚）： + +- `fastvideo/distillation/models/wangame/role_model.py`（bidirectional） +- `fastvideo/distillation/models/wangame/role_model_causal.py`（causal） +- `fastvideo/distillation/models/wangame/common.py`（共享 loader：load transformer / apply_trainable / activation checkpointing） + +--- + +## 2) 配置语义（YAML） + +### 2.1 `recipe.family`：选择 shared_context 的 family（不是“模型变体”） + +重构后建议把 `recipe.family` 的职责收敛为： + +- 选择 shared_context builder（数据 schema + shared parts 语义） + +例如： + +- `recipe.family: wangame` → build `WanGameSharedContext` +- `recipe.family: wan` → build `WanSharedContext` + +**重要变化**：`recipe.family` 不再区分 `wangame` vs `wangame_causal`。 +causal/bidi 的差异由 `roles..family` 决定（RoleModel 的选择）。 + +### 2.2 `roles..family`：选择 RoleModel 类型 + +例： + +- `roles.student.family: wangame_causal` → `WanGameCausalRoleModel` +- `roles.teacher.family: wangame` → `WanGameRoleModel` + +### 2.3 `roles.shared_component_role`：shared parts 的 owner（默认 student） + +```yaml +roles: + shared_component_role: student +``` + +约束（从简）： + +- 必须是一个存在的 role name，否则直接报错。 +- shared parts 的加载来源 = `roles[shared_component_role].path` +- `training.model_path` 必须与该 role 的 `path` 一致（不一致直接 error，防止 silent mismatch）。 + +--- + +## 3) 组装流程（dispatch/build）与调用链 + +### 3.1 build 的顺序（两段式） + +1) build shared_context（由 `recipe.family` 选择） +2) build role_models（对每个 role，根据 `roles..family` 选择 RoleModel class） +3) build method（method 自己声明需要哪些 role，并 fail-fast 校验） + +伪代码： + +```py +shared_context = build_shared_context(cfg) # recipe.family +role_models = build_role_models(cfg, shared_context) # roles..family + +method = Method.build( + cfg=cfg, + roles=RoleManager(role_models), + shared_context=shared_context, + validator=shared_context.validator, +) +``` + +### 3.2 示例调用链（Wangame：student causal self-forcing + teacher bidi） + +假设你用 YAML 启动： + +1) `fastvideo/training/distillation.py` + - `cfg = load_distill_run_config(path)` +2) `fastvideo/distillation/dispatch.py: build_from_config(cfg)` + - `shared_context = WangameSharedContext(cfg)`（从 `roles.shared_component_role.path` 加载 VAE 等） + - `student = WanGameCausalRoleModel(role="student", spec=roles.student, shared_context=...)` + - `teacher = WanGameRoleModel(role="teacher", spec=roles.teacher, shared_context=...)` + - `critic = WanGameRoleModel(role="critic", spec=roles.critic, shared_context=...)` + - `method = SelfForcingMethod.build(cfg, roles=RoleManager(...), shared_context, validator=shared_context.validator)` +3) `fastvideo/distillation/trainer.py: DistillTrainer.run(...)` + - 循环： + - `loss_map, outputs, metrics = method.single_train_step(raw_batch, step, ...)` + - `method.backward(...)`（若需要特殊 backward；否则走默认 `loss.backward()`） + - `method.optimizers_schedulers_step(...)` + - 验证： + - `method.log_validation(step)` → `validator.log_validation(step, request=...)` + +其中 Self-Forcing 的关键点： + +- 强制 `student` 是 causal role model（否则 build 阶段直接 error） +- rollout 时只使用 `student.predict_*_streaming(...)`（内部 cache），teacher/critic 用并行 forward（无需 cache） + +--- + +## 4) 需要修改/新增的文件（TODO 列表） + +> 先以 wangame 落地，跑通后再推广到 wan。 + +### 4.1 shared_context + +- [ ] `fastvideo/distillation/models/wangame/shared_context.py` + - `WanGameSharedContext` + +### 4.2 role models（每个 role 一个实例） + +- [ ] `fastvideo/distillation/models/wangame/role_model.py`（bidi） +- [ ] `fastvideo/distillation/models/wangame/role_model_causal.py`（causal + cache/streaming） +- [ ] `fastvideo/distillation/models/wangame/common.py`（共享 loader 工具） + +### 4.3 dispatch（从 “build model plugin” 改为 “build shared_context + role_models”） + +- [ ] `fastvideo/distillation/dispatch.py` + - 注册与构建：`@register_shared_context` / `@register_role_model` + +### 4.4 methods（对齐 FastGen 的“继承表达差异点”） + +- [ ] `fastvideo/distillation/methods/base.py` + - `build(..., roles, shared_context, validator)`（替代 bundle+model plugin） +- [ ] `dmd2.py` 作为 DM 基类：self_forcing 只 override rollout +- [ ] `finetune.py` 作为 SFT 基类：dfsft 只 override timestep/chunk 采样（尽量复用 validation/optim/backward） + +--- + +## 5) Refactor 后示例 YAML（Self-Forcing：student causal + teacher bidi） + +> 下面仅展示关键结构；其余 training 超参按你现有 wangame YAML 填即可。 + +```yaml +recipe: + family: wangame + method: self_forcing + +roles: + shared_component_role: student + + student: + family: wangame_causal + path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + trainable: true + + teacher: + family: wangame + path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + trainable: false + + critic: + family: wangame + path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + trainable: true + +training: + # 必须：training.model_path 与 shared_component_role.path 一致（从简、fail-fast） + model_path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + data_path: /path/to/wangame/parquet + seed: 1000 + output_dir: outputs/wangame_self_forcing_refactor + max_train_steps: 100000 + mixed_precision: bf16 + # ... 其余保持现有 + + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json + every_steps: 100 + sampling_steps: [4] + sampler_kind: sde + rollout_mode: streaming + +method_config: + rollout_mode: simulate + dmd_denoising_steps: [4] # 示例 + chunk_size: 3 + student_sample_type: sde + context_noise: 0.0 +``` + +--- + +## 6) 关键取舍（你 review 时最需要确认） + +1) `recipe.family` 是否收敛为 “shared_context family”（我建议是）。 +2) backward hook 的归属：默认 `loss.backward()`，只在必要时让 RoleModel 提供 `backward()`。 +3) validator 的依赖：validator 通常更依赖 shared_context（vae/scheduler/pipeline_config），而不是 role_models。 + From 5179f84ba694c25f379d142cd76f89f0476b0f19 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 00:33:35 +0000 Subject: [PATCH 160/214] designing refactor 2 --- dev/phase_refactor.md | 207 +++++++++++++++++++++++------------------- 1 file changed, 112 insertions(+), 95 deletions(-) diff --git a/dev/phase_refactor.md b/dev/phase_refactor.md index 2b09a41a1..22c8d761e 100644 --- a/dev/phase_refactor.md +++ b/dev/phase_refactor.md @@ -1,12 +1,68 @@ # Phase：Refactor（对齐 FastGen：Method 管“有哪些网络实例”，Student 持有 shared parts） +## 最终 YAML 示例（Self-Forcing：student=wangame_causal，teacher=wangame） + +> 这是我建议的“最终形态”。我会按 **FastGen 的 instantiate 思路** 来设计：YAML 允许用 `family/method` 简写， +> 也允许（更推荐）用 `_target_`/class-path 直接指向类，这样我们不需要维护一堆 registry 表。 + +```yaml +recipe: + family: wangame + method: self_forcing + +models: + # shared / expensive components 的来源（VAE / encoders / scheduler registry / …） + # 默认：student + shared_component_model: student + + student: + family: wangame_causal + path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + trainable: true + + teacher: + family: wangame + path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + trainable: false + + critic: + family: wangame + path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + trainable: true + +training: + # 必须：training.model_path 与 models..path 一致（从简、fail-fast） + model_path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + data_path: /path/to/wangame/parquet + seed: 1000 + output_dir: outputs/wangame_self_forcing_refactor + max_train_steps: 100000 + mixed_precision: bf16 + # ... 其余保持现有 + + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json + every_steps: 100 + sampling_steps: [4] + sampler_kind: sde + rollout_mode: streaming + +method_config: + rollout_mode: simulate + dmd_denoising_steps: [4] + chunk_size: 3 + student_sample_type: sde + context_noise: 0.0 +``` + > 本阶段是一个 **较大重构**，目标是把目前 distillation 框架里“概念过多、职责漂移、读起来像一盘散沙”的问题， > 收敛成一条更直观、FastGen 风格的主线： > > - **Method/Algorithm 决定自己需要哪些网络实例**（student/teacher/critic/fake_score/...），而不是框架层试图泛化所有 role。 > - **shared / expensive components**（VAE、encoder、scheduler、dataloader、validator 等）只构建一份， -> 默认由 `roles.shared_component_role`（通常是 student）负责加载并提供。 -> - 每个 role 变成一个 **独立的模型实例（RoleModel）**：加载自己的 transformer（bidi 或 causal），并提供 forward primitives。 +> 默认由 `models.shared_component_model`（通常是 student）负责加载并提供。 +> - 每个 role 变成一个 **独立的模型实例（Model）**：加载自己的 transformer（bidi 或 causal），并提供 forward primitives。 > > 这一思路对齐 FastGen： > - FastGen 里 `self.net`（student）初始化 preprocessors（`init_preprocessors()`），teacher 只是另一个 network 实例； @@ -20,7 +76,7 @@ 1) **去掉 run-level “大 model plugin 内部 per-role if/else 分发 transformer”** - 例如现在 `wangame_causal` 里出现 `def _transformer_cls_name_for_role(...)` 这种逻辑。 - - 重构后：每个 role 自己对应一个独立 RoleModel（bidi / causal 各是一个 class），不在一个大类里分发。 + - 重构后：每个 role 自己对应一个独立 Model（bidi / causal 各是一个 class），不在一个大类里分发。 2) **Method 明确声明“我需要哪些网络”** - `finetune`：只需要 student @@ -29,7 +85,7 @@ 3) **shared parts 只加载一份（默认 student 持有）** - VAE / text encoder / image encoder / scheduler / dataloader / validator - - 由 `roles.shared_component_role` 指定加载来源（默认 student），mismatch 就直接报错（从简、无 fallback）。 + - 由 `models.shared_component_model` 指定加载来源（默认 student），mismatch 就直接报错（从简、无 fallback）。 ### ❌ 非目标（本阶段不做） @@ -39,7 +95,7 @@ --- -## 1) 新的核心对象：`shared_context` + `RoleModel` +## 1) 新的核心对象：`shared_context` + `Model` ### 1.1 `shared_context`（按 run 只构建一次） @@ -69,9 +125,9 @@ > FastGen 不需要显式 `shared_context.py` 的原因：它把 shared parts 隐式挂在 `self.net`（student）里； > 我们这里显式拆出来，是因为 FastVideo 的 pipeline/validator/dataloader 语义更重、更适合集中持有（也更利于 checkpoint/resume）。 -### 1.2 `RoleModel`（每个 role 一个独立实例） +### 1.2 `Model`（每个 role 一个独立实例） -RoleModel 只负责该 role 的 transformer 以及 forward primitives： +Model 只负责该 role 的 transformer 以及 forward primitives： - `role: str`（用于日志/报错） - `spec: RoleSpec`（family/path/trainable/extra…） @@ -86,22 +142,34 @@ RoleModel 只负责该 role 的 transformer 以及 forward primitives： 对 causal role（需要 KV cache / streaming rollout 的角色）： -- `CausalRoleModelBase` 扩展接口： +- `CausalModelBase` 扩展接口： - `clear_caches(cache_tag="pos")` - `predict_noise_streaming(shared_context, noisy_latents, timestep, batch, store_kv, cur_start_frame, ...)` - `predict_x0_streaming(...)` -> 对齐 FastGen：cache state 是 RoleModel 的 internal state，method 不传 KV tensor。 +> 对齐 FastGen：cache state 是 Model 的 internal state，method 不传 KV tensor。 -**代码落点建议**：仍放在 `models//` 里（和 shared_context 同一目录，读起来更高内聚）： +**代码落点（按你要求）**：直接复用现有文件名，不新建额外的 model 文件： -- `fastvideo/distillation/models/wangame/role_model.py`（bidirectional） -- `fastvideo/distillation/models/wangame/role_model_causal.py`（causal） +- `fastvideo/distillation/models/wangame/wangame.py`（bidirectional model；每个 role 一个实例） +- `fastvideo/distillation/models/wangame/wangame_causal.py`（causal model；每个 role 一个实例，含 cache/streaming primitives） - `fastvideo/distillation/models/wangame/common.py`（共享 loader：load transformer / apply_trainable / activation checkpointing） --- -## 2) 配置语义（YAML） +## 2) 配置语义（YAML）与 FastGen-style instantiate + +我同意你说的：FastGen 的 `instantiate(...)` + 继承（`super().build_model()`）可以自动处理掉很多通用装配逻辑。 + +在我们这里，对应的改法是： + +- 把 `dispatch.py` 里的 `@register_model/@register_method` 这类 registry 变薄，甚至直接去掉； +- 改成一个 `instantiate_from_config()`： + - 支持从 YAML 的 `family/method` 解析到 python 类（可以是 class-path，也可以是约定式 import 路径）； + - 然后由 method 的 `build()`/`super().build()` 串起 shared_context + 多模型实例构建。 + +> 关键点：不是“没有 dispatch/registry”，而是把它变成 **config-driven instantiate**（更像 FastGen）， +> 同时让继承树表达算法差异，避免每个 method copy 组装细节。 ### 2.1 `recipe.family`：选择 shared_context 的 family（不是“模型变体”） @@ -115,26 +183,26 @@ RoleModel 只负责该 role 的 transformer 以及 forward primitives： - `recipe.family: wan` → build `WanSharedContext` **重要变化**：`recipe.family` 不再区分 `wangame` vs `wangame_causal`。 -causal/bidi 的差异由 `roles..family` 决定（RoleModel 的选择）。 +causal/bidi 的差异由 `models..family` 决定（Model 的选择）。 -### 2.2 `roles..family`：选择 RoleModel 类型 +### 2.2 `models..family`：选择 Model 类型 例： -- `roles.student.family: wangame_causal` → `WanGameCausalRoleModel` -- `roles.teacher.family: wangame` → `WanGameRoleModel` +- `models.student.family: wangame_causal` → `WanGameCausalModel` +- `models.teacher.family: wangame` → `WanGameModel` -### 2.3 `roles.shared_component_role`：shared parts 的 owner（默认 student） +### 2.3 `models.shared_component_model`：shared parts 的 owner（默认 student） ```yaml -roles: - shared_component_role: student +models: + shared_component_model: student ``` 约束（从简）： - 必须是一个存在的 role name，否则直接报错。 -- shared parts 的加载来源 = `roles[shared_component_role].path` +- shared parts 的加载来源 = `models[shared_component_model].path` - `training.model_path` 必须与该 role 的 `path` 一致（不一致直接 error，防止 silent mismatch）。 --- @@ -144,18 +212,18 @@ roles: ### 3.1 build 的顺序（两段式） 1) build shared_context（由 `recipe.family` 选择） -2) build role_models（对每个 role，根据 `roles..family` 选择 RoleModel class） +2) build models（对每个 role，根据 `models..family` 选择 Model class） 3) build method（method 自己声明需要哪些 role，并 fail-fast 校验） 伪代码： ```py shared_context = build_shared_context(cfg) # recipe.family -role_models = build_role_models(cfg, shared_context) # roles..family +models = build_models(cfg, shared_context) # models..family method = Method.build( cfg=cfg, - roles=RoleManager(role_models), + models=RoleManager(models), shared_context=shared_context, validator=shared_context.validator, ) @@ -167,12 +235,12 @@ method = Method.build( 1) `fastvideo/training/distillation.py` - `cfg = load_distill_run_config(path)` -2) `fastvideo/distillation/dispatch.py: build_from_config(cfg)` - - `shared_context = WangameSharedContext(cfg)`（从 `roles.shared_component_role.path` 加载 VAE 等） - - `student = WanGameCausalRoleModel(role="student", spec=roles.student, shared_context=...)` - - `teacher = WanGameRoleModel(role="teacher", spec=roles.teacher, shared_context=...)` - - `critic = WanGameRoleModel(role="critic", spec=roles.critic, shared_context=...)` - - `method = SelfForcingMethod.build(cfg, roles=RoleManager(...), shared_context, validator=shared_context.validator)` +2) `fastvideo/distillation/dispatch.py: build_from_config(cfg)`（或未来直接在 method.build 里完成） + - `shared_context = WangameSharedContext(cfg)`（从 `models.shared_component_model.path` 加载 VAE 等） + - `student = WanGameCausalModel(role="student", spec=models.student, shared_context=...)` + - `teacher = WanGameModel(role="teacher", spec=models.teacher, shared_context=...)` + - `critic = WanGameModel(role="critic", spec=models.critic, shared_context=...)` + - `method = SelfForcingMethod.build(cfg, models=RoleManager(...), shared_context, validator=shared_context.validator)` 3) `fastvideo/distillation/trainer.py: DistillTrainer.run(...)` - 循环： - `loss_map, outputs, metrics = method.single_train_step(raw_batch, step, ...)` @@ -183,7 +251,7 @@ method = Method.build( 其中 Self-Forcing 的关键点： -- 强制 `student` 是 causal role model（否则 build 阶段直接 error） +- 强制 `student` 是 causal model（否则 build 阶段直接 error） - rollout 时只使用 `student.predict_*_streaming(...)`（内部 cache），teacher/critic 用并行 forward（无需 cache） --- @@ -197,84 +265,33 @@ method = Method.build( - [ ] `fastvideo/distillation/models/wangame/shared_context.py` - `WanGameSharedContext` -### 4.2 role models（每个 role 一个实例） +### 4.2 models（每个 role 一个实例） -- [ ] `fastvideo/distillation/models/wangame/role_model.py`（bidi） -- [ ] `fastvideo/distillation/models/wangame/role_model_causal.py`（causal + cache/streaming） +> 这里不再引入 `RoleModel` 这个名字；每个 role 就是一个 **独立 Model 实例**。 + +- [ ] `fastvideo/distillation/models/wangame/wangame.py`（bidi；每个 role 一个实例） +- [ ] `fastvideo/distillation/models/wangame/wangame_causal.py`（causal + cache/streaming；每个 role 一个实例） - [ ] `fastvideo/distillation/models/wangame/common.py`（共享 loader 工具） -### 4.3 dispatch（从 “build model plugin” 改为 “build shared_context + role_models”） +### 4.3 dispatch（从 “registry 驱动” 改为 “instantiate 驱动”） - [ ] `fastvideo/distillation/dispatch.py` - - 注册与构建：`@register_shared_context` / `@register_role_model` + - 只做入口编排：load yaml → instantiate method → `method.build_run(...)`（或 `method.build(...)`）→ trainer.run +- [ ] `fastvideo/distillation/utils/instantiate.py` + - `instantiate(cfg)`：优先 `_target_`，否则用 `recipe.method`/`models.*.family` 的约定式解析 + - **继承 + `super().build_*()`** 负责复用通用装配逻辑（对齐 FastGen） ### 4.4 methods（对齐 FastGen 的“继承表达差异点”） - [ ] `fastvideo/distillation/methods/base.py` - - `build(..., roles, shared_context, validator)`（替代 bundle+model plugin） + - `build(..., models, shared_context, validator)`（替代 bundle+model plugin） - [ ] `dmd2.py` 作为 DM 基类：self_forcing 只 override rollout - [ ] `finetune.py` 作为 SFT 基类：dfsft 只 override timestep/chunk 采样（尽量复用 validation/optim/backward） --- -## 5) Refactor 后示例 YAML（Self-Forcing：student causal + teacher bidi） - -> 下面仅展示关键结构；其余 training 超参按你现有 wangame YAML 填即可。 - -```yaml -recipe: - family: wangame - method: self_forcing - -roles: - shared_component_role: student - - student: - family: wangame_causal - path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - trainable: true - - teacher: - family: wangame - path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers - trainable: false - - critic: - family: wangame - path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - trainable: true - -training: - # 必须：training.model_path 与 shared_component_role.path 一致（从简、fail-fast） - model_path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - data_path: /path/to/wangame/parquet - seed: 1000 - output_dir: outputs/wangame_self_forcing_refactor - max_train_steps: 100000 - mixed_precision: bf16 - # ... 其余保持现有 - - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json - every_steps: 100 - sampling_steps: [4] - sampler_kind: sde - rollout_mode: streaming - -method_config: - rollout_mode: simulate - dmd_denoising_steps: [4] # 示例 - chunk_size: 3 - student_sample_type: sde - context_noise: 0.0 -``` - ---- - ## 6) 关键取舍（你 review 时最需要确认） 1) `recipe.family` 是否收敛为 “shared_context family”（我建议是）。 -2) backward hook 的归属：默认 `loss.backward()`，只在必要时让 RoleModel 提供 `backward()`。 -3) validator 的依赖：validator 通常更依赖 shared_context（vae/scheduler/pipeline_config），而不是 role_models。 - +2) backward hook 的归属：默认 `loss.backward()`，只在必要时让 Model 提供 `backward()`。 +3) validator 的依赖：validator 通常更依赖 shared_context（vae/scheduler/pipeline_config），而不是 models。 From a82208a0c623645ae57931c7461f879662574a0e Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 00:43:18 +0000 Subject: [PATCH 161/214] designing refactor 3 --- dev/phase_refactor.md | 345 +++++++++++++++--------------------------- 1 file changed, 120 insertions(+), 225 deletions(-) diff --git a/dev/phase_refactor.md b/dev/phase_refactor.md index 22c8d761e..f31cecb52 100644 --- a/dev/phase_refactor.md +++ b/dev/phase_refactor.md @@ -1,297 +1,192 @@ -# Phase：Refactor（对齐 FastGen：Method 管“有哪些网络实例”，Student 持有 shared parts） +# Phase：Refactor（抛弃 “Distill” 命名：这是通用 Training 框架；YAML=instantiate-first） -## 最终 YAML 示例（Self-Forcing：student=wangame_causal，teacher=wangame） +## 0) 最终 YAML 示例（Self-Forcing：Student=Causal Wangame，Teacher=Bidirectional Wangame） -> 这是我建议的“最终形态”。我会按 **FastGen 的 instantiate 思路** 来设计：YAML 允许用 `family/method` 简写， -> 也允许（更推荐）用 `_target_`/class-path 直接指向类，这样我们不需要维护一堆 registry 表。 +> 你提的方向我同意：YAML 里不要再出现 `wangame/dmd2` 这种“缩写字符串”。 +> 直接写 **类名/类路径**，用 FastGen 同款 `instantiate` 思路（Hydra 风格 `_target_`），避免 registry/if-else。 ```yaml -recipe: - family: wangame - method: self_forcing +log: + project: fastvideo + group: wangame + name: self_forcing_causal_student_bidi_teacher + wandb_mode: online + +trainer: + # trainer 本身也可以不 instantiate（只当作纯参数 dict） + # 但为了彻底对齐 FastGen，我们允许它也走 _target_。 + _target_: fastvideo.distillation.trainer.Trainer + output_dir: outputs/wangame_self_forcing_refactor + max_train_steps: 100000 + seed: 1000 + mixed_precision: bf16 + grad_accum_rounds: 1 -models: - # shared / expensive components 的来源（VAE / encoders / scheduler registry / …） - # 默认：student - shared_component_model: student +data: + _target_: fastvideo.distillation.utils.dataloader.ParquetDataConfig + data_path: /path/to/wangame/parquet + dataloader_num_workers: 4 + +validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json + every_steps: 100 + sampling_steps: [4] + rollout_mode: streaming + # 不固定字段：由 method/validator 自己按需读取（从简） + pipeline: + sampler_kind: sde + scheduler: + _target_: fastvideo.models.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler + flow_shift: 3 + +shared_context: + # 替代旧的 recipe.family：shared_context 是一个“可实例化对象” + _target_: fastvideo.distillation.models.wangame.shared_context.WanGameSharedContext + # shared parts（VAE/encoders/schedulers/validator 等）只构建一份，来源由该路径决定 + model_path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 +models: + # 每个 role 一个独立 Model 实例（method 决定需要哪些 role） student: - family: wangame_causal - path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + _target_: fastvideo.distillation.models.wangame.wangame_causal.WanGameCausalModel + init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 trainable: true teacher: - family: wangame - path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + _target_: fastvideo.distillation.models.wangame.wangame.WanGameModel + init_from: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers trainable: false critic: - family: wangame - path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + _target_: fastvideo.distillation.models.wangame.wangame.WanGameModel + init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 trainable: true -training: - # 必须：training.model_path 与 models..path 一致（从简、fail-fast） - model_path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - data_path: /path/to/wangame/parquet - seed: 1000 - output_dir: outputs/wangame_self_forcing_refactor - max_train_steps: 100000 - mixed_precision: bf16 - # ... 其余保持现有 - - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json - every_steps: 100 - sampling_steps: [4] - sampler_kind: sde - rollout_mode: streaming - -method_config: +method: + _target_: fastvideo.distillation.methods.distribution_matching.self_forcing.SelfForcingMethod + # method-specific config（随 method 变化，不强行统一 schema） rollout_mode: simulate - dmd_denoising_steps: [4] chunk_size: 3 student_sample_type: sde context_noise: 0.0 ``` -> 本阶段是一个 **较大重构**，目标是把目前 distillation 框架里“概念过多、职责漂移、读起来像一盘散沙”的问题， -> 收敛成一条更直观、FastGen 风格的主线： -> -> - **Method/Algorithm 决定自己需要哪些网络实例**（student/teacher/critic/fake_score/...），而不是框架层试图泛化所有 role。 -> - **shared / expensive components**（VAE、encoder、scheduler、dataloader、validator 等）只构建一份， -> 默认由 `models.shared_component_model`（通常是 student）负责加载并提供。 -> - 每个 role 变成一个 **独立的模型实例（Model）**：加载自己的 transformer（bidi 或 causal），并提供 forward primitives。 -> -> 这一思路对齐 FastGen： -> - FastGen 里 `self.net`（student）初始化 preprocessors（`init_preprocessors()`），teacher 只是另一个 network 实例； -> - method/model class（如 `DMD2Model`/`SelfForcingModel`）硬编码构建 teacher/fake_score 等，而不是“传入 roles 列表自动生成”。 - --- -## 0) 目标 / 非目标 +## 1) 为什么要从 “Distill” 改成 “Training” -### ✅ 目标 +现状里我们已经同时支持： -1) **去掉 run-level “大 model plugin 内部 per-role if/else 分发 transformer”** - - 例如现在 `wangame_causal` 里出现 `def _transformer_cls_name_for_role(...)` 这种逻辑。 - - 重构后：每个 role 自己对应一个独立 Model（bidi / causal 各是一个 class），不在一个大类里分发。 +- few-step distillation（DMD2 / Self-Forcing） +- finetuning / SFT +- DFSFT（causal + streaming rollout） -2) **Method 明确声明“我需要哪些网络”** - - `finetune`：只需要 student - - `dmd2`：需要 student + teacher + critic（或 future：fake_score/discriminator） - - `self_forcing`：需要 student（必须 causal）+ teacher/critic（可 bidi） +所以 “distill” 只是其中一种训练策略；框架本质更像： -3) **shared parts 只加载一份（默认 student 持有）** - - VAE / text encoder / image encoder / scheduler / dataloader / validator - - 由 `models.shared_component_model` 指定加载来源（默认 student），mismatch 就直接报错（从简、无 fallback）。 +> **Trainer（loop） + Method（algorithm） + Model（per-role instance） + SharedContext（shared parts）** -### ❌ 非目标（本阶段不做） - -- 不追求兼容旧 YAML / legacy entrypoints（从简、只支持新语义）。 -- 不做跨 family 的 teacher/student（例如 teacher=SDXL、student=Wan），遇到直接 error（后续再讨论）。 -- 不在本阶段引入复杂的“shared parts 多套并存”（例如 teacher 也要自己的 VAE）。 +本次 refactor 的目标是把命名与 config 语义对齐，让人第一眼就能理解“这是训练框架”，而不是“蒸馏脚手架”。 --- -## 1) 新的核心对象：`shared_context` + `Model` - -### 1.1 `shared_context`（按 run 只构建一次） - -> 叫 `shared_context` 是为了避免和 backward 的 ctx 混淆。 - -它解决的问题：把“shared / expensive / 与 role 无关，但与 family/数据形态强相关”的东西集中管理。 - -以 **WanGame** 为例，`shared_context` 需要持有： - -- `training_args` -- `dataloader`（parquet schema + workers 等） -- `validator`（可选；具体何时调用由 method 决定） -- shared modules： - - `vae` - - （可选）`text_encoder` / `image_encoder` / tokenizer / image_processor - - `noise_scheduler`（以及 `num_train_timesteps`） -- batch 规范化： - - `prepare_batch(raw_batch) -> TrainingBatch` - - attention metadata builder（VSA/VMoBA）初始化与构建（如果属于 batch schema） -- RNG： - - 训练噪声 RNG（用于 exact resume） - -**代码落点（按你要求）**：直接放在 family 目录里，例如： - -- `fastvideo/distillation/models/wangame/shared_context.py` - -> FastGen 不需要显式 `shared_context.py` 的原因：它把 shared parts 隐式挂在 `self.net`（student）里； -> 我们这里显式拆出来，是因为 FastVideo 的 pipeline/validator/dataloader 语义更重、更适合集中持有（也更利于 checkpoint/resume）。 - -### 1.2 `Model`（每个 role 一个独立实例） +## 2) 参考 FastGen：我们要“照搬”的点是什么 -Model 只负责该 role 的 transformer 以及 forward primitives： +FastGen 的几个关键点（对应到我们要做的事）： -- `role: str`（用于日志/报错） -- `spec: RoleSpec`（family/path/trainable/extra…） -- `modules`（至少 `transformer`，可选 `transformer_2`） -- `trainable` 标记 + activation checkpointing（仅对 trainable role） +1) **instantiate-first** + - FastGen 用 `LazyCall/_target_ + instantiate()`：config 里写“我要哪个类、给它什么参数”，代码只负责实例化与编排。 + - 我们的 YAML 也应如此：不再让 `dispatch.py` 维护字符串 registry（`wangame`/`dmd2`/...）。 -核心接口（operation-centric）： +2) **继承 + `super().build_*()` 复用装配逻辑** + - 通用的：建 shared parts、建 networks、DDP/FSDP wrapper、optim/scheduler、checkpoint、logging。 + - method 子类只 override：需要哪些 models、rollout/损失、更新策略。 -- `predict_noise(shared_context, noisy_latents, timestep, batch, conditional, cfg_uncond, attn_kind)` -- `predict_x0(...)` -- （可选）`backward(loss, bwd_ctx, grad_accum_rounds)`：仅当 backward 需要恢复 forward_context 或处理 checkpoint wrapper 的特殊情况。 - -对 causal role（需要 KV cache / streaming rollout 的角色）： - -- `CausalModelBase` 扩展接口： - - `clear_caches(cache_tag="pos")` - - `predict_noise_streaming(shared_context, noisy_latents, timestep, batch, store_kv, cur_start_frame, ...)` - - `predict_x0_streaming(...)` - -> 对齐 FastGen：cache state 是 Model 的 internal state，method 不传 KV tensor。 - -**代码落点（按你要求）**：直接复用现有文件名，不新建额外的 model 文件： - -- `fastvideo/distillation/models/wangame/wangame.py`（bidirectional model；每个 role 一个实例） -- `fastvideo/distillation/models/wangame/wangame_causal.py`（causal model；每个 role 一个实例，含 cache/streaming primitives） -- `fastvideo/distillation/models/wangame/common.py`（共享 loader：load transformer / apply_trainable / activation checkpointing） +3) **Student 持有 shared parts（或显式 shared_context）** + - FastGen 把 preprocessors 隐式挂到 `self.net`（student）； + - 我们这边 shared parts 语义更重（pipeline/validator/dataloader），更适合显式 `shared_context`（但 config 同样 instantiate）。 --- -## 2) 配置语义（YAML）与 FastGen-style instantiate - -我同意你说的：FastGen 的 `instantiate(...)` + 继承（`super().build_model()`）可以自动处理掉很多通用装配逻辑。 - -在我们这里，对应的改法是： - -- 把 `dispatch.py` 里的 `@register_model/@register_method` 这类 registry 变薄，甚至直接去掉； -- 改成一个 `instantiate_from_config()`： - - 支持从 YAML 的 `family/method` 解析到 python 类（可以是 class-path，也可以是约定式 import 路径）； - - 然后由 method 的 `build()`/`super().build()` 串起 shared_context + 多模型实例构建。 +## 3) 新 YAML 的核心语义（草案） -> 关键点：不是“没有 dispatch/registry”，而是把它变成 **config-driven instantiate**（更像 FastGen）， -> 同时让继承树表达算法差异，避免每个 method copy 组装细节。 +### 3.1 `_target_`：一切都可 instantiate -### 2.1 `recipe.family`：选择 shared_context 的 family（不是“模型变体”） +- `method._target_`：算法/训练策略（SelfForcing/DMD2/Finetune/DFSFT） +- `models.._target_`：每个 role 的 Model class（bidi/causal） +- `shared_context._target_`：shared parts 的构建者（按任务/数据形态） +- （可选）`trainer._target_`：训练 loop（我们也可以先只传纯参数，避免 trainer 过度可插拔） -重构后建议把 `recipe.family` 的职责收敛为： +### 3.2 去掉 `family/method` 缩写 -- 选择 shared_context builder（数据 schema + shared parts 语义） +不再支持： -例如： +- `recipe.family: wangame` +- `models.student.family: wangame_causal` +- `recipe.method: self_forcing` -- `recipe.family: wangame` → build `WanGameSharedContext` -- `recipe.family: wan` → build `WanSharedContext` +只保留 `_target_`（或 `class_path` 同义字段），避免隐式映射导致概念膨胀。 -**重要变化**：`recipe.family` 不再区分 `wangame` vs `wangame_causal`。 -causal/bidi 的差异由 `models..family` 决定（Model 的选择）。 +### 3.3 “shared_component_model” 的位置 -### 2.2 `models..family`：选择 Model 类型 +我建议 **直接去掉 `shared_component_model`**： -例： +- shared parts 的来源已经由 `shared_context.model_path` 显式指定； +- 再用一个 role 去“引用/指代”来源，属于重复表达，反而会让人误以为 role 会影响 shared parts 的语义。 -- `models.student.family: wangame_causal` → `WanGameCausalModel` -- `models.teacher.family: wangame` → `WanGameModel` +从简后的约束： -### 2.3 `models.shared_component_model`：shared parts 的 owner（默认 student） - -```yaml -models: - shared_component_model: student -``` - -约束（从简）： - -- 必须是一个存在的 role name，否则直接报错。 -- shared parts 的加载来源 = `models[shared_component_model].path` -- `training.model_path` 必须与该 role 的 `path` 一致（不一致直接 error，防止 silent mismatch）。 +- 只要求 `shared_context.model_path` 必填； +- 是否与 `models.student.init_from` 一致由用户自行保证（如果未来要 strict 校验，再引入显式的 `shared_context.strict_match` 之类字段）。 --- -## 3) 组装流程（dispatch/build）与调用链 +## 4) 组装流程（调用链） -### 3.1 build 的顺序（两段式） +伪代码（对齐 FastGen：method 主导装配）： -1) build shared_context（由 `recipe.family` 选择） -2) build models（对每个 role，根据 `models..family` 选择 Model class） -3) build method（method 自己声明需要哪些 role，并 fail-fast 校验） +```py +cfg = load_yaml(path) -伪代码： +method = instantiate(cfg["method"], cfg=cfg) # method class -```py -shared_context = build_shared_context(cfg) # recipe.family -models = build_models(cfg, shared_context) # models..family +shared_context = instantiate(cfg["shared_context"], cfg=cfg) # build shared parts once +models = { + role: instantiate(model_cfg, role=role, shared_context=shared_context) + for role, model_cfg in cfg["models"].items() +} -method = Method.build( +run_objects = method.build_run( cfg=cfg, - models=RoleManager(models), shared_context=shared_context, - validator=shared_context.validator, + models=models, ) -``` - -### 3.2 示例调用链（Wangame：student causal self-forcing + teacher bidi） - -假设你用 YAML 启动： -1) `fastvideo/training/distillation.py` - - `cfg = load_distill_run_config(path)` -2) `fastvideo/distillation/dispatch.py: build_from_config(cfg)`（或未来直接在 method.build 里完成） - - `shared_context = WangameSharedContext(cfg)`（从 `models.shared_component_model.path` 加载 VAE 等） - - `student = WanGameCausalModel(role="student", spec=models.student, shared_context=...)` - - `teacher = WanGameModel(role="teacher", spec=models.teacher, shared_context=...)` - - `critic = WanGameModel(role="critic", spec=models.critic, shared_context=...)` - - `method = SelfForcingMethod.build(cfg, models=RoleManager(...), shared_context, validator=shared_context.validator)` -3) `fastvideo/distillation/trainer.py: DistillTrainer.run(...)` - - 循环： - - `loss_map, outputs, metrics = method.single_train_step(raw_batch, step, ...)` - - `method.backward(...)`（若需要特殊 backward；否则走默认 `loss.backward()`） - - `method.optimizers_schedulers_step(...)` - - 验证： - - `method.log_validation(step)` → `validator.log_validation(step, request=...)` +trainer = instantiate(cfg["trainer"], cfg=cfg, **run_objects.trainer_inputs) +trainer.run(**run_objects.run_inputs) +``` -其中 Self-Forcing 的关键点： +Self-Forcing 额外 fail-fast： -- 强制 `student` 是 causal model（否则 build 阶段直接 error） -- rollout 时只使用 `student.predict_*_streaming(...)`（内部 cache），teacher/critic 用并行 forward（无需 cache） +- `student` 必须是 `CausalModelBase`（不允许 cache-free 分支） --- -## 4) 需要修改/新增的文件（TODO 列表） - -> 先以 wangame 落地，跑通后再推广到 wan。 - -### 4.1 shared_context +## 5) TODO（本阶段要改哪些地方） -- [ ] `fastvideo/distillation/models/wangame/shared_context.py` - - `WanGameSharedContext` +> 先以 wangame 跑通；再推广到 wan。 -### 4.2 models（每个 role 一个实例） +- [ ] 新增 `fastvideo/distillation/utils/instantiate.py` + - 支持 `_target_`（Hydra 语义）+ 纯 dict 参数 + - 支持把 `cfg` 作为通用参数注入（FastGen 常用 pattern） -> 这里不再引入 `RoleModel` 这个名字；每个 role 就是一个 **独立 Model 实例**。 - -- [ ] `fastvideo/distillation/models/wangame/wangame.py`（bidi；每个 role 一个实例） -- [ ] `fastvideo/distillation/models/wangame/wangame_causal.py`（causal + cache/streaming；每个 role 一个实例） -- [ ] `fastvideo/distillation/models/wangame/common.py`（共享 loader 工具） - -### 4.3 dispatch（从 “registry 驱动” 改为 “instantiate 驱动”） +- [ ] `fastvideo/distillation/utils/config.py` + - 更新 yaml schema：以 `_target_` 为主（不再解析 `recipe.family/method`） - [ ] `fastvideo/distillation/dispatch.py` - - 只做入口编排：load yaml → instantiate method → `method.build_run(...)`（或 `method.build(...)`）→ trainer.run -- [ ] `fastvideo/distillation/utils/instantiate.py` - - `instantiate(cfg)`：优先 `_target_`，否则用 `recipe.method`/`models.*.family` 的约定式解析 - - **继承 + `super().build_*()`** 负责复用通用装配逻辑（对齐 FastGen） - -### 4.4 methods（对齐 FastGen 的“继承表达差异点”） - -- [ ] `fastvideo/distillation/methods/base.py` - - `build(..., models, shared_context, validator)`（替代 bundle+model plugin） -- [ ] `dmd2.py` 作为 DM 基类：self_forcing 只 override rollout -- [ ] `finetune.py` 作为 SFT 基类：dfsft 只 override timestep/chunk 采样（尽量复用 validation/optim/backward） - ---- - -## 6) 关键取舍（你 review 时最需要确认） + - 从 “registry 分发 build_*” → “instantiate + method.build_run()” -1) `recipe.family` 是否收敛为 “shared_context family”（我建议是）。 -2) backward hook 的归属：默认 `loss.backward()`，只在必要时让 Model 提供 `backward()`。 -3) validator 的依赖：validator 通常更依赖 shared_context（vae/scheduler/pipeline_config），而不是 models。 +- [ ] 命名清理（不要求一次性全改完，但新接口上应去掉 Distill） + - `DistillTrainer` → `Trainer`（内部文件夹可暂时保留） + - `DistillMethod` → `Method` + - entrypoint：`distillation.py` → `training.py`（可选；看 PR 策略） From bf4333920e3871208e753208143795ae5d0bc31f Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 06:12:39 +0000 Subject: [PATCH 162/214] designing refactor 4 --- dev/phase_refactor.md | 471 +++++++++++++++++++++++++++++++++--------- 1 file changed, 379 insertions(+), 92 deletions(-) diff --git a/dev/phase_refactor.md b/dev/phase_refactor.md index f31cecb52..6b070ddf3 100644 --- a/dev/phase_refactor.md +++ b/dev/phase_refactor.md @@ -1,9 +1,74 @@ -# Phase：Refactor（抛弃 “Distill” 命名：这是通用 Training 框架；YAML=instantiate-first） +# Phase: Refactor — `_target_` instantiate-first, drop the string registry -## 0) 最终 YAML 示例（Self-Forcing：Student=Causal Wangame，Teacher=Bidirectional Wangame） +## 0) FastGen's hierarchy (the reference we follow) -> 你提的方向我同意：YAML 里不要再出现 `wangame/dmd2` 这种“缩写字符串”。 -> 直接写 **类名/类路径**，用 FastGen 同款 `instantiate` 思路（Hydra 风格 `_target_`），避免 registry/if-else。 +``` +FastGenNetwork (ABC, nn.Module) ← backbone: owns transformer + noise_scheduler + VAE + ├── WanNetwork ← Wan-specific forward, text conditioning + └── CausalFastGenNetwork ← adds chunk_size, clear_caches() + └── CausalWanNetwork + +FastGenModel (nn.Module) ← base: owns self.net + self.teacher + optimizers + ├── DMD2Model ← adds self.fake_score, self.discriminator, DMD2 loss + ├── SelfForcingModel ← adds causal rollout logic + └── SFTModel ← vanilla finetuning +``` + +**Key lessons from FastGen:** + +1. `FastGenNetwork.forward(x_t, t, condition, fwd_pred_type)` is the standardized per-role + interface. Each role is an independent network instance. No shared dispatcher. +2. `FastGenModel` **IS the method** — it owns `self.net` (student) and `self.teacher` directly + as attributes, just like `DMD2Model.self.fake_score`. No RoleManager. +3. Optimizers live on `FastGenModel`, not on the network: `self.net_optimizer`, + `self.fake_score_optimizer`, etc. Method subclasses add their own via `init_optimizers()`. +4. VAE/text encoder live on `self.net` via `net.init_preprocessors()`. For us, this + concept maps to `SharedContext` (explained below). + +--- + +## 1) Our hierarchy + +We cannot directly subclass `FastGenNetwork` because our transformers are raw +`nn.Module` objects from diffusers (not `FastGenNetwork`). But we follow the same +**structural shape**: + +``` +SharedContextBase (ABC) ← VAE, scheduler, prepare_batch, dataloader, validator + ├── WanSharedContext ← Wan T2V: text conditioning, timestep mechanics + └── WanGameSharedContext ← WanGame I2V+action: image/action conditioning + +ModelBase (ABC) ← per-role: owns ONE transformer, predict_{noise,x0} + ├── WanModelBase ← shared bidi Wan forward logic + │ ├── WanModel ← T2V (text conditioning) + │ └── WanGameModel ← I2V+action conditioning + └── CausalModelBase ← adds streaming ops, clear_caches + ├── WanCausalModel ← causal T2V + └── WanGameCausalModel ← causal I2V+action + +DistillMethod (nn.Module, ABC) ← base: generic optimizer loop, checkpoint props + ├── DMD2Method ← self.student, self.teacher, self.fake_score + ├── SelfForcingMethod ← self.student (must be CausalModelBase), self.teacher, self.critic + └── FinetuneMethod / DFSFTMethod ← self.student only +``` + +**SharedContext** is analogous to `FastGenNetwork.init_preprocessors()` — it holds +what is truly shared across all roles. `prepare_batch` lives here because it is +pure preprocessing: it samples timesteps, normalizes latents, builds attn metadata, +and populates `batch.conditional_dict` / `batch.unconditional_dict`. It does not +touch any transformer. + +**ModelBase** is analogous to `FastGenNetwork`. Each role is an independent instance +(like FastGen's `self.net`, `self.teacher`). It owns one transformer and implements +`predict_noise`/`predict_x0` by calling that transformer + reading precomputed +fields from `TrainingBatch`. + +**DistillMethod** is analogous to `FastGenModel`. It owns the role model objects, +all optimizers, and implements `single_train_step`. RoleManager is retired. + +--- + +## 2) Final YAML ```yaml log: @@ -13,9 +78,6 @@ log: wandb_mode: online trainer: - # trainer 本身也可以不 instantiate（只当作纯参数 dict） - # 但为了彻底对齐 FastGen，我们允许它也走 _target_。 - _target_: fastvideo.distillation.trainer.Trainer output_dir: outputs/wangame_self_forcing_refactor max_train_steps: 100000 seed: 1000 @@ -32,8 +94,6 @@ validation: dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json every_steps: 100 sampling_steps: [4] - rollout_mode: streaming - # 不固定字段：由 method/validator 自己按需读取（从简） pipeline: sampler_kind: sde scheduler: @@ -41,152 +101,379 @@ validation: flow_shift: 3 shared_context: - # 替代旧的 recipe.family：shared_context 是一个“可实例化对象” - _target_: fastvideo.distillation.models.wangame.shared_context.WanGameSharedContext - # shared parts（VAE/encoders/schedulers/validator 等）只构建一份，来源由该路径决定 + _target_: fastvideo.distillation.models.wangame.WanGameSharedContext model_path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 models: - # 每个 role 一个独立 Model 实例（method 决定需要哪些 role） student: - _target_: fastvideo.distillation.models.wangame.wangame_causal.WanGameCausalModel + _target_: fastvideo.distillation.models.wangame.WanGameCausalModel init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 trainable: true - teacher: - _target_: fastvideo.distillation.models.wangame.wangame.WanGameModel + _target_: fastvideo.distillation.models.wangame.WanGameModel init_from: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers trainable: false - critic: - _target_: fastvideo.distillation.models.wangame.wangame.WanGameModel + _target_: fastvideo.distillation.models.wangame.WanGameModel init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 trainable: true method: _target_: fastvideo.distillation.methods.distribution_matching.self_forcing.SelfForcingMethod - # method-specific config（随 method 变化，不强行统一 schema） rollout_mode: simulate chunk_size: 3 student_sample_type: sde context_noise: 0.0 ``` +For a Wan T2V DMD2 run, only the `_target_` paths change: + +```yaml +shared_context: + _target_: fastvideo.distillation.models.wan.WanSharedContext + model_path: /path/to/wan_14b + +models: + student: + _target_: fastvideo.distillation.models.wan.WanModel + init_from: /path/to/wan_14b + trainable: true + teacher: + _target_: fastvideo.distillation.models.wan.WanModel + init_from: /path/to/wan_14b + trainable: false + fake_score: + _target_: fastvideo.distillation.models.wan.WanModel + init_from: /path/to/wan_14b + trainable: true + +method: + _target_: fastvideo.distillation.methods.distribution_matching.dmd2.DMD2Method + guidance_scale: 5.0 + student_update_freq: 5 +``` + --- -## 1) 为什么要从 “Distill” 改成 “Training” +## 3) Class interfaces + +### 3.1 SharedContextBase + +```python +class SharedContextBase(ABC): + """Holds preprocessing primitives and shared components (VAE, scheduler). + Analogous to FastGenNetwork.init_preprocessors() — owns what all roles share. + Does NOT own any transformer or implement predict_x0/predict_noise. + """ + + dataloader: Any + validator: Any | None + training_args: Any + noise_scheduler: Any + vae: Any + + @abstractmethod + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: Literal["data", "zeros"] = "data", + ) -> TrainingBatch: + """Normalize latents, sample timesteps, build attn_metadata, populate + batch.conditional_dict / batch.unconditional_dict. No transformer calls.""" + ... + + @abstractmethod + def add_noise( + self, clean: Tensor, noise: Tensor, timestep: Tensor + ) -> Tensor: ... + + @abstractmethod + def on_train_start(self) -> None: ... + + def get_rng_generators(self) -> dict[str, torch.Generator]: + return {} +``` -现状里我们已经同时支持： +### 3.2 ModelBase (per-role, analogous to FastGenNetwork) -- few-step distillation（DMD2 / Self-Forcing） -- finetuning / SFT -- DFSFT（causal + streaming rollout） +```python +class ModelBase(ABC): + """Per-role model. Analogous to FastGenNetwork in FastGen. + Owns ONE transformer and implements forward ops on top of shared context. + """ -所以 “distill” 只是其中一种训练策略；框架本质更像： + transformer: nn.Module + ctx: SharedContextBase # injected at construction + trainable: bool -> **Trainer（loop） + Method（algorithm） + Model（per-role instance） + SharedContext（shared parts）** + @abstractmethod + def predict_noise( + self, + noisy_latents: Tensor, + timestep: Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cfg_uncond: dict | None = None, + attn_kind: str = "dense", + ) -> Tensor: ... -本次 refactor 的目标是把命名与 config 语义对齐，让人第一眼就能理解“这是训练框架”，而不是“蒸馏脚手架”。 + @abstractmethod + def predict_x0(self, ...) -> Tensor: ... ---- + def backward( + self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int + ) -> None: + """Default backward. Causal subclass may need forward-context restore.""" + (loss / max(1, grad_accum_rounds)).backward() + + +class CausalModelBase(ModelBase): + """Causal/streaming extension. Analogous to CausalFastGenNetwork.""" -## 2) 参考 FastGen：我们要“照搬”的点是什么 + @abstractmethod + def predict_noise_streaming(self, ...) -> Tensor | None: ... -FastGen 的几个关键点（对应到我们要做的事）： + @abstractmethod + def predict_x0_streaming(self, ...) -> Tensor | None: ... -1) **instantiate-first** - - FastGen 用 `LazyCall/_target_ + instantiate()`：config 里写“我要哪个类、给它什么参数”，代码只负责实例化与编排。 - - 我们的 YAML 也应如此：不再让 `dispatch.py` 维护字符串 registry（`wangame`/`dmd2`/...）。 + @abstractmethod + def clear_caches(self, *, cache_tag: str = "pos") -> None: ... +``` -2) **继承 + `super().build_*()` 复用装配逻辑** - - 通用的：建 shared parts、建 networks、DDP/FSDP wrapper、optim/scheduler、checkpoint、logging。 - - method 子类只 override：需要哪些 models、rollout/损失、更新策略。 +### 3.3 DistillMethod (analogous to FastGenModel) + +```python +class DistillMethod(nn.Module, ABC): + """Base training method. Analogous to FastGenModel. + Owns role model objects and ALL optimizers directly as attributes. + RoleManager is retired. + """ + + @classmethod + @abstractmethod + def build( + cls, + *, + cfg: RunConfig, + shared_context: SharedContextBase, + role_models: dict[str, ModelBase], + ) -> "DistillMethod": + """Assemble the method. Analogous to FastGenModel.__init__ → build_model(). + The classmethod reads role_models, stores them as self.student / self.teacher / + etc., then calls init_optimizers(). + """ + ... + + @abstractmethod + def init_optimizers(self) -> None: + """Build self.student_optimizer, self.teacher_optimizer, etc. + Analogous to FastGenModel.init_optimizers() + DMD2Model.init_optimizers(). + Subclasses call super().init_optimizers() then add their own. + """ + ... + + @abstractmethod + def single_train_step( + self, batch: TrainingBatch, iteration: int, **kwargs + ) -> tuple[dict[str, Tensor], dict[str, Any], dict[str, float]]: ... + + @abstractmethod + def get_optimizers(self, iteration: int) -> list[Optimizer]: ... + + @abstractmethod + def get_lr_schedulers(self, iteration: int) -> list[Any]: ... + + # Checkpoint helpers (mirror FastGen's model_dict / optimizer_dict / scheduler_dict) + @property + @abstractmethod + def model_dict(self) -> dict[str, nn.Module]: ... + + @property + @abstractmethod + def optimizer_dict(self) -> dict[str, Optimizer]: ... + + @property + @abstractmethod + def scheduler_dict(self) -> dict[str, Any]: ... +``` -3) **Student 持有 shared parts（或显式 shared_context）** - - FastGen 把 preprocessors 隐式挂到 `self.net`（student）； - - 我们这边 shared parts 语义更重（pipeline/validator/dataloader），更适合显式 `shared_context`（但 config 同样 instantiate）。 +### 3.4 SelfForcingMethod (concrete example) + +```python +class SelfForcingMethod(DistillMethod): + def __init__( + self, + student: CausalModelBase, + teacher: ModelBase, + critic: ModelBase, + shared_context: SharedContextBase, + cfg: dict, + ) -> None: + super().__init__() + if not isinstance(student, CausalModelBase): + raise TypeError( + f"SelfForcingMethod requires CausalModelBase student, got {type(student).__name__}" + ) + self.student = student + self.teacher = teacher + self.critic = critic + self.ctx = shared_context + self.init_optimizers() + + @classmethod + def build(cls, *, cfg, shared_context, role_models): + return cls( + student=role_models["student"], + teacher=role_models["teacher"], + critic=role_models["critic"], + shared_context=shared_context, + cfg=cfg.method, + ) + + def init_optimizers(self) -> None: + self.student_optimizer = build_optimizer(self.student.transformer, ...) + self.student_lr_scheduler = build_lr_scheduler(self.student_optimizer, ...) + self.critic_optimizer = build_optimizer(self.critic.transformer, ...) + self.critic_lr_scheduler = build_lr_scheduler(self.critic_optimizer, ...) + + @property + def model_dict(self): + return {"student": self.student.transformer, "critic": self.critic.transformer} + + @property + def optimizer_dict(self): + return {"student": self.student_optimizer, "critic": self.critic_optimizer} + + @property + def scheduler_dict(self): + return {"student": self.student_lr_scheduler, "critic": self.critic_lr_scheduler} + + def get_optimizers(self, iteration: int) -> list: + return [self.student_optimizer, self.critic_optimizer] + + def get_lr_schedulers(self, iteration: int) -> list: + return [self.student_lr_scheduler, self.critic_lr_scheduler] +``` --- -## 3) 新 YAML 的核心语义（草案） +## 4) Assembly flow (dispatch / entrypoint) -### 3.1 `_target_`：一切都可 instantiate +```python +cfg = load_run_config(path) -- `method._target_`：算法/训练策略（SelfForcing/DMD2/Finetune/DFSFT） -- `models.._target_`：每个 role 的 Model class（bidi/causal） -- `shared_context._target_`：shared parts 的构建者（按任务/数据形态） -- （可选）`trainer._target_`：训练 loop（我们也可以先只传纯参数，避免 trainer 过度可插拔） +# 1. SharedContext — VAE, scheduler, dataloader, validator +shared_context = instantiate(cfg.shared_context) -### 3.2 去掉 `family/method` 缩写 +# 2. Per-role models — each owns one transformer + forward ops +# shared_context is injected so each model can read batch fields it needs +role_models = { + role: instantiate(model_cfg, shared_context=shared_context) + for role, model_cfg in cfg.models.items() +} -不再支持： +# 3. Method class assembles itself from role models (analogous to FastGenModel.__init__) +method_cls = resolve_target(cfg.method["_target_"]) +method = method_cls.build( + cfg=cfg, + shared_context=shared_context, + role_models=role_models, +) -- `recipe.family: wangame` -- `models.student.family: wangame_causal` -- `recipe.method: self_forcing` +# 4. Generic trainer loop, not method-aware beyond the DistillMethod interface +trainer = Trainer(cfg.trainer) +trainer.run(method, shared_context=shared_context) +``` -只保留 `_target_`（或 `class_path` 同义字段），避免隐式映射导致概念膨胀。 +--- -### 3.3 “shared_component_model” 的位置 +## 5) What is retired / changed -我建议 **直接去掉 `shared_component_model`**： +| Current | New | +|---|---| +| `dispatch.py` string registry (`_MODELS`, `_METHODS`, `@register_*`) | Deleted; `_target_` + `instantiate()` | +| `ModelBase` — one class handles ALL roles via `RoleHandle` arg | Retired; replaced by per-role `ModelBase` (one instance per role) | +| `CausalModelBase` | Becomes proper ABC for streaming, same concept | +| `RoleHandle` / `RoleManager` | Retired; method owns role objects directly | +| `DistillMethod.build(cfg, bundle, model, validator)` | `DistillMethod.build(cfg, shared_context, role_models)` | +| Optimizers on `RoleHandle.optimizers` | On method: `self.student_optimizer`, etc. | +| `recipe.family` / `recipe.method` YAML keys | Deleted; `shared_context._target_` + `method._target_` | +| `roles.*` YAML section | Replaced by `models.*` with `_target_`, `init_from`, `trainable` | +| Method checkpoint uses `bundle.roles` dict | Uses `model_dict`, `optimizer_dict`, `scheduler_dict` properties | -- shared parts 的来源已经由 `shared_context.model_path` 显式指定； -- 再用一个 role 去“引用/指代”来源，属于重复表达，反而会让人误以为 role 会影响 shared parts 的语义。 +--- -从简后的约束： +## 6) Naming cleanup (separate PR, do NOT mix in) -- 只要求 `shared_context.model_path` 必填； -- 是否与 `models.student.init_from` 一致由用户自行保证（如果未来要 strict 校验，再引入显式的 `shared_context.strict_match` 之类字段）。 +- `DistillMethod` → `Method` +- `DistillRunConfig` → `RunConfig` +- `load_distill_run_config` → `load_run_config` +- Entrypoint `distillation.py` → `training.py` (optional) --- -## 4) 组装流程（调用链） +## 7) TODO (this phase — wangame first, then extend to wan) -伪代码（对齐 FastGen：method 主导装配）： +**Infrastructure** -```py -cfg = load_yaml(path) +- [ ] `fastvideo/distillation/utils/instantiate.py` + - `resolve_target(target: str) -> type` + - `instantiate(cfg: dict, **extra) -> Any` -method = instantiate(cfg["method"], cfg=cfg) # method class +- [ ] `fastvideo/distillation/utils/config.py` + - New `RunConfig` dataclass (no `RecipeSpec`, no `RoleSpec`) + - New `load_run_config(path)` parser for new YAML schema + - Keep `load_distill_run_config` as deprecated shim -shared_context = instantiate(cfg["shared_context"], cfg=cfg) # build shared parts once -models = { - role: instantiate(model_cfg, role=role, shared_context=shared_context) - for role, model_cfg in cfg["models"].items() -} +**Base classes** -run_objects = method.build_run( - cfg=cfg, - shared_context=shared_context, - models=models, -) +- [ ] `fastvideo/distillation/models/base.py` + - New `SharedContextBase` ABC + - New `ModelBase` ABC (per-role; `self.transformer`, `self.ctx`, `predict_noise`, `predict_x0`) + - New `CausalModelBase(ModelBase)` ABC (streaming ops, `clear_caches`) + - Retire old `ModelBase` once migration is done -trainer = instantiate(cfg["trainer"], cfg=cfg, **run_objects.trainer_inputs) -trainer.run(**run_objects.run_inputs) -``` +- [ ] `fastvideo/distillation/roles.py` + - Retire `RoleHandle` / `RoleManager` (keep temporarily if trainer checkpoint code needs it, then remove) -Self-Forcing 额外 fail-fast： +- [ ] `fastvideo/distillation/methods/base.py` + - New `DistillMethod.build(cfg, shared_context, role_models)` signature + - Add abstract `init_optimizers`, `model_dict`, `optimizer_dict`, `scheduler_dict` + - Remove `bundle`, `model`, `validator` parameters -- `student` 必须是 `CausalModelBase`（不允许 cache-free 分支） +**WanGame models** ---- +- [ ] `fastvideo/distillation/models/wangame/shared_context.py` + - `WanGameSharedContext(SharedContextBase)`: extracts VAE, scheduler, `prepare_batch`, + `add_noise`, `on_train_start`, dataloader, validator from current `WanGameModel.__init__` -## 5) TODO（本阶段要改哪些地方） +- [ ] `fastvideo/distillation/models/wangame/models.py` + - `WanGameModelBase(ModelBase)`: shared bidi forward logic (input_kwargs, CFG, action cond) + - `WanGameModel(WanGameModelBase)`: bidi transformer + - `WanGameCausalModel(CausalModelBase, WanGameModelBase)`: causal transformer + streaming ops + - Transformer loading moves into each class `__init__` (replaces `common._build_wangame_role_handles`) -> 先以 wangame 跑通；再推广到 wan。 +**Wan models** -- [ ] 新增 `fastvideo/distillation/utils/instantiate.py` - - 支持 `_target_`（Hydra 语义）+ 纯 dict 参数 - - 支持把 `cfg` 作为通用参数注入（FastGen 常用 pattern） +- [ ] `fastvideo/distillation/models/wan/shared_context.py` + - `WanSharedContext(SharedContextBase)`: same extraction from current `WanModel` -- [ ] `fastvideo/distillation/utils/config.py` - - 更新 yaml schema：以 `_target_` 为主（不再解析 `recipe.family/method`） +- [ ] `fastvideo/distillation/models/wan/models.py` + - `WanModelBase(ModelBase)`: shared bidi forward logic (text conditioning, CFG) + - `WanModel(WanModelBase)`: bidi transformer + - `WanCausalModel(CausalModelBase, WanModelBase)`: causal transformer + streaming ops + +**Methods** + +- [ ] Update all method `build()` signatures to new interface: + - `SelfForcingMethod`, `DMD2Method`, `FinetuneMethod`, `DFSFTMethod` -- [ ] `fastvideo/distillation/dispatch.py` - - 从 “registry 分发 build_*” → “instantiate + method.build_run()” +**Dispatch & configs** -- [ ] 命名清理（不要求一次性全改完，但新接口上应去掉 Distill） - - `DistillTrainer` → `Trainer`（内部文件夹可暂时保留） - - `DistillMethod` → `Method` - - entrypoint：`distillation.py` → `training.py`（可选；看 PR 策略） +- [ ] `fastvideo/distillation/dispatch.py`: replace body with new assembly flow (or delete) +- [ ] New YAML configs for wangame self-forcing (validate end-to-end) +- [ ] Update existing wangame YAML configs to new schema +- [ ] Update wan YAML configs From 0e6d1090d8647c5bc07170de5dd4fc370c7bcbef Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 06:30:46 +0000 Subject: [PATCH 163/214] designing refactor 5 --- dev/phase_refactor.md | 650 ++++++++++++++++++++++++------------------ 1 file changed, 368 insertions(+), 282 deletions(-) diff --git a/dev/phase_refactor.md b/dev/phase_refactor.md index 6b070ddf3..215b07076 100644 --- a/dev/phase_refactor.md +++ b/dev/phase_refactor.md @@ -1,212 +1,122 @@ # Phase: Refactor — `_target_` instantiate-first, drop the string registry -## 0) FastGen's hierarchy (the reference we follow) +## 0) FastGen hierarchy (the reference) ``` -FastGenNetwork (ABC, nn.Module) ← backbone: owns transformer + noise_scheduler + VAE - ├── WanNetwork ← Wan-specific forward, text conditioning - └── CausalFastGenNetwork ← adds chunk_size, clear_caches() - └── CausalWanNetwork - -FastGenModel (nn.Module) ← base: owns self.net + self.teacher + optimizers - ├── DMD2Model ← adds self.fake_score, self.discriminator, DMD2 loss - ├── SelfForcingModel ← adds causal rollout logic - └── SFTModel ← vanilla finetuning -``` - -**Key lessons from FastGen:** - -1. `FastGenNetwork.forward(x_t, t, condition, fwd_pred_type)` is the standardized per-role - interface. Each role is an independent network instance. No shared dispatcher. -2. `FastGenModel` **IS the method** — it owns `self.net` (student) and `self.teacher` directly - as attributes, just like `DMD2Model.self.fake_score`. No RoleManager. -3. Optimizers live on `FastGenModel`, not on the network: `self.net_optimizer`, - `self.fake_score_optimizer`, etc. Method subclasses add their own via `init_optimizers()`. -4. VAE/text encoder live on `self.net` via `net.init_preprocessors()`. For us, this - concept maps to `SharedContext` (explained below). - ---- - -## 1) Our hierarchy - -We cannot directly subclass `FastGenNetwork` because our transformers are raw -`nn.Module` objects from diffusers (not `FastGenNetwork`). But we follow the same -**structural shape**: +FastGenNetwork (ABC, nn.Module) ← per-role backbone; owns transformer + noise_scheduler +CausalFastGenNetwork (ABC, nn.Module) ← parallel root, NOT a subclass of FastGenNetwork -``` -SharedContextBase (ABC) ← VAE, scheduler, prepare_batch, dataloader, validator - ├── WanSharedContext ← Wan T2V: text conditioning, timestep mechanics - └── WanGameSharedContext ← WanGame I2V+action: image/action conditioning - -ModelBase (ABC) ← per-role: owns ONE transformer, predict_{noise,x0} - ├── WanModelBase ← shared bidi Wan forward logic - │ ├── WanModel ← T2V (text conditioning) - │ └── WanGameModel ← I2V+action conditioning - └── CausalModelBase ← adds streaming ops, clear_caches - ├── WanCausalModel ← causal T2V - └── WanGameCausalModel ← causal I2V+action - -DistillMethod (nn.Module, ABC) ← base: generic optimizer loop, checkpoint props - ├── DMD2Method ← self.student, self.teacher, self.fake_score - ├── SelfForcingMethod ← self.student (must be CausalModelBase), self.teacher, self.critic - └── FinetuneMethod / DFSFTMethod ← self.student only +FastGenModel (nn.Module) ← base method; owns self.net, self.teacher, optimizers + DMD2Model ← adds self.fake_score, self.discriminator + SelfForcingModel ← causal rollout + SFTModel ``` -**SharedContext** is analogous to `FastGenNetwork.init_preprocessors()` — it holds -what is truly shared across all roles. `prepare_batch` lives here because it is -pure preprocessing: it samples timesteps, normalizes latents, builds attn metadata, -and populates `batch.conditional_dict` / `batch.unconditional_dict`. It does not -touch any transformer. +**Key lessons:** -**ModelBase** is analogous to `FastGenNetwork`. Each role is an independent instance -(like FastGen's `self.net`, `self.teacher`). It owns one transformer and implements -`predict_noise`/`predict_x0` by calling that transformer + reading precomputed -fields from `TrainingBatch`. +1. **`CausalFastGenNetwork` is parallel to `FastGenNetwork`**, not a subclass. -**DistillMethod** is analogous to `FastGenModel`. It owns the role model objects, -all optimizers, and implements `single_train_step`. RoleManager is retired. +2. **Every role instance owns its own `noise_scheduler`** — student and teacher each get + an independent instance of the same schedule. No sharing needed. This is because + `FastGenNetwork.__init__` always calls `self.set_noise_schedule()`. + The teacher's `forward(x_t, t, fwd_pred_type="x0")` uses its own scheduler internally + for `pred_noise → x0` conversion. No "who owns the scheduler" problem. ---- +3. **No mixin class.** FastGen doesn't need one because student and teacher are the + **same class** (`WanNetwork`). The only construction-time difference is that + `FastGenModel.build_model()` calls `self.net.init_preprocessors()` on the student, + but NOT on the teacher. The class itself is neutral — it has `init_preprocessors()` + as a method, but whether it's called is the caller's decision. + For us: same principle. Every `ModelBase` has `init_preprocessors()`. The method's + `build()` calls it only on the student. No mixin. No `is_student` flag. -## 2) Final YAML +4. **Method owns all optimizers** (`self.net_optimizer`, `self.fake_score_optimizer`, …) + via `init_optimizers()`. Subclasses extend with `super().init_optimizers()`. -```yaml -log: - project: fastvideo - group: wangame - name: self_forcing_causal_student_bidi_teacher - wandb_mode: online +5. **`model_dict` / `optimizer_dict` / `scheduler_dict`** are properties on the method, + used by the trainer for checkpointing. -trainer: - output_dir: outputs/wangame_self_forcing_refactor - max_train_steps: 100000 - seed: 1000 - mixed_precision: bf16 - grad_accum_rounds: 1 +6. **Dataloader is external** — passed into trainer separately, not owned by the model. -data: - _target_: fastvideo.distillation.utils.dataloader.ParquetDataConfig - data_path: /path/to/wangame/parquet - dataloader_num_workers: 4 - -validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json - every_steps: 100 - sampling_steps: [4] - pipeline: - sampler_kind: sde - scheduler: - _target_: fastvideo.models.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler - flow_shift: 3 - -shared_context: - _target_: fastvideo.distillation.models.wangame.WanGameSharedContext - model_path: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 +--- -models: - student: - _target_: fastvideo.distillation.models.wangame.WanGameCausalModel - init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - trainable: true - teacher: - _target_: fastvideo.distillation.models.wangame.WanGameModel - init_from: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers - trainable: false - critic: - _target_: fastvideo.distillation.models.wangame.WanGameModel - init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - trainable: true +## 1) Our hierarchy -method: - _target_: fastvideo.distillation.methods.distribution_matching.self_forcing.SelfForcingMethod - rollout_mode: simulate - chunk_size: 3 - student_sample_type: sde - context_noise: 0.0 +``` +fastvideo/distillation/ +├── models/ +│ ├── base.py ModelBase, CausalModelBase (parallel roots, no mixin) +│ ├── wan/ +│ │ └── model.py WanModel(ModelBase), WanCausalModel(CausalModelBase) +│ └── wangame/ +│ └── model.py WanGameModel(ModelBase), WanGameCausalModel(CausalModelBase) +├── methods/ +│ ├── base.py DistillMethod (analogous to FastGenModel) +│ └── distribution_matching/ +│ ├── dmd2.py DMD2Method +│ └── self_forcing.py SelfForcingMethod +└── utils/ + └── instantiate.py resolve_target, instantiate ``` -For a Wan T2V DMD2 run, only the `_target_` paths change: +**Every model instance has its own `noise_scheduler`** (constructed in `__init__`). +VAE and `prepare_batch` are initialized via `init_preprocessors()`, called only on +the student by `DistillMethod.build()`. No mixin. No flag. -```yaml -shared_context: - _target_: fastvideo.distillation.models.wan.WanSharedContext - model_path: /path/to/wan_14b - -models: - student: - _target_: fastvideo.distillation.models.wan.WanModel - init_from: /path/to/wan_14b - trainable: true - teacher: - _target_: fastvideo.distillation.models.wan.WanModel - init_from: /path/to/wan_14b - trainable: false - fake_score: - _target_: fastvideo.distillation.models.wan.WanModel - init_from: /path/to/wan_14b - trainable: true - -method: - _target_: fastvideo.distillation.methods.distribution_matching.dmd2.DMD2Method - guidance_scale: 5.0 - student_update_freq: 5 -``` +**DistillMethod** owns role model objects (`self.student`, `self.teacher`, …) and +all optimizers as attributes. `RoleManager` is retired. --- -## 3) Class interfaces +## 2) Class interfaces -### 3.1 SharedContextBase +### 2.1 ModelBase and CausalModelBase (parallel roots) ```python -class SharedContextBase(ABC): - """Holds preprocessing primitives and shared components (VAE, scheduler). - Analogous to FastGenNetwork.init_preprocessors() — owns what all roles share. - Does NOT own any transformer or implement predict_x0/predict_noise. +# fastvideo/distillation/models/base.py + +class ModelBase(ABC): + """Per-role bidirectional model. Analogous to FastGenNetwork. + Every instance owns its own noise_scheduler. + init_preprocessors() is only called on the student by DistillMethod.build(). """ + transformer: nn.Module + noise_scheduler: Any # always set in __init__ + + def init_preprocessors(self, training_args: Any) -> None: + """Load VAE, seed RNGs, build prepare_batch machinery. + Analogous to FastGenNetwork.init_preprocessors(). + Only called on the student role. Default: no-op (for teacher/critic). + Subclasses override when they are capable of being the student. + """ + pass - dataloader: Any - validator: Any | None - training_args: Any - noise_scheduler: Any - vae: Any + def on_train_start(self) -> None: + """Called by trainer before training loop. Default: no-op.""" + pass + + def get_rng_generators(self) -> dict[str, torch.Generator]: + return {} - @abstractmethod def prepare_batch( self, raw_batch: dict[str, Any], *, current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", + latents_source: str = "data", ) -> TrainingBatch: - """Normalize latents, sample timesteps, build attn_metadata, populate - batch.conditional_dict / batch.unconditional_dict. No transformer calls.""" - ... + """Build TrainingBatch from raw dataloader output. + Raises NotImplementedError unless init_preprocessors() was called. + """ + raise NotImplementedError( + f"{type(self).__name__}.prepare_batch() requires init_preprocessors() " + "to have been called (student only)." + ) - @abstractmethod def add_noise( self, clean: Tensor, noise: Tensor, timestep: Tensor - ) -> Tensor: ... - - @abstractmethod - def on_train_start(self) -> None: ... - - def get_rng_generators(self) -> dict[str, torch.Generator]: - return {} -``` - -### 3.2 ModelBase (per-role, analogous to FastGenNetwork) - -```python -class ModelBase(ABC): - """Per-role model. Analogous to FastGenNetwork in FastGen. - Owns ONE transformer and implements forward ops on top of shared context. - """ - - transformer: nn.Module - ctx: SharedContextBase # injected at construction - trainable: bool + ) -> Tensor: + raise NotImplementedError @abstractmethod def predict_noise( @@ -224,33 +134,144 @@ class ModelBase(ABC): def predict_x0(self, ...) -> Tensor: ... def backward( - self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int + self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int = 1 ) -> None: - """Default backward. Causal subclass may need forward-context restore.""" (loss / max(1, grad_accum_rounds)).backward() -class CausalModelBase(ModelBase): - """Causal/streaming extension. Analogous to CausalFastGenNetwork.""" +class CausalModelBase(ABC): + """Per-role causal model. Parallel root to ModelBase, NOT a subclass. + Analogous to CausalFastGenNetwork. + Same interface as ModelBase plus streaming ops. + """ + transformer: nn.Module + noise_scheduler: Any # always set in __init__ + + def init_preprocessors(self, training_args: Any) -> None: + pass + + def on_train_start(self) -> None: + pass + + def get_rng_generators(self) -> dict[str, torch.Generator]: + return {} + + def prepare_batch(self, ...) -> TrainingBatch: + raise NotImplementedError + + def add_noise(self, ...) -> Tensor: + raise NotImplementedError @abstractmethod - def predict_noise_streaming(self, ...) -> Tensor | None: ... + def predict_noise(self, ...) -> Tensor: ... + + @abstractmethod + def predict_x0(self, ...) -> Tensor: ... + + @abstractmethod + def predict_noise_streaming( + self, + noisy_latents: Tensor, + timestep: Tensor, + batch: TrainingBatch, + *, + conditional: bool, + cache_tag: str = "pos", + store_kv: bool = False, + cur_start_frame: int = 0, + cfg_uncond: dict | None = None, + attn_kind: str = "dense", + ) -> Tensor | None: ... @abstractmethod def predict_x0_streaming(self, ...) -> Tensor | None: ... @abstractmethod def clear_caches(self, *, cache_tag: str = "pos") -> None: ... + + def backward(self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int = 1) -> None: + (loss / max(1, grad_accum_rounds)).backward() ``` -### 3.3 DistillMethod (analogous to FastGenModel) +### 2.2 Concrete model classes (WanGame example) ```python -class DistillMethod(nn.Module, ABC): - """Base training method. Analogous to FastGenModel. - Owns role model objects and ALL optimizers directly as attributes. - RoleManager is retired. +# fastvideo/distillation/models/wangame/model.py + +class WanGameModel(ModelBase): + """Bidirectional WanGame model. Can be student or teacher/critic. + Always builds noise_scheduler in __init__. + VAE and prepare_batch are activated by init_preprocessors() — only for student. + """ + + def __init__(self, *, init_from: str, trainable: bool = True, **kwargs) -> None: + self.transformer = load_transformer(init_from, cls="WanGameActionTransformer3DModel") + apply_trainable(self.transformer, trainable=trainable) + # noise_scheduler always built — same as FastGenNetwork.__init__ calling set_noise_schedule() + self.noise_scheduler = FlowMatchEulerDiscreteScheduler(...) + # preprocessor state — inactive until init_preprocessors() is called + self.vae: Any | None = None + self._noise_gen_cpu: torch.Generator | None = None + self._noise_gen_cuda: torch.Generator | None = None + self._init_from = init_from + + def init_preprocessors(self, training_args: Any) -> None: + """Activate as student. Analogous to FastGenNetwork.init_preprocessors().""" + self.training_args = training_args + self.vae = load_module_from_path(self._init_from, module_type="vae", ...) + self.device = get_local_torch_device() + self._init_timestep_mechanics() + + def on_train_start(self) -> None: + seed = self.training_args.seed + self._noise_gen_cpu = torch.Generator(device="cpu").manual_seed(seed) + self._noise_gen_cuda = torch.Generator(device=self.device).manual_seed(seed) + + def prepare_batch(self, raw_batch, *, ...) -> TrainingBatch: + if self.vae is None: + raise RuntimeError("prepare_batch requires init_preprocessors() (student only)") + # ... full WanGame batch preparation + return training_batch + + def add_noise(self, clean, noise, timestep) -> Tensor: + return self.noise_scheduler.add_noise(...) + + def predict_noise(self, noisy_latents, timestep, batch, *, conditional, ...) -> Tensor: + # Uses self.noise_scheduler (always available) and self.transformer + with autocast(...), set_forward_context(...): + kwargs = self._build_input_kwargs(noisy_latents, timestep, batch, conditional=conditional, ...) + return self.transformer(**kwargs).permute(0, 2, 1, 3, 4) + + def predict_x0(self, noisy_latents, timestep, batch, *, conditional, ...) -> Tensor: + pred_noise = self.predict_noise(noisy_latents, timestep, batch, conditional=conditional, ...) + # Uses self.noise_scheduler — always available, no dependency on student + return pred_noise_to_pred_video(pred_noise, noisy_latents, timestep, self.noise_scheduler) + + +class WanGameCausalModel(CausalModelBase): + """Causal WanGame. Parallel root to WanGameModel. + Same init_preprocessors() pattern — called only on student. """ + def __init__(self, *, init_from: str, trainable: bool = True, **kwargs) -> None: + self.transformer = load_transformer(init_from, cls="CausalWanGameTransformer") + apply_trainable(self.transformer, trainable=trainable) + self.noise_scheduler = FlowMatchEulerDiscreteScheduler(...) + self.vae = None + ... + + def init_preprocessors(self, training_args: Any) -> None: + # Same pattern as WanGameModel.init_preprocessors() + ... + + def predict_noise_streaming(self, ...) -> Tensor | None: ... + def predict_x0_streaming(self, ...) -> Tensor | None: ... + def clear_caches(self, *, cache_tag: str = "pos") -> None: ... +``` + +### 2.3 DistillMethod (analogous to FastGenModel) + +```python +class DistillMethod(nn.Module, ABC): @classmethod @abstractmethod @@ -258,27 +279,21 @@ class DistillMethod(nn.Module, ABC): cls, *, cfg: RunConfig, - shared_context: SharedContextBase, - role_models: dict[str, ModelBase], + role_models: dict[str, ModelBase | CausalModelBase], + dataloader: Any, + validator: Any | None, ) -> "DistillMethod": - """Assemble the method. Analogous to FastGenModel.__init__ → build_model(). - The classmethod reads role_models, stores them as self.student / self.teacher / - etc., then calls init_optimizers(). + """Assemble the method. Calls init_preprocessors() on the student, + then calls init_optimizers(). + Analogous to FastGenModel.__init__ → build_model() → init_preprocessors(). """ ... @abstractmethod - def init_optimizers(self) -> None: - """Build self.student_optimizer, self.teacher_optimizer, etc. - Analogous to FastGenModel.init_optimizers() + DMD2Model.init_optimizers(). - Subclasses call super().init_optimizers() then add their own. - """ - ... + def init_optimizers(self) -> None: ... @abstractmethod - def single_train_step( - self, batch: TrainingBatch, iteration: int, **kwargs - ) -> tuple[dict[str, Tensor], dict[str, Any], dict[str, float]]: ... + def single_train_step(self, raw_batch, iteration, **kwargs) -> ...: ... @abstractmethod def get_optimizers(self, iteration: int) -> list[Optimizer]: ... @@ -286,7 +301,6 @@ class DistillMethod(nn.Module, ABC): @abstractmethod def get_lr_schedulers(self, iteration: int) -> list[Any]: ... - # Checkpoint helpers (mirror FastGen's model_dict / optimizer_dict / scheduler_dict) @property @abstractmethod def model_dict(self) -> dict[str, nn.Module]: ... @@ -300,44 +314,55 @@ class DistillMethod(nn.Module, ABC): def scheduler_dict(self) -> dict[str, Any]: ... ``` -### 3.4 SelfForcingMethod (concrete example) +### 2.4 SelfForcingMethod (example) ```python class SelfForcingMethod(DistillMethod): - def __init__( - self, - student: CausalModelBase, - teacher: ModelBase, - critic: ModelBase, - shared_context: SharedContextBase, - cfg: dict, - ) -> None: - super().__init__() + + @classmethod + def build(cls, *, cfg, role_models, dataloader, validator): + student = role_models["student"] + teacher = role_models["teacher"] + critic = role_models["critic"] + if not isinstance(student, CausalModelBase): raise TypeError( - f"SelfForcingMethod requires CausalModelBase student, got {type(student).__name__}" + f"SelfForcingMethod requires CausalModelBase student, " + f"got {type(student).__name__}" ) + + # Call init_preprocessors only on student — identical to FastGen's build_model() + # calling init_preprocessors() only on self.net, not self.teacher. + student.init_preprocessors(cfg.training_args) + + return cls( + student=student, teacher=teacher, critic=critic, + dataloader=dataloader, validator=validator, cfg=cfg.method, + ) + + def __init__(self, student, teacher, critic, dataloader, validator, cfg): + super().__init__() self.student = student self.teacher = teacher self.critic = critic - self.ctx = shared_context + self.dataloader = dataloader + self.validator = validator self.init_optimizers() - @classmethod - def build(cls, *, cfg, shared_context, role_models): - return cls( - student=role_models["student"], - teacher=role_models["teacher"], - critic=role_models["critic"], - shared_context=shared_context, - cfg=cfg.method, - ) - def init_optimizers(self) -> None: self.student_optimizer = build_optimizer(self.student.transformer, ...) - self.student_lr_scheduler = build_lr_scheduler(self.student_optimizer, ...) + self.student_lr_scheduler = build_scheduler(self.student_optimizer, ...) self.critic_optimizer = build_optimizer(self.critic.transformer, ...) - self.critic_lr_scheduler = build_lr_scheduler(self.critic_optimizer, ...) + self.critic_lr_scheduler = build_scheduler(self.critic_optimizer, ...) + + def single_train_step(self, raw_batch, iteration, **kwargs): + batch = self.student.prepare_batch(raw_batch, ...) # student owns this + noisy = self.student.add_noise(batch.latents, batch.noise, batch.timesteps) + + student_x0 = self.student.predict_x0(noisy, batch.timesteps, batch, ...) + teacher_x0 = self.teacher.predict_x0(noisy, batch.timesteps, batch, ...) + # teacher uses its OWN noise_scheduler for pred_noise→x0 — no sharing needed + ... @property def model_dict(self): @@ -350,42 +375,118 @@ class SelfForcingMethod(DistillMethod): @property def scheduler_dict(self): return {"student": self.student_lr_scheduler, "critic": self.critic_lr_scheduler} +``` - def get_optimizers(self, iteration: int) -> list: - return [self.student_optimizer, self.critic_optimizer] +--- + +## 3) Final YAML - def get_lr_schedulers(self, iteration: int) -> list: - return [self.student_lr_scheduler, self.critic_lr_scheduler] +### WanGame Self-Forcing + +```yaml +log: + project: fastvideo + group: wangame + name: self_forcing_causal_student_bidi_teacher + wandb_mode: online + +trainer: + output_dir: outputs/wangame_self_forcing_refactor + max_train_steps: 100000 + seed: 1000 + mixed_precision: bf16 + grad_accum_rounds: 1 + +data: + _target_: fastvideo.distillation.utils.dataloader.ParquetDataConfig + data_path: /path/to/wangame/parquet + dataloader_num_workers: 4 + +validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json + every_steps: 100 + sampling_steps: [4] + +models: + student: + _target_: fastvideo.distillation.models.wangame.WanGameCausalModel + init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + trainable: true + teacher: + _target_: fastvideo.distillation.models.wangame.WanGameModel + init_from: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers + trainable: false + critic: + _target_: fastvideo.distillation.models.wangame.WanGameModel + init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 + trainable: true + +method: + _target_: fastvideo.distillation.methods.distribution_matching.self_forcing.SelfForcingMethod + rollout_mode: simulate + chunk_size: 3 + student_sample_type: sde + context_noise: 0.0 +``` + +No `is_student` flag in YAML. The method's `build()` decides which role gets +`init_preprocessors()`. + +### Wan DMD2 + +```yaml +models: + student: + _target_: fastvideo.distillation.models.wan.WanModel + init_from: /path/to/wan_14b + trainable: true + teacher: + _target_: fastvideo.distillation.models.wan.WanModel + init_from: /path/to/wan_14b + trainable: false + fake_score: + _target_: fastvideo.distillation.models.wan.WanModel + init_from: /path/to/wan_14b + trainable: true + +method: + _target_: fastvideo.distillation.methods.distribution_matching.dmd2.DMD2Method + guidance_scale: 5.0 + student_update_freq: 5 ``` --- -## 4) Assembly flow (dispatch / entrypoint) +## 4) Assembly flow ```python cfg = load_run_config(path) -# 1. SharedContext — VAE, scheduler, dataloader, validator -shared_context = instantiate(cfg.shared_context) - -# 2. Per-role models — each owns one transformer + forward ops -# shared_context is injected so each model can read batch fields it needs +# 1. Build per-role models. No init_preprocessors() here — that is the method's job. +# Analogous to FastGenModel instantiating self.net and self.teacher as raw networks. role_models = { - role: instantiate(model_cfg, shared_context=shared_context) + role: instantiate(model_cfg) # just transformer + noise_scheduler for role, model_cfg in cfg.models.items() } -# 3. Method class assembles itself from role models (analogous to FastGenModel.__init__) +# 2. Dataloader and validator are external (trainer already takes dataloader separately). +dataloader = instantiate(cfg.data, training_args=training_args) +validator = build_validator(cfg.validation) if cfg.validation.enabled else None + +# 3. Method build() calls init_preprocessors() on student, then init_optimizers(). +# Analogous to FastGenModel.__init__ → build_model() → init_preprocessors(). method_cls = resolve_target(cfg.method["_target_"]) method = method_cls.build( cfg=cfg, - shared_context=shared_context, role_models=role_models, + dataloader=dataloader, + validator=validator, ) -# 4. Generic trainer loop, not method-aware beyond the DistillMethod interface -trainer = Trainer(cfg.trainer) -trainer.run(method, shared_context=shared_context) +# 4. Trainer runs the loop. +trainer = DistillTrainer(training_args) +trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps, ...) ``` --- @@ -394,28 +495,27 @@ trainer.run(method, shared_context=shared_context) | Current | New | |---|---| -| `dispatch.py` string registry (`_MODELS`, `_METHODS`, `@register_*`) | Deleted; `_target_` + `instantiate()` | -| `ModelBase` — one class handles ALL roles via `RoleHandle` arg | Retired; replaced by per-role `ModelBase` (one instance per role) | -| `CausalModelBase` | Becomes proper ABC for streaming, same concept | -| `RoleHandle` / `RoleManager` | Retired; method owns role objects directly | -| `DistillMethod.build(cfg, bundle, model, validator)` | `DistillMethod.build(cfg, shared_context, role_models)` | +| `dispatch.py` string registry | Deleted; `_target_` + `instantiate()` | +| `ModelBase` — dispatches via `RoleHandle` arg | Retired; per-role instance owns one transformer | +| `CausalModelBase(ModelBase)` subclass | `CausalModelBase` parallel root (not a subclass) | +| `RoleHandle` / `RoleManager` | Retired; method owns `self.student`, `self.teacher`, etc. | | Optimizers on `RoleHandle.optimizers` | On method: `self.student_optimizer`, etc. | -| `recipe.family` / `recipe.method` YAML keys | Deleted; `shared_context._target_` + `method._target_` | -| `roles.*` YAML section | Replaced by `models.*` with `_target_`, `init_from`, `trainable` | -| Method checkpoint uses `bundle.roles` dict | Uses `model_dict`, `optimizer_dict`, `scheduler_dict` properties | +| `DistillMethod.build(cfg, bundle, model, validator)` | `DistillMethod.build(cfg, role_models, dataloader, validator)` | +| `recipe.family` / `recipe.method` YAML | Deleted; `models.._target_` + `method._target_` | +| `roles.*` YAML section | `models.*` with `_target_`, `init_from`, `trainable` | +| `is_student` / mixin / SharedContext | None of these. `build()` calls `init_preprocessors()` on student. | --- -## 6) Naming cleanup (separate PR, do NOT mix in) +## 6) Naming cleanup (separate PR) - `DistillMethod` → `Method` - `DistillRunConfig` → `RunConfig` -- `load_distill_run_config` → `load_run_config` - Entrypoint `distillation.py` → `training.py` (optional) --- -## 7) TODO (this phase — wangame first, then extend to wan) +## 7) TODO (wangame first, then wan) **Infrastructure** @@ -424,56 +524,42 @@ trainer.run(method, shared_context=shared_context) - `instantiate(cfg: dict, **extra) -> Any` - [ ] `fastvideo/distillation/utils/config.py` - - New `RunConfig` dataclass (no `RecipeSpec`, no `RoleSpec`) - - New `load_run_config(path)` parser for new YAML schema + - New `RunConfig` dataclass + - New `load_run_config(path)` parser - Keep `load_distill_run_config` as deprecated shim **Base classes** - [ ] `fastvideo/distillation/models/base.py` - - New `SharedContextBase` ABC - - New `ModelBase` ABC (per-role; `self.transformer`, `self.ctx`, `predict_noise`, `predict_x0`) - - New `CausalModelBase(ModelBase)` ABC (streaming ops, `clear_caches`) - - Retire old `ModelBase` once migration is done + - New `ModelBase` ABC with `init_preprocessors` (no-op default), `prepare_batch` (raises), `add_noise` (raises) + - New `CausalModelBase` ABC (parallel root, same pattern + streaming methods) + - Retire old `ModelBase` / `CausalModelBase` -- [ ] `fastvideo/distillation/roles.py` - - Retire `RoleHandle` / `RoleManager` (keep temporarily if trainer checkpoint code needs it, then remove) +- [ ] `fastvideo/distillation/roles.py` — retire `RoleHandle` / `RoleManager` - [ ] `fastvideo/distillation/methods/base.py` - - New `DistillMethod.build(cfg, shared_context, role_models)` signature + - New `DistillMethod.build(cfg, role_models, dataloader, validator)` signature - Add abstract `init_optimizers`, `model_dict`, `optimizer_dict`, `scheduler_dict` - - Remove `bundle`, `model`, `validator` parameters - -**WanGame models** - -- [ ] `fastvideo/distillation/models/wangame/shared_context.py` - - `WanGameSharedContext(SharedContextBase)`: extracts VAE, scheduler, `prepare_batch`, - `add_noise`, `on_train_start`, dataloader, validator from current `WanGameModel.__init__` -- [ ] `fastvideo/distillation/models/wangame/models.py` - - `WanGameModelBase(ModelBase)`: shared bidi forward logic (input_kwargs, CFG, action cond) - - `WanGameModel(WanGameModelBase)`: bidi transformer - - `WanGameCausalModel(CausalModelBase, WanGameModelBase)`: causal transformer + streaming ops - - Transformer loading moves into each class `__init__` (replaces `common._build_wangame_role_handles`) +**WanGame** -**Wan models** +- [ ] `fastvideo/distillation/models/wangame/model.py` + - `WanGameModel(ModelBase)`: always builds `noise_scheduler`; `init_preprocessors()` loads VAE + - `WanGameCausalModel(CausalModelBase)`: same pattern + streaming -- [ ] `fastvideo/distillation/models/wan/shared_context.py` - - `WanSharedContext(SharedContextBase)`: same extraction from current `WanModel` +**Wan** -- [ ] `fastvideo/distillation/models/wan/models.py` - - `WanModelBase(ModelBase)`: shared bidi forward logic (text conditioning, CFG) - - `WanModel(WanModelBase)`: bidi transformer - - `WanCausalModel(CausalModelBase, WanModelBase)`: causal transformer + streaming ops +- [ ] `fastvideo/distillation/models/wan/model.py` + - `WanModel(ModelBase)`: always builds `noise_scheduler`; `init_preprocessors()` loads VAE + negative prompt + - `WanCausalModel(CausalModelBase)`: same + streaming **Methods** -- [ ] Update all method `build()` signatures to new interface: - - `SelfForcingMethod`, `DMD2Method`, `FinetuneMethod`, `DFSFTMethod` +- [ ] Update all `build()` signatures: `SelfForcingMethod`, `DMD2Method`, `FinetuneMethod`, `DFSFTMethod` +- [ ] `DMD2Method.build()`: calls `init_preprocessors()` on student; `init_optimizers()` adds `fake_score_optimizer` **Dispatch & configs** -- [ ] `fastvideo/distillation/dispatch.py`: replace body with new assembly flow (or delete) -- [ ] New YAML configs for wangame self-forcing (validate end-to-end) -- [ ] Update existing wangame YAML configs to new schema -- [ ] Update wan YAML configs +- [ ] `fastvideo/distillation/dispatch.py`: replace with new assembly flow (or delete) +- [ ] New YAML for wangame self-forcing; validate end-to-end +- [ ] Migrate existing wangame and wan YAML configs From 264ac26a55b2f4e77ede0c26638e174aaaef8035 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 06:47:53 +0000 Subject: [PATCH 164/214] deisgning refactor 6 --- dev/phase_refactor.md | 170 ++++++++++++++++++++++++++++-------------- 1 file changed, 115 insertions(+), 55 deletions(-) diff --git a/dev/phase_refactor.md b/dev/phase_refactor.md index 215b07076..a34d35278 100644 --- a/dev/phase_refactor.md +++ b/dev/phase_refactor.md @@ -47,9 +47,10 @@ fastvideo/distillation/ ├── models/ │ ├── base.py ModelBase, CausalModelBase (parallel roots, no mixin) │ ├── wan/ -│ │ └── model.py WanModel(ModelBase), WanCausalModel(CausalModelBase) +│ │ └── model.py WanModel(ModelBase) │ └── wangame/ -│ └── model.py WanGameModel(ModelBase), WanGameCausalModel(CausalModelBase) +│ ├── model.py WanGameModel(ModelBase) +│ └── model_causal.py WanGameCausalModel(CausalModelBase) ├── methods/ │ ├── base.py DistillMethod (analogous to FastGenModel) │ └── distribution_matching/ @@ -70,29 +71,31 @@ all optimizers as attributes. `RoleManager` is retired. ## 2) Class interfaces -### 2.1 ModelBase and CausalModelBase (parallel roots) +### 2.1 ModelBase and CausalModelBase + +`CausalModelBase` **inherits** `ModelBase` (unlike FastGen where they are parallel). +Reason: `predict_noise`/`predict_x0` have the same signature on both — causal only +adds streaming methods. Inheritance means `isinstance(causal_student, ModelBase)` is +`True`, which is useful when a method slot (e.g. `fake_score` in DMD2) can accept +either kind. ```python # fastvideo/distillation/models/base.py class ModelBase(ABC): - """Per-role bidirectional model. Analogous to FastGenNetwork. - Every instance owns its own noise_scheduler. + """Per-role model. Every instance owns its own noise_scheduler. init_preprocessors() is only called on the student by DistillMethod.build(). """ transformer: nn.Module noise_scheduler: Any # always set in __init__ def init_preprocessors(self, training_args: Any) -> None: - """Load VAE, seed RNGs, build prepare_batch machinery. - Analogous to FastGenNetwork.init_preprocessors(). - Only called on the student role. Default: no-op (for teacher/critic). - Subclasses override when they are capable of being the student. + """Load VAE, seed RNGs. Analogous to FastGenNetwork.init_preprocessors(). + Only called on the student. Default: no-op (teacher/critic skip this). """ pass def on_train_start(self) -> None: - """Called by trainer before training loop. Default: no-op.""" pass def get_rng_generators(self) -> dict[str, torch.Generator]: @@ -105,17 +108,12 @@ class ModelBase(ABC): current_vsa_sparsity: float = 0.0, latents_source: str = "data", ) -> TrainingBatch: - """Build TrainingBatch from raw dataloader output. - Raises NotImplementedError unless init_preprocessors() was called. - """ raise NotImplementedError( f"{type(self).__name__}.prepare_batch() requires init_preprocessors() " - "to have been called (student only)." + "(student only)." ) - def add_noise( - self, clean: Tensor, noise: Tensor, timestep: Tensor - ) -> Tensor: + def add_noise(self, clean: Tensor, noise: Tensor, timestep: Tensor) -> Tensor: raise NotImplementedError @abstractmethod @@ -133,40 +131,14 @@ class ModelBase(ABC): @abstractmethod def predict_x0(self, ...) -> Tensor: ... - def backward( - self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int = 1 - ) -> None: + def backward(self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int = 1) -> None: (loss / max(1, grad_accum_rounds)).backward() -class CausalModelBase(ABC): - """Per-role causal model. Parallel root to ModelBase, NOT a subclass. - Analogous to CausalFastGenNetwork. - Same interface as ModelBase plus streaming ops. +class CausalModelBase(ModelBase): + """Causal/streaming extension. Subclass of ModelBase (same predict signatures, + adds streaming ops). isinstance(causal_model, ModelBase) == True. """ - transformer: nn.Module - noise_scheduler: Any # always set in __init__ - - def init_preprocessors(self, training_args: Any) -> None: - pass - - def on_train_start(self) -> None: - pass - - def get_rng_generators(self) -> dict[str, torch.Generator]: - return {} - - def prepare_batch(self, ...) -> TrainingBatch: - raise NotImplementedError - - def add_noise(self, ...) -> Tensor: - raise NotImplementedError - - @abstractmethod - def predict_noise(self, ...) -> Tensor: ... - - @abstractmethod - def predict_x0(self, ...) -> Tensor: ... @abstractmethod def predict_noise_streaming( @@ -188,12 +160,95 @@ class CausalModelBase(ABC): @abstractmethod def clear_caches(self, *, cache_tag: str = "pos") -> None: ... +``` - def backward(self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int = 1) -> None: - (loss / max(1, grad_accum_rounds)).backward() +### 2.2 Dynamic config — `__init__` signature as schema + +We abandon structured config dataclasses (no `DistillRunConfig`, no `RecipeSpec`, +no `attrs` configs). Instead, each `_target_` class declares its own schema via its +`__init__` type annotations. The `instantiate()` utility introspects +`inspect.signature()` to: + +1. **Validate**: detect missing required args (no default) and unknown keys early. +2. **Filter**: only pass kwargs the constructor accepts, ignoring extras. +3. **Document**: `help(WanGameModel)` and IDE hover shows the exact fields. + +```python +# fastvideo/distillation/utils/instantiate.py + +import importlib, inspect +from typing import Any + + +def resolve_target(target: str) -> type: + module_path, cls_name = target.rsplit(".", 1) + return getattr(importlib.import_module(module_path), cls_name) + + +def instantiate(cfg: dict[str, Any], /, **extra: Any) -> Any: + """Hydra-style _target_ instantiation with signature-based validation. + + - Pops '_target_', resolves the class. + - Merges cfg fields with extra kwargs. + - Inspects __init__ signature: + * Raises if a required parameter is missing. + * Raises if an unrecognised key is passed (no **kwargs on target). + * Passes only accepted kwargs otherwise. + """ + cfg = dict(cfg) + target_str = cfg.pop("_target_") + cls = resolve_target(target_str) + kwargs = {**cfg, **extra} + + sig = inspect.signature(cls.__init__) + params = { + name: p + for name, p in sig.parameters.items() + if name != "self" + } + has_var_keyword = any( + p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values() + ) + + # Validate required params are present + for name, p in params.items(): + if ( + p.kind in (p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY) + and p.default is inspect.Parameter.empty + and name not in kwargs + ): + raise TypeError( + f"instantiate({target_str!r}): missing required argument {name!r}" + ) + + # Validate no unexpected keys (unless class accepts **kwargs) + if not has_var_keyword: + unexpected = set(kwargs) - set(params) + if unexpected: + raise TypeError( + f"instantiate({target_str!r}): unexpected config keys {sorted(unexpected)}" + ) + + return cls(**kwargs) ``` -### 2.2 Concrete model classes (WanGame example) +**Each class is self-documenting.** Example: + +```python +class WanGameModel(ModelBase): + def __init__( + self, + *, + init_from: str, # required — instantiate() raises if missing + trainable: bool = True, # optional with default + ) -> None: ... +``` + +YAML validation is thus free — no separate schema file, no config dataclass to +maintain in parallel. Adding a new field to a class means adding it to `__init__`. +Removing a field and forgetting to update YAML → `TypeError` at startup. + +### 2.4 Concrete model classes (WanGame example) ```python # fastvideo/distillation/models/wangame/model.py @@ -248,8 +303,11 @@ class WanGameModel(ModelBase): return pred_noise_to_pred_video(pred_noise, noisy_latents, timestep, self.noise_scheduler) + +# fastvideo/distillation/models/wangame/model_causal.py + class WanGameCausalModel(CausalModelBase): - """Causal WanGame. Parallel root to WanGameModel. + """Causal WanGame. Parallel root to WanGameModel (not a subclass). Same init_preprocessors() pattern — called only on student. """ def __init__(self, *, init_from: str, trainable: bool = True, **kwargs) -> None: @@ -268,7 +326,7 @@ class WanGameCausalModel(CausalModelBase): def clear_caches(self, *, cache_tag: str = "pos") -> None: ... ``` -### 2.3 DistillMethod (analogous to FastGenModel) +### 2.5 DistillMethod (analogous to FastGenModel) ```python class DistillMethod(nn.Module, ABC): @@ -314,7 +372,7 @@ class DistillMethod(nn.Module, ABC): def scheduler_dict(self) -> dict[str, Any]: ... ``` -### 2.4 SelfForcingMethod (example) +### 2.6 SelfForcingMethod (example) ```python class SelfForcingMethod(DistillMethod): @@ -522,6 +580,7 @@ trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps - [ ] `fastvideo/distillation/utils/instantiate.py` - `resolve_target(target: str) -> type` - `instantiate(cfg: dict, **extra) -> Any` + - Uses `inspect.signature()` for field validation and filtering (see §2.5) - [ ] `fastvideo/distillation/utils/config.py` - New `RunConfig` dataclass @@ -545,13 +604,14 @@ trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps - [ ] `fastvideo/distillation/models/wangame/model.py` - `WanGameModel(ModelBase)`: always builds `noise_scheduler`; `init_preprocessors()` loads VAE - - `WanGameCausalModel(CausalModelBase)`: same pattern + streaming + +- [ ] `fastvideo/distillation/models/wangame/model_causal.py` + - `WanGameCausalModel(CausalModelBase)`: same pattern + streaming ops **Wan** - [ ] `fastvideo/distillation/models/wan/model.py` - `WanModel(ModelBase)`: always builds `noise_scheduler`; `init_preprocessors()` loads VAE + negative prompt - - `WanCausalModel(CausalModelBase)`: same + streaming **Methods** From 3e1063f357bc80c24d2d9a66bf9cd5354d8910dd Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 07:05:41 +0000 Subject: [PATCH 165/214] refactor 7 --- dev/phase_refactor.md | 141 +++++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/dev/phase_refactor.md b/dev/phase_refactor.md index a34d35278..891210033 100644 --- a/dev/phase_refactor.md +++ b/dev/phase_refactor.md @@ -73,11 +73,11 @@ all optimizers as attributes. `RoleManager` is retired. ### 2.1 ModelBase and CausalModelBase -`CausalModelBase` **inherits** `ModelBase` (unlike FastGen where they are parallel). -Reason: `predict_noise`/`predict_x0` have the same signature on both — causal only -adds streaming methods. Inheritance means `isinstance(causal_student, ModelBase)` is -`True`, which is useful when a method slot (e.g. `fake_score` in DMD2) can accept -either kind. +`CausalModelBase` is a **parallel root** to `ModelBase`, mirroring FastGen's design. +Each concrete method slot accepts exactly one expected type — there is no legitimate +use-case where a `fake_score` or `teacher` slot would accept either kind +interchangeably. Parallel roots keep the contracts clean: `ModelBase` for +bidirectional models, `CausalModelBase` for streaming/causal models. ```python # fastvideo/distillation/models/base.py @@ -135,10 +135,46 @@ class ModelBase(ABC): (loss / max(1, grad_accum_rounds)).backward() -class CausalModelBase(ModelBase): - """Causal/streaming extension. Subclass of ModelBase (same predict signatures, - adds streaming ops). isinstance(causal_model, ModelBase) == True. +class CausalModelBase(ABC): + """Parallel root for causal/streaming models. NOT a subclass of ModelBase. + Mirrors FastGen's design: CausalFastGenNetwork is parallel to FastGenNetwork. + Concrete method slots that need a causal model declare CausalModelBase explicitly. """ + transformer: nn.Module + noise_scheduler: Any # always set in __init__ + + def init_preprocessors(self, training_args: Any) -> None: + pass + + def on_train_start(self) -> None: + pass + + def get_rng_generators(self) -> dict[str, torch.Generator]: + return {} + + def prepare_batch( + self, + raw_batch: dict[str, Any], + *, + current_vsa_sparsity: float = 0.0, + latents_source: str = "data", + ) -> TrainingBatch: + raise NotImplementedError( + f"{type(self).__name__}.prepare_batch() requires init_preprocessors() " + "(student only)." + ) + + def add_noise(self, clean: Tensor, noise: Tensor, timestep: Tensor) -> Tensor: + raise NotImplementedError + + @abstractmethod + def predict_noise(self, noisy_latents: Tensor, timestep: Tensor, batch: TrainingBatch, *, conditional: bool, cfg_uncond: dict | None = None, attn_kind: str = "dense") -> Tensor: ... + + @abstractmethod + def predict_x0(self, ...) -> Tensor: ... + + def backward(self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int = 1) -> None: + (loss / max(1, grad_accum_rounds)).backward() @abstractmethod def predict_noise_streaming( @@ -166,17 +202,20 @@ class CausalModelBase(ModelBase): We abandon structured config dataclasses (no `DistillRunConfig`, no `RecipeSpec`, no `attrs` configs). Instead, each `_target_` class declares its own schema via its -`__init__` type annotations. The `instantiate()` utility introspects -`inspect.signature()` to: +`__init__` type annotations. The `instantiate()` utility: + +1. Pops `_target_` and resolves the class. +2. Passes the **entire remaining dict** into the constructor — each class takes what + it needs via `**kwargs` and ignores the rest. -1. **Validate**: detect missing required args (no default) and unknown keys early. -2. **Filter**: only pass kwargs the constructor accepts, ignoring extras. -3. **Document**: `help(WanGameModel)` and IDE hover shows the exact fields. +This is the "whole config dict flows through" model: callers don't need to know which +keys a constructor cares about, and adding a field to a class means adding it to +`__init__` — no schema file to maintain in parallel. ```python # fastvideo/distillation/utils/instantiate.py -import importlib, inspect +import importlib from typing import Any @@ -186,68 +225,30 @@ def resolve_target(target: str) -> type: def instantiate(cfg: dict[str, Any], /, **extra: Any) -> Any: - """Hydra-style _target_ instantiation with signature-based validation. - - - Pops '_target_', resolves the class. - - Merges cfg fields with extra kwargs. - - Inspects __init__ signature: - * Raises if a required parameter is missing. - * Raises if an unrecognised key is passed (no **kwargs on target). - * Passes only accepted kwargs otherwise. + """_target_ instantiation: pop _target_, resolve class, pass everything else through. + + Each constructor accepts what it needs via explicit params + **kwargs for the rest. + No signature inspection — simplicity over strict validation. """ cfg = dict(cfg) target_str = cfg.pop("_target_") cls = resolve_target(target_str) - kwargs = {**cfg, **extra} - - sig = inspect.signature(cls.__init__) - params = { - name: p - for name, p in sig.parameters.items() - if name != "self" - } - has_var_keyword = any( - p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values() - ) - - # Validate required params are present - for name, p in params.items(): - if ( - p.kind in (p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY) - and p.default is inspect.Parameter.empty - and name not in kwargs - ): - raise TypeError( - f"instantiate({target_str!r}): missing required argument {name!r}" - ) - - # Validate no unexpected keys (unless class accepts **kwargs) - if not has_var_keyword: - unexpected = set(kwargs) - set(params) - if unexpected: - raise TypeError( - f"instantiate({target_str!r}): unexpected config keys {sorted(unexpected)}" - ) - - return cls(**kwargs) + return cls(**{**cfg, **extra}) ``` -**Each class is self-documenting.** Example: +**Each class is self-documenting via its explicit params.** Example: ```python class WanGameModel(ModelBase): def __init__( self, *, - init_from: str, # required — instantiate() raises if missing + init_from: str, # required — TypeError at startup if missing from YAML trainable: bool = True, # optional with default + **kwargs, # absorbs any other keys in the config dict ) -> None: ... ``` -YAML validation is thus free — no separate schema file, no config dataclass to -maintain in parallel. Adding a new field to a class means adding it to `__init__`. -Removing a field and forgetting to update YAML → `TypeError` at startup. - ### 2.4 Concrete model classes (WanGame example) ```python @@ -338,12 +339,12 @@ class DistillMethod(nn.Module, ABC): *, cfg: RunConfig, role_models: dict[str, ModelBase | CausalModelBase], - dataloader: Any, validator: Any | None, ) -> "DistillMethod": """Assemble the method. Calls init_preprocessors() on the student, then calls init_optimizers(). Analogous to FastGenModel.__init__ → build_model() → init_preprocessors(). + Dataloader is NOT passed here — it is owned by the trainer. """ ... @@ -378,7 +379,7 @@ class DistillMethod(nn.Module, ABC): class SelfForcingMethod(DistillMethod): @classmethod - def build(cls, *, cfg, role_models, dataloader, validator): + def build(cls, *, cfg, role_models, validator): student = role_models["student"] teacher = role_models["teacher"] critic = role_models["critic"] @@ -395,15 +396,15 @@ class SelfForcingMethod(DistillMethod): return cls( student=student, teacher=teacher, critic=critic, - dataloader=dataloader, validator=validator, cfg=cfg.method, + validator=validator, cfg=cfg.method, ) - def __init__(self, student, teacher, critic, dataloader, validator, cfg): + def __init__(self, student, teacher, critic, validator, cfg): super().__init__() self.student = student self.teacher = teacher self.critic = critic - self.dataloader = dataloader + # No self.dataloader — trainer owns it and passes raw_batch into single_train_step. self.validator = validator self.init_optimizers() @@ -528,21 +529,21 @@ role_models = { for role, model_cfg in cfg.models.items() } -# 2. Dataloader and validator are external (trainer already takes dataloader separately). +# 2. Dataloader is owned by the trainer. Build it here for trainer use only. dataloader = instantiate(cfg.data, training_args=training_args) validator = build_validator(cfg.validation) if cfg.validation.enabled else None # 3. Method build() calls init_preprocessors() on student, then init_optimizers(). +# No dataloader passed — trainer passes raw_batch into single_train_step each step. # Analogous to FastGenModel.__init__ → build_model() → init_preprocessors(). method_cls = resolve_target(cfg.method["_target_"]) method = method_cls.build( cfg=cfg, role_models=role_models, - dataloader=dataloader, validator=validator, ) -# 4. Trainer runs the loop. +# 4. Trainer runs the loop. Trainer is the sole owner of the dataloader. trainer = DistillTrainer(training_args) trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps, ...) ``` @@ -555,10 +556,10 @@ trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps |---|---| | `dispatch.py` string registry | Deleted; `_target_` + `instantiate()` | | `ModelBase` — dispatches via `RoleHandle` arg | Retired; per-role instance owns one transformer | -| `CausalModelBase(ModelBase)` subclass | `CausalModelBase` parallel root (not a subclass) | +| `CausalModelBase(ModelBase)` subclass | `CausalModelBase(ABC)` parallel root — mirrors FastGen | | `RoleHandle` / `RoleManager` | Retired; method owns `self.student`, `self.teacher`, etc. | | Optimizers on `RoleHandle.optimizers` | On method: `self.student_optimizer`, etc. | -| `DistillMethod.build(cfg, bundle, model, validator)` | `DistillMethod.build(cfg, role_models, dataloader, validator)` | +| `DistillMethod.build(cfg, bundle, model, validator)` | `DistillMethod.build(cfg, role_models, validator)` — no dataloader | | `recipe.family` / `recipe.method` YAML | Deleted; `models.._target_` + `method._target_` | | `roles.*` YAML section | `models.*` with `_target_`, `init_from`, `trainable` | | `is_student` / mixin / SharedContext | None of these. `build()` calls `init_preprocessors()` on student. | From 163605c6c519459cb78eb5e89763bdebb58f12f1 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 07:13:43 +0000 Subject: [PATCH 166/214] refactor 8 --- dev/phase_refactor.md | 246 +++++++++++++++++++++++++----------------- 1 file changed, 148 insertions(+), 98 deletions(-) diff --git a/dev/phase_refactor.md b/dev/phase_refactor.md index 891210033..802728b62 100644 --- a/dev/phase_refactor.md +++ b/dev/phase_refactor.md @@ -3,8 +3,8 @@ ## 0) FastGen hierarchy (the reference) ``` -FastGenNetwork (ABC, nn.Module) ← per-role backbone; owns transformer + noise_scheduler -CausalFastGenNetwork (ABC, nn.Module) ← parallel root, NOT a subclass of FastGenNetwork +FastGenNetwork (ABC, nn.Module) ← per-role backbone; owns transformer + noise_scheduler + CausalFastGenNetwork (FastGenNetwork) ← extends with chunked processing + KV caches FastGenModel (nn.Module) ← base method; owns self.net, self.teacher, optimizers DMD2Model ← adds self.fake_score, self.discriminator @@ -14,7 +14,8 @@ FastGenModel (nn.Module) ← base method; owns self.net, self.teach **Key lessons:** -1. **`CausalFastGenNetwork` is parallel to `FastGenNetwork`**, not a subclass. +1. **`CausalFastGenNetwork` extends `FastGenNetwork`** — it inherits the shared contract + (noise_scheduler, init_preprocessors, forward) and adds streaming/chunked methods. 2. **Every role instance owns its own `noise_scheduler`** — student and teacher each get an independent instance of the same schedule. No sharing needed. This is because @@ -45,7 +46,7 @@ FastGenModel (nn.Module) ← base method; owns self.net, self.teach ``` fastvideo/distillation/ ├── models/ -│ ├── base.py ModelBase, CausalModelBase (parallel roots, no mixin) +│ ├── base.py ModelBase; CausalModelBase(ModelBase) adds streaming │ ├── wan/ │ │ └── model.py WanModel(ModelBase) │ └── wangame/ @@ -62,7 +63,7 @@ fastvideo/distillation/ **Every model instance has its own `noise_scheduler`** (constructed in `__init__`). VAE and `prepare_batch` are initialized via `init_preprocessors()`, called only on -the student by `DistillMethod.build()`. No mixin. No flag. +the student by `DistillMethod.__init__()`. No mixin. No flag. **DistillMethod** owns role model objects (`self.student`, `self.teacher`, …) and all optimizers as attributes. `RoleManager` is retired. @@ -73,18 +74,17 @@ all optimizers as attributes. `RoleManager` is retired. ### 2.1 ModelBase and CausalModelBase -`CausalModelBase` is a **parallel root** to `ModelBase`, mirroring FastGen's design. -Each concrete method slot accepts exactly one expected type — there is no legitimate -use-case where a `fake_score` or `teacher` slot would accept either kind -interchangeably. Parallel roots keep the contracts clean: `ModelBase` for -bidirectional models, `CausalModelBase` for streaming/causal models. +`CausalModelBase` **extends** `ModelBase`, mirroring FastGen's design where +`CausalFastGenNetwork(FastGenNetwork)` inherits the shared contract and adds +streaming methods. This avoids duplicating the entire bidirectional interface. +Methods that require a causal student type-check with `isinstance(student, CausalModelBase)`. ```python # fastvideo/distillation/models/base.py class ModelBase(ABC): """Per-role model. Every instance owns its own noise_scheduler. - init_preprocessors() is only called on the student by DistillMethod.build(). + init_preprocessors() is only called on the student by DistillMethod.__init__(). """ transformer: nn.Module noise_scheduler: Any # always set in __init__ @@ -135,46 +135,11 @@ class ModelBase(ABC): (loss / max(1, grad_accum_rounds)).backward() -class CausalModelBase(ABC): - """Parallel root for causal/streaming models. NOT a subclass of ModelBase. - Mirrors FastGen's design: CausalFastGenNetwork is parallel to FastGenNetwork. - Concrete method slots that need a causal model declare CausalModelBase explicitly. +class CausalModelBase(ModelBase): + """Extends ModelBase with streaming/causal methods. + Mirrors FastGen: CausalFastGenNetwork(FastGenNetwork) adds chunked processing + KV caches. + Inherits all bidirectional methods; adds streaming variants. """ - transformer: nn.Module - noise_scheduler: Any # always set in __init__ - - def init_preprocessors(self, training_args: Any) -> None: - pass - - def on_train_start(self) -> None: - pass - - def get_rng_generators(self) -> dict[str, torch.Generator]: - return {} - - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: str = "data", - ) -> TrainingBatch: - raise NotImplementedError( - f"{type(self).__name__}.prepare_batch() requires init_preprocessors() " - "(student only)." - ) - - def add_noise(self, clean: Tensor, noise: Tensor, timestep: Tensor) -> Tensor: - raise NotImplementedError - - @abstractmethod - def predict_noise(self, noisy_latents: Tensor, timestep: Tensor, batch: TrainingBatch, *, conditional: bool, cfg_uncond: dict | None = None, attn_kind: str = "dense") -> Tensor: ... - - @abstractmethod - def predict_x0(self, ...) -> Tensor: ... - - def backward(self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int = 1) -> None: - (loss / max(1, grad_accum_rounds)).backward() @abstractmethod def predict_noise_streaming( @@ -205,19 +170,25 @@ no `attrs` configs). Instead, each `_target_` class declares its own schema via `__init__` type annotations. The `instantiate()` utility: 1. Pops `_target_` and resolves the class. -2. Passes the **entire remaining dict** into the constructor — each class takes what - it needs via `**kwargs` and ignores the rest. +2. Inspects the constructor signature to find accepted parameters. +3. Warns on any unexpected keys (catches typos like `trainabel` instead of `trainable`). +4. Passes only recognized keys to the constructor. This is the "whole config dict flows through" model: callers don't need to know which keys a constructor cares about, and adding a field to a class means adding it to -`__init__` — no schema file to maintain in parallel. +`__init__` — no schema file to maintain in parallel. But unlike a blind `**kwargs` +pass-through, we catch silent misconfiguration at startup. ```python # fastvideo/distillation/utils/instantiate.py import importlib +import inspect +import logging from typing import Any +logger = logging.getLogger(__name__) + def resolve_target(target: str) -> type: module_path, cls_name = target.rsplit(".", 1) @@ -225,18 +196,47 @@ def resolve_target(target: str) -> type: def instantiate(cfg: dict[str, Any], /, **extra: Any) -> Any: - """_target_ instantiation: pop _target_, resolve class, pass everything else through. + """_target_ instantiation with signature-based validation. - Each constructor accepts what it needs via explicit params + **kwargs for the rest. - No signature inspection — simplicity over strict validation. + 1. Pops _target_, resolves the class. + 2. Inspects __init__ signature — if it accepts **kwargs, all keys pass through. + Otherwise, warns on unrecognized keys and filters them out. + 3. Catches typos at startup instead of silently applying defaults. """ cfg = dict(cfg) target_str = cfg.pop("_target_") cls = resolve_target(target_str) - return cls(**{**cfg, **extra}) + merged = {**cfg, **extra} + + sig = inspect.signature(cls.__init__) + params = sig.parameters + has_var_keyword = any( + p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values() + ) + + if not has_var_keyword: + # Strict mode: only pass recognized params, warn on extras. + accepted = { + name for name, p in params.items() + if p.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) and name != "self" + } + unexpected = set(merged) - accepted + if unexpected: + logger.warning( + "%s.__init__ does not accept: %s (accepted: %s). " + "Check for typos in YAML.", + cls.__name__, sorted(unexpected), sorted(accepted), + ) + merged = {k: v for k, v in merged.items() if k in accepted} + + return cls(**merged) ``` -**Each class is self-documenting via its explicit params.** Example: +**Each class is self-documenting via its explicit params.** Constructors that want +to be strict omit `**kwargs`; those that legitimately need pass-through keep it. ```python class WanGameModel(ModelBase): @@ -245,7 +245,7 @@ class WanGameModel(ModelBase): *, init_from: str, # required — TypeError at startup if missing from YAML trainable: bool = True, # optional with default - **kwargs, # absorbs any other keys in the config dict + # No **kwargs — instantiate() will warn on unrecognized keys ) -> None: ... ``` @@ -260,7 +260,7 @@ class WanGameModel(ModelBase): VAE and prepare_batch are activated by init_preprocessors() — only for student. """ - def __init__(self, *, init_from: str, trainable: bool = True, **kwargs) -> None: + def __init__(self, *, init_from: str, trainable: bool = True) -> None: self.transformer = load_transformer(init_from, cls="WanGameActionTransformer3DModel") apply_trainable(self.transformer, trainable=trainable) # noise_scheduler always built — same as FastGenNetwork.__init__ calling set_noise_schedule() @@ -308,10 +308,10 @@ class WanGameModel(ModelBase): # fastvideo/distillation/models/wangame/model_causal.py class WanGameCausalModel(CausalModelBase): - """Causal WanGame. Parallel root to WanGameModel (not a subclass). + """Causal WanGame. Extends CausalModelBase with streaming ops. Same init_preprocessors() pattern — called only on student. """ - def __init__(self, *, init_from: str, trainable: bool = True, **kwargs) -> None: + def __init__(self, *, init_from: str, trainable: bool = True) -> None: self.transformer = load_transformer(init_from, cls="CausalWanGameTransformer") apply_trainable(self.transformer, trainable=trainable) self.noise_scheduler = FlowMatchEulerDiscreteScheduler(...) @@ -329,24 +329,29 @@ class WanGameCausalModel(CausalModelBase): ### 2.5 DistillMethod (analogous to FastGenModel) +Like FastGen's `FastGenModel.__init__` which calls `build_model()` → `init_preprocessors()` +internally, our `DistillMethod.__init__` is the single entry point. Each subclass's +`__init__` validates roles, calls `init_preprocessors()` on the student, and calls +`init_optimizers()`. No separate `build()` classmethod — one construction path, no +risk of someone calling `__init__` directly and skipping validation. + ```python class DistillMethod(nn.Module, ABC): - @classmethod - @abstractmethod - def build( - cls, + def __init__( + self, *, cfg: RunConfig, - role_models: dict[str, ModelBase | CausalModelBase], + role_models: dict[str, ModelBase], validator: Any | None, - ) -> "DistillMethod": - """Assemble the method. Calls init_preprocessors() on the student, - then calls init_optimizers(). - Analogous to FastGenModel.__init__ → build_model() → init_preprocessors(). - Dataloader is NOT passed here — it is owned by the trainer. + ) -> None: + """Subclasses override __init__, call super().__init__(), then: + 1. Validate and assign role models (self.student, self.teacher, ...). + 2. Call self.student.init_preprocessors(cfg.training_args). + 3. Call self.init_optimizers(). + Dataloader is NOT passed — trainer owns it. """ - ... + super().__init__() @abstractmethod def init_optimizers(self) -> None: ... @@ -378,8 +383,10 @@ class DistillMethod(nn.Module, ABC): ```python class SelfForcingMethod(DistillMethod): - @classmethod - def build(cls, *, cfg, role_models, validator): + def __init__(self, *, cfg, role_models, validator): + super().__init__(cfg=cfg, role_models=role_models, validator=validator) + + # 1. Validate and assign roles student = role_models["student"] teacher = role_models["teacher"] critic = role_models["critic"] @@ -390,22 +397,16 @@ class SelfForcingMethod(DistillMethod): f"got {type(student).__name__}" ) - # Call init_preprocessors only on student — identical to FastGen's build_model() - # calling init_preprocessors() only on self.net, not self.teacher. - student.init_preprocessors(cfg.training_args) - - return cls( - student=student, teacher=teacher, critic=critic, - validator=validator, cfg=cfg.method, - ) - - def __init__(self, student, teacher, critic, validator, cfg): - super().__init__() self.student = student self.teacher = teacher self.critic = critic - # No self.dataloader — trainer owns it and passes raw_batch into single_train_step. self.validator = validator + self.cfg = cfg.method + + # 2. Init preprocessors on student only — same as FastGen's build_model() + self.student.init_preprocessors(cfg.training_args) + + # 3. Init optimizers (after preprocessors, before FSDP wrapping) self.init_optimizers() def init_optimizers(self) -> None: @@ -533,21 +534,62 @@ role_models = { dataloader = instantiate(cfg.data, training_args=training_args) validator = build_validator(cfg.validation) if cfg.validation.enabled else None -# 3. Method build() calls init_preprocessors() on student, then init_optimizers(). -# No dataloader passed — trainer passes raw_batch into single_train_step each step. +# 3. Method __init__ validates roles, calls init_preprocessors() on student, +# then calls init_optimizers(). Single construction path — no separate build(). # Analogous to FastGenModel.__init__ → build_model() → init_preprocessors(). method_cls = resolve_target(cfg.method["_target_"]) -method = method_cls.build( +method = method_cls( cfg=cfg, role_models=role_models, validator=validator, ) -# 4. Trainer runs the loop. Trainer is the sole owner of the dataloader. +# 4. FSDP wrapping — trainer wraps models after construction, before training. +# See §4.1 for details. trainer = DistillTrainer(training_args) +trainer.setup_fsdp(method) + +# 5. Trainer runs the loop. Trainer is the sole owner of the dataloader. trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps, ...) ``` +### 4.1 FSDP integration + +FSDP wrapping happens **after method construction, before training**. The trainer +is responsible for wrapping — the method and models don't know about FSDP. + +```python +# In DistillTrainer.setup_fsdp(method): + +# 1. Wrap trainable models in FSDP using method.model_dict. +for name, module in method.model_dict.items(): + wrapped = FSDP(module, ...) + # Replace the transformer on the owning model object. + # method.student.transformer = wrapped, etc. + +# 2. Non-trainable models (teacher) are cast to precision and moved to device, +# but NOT wrapped in FSDP (no gradients needed). + +# 3. Preprocessors (VAE, text encoders) on the student are moved to device/dtype. +# They live outside FSDP — they are frozen inference-only modules. + +# 4. After FSDP wrapping, optimizers reference the wrapped parameters. +# This means init_optimizers() must run BEFORE FSDP wrapping, and the +# optimizer param groups automatically track the FSDP-wrapped params +# (PyTorch FSDP handles this transparently for params created pre-wrap). +``` + +**Key FSDP decisions:** + +| Concern | Approach | +|---|---| +| Which models get wrapped? | `method.model_dict` — only trainable models | +| Who owns wrapping? | Trainer, not method or model | +| Teacher FSDP? | Configurable: `cfg.trainer.add_teacher_to_fsdp` (default false for frozen teacher) | +| Meta-device init? | `cfg.trainer.fsdp_meta_init` — load weights on meta device, broadcast after wrap | +| Precision? | `cfg.trainer.fsdp_precision` — FSDP MixedPrecision policy applied at wrap time | +| Optimizer creation order? | `init_optimizers()` in `__init__` → FSDP wrap → optimizer params auto-track | + --- ## 5) What is retired / changed @@ -556,10 +598,10 @@ trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps |---|---| | `dispatch.py` string registry | Deleted; `_target_` + `instantiate()` | | `ModelBase` — dispatches via `RoleHandle` arg | Retired; per-role instance owns one transformer | -| `CausalModelBase(ModelBase)` subclass | `CausalModelBase(ABC)` parallel root — mirrors FastGen | +| `CausalModelBase(ModelBase)` subclass (broken contract) | `CausalModelBase(ModelBase)` clean inheritance — mirrors FastGen | | `RoleHandle` / `RoleManager` | Retired; method owns `self.student`, `self.teacher`, etc. | | Optimizers on `RoleHandle.optimizers` | On method: `self.student_optimizer`, etc. | -| `DistillMethod.build(cfg, bundle, model, validator)` | `DistillMethod.build(cfg, role_models, validator)` — no dataloader | +| `DistillMethod.build(cfg, bundle, model, validator)` | `DistillMethod.__init__(cfg, role_models, validator)` — no dataloader, no build() | | `recipe.family` / `recipe.method` YAML | Deleted; `models.._target_` + `method._target_` | | `roles.*` YAML section | `models.*` with `_target_`, `init_from`, `trainable` | | `is_student` / mixin / SharedContext | None of these. `build()` calls `init_preprocessors()` on student. | @@ -581,7 +623,7 @@ trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps - [ ] `fastvideo/distillation/utils/instantiate.py` - `resolve_target(target: str) -> type` - `instantiate(cfg: dict, **extra) -> Any` - - Uses `inspect.signature()` for field validation and filtering (see §2.5) + - Uses `inspect.signature()` for kwargs validation and warning on unrecognized keys (see §2.2) - [ ] `fastvideo/distillation/utils/config.py` - New `RunConfig` dataclass @@ -592,19 +634,20 @@ trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps - [ ] `fastvideo/distillation/models/base.py` - New `ModelBase` ABC with `init_preprocessors` (no-op default), `prepare_batch` (raises), `add_noise` (raises) - - New `CausalModelBase` ABC (parallel root, same pattern + streaming methods) + - New `CausalModelBase(ModelBase)` — inherits shared contract, adds streaming methods - Retire old `ModelBase` / `CausalModelBase` - [ ] `fastvideo/distillation/roles.py` — retire `RoleHandle` / `RoleManager` - [ ] `fastvideo/distillation/methods/base.py` - - New `DistillMethod.build(cfg, role_models, dataloader, validator)` signature + - New `DistillMethod.__init__(cfg, role_models, validator)` — single construction path, no build() - Add abstract `init_optimizers`, `model_dict`, `optimizer_dict`, `scheduler_dict` **WanGame** - [ ] `fastvideo/distillation/models/wangame/model.py` - `WanGameModel(ModelBase)`: always builds `noise_scheduler`; `init_preprocessors()` loads VAE + - No `**kwargs` — let `instantiate()` catch typos - [ ] `fastvideo/distillation/models/wangame/model_causal.py` - `WanGameCausalModel(CausalModelBase)`: same pattern + streaming ops @@ -616,8 +659,15 @@ trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps **Methods** -- [ ] Update all `build()` signatures: `SelfForcingMethod`, `DMD2Method`, `FinetuneMethod`, `DFSFTMethod` -- [ ] `DMD2Method.build()`: calls `init_preprocessors()` on student; `init_optimizers()` adds `fake_score_optimizer` +- [ ] Update all `__init__` signatures: `SelfForcingMethod`, `DMD2Method`, `FinetuneMethod`, `DFSFTMethod` +- [ ] `DMD2Method.__init__()`: calls `init_preprocessors()` on student; `init_optimizers()` adds `fake_score_optimizer` + +**FSDP** + +- [ ] `DistillTrainer.setup_fsdp(method)` — wraps `method.model_dict` modules in FSDP +- [ ] Configurable teacher FSDP: `cfg.trainer.add_teacher_to_fsdp` (default false) +- [ ] Meta-device init support: `cfg.trainer.fsdp_meta_init` +- [ ] FSDP MixedPrecision policy: `cfg.trainer.fsdp_precision` **Dispatch & configs** From 43231304a08bed41c0ab948a552e88281d527622 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 08:08:09 +0000 Subject: [PATCH 167/214] refactor init impl --- .../refactor/dfsft_wangame_causal_v3.yaml | 83 +++ .../self_forcing_wangame_causal_v3.yaml | 123 ++++ fastvideo/distillation/__init__.py | 3 - fastvideo/distillation/dispatch.py | 197 +++-- fastvideo/distillation/methods/__init__.py | 19 +- fastvideo/distillation/methods/base.py | 68 +- .../methods/distribution_matching/dmd2.py | 574 +++++++++------ .../distribution_matching/self_forcing.py | 474 ++++++------ .../distillation/methods/fine_tuning/dfsft.py | 406 +++++++---- .../methods/fine_tuning/finetune.py | 258 ++++--- fastvideo/distillation/models/base.py | 106 +-- fastvideo/distillation/models/wan/__init__.py | 10 +- fastvideo/distillation/models/wan/wan.py | 681 +++++++++++------- .../distillation/models/wangame/__init__.py | 33 +- .../distillation/models/wangame/common.py | 89 --- .../distillation/models/wangame/wangame.py | 642 ++++++++++++----- .../models/wangame/wangame_causal.py | 411 +++++------ fastvideo/distillation/roles.py | 38 - fastvideo/distillation/utils/checkpoint.py | 133 +++- fastvideo/distillation/utils/config.py | 358 ++++----- fastvideo/distillation/utils/instantiate.py | 107 +++ fastvideo/distillation/utils/optimizer.py | 77 +- fastvideo/distillation/validators/base.py | 4 +- .../test_optimizer_scheduler_alignment.py | 15 +- 24 files changed, 2949 insertions(+), 1960 deletions(-) create mode 100644 examples/distillation/refactor/dfsft_wangame_causal_v3.yaml create mode 100644 examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml delete mode 100644 fastvideo/distillation/models/wangame/common.py delete mode 100644 fastvideo/distillation/roles.py create mode 100644 fastvideo/distillation/utils/instantiate.py diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml new file mode 100644 index 000000000..415c64a07 --- /dev/null +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -0,0 +1,83 @@ +# V3 config: WanGame causal Diffusion-Forcing SFT (DFSFT). +# +# Uses _target_-based instantiation — each model role is an independent +# class instance; the method class is resolved directly from the YAML. + +models: + student: + _target_: fastvideo.distillation.models.wangame.WanGameCausalModel + init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + +method: + _target_: fastvideo.distillation.methods.fine_tuning.dfsft.DiffusionForcingSFTMethod + +training: + # Distributed + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + # Data (parquet dataset folder) + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1 + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_dfsft_causal_v3 + max_train_steps: 20000 + seed: 1000 + checkpoints_total_limit: 2 + + # Optimizer + learning_rate: 1.0e-5 + mixed_precision: bf16 + betas: "0.9,0.999" + weight_decay: 1.0e-4 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Method-agnostic knobs + training_cfg_rate: 0.0 + enable_gradient_checkpointing_type: full + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_dfsft_causal_v3 + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [40] + rollout_mode: streaming + sampler_kind: ode + ode_solver: euler + guidance_scale: 1.0 + num_frames: 75 + +default_pipeline_config: + flow_shift: 3 + sampler_kind: ode + +method_config: + attn_kind: dense + chunk_size: 3 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml new file mode 100644 index 000000000..9096399eb --- /dev/null +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -0,0 +1,123 @@ +# V3 config: WanGame causal Self-Forcing distillation (40-step teacher -> 4-step student). +# +# Uses _target_-based instantiation — each model role is an independent +# class instance; the method class is resolved directly from the YAML. +# +# - Teacher weights come from a DFSFT causal checkpoint (DCP format). +# - Student/Critic are warmstarted from the same checkpoint. + +models: + student: + _target_: fastvideo.distillation.models.wangame.WanGameCausalModel + init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + init_from_checkpoint: outputs/wangame_dfsft_causal_v3/persistent/checkpoint-22000 + teacher: + _target_: fastvideo.distillation.models.wangame.WanGameCausalModel + init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: false + disable_custom_init_weights: true + init_from_checkpoint: outputs/wangame_dfsft_causal_v3/persistent/checkpoint-22000 + init_from_checkpoint_role: student + critic: + _target_: fastvideo.distillation.models.wangame.WanGameCausalModel + init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + disable_custom_init_weights: true + init_from_checkpoint: outputs/wangame_dfsft_causal_v3/persistent/checkpoint-22000 + init_from_checkpoint_role: student + +method: + _target_: fastvideo.distillation.methods.distribution_matching.self_forcing.SelfForcingMethod + +training: + # Distributed (4 nodes x 8 GPUs = 32 ranks) + num_gpus: 32 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 32 + + # Data + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1 + dataloader_num_workers: 4 + + # Batch / shape + train_batch_size: 1 + train_sp_batch_size: 1 + gradient_accumulation_steps: 1 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + # Output / steps + output_dir: outputs/wangame_self_forcing_4steps_v3 + max_train_steps: 4000 + seed: 1000 + checkpoints_total_limit: 3 + + # Optimizer (student) + learning_rate: 2.0e-6 + mixed_precision: bf16 + betas: "0.0,0.999" + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + # Optimizer (critic / fake-score) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: "0.0,0.999" + fake_score_lr_scheduler: constant + + # Distillation knobs + training_cfg_rate: 0.0 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 + enable_gradient_checkpointing_type: null + + # Checkpointing + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + + # Tracking / validation + tracker_project_name: distillation_wangame + wandb_run_name: wangame_self_forcing_4steps_v3 + validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [4] + sampler_kind: sde + rollout_mode: streaming + guidance_scale: 1.0 + num_frames: 69 + +default_pipeline_config: + flow_shift: 3 + sampler_kind: sde + +method_config: + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + + warp_denoising_step: true + dmd_denoising_steps: [1000,750,500,250] + + chunk_size: 3 + student_sample_type: sde + same_step_across_blocks: false + last_step_only: false + context_noise: 0.0 + enable_gradient_in_rollout: true + start_gradient_frame: 0 + + cfg_uncond: + on_missing: error + action: keep + image: keep + text: keep diff --git a/fastvideo/distillation/__init__.py b/fastvideo/distillation/__init__.py index 658f32560..1c6599375 100644 --- a/fastvideo/distillation/__init__.py +++ b/fastvideo/distillation/__init__.py @@ -1,10 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -from fastvideo.distillation.roles import RoleManager, RoleHandle from fastvideo.distillation.trainer import DistillTrainer __all__ = [ "DistillTrainer", - "RoleManager", - "RoleHandle", ] diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index febf888b3..0cdf069a4 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -1,141 +1,112 @@ # SPDX-License-Identifier: Apache-2.0 +"""Assembly: build method + dataloader from a ``_target_``-based config.""" + from __future__ import annotations -from collections.abc import Callable from typing import Any, TYPE_CHECKING -from typing import Protocol -from fastvideo.distillation.methods.base import DistillMethod -from fastvideo.distillation.utils.config import DistillRunConfig +from fastvideo.distillation.utils.instantiate import ( + instantiate, + resolve_target, +) +from fastvideo.distillation.utils.config import RunConfig if TYPE_CHECKING: from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.distillation.models.base import ModelBase - - -class ModelBuilder(Protocol): - def __call__(self, *, cfg: DistillRunConfig) -> ModelBase: - ... - - -_MODELS: dict[str, ModelBuilder] = {} -_METHODS: dict[str, type[DistillMethod]] = {} -_BUILTINS_REGISTERED = False - - -def register_model(name: str) -> Callable[[ModelBuilder], ModelBuilder]: - name = str(name).strip() - if not name: - raise ValueError("model name cannot be empty") - - def decorator(builder: ModelBuilder) -> ModelBuilder: - if name in _MODELS: - raise KeyError(f"Model already registered: {name!r}") - _MODELS[name] = builder - return builder - - return decorator - - -def register_method( - name: str, -) -> Callable[[type[DistillMethod]], type[DistillMethod]]: - name = str(name).strip() - if not name: - raise ValueError("method name cannot be empty") - - def decorator(method_cls: type[DistillMethod]) -> type[DistillMethod]: - if name in _METHODS: - raise KeyError(f"Method already registered: {name!r}") - if not issubclass(method_cls, DistillMethod): - raise TypeError(f"Registered method must subclass DistillMethod: {method_cls}") - _METHODS[name] = method_cls - return method_cls - - return decorator - - -def ensure_builtin_registrations() -> None: - global _BUILTINS_REGISTERED - if _BUILTINS_REGISTERED: - return - - # NOTE: keep these imports explicit (no wildcard scanning) so registration - # order is stable and failures are debuggable. - from fastvideo.distillation.models import wan as _wan # noqa: F401 - from fastvideo.distillation.models import wangame as _wangame # noqa: F401 - from fastvideo.distillation.methods.distribution_matching import dmd2 as _dmd2 # noqa: F401 - from fastvideo.distillation.methods.distribution_matching import ( - self_forcing as _self_forcing, # noqa: F401 - ) - from fastvideo.distillation.methods.fine_tuning import finetune as _finetune # noqa: F401 - from fastvideo.distillation.methods.fine_tuning import dfsft as _dfsft # noqa: F401 - - _BUILTINS_REGISTERED = True - + from fastvideo.distillation.methods.base import DistillMethod -def available_models() -> list[str]: - return sorted(_MODELS.keys()) +def build_from_config( + cfg: RunConfig, +) -> tuple[TrainingArgs, DistillMethod, Any, int]: + """Build method + dataloader from a v3 run config. -def available_methods() -> list[str]: - return sorted(_METHODS.keys()) - - -def get_model(name: str) -> ModelBuilder: - ensure_builtin_registrations() - if name not in _MODELS: - raise KeyError(f"Unknown model {name!r}. Available: {available_models()}") - return _MODELS[name] - - -def get_method(name: str) -> type[DistillMethod]: - ensure_builtin_registrations() - if name not in _METHODS: - raise KeyError(f"Unknown method {name!r}. Available: {available_methods()}") - return _METHODS[name] - - -def build_from_config(cfg: DistillRunConfig) -> tuple[TrainingArgs, DistillMethod, Any, int]: - """Build method+dataloader from a YAML config. - - Assembles: - - model components (bundle/adapter/dataloader/validator) - - method implementation (algorithm) on top of those components + 1. Instantiate each model in ``cfg.models`` via ``_target_``. + 2. Resolve the method class from ``cfg.method["_target_"]``. + 3. Construct the method with ``(cfg=cfg, role_models=..., + validator=...)``. + 4. Return ``(training_args, method, dataloader, start_step)``. """ + from fastvideo.distillation.models.base import ModelBase - model_builder = get_model(str(cfg.recipe.family)) - model = model_builder(cfg=cfg) + # --- 1. Build role model instances --- + role_models: dict[str, ModelBase] = {} + for role, model_cfg in cfg.models.items(): + model = instantiate(model_cfg) + if not isinstance(model, ModelBase): + raise TypeError( + f"models.{role}._target_ must resolve to a " + f"ModelBase subclass, got {type(model).__name__}" + ) + role_models[role] = model - from fastvideo.distillation.utils.checkpoint import maybe_warmstart_role_modules + # --- 2. Warm-start from checkpoint if needed --- + from fastvideo.distillation.utils.checkpoint import ( + maybe_warmstart_role_modules, + ) - for role, role_cfg in cfg.roles.items(): - init_from_checkpoint = (role_cfg.extra or {}).get("init_from_checkpoint", None) + for role, model_cfg in cfg.models.items(): + init_from_checkpoint = model_cfg.get( + "init_from_checkpoint", None + ) if not init_from_checkpoint: continue - - checkpoint_role = (role_cfg.extra or {}).get("init_from_checkpoint_role", None) - if checkpoint_role is not None and not isinstance(checkpoint_role, str): + checkpoint_role = model_cfg.get( + "init_from_checkpoint_role", None + ) + if ( + checkpoint_role is not None + and not isinstance(checkpoint_role, str) + ): raise ValueError( - f"roles.{role}.init_from_checkpoint_role must be a string when set, " - f"got {type(checkpoint_role).__name__}" + f"models.{role}.init_from_checkpoint_role " + "must be a string when set, got " + f"{type(checkpoint_role).__name__}" ) - + # Warmstart uses the model's transformer directly. + model = role_models[role] maybe_warmstart_role_modules( - bundle=model.bundle, + bundle=None, role=str(role), init_from_checkpoint=str(init_from_checkpoint), - checkpoint_role=str(checkpoint_role) if checkpoint_role else None, + checkpoint_role=( + str(checkpoint_role) + if checkpoint_role + else None + ), + model=model, ) - method_cls = get_method(str(cfg.recipe.method)) - method = method_cls.build( + # --- 3. Build method --- + method_cfg = dict(cfg.method) + method_target = str(method_cfg.pop("_target_")) + method_cls = resolve_target(method_target) + + # The student model provides the validator and dataloader. + student = role_models.get("student") + validator = ( + getattr(student, "validator", None) + if student is not None + else None + ) + + method = method_cls( cfg=cfg, - bundle=model.bundle, - model=model, - validator=model.validator, + role_models=role_models, + validator=validator, + ) + + # --- 4. Gather dataloader and start_step --- + dataloader = ( + getattr(student, "dataloader", None) + if student is not None + else None + ) + start_step = int( + getattr(student, "start_step", 0) + if student is not None + else 0 ) - start_step = int(getattr(model, "start_step", 0) or 0) - return model.training_args, method, model.dataloader, start_step + return cfg.training_args, method, dataloader, start_step diff --git a/fastvideo/distillation/methods/__init__.py b/fastvideo/distillation/methods/__init__.py index f1d11ff6c..4dc7bb63a 100644 --- a/fastvideo/distillation/methods/__init__.py +++ b/fastvideo/distillation/methods/__init__.py @@ -1,30 +1,27 @@ # SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - -from typing import TYPE_CHECKING - from fastvideo.distillation.methods.base import DistillMethod -if TYPE_CHECKING: - from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method - from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod - __all__ = [ "DistillMethod", "DMD2Method", "FineTuneMethod", + "SelfForcingMethod", + "DiffusionForcingSFTMethod", ] def __getattr__(name: str) -> object: - # Lazy import to avoid circular imports during registry bring-up. if name == "DMD2Method": from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method - return DMD2Method if name == "FineTuneMethod": from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod - return FineTuneMethod + if name == "SelfForcingMethod": + from fastvideo.distillation.methods.distribution_matching.self_forcing import SelfForcingMethod + return SelfForcingMethod + if name == "DiffusionForcingSFTMethod": + from fastvideo.distillation.methods.fine_tuning.dfsft import DiffusionForcingSFTMethod + return DiffusionForcingSFTMethod raise AttributeError(name) diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py index c544ec83c..912ca97a8 100644 --- a/fastvideo/distillation/methods/base.py +++ b/fastvideo/distillation/methods/base.py @@ -4,48 +4,48 @@ from abc import ABC, abstractmethod from collections.abc import Sequence -from typing import Any, TYPE_CHECKING +from typing import Any import torch -from fastvideo.distillation.roles import RoleManager - -if TYPE_CHECKING: - from fastvideo.distillation.utils.config import DistillRunConfig +from fastvideo.distillation.models.base import ModelBase LogScalar = float | int | torch.Tensor class DistillMethod(torch.nn.Module, ABC): - def __init__(self, bundle: RoleManager) -> None: + """Base distillation method (algorithm layer). + + Subclasses own their role models (student, teacher, critic, …) as + plain attributes and manage optimizers directly — no ``RoleManager`` + or ``RoleHandle``. + + The constructor receives *role_models* (a ``dict[str, ModelBase]``) + and a *cfg* object. It calls ``init_preprocessors`` on the student + and builds ``self.role_modules`` for FSDP wrapping. + """ + + def __init__( + self, + *, + role_models: dict[str, ModelBase], + ) -> None: super().__init__() - self.bundle = bundle self.tracker: Any | None = None + # Build nn.ModuleDict for FSDP / checkpoint visibility. self.role_modules = torch.nn.ModuleDict() - for role, handle in bundle.roles.items(): - if handle.modules: - self.role_modules[role] = torch.nn.ModuleDict(handle.modules) - - @classmethod - @abstractmethod - def build( - cls, - *, - cfg: DistillRunConfig, - bundle: RoleManager, - model: Any, - validator: Any | None, - ) -> DistillMethod: - raise NotImplementedError + for role, model in role_models.items(): + mods: dict[str, torch.nn.Module] = {} + transformer = getattr(model, "transformer", None) + if isinstance(transformer, torch.nn.Module): + mods["transformer"] = transformer + transformer_2 = getattr(model, "transformer_2", None) + if isinstance(transformer_2, torch.nn.Module): + mods["transformer_2"] = transformer_2 + if mods: + self.role_modules[role] = torch.nn.ModuleDict(mods) def set_tracker(self, tracker: Any) -> None: - """Attach a tracker (infra) to this method. - - Trainer constructs/owns the tracker, but method-managed validation may - need it to log artifacts (videos/images/files). This is a best-effort - bridge that keeps model plugins free of tracker ownership. - """ - self.tracker = tracker validator = getattr(self, "validator", None) if validator is None: @@ -64,11 +64,17 @@ def single_train_step( iteration: int, *, current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: + ) -> tuple[ + dict[str, torch.Tensor], + dict[str, Any], + dict[str, LogScalar], + ]: raise NotImplementedError @abstractmethod - def get_optimizers(self, iteration: int) -> Sequence[torch.optim.Optimizer]: + def get_optimizers( + self, iteration: int + ) -> Sequence[torch.optim.Optimizer]: raise NotImplementedError @abstractmethod diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 56f042bc3..4e83b6670 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -1,28 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""DMD2 distillation method (algorithm layer). - -Config keys used (YAML schema-v2): -- `recipe.method`: must be `"dmd2"` for this method. -- `roles`: requires `student`, `teacher`, `critic` (trainable critic). -- `method_config`: - - `rollout_mode` (`simulate` or `data_latent`) - - `dmd_denoising_steps` (list[int]) - - `cfg_uncond` (optional): `on_missing` + channel policies - - optional overrides: `generator_update_interval`, `warp_denoising_step`, - `real_score_guidance_scale` -- `training` (selected fields used for optim/schedule): - - `learning_rate`, `betas`, `lr_scheduler` - - `fake_score_learning_rate`, `fake_score_betas`, `fake_score_lr_scheduler` - - `weight_decay`, `lr_warmup_steps`, `max_train_steps`, `lr_num_cycles`, - `lr_power`, `min_lr_ratio` - - `generator_update_interval`, `warp_denoising_step`, `real_score_guidance_scale`, - `max_grad_norm` -- `training.validation.*` (parsed by method; executed via validator): - - `enabled`, `every_steps`, `dataset_file`, `sampling_steps` - - optional: `sampling_timesteps`, `guidance_scale`, `sampler_kind`, `ode_solver`, - `rollout_mode`, `output_dir`, `num_frames` -""" +"""DMD2 distillation method (algorithm layer).""" from __future__ import annotations @@ -32,10 +10,9 @@ import torch.nn.functional as F from fastvideo.distillation.methods.base import DistillMethod, LogScalar -from fastvideo.distillation.dispatch import register_method -from fastvideo.distillation.roles import RoleManager +from fastvideo.distillation.models.base import ModelBase from fastvideo.distillation.utils.optimizer import ( - build_role_optimizer_and_scheduler, + build_optimizer_and_scheduler, clip_grad_norm_if_needed, ) from fastvideo.distillation.utils.validation import ( @@ -52,78 +29,73 @@ ) from fastvideo.distillation.validators.base import ValidationRequest from fastvideo.distillation.utils.config import ( - DistillRunConfig, get_optional_float, get_optional_int, parse_betas, ) if TYPE_CHECKING: - from fastvideo.distillation.models.base import ModelBase + pass -@register_method("dmd2") class DMD2Method(DistillMethod): """DMD2 distillation algorithm (method layer). - Owns the algorithmic orchestration (loss construction + update policy) and - stays independent of any specific model plugin. It requires a - :class:`~fastvideo.distillation.roles.RoleManager` containing at least the - roles ``student``, ``teacher``, and ``critic``. - - All model-plugin details (how to run student rollout, teacher CFG - prediction, critic loss, and how to safely run backward under activation - checkpointing/forward-context constraints) are delegated to the model plugin - passed in at construction time. + Owns role model instances directly: + - ``self.student`` — trainable student :class:`ModelBase` + - ``self.teacher`` — frozen teacher :class:`ModelBase` + - ``self.critic`` — trainable critic :class:`ModelBase` """ def __init__( self, *, - bundle: RoleManager, - model: ModelBase, - method_config: dict[str, Any] | None = None, - validation_config: dict[str, Any] | None = None, + cfg: Any, + role_models: dict[str, ModelBase], validator: Any | None = None, ) -> None: - super().__init__(bundle) - bundle.require_roles(["student", "teacher", "critic"]) - self.student = bundle.role("student") - self.teacher = bundle.role("teacher") - self.critic = bundle.role("critic") - if not self.student.trainable: - raise ValueError("DMD2Method requires roles.student.trainable=true") - if self.teacher.trainable: - raise ValueError("DMD2Method requires roles.teacher.trainable=false") - if not self.critic.trainable: - raise ValueError("DMD2Method requires roles.critic.trainable=true") - self.model = model + super().__init__(role_models=role_models) + + if "student" not in role_models: + raise ValueError("DMD2Method requires role 'student'") + if "teacher" not in role_models: + raise ValueError("DMD2Method requires role 'teacher'") + if "critic" not in role_models: + raise ValueError("DMD2Method requires role 'critic'") + + self.student = role_models["student"] + self.teacher = role_models["teacher"] + self.critic = role_models["critic"] + + if not getattr(self.student, "_trainable", True): + raise ValueError( + "DMD2Method requires student to be trainable" + ) + if getattr(self.teacher, "_trainable", True): + raise ValueError( + "DMD2Method requires teacher to be non-trainable" + ) + if not getattr(self.critic, "_trainable", True): + raise ValueError( + "DMD2Method requires critic to be trainable" + ) + self.validator = validator - self.training_args = model.training_args - self.method_config: dict[str, Any] = dict(method_config or {}) - self.validation_config: dict[str, Any] = dict(validation_config or {}) + self.training_args = cfg.training_args + self.method_config: dict[str, Any] = dict( + getattr(cfg, "method_config", {}) or {} + ) + self.validation_config: dict[str, Any] = dict( + getattr(cfg, "validation", {}) or {} + ) self._cfg_uncond = self._parse_cfg_uncond() self._rollout_mode = self._parse_rollout_mode() self._denoising_step_list: torch.Tensor | None = None - self._init_optimizers_and_schedulers() - # DistillMethod override: build - @classmethod - def build( - cls, - *, - cfg: DistillRunConfig, - bundle: RoleManager, - model: Any, - validator: Any | None, - ) -> DistillMethod: - return cls( - bundle=bundle, - model=model, - method_config=cfg.method_config, - validation_config=cfg.validation, - validator=validator, - ) + # Initialize preprocessors on student. + self.student.init_preprocessors(self.training_args) + + self._init_optimizers_and_schedulers() # DistillMethod override: single_train_step def single_train_step( @@ -132,12 +104,16 @@ def single_train_step( iteration: int, *, current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: + ) -> tuple[ + dict[str, torch.Tensor], + dict[str, Any], + dict[str, LogScalar], + ]: latents_source: Literal["data", "zeros"] = "data" if self._rollout_mode == "simulate": latents_source = "zeros" - training_batch = self.model.prepare_batch( + training_batch = self.student.prepare_batch( batch, current_vsa_sparsity=current_vsa_sparsity, latents_source=latents_source, @@ -152,13 +128,22 @@ def single_train_step( ) student_ctx = None if update_student: - generator_pred_x0 = self._student_rollout(training_batch, with_grad=True) - student_ctx = (training_batch.timesteps, training_batch.attn_metadata_vsa) - generator_loss = self._dmd_loss(generator_pred_x0, training_batch) + generator_pred_x0 = self._student_rollout( + training_batch, with_grad=True + ) + student_ctx = ( + training_batch.timesteps, + training_batch.attn_metadata_vsa, + ) + generator_loss = self._dmd_loss( + generator_pred_x0, training_batch + ) - fake_score_loss, critic_ctx, critic_outputs = self._critic_flow_matching_loss( - training_batch - ) + ( + fake_score_loss, + critic_ctx, + critic_outputs, + ) = self._critic_flow_matching_loss(training_batch) total_loss = generator_loss + fake_score_loss loss_map = { @@ -173,7 +158,9 @@ def single_train_step( "student_ctx": student_ctx, "critic_ctx": critic_ctx, } - metrics: dict[str, LogScalar] = {"update_student": float(update_student)} + metrics: dict[str, LogScalar] = { + "update_student": float(update_student) + } return loss_map, outputs, metrics # DistillMethod override: backward @@ -187,15 +174,23 @@ def backward( grad_accum_rounds = max(1, int(grad_accum_rounds)) backward_ctx = outputs.get("_fv_backward") if not isinstance(backward_ctx, dict): - super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + super().backward( + loss_map, + outputs, + grad_accum_rounds=grad_accum_rounds, + ) return - update_student = bool(backward_ctx.get("update_student", False)) + update_student = bool( + backward_ctx.get("update_student", False) + ) if update_student: student_ctx = backward_ctx.get("student_ctx") if student_ctx is None: - raise RuntimeError("Missing student backward context") - self.model.backward( + raise RuntimeError( + "Missing student backward context" + ) + self.student.backward( loss_map["generator_loss"], student_ctx, grad_accum_rounds=grad_accum_rounds, @@ -204,41 +199,44 @@ def backward( critic_ctx = backward_ctx.get("critic_ctx") if critic_ctx is None: raise RuntimeError("Missing critic backward context") - self.model.backward( + self.student.backward( loss_map["fake_score_loss"], critic_ctx, grad_accum_rounds=grad_accum_rounds, ) # DistillMethod override: get_optimizers - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + def get_optimizers( + self, iteration: int + ) -> list[torch.optim.Optimizer]: optimizers: list[torch.optim.Optimizer] = [] - optimizers.extend(self.critic.optimizers.values()) + optimizers.append(self._critic_optimizer) if self._should_update_student(iteration): - optimizers.extend(self.student.optimizers.values()) + optimizers.append(self._student_optimizer) return optimizers # DistillMethod override: get_lr_schedulers def get_lr_schedulers(self, iteration: int) -> list[Any]: schedulers: list[Any] = [] - schedulers.extend(self.bundle.role("critic").lr_schedulers.values()) + schedulers.append(self._critic_lr_scheduler) if self._should_update_student(iteration): - schedulers.extend(self.bundle.role("student").lr_schedulers.values()) + schedulers.append(self._student_lr_scheduler) return schedulers # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: if self._should_update_student(iteration): - for module in self.bundle.role("student").modules.values(): - clip_grad_norm_if_needed(module, self.training_args) - for module in self.bundle.role("critic").modules.values(): - clip_grad_norm_if_needed(module, self.training_args) - + clip_grad_norm_if_needed( + self.student.transformer, self.training_args + ) + clip_grad_norm_if_needed( + self.critic.transformer, self.training_args + ) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start def on_train_start(self) -> None: - self.model.on_train_start() + self.student.on_train_start() # DistillTrainer hook: log_validation def log_validation(self, iteration: int) -> None: @@ -248,42 +246,64 @@ def log_validation(self, iteration: int) -> None: if not is_validation_enabled(self.validation_config): return - every_steps = parse_validation_every_steps(self.validation_config) + every_steps = parse_validation_every_steps( + self.validation_config + ) if every_steps <= 0: return if iteration % every_steps != 0: return - dataset_file = parse_validation_dataset_file(self.validation_config) - sampling_steps = parse_validation_sampling_steps(self.validation_config) + dataset_file = parse_validation_dataset_file( + self.validation_config + ) + sampling_steps = parse_validation_sampling_steps( + self.validation_config + ) sampling_timesteps: list[int] | None = None - raw_timesteps = self.validation_config.get("sampling_timesteps", None) + raw_timesteps = self.validation_config.get( + "sampling_timesteps", None + ) if raw_timesteps is None: - raw_timesteps = self.method_config.get("dmd_denoising_steps", None) + raw_timesteps = self.method_config.get( + "dmd_denoising_steps", None + ) if isinstance(raw_timesteps, list) and raw_timesteps: sampling_timesteps = [int(s) for s in raw_timesteps] if not sampling_steps: - # Default to the few-step student rollout step count for DMD2. if sampling_timesteps is None: return sampling_steps = [int(len(sampling_timesteps))] - sampler_kind = parse_validation_sampler_kind(self.validation_config, default="sde") + sampler_kind = parse_validation_sampler_kind( + self.validation_config, default="sde" + ) ode_solver = parse_validation_ode_solver( self.validation_config, sampler_kind=sampler_kind ) - if sampling_timesteps is not None and sampler_kind != "sde": + if ( + sampling_timesteps is not None + and sampler_kind != "sde" + ): raise ValueError( - "method_config.validation.sampling_timesteps is only valid when " - "sampler_kind='sde'" + "method_config.validation.sampling_timesteps is " + "only valid when sampler_kind='sde'" ) - rollout_mode = parse_validation_rollout_mode(self.validation_config) - guidance_scale = parse_validation_guidance_scale(self.validation_config) - output_dir = parse_validation_output_dir(self.validation_config) - num_actions = parse_validation_num_frames(self.validation_config) + rollout_mode = parse_validation_rollout_mode( + self.validation_config + ) + guidance_scale = parse_validation_guidance_scale( + self.validation_config + ) + output_dir = parse_validation_output_dir( + self.validation_config + ) + num_actions = parse_validation_num_frames( + self.validation_config + ) request = ValidationRequest( sample_handle=self.student, @@ -301,30 +321,32 @@ def log_validation(self, iteration: int) -> None: # Checkpoint hook: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - generators: dict[str, torch.Generator] = {} - model = getattr(self, "model", None) - get_model_generators = getattr(model, "get_rng_generators", None) - if callable(get_model_generators): - generators.update(get_model_generators()) + student_gens = self.student.get_rng_generators() + generators.update(student_gens) validator = getattr(self, "validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) + validation_gen = getattr( + validator, "validation_random_generator", None + ) if isinstance(validation_gen, torch.Generator): generators["validation_cpu"] = validation_gen return generators - def _parse_rollout_mode(self) -> Literal["simulate", "data_latent"]: + def _parse_rollout_mode( + self, + ) -> Literal["simulate", "data_latent"]: raw = self.method_config.get("rollout_mode", None) if raw is None: - raise ValueError("method_config.rollout_mode must be set for DMD2") + raise ValueError( + "method_config.rollout_mode must be set for DMD2" + ) if not isinstance(raw, str): raise ValueError( - "method_config.rollout_mode must be a string, got " - f"{type(raw).__name__}" + "method_config.rollout_mode must be a string, " + f"got {type(raw).__name__}" ) mode = raw.strip().lower() if mode in ("simulate", "sim"): @@ -343,8 +365,8 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: return None if not isinstance(raw, dict): raise ValueError( - "method_config.cfg_uncond must be a dict when set, got " - f"{type(raw).__name__}" + "method_config.cfg_uncond must be a dict when " + f"set, got {type(raw).__name__}" ) cfg: dict[str, Any] = dict(raw) @@ -354,14 +376,14 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: on_missing_raw = "error" if not isinstance(on_missing_raw, str): raise ValueError( - "method_config.cfg_uncond.on_missing must be a string, got " - f"{type(on_missing_raw).__name__}" + "method_config.cfg_uncond.on_missing must be a " + f"string, got {type(on_missing_raw).__name__}" ) on_missing = on_missing_raw.strip().lower() if on_missing not in {"error", "ignore"}: raise ValueError( - "method_config.cfg_uncond.on_missing must be one of " - "{error, ignore}, got " + "method_config.cfg_uncond.on_missing must be one " + "of {error, ignore}, got " f"{on_missing_raw!r}" ) cfg["on_missing"] = on_missing @@ -373,7 +395,8 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: continue if not isinstance(policy_raw, str): raise ValueError( - "method_config.cfg_uncond values must be strings, got " + "method_config.cfg_uncond values must be " + "strings, got " f"{channel}={type(policy_raw).__name__}" ) policy = policy_raw.strip().lower() @@ -382,8 +405,8 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: allowed = {*allowed, "negative_prompt"} if policy not in allowed: raise ValueError( - "method_config.cfg_uncond values must be one of " - f"{sorted(allowed)}, got " + "method_config.cfg_uncond values must be one " + f"of {sorted(allowed)}, got " f"{channel}={policy_raw!r}" ) cfg[channel] = policy @@ -393,36 +416,82 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: def _init_optimizers_and_schedulers(self) -> None: training_args = self.training_args - # Student optimizer/scheduler (default training hyperparams). - student_lr = float(getattr(training_args, "learning_rate", 0.0) or 0.0) + # Student optimizer/scheduler. + student_lr = float( + getattr(training_args, "learning_rate", 0.0) or 0.0 + ) student_betas = parse_betas( getattr(training_args, "betas", None), where="training.betas", ) - student_sched = str(getattr(training_args, "lr_scheduler", "constant")) - build_role_optimizer_and_scheduler( - role="student", - handle=self.student, + student_sched = str( + getattr(training_args, "lr_scheduler", "constant") + ) + student_params = [ + p + for p in self.student.transformer.parameters() + if p.requires_grad + ] + if getattr(self.student, "transformer_2", None) is not None: + student_params.extend( + p + for p in self.student.transformer_2.parameters() + if p.requires_grad + ) + ( + self._student_optimizer, + self._student_lr_scheduler, + ) = build_optimizer_and_scheduler( + params=student_params, training_args=self.training_args, learning_rate=student_lr, betas=student_betas, scheduler_name=student_sched, ) - # Critic optimizer/scheduler (DMD2-specific overrides). - critic_lr = float(getattr(training_args, "fake_score_learning_rate", 0.0) or 0.0) + # Critic optimizer/scheduler. + critic_lr = float( + getattr( + training_args, "fake_score_learning_rate", 0.0 + ) + or 0.0 + ) if critic_lr == 0.0: critic_lr = student_lr - critic_betas_raw = getattr(training_args, "fake_score_betas", None) + critic_betas_raw = getattr( + training_args, "fake_score_betas", None + ) if critic_betas_raw is None: - critic_betas_raw = getattr(training_args, "betas", None) - critic_betas = parse_betas(critic_betas_raw, where="training.fake_score_betas") + critic_betas_raw = getattr( + training_args, "betas", None + ) + critic_betas = parse_betas( + critic_betas_raw, where="training.fake_score_betas" + ) - critic_sched = str(getattr(training_args, "fake_score_lr_scheduler", None) or student_sched) - build_role_optimizer_and_scheduler( - role="critic", - handle=self.critic, + critic_sched = str( + getattr( + training_args, "fake_score_lr_scheduler", None + ) + or student_sched + ) + critic_params = [ + p + for p in self.critic.transformer.parameters() + if p.requires_grad + ] + if getattr(self.critic, "transformer_2", None) is not None: + critic_params.extend( + p + for p in self.critic.transformer_2.parameters() + if p.requires_grad + ) + ( + self._critic_optimizer, + self._critic_lr_scheduler, + ) = build_optimizer_and_scheduler( + params=critic_params, training_args=self.training_args, learning_rate=critic_lr, betas=critic_betas, @@ -436,32 +505,51 @@ def _should_update_student(self, iteration: int) -> bool: where="method_config.generator_update_interval", ) if interval is None: - interval = int(getattr(self.training_args, "generator_update_interval", 1) or 1) + interval = int( + getattr( + self.training_args, + "generator_update_interval", + 1, + ) + or 1 + ) if interval <= 0: return True return iteration % interval == 0 - def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: - if self._denoising_step_list is not None and self._denoising_step_list.device == device: + def _get_denoising_step_list( + self, device: torch.device + ) -> torch.Tensor: + if ( + self._denoising_step_list is not None + and self._denoising_step_list.device == device + ): return self._denoising_step_list raw = self.method_config.get("dmd_denoising_steps", None) if not isinstance(raw, list) or not raw: - raise ValueError("method_config.dmd_denoising_steps must be set for DMD2 distillation") + raise ValueError( + "method_config.dmd_denoising_steps must be set " + "for DMD2 distillation" + ) - steps = torch.tensor([int(s) for s in raw], dtype=torch.long, device=device) + steps = torch.tensor( + [int(s) for s in raw], + dtype=torch.long, + device=device, + ) warp = self.method_config.get("warp_denoising_step", None) if warp is None: - warp = getattr(self.training_args, "warp_denoising_step", False) + warp = getattr( + self.training_args, "warp_denoising_step", False + ) if bool(warp): - noise_scheduler = getattr(self.model, "noise_scheduler", None) - if noise_scheduler is None: - raise ValueError("warp_denoising_step requires model.noise_scheduler.timesteps") - timesteps = torch.cat( ( - noise_scheduler.timesteps.to("cpu"), + self.student.noise_scheduler.timesteps.to( + "cpu" + ), torch.tensor([0], dtype=torch.float32), ) ).to(device) @@ -470,7 +558,9 @@ def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: self._denoising_step_list = steps return steps - def _sample_rollout_timestep(self, device: torch.device) -> torch.Tensor: + def _sample_rollout_timestep( + self, device: torch.device + ) -> torch.Tensor: step_list = self._get_denoising_step_list(device) index = torch.randint( 0, @@ -481,7 +571,9 @@ def _sample_rollout_timestep(self, device: torch.device) -> torch.Tensor: ) return step_list[index] - def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: + def _student_rollout( + self, batch: Any, *, with_grad: bool + ) -> torch.Tensor: latents = batch.latents device = latents.device dtype = latents.dtype @@ -489,10 +581,13 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: if self._rollout_mode != "simulate": timestep = self._sample_rollout_timestep(device) - noise = torch.randn(latents.shape, device=device, dtype=dtype) - noisy_latents = self.model.add_noise(latents, noise, timestep) - pred_x0 = self.model.predict_x0( - self.student, + noise = torch.randn( + latents.shape, device=device, dtype=dtype + ) + noisy_latents = self.student.add_noise( + latents, noise, timestep + ) + pred_x0 = self.student.predict_x0( noisy_latents, timestep, batch, @@ -500,7 +595,9 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: cfg_uncond=self._cfg_uncond, attn_kind="vsa", ) - batch.dmd_latent_vis_dict["generator_timestep"] = timestep + batch.dmd_latent_vis_dict[ + "generator_timestep" + ] = timestep return pred_x0 target_timestep_idx = torch.randint( @@ -510,10 +607,14 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: device=device, dtype=torch.long, ) - target_timestep_idx_int = int(target_timestep_idx.item()) + target_timestep_idx_int = int( + target_timestep_idx.item() + ) target_timestep = step_list[target_timestep_idx] - current_noise_latents = torch.randn(latents.shape, device=device, dtype=dtype) + current_noise_latents = torch.randn( + latents.shape, device=device, dtype=dtype + ) current_noise_latents_copy = current_noise_latents.clone() max_target_idx = len(step_list) - 1 @@ -524,12 +625,14 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: with torch.no_grad(): for step_idx in range(max_target_idx): current_timestep = step_list[step_idx] - current_timestep_tensor = current_timestep * torch.ones( - 1, device=device, dtype=torch.long + current_timestep_tensor = ( + current_timestep + * torch.ones( + 1, device=device, dtype=torch.long + ) ) - pred_clean = self.model.predict_x0( - self.student, + pred_clean = self.student.predict_x0( current_noise_latents, current_timestep_tensor, batch, @@ -539,27 +642,39 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: ) next_timestep = step_list[step_idx + 1] - next_timestep_tensor = next_timestep * torch.ones( - 1, device=device, dtype=torch.long + next_timestep_tensor = ( + next_timestep + * torch.ones( + 1, device=device, dtype=torch.long + ) + ) + noise = torch.randn( + latents.shape, + device=device, + dtype=pred_clean.dtype, ) - noise = torch.randn(latents.shape, device=device, dtype=pred_clean.dtype) - current_noise_latents = self.model.add_noise( - pred_clean, - noise, - next_timestep_tensor, + current_noise_latents = ( + self.student.add_noise( + pred_clean, + noise, + next_timestep_tensor, + ) + ) + noise_latents.append( + current_noise_latents.clone() ) - noise_latents.append(current_noise_latents.clone()) if noise_latent_index >= 0: if noise_latent_index >= len(noise_latents): - raise RuntimeError("noise_latent_index is out of bounds") + raise RuntimeError( + "noise_latent_index is out of bounds" + ) noisy_input = noise_latents[noise_latent_index] else: noisy_input = current_noise_latents_copy if with_grad: - pred_x0 = self.model.predict_x0( - self.student, + pred_x0 = self.student.predict_x0( noisy_input, target_timestep, batch, @@ -569,8 +684,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: ) else: with torch.no_grad(): - pred_x0 = self.model.predict_x0( - self.student, + pred_x0 = self.student.predict_x0( noisy_input, target_timestep, batch, @@ -579,32 +693,43 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: attn_kind="vsa", ) - batch.dmd_latent_vis_dict["generator_timestep"] = target_timestep.float().detach() + batch.dmd_latent_vis_dict[ + "generator_timestep" + ] = target_timestep.float().detach() return pred_x0 - def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: + def _critic_flow_matching_loss( + self, batch: Any + ) -> tuple[torch.Tensor, Any, dict[str, Any]]: with torch.no_grad(): - generator_pred_x0 = self._student_rollout(batch, with_grad=False) + generator_pred_x0 = self._student_rollout( + batch, with_grad=False + ) device = generator_pred_x0.device fake_score_timestep = torch.randint( 0, - int(self.model.num_train_timesteps), + int(self.student.num_train_timesteps), [1], device=device, dtype=torch.long, ) - fake_score_timestep = self.model.shift_and_clamp_timestep(fake_score_timestep) + fake_score_timestep = ( + self.student.shift_and_clamp_timestep( + fake_score_timestep + ) + ) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_x0 = self.model.add_noise(generator_pred_x0, noise, fake_score_timestep) + noisy_x0 = self.student.add_noise( + generator_pred_x0, noise, fake_score_timestep + ) - pred_noise = self.model.predict_noise( - self.critic, + pred_noise = self.critic.predict_noise( noisy_x0, fake_score_timestep, batch, @@ -613,44 +738,67 @@ def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dic attn_kind="dense", ) target = noise - generator_pred_x0 - flow_matching_loss = torch.mean((pred_noise - target) ** 2) + flow_matching_loss = torch.mean( + (pred_noise - target) ** 2 + ) batch.fake_score_latent_vis_dict = { "generator_pred_video": generator_pred_x0, "fake_score_timestep": fake_score_timestep, } - outputs = {"fake_score_latent_vis_dict": batch.fake_score_latent_vis_dict} - return flow_matching_loss, (batch.timesteps, batch.attn_metadata), outputs + outputs = { + "fake_score_latent_vis_dict": ( + batch.fake_score_latent_vis_dict + ) + } + return ( + flow_matching_loss, + (batch.timesteps, batch.attn_metadata), + outputs, + ) - def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor: + def _dmd_loss( + self, + generator_pred_x0: torch.Tensor, + batch: Any, + ) -> torch.Tensor: guidance_scale = get_optional_float( self.method_config, "real_score_guidance_scale", where="method_config.real_score_guidance_scale", ) if guidance_scale is None: - guidance_scale = float(getattr(self.training_args, "real_score_guidance_scale", 1.0)) + guidance_scale = float( + getattr( + self.training_args, + "real_score_guidance_scale", + 1.0, + ) + ) device = generator_pred_x0.device with torch.no_grad(): timestep = torch.randint( 0, - int(self.model.num_train_timesteps), + int(self.student.num_train_timesteps), [1], device=device, dtype=torch.long, ) - timestep = self.model.shift_and_clamp_timestep(timestep) + timestep = self.student.shift_and_clamp_timestep( + timestep + ) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_latents = self.model.add_noise(generator_pred_x0, noise, timestep) + noisy_latents = self.student.add_noise( + generator_pred_x0, noise, timestep + ) - faker_x0 = self.model.predict_x0( - self.critic, + faker_x0 = self.critic.predict_x0( noisy_latents, timestep, batch, @@ -658,8 +806,7 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_cond_x0 = self.model.predict_x0( - self.teacher, + real_cond_x0 = self.teacher.predict_x0( noisy_latents, timestep, batch, @@ -667,8 +814,7 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_uncond_x0 = self.model.predict_x0( - self.teacher, + real_uncond_x0 = self.teacher.predict_x0( noisy_latents, timestep, batch, @@ -676,14 +822,20 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale + real_cfg_x0 = real_cond_x0 + ( + real_cond_x0 - real_uncond_x0 + ) * guidance_scale - denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() + denom = torch.abs( + generator_pred_x0 - real_cfg_x0 + ).mean() grad = (faker_x0 - real_cfg_x0) / denom grad = torch.nan_to_num(grad) loss = 0.5 * F.mse_loss( generator_pred_x0.float(), - (generator_pred_x0.float() - grad.float()).detach(), + ( + generator_pred_x0.float() - grad.float() + ).detach(), ) return loss diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/distillation/methods/distribution_matching/self_forcing.py index 9ae64e529..5e9156b00 100644 --- a/fastvideo/distillation/methods/distribution_matching/self_forcing.py +++ b/fastvideo/distillation/methods/distribution_matching/self_forcing.py @@ -1,41 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Self-Forcing distillation method (algorithm layer). - -Self-Forcing is a distribution-matching variant that changes *how the student -sample (gen_data) is obtained*: instead of a single-step rollout, we perform a -multi-step simulate rollout and only keep gradients at a stochastic "exit" -denoising step per temporal block (chunk). - -This implementation follows the structure of: -- FastGen: `fastgen/methods/distribution_matching/self_forcing.py` -- FastVideo legacy: `fastvideo/training/self_forcing_distillation_pipeline.py` - -Config keys used (YAML schema-v2): -- `recipe.method`: must be `"self_forcing"` for this method. -- `roles`: requires `student`, `teacher`, `critic` (trainable critic). -- `method_config`: - - `rollout_mode`: must be `"simulate"` for self-forcing. - - `dmd_denoising_steps` (list[int]): the candidate denoising steps. - - `cfg_uncond` (optional): same as DMD2. - - self-forcing rollout knobs: - - `chunk_size` (int, optional; default=3) - - `student_sample_type` (`"sde"` or `"ode"`, optional; default="sde") - - `same_step_across_blocks` (bool, optional; default=false) - - `last_step_only` (bool, optional; default=false) - - `context_noise` (float, optional; default=0.0) - - `enable_gradient_in_rollout` (bool, optional; default=true) - - `start_gradient_frame` (int, optional; default=0) -- `training` (selected fields used for optim/schedule): same as DMD2. -- `training.validation.*`: parsed by base DMD2 method; executed via validator. - -Notes / limitations: -- This method uses `SelfForcingFlowMatchScheduler` for simulate-rollout - (matching legacy behavior). Teacher/critic losses also interpret noise under - the same scheduler for consistency. -- This method requires a causal model plugin implementing `CausalModelBase` and - a causal student role. Cache-free rollout is intentionally unsupported. -""" +"""Self-Forcing distillation method (algorithm layer).""" from __future__ import annotations @@ -44,10 +9,17 @@ import torch import torch.distributed as dist -from fastvideo.distillation.dispatch import register_method -from fastvideo.distillation.models.base import CausalModelBase -from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method -from fastvideo.distillation.utils.config import get_optional_float, get_optional_int +from fastvideo.distillation.models.base import ( + CausalModelBase, + ModelBase, +) +from fastvideo.distillation.methods.distribution_matching.dmd2 import ( + DMD2Method, +) +from fastvideo.distillation.utils.config import ( + get_optional_float, + get_optional_int, +) from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( SelfForcingFlowMatchScheduler, ) @@ -55,54 +27,60 @@ if TYPE_CHECKING: from fastvideo.pipelines import TrainingBatch - from fastvideo.distillation.roles import RoleManager - from fastvideo.distillation.utils.config import DistillRunConfig def _require_bool(raw: Any, *, where: str) -> bool: if isinstance(raw, bool): return raw - raise ValueError(f"Expected bool at {where}, got {type(raw).__name__}") + raise ValueError( + f"Expected bool at {where}, got {type(raw).__name__}" + ) def _require_str(raw: Any, *, where: str) -> str: if not isinstance(raw, str) or not raw.strip(): - raise ValueError(f"Expected non-empty string at {where}") + raise ValueError( + f"Expected non-empty string at {where}" + ) return raw -@register_method("self_forcing") class SelfForcingMethod(DMD2Method): """Self-Forcing DMD2 (distribution matching) method. - Inherits DMD2 losses/update policy, but replaces the student rollout - procedure with a self-forcing rollout. + Requires a causal student implementing ``CausalModelBase``. """ def __init__( self, *, - bundle: Any, - model: Any, - method_config: dict[str, Any] | None = None, - validation_config: dict[str, Any] | None = None, + cfg: Any, + role_models: dict[str, ModelBase], validator: Any | None = None, ) -> None: super().__init__( - bundle=bundle, - model=model, - method_config=method_config, - validation_config=validation_config, + cfg=cfg, + role_models=role_models, validator=validator, ) + # Validate causal student. + if not isinstance(self.student, CausalModelBase): + raise ValueError( + "SelfForcingMethod requires a causal student " + "implementing CausalModelBase." + ) + if self._rollout_mode != "simulate": - raise ValueError("SelfForcingMethod only supports method_config.rollout_mode='simulate'") + raise ValueError( + "SelfForcingMethod only supports " + "method_config.rollout_mode='simulate'" + ) - cfg = self.method_config + mcfg = self.method_config chunk_size = get_optional_int( - cfg, + mcfg, "chunk_size", where="method_config.chunk_size", ) @@ -110,37 +88,44 @@ def __init__( chunk_size = 3 if chunk_size <= 0: raise ValueError( - "method_config.chunk_size must be a positive integer, got " - f"{chunk_size}" + "method_config.chunk_size must be a positive " + f"integer, got {chunk_size}" ) self._chunk_size = int(chunk_size) - sample_type_raw = cfg.get("student_sample_type", "sde") - sample_type = _require_str(sample_type_raw, where="method_config.student_sample_type") + sample_type_raw = mcfg.get("student_sample_type", "sde") + sample_type = _require_str( + sample_type_raw, + where="method_config.student_sample_type", + ) sample_type = sample_type.strip().lower() if sample_type not in {"sde", "ode"}: raise ValueError( - "method_config.student_sample_type must be one of {sde, ode}, got " - f"{sample_type_raw!r}" + "method_config.student_sample_type must be one " + f"of {{sde, ode}}, got {sample_type_raw!r}" ) - self._student_sample_type: Literal["sde", "ode"] = sample_type # type: ignore[assignment] + self._student_sample_type: Literal["sde", "ode"] = ( + sample_type # type: ignore[assignment] + ) - same_step_raw = cfg.get("same_step_across_blocks", False) + same_step_raw = mcfg.get("same_step_across_blocks", False) if same_step_raw is None: same_step_raw = False self._same_step_across_blocks = _require_bool( - same_step_raw, where="method_config.same_step_across_blocks" + same_step_raw, + where="method_config.same_step_across_blocks", ) - last_step_raw = cfg.get("last_step_only", False) + last_step_raw = mcfg.get("last_step_only", False) if last_step_raw is None: last_step_raw = False self._last_step_only = _require_bool( - last_step_raw, where="method_config.last_step_only" + last_step_raw, + where="method_config.last_step_only", ) context_noise = get_optional_float( - cfg, + mcfg, "context_noise", where="method_config.context_noise", ) @@ -148,20 +133,23 @@ def __init__( context_noise = 0.0 if context_noise < 0.0: raise ValueError( - "method_config.context_noise must be >= 0, got " - f"{context_noise}" + "method_config.context_noise must be >= 0, " + f"got {context_noise}" ) self._context_noise = float(context_noise) - enable_grad_raw = cfg.get("enable_gradient_in_rollout", True) + enable_grad_raw = mcfg.get( + "enable_gradient_in_rollout", True + ) if enable_grad_raw is None: enable_grad_raw = True self._enable_gradient_in_rollout = _require_bool( - enable_grad_raw, where="method_config.enable_gradient_in_rollout" + enable_grad_raw, + where="method_config.enable_gradient_in_rollout", ) start_grad_frame = get_optional_int( - cfg, + mcfg, "start_gradient_frame", where="method_config.start_gradient_frame", ) @@ -169,65 +157,35 @@ def __init__( start_grad_frame = 0 if start_grad_frame < 0: raise ValueError( - "method_config.start_gradient_frame must be >= 0, got " - f"{start_grad_frame}" + "method_config.start_gradient_frame must be " + f">= 0, got {start_grad_frame}" ) self._start_gradient_frame = int(start_grad_frame) - # Legacy Self-Forcing uses a dedicated scheduler (different sigma_min). - shift = float(getattr(self.training_args.pipeline_config, "flow_shift", 0.0) or 0.0) + shift = float( + getattr( + self.training_args.pipeline_config, + "flow_shift", + 0.0, + ) + or 0.0 + ) self._sf_scheduler = SelfForcingFlowMatchScheduler( num_inference_steps=1000, - num_train_timesteps=int(self.model.num_train_timesteps), + num_train_timesteps=int( + self.student.num_train_timesteps + ), shift=shift, sigma_min=0.0, extra_one_step=True, training=True, ) - # Cache for warped denoising step list (device specific). self._sf_denoising_step_list: torch.Tensor | None = None - # DistillMethod override: build - @classmethod - def build( - cls, - *, - cfg: DistillRunConfig, - bundle: RoleManager, - model: Any, - validator: Any | None, - ) -> DMD2Method: - student_spec = cfg.roles.get("student") - if student_spec is None: - raise ValueError("SelfForcingMethod requires roles.student") - recipe_family = str(getattr(cfg.recipe, "family", "")).strip().lower() - if not recipe_family.endswith("_causal"): - raise ValueError( - "SelfForcingMethod requires a causal student. " - f"Got recipe.family={recipe_family!r}." - ) - student_family = str(getattr(student_spec, "family", "")).strip().lower() - if not student_family.endswith("_causal"): - raise ValueError( - "SelfForcingMethod requires roles.student.family to be a causal " - f"family (suffix '_causal'). Got roles.student.family={student_family!r}." - ) - if not isinstance(model, CausalModelBase): - raise ValueError( - "SelfForcingMethod requires a causal model plugin implementing CausalModelBase. " - "Make sure you are using a causal-capable family/model plugin." - ) - - return cls( - bundle=bundle, - model=model, - method_config=cfg.method_config, - validation_config=cfg.validation, - validator=validator, - ) - - def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: + def _get_denoising_step_list( + self, device: torch.device + ) -> torch.Tensor: if ( self._sf_denoising_step_list is not None and self._sf_denoising_step_list.device == device @@ -236,12 +194,21 @@ def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: raw = self.method_config.get("dmd_denoising_steps", None) if not isinstance(raw, list) or not raw: - raise ValueError("method_config.dmd_denoising_steps must be set for self_forcing") - steps = torch.tensor([int(s) for s in raw], dtype=torch.long, device=device) + raise ValueError( + "method_config.dmd_denoising_steps must be set " + "for self_forcing" + ) + steps = torch.tensor( + [int(s) for s in raw], + dtype=torch.long, + device=device, + ) warp = self.method_config.get("warp_denoising_step", None) if warp is None: - warp = getattr(self.training_args, "warp_denoising_step", False) + warp = getattr( + self.training_args, "warp_denoising_step", False + ) if bool(warp): timesteps = torch.cat( ( @@ -249,14 +216,16 @@ def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: torch.tensor([0], dtype=torch.float32), ) ).to(device) - steps = timesteps[int(self.model.num_train_timesteps) - steps] + steps = timesteps[ + int(self.student.num_train_timesteps) - steps + ] self._sf_denoising_step_list = steps return steps def _predict_x0_with_scheduler( self, - handle: Any, + model: ModelBase, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, @@ -264,8 +233,7 @@ def _predict_x0_with_scheduler( conditional: bool, attn_kind: Literal["dense", "vsa"], ) -> torch.Tensor: - pred_noise = self.model.predict_noise( - handle, + pred_noise = model.predict_noise( noisy_latents, timestep, batch, @@ -295,17 +263,31 @@ def _sf_add_noise( ).unflatten(0, (b, t)) return noisy - def _timestep_to_sigma(self, timestep: torch.Tensor) -> torch.Tensor: - sigmas = self._sf_scheduler.sigmas.to(device=timestep.device, dtype=torch.float32) - timesteps = self._sf_scheduler.timesteps.to(device=timestep.device, dtype=torch.float32) - t = timestep.to(device=timestep.device, dtype=torch.float32) + def _timestep_to_sigma( + self, timestep: torch.Tensor + ) -> torch.Tensor: + sigmas = self._sf_scheduler.sigmas.to( + device=timestep.device, dtype=torch.float32 + ) + timesteps = self._sf_scheduler.timesteps.to( + device=timestep.device, dtype=torch.float32 + ) + t = timestep.to( + device=timestep.device, dtype=torch.float32 + ) if t.ndim == 2: t = t.flatten(0, 1) elif t.ndim == 1 and t.numel() == 1: t = t.expand(1) elif t.ndim != 1: - raise ValueError(f"Invalid timestep shape: {tuple(timestep.shape)}") - idx = torch.argmin((timesteps.unsqueeze(0) - t.unsqueeze(1)).abs(), dim=1) + raise ValueError( + "Invalid timestep shape: " + f"{tuple(timestep.shape)}" + ) + idx = torch.argmin( + (timesteps.unsqueeze(0) - t.unsqueeze(1)).abs(), + dim=1, + ) return sigmas[idx] def _sample_exit_indices( @@ -320,7 +302,11 @@ def _sample_exit_indices( if num_steps <= 0: raise ValueError("num_steps must be positive") - shape = (1,) if self._same_step_across_blocks else (num_blocks,) + shape = ( + (1,) + if self._same_step_across_blocks + else (num_blocks,) + ) if not dist.is_initialized() or dist.get_rank() == 0: if self._last_step_only: @@ -338,30 +324,45 @@ def _sample_exit_indices( device=device, ) else: - indices = torch.empty(shape, dtype=torch.long, device=device) + indices = torch.empty( + shape, dtype=torch.long, device=device + ) if dist.is_initialized(): dist.broadcast(indices, src=0) if self._same_step_across_blocks: - return [int(indices.item()) for _ in range(num_blocks)] + return [ + int(indices.item()) for _ in range(num_blocks) + ] return [int(i) for i in indices.tolist()] - def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: - if not isinstance(self.model, CausalModelBase): + def _student_rollout( + self, batch: Any, *, with_grad: bool + ) -> torch.Tensor: + if not isinstance(self.student, CausalModelBase): raise ValueError( - "SelfForcingMethod requires a causal model plugin implementing CausalModelBase." + "SelfForcingMethod requires a causal student " + "implementing CausalModelBase." ) - return self._student_rollout_streaming(batch, with_grad=with_grad) + return self._student_rollout_streaming( + batch, with_grad=with_grad + ) - def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Tensor: + def _student_rollout_streaming( + self, batch: Any, *, with_grad: bool + ) -> torch.Tensor: + assert isinstance(self.student, CausalModelBase) latents = batch.latents if latents is None: - raise RuntimeError("TrainingBatch.latents is required for self-forcing rollout") + raise RuntimeError( + "TrainingBatch.latents is required for " + "self-forcing rollout" + ) if latents.ndim != 5: raise ValueError( - "TrainingBatch.latents must be [B, T, C, H, W], got " - f"shape={tuple(latents.shape)}" + "TrainingBatch.latents must be [B, T, C, H, W]" + f", got shape={tuple(latents.shape)}" ) device = latents.device @@ -372,7 +373,9 @@ def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Te denoising_steps = self._get_denoising_step_list(device) num_steps = int(denoising_steps.numel()) - noise_full = torch.randn_like(latents, device=device, dtype=dtype) + noise_full = torch.randn_like( + latents, device=device, dtype=dtype + ) chunk = int(self._chunk_size) if chunk <= 0: @@ -393,7 +396,7 @@ def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Te denoised_blocks: list[torch.Tensor] = [] cache_tag = "pos" - self.model.clear_caches(self.student, cache_tag=cache_tag) + self.student.clear_caches(cache_tag=cache_tag) for block_idx in range(num_blocks): if block_idx == 0: @@ -410,13 +413,18 @@ def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Te noisy_block = noise_full[:, start:end] exit_idx = int(exit_indices[block_idx]) - for step_idx, current_timestep in enumerate(denoising_steps): + for step_idx, current_timestep in enumerate( + denoising_steps + ): exit_flag = step_idx == exit_idx - timestep_block = current_timestep * torch.ones( - (batch_size, end - start), - device=device, - dtype=torch.float32, + timestep_block = ( + current_timestep + * torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + ) ) enable_grad = ( @@ -428,26 +436,30 @@ def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Te if not exit_flag: with torch.no_grad(): - pred_noise = self.model.predict_noise_streaming( - self.student, - noisy_block, - timestep_block, - batch, - conditional=True, - cache_tag=cache_tag, - store_kv=False, - cur_start_frame=start, - cfg_uncond=self._cfg_uncond, - attn_kind="vsa", + pred_noise = ( + self.student.predict_noise_streaming( + noisy_block, + timestep_block, + batch, + conditional=True, + cache_tag=cache_tag, + store_kv=False, + cur_start_frame=start, + cfg_uncond=self._cfg_uncond, + attn_kind="vsa", + ) ) if pred_noise is None: raise RuntimeError( - "predict_noise_streaming returned None " + "predict_noise_streaming " + "returned None " "(store_kv=False)" ) pred_x0_chunk = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_block.flatten(0, 1), + noise_input_latent=( + noisy_block.flatten(0, 1) + ), timestep=timestep_block, scheduler=self._sf_scheduler, ).unflatten(0, pred_noise.shape[:2]) @@ -467,7 +479,9 @@ def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Te ), ) else: - sigma_cur = self._timestep_to_sigma(timestep_block).view( + sigma_cur = self._timestep_to_sigma( + timestep_block + ).view( batch_size, end - start, 1, 1, 1 ) sigma_next = self._timestep_to_sigma( @@ -477,34 +491,43 @@ def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Te device=device, dtype=torch.float32, ) - ).view(batch_size, end - start, 1, 1, 1) - eps = (noisy_block - (1 - sigma_cur) * pred_x0_chunk) / sigma_cur.clamp_min( - 1e-8 + ).view( + batch_size, end - start, 1, 1, 1 + ) + eps = ( + noisy_block + - (1 - sigma_cur) * pred_x0_chunk + ) / sigma_cur.clamp_min(1e-8) + noisy_block = ( + (1 - sigma_next) * pred_x0_chunk + + sigma_next * eps ) - noisy_block = (1 - sigma_next) * pred_x0_chunk + sigma_next * eps continue with torch.set_grad_enabled(enable_grad): - pred_noise = self.model.predict_noise_streaming( - self.student, - noisy_block, - timestep_block, - batch, - conditional=True, - cache_tag=cache_tag, - store_kv=False, - cur_start_frame=start, - cfg_uncond=self._cfg_uncond, - attn_kind="vsa", + pred_noise = ( + self.student.predict_noise_streaming( + noisy_block, + timestep_block, + batch, + conditional=True, + cache_tag=cache_tag, + store_kv=False, + cur_start_frame=start, + cfg_uncond=self._cfg_uncond, + attn_kind="vsa", + ) ) if pred_noise is None: raise RuntimeError( - "predict_noise_streaming returned None " - "(store_kv=False)" + "predict_noise_streaming returned " + "None (store_kv=False)" ) pred_x0_chunk = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_block.flatten(0, 1), + noise_input_latent=( + noisy_block.flatten(0, 1) + ), timestep=timestep_block, scheduler=self._sf_scheduler, ).unflatten(0, pred_noise.shape[:2]) @@ -532,8 +555,7 @@ def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Te ) context_latents = pred_x0_chunk.detach() - _ = self.model.predict_noise_streaming( - self.student, + _ = self.student.predict_noise_streaming( context_latents, context_timestep, batch, @@ -546,36 +568,45 @@ def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Te ) if not denoised_blocks: - raise RuntimeError("Self-forcing rollout produced no blocks") + raise RuntimeError( + "Self-forcing rollout produced no blocks" + ) - self.model.clear_caches(self.student, cache_tag=cache_tag) + self.student.clear_caches(cache_tag=cache_tag) return torch.cat(denoised_blocks, dim=1) def _critic_flow_matching_loss( self, batch: Any ) -> tuple[torch.Tensor, Any, dict[str, Any]]: with torch.no_grad(): - generator_pred_x0 = self._student_rollout(batch, with_grad=False) + generator_pred_x0 = self._student_rollout( + batch, with_grad=False + ) device = generator_pred_x0.device fake_score_timestep = torch.randint( 0, - int(self.model.num_train_timesteps), + int(self.student.num_train_timesteps), [1], device=device, dtype=torch.long, ) - fake_score_timestep = self.model.shift_and_clamp_timestep(fake_score_timestep) + fake_score_timestep = ( + self.student.shift_and_clamp_timestep( + fake_score_timestep + ) + ) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_x0 = self._sf_add_noise(generator_pred_x0, noise, fake_score_timestep) + noisy_x0 = self._sf_add_noise( + generator_pred_x0, noise, fake_score_timestep + ) - pred_noise = self.model.predict_noise( - self.critic, + pred_noise = self.critic.predict_noise( noisy_x0, fake_score_timestep, batch, @@ -584,41 +615,65 @@ def _critic_flow_matching_loss( attn_kind="dense", ) target = noise - generator_pred_x0 - flow_matching_loss = torch.mean((pred_noise - target) ** 2) + flow_matching_loss = torch.mean( + (pred_noise - target) ** 2 + ) batch.fake_score_latent_vis_dict = { "generator_pred_video": generator_pred_x0, "fake_score_timestep": fake_score_timestep, } - outputs = {"fake_score_latent_vis_dict": batch.fake_score_latent_vis_dict} - return flow_matching_loss, (batch.timesteps, batch.attn_metadata), outputs + outputs = { + "fake_score_latent_vis_dict": ( + batch.fake_score_latent_vis_dict + ) + } + return ( + flow_matching_loss, + (batch.timesteps, batch.attn_metadata), + outputs, + ) - def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor: + def _dmd_loss( + self, + generator_pred_x0: torch.Tensor, + batch: Any, + ) -> torch.Tensor: guidance_scale = get_optional_float( self.method_config, "real_score_guidance_scale", where="method_config.real_score_guidance_scale", ) if guidance_scale is None: - guidance_scale = float(getattr(self.training_args, "real_score_guidance_scale", 1.0)) + guidance_scale = float( + getattr( + self.training_args, + "real_score_guidance_scale", + 1.0, + ) + ) device = generator_pred_x0.device with torch.no_grad(): timestep = torch.randint( 0, - int(self.model.num_train_timesteps), + int(self.student.num_train_timesteps), [1], device=device, dtype=torch.long, ) - timestep = self.model.shift_and_clamp_timestep(timestep) + timestep = self.student.shift_and_clamp_timestep( + timestep + ) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_latents = self._sf_add_noise(generator_pred_x0, noise, timestep) + noisy_latents = self._sf_add_noise( + generator_pred_x0, noise, timestep + ) faker_x0 = self._predict_x0_with_scheduler( self.critic, @@ -644,14 +699,23 @@ def _dmd_loss(self, generator_pred_x0: torch.Tensor, batch: Any) -> torch.Tensor conditional=False, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale + real_cfg_x0 = real_cond_x0 + ( + real_cond_x0 - real_uncond_x0 + ) * guidance_scale - denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() + denom = torch.abs( + generator_pred_x0 - real_cfg_x0 + ).mean() grad = (faker_x0 - real_cfg_x0) / denom grad = torch.nan_to_num(grad) loss = 0.5 * torch.mean( - (generator_pred_x0.float() - (generator_pred_x0.float() - grad.float()).detach()) + ( + generator_pred_x0.float() + - ( + generator_pred_x0.float() - grad.float() + ).detach() + ) ** 2 ) return loss diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index a81138f49..f9081c3a9 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -1,23 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Diffusion-forcing SFT method (DFSFT; algorithm layer). - -Config keys used (YAML schema-v2): -- `recipe.method`: must be `"dfsft"` for this method. -- `roles`: requires `student` (and `roles.student.trainable=true`). -- `method_config`: - - `attn_kind` (optional): `dense` or `vsa` - - `chunk_size` (optional; default=3) - - `min_timestep_ratio` / `max_timestep_ratio` (optional) -- `training` (selected fields used for optim/schedule): - - `learning_rate`, `betas`, `lr_scheduler` - - `weight_decay`, `lr_warmup_steps`, `max_train_steps`, `lr_num_cycles`, - `lr_power`, `min_lr_ratio`, `max_grad_norm` -- `training.validation.*` (parsed by method; executed via validator): - - `enabled`, `every_steps`, `dataset_file`, `sampling_steps` - - optional: `guidance_scale`, `sampler_kind`, `ode_solver`, `rollout_mode`, - `output_dir`, `num_frames` -""" +"""Diffusion-forcing SFT method (DFSFT; algorithm layer).""" from __future__ import annotations @@ -27,14 +10,10 @@ import torch.nn.functional as F from fastvideo.distillation.methods.base import DistillMethod, LogScalar -from fastvideo.distillation.dispatch import register_method -from fastvideo.distillation.utils.config import ( - DistillRunConfig, - parse_betas, -) -from fastvideo.distillation.roles import RoleManager +from fastvideo.distillation.models.base import ModelBase +from fastvideo.distillation.utils.config import parse_betas from fastvideo.distillation.utils.optimizer import ( - build_role_optimizer_and_scheduler, + build_optimizer_and_scheduler, clip_grad_norm_if_needed, ) from fastvideo.distillation.utils.validation import ( @@ -52,64 +31,56 @@ from fastvideo.distillation.validators.base import ValidationRequest if TYPE_CHECKING: - from fastvideo.distillation.models.base import ModelBase + pass -@register_method("dfsft") class DiffusionForcingSFTMethod(DistillMethod): - """Diffusion-forcing SFT (DFSFT): train only `student` with inhomogeneous timesteps. - - This is a supervised finetuning objective (flow-matching loss), except that - we sample *block-wise* (chunk-wise) inhomogeneous timesteps `t_inhom` over - the latent time dimension to expose the student to noisy-history regimes. + """Diffusion-forcing SFT (DFSFT): train only ``student`` + with inhomogeneous timesteps. """ def __init__( self, *, - bundle: RoleManager, - model: ModelBase, - method_config: dict[str, Any] | None = None, - validation_config: dict[str, Any] | None = None, + cfg: Any, + role_models: dict[str, ModelBase], validator: Any | None = None, ) -> None: - super().__init__(bundle) - bundle.require_roles(["student"]) - self.student = bundle.role("student") - if not self.student.trainable: - raise ValueError("DFSFT requires roles.student.trainable=true") + super().__init__(role_models=role_models) + + if "student" not in role_models: + raise ValueError("DFSFT requires role 'student'") + self.student = role_models["student"] + if not getattr(self.student, "_trainable", True): + raise ValueError( + "DFSFT requires student to be trainable" + ) - self.model = model self.validator = validator - self.training_args = model.training_args - self.method_config: dict[str, Any] = dict(method_config or {}) - self.validation_config: dict[str, Any] = dict(validation_config or {}) - self._attn_kind: Literal["dense", "vsa"] = self._parse_attn_kind( - self.method_config.get("attn_kind", None) + self.training_args = cfg.training_args + self.method_config: dict[str, Any] = dict( + getattr(cfg, "method_config", {}) or {} + ) + self.validation_config: dict[str, Any] = dict( + getattr(cfg, "validation", {}) or {} + ) + self._attn_kind: Literal["dense", "vsa"] = ( + self._parse_attn_kind( + self.method_config.get("attn_kind", None) + ) ) - self._chunk_size = self._parse_chunk_size(self.method_config.get("chunk_size", None)) - self._timestep_index_range = self._parse_timestep_index_range() + self._chunk_size = self._parse_chunk_size( + self.method_config.get("chunk_size", None) + ) + self._timestep_index_range = ( + self._parse_timestep_index_range() + ) - self._init_optimizers_and_schedulers() + # Initialize preprocessors on student. + self.student.init_preprocessors(self.training_args) - # DistillMethod override: build - @classmethod - def build( - cls, - *, - cfg: DistillRunConfig, - bundle: RoleManager, - model: Any, - validator: Any | None, - ) -> DistillMethod: - return cls( - bundle=bundle, - model=model, - method_config=cfg.method_config, - validation_config=cfg.validation, - validator=validator, - ) + self._init_optimizers_and_schedulers() # DistillMethod override: single_train_step def single_train_step( @@ -118,34 +89,53 @@ def single_train_step( iteration: int, *, current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: + ) -> tuple[ + dict[str, torch.Tensor], + dict[str, Any], + dict[str, LogScalar], + ]: del iteration - training_batch = self.model.prepare_batch( + training_batch = self.student.prepare_batch( batch, current_vsa_sparsity=current_vsa_sparsity, latents_source="data", ) if training_batch.latents is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.latents") + raise RuntimeError( + "prepare_batch() must set TrainingBatch.latents" + ) clean_latents = training_batch.latents if not torch.is_tensor(clean_latents): - raise TypeError("TrainingBatch.latents must be a torch.Tensor") + raise TypeError( + "TrainingBatch.latents must be a torch.Tensor" + ) if clean_latents.ndim != 5: raise ValueError( - "TrainingBatch.latents must be [B, T, C, H, W], got " + "TrainingBatch.latents must be " + "[B, T, C, H, W], got " f"shape={tuple(clean_latents.shape)}" ) - batch_size, num_latents = int(clean_latents.shape[0]), int(clean_latents.shape[1]) + batch_size, num_latents = int( + clean_latents.shape[0] + ), int(clean_latents.shape[1]) - transformer = self.student.require_module("transformer") - expected_chunk = getattr(transformer, "num_frame_per_block", None) - if expected_chunk is not None and int(expected_chunk) != int(self._chunk_size): + expected_chunk = getattr( + self.student.transformer, + "num_frame_per_block", + None, + ) + if ( + expected_chunk is not None + and int(expected_chunk) != int(self._chunk_size) + ): raise ValueError( - "DFSFT chunk_size must match transformer.num_frame_per_block for " - f"causal training (got {self._chunk_size}, expected {expected_chunk})." + "DFSFT chunk_size must match " + "transformer.num_frame_per_block for " + f"causal training (got {self._chunk_size}, " + f"expected {expected_chunk})." ) timestep_indices = self._sample_t_inhom_indices( @@ -153,20 +143,29 @@ def single_train_step( num_latents=num_latents, device=clean_latents.device, ) - sp_size = int(getattr(self.training_args, "sp_size", 1) or 1) - sp_group = getattr(self.model, "sp_group", None) - if sp_size > 1 and sp_group is not None and hasattr(sp_group, "broadcast"): + sp_size = int( + getattr(self.training_args, "sp_size", 1) or 1 + ) + sp_group = getattr(self.student, "sp_group", None) + if ( + sp_size > 1 + and sp_group is not None + and hasattr(sp_group, "broadcast") + ): sp_group.broadcast(timestep_indices, src=0) - scheduler = getattr(self.model, "noise_scheduler", None) + scheduler = self.student.noise_scheduler if scheduler is None: - raise ValueError("DFSFT requires model.noise_scheduler") + raise ValueError( + "DFSFT requires student.noise_scheduler" + ) schedule_timesteps = scheduler.timesteps.to( device=clean_latents.device, dtype=torch.float32 ) schedule_sigmas = scheduler.sigmas.to( - device=clean_latents.device, dtype=clean_latents.dtype + device=clean_latents.device, + dtype=clean_latents.dtype, ) t_inhom = schedule_timesteps[timestep_indices] @@ -175,17 +174,21 @@ def single_train_step( noise = torch.randn_like(clean_latents) else: if not torch.is_tensor(noise): - raise TypeError("TrainingBatch.noise must be a torch.Tensor when set") - noise = noise.permute(0, 2, 1, 3, 4).to(dtype=clean_latents.dtype) + raise TypeError( + "TrainingBatch.noise must be a " + "torch.Tensor when set" + ) + noise = noise.permute(0, 2, 1, 3, 4).to( + dtype=clean_latents.dtype + ) - noisy_latents = self.model.add_noise( + noisy_latents = self.student.add_noise( clean_latents, noise, t_inhom.flatten(), ) - pred = self.model.predict_noise( - self.student, + pred = self.student.predict_noise( noisy_latents, t_inhom, training_batch, @@ -193,11 +196,21 @@ def single_train_step( attn_kind=self._attn_kind, ) - if bool(getattr(self.training_args, "precondition_outputs", False)): + if bool( + getattr( + self.training_args, + "precondition_outputs", + False, + ) + ): sigmas = schedule_sigmas[timestep_indices] - sigmas = sigmas.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) + sigmas = sigmas.unsqueeze(-1).unsqueeze( + -1 + ).unsqueeze(-1) pred_x0 = noisy_latents - pred * sigmas - loss = F.mse_loss(pred_x0.float(), clean_latents.float()) + loss = F.mse_loss( + pred_x0.float(), clean_latents.float() + ) else: target = noise - clean_latents loss = F.mse_loss(pred.float(), target.float()) @@ -208,7 +221,12 @@ def single_train_step( attn_metadata = training_batch.attn_metadata loss_map = {"total_loss": loss, "dfsft_loss": loss} - outputs: dict[str, Any] = {"_fv_backward": (training_batch.timesteps, attn_metadata)} + outputs: dict[str, Any] = { + "_fv_backward": ( + training_batch.timesteps, + attn_metadata, + ) + } metrics: dict[str, LogScalar] = {} return loss_map, outputs, metrics @@ -223,33 +241,40 @@ def backward( grad_accum_rounds = max(1, int(grad_accum_rounds)) ctx = outputs.get("_fv_backward") if ctx is None: - super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + super().backward( + loss_map, + outputs, + grad_accum_rounds=grad_accum_rounds, + ) return - self.model.backward( + self.student.backward( loss_map["total_loss"], ctx, grad_accum_rounds=grad_accum_rounds, ) # DistillMethod override: get_optimizers - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + def get_optimizers( + self, iteration: int + ) -> list[torch.optim.Optimizer]: del iteration - return list(self.student.optimizers.values()) + return [self._student_optimizer] # DistillMethod override: get_lr_schedulers def get_lr_schedulers(self, iteration: int) -> list[Any]: del iteration - return list(self.student.lr_schedulers.values()) + return [self._student_lr_scheduler] # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: - for module in self.student.modules.values(): - clip_grad_norm_if_needed(module, self.training_args) + clip_grad_norm_if_needed( + self.student.transformer, self.training_args + ) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start def on_train_start(self) -> None: - self.model.on_train_start() + self.student.on_train_start() # DistillTrainer hook: log_validation def log_validation(self, iteration: int) -> None: @@ -259,19 +284,35 @@ def log_validation(self, iteration: int) -> None: if not is_validation_enabled(self.validation_config): return - every_steps = parse_validation_every_steps(self.validation_config) + every_steps = parse_validation_every_steps( + self.validation_config + ) if every_steps <= 0: return if iteration % every_steps != 0: return - dataset_file = parse_validation_dataset_file(self.validation_config) - sampling_steps = parse_validation_sampling_steps(self.validation_config) - guidance_scale = parse_validation_guidance_scale(self.validation_config) - sampler_kind = parse_validation_sampler_kind(self.validation_config, default="ode") - rollout_mode = parse_validation_rollout_mode(self.validation_config) - output_dir = parse_validation_output_dir(self.validation_config) - num_actions = parse_validation_num_frames(self.validation_config) + dataset_file = parse_validation_dataset_file( + self.validation_config + ) + sampling_steps = parse_validation_sampling_steps( + self.validation_config + ) + guidance_scale = parse_validation_guidance_scale( + self.validation_config + ) + sampler_kind = parse_validation_sampler_kind( + self.validation_config, default="ode" + ) + rollout_mode = parse_validation_rollout_mode( + self.validation_config + ) + output_dir = parse_validation_output_dir( + self.validation_config + ) + num_actions = parse_validation_num_frames( + self.validation_config + ) ode_solver = parse_validation_ode_solver( self.validation_config, sampler_kind=sampler_kind ) @@ -292,30 +333,30 @@ def log_validation(self, iteration: int) -> None: # Checkpoint hook: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - generators: dict[str, torch.Generator] = {} - model = getattr(self, "model", None) - get_model_generators = getattr(model, "get_rng_generators", None) - if callable(get_model_generators): - generators.update(get_model_generators()) + student_gens = self.student.get_rng_generators() + generators.update(student_gens) validator = getattr(self, "validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) + validation_gen = getattr( + validator, "validation_random_generator", None + ) if isinstance(validation_gen, torch.Generator): generators["validation_cpu"] = validation_gen return generators - def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: + def _parse_attn_kind( + self, raw: Any + ) -> Literal["dense", "vsa"]: if raw in (None, ""): return "dense" kind = str(raw).strip().lower() if kind not in {"dense", "vsa"}: raise ValueError( - "method_config.attn_kind must be one of {'dense', 'vsa'}, " - f"got {raw!r}." + "method_config.attn_kind must be one of " + f"{{'dense', 'vsa'}}, got {raw!r}." ) return cast(Literal["dense", "vsa"], kind) @@ -323,58 +364,98 @@ def _parse_chunk_size(self, raw: Any) -> int: if raw in (None, ""): return 3 if isinstance(raw, bool): - raise ValueError("method_config.chunk_size must be an int, got bool") + raise ValueError( + "method_config.chunk_size must be an int, " + "got bool" + ) if isinstance(raw, float) and not raw.is_integer(): - raise ValueError("method_config.chunk_size must be an int, got float") + raise ValueError( + "method_config.chunk_size must be an int, " + "got float" + ) if isinstance(raw, str) and not raw.strip(): - raise ValueError("method_config.chunk_size must be an int, got empty string") + raise ValueError( + "method_config.chunk_size must be an int, " + "got empty string" + ) try: value = int(raw) except (TypeError, ValueError) as e: raise ValueError( - "method_config.chunk_size must be an int, got " - f"{type(raw).__name__}" + "method_config.chunk_size must be an int, " + f"got {type(raw).__name__}" ) from e if value <= 0: - raise ValueError("method_config.chunk_size must be > 0") + raise ValueError( + "method_config.chunk_size must be > 0" + ) return value - def _parse_ratio(self, raw: Any, *, where: str, default: float) -> float: + def _parse_ratio( + self, raw: Any, *, where: str, default: float + ) -> float: if raw in (None, ""): return float(default) if isinstance(raw, bool): - raise ValueError(f"{where} must be a number/string, got bool") + raise ValueError( + f"{where} must be a number/string, got bool" + ) if isinstance(raw, (int, float)): return float(raw) if isinstance(raw, str) and raw.strip(): return float(raw) - raise ValueError(f"{where} must be a number/string, got {type(raw).__name__}") + raise ValueError( + f"{where} must be a number/string, " + f"got {type(raw).__name__}" + ) def _parse_timestep_index_range(self) -> tuple[int, int]: - scheduler = getattr(self.model, "noise_scheduler", None) + scheduler = self.student.noise_scheduler if scheduler is None: - raise ValueError("DFSFT requires model.noise_scheduler") - num_steps = int(getattr(scheduler, "config", scheduler).num_train_timesteps) + raise ValueError( + "DFSFT requires student.noise_scheduler" + ) + num_steps = int( + getattr(scheduler, "config", scheduler) + .num_train_timesteps + ) min_ratio = self._parse_ratio( self.method_config.get("min_timestep_ratio", None), where="method_config.min_timestep_ratio", - default=float(getattr(self.training_args, "min_timestep_ratio", 0.0) or 0.0), + default=float( + getattr( + self.training_args, + "min_timestep_ratio", + 0.0, + ) + or 0.0 + ), ) max_ratio = self._parse_ratio( self.method_config.get("max_timestep_ratio", None), where="method_config.max_timestep_ratio", - default=float(getattr(self.training_args, "max_timestep_ratio", 1.0) or 1.0), + default=float( + getattr( + self.training_args, + "max_timestep_ratio", + 1.0, + ) + or 1.0 + ), ) - if not (0.0 <= min_ratio <= 1.0 and 0.0 <= max_ratio <= 1.0): + if not ( + 0.0 <= min_ratio <= 1.0 and 0.0 <= max_ratio <= 1.0 + ): raise ValueError( - "DFSFT timestep ratios must be in [0,1], got " - f"min={min_ratio}, max={max_ratio}" + "DFSFT timestep ratios must be in [0,1], " + f"got min={min_ratio}, max={max_ratio}" ) if max_ratio < min_ratio: raise ValueError( - "method_config.max_timestep_ratio must be >= min_timestep_ratio" + "method_config.max_timestep_ratio must be " + ">= min_timestep_ratio" ) min_index = int(min_ratio * num_steps) @@ -385,31 +466,60 @@ def _parse_timestep_index_range(self) -> tuple[int, int]: if max_index <= min_index: max_index = min(num_steps - 1, min_index + 1) - # torch.randint expects [low, high), so make high exclusive. return min_index, max_index + 1 def _init_optimizers_and_schedulers(self) -> None: - student_lr = float(getattr(self.training_args, "learning_rate", 0.0) or 0.0) + student_lr = float( + getattr(self.training_args, "learning_rate", 0.0) + or 0.0 + ) if student_lr <= 0.0: - raise ValueError("training.learning_rate must be > 0 for dfsft") + raise ValueError( + "training.learning_rate must be > 0 for dfsft" + ) student_betas = parse_betas( getattr(self.training_args, "betas", None), where="training.betas", ) - student_sched = str(getattr(self.training_args, "lr_scheduler", "constant")) - build_role_optimizer_and_scheduler( - role="student", - handle=self.student, + student_sched = str( + getattr( + self.training_args, "lr_scheduler", "constant" + ) + ) + student_params = [ + p + for p in self.student.transformer.parameters() + if p.requires_grad + ] + if getattr(self.student, "transformer_2", None) is not None: + student_params.extend( + p + for p in self.student.transformer_2.parameters() + if p.requires_grad + ) + ( + self._student_optimizer, + self._student_lr_scheduler, + ) = build_optimizer_and_scheduler( + params=student_params, training_args=self.training_args, learning_rate=student_lr, betas=student_betas, scheduler_name=student_sched, ) - def _sample_t_inhom_indices(self, *, batch_size: int, num_latents: int, device: torch.device) -> torch.Tensor: + def _sample_t_inhom_indices( + self, + *, + batch_size: int, + num_latents: int, + device: torch.device, + ) -> torch.Tensor: chunk_size = self._chunk_size - num_chunks = (num_latents + chunk_size - 1) // chunk_size + num_chunks = ( + num_latents + chunk_size - 1 + ) // chunk_size low, high = self._timestep_index_range chunk_indices = torch.randint( low=low, @@ -418,5 +528,7 @@ def _sample_t_inhom_indices(self, *, batch_size: int, num_latents: int, device: device=device, dtype=torch.long, ) - expanded = chunk_indices.repeat_interleave(chunk_size, dim=1) + expanded = chunk_indices.repeat_interleave( + chunk_size, dim=1 + ) return expanded[:, :num_latents] diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index ab8f0614a..ab0df2e5a 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -1,21 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Supervised finetuning method (algorithm layer). - -Config keys used (YAML schema-v2): -- `recipe.method`: must be `"finetune"` for this method. -- `roles`: requires `student` (and `roles.student.trainable=true`). -- `method_config`: - - `attn_kind` (optional): `dense` or `vsa` -- `training` (selected fields used for optim/schedule): - - `learning_rate`, `betas`, `lr_scheduler` - - `weight_decay`, `lr_warmup_steps`, `max_train_steps`, `lr_num_cycles`, - `lr_power`, `min_lr_ratio`, `max_grad_norm` -- `training.validation.*` (parsed by method; executed via validator): - - `enabled`, `every_steps`, `dataset_file`, `sampling_steps` - - optional: `guidance_scale`, `sampler_kind`, `ode_solver`, `rollout_mode`, - `output_dir`, `num_frames` -""" +"""Supervised finetuning method (algorithm layer).""" from __future__ import annotations @@ -24,11 +9,10 @@ import torch import torch.nn.functional as F -from fastvideo.distillation.dispatch import register_method from fastvideo.distillation.methods.base import DistillMethod, LogScalar -from fastvideo.distillation.roles import RoleManager +from fastvideo.distillation.models.base import ModelBase from fastvideo.distillation.utils.optimizer import ( - build_role_optimizer_and_scheduler, + build_optimizer_and_scheduler, clip_grad_norm_if_needed, ) from fastvideo.distillation.utils.validation import ( @@ -44,68 +28,52 @@ parse_validation_sampling_steps, ) from fastvideo.distillation.validators.base import ValidationRequest -from fastvideo.distillation.utils.config import ( - DistillRunConfig, - parse_betas, -) +from fastvideo.distillation.utils.config import parse_betas if TYPE_CHECKING: - from fastvideo.distillation.models.base import ModelBase + pass -@register_method("finetune") class FineTuneMethod(DistillMethod): - """Supervised finetuning as a method: only `student` participates. - - The loss follows the same objective used by the legacy training pipeline: - - default: flow-matching target `noise - x0` - - optional (if `training.precondition_outputs=true`): precondition to `x0` - and regress `x0` directly. - """ + """Supervised finetuning: only ``student`` participates.""" def __init__( self, *, - bundle: RoleManager, - model: ModelBase, - method_config: dict[str, Any] | None = None, - validation_config: dict[str, Any] | None = None, + cfg: Any, + role_models: dict[str, ModelBase], validator: Any | None = None, ) -> None: - super().__init__(bundle) - bundle.require_roles(["student"]) - self.student = bundle.role("student") - if not self.student.trainable: - raise ValueError("FineTuneMethod requires roles.student.trainable=true") + super().__init__(role_models=role_models) + + if "student" not in role_models: + raise ValueError( + "FineTuneMethod requires role 'student'" + ) + self.student = role_models["student"] + if not getattr(self.student, "_trainable", True): + raise ValueError( + "FineTuneMethod requires student to be trainable" + ) - self.model = model self.validator = validator - self.training_args = model.training_args - self.method_config: dict[str, Any] = dict(method_config or {}) - self.validation_config: dict[str, Any] = dict(validation_config or {}) - self._attn_kind: Literal["dense", "vsa"] = self._parse_attn_kind( - self.method_config.get("attn_kind", None) + self.training_args = cfg.training_args + self.method_config: dict[str, Any] = dict( + getattr(cfg, "method_config", {}) or {} + ) + self.validation_config: dict[str, Any] = dict( + getattr(cfg, "validation", {}) or {} + ) + self._attn_kind: Literal["dense", "vsa"] = ( + self._parse_attn_kind( + self.method_config.get("attn_kind", None) + ) ) - self._init_optimizers_and_schedulers() + # Initialize preprocessors on student. + self.student.init_preprocessors(self.training_args) - # DistillMethod override: build - @classmethod - def build( - cls, - *, - cfg: DistillRunConfig, - bundle: RoleManager, - model: Any, - validator: Any | None, - ) -> DistillMethod: - return cls( - bundle=bundle, - model=model, - method_config=cfg.method_config, - validation_config=cfg.validation, - validator=validator, - ) + self._init_optimizers_and_schedulers() # DistillMethod override: single_train_step def single_train_step( @@ -114,35 +82,50 @@ def single_train_step( iteration: int, *, current_vsa_sparsity: float = 0.0, - ) -> tuple[dict[str, torch.Tensor], dict[str, Any], dict[str, LogScalar]]: + ) -> tuple[ + dict[str, torch.Tensor], + dict[str, Any], + dict[str, LogScalar], + ]: del iteration - training_batch = self.model.prepare_batch( + training_batch = self.student.prepare_batch( batch, current_vsa_sparsity=current_vsa_sparsity, latents_source="data", ) if training_batch.latents is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.latents") + raise RuntimeError( + "prepare_batch() must set TrainingBatch.latents" + ) if training_batch.noisy_model_input is None: raise RuntimeError( - "model.prepare_batch() must set TrainingBatch.noisy_model_input" + "prepare_batch() must set " + "TrainingBatch.noisy_model_input" ) if training_batch.noise is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.noise") + raise RuntimeError( + "prepare_batch() must set TrainingBatch.noise" + ) if training_batch.sigmas is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.sigmas") + raise RuntimeError( + "prepare_batch() must set TrainingBatch.sigmas" + ) if training_batch.timesteps is None: - raise RuntimeError("model.prepare_batch() must set TrainingBatch.timesteps") + raise RuntimeError( + "prepare_batch() must set " + "TrainingBatch.timesteps" + ) clean_latents = training_batch.latents - noisy_latents = training_batch.noisy_model_input.permute(0, 2, 1, 3, 4) + noisy_latents = training_batch.noisy_model_input.permute( + 0, 2, 1, 3, 4 + ) noise = training_batch.noise.permute(0, 2, 1, 3, 4) sigmas = training_batch.sigmas timesteps = training_batch.timesteps - pred = self.model.predict_noise( - self.student, + pred = self.student.predict_noise( noisy_latents, timesteps, training_batch, @@ -150,9 +133,17 @@ def single_train_step( attn_kind=self._attn_kind, ) - if bool(getattr(self.training_args, "precondition_outputs", False)): + if bool( + getattr( + self.training_args, + "precondition_outputs", + False, + ) + ): pred_x0 = noisy_latents - pred * sigmas - loss = F.mse_loss(pred_x0.float(), clean_latents.float()) + loss = F.mse_loss( + pred_x0.float(), clean_latents.float() + ) else: target = noise - clean_latents loss = F.mse_loss(pred.float(), target.float()) @@ -163,7 +154,12 @@ def single_train_step( attn_metadata = training_batch.attn_metadata loss_map = {"total_loss": loss, "finetune_loss": loss} - outputs: dict[str, Any] = {"_fv_backward": (training_batch.timesteps, attn_metadata)} + outputs: dict[str, Any] = { + "_fv_backward": ( + training_batch.timesteps, + attn_metadata, + ) + } metrics: dict[str, LogScalar] = {} return loss_map, outputs, metrics @@ -178,33 +174,40 @@ def backward( grad_accum_rounds = max(1, int(grad_accum_rounds)) ctx = outputs.get("_fv_backward") if ctx is None: - super().backward(loss_map, outputs, grad_accum_rounds=grad_accum_rounds) + super().backward( + loss_map, + outputs, + grad_accum_rounds=grad_accum_rounds, + ) return - self.model.backward( + self.student.backward( loss_map["total_loss"], ctx, grad_accum_rounds=grad_accum_rounds, ) # DistillMethod override: get_optimizers - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + def get_optimizers( + self, iteration: int + ) -> list[torch.optim.Optimizer]: del iteration - return list(self.student.optimizers.values()) + return [self._student_optimizer] # DistillMethod override: get_lr_schedulers def get_lr_schedulers(self, iteration: int) -> list[Any]: del iteration - return list(self.student.lr_schedulers.values()) + return [self._student_lr_scheduler] # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: - for module in self.student.modules.values(): - clip_grad_norm_if_needed(module, self.training_args) + clip_grad_norm_if_needed( + self.student.transformer, self.training_args + ) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start def on_train_start(self) -> None: - self.model.on_train_start() + self.student.on_train_start() # DistillTrainer hook: log_validation def log_validation(self, iteration: int) -> None: @@ -214,19 +217,35 @@ def log_validation(self, iteration: int) -> None: if not is_validation_enabled(self.validation_config): return - every_steps = parse_validation_every_steps(self.validation_config) + every_steps = parse_validation_every_steps( + self.validation_config + ) if every_steps <= 0: return if iteration % every_steps != 0: return - dataset_file = parse_validation_dataset_file(self.validation_config) - sampling_steps = parse_validation_sampling_steps(self.validation_config) - guidance_scale = parse_validation_guidance_scale(self.validation_config) - sampler_kind = parse_validation_sampler_kind(self.validation_config, default="ode") - rollout_mode = parse_validation_rollout_mode(self.validation_config) - output_dir = parse_validation_output_dir(self.validation_config) - num_actions = parse_validation_num_frames(self.validation_config) + dataset_file = parse_validation_dataset_file( + self.validation_config + ) + sampling_steps = parse_validation_sampling_steps( + self.validation_config + ) + guidance_scale = parse_validation_guidance_scale( + self.validation_config + ) + sampler_kind = parse_validation_sampler_kind( + self.validation_config, default="ode" + ) + rollout_mode = parse_validation_rollout_mode( + self.validation_config + ) + output_dir = parse_validation_output_dir( + self.validation_config + ) + num_actions = parse_validation_num_frames( + self.validation_config + ) ode_solver = parse_validation_ode_solver( self.validation_config, sampler_kind=sampler_kind ) @@ -247,48 +266,67 @@ def log_validation(self, iteration: int) -> None: # Checkpoint hook: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - generators: dict[str, torch.Generator] = {} - model = getattr(self, "model", None) - get_model_generators = getattr(model, "get_rng_generators", None) - if callable(get_model_generators): - generators.update(get_model_generators()) + student_gens = self.student.get_rng_generators() + generators.update(student_gens) validator = getattr(self, "validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) + validation_gen = getattr( + validator, "validation_random_generator", None + ) if isinstance(validation_gen, torch.Generator): generators["validation_cpu"] = validation_gen return generators - def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: + def _parse_attn_kind( + self, raw: Any + ) -> Literal["dense", "vsa"]: if raw in (None, ""): return "dense" kind = str(raw).strip().lower() if kind not in {"dense", "vsa"}: raise ValueError( - "method_config.attn_kind must be one of {'dense', 'vsa'}, " - f"got {raw!r}." + "method_config.attn_kind must be one of " + f"{{'dense', 'vsa'}}, got {raw!r}." ) return cast(Literal["dense", "vsa"], kind) def _init_optimizers_and_schedulers(self) -> None: training_args = self.training_args - student_lr = float(getattr(training_args, "learning_rate", 0.0) or 0.0) + student_lr = float( + getattr(training_args, "learning_rate", 0.0) or 0.0 + ) if student_lr <= 0.0: - raise ValueError("training.learning_rate must be > 0 for finetune") + raise ValueError( + "training.learning_rate must be > 0 for finetune" + ) student_betas = parse_betas( getattr(training_args, "betas", None), where="training.betas", ) - student_sched = str(getattr(training_args, "lr_scheduler", "constant")) - build_role_optimizer_and_scheduler( - role="student", - handle=self.student, + student_sched = str( + getattr(training_args, "lr_scheduler", "constant") + ) + student_params = [ + p + for p in self.student.transformer.parameters() + if p.requires_grad + ] + if getattr(self.student, "transformer_2", None) is not None: + student_params.extend( + p + for p in self.student.transformer_2.parameters() + if p.requires_grad + ) + ( + self._student_optimizer, + self._student_lr_scheduler, + ) = build_optimizer_and_scheduler( + params=student_params, training_args=self.training_args, learning_rate=student_lr, betas=student_betas, diff --git a/fastvideo/distillation/models/base.py b/fastvideo/distillation/models/base.py index fec1e589f..a6cc75c64 100644 --- a/fastvideo/distillation/models/base.py +++ b/fastvideo/distillation/models/base.py @@ -7,50 +7,60 @@ import torch -from fastvideo.distillation.roles import RoleHandle - if TYPE_CHECKING: from fastvideo.pipelines import TrainingBatch class ModelBase(ABC): - """Operation-centric runtime primitives implemented by a model plugin. - - This interface is intentionally *method-agnostic*: - - A method selects roles (student/teacher/critic/...) and decides how to use - them. - - The model plugin implements how to run those roles against FastVideo - pipelines, forward-context requirements, and batch normalization quirks. + """Per-role model instance. - Implementations typically live next to the model plugin - (e.g. `models/wan/wan.py`) - rather than in a global adapter registry. + Every role (student, teacher, critic, …) gets its own ``ModelBase`` + instance. Each instance owns its own ``transformer`` and + ``noise_scheduler``. Heavyweight resources (VAE, dataloader, RNG + seeds) are loaded lazily via :meth:`init_preprocessors`, which the + method calls **only on the student**. """ - training_args: Any - bundle: Any - dataloader: Any - validator: Any | None - start_step: int + transformer: torch.nn.Module + noise_scheduler: Any - @property - @abstractmethod - def num_train_timesteps(self) -> int: - """Return the scheduler's training timestep horizon (usually 1000).""" + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ - @abstractmethod - def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: - """Apply model/pipeline timestep shifting and clamp into valid range.""" + def init_preprocessors(self, training_args: Any) -> None: + """Load VAE, build dataloader, seed RNGs. + + Called only on the student by the method's ``__init__``. + Default is a no-op so teacher/critic instances skip this. + """ - @abstractmethod def on_train_start(self) -> None: - """Initialize RNG seeds and any cached conditioning needed for training.""" + """Called once before the training loop begins.""" def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - + """Return RNG generators for checkpoint resume.""" return {} + # ------------------------------------------------------------------ + # Timestep helpers + # ------------------------------------------------------------------ + + @property + def num_train_timesteps(self) -> int: + """Return the scheduler's training timestep horizon.""" + return int(self.noise_scheduler.num_train_timesteps) + + def shift_and_clamp_timestep( + self, timestep: torch.Tensor + ) -> torch.Tensor: + """Apply model/pipeline timestep shifting and clamp.""" + return timestep + + # ------------------------------------------------------------------ + # Runtime primitives + # ------------------------------------------------------------------ + @abstractmethod def prepare_batch( self, @@ -59,7 +69,7 @@ def prepare_batch( current_vsa_sparsity: float = 0.0, latents_source: Literal["data", "zeros"] = "data", ) -> TrainingBatch: - """Convert a dataloader batch into forward primitives for methods.""" + """Convert a dataloader batch into forward primitives.""" @abstractmethod def add_noise( @@ -68,12 +78,11 @@ def add_noise( noise: torch.Tensor, timestep: torch.Tensor, ) -> torch.Tensor: - """Apply forward-process noise at `timestep` for a given scheduler.""" + """Apply forward-process noise at *timestep*.""" @abstractmethod def predict_noise( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, @@ -82,12 +91,11 @@ def predict_noise( cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: - """Run a role to predict noise/flow for the given noisy latents.""" + """Predict noise/flow for the given noisy latents.""" @abstractmethod def predict_x0( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, @@ -96,32 +104,33 @@ def predict_x0( cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor: - """Run a role to predict x0 for the given noisy latents.""" + """Predict x0 for the given noisy latents.""" @abstractmethod - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: - """Backward hook that may restore forward-context for checkpointed modules.""" + def backward( + self, + loss: torch.Tensor, + ctx: Any, + *, + grad_accum_rounds: int, + ) -> None: + """Backward that may restore forward-context.""" class CausalModelBase(ModelBase): - """Optional extension for causal / streaming model plugins. - - This mirrors the FastGen design choice that *only* causal networks expose a - cache lifecycle API, while non-causal models stay on a clean `ModelBase` - interface. + """Extension for causal / streaming model plugins. - Important: methods should not pass KV-cache tensors around. Cache state is - internal to the causal model plugin and keyed by `(role handle, cache_tag)`. + Cache state is internal to the model instance and keyed by + *cache_tag* (no role handle needed). """ @abstractmethod - def clear_caches(self, handle: RoleHandle, *, cache_tag: str = "pos") -> None: - """Clear internal caches for a role before starting a new rollout.""" + def clear_caches(self, *, cache_tag: str = "pos") -> None: + """Clear internal caches before starting a new rollout.""" @abstractmethod def predict_noise_streaming( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, @@ -133,12 +142,11 @@ def predict_noise_streaming( cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor | None: - """Streaming predict-noise primitive that may update internal caches.""" + """Streaming predict-noise that may update internal caches.""" @abstractmethod def predict_x0_streaming( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, @@ -150,4 +158,4 @@ def predict_x0_streaming( cfg_uncond: dict[str, Any] | None = None, attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor | None: - """Streaming predict-x0 primitive that may update internal caches.""" + """Streaming predict-x0 that may update internal caches.""" diff --git a/fastvideo/distillation/models/wan/__init__.py b/fastvideo/distillation/models/wan/__init__.py index c9cc3e811..656b116f1 100644 --- a/fastvideo/distillation/models/wan/__init__.py +++ b/fastvideo/distillation/models/wan/__init__.py @@ -1,9 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -"""Wan model plugin package. - -Importing this package registers the `recipe.family: wan` model builder. -""" - -from fastvideo.distillation.models.wan.wan import WanModel as WanModel # noqa: F401 +"""Wan model plugin package.""" +from fastvideo.distillation.models.wan.wan import ( + WanModel as WanModel, +) diff --git a/fastvideo/distillation/models/wan/wan.py b/fastvideo/distillation/models/wan/wan.py index d4f8d3d0e..fc683eb97 100644 --- a/fastvideo/distillation/models/wan/wan.py +++ b/fastvideo/distillation/models/wan/wan.py @@ -1,24 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Wan model plugin (components + runtime adapter). - -Config keys used (YAML schema-v2): -- `recipe.family`: must be `"wan"` for this plugin. -- `roles.shared_component_role` (affects default `training.model_path`) -- `roles.`: - - `family`, `path`, `trainable`, `disable_custom_init_weights` -- `training` (selected fields): - - `seed`, `data_path`, `model_path` - - `num_height`, `num_width`, `num_latent_t` - - `min_timestep_ratio`, `max_timestep_ratio`, `boundary_ratio` (optional) - - `weighting_scheme`, `logit_mean`, `logit_std`, `mode_scale` - - `sp_size`, `tp_size`, `num_gpus`, `pin_cpu_memory` - - `pipeline_config.flow_shift`, `pipeline_config.dit_config.patch_size` - - `moba_config` (if `FASTVIDEO_ATTENTION_BACKEND=VMOBA_ATTN`) - - `enable_gradient_checkpointing_type` (optional) -- `training.validation.*` (consumed by `WanValidator` when enabled) -- `method_config.cfg_uncond.*` (consumed by `WanModel` for CFG-uncond policy) -""" +"""Wan model plugin (per-role instance).""" from __future__ import annotations @@ -43,142 +25,180 @@ from fastvideo.pipelines import TrainingBatch from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline from fastvideo.pipelines.pipeline_batch_info import ForwardBatch -from fastvideo.training.activation_checkpoint import apply_activation_checkpointing +from fastvideo.training.activation_checkpoint import ( + apply_activation_checkpointing, +) from fastvideo.training.training_utils import ( compute_density_for_timestep_sampling, get_sigmas, normalize_dit_input, shift_timestep, ) -from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed +from fastvideo.utils import ( + is_vmoba_available, + is_vsa_available, + set_random_seed, +) -from fastvideo.distillation.dispatch import register_model from fastvideo.distillation.models.base import ModelBase -from fastvideo.distillation.roles import RoleHandle, RoleManager -from fastvideo.distillation.utils.config import DistillRunConfig -from fastvideo.distillation.utils.dataloader import build_parquet_t2v_train_dataloader -from fastvideo.distillation.utils.module_state import apply_trainable -from fastvideo.distillation.utils.moduleloader import load_module_from_path +from fastvideo.distillation.utils.module_state import ( + apply_trainable, +) +from fastvideo.distillation.utils.moduleloader import ( + load_module_from_path, +) try: from fastvideo.attention.backends.video_sparse_attn import ( VideoSparseAttentionMetadataBuilder, ) - from fastvideo.attention.backends.vmoba import VideoMobaAttentionMetadataBuilder + from fastvideo.attention.backends.vmoba import ( + VideoMobaAttentionMetadataBuilder, + ) except Exception: VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] -@register_model("wan") class WanModel(ModelBase): - """Wan model plugin: loads roles + shared components and exposes runtime primitives.""" - - def __init__(self, *, cfg: DistillRunConfig) -> None: - training_args = cfg.training_args - roles_cfg = cfg.roles + """Wan per-role model: owns transformer + noise_scheduler.""" - if getattr(training_args, "seed", None) is None: - raise ValueError("training.seed must be set for distillation") - if not getattr(training_args, "data_path", ""): - raise ValueError("training.data_path must be set for distillation") - - # Load shared components (training.model_path; defaults to - # roles.shared_component_role's path). - training_args.override_transformer_cls_name = "WanTransformer3DModel" - vae = load_module_from_path( - model_path=str(training_args.model_path), - module_type="vae", - training_args=training_args, + def __init__( + self, + *, + init_from: str, + trainable: bool = True, + disable_custom_init_weights: bool = False, + flow_shift: float = 3.0, + enable_gradient_checkpointing_type: str | None = None, + ) -> None: + self._init_from = str(init_from) + self._trainable = bool(trainable) + + transformer = load_module_from_path( + model_path=self._init_from, + module_type="transformer", + training_args=None, + disable_custom_init_weights=disable_custom_init_weights, + override_transformer_cls_name="WanTransformer3DModel", ) - noise_scheduler = FlowMatchEulerDiscreteScheduler( - shift=float(training_args.pipeline_config.flow_shift or 0.0) + transformer = apply_trainable( + transformer, trainable=self._trainable ) - - role_handles: dict[str, RoleHandle] = {} - for role, role_spec in roles_cfg.items(): - if role_spec.family != "wan": - raise ValueError( - "Wan model plugin only supports roles with family='wan'; " - f"got {role}={role_spec.family!r}" - ) - - disable_custom_init_weights = bool( - getattr(role_spec, "disable_custom_init_weights", False) + if ( + self._trainable + and enable_gradient_checkpointing_type + ): + transformer = apply_activation_checkpointing( + transformer, + checkpointing_type=( + enable_gradient_checkpointing_type + ), ) - transformer = load_module_from_path( - model_path=role_spec.path, - module_type="transformer", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, + self.transformer = transformer + + # Optional MoE transformer_2. + self.transformer_2: torch.nn.Module | None = None + try: + t2 = load_module_from_path( + model_path=self._init_from, + module_type="transformer_2", + training_args=None, + disable_custom_init_weights=( + disable_custom_init_weights + ), ) - modules: dict[str, torch.nn.Module] = {"transformer": transformer} - - # Optional MoE support: load transformer_2 if present in the model. - try: - transformer_2 = load_module_from_path( - model_path=role_spec.path, - module_type="transformer_2", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, + except (ValueError, FileNotFoundError): + t2 = None + if t2 is not None: + t2 = apply_trainable(t2, trainable=self._trainable) + if ( + self._trainable + and enable_gradient_checkpointing_type + ): + t2 = apply_activation_checkpointing( + t2, + checkpointing_type=( + enable_gradient_checkpointing_type + ), ) - except ValueError: - transformer_2 = None - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 - - for name, module in list(modules.items()): - module = apply_trainable(module, trainable=bool(role_spec.trainable)) - if role_spec.trainable and getattr( - training_args, "enable_gradient_checkpointing_type", None - ): - module = apply_activation_checkpointing( - module, - checkpointing_type=training_args.enable_gradient_checkpointing_type, - ) - modules[name] = module - - role_handles[role] = RoleHandle( - modules=modules, - optimizers={}, - lr_schedulers={}, - trainable=bool(role_spec.trainable), - ) - - self.bundle = RoleManager(roles=role_handles) + self.transformer_2 = t2 - # Optional validator. - self.validator = None - validation_cfg = getattr(cfg, "validation", {}) or {} - validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) - if validation_enabled: - from fastvideo.distillation.validators.wan import WanValidator + self.noise_scheduler = FlowMatchEulerDiscreteScheduler( + shift=float(flow_shift) + ) + + # Filled by init_preprocessors (student only). + self.vae: Any = None + self.training_args: Any = None + self.dataloader: Any = None + self.validator: Any = None + self.start_step: int = 0 - self.validator = WanValidator(training_args=training_args) + self.world_group: Any = None + self.sp_group: Any = None + self.device: Any = None - # NOTE: runtime primitives need a prompt-encoding capable handle. - prompt_handle = role_handles.get("student") - if prompt_handle is None: - raise ValueError("Wan model plugin requires a 'student' role for prompt encoding") + self.noise_random_generator: torch.Generator | None = None + self.noise_gen_cuda: torch.Generator | None = None - self.prompt_handle = prompt_handle + self.negative_prompt_embeds: torch.Tensor | None = None + self.negative_prompt_attention_mask: ( + torch.Tensor | None + ) = None + + # Timestep mechanics. + self.timestep_shift: float = float(flow_shift) + self.num_train_timestep: int = int( + self.noise_scheduler.num_train_timesteps + ) + self.min_timestep: int = 0 + self.max_timestep: int = self.num_train_timestep + self.boundary_timestep: float | None = None + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def init_preprocessors(self, training_args: Any) -> None: self.training_args = training_args - self.noise_scheduler = noise_scheduler - self.vae = vae + + self.vae = load_module_from_path( + model_path=str(training_args.model_path), + module_type="vae", + training_args=training_args, + ) self.world_group = get_world_group() self.sp_group = get_sp_group() self.device = get_local_torch_device() - self.noise_random_generator: torch.Generator | None = None - self.noise_gen_cuda: torch.Generator | None = None - - self.negative_prompt_embeds: torch.Tensor | None = None - self.negative_prompt_attention_mask: torch.Tensor | None = None - self._init_timestep_mechanics() - from fastvideo.dataset.dataloader.schema import pyarrow_schema_t2v + # Optional validator. + validation_cfg = getattr( + training_args, "_validation_cfg", None + ) + if validation_cfg: + validation_enabled = bool( + validation_cfg.get( + "enabled", bool(validation_cfg) + ) + ) + if validation_enabled: + from fastvideo.distillation.validators.wan import ( + WanValidator, + ) + self.validator = WanValidator( + training_args=training_args + ) + + from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_t2v, + ) + from fastvideo.distillation.utils.dataloader import ( + build_parquet_t2v_train_dataloader, + ) self.dataloader = build_parquet_t2v_train_dataloader( training_args, @@ -186,48 +206,58 @@ def __init__(self, *, cfg: DistillRunConfig) -> None: ) self.start_step = 0 - # ModelBase override: num_train_timesteps @property def num_train_timesteps(self) -> int: return int(self.num_train_timestep) - # ModelBase override: shift_and_clamp_timestep - def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: - timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) + def shift_and_clamp_timestep( + self, timestep: torch.Tensor + ) -> torch.Tensor: + timestep = shift_timestep( + timestep, self.timestep_shift, self.num_train_timestep + ) return timestep.clamp(self.min_timestep, self.max_timestep) - # ModelBase override: on_train_start def on_train_start(self) -> None: - seed = self.training_args.seed + seed = getattr(self.training_args, "seed", None) if seed is None: - raise ValueError("training_args.seed must be set for distillation") + raise ValueError( + "training_args.seed must be set for distillation" + ) global_rank = int(getattr(self.world_group, "rank", 0)) - sp_world_size = int(getattr(self.training_args, "sp_size", 1) or 1) + sp_world_size = int( + getattr(self.training_args, "sp_size", 1) or 1 + ) if sp_world_size > 1: - sp_group_seed = int(seed) + (global_rank // sp_world_size) + sp_group_seed = int(seed) + ( + global_rank // sp_world_size + ) set_random_seed(sp_group_seed) else: set_random_seed(int(seed) + global_rank) - self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) - self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) + self.noise_random_generator = torch.Generator( + device="cpu" + ).manual_seed(int(seed)) + self.noise_gen_cuda = torch.Generator( + device=self.device + ).manual_seed(int(seed)) self.ensure_negative_conditioning() - # ModelBase override: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - generators: dict[str, torch.Generator] = {} if self.noise_random_generator is not None: generators["noise_cpu"] = self.noise_random_generator if self.noise_gen_cuda is not None: generators["noise_cuda"] = self.noise_gen_cuda - return generators - # ModelBase override: prepare_batch + # ------------------------------------------------------------------ + # Runtime primitives + # ------------------------------------------------------------------ + def prepare_batch( self, raw_batch: dict[str, Any], @@ -235,25 +265,36 @@ def prepare_batch( current_vsa_sparsity: float = 0.0, latents_source: Literal["data", "zeros"] = "data", ) -> TrainingBatch: - """Convert a dataloader batch into forward primitives for methods.""" - self.ensure_negative_conditioning() dtype = self._get_training_dtype() device = self.device - training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + training_batch = TrainingBatch( + current_vsa_sparsity=current_vsa_sparsity + ) encoder_hidden_states = raw_batch["text_embedding"] encoder_attention_mask = raw_batch["text_attention_mask"] infos = raw_batch.get("info_list") if latents_source == "zeros": batch_size = encoder_hidden_states.shape[0] - vae_config = self.training_args.pipeline_config.vae_config.arch_config + vae_config = ( + self.training_args.pipeline_config + .vae_config.arch_config + ) num_channels = vae_config.z_dim - spatial_compression_ratio = vae_config.spatial_compression_ratio - latent_height = self.training_args.num_height // spatial_compression_ratio - latent_width = self.training_args.num_width // spatial_compression_ratio + spatial_compression_ratio = ( + vae_config.spatial_compression_ratio + ) + latent_height = ( + self.training_args.num_height + // spatial_compression_ratio + ) + latent_width = ( + self.training_args.num_width + // spatial_compression_ratio + ) latents = torch.zeros( batch_size, num_channels, @@ -265,29 +306,45 @@ def prepare_batch( ) elif latents_source == "data": if "vae_latent" not in raw_batch: - raise ValueError("vae_latent not found in batch and latents_source='data'") + raise ValueError( + "vae_latent not found in batch " + "and latents_source='data'" + ) latents = raw_batch["vae_latent"] - latents = latents[:, :, : self.training_args.num_latent_t] + latents = latents[ + :, :, : self.training_args.num_latent_t + ] latents = latents.to(device, dtype=dtype) else: - raise ValueError(f"Unknown latents_source: {latents_source!r}") + raise ValueError( + f"Unknown latents_source: {latents_source!r}" + ) training_batch.latents = latents - training_batch.encoder_hidden_states = encoder_hidden_states.to(device, dtype=dtype) - training_batch.encoder_attention_mask = encoder_attention_mask.to(device, dtype=dtype) + training_batch.encoder_hidden_states = ( + encoder_hidden_states.to(device, dtype=dtype) + ) + training_batch.encoder_attention_mask = ( + encoder_attention_mask.to(device, dtype=dtype) + ) training_batch.infos = infos - training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch.latents = normalize_dit_input( + "wan", training_batch.latents, self.vae + ) training_batch = self._prepare_dit_inputs(training_batch) - training_batch = self._build_attention_metadata(training_batch) + training_batch = self._build_attention_metadata( + training_batch + ) - training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) + training_batch.attn_metadata_vsa = copy.deepcopy( + training_batch.attn_metadata + ) if training_batch.attn_metadata is not None: training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] return training_batch - # ModelBase override: add_noise def add_noise( self, clean_latents: torch.Tensor, @@ -302,10 +359,8 @@ def add_noise( ).unflatten(0, (b, t)) return noisy - # ModelBase override: predict_x0 def predict_x0( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, @@ -319,9 +374,13 @@ def predict_x0( if conditional: text_dict = batch.conditional_dict if text_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") + raise RuntimeError( + "Missing conditional_dict in TrainingBatch" + ) else: - text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) + text_dict = self._get_uncond_text_dict( + batch, cfg_uncond=cfg_uncond + ) if attn_kind == "dense": attn_metadata = batch.attn_metadata @@ -330,13 +389,19 @@ def predict_x0( else: raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - with torch.autocast(device_type, dtype=dtype), set_forward_context( + with torch.autocast( + device_type, dtype=dtype + ), set_forward_context( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, timestep, text_dict + ) + transformer = self._get_transformer(timestep) + pred_noise = transformer(**input_kwargs).permute( + 0, 2, 1, 3, 4 + ) pred_x0 = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), noise_input_latent=noisy_latents.flatten(0, 1), @@ -345,10 +410,8 @@ def predict_x0( ).unflatten(0, pred_noise.shape[:2]) return pred_x0 - # ModelBase override: predict_noise def predict_noise( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, @@ -362,9 +425,13 @@ def predict_noise( if conditional: text_dict = batch.conditional_dict if text_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") + raise RuntimeError( + "Missing conditional_dict in TrainingBatch" + ) else: - text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) + text_dict = self._get_uncond_text_dict( + batch, cfg_uncond=cfg_uncond + ) if attn_kind == "dense": attn_metadata = batch.attn_metadata @@ -373,34 +440,62 @@ def predict_noise( else: raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - with torch.autocast(device_type, dtype=dtype), set_forward_context( + with torch.autocast( + device_type, dtype=dtype + ), set_forward_context( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): - input_kwargs = self._build_distill_input_kwargs(noisy_latents, timestep, text_dict) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + input_kwargs = self._build_distill_input_kwargs( + noisy_latents, timestep, text_dict + ) + transformer = self._get_transformer(timestep) + pred_noise = transformer(**input_kwargs).permute( + 0, 2, 1, 3, 4 + ) return pred_noise - # ModelBase override: backward - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + def backward( + self, + loss: torch.Tensor, + ctx: Any, + *, + grad_accum_rounds: int, + ) -> None: timesteps, attn_metadata = ctx - with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): + with set_forward_context( + current_timestep=timesteps, + attn_metadata=attn_metadata, + ): (loss / max(1, int(grad_accum_rounds))).backward() - # --- Wan-specific helpers below --- + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ def _get_training_dtype(self) -> torch.dtype: return torch.bfloat16 def _init_timestep_mechanics(self) -> None: - self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) - self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) - self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) - self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) + self.timestep_shift = float( + self.training_args.pipeline_config.flow_shift + ) + self.num_train_timestep = int( + self.noise_scheduler.num_train_timesteps + ) + self.min_timestep = int( + self.training_args.min_timestep_ratio + * self.num_train_timestep + ) + self.max_timestep = int( + self.training_args.max_timestep_ratio + * self.num_train_timestep + ) - boundary_ratio = getattr(self.training_args, "boundary_ratio", None) - self.boundary_timestep: float | None = ( + boundary_ratio = getattr( + self.training_args, "boundary_ratio", None + ) + self.boundary_timestep = ( float(boundary_ratio) * float(self.num_train_timestep) if boundary_ratio is not None else None @@ -419,18 +514,21 @@ def ensure_negative_conditioning(self) -> None: neg_mask: torch.Tensor | None = None if world_group.rank_in_group == 0: - sampling_param = SamplingParam.from_pretrained(training_args.model_path) + sampling_param = SamplingParam.from_pretrained( + training_args.model_path + ) negative_prompt = sampling_param.negative_prompt args_copy = copy.deepcopy(training_args) args_copy.inference_mode = True - prompt_transformer = self.prompt_handle.require_module("transformer") prompt_pipeline = WanPipeline.from_pretrained( training_args.model_path, args=args_copy, inference_mode=True, - loaded_modules={"transformer": prompt_transformer}, + loaded_modules={ + "transformer": self.transformer + }, tp_size=training_args.tp_size, sp_size=training_args.sp_size, num_gpus=training_args.num_gpus, @@ -449,8 +547,12 @@ def ensure_negative_conditioning(self) -> None: training_args, ) - neg_embeds = result_batch.prompt_embeds[0].to(device=device, dtype=dtype) - neg_mask = result_batch.prompt_attention_mask[0].to(device=device, dtype=dtype) + neg_embeds = result_batch.prompt_embeds[0].to( + device=device, dtype=dtype + ) + neg_mask = result_batch.prompt_attention_mask[0].to( + device=device, dtype=dtype + ) del prompt_pipeline gc.collect() @@ -464,29 +566,48 @@ def ensure_negative_conditioning(self) -> None: meta[0] = neg_embeds.ndim meta[1] = neg_mask.ndim world_group.broadcast(meta, src=0) - embed_ndim, mask_ndim = (int(meta[0].item()), int(meta[1].item())) + embed_ndim, mask_ndim = ( + int(meta[0].item()), + int(meta[1].item()), + ) max_ndim = 8 - embed_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) - mask_shape = torch.full((max_ndim,), -1, device=device, dtype=torch.int64) + embed_shape = torch.full( + (max_ndim,), -1, device=device, dtype=torch.int64 + ) + mask_shape = torch.full( + (max_ndim,), -1, device=device, dtype=torch.int64 + ) if world_group.rank_in_group == 0: assert neg_embeds is not None assert neg_mask is not None embed_shape[:embed_ndim] = torch.tensor( - list(neg_embeds.shape), device=device, dtype=torch.int64 + list(neg_embeds.shape), + device=device, + dtype=torch.int64, ) mask_shape[:mask_ndim] = torch.tensor( - list(neg_mask.shape), device=device, dtype=torch.int64 + list(neg_mask.shape), + device=device, + dtype=torch.int64, ) world_group.broadcast(embed_shape, src=0) world_group.broadcast(mask_shape, src=0) - embed_sizes = tuple(int(x) for x in embed_shape[:embed_ndim].tolist()) - mask_sizes = tuple(int(x) for x in mask_shape[:mask_ndim].tolist()) + embed_sizes = tuple( + int(x) for x in embed_shape[:embed_ndim].tolist() + ) + mask_sizes = tuple( + int(x) for x in mask_shape[:mask_ndim].tolist() + ) if world_group.rank_in_group != 0: - neg_embeds = torch.empty(embed_sizes, device=device, dtype=dtype) - neg_mask = torch.empty(mask_sizes, device=device, dtype=dtype) + neg_embeds = torch.empty( + embed_sizes, device=device, dtype=dtype + ) + neg_mask = torch.empty( + mask_sizes, device=device, dtype=dtype + ) assert neg_embeds is not None assert neg_mask is not None @@ -496,9 +617,14 @@ def ensure_negative_conditioning(self) -> None: self.negative_prompt_embeds = neg_embeds self.negative_prompt_attention_mask = neg_mask - def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: + def _sample_timesteps( + self, batch_size: int, device: torch.device + ) -> torch.Tensor: if self.noise_random_generator is None: - raise RuntimeError("WanAdapter.on_train_start() must be called before prepare_batch()") + raise RuntimeError( + "on_train_start() must be called before " + "prepare_batch()" + ) u = compute_density_for_timestep_sampling( weighting_scheme=self.training_args.weighting_scheme, @@ -508,20 +634,33 @@ def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tens logit_std=self.training_args.logit_std, mode_scale=self.training_args.mode_scale, ) - indices = (u * self.noise_scheduler.config.num_train_timesteps).long() - return self.noise_scheduler.timesteps[indices].to(device=device) + indices = ( + u * self.noise_scheduler.config.num_train_timesteps + ).long() + return self.noise_scheduler.timesteps[indices].to( + device=device + ) - def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: + def _build_attention_metadata( + self, training_batch: TrainingBatch + ) -> TrainingBatch: latents_shape = training_batch.raw_latent_shape - patch_size = self.training_args.pipeline_config.dit_config.patch_size + patch_size = ( + self.training_args.pipeline_config.dit_config + .patch_size + ) current_vsa_sparsity = training_batch.current_vsa_sparsity assert latents_shape is not None assert training_batch.timesteps is not None if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": - if not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None: + if ( + not is_vsa_available() + or VideoSparseAttentionMetadataBuilder is None + ): raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VIDEO_SPARSE_ATTN, but fastvideo_kernel " + "FASTVIDEO_ATTENTION_BACKEND is " + "VIDEO_SPARSE_ATTN, but fastvideo_kernel " "is not correctly installed or detected." ) training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] @@ -532,16 +671,22 @@ def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBa device=self.device, ) elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": - if not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None: + if ( + not is_vmoba_available() + or VideoMobaAttentionMetadataBuilder is None + ): raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, but fastvideo_kernel " - "(or flash_attn>=2.7.4) is not correctly installed." + "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, " + "but fastvideo_kernel (or flash_attn>=2.7.4) " + "is not correctly installed." ) moba_params = self.training_args.moba_config.copy() moba_params.update( { "current_timestep": training_batch.timesteps, - "raw_latent_shape": training_batch.raw_latent_shape[2:5], + "raw_latent_shape": ( + training_batch.raw_latent_shape[2:5] + ), "patch_size": patch_size, "device": self.device, } @@ -552,13 +697,18 @@ def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBa return training_batch - def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: + def _prepare_dit_inputs( + self, training_batch: TrainingBatch + ) -> TrainingBatch: latents = training_batch.latents assert isinstance(latents, torch.Tensor) batch_size = latents.shape[0] if self.noise_gen_cuda is None: - raise RuntimeError("WanAdapter.on_train_start() must be called before prepare_batch()") + raise RuntimeError( + "on_train_start() must be called before " + "prepare_batch()" + ) noise = torch.randn( latents.shape, @@ -566,8 +716,13 @@ def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: device=latents.device, dtype=latents.dtype, ) - timesteps = self._sample_timesteps(batch_size, latents.device) - if int(getattr(self.training_args, "sp_size", 1) or 1) > 1: + timesteps = self._sample_timesteps( + batch_size, latents.device + ) + if ( + int(getattr(self.training_args, "sp_size", 1) or 1) + > 1 + ): self.sp_group.broadcast(timesteps, src=0) sigmas = get_sigmas( @@ -577,7 +732,9 @@ def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: n_dim=latents.ndim, dtype=latents.dtype, ) - noisy_model_input = (1.0 - sigmas) * latents + sigmas * noise + noisy_model_input = ( + (1.0 - sigmas) * latents + sigmas * noise + ) training_batch.noisy_model_input = noisy_model_input training_batch.timesteps = timesteps @@ -586,8 +743,12 @@ def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: training_batch.raw_latent_shape = latents.shape training_batch.conditional_dict = { - "encoder_hidden_states": training_batch.encoder_hidden_states, - "encoder_attention_mask": training_batch.encoder_attention_mask, + "encoder_hidden_states": ( + training_batch.encoder_hidden_states + ), + "encoder_attention_mask": ( + training_batch.encoder_attention_mask + ), } if ( @@ -597,15 +758,21 @@ def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: neg_embeds = self.negative_prompt_embeds neg_mask = self.negative_prompt_attention_mask if neg_embeds.shape[0] == 1 and batch_size > 1: - neg_embeds = neg_embeds.expand(batch_size, *neg_embeds.shape[1:]).contiguous() + neg_embeds = neg_embeds.expand( + batch_size, *neg_embeds.shape[1:] + ).contiguous() if neg_mask.shape[0] == 1 and batch_size > 1: - neg_mask = neg_mask.expand(batch_size, *neg_mask.shape[1:]).contiguous() + neg_mask = neg_mask.expand( + batch_size, *neg_mask.shape[1:] + ).contiguous() training_batch.unconditional_dict = { "encoder_hidden_states": neg_embeds, "encoder_attention_mask": neg_mask, } - training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) + training_batch.latents = training_batch.latents.permute( + 0, 2, 1, 3, 4 + ) return training_batch def _build_distill_input_kwargs( @@ -615,25 +782,34 @@ def _build_distill_input_kwargs( text_dict: dict[str, torch.Tensor] | None, ) -> dict[str, Any]: if text_dict is None: - raise ValueError("text_dict cannot be None for Wan distillation") + raise ValueError( + "text_dict cannot be None for Wan distillation" + ) return { - "hidden_states": noise_input.permute(0, 2, 1, 3, 4), - "encoder_hidden_states": text_dict["encoder_hidden_states"], - "encoder_attention_mask": text_dict["encoder_attention_mask"], + "hidden_states": noise_input.permute( + 0, 2, 1, 3, 4 + ), + "encoder_hidden_states": text_dict[ + "encoder_hidden_states" + ], + "encoder_attention_mask": text_dict[ + "encoder_attention_mask" + ], "timestep": timestep, "return_dict": False, } - def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: - transformer = handle.require_module("transformer") - transformer_2 = handle.modules.get("transformer_2") + def _get_transformer( + self, timestep: torch.Tensor + ) -> torch.nn.Module: if ( - transformer_2 is not None + self.transformer_2 is not None and self.boundary_timestep is not None - and float(timestep.item()) < float(self.boundary_timestep) + and float(timestep.item()) + < float(self.boundary_timestep) ): - return transformer_2 - return transformer + return self.transformer_2 + return self.transformer def _get_uncond_text_dict( self, @@ -642,28 +818,32 @@ def _get_uncond_text_dict( cfg_uncond: dict[str, Any] | None, ) -> dict[str, torch.Tensor]: if cfg_uncond is None: - text_dict = getattr(batch, "unconditional_dict", None) + text_dict = getattr( + batch, "unconditional_dict", None + ) if text_dict is None: raise RuntimeError( - "Missing unconditional_dict; ensure_negative_conditioning() may have failed" + "Missing unconditional_dict; " + "ensure_negative_conditioning() " + "may have failed" ) return text_dict on_missing_raw = cfg_uncond.get("on_missing", "error") if not isinstance(on_missing_raw, str): raise ValueError( - "method_config.cfg_uncond.on_missing must be a string, got " + "method_config.cfg_uncond.on_missing must be " + "a string, got " f"{type(on_missing_raw).__name__}" ) on_missing = on_missing_raw.strip().lower() if on_missing not in {"error", "ignore"}: raise ValueError( - "method_config.cfg_uncond.on_missing must be one of {error, ignore}, got " + "method_config.cfg_uncond.on_missing must be " + "one of {error, ignore}, got " f"{on_missing_raw!r}" ) - # Wan only supports text CFG. If users configure other channels, fail - # fast (unless explicitly ignored). for channel, policy_raw in cfg_uncond.items(): if channel in {"on_missing", "text"}: continue @@ -671,7 +851,8 @@ def _get_uncond_text_dict( continue if not isinstance(policy_raw, str): raise ValueError( - "method_config.cfg_uncond values must be strings, got " + "method_config.cfg_uncond values must be " + "strings, got " f"{channel}={type(policy_raw).__name__}" ) policy = policy_raw.strip().lower() @@ -680,9 +861,10 @@ def _get_uncond_text_dict( if on_missing == "ignore": continue raise ValueError( - "WanAdapter does not support cfg_uncond channel " + "WanModel does not support cfg_uncond channel " f"{channel!r} (policy={policy!r}). " - "Set cfg_uncond.on_missing=ignore or remove the channel." + "Set cfg_uncond.on_missing=ignore or remove " + "the channel." ) text_policy_raw = cfg_uncond.get("text", None) @@ -690,41 +872,56 @@ def _get_uncond_text_dict( text_policy = "negative_prompt" elif not isinstance(text_policy_raw, str): raise ValueError( - "method_config.cfg_uncond.text must be a string, got " + "method_config.cfg_uncond.text must be a " + "string, got " f"{type(text_policy_raw).__name__}" ) else: text_policy = text_policy_raw.strip().lower() if text_policy in {"negative_prompt"}: - text_dict = getattr(batch, "unconditional_dict", None) + text_dict = getattr( + batch, "unconditional_dict", None + ) if text_dict is None: raise RuntimeError( - "Missing unconditional_dict; ensure_negative_conditioning() may have failed" + "Missing unconditional_dict; " + "ensure_negative_conditioning() " + "may have failed" ) return text_dict if text_policy == "keep": if batch.conditional_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") + raise RuntimeError( + "Missing conditional_dict in TrainingBatch" + ) return batch.conditional_dict if text_policy == "zero": if batch.conditional_dict is None: - raise RuntimeError("Missing conditional_dict in TrainingBatch") + raise RuntimeError( + "Missing conditional_dict in TrainingBatch" + ) cond = batch.conditional_dict enc = cond["encoder_hidden_states"] mask = cond["encoder_attention_mask"] - if not torch.is_tensor(enc) or not torch.is_tensor(mask): - raise TypeError("conditional_dict must contain tensor text inputs") + if not torch.is_tensor(enc) or not torch.is_tensor( + mask + ): + raise TypeError( + "conditional_dict must contain " + "tensor text inputs" + ) return { "encoder_hidden_states": torch.zeros_like(enc), "encoder_attention_mask": torch.zeros_like(mask), } if text_policy == "drop": raise ValueError( - "cfg_uncond.text=drop is not supported for Wan. " - "Use {negative_prompt, keep, zero}." + "cfg_uncond.text=drop is not supported for Wan." + " Use {negative_prompt, keep, zero}." ) raise ValueError( - "cfg_uncond.text must be one of {negative_prompt, keep, zero, drop}, got " + "cfg_uncond.text must be one of " + "{negative_prompt, keep, zero, drop}, got " f"{text_policy_raw!r}" ) diff --git a/fastvideo/distillation/models/wangame/__init__.py b/fastvideo/distillation/models/wangame/__init__.py index e05c01e0f..bcbd06ad4 100644 --- a/fastvideo/distillation/models/wangame/__init__.py +++ b/fastvideo/distillation/models/wangame/__init__.py @@ -1,27 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 -"""WanGame model plugin package. - -This package registers the WanGame model builders: -- `recipe.family: wangame` (bidirectional) -- `recipe.family: wangame_causal` (causal-capable; supports streaming primitives) -""" - -from __future__ import annotations - -from fastvideo.distillation.dispatch import register_model -from fastvideo.distillation.utils.config import DistillRunConfig - - -@register_model("wangame") -def _build_wangame_model(*, cfg: DistillRunConfig): - from fastvideo.distillation.models.wangame.wangame import WanGameModel - - return WanGameModel(cfg=cfg) - - -@register_model("wangame_causal") -def _build_wangame_causal_model(*, cfg: DistillRunConfig): - from fastvideo.distillation.models.wangame.wangame_causal import WanGameCausalModel - - return WanGameCausalModel(cfg=cfg) +"""WanGame model plugin package.""" + +from fastvideo.distillation.models.wangame.wangame import ( + WanGameModel as WanGameModel, +) +from fastvideo.distillation.models.wangame.wangame_causal import ( + WanGameCausalModel as WanGameCausalModel, +) diff --git a/fastvideo/distillation/models/wangame/common.py b/fastvideo/distillation/models/wangame/common.py deleted file mode 100644 index 2d482fb4e..000000000 --- a/fastvideo/distillation/models/wangame/common.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -"""WanGame shared helpers. - -This module hosts "family-level" helpers used by both bidirectional and causal -WanGame model plugins (e.g. role-handle assembly). - -The goal is to avoid duplicated logic across: -- `fastvideo/distillation/models/wangame/wangame.py` -- `fastvideo/distillation/models/wangame/wangame_causal.py` - -Keep this file free of imports of the family-specific model classes to prevent -circular dependencies. -""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -import torch - -from fastvideo.training.activation_checkpoint import apply_activation_checkpointing - -from fastvideo.distillation.roles import RoleHandle -from fastvideo.distillation.utils.module_state import apply_trainable -from fastvideo.distillation.utils.moduleloader import load_module_from_path - - -def _build_wangame_role_handles( - *, - roles_cfg: dict[str, Any], - training_args: Any, - transformer_cls_name_for_role: Callable[[str, Any], str], -) -> dict[str, RoleHandle]: - role_handles: dict[str, RoleHandle] = {} - for role, role_spec in roles_cfg.items(): - if role_spec.family not in {"wangame", "wangame_causal"}: - raise ValueError( - "Wangame model plugin only supports roles with family in " - "{'wangame', 'wangame_causal'}; " - f"got {role}={role_spec.family!r}" - ) - - transformer_cls_name = transformer_cls_name_for_role(str(role), role_spec) - disable_custom_init_weights = bool( - getattr(role_spec, "disable_custom_init_weights", False) - ) - transformer = load_module_from_path( - model_path=role_spec.path, - module_type="transformer", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - override_transformer_cls_name=transformer_cls_name, - ) - modules: dict[str, torch.nn.Module] = {"transformer": transformer} - - # Optional MoE support: load transformer_2 if present in the model. - try: - transformer_2 = load_module_from_path( - model_path=role_spec.path, - module_type="transformer_2", - training_args=training_args, - disable_custom_init_weights=disable_custom_init_weights, - ) - except ValueError: - transformer_2 = None - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 - - for name, module in list(modules.items()): - module = apply_trainable(module, trainable=bool(role_spec.trainable)) - if bool(role_spec.trainable) and bool( - getattr(training_args, "enable_gradient_checkpointing_type", None) - ): - module = apply_activation_checkpointing( - module, - checkpointing_type=training_args.enable_gradient_checkpointing_type, - ) - modules[name] = module - - role_handles[str(role)] = RoleHandle( - modules=modules, - optimizers={}, - lr_schedulers={}, - trainable=bool(role_spec.trainable), - ) - - return role_handles diff --git a/fastvideo/distillation/models/wangame/wangame.py b/fastvideo/distillation/models/wangame/wangame.py index 54198dc61..6080e48f0 100644 --- a/fastvideo/distillation/models/wangame/wangame.py +++ b/fastvideo/distillation/models/wangame/wangame.py @@ -1,23 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 -"""WanGame bidirectional model plugin (components + runtime adapter). - -Config keys used (YAML schema-v2): -- `recipe.family`: must be `"wangame"` for this plugin. -- `roles.shared_component_role` (affects default `training.model_path`) -- `roles.`: - - `family`, `path`, `trainable`, `disable_custom_init_weights` -- `training` (selected fields): - - `seed`, `data_path`, `model_path` - - `num_height`, `num_width`, `num_latent_t` - - `min_timestep_ratio`, `max_timestep_ratio`, `boundary_ratio` (optional) - - `weighting_scheme`, `logit_mean`, `logit_std`, `mode_scale` - - `sp_size`, `tp_size`, `num_gpus` - - `pipeline_config.flow_shift`, `pipeline_config.dit_config.patch_size` - - `moba_config` (if `FASTVIDEO_ATTENTION_BACKEND=VMOBA_ATTN`) - - `enable_gradient_checkpointing_type` (optional) -- `training.validation.*` (consumed by `WanGameValidator` when enabled) -- `method_config.cfg_uncond.*` (consumed by `WanGameModel` for CFG-uncond policy) +"""WanGame bidirectional model plugin (per-role instance). + +Each role (student, teacher, critic) gets its own ``WanGameModel`` instance. +The constructor loads the transformer and noise scheduler. Heavyweight +resources (VAE, dataloader, RNG seeds) are loaded via +:meth:`init_preprocessors`, which the method calls only on the student. """ from __future__ import annotations @@ -39,107 +27,182 @@ ) from fastvideo.models.utils import pred_noise_to_pred_video from fastvideo.pipelines import TrainingBatch +from fastvideo.training.activation_checkpoint import ( + apply_activation_checkpointing, +) from fastvideo.training.training_utils import ( compute_density_for_timestep_sampling, get_sigmas, normalize_dit_input, shift_timestep, ) -from fastvideo.utils import is_vmoba_available, is_vsa_available, set_random_seed +from fastvideo.utils import ( + is_vmoba_available, + is_vsa_available, + set_random_seed, +) -from fastvideo.distillation.models.wangame.common import _build_wangame_role_handles from fastvideo.distillation.models.base import ModelBase -from fastvideo.distillation.roles import RoleHandle, RoleManager -from fastvideo.distillation.utils.config import DistillRunConfig -from fastvideo.distillation.utils.dataloader import build_parquet_wangame_train_dataloader +from fastvideo.distillation.utils.module_state import apply_trainable from fastvideo.distillation.utils.moduleloader import load_module_from_path try: from fastvideo.attention.backends.video_sparse_attn import ( VideoSparseAttentionMetadataBuilder, ) - from fastvideo.attention.backends.vmoba import VideoMobaAttentionMetadataBuilder + from fastvideo.attention.backends.vmoba import ( + VideoMobaAttentionMetadataBuilder, + ) except Exception: VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] class WanGameModel(ModelBase): - """WanGame model plugin: loads roles + shared components and exposes runtime primitives. + """WanGame per-role model: owns transformer + noise_scheduler. - This model plugin is intentionally *not* method-specific: - - It knows how to turn a wangame parquet batch into forward primitives. - - It knows how to run a model role (handle) for predict_noise / predict_x0. - - It does not encode DMD2/SFT/etc semantics beyond required primitives. + Constructor loads the transformer from *init_from*. VAE, dataloader, + and RNG are deferred to :meth:`init_preprocessors`. """ + # Name of the transformer class to load. Subclasses may override. + _transformer_cls_name: str = "WanGameActionTransformer3DModel" + def __init__( self, *, - cfg: DistillRunConfig, + init_from: str, + trainable: bool = True, + disable_custom_init_weights: bool = False, + flow_shift: float = 3.0, + enable_gradient_checkpointing_type: str | None = None, ) -> None: - training_args = cfg.training_args - roles_cfg = cfg.roles + self._init_from = str(init_from) + self._trainable = bool(trainable) + + # We need a minimal TrainingArgs-like object just for loading. + # The full training_args arrives via init_preprocessors(). + self.transformer = self._load_transformer( + init_from=self._init_from, + trainable=self._trainable, + disable_custom_init_weights=disable_custom_init_weights, + enable_gradient_checkpointing_type=( + enable_gradient_checkpointing_type + ), + ) - non_wangame_roles = [ - role - for role, spec in roles_cfg.items() - if str(spec.family).strip().lower() != "wangame" - ] - if non_wangame_roles: - raise ValueError( - "recipe.family=wangame only supports roles..family=wangame " - f"(got non-wangame roles: {non_wangame_roles}). " - "If you need causal roles, use recipe.family: wangame_causal." + # Optional MoE transformer_2. + self.transformer_2: torch.nn.Module | None = None + try: + t2 = load_module_from_path( + model_path=self._init_from, + module_type="transformer_2", + training_args=None, + disable_custom_init_weights=disable_custom_init_weights, ) + except (ValueError, FileNotFoundError): + t2 = None + if t2 is not None: + t2 = apply_trainable(t2, trainable=self._trainable) + if self._trainable and enable_gradient_checkpointing_type: + t2 = apply_activation_checkpointing( + t2, + checkpointing_type=enable_gradient_checkpointing_type, + ) + self.transformer_2 = t2 - if getattr(training_args, "seed", None) is None: - raise ValueError("training.seed must be set for distillation") - if not getattr(training_args, "data_path", ""): - raise ValueError("training.data_path must be set for distillation") - - # Load shared components (training.model_path; defaults to - # roles.shared_component_role's path). - vae = load_module_from_path( - model_path=str(training_args.model_path), - module_type="vae", - training_args=training_args, - ) - noise_scheduler = FlowMatchEulerDiscreteScheduler( - shift=float(training_args.pipeline_config.flow_shift or 0.0) + self.noise_scheduler = FlowMatchEulerDiscreteScheduler( + shift=float(flow_shift) ) - role_handles = _build_wangame_role_handles( - roles_cfg=roles_cfg, - training_args=training_args, - transformer_cls_name_for_role=lambda _role, _spec: "WanGameActionTransformer3DModel", - ) + # Filled by init_preprocessors (student only). + self.vae: Any = None + self.training_args: Any = None + self.dataloader: Any = None + self.validator: Any = None + self.start_step: int = 0 - self.bundle = RoleManager(roles=role_handles) + self.world_group: Any = None + self.sp_group: Any = None + self.device: Any = None - # Optional validator. - self.validator = None - validation_cfg = getattr(cfg, "validation", {}) or {} - validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) - if validation_enabled: - from fastvideo.distillation.validators.wangame import WanGameValidator + self.noise_random_generator: torch.Generator | None = None + self.noise_gen_cuda: torch.Generator | None = None - self.validator = WanGameValidator(training_args=training_args) + # Timestep mechanics (set after init_preprocessors or eagerly). + self.timestep_shift: float = float(flow_shift) + self.num_train_timestep: int = int( + self.noise_scheduler.num_train_timesteps + ) + self.min_timestep: int = 0 + self.max_timestep: int = self.num_train_timestep + self.boundary_timestep: float | None = None + def _load_transformer( + self, + *, + init_from: str, + trainable: bool, + disable_custom_init_weights: bool, + enable_gradient_checkpointing_type: str | None, + ) -> torch.nn.Module: + transformer = load_module_from_path( + model_path=init_from, + module_type="transformer", + training_args=None, + disable_custom_init_weights=disable_custom_init_weights, + override_transformer_cls_name=self._transformer_cls_name, + ) + transformer = apply_trainable(transformer, trainable=trainable) + if trainable and enable_gradient_checkpointing_type: + transformer = apply_activation_checkpointing( + transformer, + checkpointing_type=enable_gradient_checkpointing_type, + ) + return transformer + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def init_preprocessors(self, training_args: Any) -> None: + """Load VAE, build dataloader, seed RNGs. Student only.""" self.training_args = training_args - self.noise_scheduler = noise_scheduler - self.vae = vae + + self.vae = load_module_from_path( + model_path=str(training_args.model_path), + module_type="vae", + training_args=training_args, + ) self.world_group = get_world_group() self.sp_group = get_sp_group() self.device = get_local_torch_device() - self.noise_random_generator: torch.Generator | None = None - self.noise_gen_cuda: torch.Generator | None = None - self._init_timestep_mechanics() - from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame + # Optional validator. + validation_cfg = getattr(training_args, "_validation_cfg", None) + if validation_cfg: + validation_enabled = bool( + validation_cfg.get( + "enabled", bool(validation_cfg) + ) + ) + if validation_enabled: + from fastvideo.distillation.validators.wangame import ( + WanGameValidator, + ) + self.validator = WanGameValidator( + training_args=training_args + ) + + from fastvideo.dataset.dataloader.schema import ( + pyarrow_schema_wangame, + ) + from fastvideo.distillation.utils.dataloader import ( + build_parquet_wangame_train_dataloader, + ) self.dataloader = build_parquet_wangame_train_dataloader( training_args, @@ -147,37 +210,51 @@ def __init__( ) self.start_step = 0 - # ModelBase override: num_train_timesteps + # ------------------------------------------------------------------ + # ModelBase overrides: timestep helpers + # ------------------------------------------------------------------ + @property def num_train_timesteps(self) -> int: return int(self.num_train_timestep) - # ModelBase override: shift_and_clamp_timestep - def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: - timestep = shift_timestep(timestep, self.timestep_shift, self.num_train_timestep) + def shift_and_clamp_timestep( + self, timestep: torch.Tensor + ) -> torch.Tensor: + timestep = shift_timestep( + timestep, self.timestep_shift, self.num_train_timestep + ) return timestep.clamp(self.min_timestep, self.max_timestep) - # ModelBase override: on_train_start + # ------------------------------------------------------------------ + # ModelBase overrides: lifecycle hooks + # ------------------------------------------------------------------ + def on_train_start(self) -> None: seed = getattr(self.training_args, "seed", None) if seed is None: - raise ValueError("training_args.seed must be set for distillation") + raise ValueError( + "training_args.seed must be set for distillation" + ) global_rank = int(getattr(self.world_group, "rank", 0)) - sp_world_size = int(getattr(self.training_args, "sp_size", 1) or 1) + sp_world_size = int( + getattr(self.training_args, "sp_size", 1) or 1 + ) if sp_world_size > 1: sp_group_seed = int(seed) + (global_rank // sp_world_size) set_random_seed(sp_group_seed) else: set_random_seed(int(seed) + global_rank) - self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) - self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) + self.noise_random_generator = torch.Generator( + device="cpu" + ).manual_seed(int(seed)) + self.noise_gen_cuda = torch.Generator( + device=self.device + ).manual_seed(int(seed)) - # ModelBase override: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: - """Return RNG generators that should be checkpointed for exact resume.""" - generators: dict[str, torch.Generator] = {} if self.noise_random_generator is not None: generators["noise_cpu"] = self.noise_random_generator @@ -185,7 +262,10 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: generators["noise_cuda"] = self.noise_gen_cuda return generators - # ModelBase override: prepare_batch + # ------------------------------------------------------------------ + # ModelBase overrides: runtime primitives + # ------------------------------------------------------------------ + def prepare_batch( self, raw_batch: dict[str, Any], @@ -193,22 +273,33 @@ def prepare_batch( current_vsa_sparsity: float = 0.0, latents_source: Literal["data", "zeros"] = "data", ) -> TrainingBatch: - """Convert a dataloader batch into forward primitives for methods.""" - dtype = self._get_training_dtype() device = self.device - training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + training_batch = TrainingBatch( + current_vsa_sparsity=current_vsa_sparsity + ) infos = raw_batch.get("info_list") if latents_source == "zeros": clip_feature = raw_batch["clip_feature"] batch_size = int(clip_feature.shape[0]) - vae_config = self.training_args.pipeline_config.vae_config.arch_config + vae_config = ( + self.training_args.pipeline_config + .vae_config.arch_config + ) num_channels = int(vae_config.z_dim) - spatial_compression_ratio = int(vae_config.spatial_compression_ratio) - latent_height = int(self.training_args.num_height) // spatial_compression_ratio - latent_width = int(self.training_args.num_width) // spatial_compression_ratio + spatial_compression_ratio = int( + vae_config.spatial_compression_ratio + ) + latent_height = ( + int(self.training_args.num_height) + // spatial_compression_ratio + ) + latent_width = ( + int(self.training_args.num_width) + // spatial_compression_ratio + ) latents = torch.zeros( batch_size, num_channels, @@ -220,58 +311,96 @@ def prepare_batch( ) elif latents_source == "data": if "vae_latent" not in raw_batch: - raise ValueError("vae_latent not found in batch and latents_source='data'") + raise ValueError( + "vae_latent not found in batch " + "and latents_source='data'" + ) latents = raw_batch["vae_latent"] - latents = latents[:, :, : self.training_args.num_latent_t] + latents = latents[ + :, :, : self.training_args.num_latent_t + ] latents = latents.to(device, dtype=dtype) else: - raise ValueError(f"Unknown latents_source: {latents_source!r}") + raise ValueError( + f"Unknown latents_source: {latents_source!r}" + ) if "clip_feature" not in raw_batch: - raise ValueError("clip_feature must be present for WanGame") - image_embeds = raw_batch["clip_feature"].to(device, dtype=dtype) + raise ValueError( + "clip_feature must be present for WanGame" + ) + image_embeds = raw_batch["clip_feature"].to( + device, dtype=dtype + ) if "first_frame_latent" not in raw_batch: - raise ValueError("first_frame_latent must be present for WanGame") + raise ValueError( + "first_frame_latent must be present for WanGame" + ) image_latents = raw_batch["first_frame_latent"] - image_latents = image_latents[:, :, : self.training_args.num_latent_t] + image_latents = image_latents[ + :, :, : self.training_args.num_latent_t + ] image_latents = image_latents.to(device, dtype=dtype) pil_image = raw_batch.get("pil_image") if isinstance(pil_image, torch.Tensor): - training_batch.preprocessed_image = pil_image.to(device=device) + training_batch.preprocessed_image = pil_image.to( + device=device + ) else: training_batch.preprocessed_image = pil_image keyboard_cond = raw_batch.get("keyboard_cond") - if isinstance(keyboard_cond, torch.Tensor) and keyboard_cond.numel() > 0: - training_batch.keyboard_cond = keyboard_cond.to(device, dtype=dtype) + if ( + isinstance(keyboard_cond, torch.Tensor) + and keyboard_cond.numel() > 0 + ): + training_batch.keyboard_cond = keyboard_cond.to( + device, dtype=dtype + ) else: training_batch.keyboard_cond = None mouse_cond = raw_batch.get("mouse_cond") - if isinstance(mouse_cond, torch.Tensor) and mouse_cond.numel() > 0: - training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) + if ( + isinstance(mouse_cond, torch.Tensor) + and mouse_cond.numel() > 0 + ): + training_batch.mouse_cond = mouse_cond.to( + device, dtype=dtype + ) else: training_batch.mouse_cond = None temporal_compression_ratio = ( - self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + self.training_args.pipeline_config + .vae_config.arch_config.temporal_compression_ratio ) - expected_num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 - if training_batch.keyboard_cond is not None and int( - training_batch.keyboard_cond.shape[1] - ) != int(expected_num_frames): + expected_num_frames = ( + (self.training_args.num_latent_t - 1) + * temporal_compression_ratio + + 1 + ) + if ( + training_batch.keyboard_cond is not None + and int(training_batch.keyboard_cond.shape[1]) + != int(expected_num_frames) + ): raise ValueError( "keyboard_cond temporal dim mismatch: " - f"got {int(training_batch.keyboard_cond.shape[1])}, expected {int(expected_num_frames)}" + f"got {int(training_batch.keyboard_cond.shape[1])}, " + f"expected {int(expected_num_frames)}" ) - if training_batch.mouse_cond is not None and int(training_batch.mouse_cond.shape[1]) != int( - expected_num_frames + if ( + training_batch.mouse_cond is not None + and int(training_batch.mouse_cond.shape[1]) + != int(expected_num_frames) ): raise ValueError( "mouse_cond temporal dim mismatch: " - f"got {int(training_batch.mouse_cond.shape[1])}, expected {int(expected_num_frames)}" + f"got {int(training_batch.mouse_cond.shape[1])}, " + f"expected {int(expected_num_frames)}" ) training_batch.latents = latents @@ -281,23 +410,30 @@ def prepare_batch( training_batch.image_latents = image_latents training_batch.infos = infos - training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch.latents = normalize_dit_input( + "wan", training_batch.latents, self.vae + ) training_batch = self._prepare_dit_inputs(training_batch) training_batch = self._build_attention_metadata(training_batch) - training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) + training_batch.attn_metadata_vsa = copy.deepcopy( + training_batch.attn_metadata + ) if training_batch.attn_metadata is not None: training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] - training_batch.mask_lat_size = self._build_i2v_mask_latents(image_latents) - viewmats, intrinsics, action_labels = self._process_actions(training_batch) + training_batch.mask_lat_size = self._build_i2v_mask_latents( + image_latents + ) + viewmats, intrinsics, action_labels = self._process_actions( + training_batch + ) training_batch.viewmats = viewmats training_batch.Ks = intrinsics training_batch.action = action_labels return training_batch - # ModelBase override: add_noise def add_noise( self, clean_latents: torch.Tensor, @@ -312,10 +448,8 @@ def add_noise( ).unflatten(0, (b, t)) return noisy - # ModelBase override: predict_x0 def predict_x0( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, @@ -334,7 +468,9 @@ def predict_x0( else: raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - with torch.autocast(device_type, dtype=dtype), set_forward_context( + with torch.autocast( + device_type, dtype=dtype + ), set_forward_context( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): @@ -355,8 +491,10 @@ def predict_x0( mouse_cond=cond_inputs["mouse_cond"], keyboard_cond=cond_inputs["keyboard_cond"], ) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + transformer = self._get_transformer(timestep) + pred_noise = transformer(**input_kwargs).permute( + 0, 2, 1, 3, 4 + ) pred_x0 = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), noise_input_latent=noisy_latents.flatten(0, 1), @@ -365,10 +503,8 @@ def predict_x0( ).unflatten(0, pred_noise.shape[:2]) return pred_x0 - # ModelBase override: predict_noise def predict_noise( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: TrainingBatch, @@ -387,7 +523,9 @@ def predict_noise( else: raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - with torch.autocast(device_type, dtype=dtype), set_forward_context( + with torch.autocast( + device_type, dtype=dtype + ), set_forward_context( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): @@ -408,38 +546,64 @@ def predict_noise( mouse_cond=cond_inputs["mouse_cond"], keyboard_cond=cond_inputs["keyboard_cond"], ) - transformer = self._get_transformer(handle, timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + transformer = self._get_transformer(timestep) + pred_noise = transformer(**input_kwargs).permute( + 0, 2, 1, 3, 4 + ) return pred_noise - # ModelBase override: backward - def backward(self, loss: torch.Tensor, ctx: Any, *, grad_accum_rounds: int) -> None: + def backward( + self, + loss: torch.Tensor, + ctx: Any, + *, + grad_accum_rounds: int, + ) -> None: timesteps, attn_metadata = ctx - with set_forward_context(current_timestep=timesteps, attn_metadata=attn_metadata): + with set_forward_context( + current_timestep=timesteps, attn_metadata=attn_metadata + ): (loss / max(1, int(grad_accum_rounds))).backward() - # --- WanGame-specific helpers below --- + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ def _get_training_dtype(self) -> torch.dtype: return torch.bfloat16 def _init_timestep_mechanics(self) -> None: - self.timestep_shift = float(self.training_args.pipeline_config.flow_shift) - self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) - self.min_timestep = int(self.training_args.min_timestep_ratio * self.num_train_timestep) - self.max_timestep = int(self.training_args.max_timestep_ratio * self.num_train_timestep) + self.timestep_shift = float( + self.training_args.pipeline_config.flow_shift + ) + self.num_train_timestep = int( + self.noise_scheduler.num_train_timesteps + ) + self.min_timestep = int( + self.training_args.min_timestep_ratio + * self.num_train_timestep + ) + self.max_timestep = int( + self.training_args.max_timestep_ratio + * self.num_train_timestep + ) - boundary_ratio = getattr(self.training_args, "boundary_ratio", None) - self.boundary_timestep: float | None = ( + boundary_ratio = getattr( + self.training_args, "boundary_ratio", None + ) + self.boundary_timestep = ( float(boundary_ratio) * float(self.num_train_timestep) if boundary_ratio is not None else None ) - def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: + def _sample_timesteps( + self, batch_size: int, device: torch.device + ) -> torch.Tensor: if self.noise_random_generator is None: raise RuntimeError( - "WanGameModel.on_train_start() must be called before prepare_batch()" + "on_train_start() must be called before " + "prepare_batch()" ) u = compute_density_for_timestep_sampling( @@ -450,20 +614,32 @@ def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tens logit_std=self.training_args.logit_std, mode_scale=self.training_args.mode_scale, ) - indices = (u * self.noise_scheduler.config.num_train_timesteps).long() - return self.noise_scheduler.timesteps[indices].to(device=device) + indices = ( + u * self.noise_scheduler.config.num_train_timesteps + ).long() + return self.noise_scheduler.timesteps[indices].to( + device=device + ) - def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: + def _build_attention_metadata( + self, training_batch: TrainingBatch + ) -> TrainingBatch: latents_shape = training_batch.raw_latent_shape - patch_size = self.training_args.pipeline_config.dit_config.patch_size + patch_size = ( + self.training_args.pipeline_config.dit_config.patch_size + ) current_vsa_sparsity = training_batch.current_vsa_sparsity assert latents_shape is not None assert training_batch.timesteps is not None if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": - if not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None: + if ( + not is_vsa_available() + or VideoSparseAttentionMetadataBuilder is None + ): raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VIDEO_SPARSE_ATTN, but fastvideo_kernel " + "FASTVIDEO_ATTENTION_BACKEND is " + "VIDEO_SPARSE_ATTN, but fastvideo_kernel " "is not correctly installed or detected." ) training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] @@ -474,16 +650,22 @@ def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBa device=self.device, ) elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": - if not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None: + if ( + not is_vmoba_available() + or VideoMobaAttentionMetadataBuilder is None + ): raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, but fastvideo_kernel " - "(or flash_attn>=2.7.4) is not correctly installed." + "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, " + "but fastvideo_kernel (or flash_attn>=2.7.4) " + "is not correctly installed." ) moba_params = self.training_args.moba_config.copy() moba_params.update( { "current_timestep": training_batch.timesteps, - "raw_latent_shape": training_batch.raw_latent_shape[2:5], + "raw_latent_shape": ( + training_batch.raw_latent_shape[2:5] + ), "patch_size": patch_size, "device": self.device, } @@ -494,14 +676,17 @@ def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBa return training_batch - def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: + def _prepare_dit_inputs( + self, training_batch: TrainingBatch + ) -> TrainingBatch: latents = training_batch.latents assert isinstance(latents, torch.Tensor) batch_size = latents.shape[0] if self.noise_gen_cuda is None: raise RuntimeError( - "WanGameModel.on_train_start() must be called before prepare_batch()" + "on_train_start() must be called before " + "prepare_batch()" ) noise = torch.randn( @@ -510,8 +695,12 @@ def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: device=latents.device, dtype=latents.dtype, ) - timesteps = self._sample_timesteps(batch_size, latents.device) - if int(getattr(self.training_args, "sp_size", 1) or 1) > 1: + timesteps = self._sample_timesteps( + batch_size, latents.device + ) + if ( + int(getattr(self.training_args, "sp_size", 1) or 1) > 1 + ): self.sp_group.broadcast(timesteps, src=0) sigmas = get_sigmas( @@ -529,17 +718,30 @@ def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: training_batch.noise = noise training_batch.raw_latent_shape = latents.shape - training_batch.latents = training_batch.latents.permute(0, 2, 1, 3, 4) + training_batch.latents = training_batch.latents.permute( + 0, 2, 1, 3, 4 + ) return training_batch - def _build_i2v_mask_latents(self, image_latents: torch.Tensor) -> torch.Tensor: + def _build_i2v_mask_latents( + self, image_latents: torch.Tensor + ) -> torch.Tensor: temporal_compression_ratio = ( - self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + self.training_args.pipeline_config + .vae_config.arch_config.temporal_compression_ratio + ) + num_frames = ( + (self.training_args.num_latent_t - 1) + * temporal_compression_ratio + + 1 ) - num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_ratio + 1 - batch_size, _num_channels, _t, latent_height, latent_width = image_latents.shape - mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width) + batch_size, _num_channels, _t, latent_height, latent_width = ( + image_latents.shape + ) + mask_lat_size = torch.ones( + batch_size, 1, num_frames, latent_height, latent_width + ) mask_lat_size[:, :, 1:] = 0 first_frame_mask = mask_lat_size[:, :, :1] @@ -548,7 +750,9 @@ def _build_i2v_mask_latents(self, image_latents: torch.Tensor) -> torch.Tensor: dim=2, repeats=temporal_compression_ratio, ) - mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], dim=2) + mask_lat_size = torch.cat( + [first_frame_mask, mask_lat_size[:, :, 1:]], dim=2 + ) mask_lat_size = mask_lat_size.view( batch_size, -1, @@ -557,7 +761,9 @@ def _build_i2v_mask_latents(self, image_latents: torch.Tensor) -> torch.Tensor: latent_width, ) mask_lat_size = mask_lat_size.transpose(1, 2) - return mask_lat_size.to(device=image_latents.device, dtype=image_latents.dtype) + return mask_lat_size.to( + device=image_latents.device, dtype=image_latents.dtype + ) def _process_actions( self, training_batch: TrainingBatch @@ -565,21 +771,32 @@ def _process_actions( keyboard_cond = getattr(training_batch, "keyboard_cond", None) mouse_cond = getattr(training_batch, "mouse_cond", None) if keyboard_cond is None or mouse_cond is None: - raise ValueError("WanGame batch must provide keyboard_cond and mouse_cond") + raise ValueError( + "WanGame batch must provide keyboard_cond " + "and mouse_cond" + ) - from fastvideo.models.dits.hyworld.pose import process_custom_actions + from fastvideo.models.dits.hyworld.pose import ( + process_custom_actions, + ) - batch_size = int(training_batch.noisy_model_input.shape[0]) # type: ignore[union-attr] + batch_size = int( + training_batch.noisy_model_input.shape[0] # type: ignore[union-attr] + ) viewmats_list: list[torch.Tensor] = [] intrinsics_list: list[torch.Tensor] = [] action_labels_list: list[torch.Tensor] = [] for b in range(batch_size): - v, i, a = process_custom_actions(keyboard_cond[b], mouse_cond[b]) + v, i, a = process_custom_actions( + keyboard_cond[b], mouse_cond[b] + ) viewmats_list.append(v) intrinsics_list.append(i) action_labels_list.append(a) - viewmats = torch.stack(viewmats_list, dim=0).to(device=self.device, dtype=torch.bfloat16) + viewmats = torch.stack(viewmats_list, dim=0).to( + device=self.device, dtype=torch.bfloat16 + ) intrinsics = torch.stack(intrinsics_list, dim=0).to( device=self.device, dtype=torch.bfloat16 ) @@ -587,16 +804,20 @@ def _process_actions( device=self.device, dtype=torch.bfloat16 ) - num_latent_t = int(training_batch.noisy_model_input.shape[2]) # type: ignore[union-attr] + num_latent_t = int( + training_batch.noisy_model_input.shape[2] # type: ignore[union-attr] + ) if int(action_labels.shape[1]) != num_latent_t: raise ValueError( "Action conditioning temporal dim mismatch: " - f"action={tuple(action_labels.shape)} vs latent_t={num_latent_t}" + f"action={tuple(action_labels.shape)} " + f"vs latent_t={num_latent_t}" ) if int(viewmats.shape[1]) != num_latent_t: raise ValueError( "Viewmats temporal dim mismatch: " - f"viewmats={tuple(viewmats.shape)} vs latent_t={num_latent_t}" + f"viewmats={tuple(viewmats.shape)} " + f"vs latent_t={num_latent_t}" ) return viewmats, intrinsics, action_labels @@ -626,7 +847,9 @@ def _build_distill_input_kwargs( return { "hidden_states": hidden_states, "encoder_hidden_states": None, - "timestep": timestep.to(device=self.device, dtype=torch.bfloat16), + "timestep": timestep.to( + device=self.device, dtype=torch.bfloat16 + ), "encoder_hidden_states_image": image_embeds, "viewmats": viewmats, "Ks": Ks, @@ -647,11 +870,17 @@ def _select_cfg_condition_inputs( image_latents = batch.image_latents mask_lat_size = batch.mask_lat_size if image_embeds is None: - raise RuntimeError("WanGameModel requires TrainingBatch.image_embeds") + raise RuntimeError( + "WanGameModel requires TrainingBatch.image_embeds" + ) if image_latents is None: - raise RuntimeError("WanGameModel requires TrainingBatch.image_latents") + raise RuntimeError( + "WanGameModel requires TrainingBatch.image_latents" + ) if mask_lat_size is None: - raise RuntimeError("WanGameModel requires TrainingBatch.mask_lat_size") + raise RuntimeError( + "WanGameModel requires TrainingBatch.mask_lat_size" + ) viewmats = getattr(batch, "viewmats", None) Ks = getattr(batch, "Ks", None) @@ -674,13 +903,15 @@ def _select_cfg_condition_inputs( on_missing_raw = cfg_uncond.get("on_missing", "error") if not isinstance(on_missing_raw, str): raise ValueError( - "method_config.cfg_uncond.on_missing must be a string, got " + "method_config.cfg_uncond.on_missing must be " + "a string, got " f"{type(on_missing_raw).__name__}" ) on_missing = on_missing_raw.strip().lower() if on_missing not in {"error", "ignore"}: raise ValueError( - "method_config.cfg_uncond.on_missing must be one of {error, ignore}, got " + "method_config.cfg_uncond.on_missing must be " + "one of {error, ignore}, got " f"{on_missing_raw!r}" ) @@ -694,7 +925,8 @@ def _select_cfg_condition_inputs( continue if not isinstance(policy_raw, str): raise ValueError( - "method_config.cfg_uncond values must be strings, got " + "method_config.cfg_uncond values must be " + "strings, got " f"{channel}={type(policy_raw).__name__}" ) policy = policy_raw.strip().lower() @@ -703,9 +935,10 @@ def _select_cfg_condition_inputs( if on_missing == "ignore": continue raise ValueError( - "WanGameModel does not support cfg_uncond channel " - f"{channel!r} (policy={policy!r}). " - "Set cfg_uncond.on_missing=ignore or remove the channel." + "WanGameModel does not support cfg_uncond " + f"channel {channel!r} (policy={policy!r}). " + "Set cfg_uncond.on_missing=ignore or remove " + "the channel." ) def _get_policy(channel: str) -> str: @@ -714,13 +947,15 @@ def _get_policy(channel: str) -> str: return "keep" if not isinstance(raw, str): raise ValueError( - "method_config.cfg_uncond values must be strings, got " + "method_config.cfg_uncond values must be " + "strings, got " f"{channel}={type(raw).__name__}" ) policy = raw.strip().lower() if policy not in {"keep", "zero", "drop"}: raise ValueError( - "method_config.cfg_uncond values must be one of {keep, zero, drop}, got " + "method_config.cfg_uncond values must be " + "one of {keep, zero, drop}, got " f"{channel}={raw!r}" ) return policy @@ -732,19 +967,24 @@ def _get_policy(channel: str) -> str: mask_lat_size = torch.zeros_like(mask_lat_size) elif image_policy == "drop": raise ValueError( - "cfg_uncond.image=drop is not supported for WanGame I2V; " - "use cfg_uncond.image=zero or keep." + "cfg_uncond.image=drop is not supported for " + "WanGame I2V; use cfg_uncond.image=zero or keep." ) action_policy = _get_policy("action") if action_policy == "zero": - if viewmats is None or Ks is None or action is None: + if ( + viewmats is None + or Ks is None + or action is None + ): if on_missing == "ignore": pass else: raise ValueError( - "cfg_uncond.action=zero requires action conditioning tensors, " - "but TrainingBatch is missing {viewmats, Ks, action}." + "cfg_uncond.action=zero requires action " + "conditioning tensors, but TrainingBatch " + "is missing {viewmats, Ks, action}." ) else: viewmats = torch.zeros_like(viewmats) @@ -772,15 +1012,21 @@ def _get_policy(channel: str) -> str: "keyboard_cond": keyboard_cond, } - def _get_transformer(self, handle: RoleHandle, timestep: torch.Tensor) -> torch.nn.Module: - transformer = handle.require_module("transformer") - transformer_2 = handle.modules.get("transformer_2") - if transformer_2 is not None and self.boundary_timestep is not None: + def _get_transformer( + self, timestep: torch.Tensor + ) -> torch.nn.Module: + if ( + self.transformer_2 is not None + and self.boundary_timestep is not None + ): if timestep.numel() != 1: raise ValueError( - "MoE boundary selection requires a scalar timestep, got " + "MoE boundary selection requires a scalar " + "timestep, got " f"shape={tuple(timestep.shape)}" ) - if float(timestep.item()) < float(self.boundary_timestep): - return transformer_2 - return transformer + if float(timestep.item()) < float( + self.boundary_timestep + ): + return self.transformer_2 + return self.transformer diff --git a/fastvideo/distillation/models/wangame/wangame_causal.py b/fastvideo/distillation/models/wangame/wangame_causal.py index 0f2a6e533..269726c0c 100644 --- a/fastvideo/distillation/models/wangame/wangame_causal.py +++ b/fastvideo/distillation/models/wangame/wangame_causal.py @@ -1,22 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""WanGame causal model plugin (streaming/cache primitives). - -Config keys used (YAML schema-v2): -- `recipe.family`: must be `"wangame_causal"` for this plugin. -- `roles.shared_component_role` (affects default `training.model_path`). -- `roles.`: - - `family`, `path`, `trainable`, `disable_custom_init_weights` -- `training` (selected fields): same as `WanGameModel` (see - `models/wangame/wangame.py`). -- `training.validation.*` (consumed by `WanGameValidator` when enabled) -- `method_config.cfg_uncond.*` (consumed by base WanGame primitives for CFG) - -Key differences vs. `models/wangame/wangame.py`: -- Supports causal transformers and streaming rollouts via `CausalModelBase`. -- Exposes cache lifecycle + streaming primitives (`predict_*_streaming`), so - methods can drive causal rollouts without passing KV-cache tensors around. -""" +"""WanGame causal model plugin (per-role instance, streaming/cache).""" from __future__ import annotations @@ -29,10 +13,6 @@ from fastvideo.models.utils import pred_noise_to_pred_video from fastvideo.distillation.models.base import CausalModelBase -from fastvideo.distillation.roles import RoleHandle -from fastvideo.distillation.utils.config import DistillRunConfig - -from fastvideo.distillation.models.wangame.common import _build_wangame_role_handles from fastvideo.distillation.models.wangame.wangame import WanGameModel @@ -49,57 +29,43 @@ class _StreamingCaches: class WanGameCausalModel(WanGameModel, CausalModelBase): - """WanGame model plugin with optional causal/streaming primitives.""" - - def __init__(self, *, cfg: DistillRunConfig) -> None: - training_args = cfg.training_args - roles_cfg = cfg.roles - - if getattr(training_args, "seed", None) is None: - raise ValueError("training.seed must be set for distillation") - if not getattr(training_args, "data_path", ""): - raise ValueError("training.data_path must be set for distillation") - - # Load shared components (training.model_path; defaults to - # roles.shared_component_role's path). - vae = self._load_shared_vae(training_args) - noise_scheduler = self._build_noise_scheduler(training_args) - - def _transformer_cls_name_for_role(role: str, role_spec: Any) -> str: - family = str(getattr(role_spec, "family", "")).strip().lower() - if family == "wangame": - return "WanGameActionTransformer3DModel" - if family == "wangame_causal": - return "CausalWanGameActionTransformer3DModel" - raise ValueError( - f"Unknown roles.{role}.family for wangame_causal: {family!r}. " - "Expected 'wangame' or 'wangame_causal'." - ) + """WanGame per-role model with causal/streaming primitives.""" - role_handles = _build_wangame_role_handles( - roles_cfg=roles_cfg, - training_args=training_args, - transformer_cls_name_for_role=_transformer_cls_name_for_role, - ) + _transformer_cls_name: str = ( + "CausalWanGameActionTransformer3DModel" + ) - # NOTE: re-run the rest of WanGameModel init without rebuilding roles. - self._init_from_built_roles( - cfg=cfg, - role_handles=role_handles, - vae=vae, - noise_scheduler=noise_scheduler, + def __init__( + self, + *, + init_from: str, + trainable: bool = True, + disable_custom_init_weights: bool = False, + flow_shift: float = 3.0, + enable_gradient_checkpointing_type: str | None = None, + ) -> None: + super().__init__( + init_from=init_from, + trainable=trainable, + disable_custom_init_weights=disable_custom_init_weights, + flow_shift=flow_shift, + enable_gradient_checkpointing_type=( + enable_gradient_checkpointing_type + ), ) - - self._streaming_caches: dict[tuple[int, str], _StreamingCaches] = {} + self._streaming_caches: dict[ + tuple[int, str], _StreamingCaches + ] = {} # --- CausalModelBase override: clear_caches --- - def clear_caches(self, handle: RoleHandle, *, cache_tag: str = "pos") -> None: - self._streaming_caches.pop((id(handle), str(cache_tag)), None) + def clear_caches(self, *, cache_tag: str = "pos") -> None: + self._streaming_caches.pop( + (id(self), str(cache_tag)), None + ) # --- CausalModelBase override: predict_noise_streaming --- def predict_noise_streaming( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: Any, @@ -123,7 +89,6 @@ def predict_noise_streaming( if cur_start_frame < 0: raise ValueError("cur_start_frame must be >= 0") - # Ensure per-frame timestep shape [B, T] (mirrors pipeline causal inference). batch_size = int(noisy_latents.shape[0]) num_frames = int(noisy_latents.shape[1]) timestep_full = self._ensure_per_frame_timestep( @@ -133,9 +98,8 @@ def predict_noise_streaming( device=noisy_latents.device, ) - transformer = self._get_transformer(handle, timestep_full) + transformer = self._get_transformer(timestep_full) caches = self._get_or_init_streaming_caches( - handle=handle, cache_tag=cache_tag, transformer=transformer, noisy_latents=noisy_latents, @@ -145,14 +109,10 @@ def predict_noise_streaming( kv_cache = caches.kv_cache crossattn_cache = caches.crossattn_cache - # When activation checkpointing is enabled, torch will recompute - # transformer blocks during backward. Self-forcing streaming rollout - # mutates KV-cache *between* forward and backward (store_kv=True), which - # can break checkpoint recompute with metadata mismatches. - # - # Scheme A: snapshot the cache end-index tensors per call, so backward - # recompute sees the same "effective cache length" as forward. - if self._should_snapshot_streaming_cache(handle) and torch.is_grad_enabled(): + if ( + self._should_snapshot_streaming_cache() + and torch.is_grad_enabled() + ): kv_cache = self._snapshot_kv_cache_indices(kv_cache) model_kwargs: dict[str, Any] = { @@ -165,7 +125,9 @@ def predict_noise_streaming( device_type = self.device.type dtype = noisy_latents.dtype - with torch.autocast(device_type, dtype=dtype), set_forward_context( + with torch.autocast( + device_type, dtype=dtype + ), set_forward_context( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): @@ -192,7 +154,6 @@ def predict_noise_streaming( keyboard_cond=cond_inputs["keyboard_cond"], ) - # Override timestep dtype: causal inference expects integer timesteps. input_kwargs["timestep"] = timestep_full.to( device=self.device, dtype=torch.long ) @@ -203,13 +164,14 @@ def predict_noise_streaming( _ = transformer(**input_kwargs) return None - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) + pred_noise = transformer(**input_kwargs).permute( + 0, 2, 1, 3, 4 + ) return pred_noise # --- CausalModelBase override: predict_x0_streaming --- def predict_x0_streaming( self, - handle: RoleHandle, noisy_latents: torch.Tensor, timestep: torch.Tensor, batch: Any, @@ -222,7 +184,6 @@ def predict_x0_streaming( attn_kind: Literal["dense", "vsa"] = "dense", ) -> torch.Tensor | None: pred_noise = self.predict_noise_streaming( - handle, noisy_latents, timestep, batch, @@ -252,81 +213,6 @@ def predict_x0_streaming( return pred_x0 # --- internal helpers --- - def _load_shared_vae(self, training_args: Any) -> torch.nn.Module: - from fastvideo.distillation.utils.moduleloader import load_module_from_path - - return load_module_from_path( - model_path=str(training_args.model_path), - module_type="vae", - training_args=training_args, - ) - - def _build_noise_scheduler(self, training_args: Any): - from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, - ) - - return FlowMatchEulerDiscreteScheduler( - shift=float(training_args.pipeline_config.flow_shift or 0.0) - ) - - def _init_from_built_roles( - self, - *, - cfg: DistillRunConfig, - role_handles: dict[str, RoleHandle], - vae: torch.nn.Module, - noise_scheduler: Any, - ) -> None: - # This is a small, explicit extraction of `WanGameModel.__init__` so the - # causal model can reuse all non-causal primitives without duplicating - # the full class body. - training_args = cfg.training_args - - self.bundle = self._build_bundle(role_handles) - - self.validator = None - validation_cfg = getattr(cfg, "validation", {}) or {} - validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) - if validation_enabled: - from fastvideo.distillation.validators.wangame import WanGameValidator - - self.validator = WanGameValidator(training_args=training_args) - - self.training_args = training_args - self.noise_scheduler = noise_scheduler - self.vae = vae - - from fastvideo.distributed import ( - get_local_torch_device, - get_sp_group, - get_world_group, - ) - - self.world_group = get_world_group() - self.sp_group = get_sp_group() - self.device = get_local_torch_device() - - self.noise_random_generator = None - self.noise_gen_cuda = None - - self._init_timestep_mechanics() - - from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame - from fastvideo.distillation.utils.dataloader import ( - build_parquet_wangame_train_dataloader, - ) - - self.dataloader = build_parquet_wangame_train_dataloader( - training_args, - parquet_schema=pyarrow_schema_wangame, - ) - self.start_step = 0 - - def _build_bundle(self, role_handles: dict[str, RoleHandle]): - from fastvideo.distillation.roles import RoleManager - - return RoleManager(roles=role_handles) def _ensure_per_frame_timestep( self, @@ -337,19 +223,27 @@ def _ensure_per_frame_timestep( device: torch.device, ) -> torch.Tensor: if timestep.ndim == 0: - return timestep.view(1, 1).expand(batch_size, num_frames).to(device=device) + return ( + timestep.view(1, 1) + .expand(batch_size, num_frames) + .to(device=device) + ) if timestep.ndim == 1: if int(timestep.shape[0]) == batch_size: - return timestep.view(batch_size, 1).expand(batch_size, num_frames).to(device=device) + return ( + timestep.view(batch_size, 1) + .expand(batch_size, num_frames) + .to(device=device) + ) raise ValueError( - "streaming timestep must be scalar, [B], or [B, T]; got " - f"shape={tuple(timestep.shape)}" + "streaming timestep must be scalar, [B], or " + f"[B, T]; got shape={tuple(timestep.shape)}" ) if timestep.ndim == 2: return timestep.to(device=device) raise ValueError( - "streaming timestep must be scalar, [B], or [B, T]; got " - f"ndim={int(timestep.ndim)}" + "streaming timestep must be scalar, [B], or [B, T]; " + f"got ndim={int(timestep.ndim)}" ) def _slice_cond_inputs_for_streaming( @@ -362,9 +256,13 @@ def _slice_cond_inputs_for_streaming( start = int(cur_start_frame) num_frames = int(num_frames) if num_frames <= 0: - raise ValueError("num_frames must be positive for streaming") + raise ValueError( + "num_frames must be positive for streaming" + ) if start < 0: - raise ValueError("cur_start_frame must be >= 0 for streaming") + raise ValueError( + "cur_start_frame must be >= 0 for streaming" + ) end = start + num_frames sliced: dict[str, Any] = dict(cond_inputs) @@ -390,9 +288,12 @@ def _slice_cond_inputs_for_streaming( sliced["action"] = action[:, start:end] temporal_compression_ratio = int( - self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio + self.training_args.pipeline_config + .vae_config.arch_config.temporal_compression_ratio + ) + raw_end_frame_idx = ( + 1 + temporal_compression_ratio * max(0, end - 1) ) - raw_end_frame_idx = 1 + temporal_compression_ratio * max(0, end - 1) mouse_cond = cond_inputs.get("mouse_cond") if isinstance(mouse_cond, torch.Tensor): @@ -400,28 +301,33 @@ def _slice_cond_inputs_for_streaming( keyboard_cond = cond_inputs.get("keyboard_cond") if isinstance(keyboard_cond, torch.Tensor): - sliced["keyboard_cond"] = keyboard_cond[:, :raw_end_frame_idx] + sliced["keyboard_cond"] = keyboard_cond[ + :, :raw_end_frame_idx + ] return sliced def _get_or_init_streaming_caches( self, *, - handle: RoleHandle, cache_tag: str, transformer: torch.nn.Module, noisy_latents: torch.Tensor, ) -> _StreamingCaches: - key = (id(handle), cache_tag) + key = (id(self), cache_tag) cached = self._streaming_caches.get(key) batch_size = int(noisy_latents.shape[0]) dtype = noisy_latents.dtype device = noisy_latents.device - frame_seq_length = self._compute_frame_seq_length(transformer, noisy_latents) + frame_seq_length = self._compute_frame_seq_length( + transformer, noisy_latents + ) local_attn_size = self._get_local_attn_size(transformer) - sliding_window_num_frames = self._get_sliding_window_num_frames(transformer) + sliding_window_num_frames = ( + self._get_sliding_window_num_frames(transformer) + ) meta = ( frame_seq_length, @@ -452,9 +358,13 @@ def _get_or_init_streaming_caches( frame_seq_length=frame_seq_length, local_attn_size=local_attn_size, sliding_window_num_frames=sliding_window_num_frames, - checkpoint_safe=self._should_use_checkpoint_safe_kv_cache(handle), + checkpoint_safe=( + self._should_use_checkpoint_safe_kv_cache() + ), + ) + crossattn_cache = self._initialize_crossattn_cache( + transformer=transformer, device=device ) - crossattn_cache = self._initialize_crossattn_cache(transformer=transformer, device=device) caches = _StreamingCaches( kv_cache=kv_cache, @@ -469,28 +379,51 @@ def _get_or_init_streaming_caches( self._streaming_caches[key] = caches return caches - def _compute_frame_seq_length(self, transformer: torch.nn.Module, noisy_latents: torch.Tensor) -> int: - latent_seq_length = int(noisy_latents.shape[-1]) * int(noisy_latents.shape[-2]) + def _compute_frame_seq_length( + self, + transformer: torch.nn.Module, + noisy_latents: torch.Tensor, + ) -> int: + latent_seq_length = int(noisy_latents.shape[-1]) * int( + noisy_latents.shape[-2] + ) patch_size = getattr(transformer, "patch_size", None) if patch_size is None: - patch_size = getattr(getattr(transformer, "config", None), "arch_config", None) + patch_size = getattr( + getattr(transformer, "config", None), + "arch_config", + None, + ) patch_size = getattr(patch_size, "patch_size", None) if patch_size is None: - raise ValueError("Unable to determine transformer.patch_size for causal streaming") + raise ValueError( + "Unable to determine transformer.patch_size " + "for causal streaming" + ) patch_ratio = int(patch_size[-1]) * int(patch_size[-2]) if patch_ratio <= 0: - raise ValueError("Invalid patch_size for causal streaming") + raise ValueError( + "Invalid patch_size for causal streaming" + ) return latent_seq_length // patch_ratio - def _get_sliding_window_num_frames(self, transformer: torch.nn.Module) -> int: + def _get_sliding_window_num_frames( + self, transformer: torch.nn.Module + ) -> int: cfg = getattr(transformer, "config", None) arch_cfg = getattr(cfg, "arch_config", None) - value = getattr(arch_cfg, "sliding_window_num_frames", None) if arch_cfg is not None else None + value = ( + getattr(arch_cfg, "sliding_window_num_frames", None) + if arch_cfg is not None + else None + ) if value is None: return 15 return int(value) - def _get_local_attn_size(self, transformer: torch.nn.Module) -> int: + def _get_local_attn_size( + self, transformer: torch.nn.Module + ) -> int: try: value = getattr(transformer, "local_attn_size", -1) except Exception: @@ -513,68 +446,107 @@ def _initialize_kv_cache( ) -> list[dict[str, Any]]: num_blocks = len(getattr(transformer, "blocks", [])) if num_blocks <= 0: - raise ValueError("Unexpected transformer.blocks for causal streaming") + raise ValueError( + "Unexpected transformer.blocks for causal " + "streaming" + ) try: - num_attention_heads = int(transformer.num_attention_heads) # type: ignore[attr-defined] + num_attention_heads = int( + transformer.num_attention_heads # type: ignore[attr-defined] + ) except AttributeError as e: - raise ValueError("Transformer is missing num_attention_heads") from e + raise ValueError( + "Transformer is missing num_attention_heads" + ) from e try: - attention_head_dim = int(transformer.attention_head_dim) # type: ignore[attr-defined] + attention_head_dim = int( + transformer.attention_head_dim # type: ignore[attr-defined] + ) except AttributeError: try: - hidden_size = int(transformer.hidden_size) # type: ignore[attr-defined] + hidden_size = int( + transformer.hidden_size # type: ignore[attr-defined] + ) except AttributeError as e: - raise ValueError("Transformer is missing attention_head_dim and hidden_size") from e - attention_head_dim = hidden_size // max(1, num_attention_heads) + raise ValueError( + "Transformer is missing attention_head_dim " + "and hidden_size" + ) from e + attention_head_dim = hidden_size // max( + 1, num_attention_heads + ) if local_attn_size != -1: - kv_cache_size = int(local_attn_size) * int(frame_seq_length) + kv_cache_size = ( + int(local_attn_size) * int(frame_seq_length) + ) else: - kv_cache_size = int(frame_seq_length) * int(sliding_window_num_frames) + kv_cache_size = int(frame_seq_length) * int( + sliding_window_num_frames + ) - # Checkpoint-safe mode: allocate enough cache capacity to avoid rolling - # / eviction during a forward->backward lifetime. This keeps the KV - # cache append-only, which is required for correct activation - # checkpoint recompute in streaming methods (e.g., self-forcing). if checkpoint_safe: - total_frames = int(getattr(self.training_args, "num_frames", 0) or 0) + total_frames = int( + getattr(self.training_args, "num_frames", 0) or 0 + ) if total_frames <= 0: raise ValueError( - "training.num_frames must be set to enable checkpoint-safe " - f"streaming KV cache; got {total_frames}" + "training.num_frames must be set to enable " + "checkpoint-safe streaming KV cache; " + f"got {total_frames}" ) - kv_cache_size = max(kv_cache_size, int(frame_seq_length) * total_frames) + kv_cache_size = max( + kv_cache_size, + int(frame_seq_length) * total_frames, + ) kv_cache: list[dict[str, Any]] = [] for _ in range(num_blocks): kv_cache.append( { "k": torch.zeros( - [batch_size, kv_cache_size, num_attention_heads, attention_head_dim], + [ + batch_size, + kv_cache_size, + num_attention_heads, + attention_head_dim, + ], dtype=dtype, device=device, ), "v": torch.zeros( - [batch_size, kv_cache_size, num_attention_heads, attention_head_dim], + [ + batch_size, + kv_cache_size, + num_attention_heads, + attention_head_dim, + ], dtype=dtype, device=device, ), - "global_end_index": torch.zeros((), dtype=torch.long, device=device), - "local_end_index": torch.zeros((), dtype=torch.long, device=device), + "global_end_index": torch.zeros( + (), dtype=torch.long, device=device + ), + "local_end_index": torch.zeros( + (), dtype=torch.long, device=device + ), } ) return kv_cache - def _should_use_checkpoint_safe_kv_cache(self, handle: RoleHandle) -> bool: - checkpointing_type = getattr(self.training_args, "enable_gradient_checkpointing_type", None) - return bool(checkpointing_type) and bool(getattr(handle, "trainable", False)) + def _should_use_checkpoint_safe_kv_cache(self) -> bool: + checkpointing_type = getattr( + self.training_args, + "enable_gradient_checkpointing_type", + None, + ) + return bool(checkpointing_type) and bool(self._trainable) - def _should_snapshot_streaming_cache(self, handle: RoleHandle) -> bool: - # Snapshotting indices is only needed when recompute may happen. - return self._should_use_checkpoint_safe_kv_cache(handle) + def _should_snapshot_streaming_cache(self) -> bool: + return self._should_use_checkpoint_safe_kv_cache() def _snapshot_kv_cache_indices( self, kv_cache: list[dict[str, Any]] @@ -583,17 +555,22 @@ def _snapshot_kv_cache_indices( for block_cache in kv_cache: global_end_index = block_cache.get("global_end_index") local_end_index = block_cache.get("local_end_index") - if not isinstance(global_end_index, torch.Tensor) or not isinstance( - local_end_index, torch.Tensor - ): + if not isinstance( + global_end_index, torch.Tensor + ) or not isinstance(local_end_index, torch.Tensor): raise ValueError( - "Unexpected kv_cache index tensors; expected tensors at " - "kv_cache[*].{global_end_index, local_end_index}" + "Unexpected kv_cache index tensors; expected " + "tensors at kv_cache[*].{global_end_index, " + "local_end_index}" ) copied = dict(block_cache) - copied["global_end_index"] = global_end_index.detach().clone() - copied["local_end_index"] = local_end_index.detach().clone() + copied["global_end_index"] = ( + global_end_index.detach().clone() + ) + copied["local_end_index"] = ( + local_end_index.detach().clone() + ) snapshot.append(copied) return snapshot @@ -603,12 +580,14 @@ def _initialize_crossattn_cache( transformer: torch.nn.Module, device: torch.device, ) -> list[dict[str, Any]] | None: - # WanGame uses image conditioning; caching the image K/V is optional but - # helps avoid repeated projections across timesteps in a rollout. num_blocks = len(getattr(transformer, "blocks", [])) if num_blocks <= 0: return None return [ - {"is_init": False, "k": torch.empty(0, device=device), "v": torch.empty(0, device=device)} + { + "is_init": False, + "k": torch.empty(0, device=device), + "v": torch.empty(0, device=device), + } for _ in range(num_blocks) ] diff --git a/fastvideo/distillation/roles.py b/fastvideo/distillation/roles.py deleted file mode 100644 index 50330b376..000000000 --- a/fastvideo/distillation/roles.py +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - -import torch - -RoleName = str - - -@dataclass(slots=True) -class RoleHandle: - modules: dict[str, torch.nn.Module] = field(default_factory=dict) - optimizers: dict[str, torch.optim.Optimizer] = field(default_factory=dict) - lr_schedulers: dict[str, Any] = field(default_factory=dict) - trainable: bool = True - - def require_module(self, name: str) -> torch.nn.Module: - if name not in self.modules: - raise KeyError(f"Missing module '{name}'") - return self.modules[name] - - -@dataclass(slots=True) -class RoleManager: - roles: dict[RoleName, RoleHandle] - - def require_roles(self, roles: list[RoleName]) -> None: - missing = [role for role in roles if role not in self.roles] - if missing: - raise KeyError(f"Missing roles: {missing}") - - def role(self, role: RoleName) -> RoleHandle: - if role not in self.roles: - raise KeyError(f"Unknown role: {role}") - return self.roles[role] diff --git a/fastvideo/distillation/utils/checkpoint.py b/fastvideo/distillation/utils/checkpoint.py index 61244d85e..dab1482a5 100644 --- a/fastvideo/distillation/utils/checkpoint.py +++ b/fastvideo/distillation/utils/checkpoint.py @@ -19,7 +19,7 @@ set_model_state_dict, ) -from fastvideo.distillation.roles import RoleHandle, RoleManager +from fastvideo.distillation.models.base import ModelBase from fastvideo.logger import init_logger from fastvideo.training.checkpointing_utils import ( ModelWrapper, @@ -211,10 +211,11 @@ def load_state_dict(self, state_dict: dict[str, Any]) -> None: def maybe_warmstart_role_modules( *, - bundle: RoleManager, + bundle: Any = None, role: str, init_from_checkpoint: str | None, checkpoint_role: str | None = None, + model: ModelBase | None = None, ) -> None: """Warmstart model modules for `role` from a Phase 2/3 DCP checkpoint. @@ -222,7 +223,11 @@ def maybe_warmstart_role_modules( - only loads role modules (no optimizer/scheduler/dataloader/RNG state) - does not advance `start_step` - The checkpoint directory is expected to be `checkpoint-/dcp/*.distcp`. + The checkpoint directory is expected to be ``checkpoint-/dcp/*.distcp``. + + In the new ``_target_``-based flow, pass ``model`` (a ``ModelBase`` instance) + instead of ``bundle``. The legacy ``bundle`` parameter is accepted but + ignored when ``model`` is provided. """ if not init_from_checkpoint: @@ -239,11 +244,27 @@ def maybe_warmstart_role_modules( if not dcp_dir.is_dir(): raise FileNotFoundError(f"Missing dcp dir under checkpoint: {dcp_dir}") - handle = bundle.role(str(role)) + # Build modules dict: new path uses ModelBase, legacy uses RoleManager. + modules: dict[str, torch.nn.Module] = {} + if model is not None: + if model.transformer is not None: + modules["transformer"] = model.transformer + transformer_2 = getattr(model, "transformer_2", None) + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + elif bundle is not None: + handle = bundle.role(str(role)) + modules = dict(handle.modules) + else: + raise ValueError( + "maybe_warmstart_role_modules requires either " + "'model' or 'bundle'" + ) + available_modules = _get_dcp_role_module_names(dcp_dir, role=str(checkpoint_role)) states: dict[str, Any] = {} - for module_name, module in handle.modules.items(): + for module_name, module in modules.items(): if module_name not in available_modules: continue allowed_keys = _get_dcp_role_module_param_keys( @@ -279,23 +300,23 @@ def maybe_warmstart_role_modules( def save_role_pretrained( *, - bundle: RoleManager, + bundle: Any = None, role: str, base_model_path: str, output_dir: str, module_names: list[str] | None = None, overwrite: bool = False, + model: ModelBase | None = None, ) -> str: """Export a role's modules into a diffusers-style model directory. - This is intended to produce a `model_path` that can be loaded by - `PipelineComponentLoader` (i.e., has `model_index.json`, `transformer/`, - `vae/`, and other pipeline components copied from `base_model_path`). + This is intended to produce a ``model_path`` that can be loaded by + ``PipelineComponentLoader`` (i.e., has ``model_index.json``, + ``transformer/``, ``vae/``, and other pipeline components copied + from ``base_model_path``). - Notes: - - Only role modules are exported (e.g. `transformer`, optionally `transformer_2`). - - The output directory is based on `base_model_path`, then overwritten with - current in-memory module weights. + In the new ``_target_``-based flow, pass ``model`` (a ``ModelBase`` + instance) instead of ``bundle``. """ # Resolve HF IDs to local directories (same behavior as module loader). @@ -325,8 +346,23 @@ def _copy_or_link(src: str, dest: str) -> None: _barrier() - handle = bundle.role(str(role)) - modules = dict(handle.modules) + # Build modules dict from model or bundle. + modules: dict[str, torch.nn.Module] = {} + if model is not None: + if model.transformer is not None: + modules["transformer"] = model.transformer + transformer_2 = getattr(model, "transformer_2", None) + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + elif bundle is not None: + handle = bundle.role(str(role)) + modules = dict(handle.modules) + else: + raise ValueError( + "save_role_pretrained requires either " + "'model' or 'bundle'" + ) + if module_names is None: module_names = sorted(modules.keys()) @@ -396,29 +432,72 @@ class DistillCheckpointConfig: class DistillCheckpointManager: - """Role-based checkpoint manager for Phase 2 distillation runtime. + """Role-based checkpoint manager for distillation runtime. - Checkpoint policy lives in YAML (via TrainingArgs fields). - - Resume path is typically provided via CLI (`--resume-from-checkpoint`). + - Resume path is typically provided via CLI (``--resume-from-checkpoint``). + + Accepts either a ``role_models`` dict (new ``_target_``-based flow) + or a legacy ``bundle`` (``RoleManager``). """ def __init__( self, *, - bundle: RoleManager, + bundle: Any = None, + role_models: dict[str, ModelBase] | None = None, + optimizers: dict[str, torch.optim.Optimizer] | None = None, + lr_schedulers: dict[str, Any] | None = None, dataloader: Any, output_dir: str, config: DistillCheckpointConfig, get_rng_generators: Callable[[], dict[str, torch.Generator]] | None = None, ) -> None: self.bundle = bundle + self.role_models = role_models or {} + self.optimizers = optimizers or {} + self.lr_schedulers = lr_schedulers or {} self.dataloader = dataloader self.output_dir = str(output_dir) self.config = config self._get_rng_generators = get_rng_generators self._last_saved_step: int | None = None - def _build_role_states(self, role: str, handle: RoleHandle) -> dict[str, Any]: + def _build_role_states_from_model( + self, role: str, model: ModelBase, + ) -> dict[str, Any]: + if not getattr(model, "_trainable", False): + return {} + + states: dict[str, Any] = {} + modules: dict[str, torch.nn.Module] = {} + if model.transformer is not None: + modules["transformer"] = model.transformer + transformer_2 = getattr(model, "transformer_2", None) + if transformer_2 is not None: + modules["transformer_2"] = transformer_2 + + container = _RoleModuleContainer(modules) + + for module_name, module in modules.items(): + states[f"roles.{role}.{module_name}"] = ModelWrapper(module) + + # Optimizers/schedulers are keyed by role name in the flat dicts. + for name, optimizer in self.optimizers.items(): + if name.startswith(f"{role}.") or name == role: + states[f"optimizers.{name}"] = OptimizerWrapper( + container, optimizer, + ) + + for name, scheduler in self.lr_schedulers.items(): + if name.startswith(f"{role}.") or name == role: + states[f"schedulers.{name}"] = SchedulerWrapper(scheduler) + + return states + + def _build_role_states_from_handle( + self, role: str, handle: Any, + ) -> dict[str, Any]: if not handle.trainable: return {} @@ -429,7 +508,9 @@ def _build_role_states(self, role: str, handle: RoleHandle) -> dict[str, Any]: states[f"roles.{role}.{module_name}"] = ModelWrapper(module) for name, optimizer in handle.optimizers.items(): - states[f"optimizers.{role}.{name}"] = OptimizerWrapper(container, optimizer) + states[f"optimizers.{role}.{name}"] = OptimizerWrapper( + container, optimizer, + ) for name, scheduler in handle.lr_schedulers.items(): states[f"schedulers.{role}.{name}"] = SchedulerWrapper(scheduler) @@ -440,8 +521,16 @@ def _build_states(self) -> dict[str, Any]: states: dict[str, Any] = {} # Models/opts/schedulers are role-scoped. - for role, handle in self.bundle.roles.items(): - states.update(self._build_role_states(role, handle)) + if self.role_models: + for role, model in self.role_models.items(): + states.update( + self._build_role_states_from_model(role, model) + ) + elif self.bundle is not None: + for role, handle in self.bundle.roles.items(): + states.update( + self._build_role_states_from_handle(role, handle) + ) # Dataloader (optional but recommended for exact resume). if _is_stateful(self.dataloader): diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index 9a8c4e33f..ab43e4446 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 +"""Distillation run config (v3 — ``_target_`` based).""" + from __future__ import annotations import os @@ -12,247 +14,198 @@ if TYPE_CHECKING: from fastvideo.fastvideo_args import TrainingArgs -RoleName = str - - -@dataclass(slots=True) -class RecipeSpec: - """Selects the model plugin key (``recipe.family``) + training method. - - This is intentionally small: everything else (roles, training args, and - pipeline config) lives in the run config. - """ - - family: str - method: str - - -@dataclass(slots=True) -class RoleSpec: - """Describes a role's model source and whether it should be trained.""" - - family: str - path: str - trainable: bool = True - disable_custom_init_weights: bool = False - extra: dict[str, Any] = field(default_factory=dict) - @dataclass(slots=True) -class DistillRunConfig: - """Parsed distillation run config loaded from schema-v2 YAML.""" +class RunConfig: + """Parsed distillation run config loaded from v3 YAML.""" - recipe: RecipeSpec - shared_component_role: RoleName - roles: dict[RoleName, RoleSpec] + models: dict[str, dict[str, Any]] + method: dict[str, Any] training_args: TrainingArgs validation: dict[str, Any] method_config: dict[str, Any] raw: dict[str, Any] -def _resolve_existing_file(path: str) -> str: - """Resolve a user-provided config path and require it exists. +# ---- parsing helpers (kept for use by methods) ---- - Distillation intentionally does not perform any "overlay" path rewriting. - The caller must pass a real file path (typically under - ``examples/distillation/``). - """ +def _resolve_existing_file(path: str) -> str: if not path: return path - expanded = os.path.expanduser(path) resolved = Path(expanded).resolve() if not resolved.exists(): - raise FileNotFoundError(f"Config file not found: {resolved}") + raise FileNotFoundError( + f"Config file not found: {resolved}" + ) if not resolved.is_file(): - raise ValueError(f"Expected a file path, got: {resolved}") + raise ValueError( + f"Expected a file path, got: {resolved}" + ) return str(resolved) def _require_mapping(raw: Any, *, where: str) -> dict[str, Any]: if not isinstance(raw, dict): - raise ValueError(f"Expected mapping at {where}, got {type(raw).__name__}") + raise ValueError( + f"Expected mapping at {where}, " + f"got {type(raw).__name__}" + ) return raw def _require_str(raw: Any, *, where: str) -> str: if not isinstance(raw, str) or not raw.strip(): - raise ValueError(f"Expected non-empty string at {where}") + raise ValueError( + f"Expected non-empty string at {where}" + ) return raw -def _get_bool(raw: Any, *, where: str, default: bool) -> bool: +def _get_bool( + raw: Any, *, where: str, default: bool +) -> bool: if raw is None: return default if isinstance(raw, bool): return raw - raise ValueError(f"Expected bool at {where}, got {type(raw).__name__}") + raise ValueError( + f"Expected bool at {where}, " + f"got {type(raw).__name__}" + ) -def get_optional_int(mapping: dict[str, Any], key: str, *, where: str) -> int | None: +def get_optional_int( + mapping: dict[str, Any], key: str, *, where: str +) -> int | None: raw = mapping.get(key) if raw is None: return None if isinstance(raw, bool): - raise ValueError(f"Expected int at {where}, got bool") + raise ValueError( + f"Expected int at {where}, got bool" + ) if isinstance(raw, int): return int(raw) if isinstance(raw, float) and raw.is_integer(): return int(raw) if isinstance(raw, str) and raw.strip(): return int(raw) - raise ValueError(f"Expected int at {where}, got {type(raw).__name__}") + raise ValueError( + f"Expected int at {where}, " + f"got {type(raw).__name__}" + ) -def get_optional_float(mapping: dict[str, Any], key: str, *, where: str) -> float | None: +def get_optional_float( + mapping: dict[str, Any], key: str, *, where: str +) -> float | None: raw = mapping.get(key) if raw is None: return None if isinstance(raw, bool): - raise ValueError(f"Expected float at {where}, got bool") + raise ValueError( + f"Expected float at {where}, got bool" + ) if isinstance(raw, (int, float)): return float(raw) if isinstance(raw, str) and raw.strip(): return float(raw) - raise ValueError(f"Expected float at {where}, got {type(raw).__name__}") + raise ValueError( + f"Expected float at {where}, " + f"got {type(raw).__name__}" + ) -def parse_betas(raw: Any, *, where: str) -> tuple[float, float]: +def parse_betas( + raw: Any, *, where: str +) -> tuple[float, float]: if raw is None: raise ValueError(f"Missing betas for {where}") if isinstance(raw, (tuple, list)) and len(raw) == 2: return float(raw[0]), float(raw[1]) if isinstance(raw, str): - parts = [p.strip() for p in raw.split(",") if p.strip()] + parts = [ + p.strip() for p in raw.split(",") if p.strip() + ] if len(parts) != 2: - raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {raw!r}") + raise ValueError( + f"Expected betas as 'b1,b2' at {where}, " + f"got {raw!r}" + ) return float(parts[0]), float(parts[1]) - raise ValueError(f"Expected betas as 'b1,b2' at {where}, got {type(raw).__name__}") + raise ValueError( + f"Expected betas as 'b1,b2' at {where}, " + f"got {type(raw).__name__}" + ) -def load_distill_run_config(path: str) -> DistillRunConfig: - """Load a distillation run config from schema-v2 YAML. +def load_run_config(path: str) -> RunConfig: + """Load a distillation run config from v3 YAML. - This loader intentionally does **not** merge with legacy CLI args. The YAML - file is the single source of truth for a run. + V3 format uses ``models:`` with ``_target_`` per role and + ``method:`` with ``_target_`` for the algorithm class. """ - - from fastvideo.fastvideo_args import ExecutionMode, TrainingArgs + from fastvideo.fastvideo_args import ( + ExecutionMode, + TrainingArgs, + ) path = _resolve_existing_file(path) with open(path, encoding="utf-8") as f: raw = yaml.safe_load(f) cfg = _require_mapping(raw, where=path) - recipe_raw = _require_mapping(cfg.get("recipe"), where="recipe") - recipe_family = _require_str(recipe_raw.get("family"), where="recipe.family") - recipe_method = _require_str(recipe_raw.get("method"), where="recipe.method") - recipe = RecipeSpec(family=recipe_family, method=recipe_method) - - models_raw = cfg.get("models", None) - if models_raw is not None: - raise ValueError( - "Top-level `models` is not supported in schema-v2. " - "Use `roles.shared_component_role` and `roles..*` instead." - ) - - roles_raw = _require_mapping(cfg.get("roles"), where="roles") - shared_component_role_raw = roles_raw.get("shared_component_role", None) - if shared_component_role_raw is None: - shared_component_role = "student" - else: - shared_component_role = _require_str( - shared_component_role_raw, - where="roles.shared_component_role", + # --- models section --- + models_raw = _require_mapping( + cfg.get("models"), where="models" + ) + models: dict[str, dict[str, Any]] = {} + for role, model_cfg_raw in models_raw.items(): + role_str = _require_str(role, where="models.") + model_cfg = _require_mapping( + model_cfg_raw, where=f"models.{role_str}" ) - roles: dict[RoleName, RoleSpec] = {} - for role, role_cfg_raw in roles_raw.items(): - if role == "shared_component_role": - continue - role_str = _require_str(role, where="roles.") - role_cfg = _require_mapping(role_cfg_raw, where=f"roles.{role_str}") - if "variant" in role_cfg: + if "_target_" not in model_cfg: raise ValueError( - f"roles.{role_str}.variant is not supported in schema-v2. " - "Use roles..family to select the model family instead " - "(e.g. family: wangame_causal)." + f"models.{role_str} must have a '_target_' key" ) - family = role_cfg.get("family") or recipe_family - family = _require_str(family, where=f"roles.{role_str}.family") - model_path = _require_str(role_cfg.get("path"), where=f"roles.{role_str}.path") - trainable = _get_bool( - role_cfg.get("trainable"), - where=f"roles.{role_str}.trainable", - default=True, - ) - disable_custom_init_weights = _get_bool( - role_cfg.get("disable_custom_init_weights"), - where=f"roles.{role_str}.disable_custom_init_weights", - default=False, - ) - extra = { - key: value - for key, value in role_cfg.items() - if key - not in { - "family", - "path", - "trainable", - "disable_custom_init_weights", - } - } - roles[role_str] = RoleSpec( - family=family, - path=model_path, - trainable=trainable, - disable_custom_init_weights=disable_custom_init_weights, - extra=extra, - ) + models[role_str] = dict(model_cfg) - shared_component_role = str(shared_component_role).strip() - if not shared_component_role: - raise ValueError("roles.shared_component_role cannot be empty") - if shared_component_role not in roles: - raise ValueError( - "roles.shared_component_role must be a role name under roles.*, got " - f"{shared_component_role!r}" - ) + # --- method section --- + method_raw = _require_mapping( + cfg.get("method"), where="method" + ) + if "_target_" not in method_raw: + raise ValueError("method must have a '_target_' key") + method = dict(method_raw) - training_raw = _require_mapping(cfg.get("training"), where="training") + # --- method_config section --- + method_config_raw = cfg.get("method_config", None) + if method_config_raw is None: + method_config: dict[str, Any] = {} + else: + method_config = _require_mapping( + method_config_raw, where="method_config" + ) - legacy_validation_keys = { - "log_validation", - "validation_dataset_file", - "validation_steps", - "validation_sampling_steps", - "validation_guidance_scale", - } - has_legacy_validation = any(key in training_raw for key in legacy_validation_keys) + # --- training section --- + training_raw = _require_mapping( + cfg.get("training"), where="training" + ) - training_validation_raw = training_raw.get("validation", None) + # Validation sub-section. + training_validation_raw = training_raw.get( + "validation", None + ) if training_validation_raw is None: - if has_legacy_validation: - raise ValueError( - "Validation config has moved under training.validation " - "(enabled/dataset_file/every_steps/sampling_steps/...). " - "Do not use legacy training.validation_* keys." - ) validation: dict[str, Any] = {} else: - if has_legacy_validation: - raise ValueError( - "Do not mix training.validation with legacy training.validation_* keys. " - "Put all validation fields under training.validation." - ) - validation = _require_mapping(training_validation_raw, where="training.validation") - - method_config_raw = cfg.get("method_config", None) - if method_config_raw is None: - method_config: dict[str, Any] = {} - else: - method_config = _require_mapping(method_config_raw, where="method_config") + validation = _require_mapping( + training_validation_raw, + where="training.validation", + ) training_kwargs: dict[str, Any] = dict(training_raw) training_kwargs.pop("validation", None) @@ -260,62 +213,61 @@ def load_distill_run_config(path: str) -> DistillRunConfig: # Entrypoint invariants. training_kwargs["mode"] = ExecutionMode.DISTILLATION training_kwargs["inference_mode"] = False - # Match the training-mode loader behavior in `ComposedPipelineBase`: - # training uses fp32 master weights and should not CPU-offload DiT weights. training_kwargs.setdefault("dit_precision", "fp32") training_kwargs["dit_cpu_offload"] = False - # Default distributed sizes. These must be set *before* TrainingArgs - # construction because `check_fastvideo_args()` asserts they are not -1 in - # training mode. - num_gpus = int(training_kwargs.get("num_gpus", 1) or 1) + num_gpus = int( + training_kwargs.get("num_gpus", 1) or 1 + ) training_kwargs.setdefault("num_gpus", num_gpus) training_kwargs.setdefault("tp_size", 1) training_kwargs.setdefault("sp_size", num_gpus) training_kwargs.setdefault("hsdp_replicate_dim", 1) training_kwargs.setdefault("hsdp_shard_dim", num_gpus) - # Use the shared-component role path as the default base model_path. This - # is needed for PipelineConfig registry lookup and shared component loading. - shared_role_spec = roles.get(shared_component_role) - if shared_role_spec is None: - raise ValueError( - "roles.shared_component_role must reference an existing role under roles.*, got " - f"{shared_component_role!r}" - ) - - if "model_path" not in training_kwargs: - training_kwargs["model_path"] = shared_role_spec.path - else: - model_path_raw = training_kwargs.get("model_path") - model_path = _require_str(model_path_raw, where="training.model_path").rstrip("/") - expected = str(shared_role_spec.path).rstrip("/") - if model_path != expected: - raise ValueError( - "training.model_path must match roles..path. " - f"Got training.model_path={model_path_raw!r}, " - f"roles.{shared_component_role}.path={shared_role_spec.path!r}" - ) + # Use the student model path as default model_path. + student_cfg = models.get("student") + if student_cfg is not None and "model_path" not in training_kwargs: + init_from = student_cfg.get("init_from") + if init_from is not None: + training_kwargs["model_path"] = str(init_from) if "pretrained_model_name_or_path" not in training_kwargs: - training_kwargs["pretrained_model_name_or_path"] = training_kwargs["model_path"] + training_kwargs["pretrained_model_name_or_path"] = ( + training_kwargs.get("model_path", "") + ) - default_pipeline_cfg_raw = cfg.get("default_pipeline_config", None) - default_pipeline_cfg_path = cfg.get("default_pipeline_config_path", None) + # Pipeline config. + default_pipeline_cfg_raw = cfg.get( + "default_pipeline_config", None + ) + default_pipeline_cfg_path = cfg.get( + "default_pipeline_config_path", None + ) pipeline_cfg_raw = cfg.get("pipeline_config", None) pipeline_cfg_path = cfg.get("pipeline_config_path", None) - if (default_pipeline_cfg_raw is not None or default_pipeline_cfg_path is not None) and ( - pipeline_cfg_raw is not None or pipeline_cfg_path is not None + if ( + default_pipeline_cfg_raw is not None + or default_pipeline_cfg_path is not None + ) and ( + pipeline_cfg_raw is not None + or pipeline_cfg_path is not None ): raise ValueError( - "Provide either default_pipeline_config(_path) or the legacy " - "pipeline_config(_path), not both" + "Provide either default_pipeline_config(_path) or " + "the legacy pipeline_config(_path), not both" ) - cfg_raw = default_pipeline_cfg_raw if default_pipeline_cfg_raw is not None else pipeline_cfg_raw + cfg_raw = ( + default_pipeline_cfg_raw + if default_pipeline_cfg_raw is not None + else pipeline_cfg_raw + ) cfg_path = ( - default_pipeline_cfg_path if default_pipeline_cfg_path is not None else pipeline_cfg_path + default_pipeline_cfg_path + if default_pipeline_cfg_path is not None + else pipeline_cfg_path ) if cfg_path is not None: @@ -327,23 +279,31 @@ def load_distill_run_config(path: str) -> DistillRunConfig: else "pipeline_config_path" ), ) - training_kwargs["pipeline_config"] = _resolve_existing_file(cfg_path) + training_kwargs["pipeline_config"] = ( + _resolve_existing_file(cfg_path) + ) elif cfg_raw is not None: if isinstance(cfg_raw, str): - training_kwargs["pipeline_config"] = _resolve_existing_file(cfg_raw) + training_kwargs["pipeline_config"] = ( + _resolve_existing_file(cfg_raw) + ) elif isinstance(cfg_raw, dict): training_kwargs["pipeline_config"] = cfg_raw else: raise ValueError( - "default_pipeline_config must be a mapping or a path string" + "default_pipeline_config must be a mapping " + "or a path string" ) training_args = TrainingArgs.from_kwargs(**training_kwargs) - return DistillRunConfig( - recipe=recipe, - shared_component_role=shared_component_role, - roles=roles, + # Stash validation config on training_args for + # init_preprocessors to pick up. + training_args._validation_cfg = validation # type: ignore[attr-defined] + + return RunConfig( + models=models, + method=method, training_args=training_args, validation=validation, method_config=method_config, diff --git a/fastvideo/distillation/utils/instantiate.py b/fastvideo/distillation/utils/instantiate.py new file mode 100644 index 000000000..b99ca16f0 --- /dev/null +++ b/fastvideo/distillation/utils/instantiate.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""``_target_``-based instantiation utilities. + +These helpers resolve a dotted Python path to a class and instantiate it, +filtering constructor kwargs through ``inspect.signature`` so that only +recognized parameters are forwarded. Unrecognized keys emit a warning +rather than raising — this keeps YAML configs forward-compatible when +a class drops a parameter in a later version. +""" + +from __future__ import annotations + +import importlib +import inspect +import warnings +from typing import Any + + +def resolve_target(target: str) -> type: + """Import and return the class (or callable) at *target*. + + *target* must be a fully-qualified dotted path, e.g. + ``"fastvideo.distillation.models.wangame.wangame.WanGameModel"``. + """ + if not isinstance(target, str) or not target.strip(): + raise ValueError( + f"_target_ must be a non-empty dotted path string, " + f"got {target!r}" + ) + target = target.strip() + parts = target.rsplit(".", 1) + if len(parts) != 2: + raise ValueError( + f"_target_ must contain at least one dot " + f"(module.ClassName), got {target!r}" + ) + module_path, attr_name = parts + try: + module = importlib.import_module(module_path) + except ModuleNotFoundError as exc: + raise ImportError( + f"Cannot import module {module_path!r} " + f"(from _target_={target!r})" + ) from exc + try: + cls = getattr(module, attr_name) + except AttributeError as exc: + raise ImportError( + f"Module {module_path!r} has no attribute " + f"{attr_name!r} (from _target_={target!r})" + ) from exc + return cls + + +def instantiate(cfg: dict[str, Any], **extra: Any) -> Any: + """Instantiate the class specified by ``cfg["_target_"]``. + + All remaining keys in *cfg* (minus ``_target_``) plus any *extra* + keyword arguments are forwarded to the constructor. Keys that do + not match an ``__init__`` parameter are silently warned about and + dropped, so callers can safely pass a superset. + """ + if not isinstance(cfg, dict): + raise TypeError( + f"instantiate() expects a dict with '_target_', " + f"got {type(cfg).__name__}" + ) + target_str = cfg.get("_target_") + if target_str is None: + raise KeyError("Config dict is missing '_target_' key") + + cls = resolve_target(str(target_str)) + kwargs: dict[str, Any] = { + k: v for k, v in cfg.items() if k != "_target_" + } + kwargs.update(extra) + + sig = inspect.signature(cls.__init__) + params = sig.parameters + has_var_keyword = any( + p.kind == inspect.Parameter.VAR_KEYWORD + for p in params.values() + ) + + if not has_var_keyword: + valid_names = { + name + for name, p in params.items() + if p.kind + in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + } + valid_names.discard("self") + unrecognized = set(kwargs) - valid_names + if unrecognized: + warnings.warn( + f"instantiate({target_str}): dropping unrecognized " + f"kwargs {sorted(unrecognized)}", + stacklevel=2, + ) + for key in unrecognized: + del kwargs[key] + + return cls(**kwargs) diff --git a/fastvideo/distillation/utils/optimizer.py b/fastvideo/distillation/utils/optimizer.py index 66913329d..ce707e230 100644 --- a/fastvideo/distillation/utils/optimizer.py +++ b/fastvideo/distillation/utils/optimizer.py @@ -6,8 +6,6 @@ import torch -from fastvideo.distillation.roles import RoleHandle - from fastvideo.training.training_utils import ( clip_grad_norm_while_handling_failing_dtensor_cases, get_scheduler, @@ -17,62 +15,83 @@ from fastvideo.fastvideo_args import TrainingArgs -def build_role_optimizer_and_scheduler( +def build_optimizer_and_scheduler( *, - role: str, - handle: RoleHandle, + params: list[torch.nn.Parameter], training_args: TrainingArgs, learning_rate: float, betas: tuple[float, float], scheduler_name: str, -) -> None: - modules = handle.modules - params: list[torch.nn.Parameter] = [] - for module in modules.values(): - params.extend([p for p in module.parameters() if p.requires_grad]) +) -> tuple[torch.optim.Optimizer, object]: + """Build an AdamW optimizer and LR scheduler. + + Returns ``(optimizer, lr_scheduler)`` so the caller can store them + as method-level attributes. + """ if not params: - raise ValueError(f"Role {role!r} is trainable but has no trainable parameters") + raise ValueError( + "No trainable parameters passed to " + "build_optimizer_and_scheduler" + ) optimizer = torch.optim.AdamW( params, lr=float(learning_rate), betas=betas, - weight_decay=float(getattr(training_args, "weight_decay", 0.0) or 0.0), + weight_decay=float( + getattr(training_args, "weight_decay", 0.0) or 0.0 + ), eps=1e-8, ) scheduler = get_scheduler( str(scheduler_name), optimizer=optimizer, - num_warmup_steps=int(getattr(training_args, "lr_warmup_steps", 0) or 0), - num_training_steps=int(getattr(training_args, "max_train_steps", 0) or 0), - num_cycles=int(getattr(training_args, "lr_num_cycles", 0) or 0), - power=float(getattr(training_args, "lr_power", 0.0) or 0.0), - min_lr_ratio=float(getattr(training_args, "min_lr_ratio", 0.5) or 0.5), + num_warmup_steps=int( + getattr(training_args, "lr_warmup_steps", 0) or 0 + ), + num_training_steps=int( + getattr(training_args, "max_train_steps", 0) or 0 + ), + num_cycles=int( + getattr(training_args, "lr_num_cycles", 0) or 0 + ), + power=float( + getattr(training_args, "lr_power", 0.0) or 0.0 + ), + min_lr_ratio=float( + getattr(training_args, "min_lr_ratio", 0.5) or 0.5 + ), last_epoch=-1, ) - handle.optimizers = {"main": optimizer} - handle.lr_schedulers = {"main": scheduler} + return optimizer, scheduler -def clip_grad_norm_if_needed(module: torch.nn.Module, training_args: TrainingArgs) -> float: - max_grad_norm_raw = getattr(training_args, "max_grad_norm", None) +def clip_grad_norm_if_needed( + module: torch.nn.Module, training_args: TrainingArgs +) -> float: + max_grad_norm_raw = getattr( + training_args, "max_grad_norm", None + ) if max_grad_norm_raw is None: return 0.0 try: max_grad_norm = float(max_grad_norm_raw) except (TypeError, ValueError) as e: raise ValueError( - "training.max_grad_norm must be a number when set, got " - f"{max_grad_norm_raw!r}" + "training.max_grad_norm must be a number when set, " + f"got {max_grad_norm_raw!r}" ) from e if max_grad_norm <= 0.0: return 0.0 - grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( - [p for p in module.parameters()], - max_grad_norm, - foreach=None, + grad_norm = ( + clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in module.parameters()], + max_grad_norm, + foreach=None, + ) + ) + return ( + float(grad_norm.item()) if grad_norm is not None else 0.0 ) - return float(grad_norm.item()) if grad_norm is not None else 0.0 - diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/distillation/validators/base.py index 2aef6b401..ffd9797d4 100644 --- a/fastvideo/distillation/validators/base.py +++ b/fastvideo/distillation/validators/base.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from typing import Literal -from fastvideo.distillation.roles import RoleHandle +from fastvideo.distillation.models.base import ModelBase @dataclass(slots=True) @@ -18,7 +18,7 @@ class ValidationRequest: request object here. """ - sample_handle: RoleHandle | None = None + sample_handle: ModelBase | None = None dataset_file: str | None = None sampling_steps: list[int] | None = None sampler_kind: Literal["ode", "sde"] | None = None diff --git a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py index 7daa66b9d..745e25729 100644 --- a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py +++ b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py @@ -1,6 +1,5 @@ import torch -from fastvideo.distillation.roles import RoleHandle, RoleManager from fastvideo.distillation.methods.base import DistillMethod @@ -33,19 +32,7 @@ def __init__(self, interval: int) -> None: self.critic_opt = _FakeOptimizer() self.student_sched = _FakeScheduler() self.critic_sched = _FakeScheduler() - bundle = RoleManager( - roles={ - "student": RoleHandle( - optimizers={"main": self.student_opt}, - lr_schedulers={"main": self.student_sched}, - ), - "critic": RoleHandle( - optimizers={"main": self.critic_opt}, - lr_schedulers={"main": self.critic_sched}, - ), - } - ) - super().__init__(bundle) + super().__init__(role_models={}) self.interval = interval def _update_student(self, iteration: int) -> bool: From ac5a2f7565ee7049574126b3817e3eb12da451e8 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 08:11:43 +0000 Subject: [PATCH 168/214] remove moe support for now --- fastvideo/distillation/methods/base.py | 3 -- .../methods/distribution_matching/dmd2.py | 12 ----- .../distillation/methods/fine_tuning/dfsft.py | 6 --- .../methods/fine_tuning/finetune.py | 6 --- fastvideo/distillation/models/wan/wan.py | 44 ------------------ .../distillation/models/wangame/wangame.py | 45 ------------------- fastvideo/distillation/utils/checkpoint.py | 9 ---- 7 files changed, 125 deletions(-) diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py index 912ca97a8..e32ac6b23 100644 --- a/fastvideo/distillation/methods/base.py +++ b/fastvideo/distillation/methods/base.py @@ -39,9 +39,6 @@ def __init__( transformer = getattr(model, "transformer", None) if isinstance(transformer, torch.nn.Module): mods["transformer"] = transformer - transformer_2 = getattr(model, "transformer_2", None) - if isinstance(transformer_2, torch.nn.Module): - mods["transformer_2"] = transformer_2 if mods: self.role_modules[role] = torch.nn.ModuleDict(mods) diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 4e83b6670..cc67a3c9b 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -432,12 +432,6 @@ def _init_optimizers_and_schedulers(self) -> None: for p in self.student.transformer.parameters() if p.requires_grad ] - if getattr(self.student, "transformer_2", None) is not None: - student_params.extend( - p - for p in self.student.transformer_2.parameters() - if p.requires_grad - ) ( self._student_optimizer, self._student_lr_scheduler, @@ -481,12 +475,6 @@ def _init_optimizers_and_schedulers(self) -> None: for p in self.critic.transformer.parameters() if p.requires_grad ] - if getattr(self.critic, "transformer_2", None) is not None: - critic_params.extend( - p - for p in self.critic.transformer_2.parameters() - if p.requires_grad - ) ( self._critic_optimizer, self._critic_lr_scheduler, diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index f9081c3a9..5a52d90ef 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -492,12 +492,6 @@ def _init_optimizers_and_schedulers(self) -> None: for p in self.student.transformer.parameters() if p.requires_grad ] - if getattr(self.student, "transformer_2", None) is not None: - student_params.extend( - p - for p in self.student.transformer_2.parameters() - if p.requires_grad - ) ( self._student_optimizer, self._student_lr_scheduler, diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index ab0df2e5a..1ce3a7218 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -316,12 +316,6 @@ def _init_optimizers_and_schedulers(self) -> None: for p in self.student.transformer.parameters() if p.requires_grad ] - if getattr(self.student, "transformer_2", None) is not None: - student_params.extend( - p - for p in self.student.transformer_2.parameters() - if p.requires_grad - ) ( self._student_optimizer, self._student_lr_scheduler, diff --git a/fastvideo/distillation/models/wan/wan.py b/fastvideo/distillation/models/wan/wan.py index fc683eb97..caebc2427 100644 --- a/fastvideo/distillation/models/wan/wan.py +++ b/fastvideo/distillation/models/wan/wan.py @@ -97,33 +97,6 @@ def __init__( ) self.transformer = transformer - # Optional MoE transformer_2. - self.transformer_2: torch.nn.Module | None = None - try: - t2 = load_module_from_path( - model_path=self._init_from, - module_type="transformer_2", - training_args=None, - disable_custom_init_weights=( - disable_custom_init_weights - ), - ) - except (ValueError, FileNotFoundError): - t2 = None - if t2 is not None: - t2 = apply_trainable(t2, trainable=self._trainable) - if ( - self._trainable - and enable_gradient_checkpointing_type - ): - t2 = apply_activation_checkpointing( - t2, - checkpointing_type=( - enable_gradient_checkpointing_type - ), - ) - self.transformer_2 = t2 - self.noise_scheduler = FlowMatchEulerDiscreteScheduler( shift=float(flow_shift) ) @@ -154,7 +127,6 @@ def __init__( ) self.min_timestep: int = 0 self.max_timestep: int = self.num_train_timestep - self.boundary_timestep: float | None = None # ------------------------------------------------------------------ # Lifecycle @@ -492,15 +464,6 @@ def _init_timestep_mechanics(self) -> None: * self.num_train_timestep ) - boundary_ratio = getattr( - self.training_args, "boundary_ratio", None - ) - self.boundary_timestep = ( - float(boundary_ratio) * float(self.num_train_timestep) - if boundary_ratio is not None - else None - ) - def ensure_negative_conditioning(self) -> None: if self.negative_prompt_embeds is not None: return @@ -802,13 +765,6 @@ def _build_distill_input_kwargs( def _get_transformer( self, timestep: torch.Tensor ) -> torch.nn.Module: - if ( - self.transformer_2 is not None - and self.boundary_timestep is not None - and float(timestep.item()) - < float(self.boundary_timestep) - ): - return self.transformer_2 return self.transformer def _get_uncond_text_dict( diff --git a/fastvideo/distillation/models/wangame/wangame.py b/fastvideo/distillation/models/wangame/wangame.py index 6080e48f0..2ed59f594 100644 --- a/fastvideo/distillation/models/wangame/wangame.py +++ b/fastvideo/distillation/models/wangame/wangame.py @@ -91,26 +91,6 @@ def __init__( ), ) - # Optional MoE transformer_2. - self.transformer_2: torch.nn.Module | None = None - try: - t2 = load_module_from_path( - model_path=self._init_from, - module_type="transformer_2", - training_args=None, - disable_custom_init_weights=disable_custom_init_weights, - ) - except (ValueError, FileNotFoundError): - t2 = None - if t2 is not None: - t2 = apply_trainable(t2, trainable=self._trainable) - if self._trainable and enable_gradient_checkpointing_type: - t2 = apply_activation_checkpointing( - t2, - checkpointing_type=enable_gradient_checkpointing_type, - ) - self.transformer_2 = t2 - self.noise_scheduler = FlowMatchEulerDiscreteScheduler( shift=float(flow_shift) ) @@ -136,8 +116,6 @@ def __init__( ) self.min_timestep: int = 0 self.max_timestep: int = self.num_train_timestep - self.boundary_timestep: float | None = None - def _load_transformer( self, *, @@ -588,15 +566,6 @@ def _init_timestep_mechanics(self) -> None: * self.num_train_timestep ) - boundary_ratio = getattr( - self.training_args, "boundary_ratio", None - ) - self.boundary_timestep = ( - float(boundary_ratio) * float(self.num_train_timestep) - if boundary_ratio is not None - else None - ) - def _sample_timesteps( self, batch_size: int, device: torch.device ) -> torch.Tensor: @@ -1015,18 +984,4 @@ def _get_policy(channel: str) -> str: def _get_transformer( self, timestep: torch.Tensor ) -> torch.nn.Module: - if ( - self.transformer_2 is not None - and self.boundary_timestep is not None - ): - if timestep.numel() != 1: - raise ValueError( - "MoE boundary selection requires a scalar " - "timestep, got " - f"shape={tuple(timestep.shape)}" - ) - if float(timestep.item()) < float( - self.boundary_timestep - ): - return self.transformer_2 return self.transformer diff --git a/fastvideo/distillation/utils/checkpoint.py b/fastvideo/distillation/utils/checkpoint.py index dab1482a5..42239ef63 100644 --- a/fastvideo/distillation/utils/checkpoint.py +++ b/fastvideo/distillation/utils/checkpoint.py @@ -249,9 +249,6 @@ def maybe_warmstart_role_modules( if model is not None: if model.transformer is not None: modules["transformer"] = model.transformer - transformer_2 = getattr(model, "transformer_2", None) - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 elif bundle is not None: handle = bundle.role(str(role)) modules = dict(handle.modules) @@ -351,9 +348,6 @@ def _copy_or_link(src: str, dest: str) -> None: if model is not None: if model.transformer is not None: modules["transformer"] = model.transformer - transformer_2 = getattr(model, "transformer_2", None) - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 elif bundle is not None: handle = bundle.role(str(role)) modules = dict(handle.modules) @@ -473,9 +467,6 @@ def _build_role_states_from_model( modules: dict[str, torch.nn.Module] = {} if model.transformer is not None: modules["transformer"] = model.transformer - transformer_2 = getattr(model, "transformer_2", None) - if transformer_2 is not None: - modules["transformer_2"] = transformer_2 container = _RoleModuleContainer(modules) From 472489344cad2948e18b245adac4e73efa5a5ed0 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 08:38:23 +0000 Subject: [PATCH 169/214] fastgen structure --- dev/fastgen.md | 467 +++++++++++++++++++++++++++++++++++++++ dev/fastgen_structure.md | 268 ---------------------- 2 files changed, 467 insertions(+), 268 deletions(-) create mode 100644 dev/fastgen.md delete mode 100644 dev/fastgen_structure.md diff --git a/dev/fastgen.md b/dev/fastgen.md new file mode 100644 index 000000000..295d296ad --- /dev/null +++ b/dev/fastgen.md @@ -0,0 +1,467 @@ +# FastGen Framework Summary + +Reference codebase: `~/alex/pkgs/FastGen` + +## Naming Convention (Important!) + +FastGen's naming maps to concepts differently than FastVideo: + +| FastGen term | FastVideo term | What it actually is | +|---|---|---| +| `FastGenNetwork` | `ModelBase` | Neural network + noise schedule | +| `FastGenModel` (e.g. `DMD2Model`) | `DistillMethod` (e.g. `DMD2Method`) | Training algorithm / method | +| `Trainer` | `DistillTrainer` | Training loop orchestrator | + +In FastGen: **"model" = method**, **"network" = model**. + +--- + +## Architecture Overview + +``` +train.py + -> load python config (attrs + OmegaConf + Hydra overrides) + -> model = instantiate(config.model_class) # a FastGenModel subclass + -> Trainer(config).run(model) + -> DDP/FSDP wrap + -> model.init_optimizers() + -> checkpointer.load(...) + -> for iter in range(max_iter): + for accum_iter: + data = preprocess_data(batch) # VAE/text encode + loss_map, outputs = model.single_train_step(data, iter) + backward(loss_map["total_loss"]) + model.optimizers_schedulers_step(iter) + model.optimizers_zero_grad(iter) + maybe_validate(model) + maybe_save_checkpoint(model) +``` + +**Key invariant**: Trainer only ever calls `single_train_step()` and reads +`loss_map["total_loss"]`. All algorithm complexity lives inside the model +(method) class. + +--- + +## Layer 1: FastGenNetwork (the neural network) + +**File**: `fastgen/networks/network.py` + +Abstract base that wraps a transformer/UNet with its noise schedule: + +```python +class FastGenNetwork(ABC, torch.nn.Module): + def __init__(self, net_pred_type="x0", schedule_type="edm", **kwargs): + self.net_pred_type = net_pred_type # "x0", "eps", "v", "flow" + self.noise_scheduler = get_noise_schedule(schedule_type, **kwargs) + + @abstractmethod + def forward(self, x_t, t, condition=None, *, + fwd_pred_type=None, # override pred type + return_features_early=False, # for discriminator + feature_indices=None, + return_logvar=False, + **fwd_kwargs) -> Tensor: ... + +class CausalFastGenNetwork(FastGenNetwork): + """Adds chunk_size, total_num_frames, clear_caches().""" + @abstractmethod + def clear_caches(self): ... +``` + +The network owns the noise schedule, which provides: +- `forward_process(x0, eps, t)` — add noise: `alpha(t)*x0 + sigma(t)*eps` +- `sample_t(n, time_dist_type, ...)` — sample training timesteps +- `sample_t_inhom(n, seq_len, chunk_size, ...)` — per-chunk timestep sampling (causal) +- `x0_to_eps()`, `eps_to_x0()`, `flow_to_x0()`, `convert_model_output()` — conversions +- `latents(noise, t_init)` — scale noise by sigma for initial latent + +### Noise Schedule Hierarchy + +``` +BaseNoiseSchedule (abstract) +├── EDMNoiseSchedule # sigma(t)=t, alpha(t)=1, t∈[0.002, 80] +├── AlphasNoiseSchedule # from alphas_cumprod +│ ├── SDNoiseSchedule +│ ├── SDXLNoiseSchedule +│ └── CogVideoXNoiseSchedule +├── RFNoiseSchedule # rectified flow: alpha(t)=1-t, sigma(t)=t +└── TrigNoiseSchedule # alpha(t)=cos(t), sigma(t)=sin(t) +``` + +Time sampling distributions: `uniform`, `lognormal`, `logitnormal`, `polynomial`, +`shifted`, `log_t`. + +--- + +## Layer 2: FastGenModel (the training method) + +**File**: `fastgen/methods/model.py` (~717 lines) + +Template-method base class that all training algorithms inherit from: + +```python +class FastGenModel(torch.nn.Module): + # --- Construction --- + def build_model(self): + self.net = instantiate(config.net) # student network + self.build_teacher() # optional frozen teacher + self._setup_ema() # optional EMA copies + + def build_teacher(self): + self.teacher = instantiate(config.teacher or config.net) + self.teacher.eval().requires_grad_(False) + + # --- Polymorphic dictionaries (key for checkpoint/FSDP) --- + @property + def model_dict(self) -> dict: # {"net": ..., "teacher": ..., "ema": ...} + @property + def fsdp_dict(self) -> dict: # subset to be FSDP-sharded + @property + def ema_dict(self) -> dict: # all EMA networks + @property + def optimizer_dict(self) -> dict: + @property + def scheduler_dict(self) -> dict: + + # --- Training interface (overridden by subclasses) --- + @abstractmethod + def single_train_step(self, data, iteration) -> (loss_map, outputs): ... + + @abstractmethod + def _get_outputs(self, gen_data, ...) -> dict: ... + + # --- Optimizer management --- + def init_optimizers(self): ... + def get_optimizers(self, iteration) -> list[Optimizer]: ... + def get_lr_schedulers(self, iteration) -> list[Scheduler]: ... + def optimizers_schedulers_step(self, iteration): ... + def optimizers_zero_grad(self, iteration): ... + + # --- Inference --- + @staticmethod + def generator_fn(net, noise, condition, t_list, ...): ... + def _student_sample_loop(self, net, noise, condition, ...): ... + + # --- Precision --- + def set_precision(self, precision, precision_amp, ...): ... +``` + +### Method Inheritance Tree + +``` +FastGenModel +├── SFTModel # supervised fine-tuning +│ └── CausalSFTModel +├── KDModel # knowledge distillation (pre-paired data) +│ └── CausalKDModel +├── CMModel # consistency model +│ ├── TCMModel # two-stage consistency +│ ├── SCMModel # simplified consistency (TrigFlow) +│ └── MeanFlowModel # mean flow matching +├── DMD2Model # distribution matching distillation v2 +│ ├── FdistillModel # f-divergence weighted DMD +│ ├── LADDModel # latent adversarial diffusion distillation +│ └── CausVidModel # causal video DMD +│ └── SelfForcingModel # self-forcing (causal rollout) +``` + +--- + +## Layer 3: Trainer (training loop) + +**File**: `fastgen/trainer.py` (~544 lines) + +Completely algorithm-agnostic. Handles: +- DDP/FSDP wrapping +- Gradient accumulation with sync control +- Data preprocessing (VAE encode, text encode) +- Validation (reuses `single_train_step` with no_grad) +- Checkpoint save/load +- Callback system (EMA update, grad clip, logging, profiling) + +The trainer never knows about roles, multiple networks, or alternating updates. +All of that is encapsulated in the model (method) class. + +--- + +## Method Details + +### SFTModel (Supervised Fine-Tuning) + +**File**: `fastgen/methods/fine_tuning/sft.py` + +Simple diffusion training with optional CFG dropout: + +``` +t = sample_t(batch_size) +eps = randn_like(real_data) +x_t = forward_process(real_data, eps, t) +condition = mix_with_neg_condition(condition, cond_dropout_prob) +pred = net(x_t, t, condition) +loss = denoising_score_matching_loss(pred, x0=real_data, eps=eps, t=t) +``` + +Condition dropout: randomly replaces condition with `neg_condition` per sample, +with configurable `cond_dropout_prob` and `keys_no_dropout` for selective dropout. + +### KDModel (Knowledge Distillation) + +**File**: `fastgen/methods/knowledge_distillation/KD.py` + +Learns from pre-constructed teacher ODE trajectories: + +- **Single-step**: student maps pure noise → x0, loss = MSE(pred, target) +- **Multi-step**: student maps intermediate path point → x0 + - `path` tensor: `[B, num_inf_steps, C, F, H, W]` (pre-computed teacher trajectory) + - Samples random t from `t_list`, gathers corresponding path point + - Supports 2-step and 4-step distillation + +**CausalKDModel**: Uses `sample_t_inhom()` for per-chunk timestep sampling. + +### DMD2Model (Distribution Matching Distillation v2) + +**File**: `fastgen/methods/distribution_matching/dmd2.py` (~532 lines) + +Three networks: student (`net`), teacher (frozen), fake_score (critic). +Optional: discriminator (GAN loss). + +**Alternating updates** controlled by `student_update_freq`: + +``` +if iter % student_update_freq == 0: + # Student update + gen_data = net(input, t_student) # generate x0 + x_t = forward_process(gen_data, eps, t) # re-noise + teacher_x0 = teacher(x_t, t, cfg=True) # teacher prediction (CFG) + fake_score_x0 = fake_score(x_t, t).detach() # critic prediction + + vsd_loss = variational_score_distillation_loss(gen_data, teacher_x0, fake_score_x0) + gan_loss = gan_loss_generator(discriminator(features)) # optional + total_loss = vsd_loss + gan_weight * gan_loss + + optimizers = [net_optimizer] +else: + # Critic + discriminator update + fake_score_loss = denoising_score_matching_loss(fake_score(x_t, t), gen_data, eps, t) + gan_loss = gan_loss_discriminator(real_logits, fake_logits) # optional + + optimizers = [fake_score_optimizer, discriminator_optimizer] +``` + +**VSD Loss** (the core DMD2 gradient): + +```python +def variational_score_distillation_loss(gen_data, teacher_x0, fake_score_x0, + additional_scale=None): + w = 1 / (|gen_data - teacher_x0|.mean() + eps) # adaptive weight + if additional_scale: w *= additional_scale # f-div weighting + vsd_grad = (fake_score_x0 - teacher_x0) * w + pseudo_target = gen_data - vsd_grad + return 0.5 * MSE(gen_data, pseudo_target) +``` + +### FdistillModel (F-Divergence Distillation) + +**File**: `fastgen/methods/distribution_matching/f_distill.py` + +Extends DMD2 with importance weighting via learned discriminator logits: + +```python +f_div_weighting = { + "rkl": lambda r: 1, # reverse KL + "kl": lambda r: r, # forward KL + "js": lambda r: 1-1/(1+r), # Jensen-Shannon + "sf": lambda r: 1/(1+r), # squared Hellinger + "sh": lambda r: r**0.5, # Hellinger + ... +} +``` + +Density ratio estimated from discriminator logits → used as `additional_scale` +in VSD loss. Optional EMA histogram normalization across timestep bins. + +### CausVidModel / SelfForcingModel (Causal Video Distillation) + +**Files**: `fastgen/methods/distribution_matching/causvid.py`, `self_forcing.py` + +**CausVidModel** extends DMD2 for causal (autoregressive) video generation: +- Per-chunk timestep sampling via `sample_t_inhom()` +- Autoregressive rollout with KV cache management +- Context noise injection for cache warmup + +**SelfForcingModel** extends CausVid, only overrides `gen_data_from_net()`: +- Blockwise causal rollout with random exit timesteps +- Only exit steps retain gradients (memory efficient) +- Exit steps broadcast-synced across ranks + +Inheritance chain: `SelfForcingModel → CausVidModel → DMD2Model → FastGenModel` + +### CMModel (Consistency Model) + +**File**: `fastgen/methods/consistency_model/CM.py` + +Learns consistency trajectory: + +``` +r = t_to_r_sigmoid(t, ratio) # map t → smaller r +y_t = forward_process(x0, eps, t) +y_r = ode_solver(teacher, y_t, t, r) # CD mode (or forward_process for CT) +D_yt = net(y_t, t) +D_yr = net(y_r, r).detach() +loss = ||D_yt - D_yr|| / (t - r) # weighted consistency loss +``` + +Weighting options: `default` (1/(t-r)), `c_out`, `sigma_sq`. + +### TCMModel (Two-Stage Consistency) + +**File**: `fastgen/methods/consistency_model/TCM.py` + +Wraps network in `TCMPrecond` (boundary conditions for consistency). +Two training stages with different loss formulations. + +### SCMModel (Simplified Consistency / TrigFlow) + +**File**: `fastgen/methods/consistency_model/sCM.py` + +Uses `TrigFlowPrecond` with trigonometric noise schedule. +Supports pseudo-Huber loss and logvar-based adaptive weighting. + +### MeanFlowModel + +**File**: `fastgen/methods/consistency_model/mean_flow.py` (~503 lines) + +Velocity-parameterized consistency training with mean flow matching. + +--- + +## Loss Functions (Centralized) + +**File**: `fastgen/methods/common_loss.py` + +```python +def denoising_score_matching_loss(pred_type, net_pred, x0, eps, t): + """Unified DSM loss for all prediction types.""" + if pred_type == "x0": target = x0 + elif pred_type == "eps": target = eps + elif pred_type == "v": target = alpha(t)*eps - sigma(t)*x0 + elif pred_type == "flow": target = eps - x0 + return MSE(net_pred, target) + +def variational_score_distillation_loss(gen_data, teacher_x0, fake_score_x0, + additional_scale=None): + """VSD loss with adaptive weighting (DMD2 core).""" + w = 1 / (|gen_data - teacher_x0|.mean() + eps) + if additional_scale: w *= additional_scale + pseudo_target = gen_data - (fake_score_x0 - teacher_x0) * w + return 0.5 * MSE(gen_data, pseudo_target) + +def gan_loss_generator(fake_logits): + return softplus(-fake_logits).mean() + +def gan_loss_discriminator(real_logits, fake_logits): + return softplus(fake_logits).mean() + softplus(-real_logits).mean() +``` + +--- + +## Configuration System + +**Structure**: attrs dataclasses + OmegaConf + LazyCall + Hydra overrides + +### Three-level config hierarchy + +1. **Base config** (`fastgen/configs/config.py`): + ```python + @attrs.define + class BaseModelConfig: + net: dict # network config (LazyCall) + teacher: Optional[dict] = None + guidance_scale: Optional[float] + net_optimizer: dict + net_scheduler: dict + sample_t_cfg: SampleTConfig # timestep distribution params + input_shape: list[int] + pretrained_model_path: str + use_ema: Any = False + student_sample_steps: int = 1 + fsdp_meta_init: bool = False + + @attrs.define + class SampleTConfig: + time_dist_type: str = "uniform" + train_p_mean: float = -1.1 + train_p_std: float = 2.0 + min_t: float = 0.002 + max_t: float = 80.0 + t_list: Optional[list[float]] = None + ``` + +2. **Method configs** (`fastgen/configs/methods/config_*.py`): + - `config_dmd2.py`: adds `fake_score_optimizer`, `discriminator`, `student_update_freq` + - `config_f_distill.py`: adds `FdistillConfig` (f_div type, ratio bounds) + - `config_cm.py`: adds `CMLossConfig` (weighting, use_cd, ratio) + +3. **Experiment configs** (`fastgen/configs/experiments//config_.py`): + - Concrete settings: model paths, input_shape, lr, t_list, etc. + +### LazyCall / instantiate + +Config entries use `LazyCall` to record `_target_` + kwargs. Objects are created +via `instantiate(cfg)` at runtime, enabling full pluggability. + +--- + +## Distributed Training + +### DDP Trick + +`DDPWrapper` temporarily redirects `module.forward` → `single_train_step` so +that DDP's forward/backward hooks fire correctly even though the training logic +isn't a standard forward pass. + +### FSDP2 with Meta-Init + +For large models (10B+): +1. Non-rank0 processes build model on `torch.device("meta")` (zero memory) +2. Rank0 loads weights +3. FSDP wrap with `sync_module_states` broadcasts weights + +### Checkpoint + +- **Non-FSDP**: rank0 saves single `.pth` (model + optim + scheduler + iteration) +- **FSDP**: `torch.distributed.checkpoint` per `model_dict` key (sharded), + plus scalar state in `.pth` + +Checkpoint keys derived directly from `model_dict` / `optimizer_dict` / `scheduler_dict` +properties — naturally supports multi-network methods. + +--- + +## Data Pipeline + +**File**: `fastgen/datasets/wds_dataloaders.py` + +WebDataset-based with two key features: + +- `files_map`: load constants from files (e.g., pre-computed neg_condition embedding) +- `presets_map`: inject literal constants into every batch + +`Trainer.preprocess_data()` handles optional online encoding: +- VAE: `data["real"] = net.vae.encode(data["real"])` +- Text: `data["condition"] = net.text_encoder.encode(data["condition"])` +- I2V/V2W: first-frame conditioning, image embeddings + +--- + +## Key Design Principles + +1. **Trainer is algorithm-agnostic**: only knows `single_train_step()` + `total_loss` +2. **Method = multi-network container + algorithm**: owns nets, optimizers, update logic +3. **Alternating updates via `get_optimizers(iter)`**: no trainer changes needed +4. **Network owns noise schedule**: method layer doesn't reimplement diffusion math +5. **Centralized loss functions**: all methods share `common_loss.py` +6. **Inheritance for algorithm variants**: SelfForcing only overrides `gen_data_from_net()` +7. **Polymorphic dicts for checkpoint**: `model_dict`/`optimizer_dict` scale to any number of roles +8. **Config-driven pluggability**: LazyCall + instantiate for all components diff --git a/dev/fastgen_structure.md b/dev/fastgen_structure.md deleted file mode 100644 index 0421e4a3a..000000000 --- a/dev/fastgen_structure.md +++ /dev/null @@ -1,268 +0,0 @@ -# FastGen 的 distillation 设计梳理（可借鉴点） - -这份文档是我在本机 `~/alex/FastGen` 仓库里阅读 distillation 相关代码后的结构总结， -重点关注 “架构设计优点/可复用模式”，用于反哺 FastVideo 的 distill 重构。 - -> 结论先行：FastGen 把 distillation 的关键复杂度分摊到了 -> `Trainer(基础设施)` / `Method(算法+多网络训练)` / `Network(架构+采样)` / -> `Dataset(数据与conditioning供给)` 四层，并用 config + callback + checkpoint -> 把它们解耦起来。 - -## 1. FastGen 的核心分层（强推荐） - -FastGen 的高层调用链可以简化成： - -``` -train.py - -> load config (python config + Hydra overrides) - -> instantiate(config.model_class) # "method" = FastGenModel 子类 - -> Trainer(config).run(model) - -> (DDP/FSDP wrap) + callbacks + checkpointer - -> dataloader + preprocess_data - -> for iter: - loss_map, outputs = model_ddp.single_train_step(data, iter) - backward(total_loss) - model.optimizers_schedulers_step(iter) - model.optimizers_zero_grad(iter) -``` - -这套结构的关键点在于： - -- **Trainer 永远只认识一个接口：`single_train_step()` + `total_loss`** -- “distillation 算法/多网络/多优化器/交替更新”等复杂逻辑都封装进 **method 类** -- **network 类只负责 forward / noise schedule /（可选）sample**，并提供少量 hook -- dataset 负责把 `real/condition/neg_condition` 等输入准备齐全（甚至可以预存为常量） - -这种分层特别适合你想做的 “`models={teacher, student, critic,...}` + 算法解耦”。 - -## 2. 仓库组织方式（distill 相关） - -FastGen 的 repo 结构（与 distill 相关部分）： - -- `train.py`：入口，加载 config，实例化 `model_class`，调用 `Trainer.run` -- `fastgen/trainer.py`：通用训练循环（DDP/FSDP、grad accum、validate、save/load、callbacks） -- `fastgen/methods/`：**训练方法/算法**（distill 的主体逻辑在这里） - - `methods/model.py`：`FastGenModel` 基类（多网络训练接口、precision、EMA、采样 loop 等） - - `methods/distribution_matching/dmd2.py`：DMD2（student/teacher/fake_score + 可选 discriminator） - - `methods/distribution_matching/causvid.py`：CausVid（因果 video distill，复用 DMD2 框架） - - `methods/distribution_matching/self_forcing.py`：Self-Forcing（继承 CausVid/DMD2，仅改 student rollout） - - 其它：CM/sCM/TCM/MeanFlow、KD、SFT 等（同一接口体系） -- `fastgen/networks/`：架构实现（EDM/SD/Wan/CogVideoX/Cosmos…） - - `networks/network.py`：`FastGenNetwork` / `CausalFastGenNetwork` 抽象接口 -- `fastgen/datasets/`：数据加载（webdataset + 预计算 latent/embedding） - - `datasets/wds_dataloaders.py`：支持 `files_map/presets_map` 注入常量 conditioning（比如 neg prompt） -- `fastgen/configs/`：配置系统（方法 config、实验 config、网络 config、数据 config） -- `fastgen/utils/`：基础设施（instantiate、distributed ddp/fsdp、checkpointer、logging、autoresume） -- `fastgen/callbacks/`：回调系统（EMA、grad clip、wandb、profiler、param count…） - -## 3. 配置系统：python config + LazyCall + Hydra override（优秀） - -FastGen 采用： - -- `attrs` 定义结构化 config（如 `BaseConfig/BaseModelConfig/BaseTrainerConfig`） -- `OmegaConf/DictConfig` 存储 LazyCall 结构并支持 object -- Hydra 的 override 语法做命令行参数覆盖：`python train.py --config=... - key=value` -- 训练启动时把 resolve 后的 config 序列化为 `config.yaml`，保证可复现 - -关键模式： - -1) **三层 config 分离** - -- `configs/config.py`：BaseConfig（训练通用字段） -- `configs/methods/config_*.py`：方法级默认参数（比如 DMD2 要 fake_score_optimizer） -- `configs/experiments/**/config_*.py`：具体实验（比如 WanT2V 的 input_shape、t_list、lr） - -2) **LazyCall/instantiate（“延迟实例化”）** - -- config 内用 `LazyCall` 记录 `_target_` + kwargs -- 真正创建对象统一走 `instantiate(cfg)` - -价值： - -- algorithm/network/dataloader/callback 全部可插拔 -- method 和 trainer 都不需要硬编码具体类名 - -## 4. Trainer：完全算法无关的训练循环（非常干净） - -`fastgen/trainer.py` 的设计要点： - -- **只要求模型实现 `single_train_step(data, iteration)`** -- backward 统一对 `loss_map["total_loss"]` 做 -- grad accumulation 通过： - - DDP：`ddp_sync_grad(model_ddp, sync_grads)` - - FSDP：`fsdp_sync_grad(model, sync_grads)` -- optimizer step / scheduler step 通过调用 model 的接口完成： - - `model.optimizers_schedulers_step(iteration)` - - `model.optimizers_zero_grad(iteration)` -- validate 可以复用 `single_train_step`（no_grad + autocast），并且支持 - `global_vars_val` 在一次 validation 中跑多个设置（例如限制 `MAX_VAL_STEPS`） - -这个框架天然支持你想要的 “算法与模型解耦”：Trainer 永远不关心 roles。 - -## 5. Method（算法层）：用“对象”承载多网络与更新策略（关键借鉴点） - -### 5.1 FastGenModel（统一训练接口 + 多网络容器） - -`fastgen/methods/model.py::FastGenModel` 承担了大量“应放在 distill 框架层”的事： - -- precision / AMP / FSDP precision 管理（`precision`, `precision_amp`, `precision_fsdp`） -- teacher 构建与冻结（`build_teacher()`） -- student 初始化（可从 teacher 或单独 ckpt）+ EMA 初始化（`_setup_ema()`） -- inference 的统一入口（`generator_fn` / `_student_sample_loop` / `sample`） -- checkpoint 需要的统一映射： - - `model_dict` / `optimizer_dict` / `scheduler_dict` - - `fsdp_dict`（决定哪些 module 要被 FSDP sharding；可选择把 teacher 放进去） - -> 这几乎就是你想要的 `ModelBundle/ModelHandle`，只是 FastGen 的实现把它融合在 -> `FastGenModel` 类里（OOP 风格）。 - -### 5.2 DMD2Model：多优化器/交替更新，Trainer 完全不需要知道 - -`fastgen/methods/distribution_matching/dmd2.py::DMD2Model` 的设计非常值得抄： - -- 模型组成： - - `net`：student（训练的 generator） - - `teacher`：冻结 - - `fake_score`：critic（训练） - - `discriminator`：可选（GAN loss 时训练） -- **交替更新策略**由 config 控制：`student_update_freq` -- 关键：通过覆写 `get_optimizers()` / `get_lr_schedulers()` 实现 - “每个 iteration step 哪些 optimizer/scheduler 走一步” - -伪代码大概是： - -```python -if iter % student_update_freq == 0: - optimizers = [net_optimizer] -else: - optimizers = [fake_score_optimizer, (optional) discriminator_optimizer] -``` - -这种方式的价值： - -- 训练 loop 不会膨胀（Trainer 永远固定） -- method 可以自由扩展更多 role/更多 optimizer，而不用改 Trainer -- scheduler 的 step 粒度天然与 optimizer step 对齐（避免 update ratio 引入 lr schedule 偏差） - -### 5.3 SelfForcingModel：继承链复用算法框架，只替换 student rollout - -Self-forcing 在 FastGen 里是： - -``` -SelfForcingModel -> CausVidModel -> DMD2Model -> FastGenModel -``` - -它的主要改动非常克制： - -- 不改 Trainer，不改 DMD2 的 loss 框架 -- 只覆写 `gen_data_from_net()`：从普通 student forward 换成 - `rollout_with_gradient()`（blockwise causal rollout，只有 exit step 保留梯度） -- rollout 的随机 exit steps 用广播同步（rank0 采样，其他 rank broadcast） -- KV cache 用 **network 内部缓存**（`CausalFastGenNetwork.clear_caches()`）， - method 侧只调用 `store_kv=True/False` 的 forward - -这体现出非常强的“算法复用”能力：Self-forcing 只是 DMD2 的一个 student 采样策略。 - -## 6. Network 抽象：统一 forward contract + causal 扩展（非常实用） - -`fastgen/networks/network.py` 的接口设计对 distill 非常友好： - -- `FastGenNetwork.forward(x_t, t, condition=..., fwd_pred_type=..., feature_indices=...)` - - 允许 method 侧用统一的方式拿到： - - x0/eps/flow 等不同 pred_type - - 中间 features（给 discriminator） - - logvar（给某些 consistency/uncertainty 变体） -- noise schedule 在 network 内统一（`self.noise_scheduler`），method 层不用各写一遍 -- `CausalFastGenNetwork` 增加： - - `chunk_size/total_num_frames` - - `clear_caches()` 抽象，明确缓存生命周期 - -另一个小但很关键的点： - -- `FastGenModel._student_sample_loop` 是通用 multistep loop， - 但如果 network 实现了 `preserve_conditioning(x, condition)`， - loop 会自动调用它来保留 I2V / V2W 的 conditioning 帧/掩码（避免 loop 被各种模型特例污染） - -## 7. 数据/conditioning 供给：把 uncond/neg prompt 变成“数据常量”（非常推荐） - -FastGen 的 WebDataset loader（`datasets/wds_dataloaders.py`）提供了两个很棒的能力： - -- `files_map`：把某些 key 从外部文件加载为常量（每个 batch 都带上） - - 典型用法：`neg_condition`（negative prompt embedding）从 `.npy` 读取一次 -- `presets_map`：把某些 key 直接用预设常量填充（比如 WAN 的负面 prompt 字符串） - -这带来的直接收益： - -- CFG/uncond 的输入不依赖 “validation 是否跑过” 或 “训练时是否临时 encode” -- 能把 expensive 的 negative prompt embedding 预先算好并缓存 -- 对 offline / 大规模训练更友好（避免每 step 重新 encode） - -`Trainer.preprocess_data()` 进一步提供可选的在线 encode： - -- 若 batch 里是 raw video/image/text，则自动用 network 内的 - `vae/text_encoder/image_encoder` 编码成 latent/embeddings -- 同时保留 `*_raw` 字段，便于日志/可视化 - -这等价于把“pipeline stage”做成一个轻量的 preprocessing hook，集中在 Trainer。 - -## 8. 分布式与 checkpoint：围绕 `model_dict` 泛化（非常可维护） - -### 8.1 DDP training_step 的包装（很巧） - -`fastgen/utils/distributed/ddp.py::DDPWrapper` 做了一个小技巧： - -- 训练逻辑在 `single_train_step`，但 DDP 的 hook 是绑定在 `forward()` 上的 -- wrapper 临时把 `module.forward` 指到 `single_train_step`，然后调用 `self(...)` - 以触发 DDP 的 forward/backward hook - -这让 “训练 step 不是 forward” 的设计依然能吃到 DDP 的正确行为。 - -### 8.2 FSDP2：支持 meta-init 的内存友好加载 - -`FastGenModel._get_meta_init_context()` + `utils/distributed/fsdp.py::model_to_fsdp` 支持： - -- 非 rank0 在 `torch.device("meta")` 上构建大模型（几乎零内存） -- rank0 负责加载权重 -- FSDP wrap 后通过 `sync_module_states` 广播权重到所有 rank - -对于 10B+ 模型，这个设计非常关键：大幅降低启动时间与 I/O contention。 - -### 8.3 Checkpointer：同一套保存/加载适配 DDP 与 FSDP - -`utils/checkpointer.py`： - -- 非 FSDP：rank0 写单个 `.pth`（包含 model/optim/scheduler/grad_scaler/callbacks/iteration） -- FSDP：用 `torch.distributed.checkpoint` 分别保存每个 `model_dict` key 的 - sharded state（例如 `.net_model`、`.fake_score_model`），并且把 - scheduler/grad_scaler/callbacks/iteration 仍写到 `.pth` - -这种按 `model_dict` key 泛化的 checkpoint 方式，对多网络 distill 非常自然。 - -## 9. 对 FastVideo 的直接启发（建议落地清单） - -结合你要做的 “`models={teacher, student, critic, ...}` + 算法解耦”，FastGen -给出的可落地模式： - -1) **Trainer 只依赖统一接口** -- 固化为：`algorithm/model.single_train_step()` 返回 `loss_map["total_loss"]` -- 其余都放到算法层：update ratio、多优化器 step、cache 管理等 - -2) **把多网络/多 optimizer 的“调度”做成可覆盖函数** -- `get_optimizers(iter)` / `get_lr_schedulers(iter)` 的模式非常清晰 -- 这比在 Trainer 里写 if/else 或在 pipeline 里复制 train_loop 更可维护 - -3) **role → checkpoint 的映射要结构化** -- 借鉴 `model_dict/optimizer_dict/scheduler_dict` 的思路： - `models` 映射天然就是 checkpoint 的 namespace - -4) **uncond/negative conditioning 不要依赖 validation** -- 学 FastGen：把 `neg_condition` 做成 dataset 常量（files_map/presets_map） - 或者训练开始时一次性 encode 并缓存/广播 - -5) **network 抽象要显式支持 distill 需求** -- forward contract 支持 `fwd_pred_type`、features、可选采样 -- causal network 明确 cache 生命周期（`clear_caches()`） - -6) **大模型加载要考虑 meta-init + FSDP2** -- 如果 FastVideo 也要跑 14B/更大 teacher，多机多卡启动成本会很明显 -- meta-init + rank0 broadcast 是成熟方案 From c9de04931f446a2220082e515905cdff44b2437a Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 19:56:29 +0000 Subject: [PATCH 170/214] better yaml file structure --- .../refactor/dfsft_wangame_causal_v3.yaml | 37 +- .../self_forcing_wangame_causal_v3.yaml | 73 ++-- .../methods/distribution_matching/dmd2.py | 386 ++++++------------ .../distribution_matching/self_forcing.py | 363 ++++++---------- .../distillation/methods/fine_tuning/dfsft.py | 288 ++++--------- .../methods/fine_tuning/finetune.py | 143 ++----- fastvideo/distillation/utils/config.py | 358 +++++++++------- 7 files changed, 630 insertions(+), 1018 deletions(-) diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index 415c64a07..14358630c 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -11,6 +11,10 @@ models: method: _target_: fastvideo.distillation.methods.fine_tuning.dfsft.DiffusionForcingSFTMethod + attn_kind: dense + chunk_size: 3 + min_timestep_ratio: 0.02 + max_timestep_ratio: 0.98 training: # Distributed @@ -44,7 +48,7 @@ training: # Optimizer learning_rate: 1.0e-5 mixed_precision: bf16 - betas: "0.9,0.999" + betas: [0.9, 0.999] weight_decay: 1.0e-4 lr_scheduler: constant lr_warmup_steps: 0 @@ -58,26 +62,21 @@ training: training_cfg_rate: 0.0 enable_gradient_checkpointing_type: full - # Tracking / validation + # Tracking tracker_project_name: distillation_wangame wandb_run_name: wangame_dfsft_causal_v3 - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [40] - rollout_mode: streaming - sampler_kind: ode - ode_solver: euler - guidance_scale: 1.0 - num_frames: 75 -default_pipeline_config: - flow_shift: 3 +validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [40] + rollout_mode: streaming sampler_kind: ode + ode_solver: euler + guidance_scale: 1.0 + num_frames: 75 -method_config: - attn_kind: dense - chunk_size: 3 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 +pipeline: + flow_shift: 3 + sampler_kind: ode diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index 9096399eb..7959bf541 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -29,6 +29,26 @@ models: method: _target_: fastvideo.distillation.methods.distribution_matching.self_forcing.SelfForcingMethod + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + + warp_denoising_step: true + dmd_denoising_steps: [1000,750,500,250] + + chunk_size: 3 + student_sample_type: sde + same_step_across_blocks: false + last_step_only: false + context_noise: 0.0 + enable_gradient_in_rollout: true + start_gradient_frame: 0 + + cfg_uncond: + on_missing: error + action: keep + image: keep + text: keep training: # Distributed (4 nodes x 8 GPUs = 32 ranks) @@ -62,7 +82,7 @@ training: # Optimizer (student) learning_rate: 2.0e-6 mixed_precision: bf16 - betas: "0.0,0.999" + betas: [0.0, 0.999] weight_decay: 0.01 lr_scheduler: constant lr_warmup_steps: 0 @@ -70,54 +90,31 @@ training: # Optimizer (critic / fake-score) fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" + fake_score_betas: [0.0, 0.999] fake_score_lr_scheduler: constant # Distillation knobs training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 enable_gradient_checkpointing_type: null # Checkpointing training_state_checkpointing_steps: 1000 weight_only_checkpointing_steps: 1000 - # Tracking / validation + # Tracking tracker_project_name: distillation_wangame wandb_run_name: wangame_self_forcing_4steps_v3 - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [4] - sampler_kind: sde - rollout_mode: streaming - guidance_scale: 1.0 - num_frames: 69 - -default_pipeline_config: - flow_shift: 3 - sampler_kind: sde -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - - warp_denoising_step: true - dmd_denoising_steps: [1000,750,500,250] - - chunk_size: 3 - student_sample_type: sde - same_step_across_blocks: false - last_step_only: false - context_noise: 0.0 - enable_gradient_in_rollout: true - start_gradient_frame: 0 +validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [4] + sampler_kind: sde + rollout_mode: streaming + guidance_scale: 1.0 + num_frames: 69 - cfg_uncond: - on_missing: error - action: keep - image: keep - text: keep +pipeline: + flow_shift: 3 + sampler_kind: sde diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index cc67a3c9b..cdac7034b 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """DMD2 distillation method (algorithm layer).""" from __future__ import annotations @@ -68,26 +67,17 @@ def __init__( self.critic = role_models["critic"] if not getattr(self.student, "_trainable", True): - raise ValueError( - "DMD2Method requires student to be trainable" - ) + raise ValueError("DMD2Method requires student to be trainable") if getattr(self.teacher, "_trainable", True): - raise ValueError( - "DMD2Method requires teacher to be non-trainable" - ) + raise ValueError("DMD2Method requires teacher to be non-trainable") if not getattr(self.critic, "_trainable", True): - raise ValueError( - "DMD2Method requires critic to be trainable" - ) + raise ValueError("DMD2Method requires critic to be trainable") self.validator = validator self.training_args = cfg.training_args - self.method_config: dict[str, Any] = dict( - getattr(cfg, "method_config", {}) or {} - ) + self.method_config: dict[str, Any] = dict(cfg.method) self.validation_config: dict[str, Any] = dict( - getattr(cfg, "validation", {}) or {} - ) + getattr(cfg, "validation", {}) or {}) self._cfg_uncond = self._parse_cfg_uncond() self._rollout_mode = self._parse_rollout_mode() self._denoising_step_list: torch.Tensor | None = None @@ -105,9 +95,9 @@ def single_train_step( *, current_vsa_sparsity: float = 0.0, ) -> tuple[ - dict[str, torch.Tensor], - dict[str, Any], - dict[str, LogScalar], + dict[str, torch.Tensor], + dict[str, Any], + dict[str, LogScalar], ]: latents_source: Literal["data", "zeros"] = "data" if self._rollout_mode == "simulate": @@ -128,16 +118,13 @@ def single_train_step( ) student_ctx = None if update_student: - generator_pred_x0 = self._student_rollout( - training_batch, with_grad=True - ) + generator_pred_x0 = self._student_rollout(training_batch, + with_grad=True) student_ctx = ( training_batch.timesteps, training_batch.attn_metadata_vsa, ) - generator_loss = self._dmd_loss( - generator_pred_x0, training_batch - ) + generator_loss = self._dmd_loss(generator_pred_x0, training_batch) ( fake_score_loss, @@ -181,15 +168,11 @@ def backward( ) return - update_student = bool( - backward_ctx.get("update_student", False) - ) + update_student = bool(backward_ctx.get("update_student", False)) if update_student: student_ctx = backward_ctx.get("student_ctx") if student_ctx is None: - raise RuntimeError( - "Missing student backward context" - ) + raise RuntimeError("Missing student backward context") self.student.backward( loss_map["generator_loss"], student_ctx, @@ -206,9 +189,7 @@ def backward( ) # DistillMethod override: get_optimizers - def get_optimizers( - self, iteration: int - ) -> list[torch.optim.Optimizer]: + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: optimizers: list[torch.optim.Optimizer] = [] optimizers.append(self._critic_optimizer) if self._should_update_student(iteration): @@ -226,12 +207,9 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: if self._should_update_student(iteration): - clip_grad_norm_if_needed( - self.student.transformer, self.training_args - ) - clip_grad_norm_if_needed( - self.critic.transformer, self.training_args - ) + clip_grad_norm_if_needed(self.student.transformer, + self.training_args) + clip_grad_norm_if_needed(self.critic.transformer, self.training_args) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start @@ -246,29 +224,19 @@ def log_validation(self, iteration: int) -> None: if not is_validation_enabled(self.validation_config): return - every_steps = parse_validation_every_steps( - self.validation_config - ) + every_steps = parse_validation_every_steps(self.validation_config) if every_steps <= 0: return if iteration % every_steps != 0: return - dataset_file = parse_validation_dataset_file( - self.validation_config - ) - sampling_steps = parse_validation_sampling_steps( - self.validation_config - ) + dataset_file = parse_validation_dataset_file(self.validation_config) + sampling_steps = parse_validation_sampling_steps(self.validation_config) sampling_timesteps: list[int] | None = None - raw_timesteps = self.validation_config.get( - "sampling_timesteps", None - ) + raw_timesteps = self.validation_config.get("sampling_timesteps", None) if raw_timesteps is None: - raw_timesteps = self.method_config.get( - "dmd_denoising_steps", None - ) + raw_timesteps = self.method_config.get("dmd_denoising_steps", None) if isinstance(raw_timesteps, list) and raw_timesteps: sampling_timesteps = [int(s) for s in raw_timesteps] @@ -277,33 +245,18 @@ def log_validation(self, iteration: int) -> None: return sampling_steps = [int(len(sampling_timesteps))] - sampler_kind = parse_validation_sampler_kind( - self.validation_config, default="sde" - ) - ode_solver = parse_validation_ode_solver( - self.validation_config, sampler_kind=sampler_kind - ) - if ( - sampling_timesteps is not None - and sampler_kind != "sde" - ): - raise ValueError( - "method_config.validation.sampling_timesteps is " - "only valid when sampler_kind='sde'" - ) + sampler_kind = parse_validation_sampler_kind(self.validation_config, + default="sde") + ode_solver = parse_validation_ode_solver(self.validation_config, + sampler_kind=sampler_kind) + if (sampling_timesteps is not None and sampler_kind != "sde"): + raise ValueError("method_config.validation.sampling_timesteps is " + "only valid when sampler_kind='sde'") - rollout_mode = parse_validation_rollout_mode( - self.validation_config - ) - guidance_scale = parse_validation_guidance_scale( - self.validation_config - ) - output_dir = parse_validation_output_dir( - self.validation_config - ) - num_actions = parse_validation_num_frames( - self.validation_config - ) + rollout_mode = parse_validation_rollout_mode(self.validation_config) + guidance_scale = parse_validation_guidance_scale(self.validation_config) + output_dir = parse_validation_output_dir(self.validation_config) + num_actions = parse_validation_num_frames(self.validation_config) request = ValidationRequest( sample_handle=self.student, @@ -327,47 +280,35 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: generators.update(student_gens) validator = getattr(self, "validator", None) - validation_gen = getattr( - validator, "validation_random_generator", None - ) + validation_gen = getattr(validator, "validation_random_generator", None) if isinstance(validation_gen, torch.Generator): generators["validation_cpu"] = validation_gen return generators - def _parse_rollout_mode( - self, - ) -> Literal["simulate", "data_latent"]: + def _parse_rollout_mode(self, ) -> Literal["simulate", "data_latent"]: raw = self.method_config.get("rollout_mode", None) if raw is None: - raise ValueError( - "method_config.rollout_mode must be set for DMD2" - ) + raise ValueError("method_config.rollout_mode must be set for DMD2") if not isinstance(raw, str): - raise ValueError( - "method_config.rollout_mode must be a string, " - f"got {type(raw).__name__}" - ) + raise ValueError("method_config.rollout_mode must be a string, " + f"got {type(raw).__name__}") mode = raw.strip().lower() if mode in ("simulate", "sim"): return "simulate" if mode in ("data_latent", "data", "vae_latent"): return "data_latent" - raise ValueError( - "method_config.rollout_mode must be one of " - "{simulate, data_latent}, got " - f"{raw!r}" - ) + raise ValueError("method_config.rollout_mode must be one of " + "{simulate, data_latent}, got " + f"{raw!r}") def _parse_cfg_uncond(self) -> dict[str, Any] | None: raw = self.method_config.get("cfg_uncond", None) if raw is None: return None if not isinstance(raw, dict): - raise ValueError( - "method_config.cfg_uncond must be a dict when " - f"set, got {type(raw).__name__}" - ) + raise ValueError("method_config.cfg_uncond must be a dict when " + f"set, got {type(raw).__name__}") cfg: dict[str, Any] = dict(raw) @@ -375,17 +316,13 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: if on_missing_raw is None: on_missing_raw = "error" if not isinstance(on_missing_raw, str): - raise ValueError( - "method_config.cfg_uncond.on_missing must be a " - f"string, got {type(on_missing_raw).__name__}" - ) + raise ValueError("method_config.cfg_uncond.on_missing must be a " + f"string, got {type(on_missing_raw).__name__}") on_missing = on_missing_raw.strip().lower() if on_missing not in {"error", "ignore"}: - raise ValueError( - "method_config.cfg_uncond.on_missing must be one " - "of {error, ignore}, got " - f"{on_missing_raw!r}" - ) + raise ValueError("method_config.cfg_uncond.on_missing must be one " + "of {error, ignore}, got " + f"{on_missing_raw!r}") cfg["on_missing"] = on_missing for channel, policy_raw in list(cfg.items()): @@ -394,21 +331,17 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: if policy_raw is None: continue if not isinstance(policy_raw, str): - raise ValueError( - "method_config.cfg_uncond values must be " - "strings, got " - f"{channel}={type(policy_raw).__name__}" - ) + raise ValueError("method_config.cfg_uncond values must be " + "strings, got " + f"{channel}={type(policy_raw).__name__}") policy = policy_raw.strip().lower() allowed = {"keep", "zero", "drop"} if channel == "text": allowed = {*allowed, "negative_prompt"} if policy not in allowed: - raise ValueError( - "method_config.cfg_uncond values must be one " - f"of {sorted(allowed)}, got " - f"{channel}={policy_raw!r}" - ) + raise ValueError("method_config.cfg_uncond values must be one " + f"of {sorted(allowed)}, got " + f"{channel}={policy_raw!r}") cfg[channel] = policy return cfg @@ -417,20 +350,14 @@ def _init_optimizers_and_schedulers(self) -> None: training_args = self.training_args # Student optimizer/scheduler. - student_lr = float( - getattr(training_args, "learning_rate", 0.0) or 0.0 - ) + student_lr = float(getattr(training_args, "learning_rate", 0.0) or 0.0) student_betas = parse_betas( getattr(training_args, "betas", None), where="training.betas", ) - student_sched = str( - getattr(training_args, "lr_scheduler", "constant") - ) + student_sched = str(getattr(training_args, "lr_scheduler", "constant")) student_params = [ - p - for p in self.student.transformer.parameters() - if p.requires_grad + p for p in self.student.transformer.parameters() if p.requires_grad ] ( self._student_optimizer, @@ -445,35 +372,21 @@ def _init_optimizers_and_schedulers(self) -> None: # Critic optimizer/scheduler. critic_lr = float( - getattr( - training_args, "fake_score_learning_rate", 0.0 - ) - or 0.0 - ) + getattr(training_args, "fake_score_learning_rate", 0.0) or 0.0) if critic_lr == 0.0: critic_lr = student_lr - critic_betas_raw = getattr( - training_args, "fake_score_betas", None - ) + critic_betas_raw = getattr(training_args, "fake_score_betas", None) if critic_betas_raw is None: - critic_betas_raw = getattr( - training_args, "betas", None - ) - critic_betas = parse_betas( - critic_betas_raw, where="training.fake_score_betas" - ) + critic_betas_raw = getattr(training_args, "betas", None) + critic_betas = parse_betas(critic_betas_raw, + where="training.fake_score_betas") critic_sched = str( - getattr( - training_args, "fake_score_lr_scheduler", None - ) - or student_sched - ) + getattr(training_args, "fake_score_lr_scheduler", None) + or student_sched) critic_params = [ - p - for p in self.critic.transformer.parameters() - if p.requires_grad + p for p in self.critic.transformer.parameters() if p.requires_grad ] ( self._critic_optimizer, @@ -490,36 +403,23 @@ def _should_update_student(self, iteration: int) -> bool: interval = get_optional_int( self.method_config, "generator_update_interval", - where="method_config.generator_update_interval", + where="method.generator_update_interval", ) if interval is None: - interval = int( - getattr( - self.training_args, - "generator_update_interval", - 1, - ) - or 1 - ) + interval = 1 if interval <= 0: return True return iteration % interval == 0 - def _get_denoising_step_list( - self, device: torch.device - ) -> torch.Tensor: - if ( - self._denoising_step_list is not None - and self._denoising_step_list.device == device - ): + def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: + if (self._denoising_step_list is not None + and self._denoising_step_list.device == device): return self._denoising_step_list raw = self.method_config.get("dmd_denoising_steps", None) if not isinstance(raw, list) or not raw: - raise ValueError( - "method_config.dmd_denoising_steps must be set " - "for DMD2 distillation" - ) + raise ValueError("method_config.dmd_denoising_steps must be set " + "for DMD2 distillation") steps = torch.tensor( [int(s) for s in raw], @@ -529,26 +429,18 @@ def _get_denoising_step_list( warp = self.method_config.get("warp_denoising_step", None) if warp is None: - warp = getattr( - self.training_args, "warp_denoising_step", False - ) + warp = getattr(self.training_args, "warp_denoising_step", False) if bool(warp): - timesteps = torch.cat( - ( - self.student.noise_scheduler.timesteps.to( - "cpu" - ), - torch.tensor([0], dtype=torch.float32), - ) - ).to(device) + timesteps = torch.cat(( + self.student.noise_scheduler.timesteps.to("cpu"), + torch.tensor([0], dtype=torch.float32), + )).to(device) steps = timesteps[1000 - steps] self._denoising_step_list = steps return steps - def _sample_rollout_timestep( - self, device: torch.device - ) -> torch.Tensor: + def _sample_rollout_timestep(self, device: torch.device) -> torch.Tensor: step_list = self._get_denoising_step_list(device) index = torch.randint( 0, @@ -559,9 +451,7 @@ def _sample_rollout_timestep( ) return step_list[index] - def _student_rollout( - self, batch: Any, *, with_grad: bool - ) -> torch.Tensor: + def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: latents = batch.latents device = latents.device dtype = latents.dtype @@ -569,12 +459,8 @@ def _student_rollout( if self._rollout_mode != "simulate": timestep = self._sample_rollout_timestep(device) - noise = torch.randn( - latents.shape, device=device, dtype=dtype - ) - noisy_latents = self.student.add_noise( - latents, noise, timestep - ) + noise = torch.randn(latents.shape, device=device, dtype=dtype) + noisy_latents = self.student.add_noise(latents, noise, timestep) pred_x0 = self.student.predict_x0( noisy_latents, timestep, @@ -583,9 +469,7 @@ def _student_rollout( cfg_uncond=self._cfg_uncond, attn_kind="vsa", ) - batch.dmd_latent_vis_dict[ - "generator_timestep" - ] = timestep + batch.dmd_latent_vis_dict["generator_timestep"] = timestep return pred_x0 target_timestep_idx = torch.randint( @@ -595,14 +479,12 @@ def _student_rollout( device=device, dtype=torch.long, ) - target_timestep_idx_int = int( - target_timestep_idx.item() - ) + target_timestep_idx_int = int(target_timestep_idx.item()) target_timestep = step_list[target_timestep_idx] - current_noise_latents = torch.randn( - latents.shape, device=device, dtype=dtype - ) + current_noise_latents = torch.randn(latents.shape, + device=device, + dtype=dtype) current_noise_latents_copy = current_noise_latents.clone() max_target_idx = len(step_list) - 1 @@ -614,11 +496,8 @@ def _student_rollout( for step_idx in range(max_target_idx): current_timestep = step_list[step_idx] current_timestep_tensor = ( - current_timestep - * torch.ones( - 1, device=device, dtype=torch.long - ) - ) + current_timestep * + torch.ones(1, device=device, dtype=torch.long)) pred_clean = self.student.predict_x0( current_noise_latents, @@ -631,32 +510,23 @@ def _student_rollout( next_timestep = step_list[step_idx + 1] next_timestep_tensor = ( - next_timestep - * torch.ones( - 1, device=device, dtype=torch.long - ) - ) + next_timestep * + torch.ones(1, device=device, dtype=torch.long)) noise = torch.randn( latents.shape, device=device, dtype=pred_clean.dtype, ) - current_noise_latents = ( - self.student.add_noise( - pred_clean, - noise, - next_timestep_tensor, - ) - ) - noise_latents.append( - current_noise_latents.clone() - ) + current_noise_latents = (self.student.add_noise( + pred_clean, + noise, + next_timestep_tensor, + )) + noise_latents.append(current_noise_latents.clone()) if noise_latent_index >= 0: if noise_latent_index >= len(noise_latents): - raise RuntimeError( - "noise_latent_index is out of bounds" - ) + raise RuntimeError("noise_latent_index is out of bounds") noisy_input = noise_latents[noise_latent_index] else: noisy_input = current_noise_latents_copy @@ -681,18 +551,14 @@ def _student_rollout( attn_kind="vsa", ) - batch.dmd_latent_vis_dict[ - "generator_timestep" - ] = target_timestep.float().detach() + batch.dmd_latent_vis_dict["generator_timestep"] = target_timestep.float( + ).detach() return pred_x0 def _critic_flow_matching_loss( - self, batch: Any - ) -> tuple[torch.Tensor, Any, dict[str, Any]]: + self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: with torch.no_grad(): - generator_pred_x0 = self._student_rollout( - batch, with_grad=False - ) + generator_pred_x0 = self._student_rollout(batch, with_grad=False) device = generator_pred_x0.device fake_score_timestep = torch.randint( @@ -703,19 +569,15 @@ def _critic_flow_matching_loss( dtype=torch.long, ) fake_score_timestep = ( - self.student.shift_and_clamp_timestep( - fake_score_timestep - ) - ) + self.student.shift_and_clamp_timestep(fake_score_timestep)) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_x0 = self.student.add_noise( - generator_pred_x0, noise, fake_score_timestep - ) + noisy_x0 = self.student.add_noise(generator_pred_x0, noise, + fake_score_timestep) pred_noise = self.critic.predict_noise( noisy_x0, @@ -726,18 +588,14 @@ def _critic_flow_matching_loss( attn_kind="dense", ) target = noise - generator_pred_x0 - flow_matching_loss = torch.mean( - (pred_noise - target) ** 2 - ) + flow_matching_loss = torch.mean((pred_noise - target)**2) batch.fake_score_latent_vis_dict = { "generator_pred_video": generator_pred_x0, "fake_score_timestep": fake_score_timestep, } outputs = { - "fake_score_latent_vis_dict": ( - batch.fake_score_latent_vis_dict - ) + "fake_score_latent_vis_dict": (batch.fake_score_latent_vis_dict) } return ( flow_matching_loss, @@ -753,16 +611,10 @@ def _dmd_loss( guidance_scale = get_optional_float( self.method_config, "real_score_guidance_scale", - where="method_config.real_score_guidance_scale", + where="method.real_score_guidance_scale", ) if guidance_scale is None: - guidance_scale = float( - getattr( - self.training_args, - "real_score_guidance_scale", - 1.0, - ) - ) + guidance_scale = 1.0 device = generator_pred_x0.device with torch.no_grad(): @@ -773,18 +625,15 @@ def _dmd_loss( device=device, dtype=torch.long, ) - timestep = self.student.shift_and_clamp_timestep( - timestep - ) + timestep = self.student.shift_and_clamp_timestep(timestep) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_latents = self.student.add_noise( - generator_pred_x0, noise, timestep - ) + noisy_latents = self.student.add_noise(generator_pred_x0, noise, + timestep) faker_x0 = self.critic.predict_x0( noisy_latents, @@ -810,20 +659,15 @@ def _dmd_loss( cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + ( - real_cond_x0 - real_uncond_x0 - ) * guidance_scale + real_cfg_x0 = real_cond_x0 + (real_cond_x0 - + real_uncond_x0) * guidance_scale - denom = torch.abs( - generator_pred_x0 - real_cfg_x0 - ).mean() + denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() grad = (faker_x0 - real_cfg_x0) / denom grad = torch.nan_to_num(grad) loss = 0.5 * F.mse_loss( generator_pred_x0.float(), - ( - generator_pred_x0.float() - grad.float() - ).detach(), + (generator_pred_x0.float() - grad.float()).detach(), ) return loss diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/distillation/methods/distribution_matching/self_forcing.py index 5e9156b00..10fe07be5 100644 --- a/fastvideo/distillation/methods/distribution_matching/self_forcing.py +++ b/fastvideo/distillation/methods/distribution_matching/self_forcing.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """Self-Forcing distillation method (algorithm layer).""" from __future__ import annotations @@ -14,15 +13,13 @@ ModelBase, ) from fastvideo.distillation.methods.distribution_matching.dmd2 import ( - DMD2Method, -) + DMD2Method, ) from fastvideo.distillation.utils.config import ( get_optional_float, get_optional_int, ) from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( - SelfForcingFlowMatchScheduler, -) + SelfForcingFlowMatchScheduler, ) from fastvideo.models.utils import pred_noise_to_pred_video if TYPE_CHECKING: @@ -32,16 +29,12 @@ def _require_bool(raw: Any, *, where: str) -> bool: if isinstance(raw, bool): return raw - raise ValueError( - f"Expected bool at {where}, got {type(raw).__name__}" - ) + raise ValueError(f"Expected bool at {where}, got {type(raw).__name__}") def _require_str(raw: Any, *, where: str) -> str: if not isinstance(raw, str) or not raw.strip(): - raise ValueError( - f"Expected non-empty string at {where}" - ) + raise ValueError(f"Expected non-empty string at {where}") return raw @@ -66,16 +59,12 @@ def __init__( # Validate causal student. if not isinstance(self.student, CausalModelBase): - raise ValueError( - "SelfForcingMethod requires a causal student " - "implementing CausalModelBase." - ) + raise ValueError("SelfForcingMethod requires a causal student " + "implementing CausalModelBase.") if self._rollout_mode != "simulate": - raise ValueError( - "SelfForcingMethod only supports " - "method_config.rollout_mode='simulate'" - ) + raise ValueError("SelfForcingMethod only supports " + "method_config.rollout_mode='simulate'") mcfg = self.method_config @@ -87,10 +76,8 @@ def __init__( if chunk_size is None: chunk_size = 3 if chunk_size <= 0: - raise ValueError( - "method_config.chunk_size must be a positive " - f"integer, got {chunk_size}" - ) + raise ValueError("method_config.chunk_size must be a positive " + f"integer, got {chunk_size}") self._chunk_size = int(chunk_size) sample_type_raw = mcfg.get("student_sample_type", "sde") @@ -100,10 +87,8 @@ def __init__( ) sample_type = sample_type.strip().lower() if sample_type not in {"sde", "ode"}: - raise ValueError( - "method_config.student_sample_type must be one " - f"of {{sde, ode}}, got {sample_type_raw!r}" - ) + raise ValueError("method_config.student_sample_type must be one " + f"of {{sde, ode}}, got {sample_type_raw!r}") self._student_sample_type: Literal["sde", "ode"] = ( sample_type # type: ignore[assignment] ) @@ -132,15 +117,11 @@ def __init__( if context_noise is None: context_noise = 0.0 if context_noise < 0.0: - raise ValueError( - "method_config.context_noise must be >= 0, " - f"got {context_noise}" - ) + raise ValueError("method_config.context_noise must be >= 0, " + f"got {context_noise}") self._context_noise = float(context_noise) - enable_grad_raw = mcfg.get( - "enable_gradient_in_rollout", True - ) + enable_grad_raw = mcfg.get("enable_gradient_in_rollout", True) if enable_grad_raw is None: enable_grad_raw = True self._enable_gradient_in_rollout = _require_bool( @@ -156,10 +137,8 @@ def __init__( if start_grad_frame is None: start_grad_frame = 0 if start_grad_frame < 0: - raise ValueError( - "method_config.start_gradient_frame must be " - f">= 0, got {start_grad_frame}" - ) + raise ValueError("method_config.start_gradient_frame must be " + f">= 0, got {start_grad_frame}") self._start_gradient_frame = int(start_grad_frame) shift = float( @@ -167,14 +146,10 @@ def __init__( self.training_args.pipeline_config, "flow_shift", 0.0, - ) - or 0.0 - ) + ) or 0.0) self._sf_scheduler = SelfForcingFlowMatchScheduler( num_inference_steps=1000, - num_train_timesteps=int( - self.student.num_train_timesteps - ), + num_train_timesteps=int(self.student.num_train_timesteps), shift=shift, sigma_min=0.0, extra_one_step=True, @@ -183,21 +158,15 @@ def __init__( self._sf_denoising_step_list: torch.Tensor | None = None - def _get_denoising_step_list( - self, device: torch.device - ) -> torch.Tensor: - if ( - self._sf_denoising_step_list is not None - and self._sf_denoising_step_list.device == device - ): + def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: + if (self._sf_denoising_step_list is not None + and self._sf_denoising_step_list.device == device): return self._sf_denoising_step_list raw = self.method_config.get("dmd_denoising_steps", None) if not isinstance(raw, list) or not raw: - raise ValueError( - "method_config.dmd_denoising_steps must be set " - "for self_forcing" - ) + raise ValueError("method_config.dmd_denoising_steps must be set " + "for self_forcing") steps = torch.tensor( [int(s) for s in raw], dtype=torch.long, @@ -206,19 +175,13 @@ def _get_denoising_step_list( warp = self.method_config.get("warp_denoising_step", None) if warp is None: - warp = getattr( - self.training_args, "warp_denoising_step", False - ) + warp = getattr(self.training_args, "warp_denoising_step", False) if bool(warp): - timesteps = torch.cat( - ( - self._sf_scheduler.timesteps.to("cpu"), - torch.tensor([0], dtype=torch.float32), - ) - ).to(device) - steps = timesteps[ - int(self.student.num_train_timesteps) - steps - ] + timesteps = torch.cat(( + self._sf_scheduler.timesteps.to("cpu"), + torch.tensor([0], dtype=torch.float32), + )).to(device) + steps = timesteps[int(self.student.num_train_timesteps) - steps] self._sf_denoising_step_list = steps return steps @@ -263,27 +226,19 @@ def _sf_add_noise( ).unflatten(0, (b, t)) return noisy - def _timestep_to_sigma( - self, timestep: torch.Tensor - ) -> torch.Tensor: - sigmas = self._sf_scheduler.sigmas.to( - device=timestep.device, dtype=torch.float32 - ) - timesteps = self._sf_scheduler.timesteps.to( - device=timestep.device, dtype=torch.float32 - ) - t = timestep.to( - device=timestep.device, dtype=torch.float32 - ) + def _timestep_to_sigma(self, timestep: torch.Tensor) -> torch.Tensor: + sigmas = self._sf_scheduler.sigmas.to(device=timestep.device, + dtype=torch.float32) + timesteps = self._sf_scheduler.timesteps.to(device=timestep.device, + dtype=torch.float32) + t = timestep.to(device=timestep.device, dtype=torch.float32) if t.ndim == 2: t = t.flatten(0, 1) elif t.ndim == 1 and t.numel() == 1: t = t.expand(1) elif t.ndim != 1: - raise ValueError( - "Invalid timestep shape: " - f"{tuple(timestep.shape)}" - ) + raise ValueError("Invalid timestep shape: " + f"{tuple(timestep.shape)}") idx = torch.argmin( (timesteps.unsqueeze(0) - t.unsqueeze(1)).abs(), dim=1, @@ -302,11 +257,7 @@ def _sample_exit_indices( if num_steps <= 0: raise ValueError("num_steps must be positive") - shape = ( - (1,) - if self._same_step_across_blocks - else (num_blocks,) - ) + shape = ((1, ) if self._same_step_across_blocks else (num_blocks, )) if not dist.is_initialized() or dist.get_rank() == 0: if self._last_step_only: @@ -324,46 +275,31 @@ def _sample_exit_indices( device=device, ) else: - indices = torch.empty( - shape, dtype=torch.long, device=device - ) + indices = torch.empty(shape, dtype=torch.long, device=device) if dist.is_initialized(): dist.broadcast(indices, src=0) if self._same_step_across_blocks: - return [ - int(indices.item()) for _ in range(num_blocks) - ] + return [int(indices.item()) for _ in range(num_blocks)] return [int(i) for i in indices.tolist()] - def _student_rollout( - self, batch: Any, *, with_grad: bool - ) -> torch.Tensor: + def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: if not isinstance(self.student, CausalModelBase): - raise ValueError( - "SelfForcingMethod requires a causal student " - "implementing CausalModelBase." - ) - return self._student_rollout_streaming( - batch, with_grad=with_grad - ) + raise ValueError("SelfForcingMethod requires a causal student " + "implementing CausalModelBase.") + return self._student_rollout_streaming(batch, with_grad=with_grad) - def _student_rollout_streaming( - self, batch: Any, *, with_grad: bool - ) -> torch.Tensor: + def _student_rollout_streaming(self, batch: Any, *, + with_grad: bool) -> torch.Tensor: assert isinstance(self.student, CausalModelBase) latents = batch.latents if latents is None: - raise RuntimeError( - "TrainingBatch.latents is required for " - "self-forcing rollout" - ) + raise RuntimeError("TrainingBatch.latents is required for " + "self-forcing rollout") if latents.ndim != 5: - raise ValueError( - "TrainingBatch.latents must be [B, T, C, H, W]" - f", got shape={tuple(latents.shape)}" - ) + raise ValueError("TrainingBatch.latents must be [B, T, C, H, W]" + f", got shape={tuple(latents.shape)}") device = latents.device dtype = latents.dtype @@ -373,9 +309,7 @@ def _student_rollout_streaming( denoising_steps = self._get_denoising_step_list(device) num_steps = int(denoising_steps.numel()) - noise_full = torch.randn_like( - latents, device=device, dtype=dtype - ) + noise_full = torch.randn_like(latents, device=device, dtype=dtype) chunk = int(self._chunk_size) if chunk <= 0: @@ -413,53 +347,40 @@ def _student_rollout_streaming( noisy_block = noise_full[:, start:end] exit_idx = int(exit_indices[block_idx]) - for step_idx, current_timestep in enumerate( - denoising_steps - ): + for step_idx, current_timestep in enumerate(denoising_steps): exit_flag = step_idx == exit_idx - timestep_block = ( - current_timestep - * torch.ones( - (batch_size, end - start), - device=device, - dtype=torch.float32, - ) - ) + timestep_block = (current_timestep * torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + )) - enable_grad = ( - bool(with_grad) - and bool(self._enable_gradient_in_rollout) - and torch.is_grad_enabled() - and start >= int(self._start_gradient_frame) - ) + enable_grad = (bool(with_grad) + and bool(self._enable_gradient_in_rollout) + and torch.is_grad_enabled() + and start >= int(self._start_gradient_frame)) if not exit_flag: with torch.no_grad(): - pred_noise = ( - self.student.predict_noise_streaming( - noisy_block, - timestep_block, - batch, - conditional=True, - cache_tag=cache_tag, - store_kv=False, - cur_start_frame=start, - cfg_uncond=self._cfg_uncond, - attn_kind="vsa", - ) - ) + pred_noise = (self.student.predict_noise_streaming( + noisy_block, + timestep_block, + batch, + conditional=True, + cache_tag=cache_tag, + store_kv=False, + cur_start_frame=start, + cfg_uncond=self._cfg_uncond, + attn_kind="vsa", + )) if pred_noise is None: - raise RuntimeError( - "predict_noise_streaming " - "returned None " - "(store_kv=False)" - ) + raise RuntimeError("predict_noise_streaming " + "returned None " + "(store_kv=False)") pred_x0_chunk = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=( - noisy_block.flatten(0, 1) - ), + noise_input_latent=(noisy_block.flatten(0, 1)), timestep=timestep_block, scheduler=self._sf_scheduler, ).unflatten(0, pred_noise.shape[:2]) @@ -471,8 +392,7 @@ def _student_rollout_streaming( noisy_block = self._sf_add_noise( pred_x0_chunk, torch.randn_like(pred_x0_chunk), - next_timestep - * torch.ones( + next_timestep * torch.ones( (batch_size, end - start), device=device, dtype=torch.float32, @@ -480,54 +400,38 @@ def _student_rollout_streaming( ) else: sigma_cur = self._timestep_to_sigma( - timestep_block - ).view( - batch_size, end - start, 1, 1, 1 - ) + timestep_block).view(batch_size, end - start, 1, 1, + 1) sigma_next = self._timestep_to_sigma( - next_timestep - * torch.ones( + next_timestep * torch.ones( (batch_size, end - start), device=device, dtype=torch.float32, - ) - ).view( - batch_size, end - start, 1, 1, 1 - ) - eps = ( - noisy_block - - (1 - sigma_cur) * pred_x0_chunk - ) / sigma_cur.clamp_min(1e-8) - noisy_block = ( - (1 - sigma_next) * pred_x0_chunk - + sigma_next * eps - ) + )).view(batch_size, end - start, 1, 1, 1) + eps = (noisy_block - (1 - sigma_cur) * + pred_x0_chunk) / sigma_cur.clamp_min(1e-8) + noisy_block = ((1 - sigma_next) * pred_x0_chunk + + sigma_next * eps) continue with torch.set_grad_enabled(enable_grad): - pred_noise = ( - self.student.predict_noise_streaming( - noisy_block, - timestep_block, - batch, - conditional=True, - cache_tag=cache_tag, - store_kv=False, - cur_start_frame=start, - cfg_uncond=self._cfg_uncond, - attn_kind="vsa", - ) - ) + pred_noise = (self.student.predict_noise_streaming( + noisy_block, + timestep_block, + batch, + conditional=True, + cache_tag=cache_tag, + store_kv=False, + cur_start_frame=start, + cfg_uncond=self._cfg_uncond, + attn_kind="vsa", + )) if pred_noise is None: - raise RuntimeError( - "predict_noise_streaming returned " - "None (store_kv=False)" - ) + raise RuntimeError("predict_noise_streaming returned " + "None (store_kv=False)") pred_x0_chunk = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=( - noisy_block.flatten(0, 1) - ), + noise_input_latent=(noisy_block.flatten(0, 1)), timestep=timestep_block, scheduler=self._sf_scheduler, ).unflatten(0, pred_noise.shape[:2]) @@ -568,20 +472,15 @@ def _student_rollout_streaming( ) if not denoised_blocks: - raise RuntimeError( - "Self-forcing rollout produced no blocks" - ) + raise RuntimeError("Self-forcing rollout produced no blocks") self.student.clear_caches(cache_tag=cache_tag) return torch.cat(denoised_blocks, dim=1) def _critic_flow_matching_loss( - self, batch: Any - ) -> tuple[torch.Tensor, Any, dict[str, Any]]: + self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: with torch.no_grad(): - generator_pred_x0 = self._student_rollout( - batch, with_grad=False - ) + generator_pred_x0 = self._student_rollout(batch, with_grad=False) device = generator_pred_x0.device fake_score_timestep = torch.randint( @@ -592,19 +491,15 @@ def _critic_flow_matching_loss( dtype=torch.long, ) fake_score_timestep = ( - self.student.shift_and_clamp_timestep( - fake_score_timestep - ) - ) + self.student.shift_and_clamp_timestep(fake_score_timestep)) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_x0 = self._sf_add_noise( - generator_pred_x0, noise, fake_score_timestep - ) + noisy_x0 = self._sf_add_noise(generator_pred_x0, noise, + fake_score_timestep) pred_noise = self.critic.predict_noise( noisy_x0, @@ -615,18 +510,14 @@ def _critic_flow_matching_loss( attn_kind="dense", ) target = noise - generator_pred_x0 - flow_matching_loss = torch.mean( - (pred_noise - target) ** 2 - ) + flow_matching_loss = torch.mean((pred_noise - target)**2) batch.fake_score_latent_vis_dict = { "generator_pred_video": generator_pred_x0, "fake_score_timestep": fake_score_timestep, } outputs = { - "fake_score_latent_vis_dict": ( - batch.fake_score_latent_vis_dict - ) + "fake_score_latent_vis_dict": (batch.fake_score_latent_vis_dict) } return ( flow_matching_loss, @@ -642,16 +533,10 @@ def _dmd_loss( guidance_scale = get_optional_float( self.method_config, "real_score_guidance_scale", - where="method_config.real_score_guidance_scale", + where="method.real_score_guidance_scale", ) if guidance_scale is None: - guidance_scale = float( - getattr( - self.training_args, - "real_score_guidance_scale", - 1.0, - ) - ) + guidance_scale = 1.0 device = generator_pred_x0.device with torch.no_grad(): @@ -662,18 +547,15 @@ def _dmd_loss( device=device, dtype=torch.long, ) - timestep = self.student.shift_and_clamp_timestep( - timestep - ) + timestep = self.student.shift_and_clamp_timestep(timestep) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_latents = self._sf_add_noise( - generator_pred_x0, noise, timestep - ) + noisy_latents = self._sf_add_noise(generator_pred_x0, noise, + timestep) faker_x0 = self._predict_x0_with_scheduler( self.critic, @@ -699,23 +581,14 @@ def _dmd_loss( conditional=False, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + ( - real_cond_x0 - real_uncond_x0 - ) * guidance_scale + real_cfg_x0 = real_cond_x0 + (real_cond_x0 - + real_uncond_x0) * guidance_scale - denom = torch.abs( - generator_pred_x0 - real_cfg_x0 - ).mean() + denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() grad = (faker_x0 - real_cfg_x0) / denom grad = torch.nan_to_num(grad) loss = 0.5 * torch.mean( - ( - generator_pred_x0.float() - - ( - generator_pred_x0.float() - grad.float() - ).detach() - ) - ** 2 - ) + (generator_pred_x0.float() - + (generator_pred_x0.float() - grad.float()).detach())**2) return loss diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index 5a52d90ef..cb1fe477e 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """Diffusion-forcing SFT method (DFSFT; algorithm layer).""" from __future__ import annotations @@ -52,30 +51,19 @@ def __init__( raise ValueError("DFSFT requires role 'student'") self.student = role_models["student"] if not getattr(self.student, "_trainable", True): - raise ValueError( - "DFSFT requires student to be trainable" - ) + raise ValueError("DFSFT requires student to be trainable") self.validator = validator self.training_args = cfg.training_args - self.method_config: dict[str, Any] = dict( - getattr(cfg, "method_config", {}) or {} - ) + self.method_config: dict[str, Any] = dict(cfg.method) self.validation_config: dict[str, Any] = dict( - getattr(cfg, "validation", {}) or {} - ) - self._attn_kind: Literal["dense", "vsa"] = ( - self._parse_attn_kind( - self.method_config.get("attn_kind", None) - ) - ) + getattr(cfg, "validation", {}) or {}) + self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind( + self.method_config.get("attn_kind", None))) self._chunk_size = self._parse_chunk_size( - self.method_config.get("chunk_size", None) - ) - self._timestep_index_range = ( - self._parse_timestep_index_range() - ) + self.method_config.get("chunk_size", None)) + self._timestep_index_range = (self._parse_timestep_index_range()) # Initialize preprocessors on student. self.student.init_preprocessors(self.training_args) @@ -90,9 +78,9 @@ def single_train_step( *, current_vsa_sparsity: float = 0.0, ) -> tuple[ - dict[str, torch.Tensor], - dict[str, Any], - dict[str, LogScalar], + dict[str, torch.Tensor], + dict[str, Any], + dict[str, LogScalar], ]: del iteration training_batch = self.student.prepare_batch( @@ -102,67 +90,48 @@ def single_train_step( ) if training_batch.latents is None: - raise RuntimeError( - "prepare_batch() must set TrainingBatch.latents" - ) + raise RuntimeError("prepare_batch() must set TrainingBatch.latents") clean_latents = training_batch.latents if not torch.is_tensor(clean_latents): - raise TypeError( - "TrainingBatch.latents must be a torch.Tensor" - ) + raise TypeError("TrainingBatch.latents must be a torch.Tensor") if clean_latents.ndim != 5: - raise ValueError( - "TrainingBatch.latents must be " - "[B, T, C, H, W], got " - f"shape={tuple(clean_latents.shape)}" - ) + raise ValueError("TrainingBatch.latents must be " + "[B, T, C, H, W], got " + f"shape={tuple(clean_latents.shape)}") - batch_size, num_latents = int( - clean_latents.shape[0] - ), int(clean_latents.shape[1]) + batch_size, num_latents = int(clean_latents.shape[0]), int( + clean_latents.shape[1]) expected_chunk = getattr( self.student.transformer, "num_frame_per_block", None, ) - if ( - expected_chunk is not None - and int(expected_chunk) != int(self._chunk_size) - ): - raise ValueError( - "DFSFT chunk_size must match " - "transformer.num_frame_per_block for " - f"causal training (got {self._chunk_size}, " - f"expected {expected_chunk})." - ) + if (expected_chunk is not None + and int(expected_chunk) != int(self._chunk_size)): + raise ValueError("DFSFT chunk_size must match " + "transformer.num_frame_per_block for " + f"causal training (got {self._chunk_size}, " + f"expected {expected_chunk}).") timestep_indices = self._sample_t_inhom_indices( batch_size=batch_size, num_latents=num_latents, device=clean_latents.device, ) - sp_size = int( - getattr(self.training_args, "sp_size", 1) or 1 - ) + sp_size = int(getattr(self.training_args, "sp_size", 1) or 1) sp_group = getattr(self.student, "sp_group", None) - if ( - sp_size > 1 - and sp_group is not None - and hasattr(sp_group, "broadcast") - ): + if (sp_size > 1 and sp_group is not None + and hasattr(sp_group, "broadcast")): sp_group.broadcast(timestep_indices, src=0) scheduler = self.student.noise_scheduler if scheduler is None: - raise ValueError( - "DFSFT requires student.noise_scheduler" - ) + raise ValueError("DFSFT requires student.noise_scheduler") - schedule_timesteps = scheduler.timesteps.to( - device=clean_latents.device, dtype=torch.float32 - ) + schedule_timesteps = scheduler.timesteps.to(device=clean_latents.device, + dtype=torch.float32) schedule_sigmas = scheduler.sigmas.to( device=clean_latents.device, dtype=clean_latents.dtype, @@ -174,13 +143,9 @@ def single_train_step( noise = torch.randn_like(clean_latents) else: if not torch.is_tensor(noise): - raise TypeError( - "TrainingBatch.noise must be a " - "torch.Tensor when set" - ) - noise = noise.permute(0, 2, 1, 3, 4).to( - dtype=clean_latents.dtype - ) + raise TypeError("TrainingBatch.noise must be a " + "torch.Tensor when set") + noise = noise.permute(0, 2, 1, 3, 4).to(dtype=clean_latents.dtype) noisy_latents = self.student.add_noise( clean_latents, @@ -196,21 +161,15 @@ def single_train_step( attn_kind=self._attn_kind, ) - if bool( - getattr( + if bool(getattr( self.training_args, "precondition_outputs", False, - ) - ): + )): sigmas = schedule_sigmas[timestep_indices] - sigmas = sigmas.unsqueeze(-1).unsqueeze( - -1 - ).unsqueeze(-1) + sigmas = sigmas.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) pred_x0 = noisy_latents - pred * sigmas - loss = F.mse_loss( - pred_x0.float(), clean_latents.float() - ) + loss = F.mse_loss(pred_x0.float(), clean_latents.float()) else: target = noise - clean_latents loss = F.mse_loss(pred.float(), target.float()) @@ -254,9 +213,7 @@ def backward( ) # DistillMethod override: get_optimizers - def get_optimizers( - self, iteration: int - ) -> list[torch.optim.Optimizer]: + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: del iteration return [self._student_optimizer] @@ -267,9 +224,7 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: - clip_grad_norm_if_needed( - self.student.transformer, self.training_args - ) + clip_grad_norm_if_needed(self.student.transformer, self.training_args) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start @@ -284,38 +239,22 @@ def log_validation(self, iteration: int) -> None: if not is_validation_enabled(self.validation_config): return - every_steps = parse_validation_every_steps( - self.validation_config - ) + every_steps = parse_validation_every_steps(self.validation_config) if every_steps <= 0: return if iteration % every_steps != 0: return - dataset_file = parse_validation_dataset_file( - self.validation_config - ) - sampling_steps = parse_validation_sampling_steps( - self.validation_config - ) - guidance_scale = parse_validation_guidance_scale( - self.validation_config - ) - sampler_kind = parse_validation_sampler_kind( - self.validation_config, default="ode" - ) - rollout_mode = parse_validation_rollout_mode( - self.validation_config - ) - output_dir = parse_validation_output_dir( - self.validation_config - ) - num_actions = parse_validation_num_frames( - self.validation_config - ) - ode_solver = parse_validation_ode_solver( - self.validation_config, sampler_kind=sampler_kind - ) + dataset_file = parse_validation_dataset_file(self.validation_config) + sampling_steps = parse_validation_sampling_steps(self.validation_config) + guidance_scale = parse_validation_guidance_scale(self.validation_config) + sampler_kind = parse_validation_sampler_kind(self.validation_config, + default="ode") + rollout_mode = parse_validation_rollout_mode(self.validation_config) + output_dir = parse_validation_output_dir(self.validation_config) + num_actions = parse_validation_num_frames(self.validation_config) + ode_solver = parse_validation_ode_solver(self.validation_config, + sampler_kind=sampler_kind) request = ValidationRequest( sample_handle=self.student, @@ -339,124 +278,78 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: generators.update(student_gens) validator = getattr(self, "validator", None) - validation_gen = getattr( - validator, "validation_random_generator", None - ) + validation_gen = getattr(validator, "validation_random_generator", None) if isinstance(validation_gen, torch.Generator): generators["validation_cpu"] = validation_gen return generators - def _parse_attn_kind( - self, raw: Any - ) -> Literal["dense", "vsa"]: + def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: if raw in (None, ""): return "dense" kind = str(raw).strip().lower() if kind not in {"dense", "vsa"}: - raise ValueError( - "method_config.attn_kind must be one of " - f"{{'dense', 'vsa'}}, got {raw!r}." - ) + raise ValueError("method_config.attn_kind must be one of " + f"{{'dense', 'vsa'}}, got {raw!r}.") return cast(Literal["dense", "vsa"], kind) def _parse_chunk_size(self, raw: Any) -> int: if raw in (None, ""): return 3 if isinstance(raw, bool): - raise ValueError( - "method_config.chunk_size must be an int, " - "got bool" - ) + raise ValueError("method_config.chunk_size must be an int, " + "got bool") if isinstance(raw, float) and not raw.is_integer(): - raise ValueError( - "method_config.chunk_size must be an int, " - "got float" - ) + raise ValueError("method_config.chunk_size must be an int, " + "got float") if isinstance(raw, str) and not raw.strip(): - raise ValueError( - "method_config.chunk_size must be an int, " - "got empty string" - ) + raise ValueError("method_config.chunk_size must be an int, " + "got empty string") try: value = int(raw) except (TypeError, ValueError) as e: - raise ValueError( - "method_config.chunk_size must be an int, " - f"got {type(raw).__name__}" - ) from e + raise ValueError("method_config.chunk_size must be an int, " + f"got {type(raw).__name__}") from e if value <= 0: - raise ValueError( - "method_config.chunk_size must be > 0" - ) + raise ValueError("method_config.chunk_size must be > 0") return value - def _parse_ratio( - self, raw: Any, *, where: str, default: float - ) -> float: + def _parse_ratio(self, raw: Any, *, where: str, default: float) -> float: if raw in (None, ""): return float(default) if isinstance(raw, bool): - raise ValueError( - f"{where} must be a number/string, got bool" - ) - if isinstance(raw, (int, float)): + raise ValueError(f"{where} must be a number/string, got bool") + if isinstance(raw, int | float): return float(raw) if isinstance(raw, str) and raw.strip(): return float(raw) - raise ValueError( - f"{where} must be a number/string, " - f"got {type(raw).__name__}" - ) + raise ValueError(f"{where} must be a number/string, " + f"got {type(raw).__name__}") def _parse_timestep_index_range(self) -> tuple[int, int]: scheduler = self.student.noise_scheduler if scheduler is None: - raise ValueError( - "DFSFT requires student.noise_scheduler" - ) + raise ValueError("DFSFT requires student.noise_scheduler") num_steps = int( - getattr(scheduler, "config", scheduler) - .num_train_timesteps - ) + getattr(scheduler, "config", scheduler).num_train_timesteps) min_ratio = self._parse_ratio( self.method_config.get("min_timestep_ratio", None), - where="method_config.min_timestep_ratio", - default=float( - getattr( - self.training_args, - "min_timestep_ratio", - 0.0, - ) - or 0.0 - ), + where="method.min_timestep_ratio", + default=0.0, ) max_ratio = self._parse_ratio( self.method_config.get("max_timestep_ratio", None), - where="method_config.max_timestep_ratio", - default=float( - getattr( - self.training_args, - "max_timestep_ratio", - 1.0, - ) - or 1.0 - ), + where="method.max_timestep_ratio", + default=1.0, ) - if not ( - 0.0 <= min_ratio <= 1.0 and 0.0 <= max_ratio <= 1.0 - ): - raise ValueError( - "DFSFT timestep ratios must be in [0,1], " - f"got min={min_ratio}, max={max_ratio}" - ) + if not (0.0 <= min_ratio <= 1.0 and 0.0 <= max_ratio <= 1.0): + raise ValueError("DFSFT timestep ratios must be in [0,1], " + f"got min={min_ratio}, max={max_ratio}") if max_ratio < min_ratio: - raise ValueError( - "method_config.max_timestep_ratio must be " - ">= min_timestep_ratio" - ) + raise ValueError("method_config.max_timestep_ratio must be " + ">= min_timestep_ratio") min_index = int(min_ratio * num_steps) max_index = int(max_ratio * num_steps) @@ -470,27 +363,18 @@ def _parse_timestep_index_range(self) -> tuple[int, int]: def _init_optimizers_and_schedulers(self) -> None: student_lr = float( - getattr(self.training_args, "learning_rate", 0.0) - or 0.0 - ) + getattr(self.training_args, "learning_rate", 0.0) or 0.0) if student_lr <= 0.0: - raise ValueError( - "training.learning_rate must be > 0 for dfsft" - ) + raise ValueError("training.learning_rate must be > 0 for dfsft") student_betas = parse_betas( getattr(self.training_args, "betas", None), where="training.betas", ) student_sched = str( - getattr( - self.training_args, "lr_scheduler", "constant" - ) - ) + getattr(self.training_args, "lr_scheduler", "constant")) student_params = [ - p - for p in self.student.transformer.parameters() - if p.requires_grad + p for p in self.student.transformer.parameters() if p.requires_grad ] ( self._student_optimizer, @@ -511,9 +395,7 @@ def _sample_t_inhom_indices( device: torch.device, ) -> torch.Tensor: chunk_size = self._chunk_size - num_chunks = ( - num_latents + chunk_size - 1 - ) // chunk_size + num_chunks = (num_latents + chunk_size - 1) // chunk_size low, high = self._timestep_index_range chunk_indices = torch.randint( low=low, @@ -522,7 +404,5 @@ def _sample_t_inhom_indices( device=device, dtype=torch.long, ) - expanded = chunk_indices.repeat_interleave( - chunk_size, dim=1 - ) + expanded = chunk_indices.repeat_interleave(chunk_size, dim=1) return expanded[:, :num_latents] diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 1ce3a7218..9d042b256 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """Supervised finetuning method (algorithm layer).""" from __future__ import annotations @@ -47,28 +46,18 @@ def __init__( super().__init__(role_models=role_models) if "student" not in role_models: - raise ValueError( - "FineTuneMethod requires role 'student'" - ) + raise ValueError("FineTuneMethod requires role 'student'") self.student = role_models["student"] if not getattr(self.student, "_trainable", True): - raise ValueError( - "FineTuneMethod requires student to be trainable" - ) + raise ValueError("FineTuneMethod requires student to be trainable") self.validator = validator self.training_args = cfg.training_args - self.method_config: dict[str, Any] = dict( - getattr(cfg, "method_config", {}) or {} - ) + self.method_config: dict[str, Any] = dict(cfg.method) self.validation_config: dict[str, Any] = dict( - getattr(cfg, "validation", {}) or {} - ) - self._attn_kind: Literal["dense", "vsa"] = ( - self._parse_attn_kind( - self.method_config.get("attn_kind", None) - ) - ) + getattr(cfg, "validation", {}) or {}) + self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind( + self.method_config.get("attn_kind", None))) # Initialize preprocessors on student. self.student.init_preprocessors(self.training_args) @@ -83,9 +72,9 @@ def single_train_step( *, current_vsa_sparsity: float = 0.0, ) -> tuple[ - dict[str, torch.Tensor], - dict[str, Any], - dict[str, LogScalar], + dict[str, torch.Tensor], + dict[str, Any], + dict[str, LogScalar], ]: del iteration training_batch = self.student.prepare_batch( @@ -95,32 +84,20 @@ def single_train_step( ) if training_batch.latents is None: - raise RuntimeError( - "prepare_batch() must set TrainingBatch.latents" - ) + raise RuntimeError("prepare_batch() must set TrainingBatch.latents") if training_batch.noisy_model_input is None: - raise RuntimeError( - "prepare_batch() must set " - "TrainingBatch.noisy_model_input" - ) + raise RuntimeError("prepare_batch() must set " + "TrainingBatch.noisy_model_input") if training_batch.noise is None: - raise RuntimeError( - "prepare_batch() must set TrainingBatch.noise" - ) + raise RuntimeError("prepare_batch() must set TrainingBatch.noise") if training_batch.sigmas is None: - raise RuntimeError( - "prepare_batch() must set TrainingBatch.sigmas" - ) + raise RuntimeError("prepare_batch() must set TrainingBatch.sigmas") if training_batch.timesteps is None: - raise RuntimeError( - "prepare_batch() must set " - "TrainingBatch.timesteps" - ) + raise RuntimeError("prepare_batch() must set " + "TrainingBatch.timesteps") clean_latents = training_batch.latents - noisy_latents = training_batch.noisy_model_input.permute( - 0, 2, 1, 3, 4 - ) + noisy_latents = training_batch.noisy_model_input.permute(0, 2, 1, 3, 4) noise = training_batch.noise.permute(0, 2, 1, 3, 4) sigmas = training_batch.sigmas timesteps = training_batch.timesteps @@ -133,17 +110,13 @@ def single_train_step( attn_kind=self._attn_kind, ) - if bool( - getattr( + if bool(getattr( self.training_args, "precondition_outputs", False, - ) - ): + )): pred_x0 = noisy_latents - pred * sigmas - loss = F.mse_loss( - pred_x0.float(), clean_latents.float() - ) + loss = F.mse_loss(pred_x0.float(), clean_latents.float()) else: target = noise - clean_latents loss = F.mse_loss(pred.float(), target.float()) @@ -187,9 +160,7 @@ def backward( ) # DistillMethod override: get_optimizers - def get_optimizers( - self, iteration: int - ) -> list[torch.optim.Optimizer]: + def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: del iteration return [self._student_optimizer] @@ -200,9 +171,7 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: - clip_grad_norm_if_needed( - self.student.transformer, self.training_args - ) + clip_grad_norm_if_needed(self.student.transformer, self.training_args) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start @@ -217,38 +186,22 @@ def log_validation(self, iteration: int) -> None: if not is_validation_enabled(self.validation_config): return - every_steps = parse_validation_every_steps( - self.validation_config - ) + every_steps = parse_validation_every_steps(self.validation_config) if every_steps <= 0: return if iteration % every_steps != 0: return - dataset_file = parse_validation_dataset_file( - self.validation_config - ) - sampling_steps = parse_validation_sampling_steps( - self.validation_config - ) - guidance_scale = parse_validation_guidance_scale( - self.validation_config - ) - sampler_kind = parse_validation_sampler_kind( - self.validation_config, default="ode" - ) - rollout_mode = parse_validation_rollout_mode( - self.validation_config - ) - output_dir = parse_validation_output_dir( - self.validation_config - ) - num_actions = parse_validation_num_frames( - self.validation_config - ) - ode_solver = parse_validation_ode_solver( - self.validation_config, sampler_kind=sampler_kind - ) + dataset_file = parse_validation_dataset_file(self.validation_config) + sampling_steps = parse_validation_sampling_steps(self.validation_config) + guidance_scale = parse_validation_guidance_scale(self.validation_config) + sampler_kind = parse_validation_sampler_kind(self.validation_config, + default="ode") + rollout_mode = parse_validation_rollout_mode(self.validation_config) + output_dir = parse_validation_output_dir(self.validation_config) + num_actions = parse_validation_num_frames(self.validation_config) + ode_solver = parse_validation_ode_solver(self.validation_config, + sampler_kind=sampler_kind) request = ValidationRequest( sample_handle=self.student, @@ -272,49 +225,35 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: generators.update(student_gens) validator = getattr(self, "validator", None) - validation_gen = getattr( - validator, "validation_random_generator", None - ) + validation_gen = getattr(validator, "validation_random_generator", None) if isinstance(validation_gen, torch.Generator): generators["validation_cpu"] = validation_gen return generators - def _parse_attn_kind( - self, raw: Any - ) -> Literal["dense", "vsa"]: + def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: if raw in (None, ""): return "dense" kind = str(raw).strip().lower() if kind not in {"dense", "vsa"}: - raise ValueError( - "method_config.attn_kind must be one of " - f"{{'dense', 'vsa'}}, got {raw!r}." - ) + raise ValueError("method_config.attn_kind must be one of " + f"{{'dense', 'vsa'}}, got {raw!r}.") return cast(Literal["dense", "vsa"], kind) def _init_optimizers_and_schedulers(self) -> None: training_args = self.training_args - student_lr = float( - getattr(training_args, "learning_rate", 0.0) or 0.0 - ) + student_lr = float(getattr(training_args, "learning_rate", 0.0) or 0.0) if student_lr <= 0.0: - raise ValueError( - "training.learning_rate must be > 0 for finetune" - ) + raise ValueError("training.learning_rate must be > 0 for finetune") student_betas = parse_betas( getattr(training_args, "betas", None), where="training.betas", ) - student_sched = str( - getattr(training_args, "lr_scheduler", "constant") - ) + student_sched = str(getattr(training_args, "lr_scheduler", "constant")) student_params = [ - p - for p in self.student.transformer.parameters() - if p.requires_grad + p for p in self.student.transformer.parameters() if p.requires_grad ] ( self._student_optimizer, diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index ab43e4446..9dfca8e67 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 - """Distillation run config (v3 — ``_target_`` based).""" from __future__ import annotations import os -from dataclasses import dataclass, field +import warnings +from dataclasses import dataclass from pathlib import Path from typing import Any, TYPE_CHECKING @@ -23,7 +23,6 @@ class RunConfig: method: dict[str, Any] training_args: TrainingArgs validation: dict[str, Any] - method_config: dict[str, Any] raw: dict[str, Any] @@ -36,109 +35,191 @@ def _resolve_existing_file(path: str) -> str: expanded = os.path.expanduser(path) resolved = Path(expanded).resolve() if not resolved.exists(): - raise FileNotFoundError( - f"Config file not found: {resolved}" - ) + raise FileNotFoundError(f"Config file not found: {resolved}") if not resolved.is_file(): - raise ValueError( - f"Expected a file path, got: {resolved}" - ) + raise ValueError(f"Expected a file path, got: {resolved}") return str(resolved) def _require_mapping(raw: Any, *, where: str) -> dict[str, Any]: if not isinstance(raw, dict): - raise ValueError( - f"Expected mapping at {where}, " - f"got {type(raw).__name__}" - ) + raise ValueError(f"Expected mapping at {where}, " + f"got {type(raw).__name__}") return raw def _require_str(raw: Any, *, where: str) -> str: if not isinstance(raw, str) or not raw.strip(): - raise ValueError( - f"Expected non-empty string at {where}" - ) + raise ValueError(f"Expected non-empty string at {where}") return raw -def _get_bool( - raw: Any, *, where: str, default: bool -) -> bool: +def _get_bool(raw: Any, *, where: str, default: bool) -> bool: if raw is None: return default if isinstance(raw, bool): return raw - raise ValueError( - f"Expected bool at {where}, " - f"got {type(raw).__name__}" - ) + raise ValueError(f"Expected bool at {where}, " + f"got {type(raw).__name__}") -def get_optional_int( - mapping: dict[str, Any], key: str, *, where: str -) -> int | None: +def get_optional_int(mapping: dict[str, Any], key: str, *, + where: str) -> int | None: raw = mapping.get(key) if raw is None: return None if isinstance(raw, bool): - raise ValueError( - f"Expected int at {where}, got bool" - ) + raise ValueError(f"Expected int at {where}, got bool") if isinstance(raw, int): return int(raw) if isinstance(raw, float) and raw.is_integer(): return int(raw) if isinstance(raw, str) and raw.strip(): return int(raw) - raise ValueError( - f"Expected int at {where}, " - f"got {type(raw).__name__}" - ) + raise ValueError(f"Expected int at {where}, " + f"got {type(raw).__name__}") -def get_optional_float( - mapping: dict[str, Any], key: str, *, where: str -) -> float | None: +def get_optional_float(mapping: dict[str, Any], key: str, *, + where: str) -> float | None: raw = mapping.get(key) if raw is None: return None if isinstance(raw, bool): - raise ValueError( - f"Expected float at {where}, got bool" - ) - if isinstance(raw, (int, float)): + raise ValueError(f"Expected float at {where}, got bool") + if isinstance(raw, int | float): return float(raw) if isinstance(raw, str) and raw.strip(): return float(raw) - raise ValueError( - f"Expected float at {where}, " - f"got {type(raw).__name__}" - ) + raise ValueError(f"Expected float at {where}, " + f"got {type(raw).__name__}") -def parse_betas( - raw: Any, *, where: str -) -> tuple[float, float]: +def parse_betas(raw: Any, *, where: str) -> tuple[float, float]: if raw is None: raise ValueError(f"Missing betas for {where}") - if isinstance(raw, (tuple, list)) and len(raw) == 2: + if isinstance(raw, tuple | list) and len(raw) == 2: return float(raw[0]), float(raw[1]) if isinstance(raw, str): - parts = [ - p.strip() for p in raw.split(",") if p.strip() - ] + parts = [p.strip() for p in raw.split(",") if p.strip()] if len(parts) != 2: - raise ValueError( - f"Expected betas as 'b1,b2' at {where}, " - f"got {raw!r}" - ) + raise ValueError(f"Expected betas as 'b1,b2' at {where}, " + f"got {raw!r}") return float(parts[0]), float(parts[1]) - raise ValueError( - f"Expected betas as 'b1,b2' at {where}, " - f"got {type(raw).__name__}" - ) + raise ValueError(f"Expected betas as 'b1,b2' at {where}, " + f"got {type(raw).__name__}") + + +# ---- config convenience helpers ---- + + +def require_positive_int( + mapping: dict[str, Any], + key: str, + *, + default: int | None = None, + where: str | None = None, +) -> int: + """Read an int that must be > 0.""" + loc = where or key + raw = mapping.get(key) + if raw is None: + if default is not None: + return default + raise ValueError(f"Missing required key {loc!r}") + val = get_optional_int(mapping, key, where=loc) + if val is None or val <= 0: + raise ValueError(f"{loc} must be a positive integer, got {raw!r}") + return val + + +def require_non_negative_int( + mapping: dict[str, Any], + key: str, + *, + default: int | None = None, + where: str | None = None, +) -> int: + """Read an int that must be >= 0.""" + loc = where or key + raw = mapping.get(key) + if raw is None: + if default is not None: + return default + raise ValueError(f"Missing required key {loc!r}") + val = get_optional_int(mapping, key, where=loc) + if val is None or val < 0: + raise ValueError(f"{loc} must be a non-negative integer, " + f"got {raw!r}") + return val + + +def require_non_negative_float( + mapping: dict[str, Any], + key: str, + *, + default: float | None = None, + where: str | None = None, +) -> float: + """Read a float that must be >= 0.""" + loc = where or key + raw = mapping.get(key) + if raw is None: + if default is not None: + return default + raise ValueError(f"Missing required key {loc!r}") + val = get_optional_float(mapping, key, where=loc) + if val is None or val < 0.0: + raise ValueError(f"{loc} must be a non-negative float, " + f"got {raw!r}") + return val + + +def require_choice( + mapping: dict[str, Any], + key: str, + choices: set[str] | frozenset[str], + *, + default: str | None = None, + where: str | None = None, +) -> str: + """Read a string that must be one of *choices*.""" + loc = where or key + raw = mapping.get(key) + if raw is None: + if default is not None: + if default not in choices: + raise ValueError(f"Default {default!r} not in {choices}") + return default + raise ValueError(f"Missing required key {loc!r}") + if not isinstance(raw, str) or not raw.strip(): + raise ValueError(f"{loc} must be a non-empty string, " + f"got {type(raw).__name__}") + val = raw.strip().lower() + if val not in choices: + raise ValueError(f"{loc} must be one of {sorted(choices)}, " + f"got {raw!r}") + return val + + +def require_bool( + mapping: dict[str, Any], + key: str, + *, + default: bool | None = None, + where: str | None = None, +) -> bool: + """Read a bool value.""" + loc = where or key + raw = mapping.get(key) + if raw is None: + if default is not None: + return default + raise ValueError(f"Missing required key {loc!r}") + if not isinstance(raw, bool): + raise ValueError(f"{loc} must be a bool, " + f"got {type(raw).__name__}") + return raw def load_run_config(path: str) -> RunConfig: @@ -158,53 +239,69 @@ def load_run_config(path: str) -> RunConfig: cfg = _require_mapping(raw, where=path) # --- models section --- - models_raw = _require_mapping( - cfg.get("models"), where="models" - ) + models_raw = _require_mapping(cfg.get("models"), where="models") models: dict[str, dict[str, Any]] = {} for role, model_cfg_raw in models_raw.items(): role_str = _require_str(role, where="models.") - model_cfg = _require_mapping( - model_cfg_raw, where=f"models.{role_str}" - ) + model_cfg = _require_mapping(model_cfg_raw, where=f"models.{role_str}") if "_target_" not in model_cfg: - raise ValueError( - f"models.{role_str} must have a '_target_' key" - ) + raise ValueError(f"models.{role_str} must have a '_target_' key") models[role_str] = dict(model_cfg) # --- method section --- - method_raw = _require_mapping( - cfg.get("method"), where="method" - ) + method_raw = _require_mapping(cfg.get("method"), where="method") if "_target_" not in method_raw: raise ValueError("method must have a '_target_' key") method = dict(method_raw) - # --- method_config section --- + # --- backward compat: merge method_config into method --- method_config_raw = cfg.get("method_config", None) - if method_config_raw is None: - method_config: dict[str, Any] = {} - else: - method_config = _require_mapping( - method_config_raw, where="method_config" + if method_config_raw is not None: + warnings.warn( + "The top-level 'method_config:' section is " + "deprecated. Move its keys into 'method:' " + "directly.", + DeprecationWarning, + stacklevel=2, ) + mc = _require_mapping(method_config_raw, where="method_config") + for k, v in mc.items(): + if k in method and k != "_target_": + warnings.warn( + f"method_config.{k} overrides " + f"method.{k} — prefer using " + "method: only", + DeprecationWarning, + stacklevel=2, + ) + method.setdefault(k, v) + + # --- validation section (top-level or under training) --- + validation_raw = cfg.get("validation", None) # --- training section --- - training_raw = _require_mapping( - cfg.get("training"), where="training" - ) + training_raw = _require_mapping(cfg.get("training"), where="training") + + # Support validation under training: for backward compat. + training_validation_raw = training_raw.get("validation", None) + if training_validation_raw is not None: + if validation_raw is not None: + raise ValueError("Provide 'validation:' at top-level or under " + "'training:', not both") + warnings.warn( + "Nesting 'validation:' under 'training:' is " + "deprecated. Move it to the top level.", + DeprecationWarning, + stacklevel=2, + ) + validation_raw = training_validation_raw - # Validation sub-section. - training_validation_raw = training_raw.get( - "validation", None - ) - if training_validation_raw is None: + if validation_raw is None: validation: dict[str, Any] = {} else: validation = _require_mapping( - training_validation_raw, - where="training.validation", + validation_raw, + where="validation", ) training_kwargs: dict[str, Any] = dict(training_raw) @@ -216,9 +313,7 @@ def load_run_config(path: str) -> RunConfig: training_kwargs.setdefault("dit_precision", "fp32") training_kwargs["dit_cpu_offload"] = False - num_gpus = int( - training_kwargs.get("num_gpus", 1) or 1 - ) + num_gpus = int(training_kwargs.get("num_gpus", 1) or 1) training_kwargs.setdefault("num_gpus", num_gpus) training_kwargs.setdefault("tp_size", 1) training_kwargs.setdefault("sp_size", num_gpus) @@ -233,72 +328,58 @@ def load_run_config(path: str) -> RunConfig: training_kwargs["model_path"] = str(init_from) if "pretrained_model_name_or_path" not in training_kwargs: - training_kwargs["pretrained_model_name_or_path"] = ( - training_kwargs.get("model_path", "") - ) - - # Pipeline config. - default_pipeline_cfg_raw = cfg.get( - "default_pipeline_config", None - ) - default_pipeline_cfg_path = cfg.get( - "default_pipeline_config_path", None - ) + training_kwargs["pretrained_model_name_or_path"] = (training_kwargs.get( + "model_path", "")) + + # Pipeline config — support both ``pipeline:`` (new) and + # ``default_pipeline_config:`` / ``pipeline_config:`` + # (legacy). + pipeline_raw = cfg.get("pipeline", None) + default_pipeline_cfg_raw = cfg.get("default_pipeline_config", None) + default_pipeline_cfg_path = cfg.get("default_pipeline_config_path", None) pipeline_cfg_raw = cfg.get("pipeline_config", None) pipeline_cfg_path = cfg.get("pipeline_config_path", None) - if ( - default_pipeline_cfg_raw is not None - or default_pipeline_cfg_path is not None - ) and ( - pipeline_cfg_raw is not None - or pipeline_cfg_path is not None - ): - raise ValueError( - "Provide either default_pipeline_config(_path) or " - "the legacy pipeline_config(_path), not both" - ) + # Merge pipeline: into default_pipeline_config: for compat. + if pipeline_raw is not None: + if default_pipeline_cfg_raw is not None: + raise ValueError("Provide either 'pipeline:' or " + "'default_pipeline_config:', not both") + default_pipeline_cfg_raw = pipeline_raw - cfg_raw = ( - default_pipeline_cfg_raw - if default_pipeline_cfg_raw is not None - else pipeline_cfg_raw - ) - cfg_path = ( - default_pipeline_cfg_path - if default_pipeline_cfg_path is not None - else pipeline_cfg_path - ) + if (default_pipeline_cfg_raw is not None or default_pipeline_cfg_path + is not None) and (pipeline_cfg_raw is not None + or pipeline_cfg_path is not None): + raise ValueError("Provide either default_pipeline_config(_path) or " + "the legacy pipeline_config(_path), not both") + + cfg_raw = (default_pipeline_cfg_raw + if default_pipeline_cfg_raw is not None else pipeline_cfg_raw) + cfg_path = (default_pipeline_cfg_path + if default_pipeline_cfg_path is not None else pipeline_cfg_path) if cfg_path is not None: cfg_path = _require_str( cfg_path, - where=( - "default_pipeline_config_path" - if default_pipeline_cfg_path is not None - else "pipeline_config_path" - ), - ) - training_kwargs["pipeline_config"] = ( - _resolve_existing_file(cfg_path) + where=("default_pipeline_config_path" if default_pipeline_cfg_path + is not None else "pipeline_config_path"), ) + training_kwargs["pipeline_config"] = (_resolve_existing_file(cfg_path)) elif cfg_raw is not None: if isinstance(cfg_raw, str): training_kwargs["pipeline_config"] = ( - _resolve_existing_file(cfg_raw) - ) + _resolve_existing_file(cfg_raw)) elif isinstance(cfg_raw, dict): training_kwargs["pipeline_config"] = cfg_raw else: - raise ValueError( - "default_pipeline_config must be a mapping " - "or a path string" - ) + raise ValueError("default_pipeline_config must be a mapping " + "or a path string") training_args = TrainingArgs.from_kwargs(**training_kwargs) - # Stash validation config on training_args for - # init_preprocessors to pick up. + # Legacy: models read _validation_cfg from training_args + # during their __init__. Remove once models are updated to + # receive validation config through a different mechanism. training_args._validation_cfg = validation # type: ignore[attr-defined] return RunConfig( @@ -306,6 +387,5 @@ def load_run_config(path: str) -> RunConfig: method=method, training_args=training_args, validation=validation, - method_config=method_config, raw=cfg, ) From a114c14d62d30a00aa037722b9bb0125a54b6ae8 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 20:46:24 +0000 Subject: [PATCH 171/214] method specific config move to right place --- examples/distillation/refactor/run.sh | 48 +++++++++++++++++++ .../self_forcing_wangame_causal_v3.yaml | 10 ++-- .../methods/distribution_matching/dmd2.py | 20 ++++---- 3 files changed, 65 insertions(+), 13 deletions(-) create mode 100755 examples/distillation/refactor/run.sh diff --git a/examples/distillation/refactor/run.sh b/examples/distillation/refactor/run.sh new file mode 100755 index 000000000..bb732624a --- /dev/null +++ b/examples/distillation/refactor/run.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Launch distillation training from a v3 YAML config. +# +# Usage: +# bash dev/refactor/run.sh [extra torchrun/script flags] +# +# Examples: +# # 8-GPU self-forcing run +# bash dev/refactor/run.sh examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +# +# # DFSFT with output dir override +# bash dev/refactor/run.sh examples/distillation/refactor/dfsft_wangame_causal_v3.yaml \ +# --override-output-dir outputs/my_run +# +# # Dry-run (parse config only, no training) +# bash dev/refactor/run.sh examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml --dry-run + +set -euo pipefail + +CONFIG="${1:?Usage: $0 [extra flags...]}" +shift + +# ── GPU / node settings ────────────────────────────────────────────── +NUM_GPUS="${NUM_GPUS:-$(nvidia-smi -L 2>/dev/null | wc -l)}" +NUM_GPUS="${NUM_GPUS:-1}" +NNODES="${NNODES:-1}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-29500}" + +echo "=== Distillation Training ===" +echo "Config: ${CONFIG}" +echo "Num GPUs: ${NUM_GPUS}" +echo "Num Nodes: ${NNODES}" +echo "Node Rank: ${NODE_RANK}" +echo "Master: ${MASTER_ADDR}:${MASTER_PORT}" +echo "Extra args: $*" +echo "==============================" + +torchrun \ + --nnodes "${NNODES}" \ + --node_rank "${NODE_RANK}" \ + --nproc_per_node "${NUM_GPUS}" \ + --master_addr "${MASTER_ADDR}" \ + --master_port "${MASTER_PORT}" \ + fastvideo/training/distillation.py \ + --config "${CONFIG}" \ + "$@" diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index 7959bf541..6de730c33 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -33,6 +33,11 @@ method: generator_update_interval: 5 real_score_guidance_scale: 3.5 + # Critic / fake-score optimizer + fake_score_learning_rate: 8.0e-6 + fake_score_betas: [0.0, 0.999] + fake_score_lr_scheduler: constant + warp_denoising_step: true dmd_denoising_steps: [1000,750,500,250] @@ -88,11 +93,6 @@ training: lr_warmup_steps: 0 max_grad_norm: 1.0 - # Optimizer (critic / fake-score) - fake_score_learning_rate: 8.0e-6 - fake_score_betas: [0.0, 0.999] - fake_score_lr_scheduler: constant - # Distillation knobs training_cfg_rate: 0.0 enable_gradient_checkpointing_type: null diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index cdac7034b..aafad65ee 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -370,21 +370,25 @@ def _init_optimizers_and_schedulers(self) -> None: scheduler_name=student_sched, ) - # Critic optimizer/scheduler. - critic_lr = float( - getattr(training_args, "fake_score_learning_rate", 0.0) or 0.0) + # Critic optimizer/scheduler — read from method config. + critic_lr_raw = get_optional_float( + self.method_config, + "fake_score_learning_rate", + where="method.fake_score_learning_rate", + ) + critic_lr = float(critic_lr_raw or 0.0) if critic_lr == 0.0: critic_lr = student_lr - critic_betas_raw = getattr(training_args, "fake_score_betas", None) + critic_betas_raw = self.method_config.get("fake_score_betas", None) if critic_betas_raw is None: critic_betas_raw = getattr(training_args, "betas", None) critic_betas = parse_betas(critic_betas_raw, - where="training.fake_score_betas") + where="method.fake_score_betas") - critic_sched = str( - getattr(training_args, "fake_score_lr_scheduler", None) - or student_sched) + critic_sched_raw = self.method_config.get("fake_score_lr_scheduler", + None) + critic_sched = str(critic_sched_raw or student_sched) critic_params = [ p for p in self.critic.transformer.parameters() if p.requires_grad ] From e04ced0e2dced44c3633145bab72015560a65703 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 20:51:38 +0000 Subject: [PATCH 172/214] run scripts --- .../refactor/dfsft_wangame_causal_v3.yaml | 10 +++++-- examples/distillation/refactor/run.sh | 28 +++++++++++++------ .../self_forcing_wangame_causal_v3.yaml | 2 +- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index 14358630c..197a3212d 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -26,8 +26,14 @@ training: # Data (parquet dataset folder) data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1 + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 dataloader_num_workers: 4 # Batch / shape @@ -63,7 +69,7 @@ training: enable_gradient_checkpointing_type: full # Tracking - tracker_project_name: distillation_wangame + tracker_project_name: distillation_wangame_r wandb_run_name: wangame_dfsft_causal_v3 validation: diff --git a/examples/distillation/refactor/run.sh b/examples/distillation/refactor/run.sh index bb732624a..97e3c675e 100755 --- a/examples/distillation/refactor/run.sh +++ b/examples/distillation/refactor/run.sh @@ -2,18 +2,15 @@ # Launch distillation training from a v3 YAML config. # # Usage: -# bash dev/refactor/run.sh [extra torchrun/script flags] +# bash examples/distillation/refactor/run.sh [extra flags] # # Examples: -# # 8-GPU self-forcing run -# bash dev/refactor/run.sh examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml -# -# # DFSFT with output dir override -# bash dev/refactor/run.sh examples/distillation/refactor/dfsft_wangame_causal_v3.yaml \ +# bash examples/distillation/refactor/run.sh examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +# bash examples/distillation/refactor/run.sh examples/distillation/refactor/dfsft_wangame_causal_v3.yaml --dry-run +# bash examples/distillation/refactor/run.sh examples/distillation/refactor/dfsft_wangame_causal_v3.yaml \ # --override-output-dir outputs/my_run # -# # Dry-run (parse config only, no training) -# bash dev/refactor/run.sh examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml --dry-run +# Logs are written to logs/_.log (and also printed to stdout). set -euo pipefail @@ -28,6 +25,17 @@ NODE_RANK="${NODE_RANK:-0}" MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" MASTER_PORT="${MASTER_PORT:-29500}" +# ── W&B ────────────────────────────────────────────────────────────── +export WANDB_API_KEY="${WANDB_API_KEY:-}" +export WANDB_MODE="${WANDB_MODE:-online}" + +# ── Log file ───────────────────────────────────────────────────────── +CONFIG_NAME="$(basename "${CONFIG}" .yaml)" +TIMESTAMP="$(date +%Y%m%d_%H%M%S)" +LOG_DIR="${LOG_DIR:-logs}" +mkdir -p "${LOG_DIR}" +LOG_FILE="${LOG_DIR}/${CONFIG_NAME}_${TIMESTAMP}.log" + echo "=== Distillation Training ===" echo "Config: ${CONFIG}" echo "Num GPUs: ${NUM_GPUS}" @@ -35,6 +43,7 @@ echo "Num Nodes: ${NNODES}" echo "Node Rank: ${NODE_RANK}" echo "Master: ${MASTER_ADDR}:${MASTER_PORT}" echo "Extra args: $*" +echo "Log file: ${LOG_FILE}" echo "==============================" torchrun \ @@ -45,4 +54,5 @@ torchrun \ --master_port "${MASTER_PORT}" \ fastvideo/training/distillation.py \ --config "${CONFIG}" \ - "$@" + "$@" \ + 2>&1 | tee "${LOG_FILE}" diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index 6de730c33..6e560b197 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -102,7 +102,7 @@ training: weight_only_checkpointing_steps: 1000 # Tracking - tracker_project_name: distillation_wangame + tracker_project_name: distillation_wangame_r wandb_run_name: wangame_self_forcing_4steps_v3 validation: From dbca2a5154bef05b39940cbf7ce2c6cde7499180 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 20:55:25 +0000 Subject: [PATCH 173/214] 120 col --- examples/distillation/refactor/run.sh | 2 +- fastvideo/distillation/.style.yapf | 3 + fastvideo/distillation/dispatch.py | 59 ++++----------- .../methods/distribution_matching/dmd2.py | 70 ++++++------------ .../distribution_matching/self_forcing.py | 71 +++++++------------ .../distillation/methods/fine_tuning/dfsft.py | 40 ++++------- .../methods/fine_tuning/finetune.py | 16 ++--- fastvideo/distillation/utils/config.py | 24 +++---- 8 files changed, 90 insertions(+), 195 deletions(-) create mode 100644 fastvideo/distillation/.style.yapf diff --git a/examples/distillation/refactor/run.sh b/examples/distillation/refactor/run.sh index 97e3c675e..b43004fb8 100755 --- a/examples/distillation/refactor/run.sh +++ b/examples/distillation/refactor/run.sh @@ -32,7 +32,7 @@ export WANDB_MODE="${WANDB_MODE:-online}" # ── Log file ───────────────────────────────────────────────────────── CONFIG_NAME="$(basename "${CONFIG}" .yaml)" TIMESTAMP="$(date +%Y%m%d_%H%M%S)" -LOG_DIR="${LOG_DIR:-logs}" +LOG_DIR="${LOG_DIR:-examples/distillation/refactor}" mkdir -p "${LOG_DIR}" LOG_FILE="${LOG_DIR}/${CONFIG_NAME}_${TIMESTAMP}.log" diff --git a/fastvideo/distillation/.style.yapf b/fastvideo/distillation/.style.yapf new file mode 100644 index 000000000..c9a88d5a6 --- /dev/null +++ b/fastvideo/distillation/.style.yapf @@ -0,0 +1,3 @@ +[style] +based_on_style = pep8 +column_limit = 120 diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index 0cdf069a4..214bfbf4c 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """Assembly: build method + dataloader from a ``_target_``-based config.""" from __future__ import annotations @@ -17,9 +16,7 @@ from fastvideo.distillation.methods.base import DistillMethod -def build_from_config( - cfg: RunConfig, -) -> tuple[TrainingArgs, DistillMethod, Any, int]: +def build_from_config(cfg: RunConfig, ) -> tuple[TrainingArgs, DistillMethod, Any, int]: """Build method + dataloader from a v3 run config. 1. Instantiate each model in ``cfg.models`` via ``_target_``. @@ -35,46 +32,30 @@ def build_from_config( for role, model_cfg in cfg.models.items(): model = instantiate(model_cfg) if not isinstance(model, ModelBase): - raise TypeError( - f"models.{role}._target_ must resolve to a " - f"ModelBase subclass, got {type(model).__name__}" - ) + raise TypeError(f"models.{role}._target_ must resolve to a " + f"ModelBase subclass, got {type(model).__name__}") role_models[role] = model # --- 2. Warm-start from checkpoint if needed --- from fastvideo.distillation.utils.checkpoint import ( - maybe_warmstart_role_modules, - ) + maybe_warmstart_role_modules, ) for role, model_cfg in cfg.models.items(): - init_from_checkpoint = model_cfg.get( - "init_from_checkpoint", None - ) + init_from_checkpoint = model_cfg.get("init_from_checkpoint", None) if not init_from_checkpoint: continue - checkpoint_role = model_cfg.get( - "init_from_checkpoint_role", None - ) - if ( - checkpoint_role is not None - and not isinstance(checkpoint_role, str) - ): - raise ValueError( - f"models.{role}.init_from_checkpoint_role " - "must be a string when set, got " - f"{type(checkpoint_role).__name__}" - ) + checkpoint_role = model_cfg.get("init_from_checkpoint_role", None) + if (checkpoint_role is not None and not isinstance(checkpoint_role, str)): + raise ValueError(f"models.{role}.init_from_checkpoint_role " + "must be a string when set, got " + f"{type(checkpoint_role).__name__}") # Warmstart uses the model's transformer directly. model = role_models[role] maybe_warmstart_role_modules( bundle=None, role=str(role), init_from_checkpoint=str(init_from_checkpoint), - checkpoint_role=( - str(checkpoint_role) - if checkpoint_role - else None - ), + checkpoint_role=(str(checkpoint_role) if checkpoint_role else None), model=model, ) @@ -85,11 +66,7 @@ def build_from_config( # The student model provides the validator and dataloader. student = role_models.get("student") - validator = ( - getattr(student, "validator", None) - if student is not None - else None - ) + validator = (getattr(student, "validator", None) if student is not None else None) method = method_cls( cfg=cfg, @@ -98,15 +75,7 @@ def build_from_config( ) # --- 4. Gather dataloader and start_step --- - dataloader = ( - getattr(student, "dataloader", None) - if student is not None - else None - ) - start_step = int( - getattr(student, "start_step", 0) - if student is not None - else 0 - ) + dataloader = (getattr(student, "dataloader", None) if student is not None else None) + start_step = int(getattr(student, "start_step", 0) if student is not None else 0) return cfg.training_args, method, dataloader, start_step diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index aafad65ee..083e7f2e1 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -76,8 +76,7 @@ def __init__( self.validator = validator self.training_args = cfg.training_args self.method_config: dict[str, Any] = dict(cfg.method) - self.validation_config: dict[str, Any] = dict( - getattr(cfg, "validation", {}) or {}) + self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) self._cfg_uncond = self._parse_cfg_uncond() self._rollout_mode = self._parse_rollout_mode() self._denoising_step_list: torch.Tensor | None = None @@ -118,8 +117,7 @@ def single_train_step( ) student_ctx = None if update_student: - generator_pred_x0 = self._student_rollout(training_batch, - with_grad=True) + generator_pred_x0 = self._student_rollout(training_batch, with_grad=True) student_ctx = ( training_batch.timesteps, training_batch.attn_metadata_vsa, @@ -145,9 +143,7 @@ def single_train_step( "student_ctx": student_ctx, "critic_ctx": critic_ctx, } - metrics: dict[str, LogScalar] = { - "update_student": float(update_student) - } + metrics: dict[str, LogScalar] = {"update_student": float(update_student)} return loss_map, outputs, metrics # DistillMethod override: backward @@ -207,8 +203,7 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: if self._should_update_student(iteration): - clip_grad_norm_if_needed(self.student.transformer, - self.training_args) + clip_grad_norm_if_needed(self.student.transformer, self.training_args) clip_grad_norm_if_needed(self.critic.transformer, self.training_args) super().optimizers_schedulers_step(iteration) @@ -245,10 +240,8 @@ def log_validation(self, iteration: int) -> None: return sampling_steps = [int(len(sampling_timesteps))] - sampler_kind = parse_validation_sampler_kind(self.validation_config, - default="sde") - ode_solver = parse_validation_ode_solver(self.validation_config, - sampler_kind=sampler_kind) + sampler_kind = parse_validation_sampler_kind(self.validation_config, default="sde") + ode_solver = parse_validation_ode_solver(self.validation_config, sampler_kind=sampler_kind) if (sampling_timesteps is not None and sampler_kind != "sde"): raise ValueError("method_config.validation.sampling_timesteps is " "only valid when sampler_kind='sde'") @@ -356,9 +349,7 @@ def _init_optimizers_and_schedulers(self) -> None: where="training.betas", ) student_sched = str(getattr(training_args, "lr_scheduler", "constant")) - student_params = [ - p for p in self.student.transformer.parameters() if p.requires_grad - ] + student_params = [p for p in self.student.transformer.parameters() if p.requires_grad] ( self._student_optimizer, self._student_lr_scheduler, @@ -383,15 +374,11 @@ def _init_optimizers_and_schedulers(self) -> None: critic_betas_raw = self.method_config.get("fake_score_betas", None) if critic_betas_raw is None: critic_betas_raw = getattr(training_args, "betas", None) - critic_betas = parse_betas(critic_betas_raw, - where="method.fake_score_betas") + critic_betas = parse_betas(critic_betas_raw, where="method.fake_score_betas") - critic_sched_raw = self.method_config.get("fake_score_lr_scheduler", - None) + critic_sched_raw = self.method_config.get("fake_score_lr_scheduler", None) critic_sched = str(critic_sched_raw or student_sched) - critic_params = [ - p for p in self.critic.transformer.parameters() if p.requires_grad - ] + critic_params = [p for p in self.critic.transformer.parameters() if p.requires_grad] ( self._critic_optimizer, self._critic_lr_scheduler, @@ -416,8 +403,7 @@ def _should_update_student(self, iteration: int) -> bool: return iteration % interval == 0 def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: - if (self._denoising_step_list is not None - and self._denoising_step_list.device == device): + if (self._denoising_step_list is not None and self._denoising_step_list.device == device): return self._denoising_step_list raw = self.method_config.get("dmd_denoising_steps", None) @@ -486,9 +472,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: target_timestep_idx_int = int(target_timestep_idx.item()) target_timestep = step_list[target_timestep_idx] - current_noise_latents = torch.randn(latents.shape, - device=device, - dtype=dtype) + current_noise_latents = torch.randn(latents.shape, device=device, dtype=dtype) current_noise_latents_copy = current_noise_latents.clone() max_target_idx = len(step_list) - 1 @@ -499,9 +483,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: with torch.no_grad(): for step_idx in range(max_target_idx): current_timestep = step_list[step_idx] - current_timestep_tensor = ( - current_timestep * - torch.ones(1, device=device, dtype=torch.long)) + current_timestep_tensor = (current_timestep * torch.ones(1, device=device, dtype=torch.long)) pred_clean = self.student.predict_x0( current_noise_latents, @@ -513,9 +495,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: ) next_timestep = step_list[step_idx + 1] - next_timestep_tensor = ( - next_timestep * - torch.ones(1, device=device, dtype=torch.long)) + next_timestep_tensor = (next_timestep * torch.ones(1, device=device, dtype=torch.long)) noise = torch.randn( latents.shape, device=device, @@ -555,12 +535,10 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: attn_kind="vsa", ) - batch.dmd_latent_vis_dict["generator_timestep"] = target_timestep.float( - ).detach() + batch.dmd_latent_vis_dict["generator_timestep"] = target_timestep.float().detach() return pred_x0 - def _critic_flow_matching_loss( - self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: + def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: with torch.no_grad(): generator_pred_x0 = self._student_rollout(batch, with_grad=False) @@ -572,16 +550,14 @@ def _critic_flow_matching_loss( device=device, dtype=torch.long, ) - fake_score_timestep = ( - self.student.shift_and_clamp_timestep(fake_score_timestep)) + fake_score_timestep = (self.student.shift_and_clamp_timestep(fake_score_timestep)) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_x0 = self.student.add_noise(generator_pred_x0, noise, - fake_score_timestep) + noisy_x0 = self.student.add_noise(generator_pred_x0, noise, fake_score_timestep) pred_noise = self.critic.predict_noise( noisy_x0, @@ -598,9 +574,7 @@ def _critic_flow_matching_loss( "generator_pred_video": generator_pred_x0, "fake_score_timestep": fake_score_timestep, } - outputs = { - "fake_score_latent_vis_dict": (batch.fake_score_latent_vis_dict) - } + outputs = {"fake_score_latent_vis_dict": (batch.fake_score_latent_vis_dict)} return ( flow_matching_loss, (batch.timesteps, batch.attn_metadata), @@ -636,8 +610,7 @@ def _dmd_loss( device=device, dtype=generator_pred_x0.dtype, ) - noisy_latents = self.student.add_noise(generator_pred_x0, noise, - timestep) + noisy_latents = self.student.add_noise(generator_pred_x0, noise, timestep) faker_x0 = self.critic.predict_x0( noisy_latents, @@ -663,8 +636,7 @@ def _dmd_loss( cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + (real_cond_x0 - - real_uncond_x0) * guidance_scale + real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() grad = (faker_x0 - real_cfg_x0) / denom diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/distillation/methods/distribution_matching/self_forcing.py index 10fe07be5..11e61ee09 100644 --- a/fastvideo/distillation/methods/distribution_matching/self_forcing.py +++ b/fastvideo/distillation/methods/distribution_matching/self_forcing.py @@ -141,12 +141,11 @@ def __init__( f">= 0, got {start_grad_frame}") self._start_gradient_frame = int(start_grad_frame) - shift = float( - getattr( - self.training_args.pipeline_config, - "flow_shift", - 0.0, - ) or 0.0) + shift = float(getattr( + self.training_args.pipeline_config, + "flow_shift", + 0.0, + ) or 0.0) self._sf_scheduler = SelfForcingFlowMatchScheduler( num_inference_steps=1000, num_train_timesteps=int(self.student.num_train_timesteps), @@ -159,8 +158,7 @@ def __init__( self._sf_denoising_step_list: torch.Tensor | None = None def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: - if (self._sf_denoising_step_list is not None - and self._sf_denoising_step_list.device == device): + if (self._sf_denoising_step_list is not None and self._sf_denoising_step_list.device == device): return self._sf_denoising_step_list raw = self.method_config.get("dmd_denoising_steps", None) @@ -227,10 +225,8 @@ def _sf_add_noise( return noisy def _timestep_to_sigma(self, timestep: torch.Tensor) -> torch.Tensor: - sigmas = self._sf_scheduler.sigmas.to(device=timestep.device, - dtype=torch.float32) - timesteps = self._sf_scheduler.timesteps.to(device=timestep.device, - dtype=torch.float32) + sigmas = self._sf_scheduler.sigmas.to(device=timestep.device, dtype=torch.float32) + timesteps = self._sf_scheduler.timesteps.to(device=timestep.device, dtype=torch.float32) t = timestep.to(device=timestep.device, dtype=torch.float32) if t.ndim == 2: t = t.flatten(0, 1) @@ -290,8 +286,7 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: "implementing CausalModelBase.") return self._student_rollout_streaming(batch, with_grad=with_grad) - def _student_rollout_streaming(self, batch: Any, *, - with_grad: bool) -> torch.Tensor: + def _student_rollout_streaming(self, batch: Any, *, with_grad: bool) -> torch.Tensor: assert isinstance(self.student, CausalModelBase) latents = batch.latents if latents is None: @@ -356,9 +351,7 @@ def _student_rollout_streaming(self, batch: Any, *, dtype=torch.float32, )) - enable_grad = (bool(with_grad) - and bool(self._enable_gradient_in_rollout) - and torch.is_grad_enabled() + enable_grad = (bool(with_grad) and bool(self._enable_gradient_in_rollout) and torch.is_grad_enabled() and start >= int(self._start_gradient_frame)) if not exit_flag: @@ -399,19 +392,14 @@ def _student_rollout_streaming(self, batch: Any, *, ), ) else: - sigma_cur = self._timestep_to_sigma( - timestep_block).view(batch_size, end - start, 1, 1, - 1) - sigma_next = self._timestep_to_sigma( - next_timestep * torch.ones( - (batch_size, end - start), - device=device, - dtype=torch.float32, - )).view(batch_size, end - start, 1, 1, 1) - eps = (noisy_block - (1 - sigma_cur) * - pred_x0_chunk) / sigma_cur.clamp_min(1e-8) - noisy_block = ((1 - sigma_next) * pred_x0_chunk + - sigma_next * eps) + sigma_cur = self._timestep_to_sigma(timestep_block).view(batch_size, end - start, 1, 1, 1) + sigma_next = self._timestep_to_sigma(next_timestep * torch.ones( + (batch_size, end - start), + device=device, + dtype=torch.float32, + )).view(batch_size, end - start, 1, 1, 1) + eps = (noisy_block - (1 - sigma_cur) * pred_x0_chunk) / sigma_cur.clamp_min(1e-8) + noisy_block = ((1 - sigma_next) * pred_x0_chunk + sigma_next * eps) continue with torch.set_grad_enabled(enable_grad): @@ -477,8 +465,7 @@ def _student_rollout_streaming(self, batch: Any, *, self.student.clear_caches(cache_tag=cache_tag) return torch.cat(denoised_blocks, dim=1) - def _critic_flow_matching_loss( - self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: + def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: with torch.no_grad(): generator_pred_x0 = self._student_rollout(batch, with_grad=False) @@ -490,16 +477,14 @@ def _critic_flow_matching_loss( device=device, dtype=torch.long, ) - fake_score_timestep = ( - self.student.shift_and_clamp_timestep(fake_score_timestep)) + fake_score_timestep = (self.student.shift_and_clamp_timestep(fake_score_timestep)) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_x0 = self._sf_add_noise(generator_pred_x0, noise, - fake_score_timestep) + noisy_x0 = self._sf_add_noise(generator_pred_x0, noise, fake_score_timestep) pred_noise = self.critic.predict_noise( noisy_x0, @@ -516,9 +501,7 @@ def _critic_flow_matching_loss( "generator_pred_video": generator_pred_x0, "fake_score_timestep": fake_score_timestep, } - outputs = { - "fake_score_latent_vis_dict": (batch.fake_score_latent_vis_dict) - } + outputs = {"fake_score_latent_vis_dict": (batch.fake_score_latent_vis_dict)} return ( flow_matching_loss, (batch.timesteps, batch.attn_metadata), @@ -554,8 +537,7 @@ def _dmd_loss( device=device, dtype=generator_pred_x0.dtype, ) - noisy_latents = self._sf_add_noise(generator_pred_x0, noise, - timestep) + noisy_latents = self._sf_add_noise(generator_pred_x0, noise, timestep) faker_x0 = self._predict_x0_with_scheduler( self.critic, @@ -581,14 +563,11 @@ def _dmd_loss( conditional=False, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + (real_cond_x0 - - real_uncond_x0) * guidance_scale + real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() grad = (faker_x0 - real_cfg_x0) / denom grad = torch.nan_to_num(grad) - loss = 0.5 * torch.mean( - (generator_pred_x0.float() - - (generator_pred_x0.float() - grad.float()).detach())**2) + loss = 0.5 * torch.mean((generator_pred_x0.float() - (generator_pred_x0.float() - grad.float()).detach())**2) return loss diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index cb1fe477e..ceead70a8 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -56,13 +56,10 @@ def __init__( self.validator = validator self.training_args = cfg.training_args self.method_config: dict[str, Any] = dict(cfg.method) - self.validation_config: dict[str, Any] = dict( - getattr(cfg, "validation", {}) or {}) - self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind( - self.method_config.get("attn_kind", None))) + self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) + self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind(self.method_config.get("attn_kind", None))) - self._chunk_size = self._parse_chunk_size( - self.method_config.get("chunk_size", None)) + self._chunk_size = self._parse_chunk_size(self.method_config.get("chunk_size", None)) self._timestep_index_range = (self._parse_timestep_index_range()) # Initialize preprocessors on student. @@ -100,16 +97,14 @@ def single_train_step( "[B, T, C, H, W], got " f"shape={tuple(clean_latents.shape)}") - batch_size, num_latents = int(clean_latents.shape[0]), int( - clean_latents.shape[1]) + batch_size, num_latents = int(clean_latents.shape[0]), int(clean_latents.shape[1]) expected_chunk = getattr( self.student.transformer, "num_frame_per_block", None, ) - if (expected_chunk is not None - and int(expected_chunk) != int(self._chunk_size)): + if (expected_chunk is not None and int(expected_chunk) != int(self._chunk_size)): raise ValueError("DFSFT chunk_size must match " "transformer.num_frame_per_block for " f"causal training (got {self._chunk_size}, " @@ -122,16 +117,14 @@ def single_train_step( ) sp_size = int(getattr(self.training_args, "sp_size", 1) or 1) sp_group = getattr(self.student, "sp_group", None) - if (sp_size > 1 and sp_group is not None - and hasattr(sp_group, "broadcast")): + if (sp_size > 1 and sp_group is not None and hasattr(sp_group, "broadcast")): sp_group.broadcast(timestep_indices, src=0) scheduler = self.student.noise_scheduler if scheduler is None: raise ValueError("DFSFT requires student.noise_scheduler") - schedule_timesteps = scheduler.timesteps.to(device=clean_latents.device, - dtype=torch.float32) + schedule_timesteps = scheduler.timesteps.to(device=clean_latents.device, dtype=torch.float32) schedule_sigmas = scheduler.sigmas.to( device=clean_latents.device, dtype=clean_latents.dtype, @@ -248,13 +241,11 @@ def log_validation(self, iteration: int) -> None: dataset_file = parse_validation_dataset_file(self.validation_config) sampling_steps = parse_validation_sampling_steps(self.validation_config) guidance_scale = parse_validation_guidance_scale(self.validation_config) - sampler_kind = parse_validation_sampler_kind(self.validation_config, - default="ode") + sampler_kind = parse_validation_sampler_kind(self.validation_config, default="ode") rollout_mode = parse_validation_rollout_mode(self.validation_config) output_dir = parse_validation_output_dir(self.validation_config) num_actions = parse_validation_num_frames(self.validation_config) - ode_solver = parse_validation_ode_solver(self.validation_config, - sampler_kind=sampler_kind) + ode_solver = parse_validation_ode_solver(self.validation_config, sampler_kind=sampler_kind) request = ValidationRequest( sample_handle=self.student, @@ -330,8 +321,7 @@ def _parse_timestep_index_range(self) -> tuple[int, int]: scheduler = self.student.noise_scheduler if scheduler is None: raise ValueError("DFSFT requires student.noise_scheduler") - num_steps = int( - getattr(scheduler, "config", scheduler).num_train_timesteps) + num_steps = int(getattr(scheduler, "config", scheduler).num_train_timesteps) min_ratio = self._parse_ratio( self.method_config.get("min_timestep_ratio", None), @@ -362,8 +352,7 @@ def _parse_timestep_index_range(self) -> tuple[int, int]: return min_index, max_index + 1 def _init_optimizers_and_schedulers(self) -> None: - student_lr = float( - getattr(self.training_args, "learning_rate", 0.0) or 0.0) + student_lr = float(getattr(self.training_args, "learning_rate", 0.0) or 0.0) if student_lr <= 0.0: raise ValueError("training.learning_rate must be > 0 for dfsft") @@ -371,11 +360,8 @@ def _init_optimizers_and_schedulers(self) -> None: getattr(self.training_args, "betas", None), where="training.betas", ) - student_sched = str( - getattr(self.training_args, "lr_scheduler", "constant")) - student_params = [ - p for p in self.student.transformer.parameters() if p.requires_grad - ] + student_sched = str(getattr(self.training_args, "lr_scheduler", "constant")) + student_params = [p for p in self.student.transformer.parameters() if p.requires_grad] ( self._student_optimizer, self._student_lr_scheduler, diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 9d042b256..9c890951a 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -54,10 +54,8 @@ def __init__( self.validator = validator self.training_args = cfg.training_args self.method_config: dict[str, Any] = dict(cfg.method) - self.validation_config: dict[str, Any] = dict( - getattr(cfg, "validation", {}) or {}) - self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind( - self.method_config.get("attn_kind", None))) + self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) + self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind(self.method_config.get("attn_kind", None))) # Initialize preprocessors on student. self.student.init_preprocessors(self.training_args) @@ -195,13 +193,11 @@ def log_validation(self, iteration: int) -> None: dataset_file = parse_validation_dataset_file(self.validation_config) sampling_steps = parse_validation_sampling_steps(self.validation_config) guidance_scale = parse_validation_guidance_scale(self.validation_config) - sampler_kind = parse_validation_sampler_kind(self.validation_config, - default="ode") + sampler_kind = parse_validation_sampler_kind(self.validation_config, default="ode") rollout_mode = parse_validation_rollout_mode(self.validation_config) output_dir = parse_validation_output_dir(self.validation_config) num_actions = parse_validation_num_frames(self.validation_config) - ode_solver = parse_validation_ode_solver(self.validation_config, - sampler_kind=sampler_kind) + ode_solver = parse_validation_ode_solver(self.validation_config, sampler_kind=sampler_kind) request = ValidationRequest( sample_handle=self.student, @@ -252,9 +248,7 @@ def _init_optimizers_and_schedulers(self) -> None: where="training.betas", ) student_sched = str(getattr(training_args, "lr_scheduler", "constant")) - student_params = [ - p for p in self.student.transformer.parameters() if p.requires_grad - ] + student_params = [p for p in self.student.transformer.parameters() if p.requires_grad] ( self._student_optimizer, self._student_lr_scheduler, diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index 9dfca8e67..cbea20e81 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -63,8 +63,7 @@ def _get_bool(raw: Any, *, where: str, default: bool) -> bool: f"got {type(raw).__name__}") -def get_optional_int(mapping: dict[str, Any], key: str, *, - where: str) -> int | None: +def get_optional_int(mapping: dict[str, Any], key: str, *, where: str) -> int | None: raw = mapping.get(key) if raw is None: return None @@ -80,8 +79,7 @@ def get_optional_int(mapping: dict[str, Any], key: str, *, f"got {type(raw).__name__}") -def get_optional_float(mapping: dict[str, Any], key: str, *, - where: str) -> float | None: +def get_optional_float(mapping: dict[str, Any], key: str, *, where: str) -> float | None: raw = mapping.get(key) if raw is None: return None @@ -328,8 +326,7 @@ def load_run_config(path: str) -> RunConfig: training_kwargs["model_path"] = str(init_from) if "pretrained_model_name_or_path" not in training_kwargs: - training_kwargs["pretrained_model_name_or_path"] = (training_kwargs.get( - "model_path", "")) + training_kwargs["pretrained_model_name_or_path"] = (training_kwargs.get("model_path", "")) # Pipeline config — support both ``pipeline:`` (new) and # ``default_pipeline_config:`` / ``pipeline_config:`` @@ -348,27 +345,22 @@ def load_run_config(path: str) -> RunConfig: default_pipeline_cfg_raw = pipeline_raw if (default_pipeline_cfg_raw is not None or default_pipeline_cfg_path - is not None) and (pipeline_cfg_raw is not None - or pipeline_cfg_path is not None): + is not None) and (pipeline_cfg_raw is not None or pipeline_cfg_path is not None): raise ValueError("Provide either default_pipeline_config(_path) or " "the legacy pipeline_config(_path), not both") - cfg_raw = (default_pipeline_cfg_raw - if default_pipeline_cfg_raw is not None else pipeline_cfg_raw) - cfg_path = (default_pipeline_cfg_path - if default_pipeline_cfg_path is not None else pipeline_cfg_path) + cfg_raw = (default_pipeline_cfg_raw if default_pipeline_cfg_raw is not None else pipeline_cfg_raw) + cfg_path = (default_pipeline_cfg_path if default_pipeline_cfg_path is not None else pipeline_cfg_path) if cfg_path is not None: cfg_path = _require_str( cfg_path, - where=("default_pipeline_config_path" if default_pipeline_cfg_path - is not None else "pipeline_config_path"), + where=("default_pipeline_config_path" if default_pipeline_cfg_path is not None else "pipeline_config_path"), ) training_kwargs["pipeline_config"] = (_resolve_existing_file(cfg_path)) elif cfg_raw is not None: if isinstance(cfg_raw, str): - training_kwargs["pipeline_config"] = ( - _resolve_existing_file(cfg_raw)) + training_kwargs["pipeline_config"] = (_resolve_existing_file(cfg_raw)) elif isinstance(cfg_raw, dict): training_kwargs["pipeline_config"] = cfg_raw else: From cb7247da65a6ae4f17f4be3c71c33bc3c1236301 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 21:19:10 +0000 Subject: [PATCH 174/214] bugfix --- fastvideo/distillation/methods/base.py | 1 + .../methods/distribution_matching/dmd2.py | 18 ++++++++++++++++-- .../distillation/methods/fine_tuning/dfsft.py | 8 ++++++++ .../methods/fine_tuning/finetune.py | 8 ++++++++ fastvideo/distillation/utils/moduleloader.py | 8 ++++++++ fastvideo/training/distillation.py | 10 +++++++--- 6 files changed, 48 insertions(+), 5 deletions(-) diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/distillation/methods/base.py index e32ac6b23..6c08e0e5f 100644 --- a/fastvideo/distillation/methods/base.py +++ b/fastvideo/distillation/methods/base.py @@ -32,6 +32,7 @@ def __init__( ) -> None: super().__init__() self.tracker: Any | None = None + self._role_models: dict[str, ModelBase] = dict(role_models) # Build nn.ModuleDict for FSDP / checkpoint visibility. self.role_modules = torch.nn.ModuleDict() for role, model in role_models.items(): diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 083e7f2e1..346ab1500 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -86,6 +86,20 @@ def __init__( self._init_optimizers_and_schedulers() + @property + def _optimizer_dict(self) -> dict[str, torch.optim.Optimizer]: + return { + "student": self._student_optimizer, + "critic": self._critic_optimizer, + } + + @property + def _lr_scheduler_dict(self) -> dict[str, Any]: + return { + "student": self._student_lr_scheduler, + "critic": self._critic_lr_scheduler, + } + # DistillMethod override: single_train_step def single_train_step( self, @@ -178,7 +192,7 @@ def backward( critic_ctx = backward_ctx.get("critic_ctx") if critic_ctx is None: raise RuntimeError("Missing critic backward context") - self.student.backward( + self.critic.backward( loss_map["fake_score_loss"], critic_ctx, grad_accum_rounds=grad_accum_rounds, @@ -636,7 +650,7 @@ def _dmd_loss( cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale + real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * (guidance_scale - 1) denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() grad = (faker_x0 - real_cfg_x0) / denom diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index ceead70a8..a2dd0f634 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -67,6 +67,14 @@ def __init__( self._init_optimizers_and_schedulers() + @property + def _optimizer_dict(self) -> dict[str, Any]: + return {"student": self._student_optimizer} + + @property + def _lr_scheduler_dict(self) -> dict[str, Any]: + return {"student": self._student_lr_scheduler} + # DistillMethod override: single_train_step def single_train_step( self, diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 9c890951a..87a950a3c 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -62,6 +62,14 @@ def __init__( self._init_optimizers_and_schedulers() + @property + def _optimizer_dict(self) -> dict[str, Any]: + return {"student": self._student_optimizer} + + @property + def _lr_scheduler_dict(self) -> dict[str, Any]: + return {"student": self._student_lr_scheduler} + # DistillMethod override: single_train_step def single_train_step( self, diff --git a/fastvideo/distillation/utils/moduleloader.py b/fastvideo/distillation/utils/moduleloader.py index 673f44a72..0be538938 100644 --- a/fastvideo/distillation/utils/moduleloader.py +++ b/fastvideo/distillation/utils/moduleloader.py @@ -40,6 +40,14 @@ def load_module_from_path( transformers_or_diffusers, _architecture = module_info component_path = os.path.join(local_model_path, module_type) + # When training_args is None (e.g. during model-only loading in + # distillation role construction), create a lightweight stand-in so + # that override_transformer_cls_name and disable_custom_init_weights + # flags can still be forwarded to PipelineComponentLoader. + if training_args is None: + from types import SimpleNamespace + training_args = SimpleNamespace() + old_override_transformer_cls_name: str | None = None if override_transformer_cls_name is not None: old_override_transformer_cls_name = getattr( diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 72c8e24e8..912b58c94 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -28,9 +28,9 @@ def run_distillation_from_config( DistillCheckpointManager, ) from fastvideo.distillation.dispatch import build_from_config - from fastvideo.distillation.utils.config import load_distill_run_config + from fastvideo.distillation.utils.config import load_run_config - cfg = load_distill_run_config(config_path) + cfg = load_run_config(config_path) training_args = cfg.training_args if resume_from_checkpoint is not None: @@ -68,7 +68,11 @@ def run_distillation_from_config( get_rng_generators = None checkpoint_manager = DistillCheckpointManager( - bundle=method.bundle, + role_models=getattr(method, '_role_models', None) or {}, + optimizers=getattr(method, '_optimizer_dict', None) or {}, + lr_schedulers=getattr( + method, '_lr_scheduler_dict', None + ) or {}, dataloader=dataloader, output_dir=training_args.output_dir, config=ckpt_config, From 9d06875c13ae5a678ecf21dece4fda2f62145ea4 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 22:18:36 +0000 Subject: [PATCH 175/214] remove fastvideo.training_args dependency --- .../refactor/dfsft_wangame_causal_v3.yaml | 91 ++- .../self_forcing_wangame_causal_v3.yaml | 91 ++- fastvideo/distillation/dispatch.py | 7 +- .../methods/distribution_matching/dmd2.py | 30 +- .../distribution_matching/self_forcing.py | 4 +- .../distillation/methods/fine_tuning/dfsft.py | 28 +- .../methods/fine_tuning/finetune.py | 27 +- fastvideo/distillation/models/base.py | 8 +- fastvideo/distillation/models/wan/wan.py | 402 ++++++++----- .../distillation/models/wangame/wangame.py | 565 +++++++++++------- .../models/wangame/wangame_causal.py | 290 +++------ fastvideo/distillation/trainer.py | 77 +-- fastvideo/distillation/utils/config.py | 383 +++++++++--- fastvideo/distillation/utils/dataloader.py | 53 +- .../distillation/utils/distill_config.py | 100 ++++ fastvideo/distillation/utils/loader_args.py | 59 ++ fastvideo/distillation/utils/moduleloader.py | 79 ++- fastvideo/distillation/utils/optimizer.py | 69 +-- fastvideo/distillation/utils/tracking.py | 41 +- fastvideo/distillation/validators/wan.py | 212 ++++--- fastvideo/distillation/validators/wangame.py | 442 ++++++++------ fastvideo/training/distillation.py | 75 +-- 22 files changed, 1878 insertions(+), 1255 deletions(-) create mode 100644 fastvideo/distillation/utils/distill_config.py create mode 100644 fastvideo/distillation/utils/loader_args.py diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index 197a3212d..c5186e2a4 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -17,60 +17,55 @@ method: max_timestep_ratio: 0.98 training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 + distributed: + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 - # Data (parquet dataset folder) - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 - dataloader_num_workers: 4 + data: + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 + dataloader_num_workers: 4 + train_batch_size: 1 + training_cfg_rate: 0.0 + seed: 1000 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 + optimizer: + learning_rate: 1.0e-5 + betas: [0.9, 0.999] + weight_decay: 1.0e-4 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 - # Output / steps - output_dir: outputs/wangame_dfsft_causal_v3 - max_train_steps: 20000 - seed: 1000 - checkpoints_total_limit: 2 + loop: + max_train_steps: 20000 + gradient_accumulation_steps: 1 - # Optimizer - learning_rate: 1.0e-5 - mixed_precision: bf16 - betas: [0.9, 0.999] - weight_decay: 1.0e-4 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 + checkpoint: + output_dir: outputs/wangame_dfsft_causal_v3 + training_state_checkpointing_steps: 1000 + checkpoints_total_limit: 2 - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 + tracker: + project_name: distillation_wangame_r + run_name: wangame_dfsft_causal_v3 - # Method-agnostic knobs - training_cfg_rate: 0.0 - enable_gradient_checkpointing_type: full - - # Tracking - tracker_project_name: distillation_wangame_r - wandb_run_name: wangame_dfsft_causal_v3 + model: + enable_gradient_checkpointing_type: full validation: enabled: true diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index 6e560b197..3565f2831 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -56,54 +56,49 @@ method: text: keep training: - # Distributed (4 nodes x 8 GPUs = 32 ranks) - num_gpus: 32 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 1 - hsdp_shard_dim: 32 - - # Data - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1 - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_self_forcing_4steps_v3 - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (student) - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: [0.0, 0.999] - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Distillation knobs - training_cfg_rate: 0.0 - enable_gradient_checkpointing_type: null - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Tracking - tracker_project_name: distillation_wangame_r - wandb_run_name: wangame_self_forcing_4steps_v3 + distributed: + num_gpus: 32 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 32 + + data: + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1 + dataloader_num_workers: 4 + train_batch_size: 1 + training_cfg_rate: 0.0 + seed: 1000 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + optimizer: + learning_rate: 2.0e-6 + betas: [0.0, 0.999] + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + loop: + max_train_steps: 4000 + gradient_accumulation_steps: 1 + + checkpoint: + output_dir: outputs/wangame_self_forcing_4steps_v3 + training_state_checkpointing_steps: 1000 + checkpoints_total_limit: 3 + + tracker: + project_name: distillation_wangame_r + run_name: wangame_self_forcing_4steps_v3 + + model: + enable_gradient_checkpointing_type: null validation: enabled: true diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/distillation/dispatch.py index 214bfbf4c..f465abe81 100644 --- a/fastvideo/distillation/dispatch.py +++ b/fastvideo/distillation/dispatch.py @@ -12,11 +12,12 @@ from fastvideo.distillation.utils.config import RunConfig if TYPE_CHECKING: - from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.distillation.utils.distill_config import ( + DistillTrainingConfig, ) from fastvideo.distillation.methods.base import DistillMethod -def build_from_config(cfg: RunConfig, ) -> tuple[TrainingArgs, DistillMethod, Any, int]: +def build_from_config(cfg: RunConfig, ) -> tuple[DistillTrainingConfig, DistillMethod, Any, int]: """Build method + dataloader from a v3 run config. 1. Instantiate each model in ``cfg.models`` via ``_target_``. @@ -78,4 +79,4 @@ def build_from_config(cfg: RunConfig, ) -> tuple[TrainingArgs, DistillMethod, An dataloader = (getattr(student, "dataloader", None) if student is not None else None) start_step = int(getattr(student, "start_step", 0) if student is not None else 0) - return cfg.training_args, method, dataloader, start_step + return cfg.training, method, dataloader, start_step diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/distillation/methods/distribution_matching/dmd2.py index 346ab1500..63dba055e 100644 --- a/fastvideo/distillation/methods/distribution_matching/dmd2.py +++ b/fastvideo/distillation/methods/distribution_matching/dmd2.py @@ -74,7 +74,7 @@ def __init__( raise ValueError("DMD2Method requires critic to be trainable") self.validator = validator - self.training_args = cfg.training_args + self.training_config = cfg.training self.method_config: dict[str, Any] = dict(cfg.method) self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) self._cfg_uncond = self._parse_cfg_uncond() @@ -82,7 +82,7 @@ def __init__( self._denoising_step_list: torch.Tensor | None = None # Initialize preprocessors on student. - self.student.init_preprocessors(self.training_args) + self.student.init_preprocessors(self.training_config) self._init_optimizers_and_schedulers() @@ -216,9 +216,10 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: + max_grad_norm = self.training_config.optimizer.max_grad_norm if self._should_update_student(iteration): - clip_grad_norm_if_needed(self.student.transformer, self.training_args) - clip_grad_norm_if_needed(self.critic.transformer, self.training_args) + clip_grad_norm_if_needed(self.student.transformer, max_grad_norm) + clip_grad_norm_if_needed(self.critic.transformer, max_grad_norm) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start @@ -354,22 +355,20 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: return cfg def _init_optimizers_and_schedulers(self) -> None: - training_args = self.training_args + tc = self.training_config # Student optimizer/scheduler. - student_lr = float(getattr(training_args, "learning_rate", 0.0) or 0.0) - student_betas = parse_betas( - getattr(training_args, "betas", None), - where="training.betas", - ) - student_sched = str(getattr(training_args, "lr_scheduler", "constant")) + student_lr = float(tc.optimizer.learning_rate) + student_betas = tc.optimizer.betas + student_sched = str(tc.optimizer.lr_scheduler) student_params = [p for p in self.student.transformer.parameters() if p.requires_grad] ( self._student_optimizer, self._student_lr_scheduler, ) = build_optimizer_and_scheduler( params=student_params, - training_args=self.training_args, + optimizer_config=tc.optimizer, + loop_config=tc.loop, learning_rate=student_lr, betas=student_betas, scheduler_name=student_sched, @@ -387,7 +386,7 @@ def _init_optimizers_and_schedulers(self) -> None: critic_betas_raw = self.method_config.get("fake_score_betas", None) if critic_betas_raw is None: - critic_betas_raw = getattr(training_args, "betas", None) + critic_betas_raw = tc.optimizer.betas critic_betas = parse_betas(critic_betas_raw, where="method.fake_score_betas") critic_sched_raw = self.method_config.get("fake_score_lr_scheduler", None) @@ -398,7 +397,8 @@ def _init_optimizers_and_schedulers(self) -> None: self._critic_lr_scheduler, ) = build_optimizer_and_scheduler( params=critic_params, - training_args=self.training_args, + optimizer_config=tc.optimizer, + loop_config=tc.loop, learning_rate=critic_lr, betas=critic_betas, scheduler_name=critic_sched, @@ -433,7 +433,7 @@ def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: warp = self.method_config.get("warp_denoising_step", None) if warp is None: - warp = getattr(self.training_args, "warp_denoising_step", False) + warp = False if bool(warp): timesteps = torch.cat(( self.student.noise_scheduler.timesteps.to("cpu"), diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/distillation/methods/distribution_matching/self_forcing.py index 11e61ee09..de6d42b4a 100644 --- a/fastvideo/distillation/methods/distribution_matching/self_forcing.py +++ b/fastvideo/distillation/methods/distribution_matching/self_forcing.py @@ -142,7 +142,7 @@ def __init__( self._start_gradient_frame = int(start_grad_frame) shift = float(getattr( - self.training_args.pipeline_config, + self.training_config.pipeline_config, "flow_shift", 0.0, ) or 0.0) @@ -173,7 +173,7 @@ def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: warp = self.method_config.get("warp_denoising_step", None) if warp is None: - warp = getattr(self.training_args, "warp_denoising_step", False) + warp = False if bool(warp): timesteps = torch.cat(( self._sf_scheduler.timesteps.to("cpu"), diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/distillation/methods/fine_tuning/dfsft.py index a2dd0f634..f91a28f0a 100644 --- a/fastvideo/distillation/methods/fine_tuning/dfsft.py +++ b/fastvideo/distillation/methods/fine_tuning/dfsft.py @@ -10,7 +10,6 @@ from fastvideo.distillation.methods.base import DistillMethod, LogScalar from fastvideo.distillation.models.base import ModelBase -from fastvideo.distillation.utils.config import parse_betas from fastvideo.distillation.utils.optimizer import ( build_optimizer_and_scheduler, clip_grad_norm_if_needed, @@ -54,7 +53,7 @@ def __init__( raise ValueError("DFSFT requires student to be trainable") self.validator = validator - self.training_args = cfg.training_args + self.training_config = cfg.training self.method_config: dict[str, Any] = dict(cfg.method) self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind(self.method_config.get("attn_kind", None))) @@ -63,7 +62,7 @@ def __init__( self._timestep_index_range = (self._parse_timestep_index_range()) # Initialize preprocessors on student. - self.student.init_preprocessors(self.training_args) + self.student.init_preprocessors(self.training_config) self._init_optimizers_and_schedulers() @@ -123,7 +122,7 @@ def single_train_step( num_latents=num_latents, device=clean_latents.device, ) - sp_size = int(getattr(self.training_args, "sp_size", 1) or 1) + sp_size = int(self.training_config.distributed.sp_size) sp_group = getattr(self.student, "sp_group", None) if (sp_size > 1 and sp_group is not None and hasattr(sp_group, "broadcast")): sp_group.broadcast(timestep_indices, src=0) @@ -162,11 +161,7 @@ def single_train_step( attn_kind=self._attn_kind, ) - if bool(getattr( - self.training_args, - "precondition_outputs", - False, - )): + if bool(self.training_config.model.precondition_outputs): sigmas = schedule_sigmas[timestep_indices] sigmas = sigmas.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) pred_x0 = noisy_latents - pred * sigmas @@ -225,7 +220,7 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: - clip_grad_norm_if_needed(self.student.transformer, self.training_args) + clip_grad_norm_if_needed(self.student.transformer, self.training_config.optimizer.max_grad_norm) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start @@ -360,22 +355,21 @@ def _parse_timestep_index_range(self) -> tuple[int, int]: return min_index, max_index + 1 def _init_optimizers_and_schedulers(self) -> None: - student_lr = float(getattr(self.training_args, "learning_rate", 0.0) or 0.0) + tc = self.training_config + student_lr = float(tc.optimizer.learning_rate) if student_lr <= 0.0: raise ValueError("training.learning_rate must be > 0 for dfsft") - student_betas = parse_betas( - getattr(self.training_args, "betas", None), - where="training.betas", - ) - student_sched = str(getattr(self.training_args, "lr_scheduler", "constant")) + student_betas = tc.optimizer.betas + student_sched = str(tc.optimizer.lr_scheduler) student_params = [p for p in self.student.transformer.parameters() if p.requires_grad] ( self._student_optimizer, self._student_lr_scheduler, ) = build_optimizer_and_scheduler( params=student_params, - training_args=self.training_args, + optimizer_config=tc.optimizer, + loop_config=tc.loop, learning_rate=student_lr, betas=student_betas, scheduler_name=student_sched, diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/distillation/methods/fine_tuning/finetune.py index 87a950a3c..c2f274be7 100644 --- a/fastvideo/distillation/methods/fine_tuning/finetune.py +++ b/fastvideo/distillation/methods/fine_tuning/finetune.py @@ -27,7 +27,6 @@ parse_validation_sampling_steps, ) from fastvideo.distillation.validators.base import ValidationRequest -from fastvideo.distillation.utils.config import parse_betas if TYPE_CHECKING: pass @@ -52,13 +51,13 @@ def __init__( raise ValueError("FineTuneMethod requires student to be trainable") self.validator = validator - self.training_args = cfg.training_args + self.training_config = cfg.training self.method_config: dict[str, Any] = dict(cfg.method) self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind(self.method_config.get("attn_kind", None))) # Initialize preprocessors on student. - self.student.init_preprocessors(self.training_args) + self.student.init_preprocessors(self.training_config) self._init_optimizers_and_schedulers() @@ -116,11 +115,7 @@ def single_train_step( attn_kind=self._attn_kind, ) - if bool(getattr( - self.training_args, - "precondition_outputs", - False, - )): + if bool(self.training_config.model.precondition_outputs): pred_x0 = noisy_latents - pred * sigmas loss = F.mse_loss(pred_x0.float(), clean_latents.float()) else: @@ -177,7 +172,7 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: # DistillMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: - clip_grad_norm_if_needed(self.student.transformer, self.training_args) + clip_grad_norm_if_needed(self.student.transformer, self.training_config.optimizer.max_grad_norm) super().optimizers_schedulers_step(iteration) # DistillTrainer hook: on_train_start @@ -245,24 +240,22 @@ def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: return cast(Literal["dense", "vsa"], kind) def _init_optimizers_and_schedulers(self) -> None: - training_args = self.training_args + tc = self.training_config - student_lr = float(getattr(training_args, "learning_rate", 0.0) or 0.0) + student_lr = float(tc.optimizer.learning_rate) if student_lr <= 0.0: raise ValueError("training.learning_rate must be > 0 for finetune") - student_betas = parse_betas( - getattr(training_args, "betas", None), - where="training.betas", - ) - student_sched = str(getattr(training_args, "lr_scheduler", "constant")) + student_betas = tc.optimizer.betas + student_sched = str(tc.optimizer.lr_scheduler) student_params = [p for p in self.student.transformer.parameters() if p.requires_grad] ( self._student_optimizer, self._student_lr_scheduler, ) = build_optimizer_and_scheduler( params=student_params, - training_args=self.training_args, + optimizer_config=tc.optimizer, + loop_config=tc.loop, learning_rate=student_lr, betas=student_betas, scheduler_name=student_sched, diff --git a/fastvideo/distillation/models/base.py b/fastvideo/distillation/models/base.py index a6cc75c64..00349922a 100644 --- a/fastvideo/distillation/models/base.py +++ b/fastvideo/distillation/models/base.py @@ -8,6 +8,8 @@ import torch if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + DistillTrainingConfig, ) from fastvideo.pipelines import TrainingBatch @@ -28,7 +30,7 @@ class ModelBase(ABC): # Lifecycle # ------------------------------------------------------------------ - def init_preprocessors(self, training_args: Any) -> None: + def init_preprocessors(self, training_config: DistillTrainingConfig) -> None: """Load VAE, build dataloader, seed RNGs. Called only on the student by the method's ``__init__``. @@ -51,9 +53,7 @@ def num_train_timesteps(self) -> int: """Return the scheduler's training timestep horizon.""" return int(self.noise_scheduler.num_train_timesteps) - def shift_and_clamp_timestep( - self, timestep: torch.Tensor - ) -> torch.Tensor: + def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: """Apply model/pipeline timestep shifting and clamp.""" return timestep diff --git a/fastvideo/distillation/models/wan/wan.py b/fastvideo/distillation/models/wan/wan.py index caebc2427..ab77764c9 100644 --- a/fastvideo/distillation/models/wan/wan.py +++ b/fastvideo/distillation/models/wan/wan.py @@ -6,7 +6,7 @@ import copy import gc -from typing import Any, Literal +from typing import Any, Literal, TYPE_CHECKING import torch @@ -23,8 +23,12 @@ ) from fastvideo.models.utils import pred_noise_to_pred_video from fastvideo.pipelines import TrainingBatch -from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch +from fastvideo.pipelines.basic.wan.wan_pipeline import ( + WanPipeline, +) +from fastvideo.pipelines.pipeline_batch_info import ( + ForwardBatch, +) from fastvideo.training.activation_checkpoint import ( apply_activation_checkpointing, ) @@ -48,6 +52,11 @@ load_module_from_path, ) +if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + DistillTrainingConfig, + ) + try: from fastvideo.attention.backends.video_sparse_attn import ( VideoSparseAttentionMetadataBuilder, @@ -70,7 +79,8 @@ def __init__( trainable: bool = True, disable_custom_init_weights: bool = False, flow_shift: float = 3.0, - enable_gradient_checkpointing_type: str | None = None, + enable_gradient_checkpointing_type: str + | None = None, ) -> None: self._init_from = str(init_from) self._trainable = bool(trainable) @@ -78,9 +88,13 @@ def __init__( transformer = load_module_from_path( model_path=self._init_from, module_type="transformer", - training_args=None, - disable_custom_init_weights=disable_custom_init_weights, - override_transformer_cls_name="WanTransformer3DModel", + loader_args=None, + disable_custom_init_weights=( + disable_custom_init_weights + ), + override_transformer_cls_name=( + "WanTransformer3DModel" + ), ) transformer = apply_trainable( transformer, trainable=self._trainable @@ -97,13 +111,17 @@ def __init__( ) self.transformer = transformer - self.noise_scheduler = FlowMatchEulerDiscreteScheduler( - shift=float(flow_shift) + self.noise_scheduler = ( + FlowMatchEulerDiscreteScheduler( + shift=float(flow_shift) + ) ) # Filled by init_preprocessors (student only). self.vae: Any = None - self.training_args: Any = None + self.training_config: DistillTrainingConfig | None = ( + None + ) self.dataloader: Any = None self.validator: Any = None self.start_step: int = 0 @@ -112,10 +130,14 @@ def __init__( self.sp_group: Any = None self.device: Any = None - self.noise_random_generator: torch.Generator | None = None + self.noise_random_generator: ( + torch.Generator | None + ) = None self.noise_gen_cuda: torch.Generator | None = None - self.negative_prompt_embeds: torch.Tensor | None = None + self.negative_prompt_embeds: ( + torch.Tensor | None + ) = None self.negative_prompt_attention_mask: ( torch.Tensor | None ) = None @@ -132,13 +154,24 @@ def __init__( # Lifecycle # ------------------------------------------------------------------ - def init_preprocessors(self, training_args: Any) -> None: - self.training_args = training_args + def init_preprocessors( + self, training_config: DistillTrainingConfig + ) -> None: + self.training_config = training_config + + from fastvideo.distillation.utils.loader_args import ( + DistillLoaderArgs, + ) + + loader_args = DistillLoaderArgs.from_training_config( + training_config, + model_path=training_config.model_path, + ) self.vae = load_module_from_path( - model_path=str(training_args.model_path), + model_path=str(training_config.model_path), module_type="vae", - training_args=training_args, + loader_args=loader_args, ) self.world_group = get_world_group() @@ -149,7 +182,7 @@ def init_preprocessors(self, training_args: Any) -> None: # Optional validator. validation_cfg = getattr( - training_args, "_validation_cfg", None + training_config, "_validation_cfg", None ) if validation_cfg: validation_enabled = bool( @@ -161,8 +194,9 @@ def init_preprocessors(self, training_args: Any) -> None: from fastvideo.distillation.validators.wan import ( WanValidator, ) + self.validator = WanValidator( - training_args=training_args + training_config=training_config ) from fastvideo.dataset.dataloader.schema import ( @@ -172,8 +206,14 @@ def init_preprocessors(self, training_args: Any) -> None: build_parquet_t2v_train_dataloader, ) + text_len = ( + training_config.pipeline_config.text_encoder_configs[ # type: ignore[union-attr] + 0 + ].arch_config.text_len + ) self.dataloader = build_parquet_t2v_train_dataloader( - training_args, + training_config.data, + text_len=int(text_len), parquet_schema=pyarrow_schema_t2v, ) self.start_step = 0 @@ -186,20 +226,28 @@ def shift_and_clamp_timestep( self, timestep: torch.Tensor ) -> torch.Tensor: timestep = shift_timestep( - timestep, self.timestep_shift, self.num_train_timestep + timestep, + self.timestep_shift, + self.num_train_timestep, + ) + return timestep.clamp( + self.min_timestep, self.max_timestep ) - return timestep.clamp(self.min_timestep, self.max_timestep) def on_train_start(self) -> None: - seed = getattr(self.training_args, "seed", None) + assert self.training_config is not None + seed = self.training_config.data.seed if seed is None: raise ValueError( - "training_args.seed must be set for distillation" + "training.data.seed must be set " + "for distillation" ) - global_rank = int(getattr(self.world_group, "rank", 0)) + global_rank = int( + getattr(self.world_group, "rank", 0) + ) sp_world_size = int( - getattr(self.training_args, "sp_size", 1) or 1 + self.training_config.distributed.sp_size or 1 ) if sp_world_size > 1: sp_group_seed = int(seed) + ( @@ -218,10 +266,14 @@ def on_train_start(self) -> None: self.ensure_negative_conditioning() - def get_rng_generators(self) -> dict[str, torch.Generator]: + def get_rng_generators( + self, + ) -> dict[str, torch.Generator]: generators: dict[str, torch.Generator] = {} if self.noise_random_generator is not None: - generators["noise_cpu"] = self.noise_random_generator + generators["noise_cpu"] = ( + self.noise_random_generator + ) if self.noise_gen_cuda is not None: generators["noise_cuda"] = self.noise_gen_cuda return generators @@ -238,6 +290,8 @@ def prepare_batch( latents_source: Literal["data", "zeros"] = "data", ) -> TrainingBatch: self.ensure_negative_conditioning() + assert self.training_config is not None + tc = self.training_config dtype = self._get_training_dtype() device = self.device @@ -245,32 +299,35 @@ def prepare_batch( training_batch = TrainingBatch( current_vsa_sparsity=current_vsa_sparsity ) - encoder_hidden_states = raw_batch["text_embedding"] - encoder_attention_mask = raw_batch["text_attention_mask"] + encoder_hidden_states = raw_batch[ + "text_embedding" + ] + encoder_attention_mask = raw_batch[ + "text_attention_mask" + ] infos = raw_batch.get("info_list") if latents_source == "zeros": batch_size = encoder_hidden_states.shape[0] vae_config = ( - self.training_args.pipeline_config - .vae_config.arch_config + tc.pipeline_config.vae_config.arch_config # type: ignore[union-attr] ) num_channels = vae_config.z_dim spatial_compression_ratio = ( vae_config.spatial_compression_ratio ) latent_height = ( - self.training_args.num_height + tc.data.num_height // spatial_compression_ratio ) latent_width = ( - self.training_args.num_width + tc.data.num_width // spatial_compression_ratio ) latents = torch.zeros( batch_size, num_channels, - self.training_args.num_latent_t, + tc.data.num_latent_t, latent_height, latent_width, device=device, @@ -284,12 +341,13 @@ def prepare_batch( ) latents = raw_batch["vae_latent"] latents = latents[ - :, :, : self.training_args.num_latent_t + :, :, : tc.data.num_latent_t ] latents = latents.to(device, dtype=dtype) else: raise ValueError( - f"Unknown latents_source: {latents_source!r}" + f"Unknown latents_source: " + f"{latents_source!r}" ) training_batch.latents = latents @@ -304,7 +362,9 @@ def prepare_batch( training_batch.latents = normalize_dit_input( "wan", training_batch.latents, self.vae ) - training_batch = self._prepare_dit_inputs(training_batch) + training_batch = self._prepare_dit_inputs( + training_batch + ) training_batch = self._build_attention_metadata( training_batch ) @@ -347,7 +407,8 @@ def predict_x0( text_dict = batch.conditional_dict if text_dict is None: raise RuntimeError( - "Missing conditional_dict in TrainingBatch" + "Missing conditional_dict in " + "TrainingBatch" ) else: text_dict = self._get_uncond_text_dict( @@ -359,7 +420,9 @@ def predict_x0( elif attn_kind == "vsa": attn_metadata = batch.attn_metadata_vsa else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + raise ValueError( + f"Unknown attn_kind: {attn_kind!r}" + ) with torch.autocast( device_type, dtype=dtype @@ -367,16 +430,20 @@ def predict_x0( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, timestep, text_dict + input_kwargs = ( + self._build_distill_input_kwargs( + noisy_latents, timestep, text_dict + ) ) transformer = self._get_transformer(timestep) - pred_noise = transformer(**input_kwargs).permute( - 0, 2, 1, 3, 4 - ) + pred_noise = transformer( + **input_kwargs + ).permute(0, 2, 1, 3, 4) pred_x0 = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), + noise_input_latent=noisy_latents.flatten( + 0, 1 + ), timestep=timestep, scheduler=self.noise_scheduler, ).unflatten(0, pred_noise.shape[:2]) @@ -398,7 +465,8 @@ def predict_noise( text_dict = batch.conditional_dict if text_dict is None: raise RuntimeError( - "Missing conditional_dict in TrainingBatch" + "Missing conditional_dict in " + "TrainingBatch" ) else: text_dict = self._get_uncond_text_dict( @@ -410,7 +478,9 @@ def predict_noise( elif attn_kind == "vsa": attn_metadata = batch.attn_metadata_vsa else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + raise ValueError( + f"Unknown attn_kind: {attn_kind!r}" + ) with torch.autocast( device_type, dtype=dtype @@ -418,13 +488,15 @@ def predict_noise( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, timestep, text_dict + input_kwargs = ( + self._build_distill_input_kwargs( + noisy_latents, timestep, text_dict + ) ) transformer = self._get_transformer(timestep) - pred_noise = transformer(**input_kwargs).permute( - 0, 2, 1, 3, 4 - ) + pred_noise = transformer( + **input_kwargs + ).permute(0, 2, 1, 3, 4) return pred_noise def backward( @@ -439,7 +511,9 @@ def backward( current_timestep=timesteps, attn_metadata=attn_metadata, ): - (loss / max(1, int(grad_accum_rounds))).backward() + ( + loss / max(1, int(grad_accum_rounds)) + ).backward() # ------------------------------------------------------------------ # Internal helpers @@ -449,53 +523,61 @@ def _get_training_dtype(self) -> torch.dtype: return torch.bfloat16 def _init_timestep_mechanics(self) -> None: + assert self.training_config is not None + tc = self.training_config self.timestep_shift = float( - self.training_args.pipeline_config.flow_shift + tc.pipeline_config.flow_shift # type: ignore[union-attr] ) self.num_train_timestep = int( self.noise_scheduler.num_train_timesteps ) - self.min_timestep = int( - self.training_args.min_timestep_ratio - * self.num_train_timestep - ) - self.max_timestep = int( - self.training_args.max_timestep_ratio - * self.num_train_timestep - ) + # min/max timestep ratios now come from method_config; + # default to full range. + self.min_timestep = 0 + self.max_timestep = self.num_train_timestep def ensure_negative_conditioning(self) -> None: if self.negative_prompt_embeds is not None: return - training_args = self.training_args + assert self.training_config is not None + tc = self.training_config world_group = self.world_group device = self.device dtype = self._get_training_dtype() + from fastvideo.distillation.utils.loader_args import ( + DistillLoaderArgs, + ) + neg_embeds: torch.Tensor | None = None neg_mask: torch.Tensor | None = None if world_group.rank_in_group == 0: sampling_param = SamplingParam.from_pretrained( - training_args.model_path + tc.model_path ) negative_prompt = sampling_param.negative_prompt - args_copy = copy.deepcopy(training_args) - args_copy.inference_mode = True + inference_args = ( + DistillLoaderArgs.for_inference( + tc, model_path=tc.model_path + ) + ) prompt_pipeline = WanPipeline.from_pretrained( - training_args.model_path, - args=args_copy, + tc.model_path, + args=inference_args, inference_mode=True, loaded_modules={ "transformer": self.transformer }, - tp_size=training_args.tp_size, - sp_size=training_args.sp_size, - num_gpus=training_args.num_gpus, - pin_cpu_memory=training_args.pin_cpu_memory, + tp_size=tc.distributed.tp_size, + sp_size=tc.distributed.sp_size, + num_gpus=tc.distributed.num_gpus, + pin_cpu_memory=( + tc.distributed.pin_cpu_memory + ), dit_cpu_offload=True, ) @@ -507,14 +589,16 @@ def ensure_negative_conditioning(self) -> None: ) result_batch = prompt_pipeline.prompt_encoding_stage( # type: ignore[attr-defined] batch_negative, - training_args, + inference_args, ) neg_embeds = result_batch.prompt_embeds[0].to( device=device, dtype=dtype ) - neg_mask = result_batch.prompt_attention_mask[0].to( - device=device, dtype=dtype + neg_mask = ( + result_batch.prompt_attention_mask[0].to( + device=device, dtype=dtype + ) ) del prompt_pipeline @@ -522,7 +606,9 @@ def ensure_negative_conditioning(self) -> None: if torch.cuda.is_available(): torch.cuda.empty_cache() - meta = torch.zeros((2,), device=device, dtype=torch.int64) + meta = torch.zeros( + (2,), device=device, dtype=torch.int64 + ) if world_group.rank_in_group == 0: assert neg_embeds is not None assert neg_mask is not None @@ -558,10 +644,12 @@ def ensure_negative_conditioning(self) -> None: world_group.broadcast(mask_shape, src=0) embed_sizes = tuple( - int(x) for x in embed_shape[:embed_ndim].tolist() + int(x) + for x in embed_shape[:embed_ndim].tolist() ) mask_sizes = tuple( - int(x) for x in mask_shape[:mask_ndim].tolist() + int(x) + for x in mask_shape[:mask_ndim].tolist() ) if world_group.rank_in_group != 0: @@ -588,17 +676,20 @@ def _sample_timesteps( "on_train_start() must be called before " "prepare_batch()" ) + assert self.training_config is not None + tc = self.training_config u = compute_density_for_timestep_sampling( - weighting_scheme=self.training_args.weighting_scheme, + weighting_scheme=tc.model.weighting_scheme, batch_size=batch_size, generator=self.noise_random_generator, - logit_mean=self.training_args.logit_mean, - logit_std=self.training_args.logit_std, - mode_scale=self.training_args.mode_scale, + logit_mean=tc.model.logit_mean, + logit_std=tc.model.logit_std, + mode_scale=tc.model.mode_scale, ) indices = ( - u * self.noise_scheduler.config.num_train_timesteps + u + * self.noise_scheduler.config.num_train_timesteps ).long() return self.noise_scheduler.timesteps[indices].to( device=device @@ -607,48 +698,67 @@ def _sample_timesteps( def _build_attention_metadata( self, training_batch: TrainingBatch ) -> TrainingBatch: + assert self.training_config is not None + tc = self.training_config latents_shape = training_batch.raw_latent_shape patch_size = ( - self.training_args.pipeline_config.dit_config - .patch_size + tc.pipeline_config.dit_config.patch_size # type: ignore[union-attr] + ) + current_vsa_sparsity = ( + training_batch.current_vsa_sparsity ) - current_vsa_sparsity = training_batch.current_vsa_sparsity assert latents_shape is not None assert training_batch.timesteps is not None - if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": + if ( + envs.FASTVIDEO_ATTENTION_BACKEND + == "VIDEO_SPARSE_ATTN" + ): if ( not is_vsa_available() - or VideoSparseAttentionMetadataBuilder is None + or VideoSparseAttentionMetadataBuilder + is None ): raise ImportError( "FASTVIDEO_ATTENTION_BACKEND is " - "VIDEO_SPARSE_ATTN, but fastvideo_kernel " - "is not correctly installed or detected." + "VIDEO_SPARSE_ATTN, but " + "fastvideo_kernel is not correctly " + "installed or detected." ) training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] raw_latent_shape=latents_shape[2:5], - current_timestep=training_batch.timesteps, + current_timestep=( + training_batch.timesteps + ), patch_size=patch_size, VSA_sparsity=current_vsa_sparsity, device=self.device, ) - elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": + elif ( + envs.FASTVIDEO_ATTENTION_BACKEND + == "VMOBA_ATTN" + ): if ( not is_vmoba_available() - or VideoMobaAttentionMetadataBuilder is None + or VideoMobaAttentionMetadataBuilder + is None ): raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, " - "but fastvideo_kernel (or flash_attn>=2.7.4) " - "is not correctly installed." + "FASTVIDEO_ATTENTION_BACKEND is " + "VMOBA_ATTN, but fastvideo_kernel " + "(or flash_attn>=2.7.4) is not " + "correctly installed." ) - moba_params = self.training_args.moba_config.copy() + moba_params = tc.model.moba_config.copy() moba_params.update( { - "current_timestep": training_batch.timesteps, + "current_timestep": ( + training_batch.timesteps + ), "raw_latent_shape": ( - training_batch.raw_latent_shape[2:5] + training_batch.raw_latent_shape[ + 2:5 + ] ), "patch_size": patch_size, "device": self.device, @@ -663,6 +773,8 @@ def _build_attention_metadata( def _prepare_dit_inputs( self, training_batch: TrainingBatch ) -> TrainingBatch: + assert self.training_config is not None + tc = self.training_config latents = training_batch.latents assert isinstance(latents, torch.Tensor) batch_size = latents.shape[0] @@ -682,10 +794,7 @@ def _prepare_dit_inputs( timesteps = self._sample_timesteps( batch_size, latents.device ) - if ( - int(getattr(self.training_args, "sp_size", 1) or 1) - > 1 - ): + if int(tc.distributed.sp_size or 1) > 1: self.sp_group.broadcast(timesteps, src=0) sigmas = get_sigmas( @@ -699,7 +808,9 @@ def _prepare_dit_inputs( (1.0 - sigmas) * latents + sigmas * noise ) - training_batch.noisy_model_input = noisy_model_input + training_batch.noisy_model_input = ( + noisy_model_input + ) training_batch.timesteps = timesteps training_batch.sigmas = sigmas training_batch.noise = noise @@ -716,15 +827,24 @@ def _prepare_dit_inputs( if ( self.negative_prompt_embeds is not None - and self.negative_prompt_attention_mask is not None + and self.negative_prompt_attention_mask + is not None ): neg_embeds = self.negative_prompt_embeds - neg_mask = self.negative_prompt_attention_mask - if neg_embeds.shape[0] == 1 and batch_size > 1: + neg_mask = ( + self.negative_prompt_attention_mask + ) + if ( + neg_embeds.shape[0] == 1 + and batch_size > 1 + ): neg_embeds = neg_embeds.expand( batch_size, *neg_embeds.shape[1:] ).contiguous() - if neg_mask.shape[0] == 1 and batch_size > 1: + if ( + neg_mask.shape[0] == 1 + and batch_size > 1 + ): neg_mask = neg_mask.expand( batch_size, *neg_mask.shape[1:] ).contiguous() @@ -733,8 +853,8 @@ def _prepare_dit_inputs( "encoder_attention_mask": neg_mask, } - training_batch.latents = training_batch.latents.permute( - 0, 2, 1, 3, 4 + training_batch.latents = ( + training_batch.latents.permute(0, 2, 1, 3, 4) ) return training_batch @@ -746,7 +866,8 @@ def _build_distill_input_kwargs( ) -> dict[str, Any]: if text_dict is None: raise ValueError( - "text_dict cannot be None for Wan distillation" + "text_dict cannot be None for " + "Wan distillation" ) return { "hidden_states": noise_input.permute( @@ -785,18 +906,20 @@ def _get_uncond_text_dict( ) return text_dict - on_missing_raw = cfg_uncond.get("on_missing", "error") + on_missing_raw = cfg_uncond.get( + "on_missing", "error" + ) if not isinstance(on_missing_raw, str): raise ValueError( - "method_config.cfg_uncond.on_missing must be " - "a string, got " + "method_config.cfg_uncond.on_missing " + "must be a string, got " f"{type(on_missing_raw).__name__}" ) on_missing = on_missing_raw.strip().lower() if on_missing not in {"error", "ignore"}: raise ValueError( - "method_config.cfg_uncond.on_missing must be " - "one of {error, ignore}, got " + "method_config.cfg_uncond.on_missing " + "must be one of {error, ignore}, got " f"{on_missing_raw!r}" ) @@ -807,9 +930,10 @@ def _get_uncond_text_dict( continue if not isinstance(policy_raw, str): raise ValueError( - "method_config.cfg_uncond values must be " - "strings, got " - f"{channel}={type(policy_raw).__name__}" + "method_config.cfg_uncond values " + "must be strings, got " + f"{channel}=" + f"{type(policy_raw).__name__}" ) policy = policy_raw.strip().lower() if policy == "keep": @@ -817,10 +941,11 @@ def _get_uncond_text_dict( if on_missing == "ignore": continue raise ValueError( - "WanModel does not support cfg_uncond channel " + "WanModel does not support " + "cfg_uncond channel " f"{channel!r} (policy={policy!r}). " - "Set cfg_uncond.on_missing=ignore or remove " - "the channel." + "Set cfg_uncond.on_missing=ignore or " + "remove the channel." ) text_policy_raw = cfg_uncond.get("text", None) @@ -828,12 +953,14 @@ def _get_uncond_text_dict( text_policy = "negative_prompt" elif not isinstance(text_policy_raw, str): raise ValueError( - "method_config.cfg_uncond.text must be a " - "string, got " + "method_config.cfg_uncond.text must be " + "a string, got " f"{type(text_policy_raw).__name__}" ) else: - text_policy = text_policy_raw.strip().lower() + text_policy = ( + text_policy_raw.strip().lower() + ) if text_policy in {"negative_prompt"}: text_dict = getattr( @@ -849,32 +976,39 @@ def _get_uncond_text_dict( if text_policy == "keep": if batch.conditional_dict is None: raise RuntimeError( - "Missing conditional_dict in TrainingBatch" + "Missing conditional_dict in " + "TrainingBatch" ) return batch.conditional_dict if text_policy == "zero": if batch.conditional_dict is None: raise RuntimeError( - "Missing conditional_dict in TrainingBatch" + "Missing conditional_dict in " + "TrainingBatch" ) cond = batch.conditional_dict enc = cond["encoder_hidden_states"] mask = cond["encoder_attention_mask"] - if not torch.is_tensor(enc) or not torch.is_tensor( - mask - ): + if not torch.is_tensor( + enc + ) or not torch.is_tensor(mask): raise TypeError( "conditional_dict must contain " "tensor text inputs" ) return { - "encoder_hidden_states": torch.zeros_like(enc), - "encoder_attention_mask": torch.zeros_like(mask), + "encoder_hidden_states": ( + torch.zeros_like(enc) + ), + "encoder_attention_mask": ( + torch.zeros_like(mask) + ), } if text_policy == "drop": raise ValueError( - "cfg_uncond.text=drop is not supported for Wan." - " Use {negative_prompt, keep, zero}." + "cfg_uncond.text=drop is not supported " + "for Wan. Use " + "{negative_prompt, keep, zero}." ) raise ValueError( "cfg_uncond.text must be one of " diff --git a/fastvideo/distillation/models/wangame/wangame.py b/fastvideo/distillation/models/wangame/wangame.py index 2ed59f594..842d40522 100644 --- a/fastvideo/distillation/models/wangame/wangame.py +++ b/fastvideo/distillation/models/wangame/wangame.py @@ -1,17 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 -"""WanGame bidirectional model plugin (per-role instance). - -Each role (student, teacher, critic) gets its own ``WanGameModel`` instance. -The constructor loads the transformer and noise scheduler. Heavyweight -resources (VAE, dataloader, RNG seeds) are loaded via -:meth:`init_preprocessors`, which the method calls only on the student. -""" +"""WanGame bidirectional model plugin (per-role instance).""" from __future__ import annotations import copy -from typing import Any, Literal +from typing import Any, Literal, TYPE_CHECKING import torch @@ -43,8 +37,17 @@ ) from fastvideo.distillation.models.base import ModelBase -from fastvideo.distillation.utils.module_state import apply_trainable -from fastvideo.distillation.utils.moduleloader import load_module_from_path +from fastvideo.distillation.utils.module_state import ( + apply_trainable, +) +from fastvideo.distillation.utils.moduleloader import ( + load_module_from_path, +) + +if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + DistillTrainingConfig, + ) try: from fastvideo.attention.backends.video_sparse_attn import ( @@ -59,14 +62,11 @@ class WanGameModel(ModelBase): - """WanGame per-role model: owns transformer + noise_scheduler. - - Constructor loads the transformer from *init_from*. VAE, dataloader, - and RNG are deferred to :meth:`init_preprocessors`. - """ + """WanGame per-role model: owns transformer + noise_scheduler.""" - # Name of the transformer class to load. Subclasses may override. - _transformer_cls_name: str = "WanGameActionTransformer3DModel" + _transformer_cls_name: str = ( + "WanGameActionTransformer3DModel" + ) def __init__( self, @@ -75,29 +75,34 @@ def __init__( trainable: bool = True, disable_custom_init_weights: bool = False, flow_shift: float = 3.0, - enable_gradient_checkpointing_type: str | None = None, + enable_gradient_checkpointing_type: str + | None = None, ) -> None: self._init_from = str(init_from) self._trainable = bool(trainable) - # We need a minimal TrainingArgs-like object just for loading. - # The full training_args arrives via init_preprocessors(). self.transformer = self._load_transformer( init_from=self._init_from, trainable=self._trainable, - disable_custom_init_weights=disable_custom_init_weights, + disable_custom_init_weights=( + disable_custom_init_weights + ), enable_gradient_checkpointing_type=( enable_gradient_checkpointing_type ), ) - self.noise_scheduler = FlowMatchEulerDiscreteScheduler( - shift=float(flow_shift) + self.noise_scheduler = ( + FlowMatchEulerDiscreteScheduler( + shift=float(flow_shift) + ) ) # Filled by init_preprocessors (student only). self.vae: Any = None - self.training_args: Any = None + self.training_config: DistillTrainingConfig | None = ( + None + ) self.dataloader: Any = None self.validator: Any = None self.start_step: int = 0 @@ -106,16 +111,18 @@ def __init__( self.sp_group: Any = None self.device: Any = None - self.noise_random_generator: torch.Generator | None = None + self.noise_random_generator: ( + torch.Generator | None + ) = None self.noise_gen_cuda: torch.Generator | None = None - # Timestep mechanics (set after init_preprocessors or eagerly). self.timestep_shift: float = float(flow_shift) self.num_train_timestep: int = int( self.noise_scheduler.num_train_timesteps ) self.min_timestep: int = 0 self.max_timestep: int = self.num_train_timestep + def _load_transformer( self, *, @@ -127,15 +134,26 @@ def _load_transformer( transformer = load_module_from_path( model_path=init_from, module_type="transformer", - training_args=None, - disable_custom_init_weights=disable_custom_init_weights, - override_transformer_cls_name=self._transformer_cls_name, + loader_args=None, + disable_custom_init_weights=( + disable_custom_init_weights + ), + override_transformer_cls_name=( + self._transformer_cls_name + ), + ) + transformer = apply_trainable( + transformer, trainable=trainable ) - transformer = apply_trainable(transformer, trainable=trainable) - if trainable and enable_gradient_checkpointing_type: + if ( + trainable + and enable_gradient_checkpointing_type + ): transformer = apply_activation_checkpointing( transformer, - checkpointing_type=enable_gradient_checkpointing_type, + checkpointing_type=( + enable_gradient_checkpointing_type + ), ) return transformer @@ -143,14 +161,25 @@ def _load_transformer( # Lifecycle # ------------------------------------------------------------------ - def init_preprocessors(self, training_args: Any) -> None: - """Load VAE, build dataloader, seed RNGs. Student only.""" - self.training_args = training_args + def init_preprocessors( + self, training_config: DistillTrainingConfig + ) -> None: + """Load VAE, build dataloader, seed RNGs.""" + self.training_config = training_config + + from fastvideo.distillation.utils.loader_args import ( + DistillLoaderArgs, + ) + + loader_args = DistillLoaderArgs.from_training_config( + training_config, + model_path=training_config.model_path, + ) self.vae = load_module_from_path( - model_path=str(training_args.model_path), + model_path=str(training_config.model_path), module_type="vae", - training_args=training_args, + loader_args=loader_args, ) self.world_group = get_world_group() @@ -160,7 +189,9 @@ def init_preprocessors(self, training_args: Any) -> None: self._init_timestep_mechanics() # Optional validator. - validation_cfg = getattr(training_args, "_validation_cfg", None) + validation_cfg = getattr( + training_config, "_validation_cfg", None + ) if validation_cfg: validation_enabled = bool( validation_cfg.get( @@ -171,8 +202,9 @@ def init_preprocessors(self, training_args: Any) -> None: from fastvideo.distillation.validators.wangame import ( WanGameValidator, ) + self.validator = WanGameValidator( - training_args=training_args + training_config=training_config ) from fastvideo.dataset.dataloader.schema import ( @@ -182,9 +214,11 @@ def init_preprocessors(self, training_args: Any) -> None: build_parquet_wangame_train_dataloader, ) - self.dataloader = build_parquet_wangame_train_dataloader( - training_args, - parquet_schema=pyarrow_schema_wangame, + self.dataloader = ( + build_parquet_wangame_train_dataloader( + training_config.data, + parquet_schema=pyarrow_schema_wangame, + ) ) self.start_step = 0 @@ -200,27 +234,36 @@ def shift_and_clamp_timestep( self, timestep: torch.Tensor ) -> torch.Tensor: timestep = shift_timestep( - timestep, self.timestep_shift, self.num_train_timestep + timestep, + self.timestep_shift, + self.num_train_timestep, + ) + return timestep.clamp( + self.min_timestep, self.max_timestep ) - return timestep.clamp(self.min_timestep, self.max_timestep) # ------------------------------------------------------------------ # ModelBase overrides: lifecycle hooks # ------------------------------------------------------------------ def on_train_start(self) -> None: - seed = getattr(self.training_args, "seed", None) + assert self.training_config is not None + tc = self.training_config + seed = tc.data.seed if seed is None: raise ValueError( - "training_args.seed must be set for distillation" + "training.data.seed must be set " + "for distillation" ) - global_rank = int(getattr(self.world_group, "rank", 0)) - sp_world_size = int( - getattr(self.training_args, "sp_size", 1) or 1 + global_rank = int( + getattr(self.world_group, "rank", 0) ) + sp_world_size = int(tc.distributed.sp_size or 1) if sp_world_size > 1: - sp_group_seed = int(seed) + (global_rank // sp_world_size) + sp_group_seed = int(seed) + ( + global_rank // sp_world_size + ) set_random_seed(sp_group_seed) else: set_random_seed(int(seed) + global_rank) @@ -232,10 +275,14 @@ def on_train_start(self) -> None: device=self.device ).manual_seed(int(seed)) - def get_rng_generators(self) -> dict[str, torch.Generator]: + def get_rng_generators( + self, + ) -> dict[str, torch.Generator]: generators: dict[str, torch.Generator] = {} if self.noise_random_generator is not None: - generators["noise_cpu"] = self.noise_random_generator + generators["noise_cpu"] = ( + self.noise_random_generator + ) if self.noise_gen_cuda is not None: generators["noise_cuda"] = self.noise_gen_cuda return generators @@ -251,6 +298,8 @@ def prepare_batch( current_vsa_sparsity: float = 0.0, latents_source: Literal["data", "zeros"] = "data", ) -> TrainingBatch: + assert self.training_config is not None + tc = self.training_config dtype = self._get_training_dtype() device = self.device @@ -263,25 +312,24 @@ def prepare_batch( clip_feature = raw_batch["clip_feature"] batch_size = int(clip_feature.shape[0]) vae_config = ( - self.training_args.pipeline_config - .vae_config.arch_config + tc.pipeline_config.vae_config.arch_config # type: ignore[union-attr] ) num_channels = int(vae_config.z_dim) spatial_compression_ratio = int( vae_config.spatial_compression_ratio ) latent_height = ( - int(self.training_args.num_height) + int(tc.data.num_height) // spatial_compression_ratio ) latent_width = ( - int(self.training_args.num_width) + int(tc.data.num_width) // spatial_compression_ratio ) latents = torch.zeros( batch_size, num_channels, - int(self.training_args.num_latent_t), + int(tc.data.num_latent_t), latent_height, latent_width, device=device, @@ -295,12 +343,13 @@ def prepare_batch( ) latents = raw_batch["vae_latent"] latents = latents[ - :, :, : self.training_args.num_latent_t + :, :, : tc.data.num_latent_t ] latents = latents.to(device, dtype=dtype) else: raise ValueError( - f"Unknown latents_source: {latents_source!r}" + f"Unknown latents_source: " + f"{latents_source!r}" ) if "clip_feature" not in raw_batch: @@ -313,18 +362,21 @@ def prepare_batch( if "first_frame_latent" not in raw_batch: raise ValueError( - "first_frame_latent must be present for WanGame" + "first_frame_latent must be present " + "for WanGame" ) image_latents = raw_batch["first_frame_latent"] image_latents = image_latents[ - :, :, : self.training_args.num_latent_t + :, :, : tc.data.num_latent_t ] - image_latents = image_latents.to(device, dtype=dtype) + image_latents = image_latents.to( + device, dtype=dtype + ) pil_image = raw_batch.get("pil_image") if isinstance(pil_image, torch.Tensor): - training_batch.preprocessed_image = pil_image.to( - device=device + training_batch.preprocessed_image = ( + pil_image.to(device=device) ) else: training_batch.preprocessed_image = pil_image @@ -334,8 +386,8 @@ def prepare_batch( isinstance(keyboard_cond, torch.Tensor) and keyboard_cond.numel() > 0 ): - training_batch.keyboard_cond = keyboard_cond.to( - device, dtype=dtype + training_batch.keyboard_cond = ( + keyboard_cond.to(device, dtype=dtype) ) else: training_batch.keyboard_cond = None @@ -352,11 +404,10 @@ def prepare_batch( training_batch.mouse_cond = None temporal_compression_ratio = ( - self.training_args.pipeline_config - .vae_config.arch_config.temporal_compression_ratio + tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio # type: ignore[union-attr] ) expected_num_frames = ( - (self.training_args.num_latent_t - 1) + (tc.data.num_latent_t - 1) * temporal_compression_ratio + 1 ) @@ -391,8 +442,12 @@ def prepare_batch( training_batch.latents = normalize_dit_input( "wan", training_batch.latents, self.vae ) - training_batch = self._prepare_dit_inputs(training_batch) - training_batch = self._build_attention_metadata(training_batch) + training_batch = self._prepare_dit_inputs( + training_batch + ) + training_batch = self._build_attention_metadata( + training_batch + ) training_batch.attn_metadata_vsa = copy.deepcopy( training_batch.attn_metadata @@ -400,11 +455,11 @@ def prepare_batch( if training_batch.attn_metadata is not None: training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] - training_batch.mask_lat_size = self._build_i2v_mask_latents( - image_latents + training_batch.mask_lat_size = ( + self._build_i2v_mask_latents(image_latents) ) - viewmats, intrinsics, action_labels = self._process_actions( - training_batch + viewmats, intrinsics, action_labels = ( + self._process_actions(training_batch) ) training_batch.viewmats = viewmats training_batch.Ks = intrinsics @@ -444,7 +499,9 @@ def predict_x0( elif attn_kind == "vsa": attn_metadata = batch.attn_metadata_vsa else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + raise ValueError( + f"Unknown attn_kind: {attn_kind!r}" + ) with torch.autocast( device_type, dtype=dtype @@ -452,30 +509,44 @@ def predict_x0( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): - cond_inputs = self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - ) - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs["image_embeds"], - image_latents=cond_inputs["image_latents"], - mask_lat_size=cond_inputs["mask_lat_size"], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs["keyboard_cond"], + cond_inputs = ( + self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + ) ) - transformer = self._get_transformer(timestep) - pred_noise = transformer(**input_kwargs).permute( - 0, 2, 1, 3, 4 + input_kwargs = ( + self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs[ + "image_embeds" + ], + image_latents=cond_inputs[ + "image_latents" + ], + mask_lat_size=cond_inputs[ + "mask_lat_size" + ], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs[ + "keyboard_cond" + ], + ) ) + transformer = self._get_transformer(timestep) + pred_noise = transformer( + **input_kwargs + ).permute(0, 2, 1, 3, 4) pred_x0 = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), + noise_input_latent=noisy_latents.flatten( + 0, 1 + ), timestep=timestep, scheduler=self.noise_scheduler, ).unflatten(0, pred_noise.shape[:2]) @@ -499,7 +570,9 @@ def predict_noise( elif attn_kind == "vsa": attn_metadata = batch.attn_metadata_vsa else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") + raise ValueError( + f"Unknown attn_kind: {attn_kind!r}" + ) with torch.autocast( device_type, dtype=dtype @@ -507,27 +580,39 @@ def predict_noise( current_timestep=batch.timesteps, attn_metadata=attn_metadata, ): - cond_inputs = self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - ) - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs["image_embeds"], - image_latents=cond_inputs["image_latents"], - mask_lat_size=cond_inputs["mask_lat_size"], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs["keyboard_cond"], + cond_inputs = ( + self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + ) ) - transformer = self._get_transformer(timestep) - pred_noise = transformer(**input_kwargs).permute( - 0, 2, 1, 3, 4 + input_kwargs = ( + self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs[ + "image_embeds" + ], + image_latents=cond_inputs[ + "image_latents" + ], + mask_lat_size=cond_inputs[ + "mask_lat_size" + ], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs[ + "keyboard_cond" + ], + ) ) + transformer = self._get_transformer(timestep) + pred_noise = transformer( + **input_kwargs + ).permute(0, 2, 1, 3, 4) return pred_noise def backward( @@ -539,9 +624,12 @@ def backward( ) -> None: timesteps, attn_metadata = ctx with set_forward_context( - current_timestep=timesteps, attn_metadata=attn_metadata + current_timestep=timesteps, + attn_metadata=attn_metadata, ): - (loss / max(1, int(grad_accum_rounds))).backward() + ( + loss / max(1, int(grad_accum_rounds)) + ).backward() # ------------------------------------------------------------------ # Internal helpers @@ -551,40 +639,39 @@ def _get_training_dtype(self) -> torch.dtype: return torch.bfloat16 def _init_timestep_mechanics(self) -> None: + assert self.training_config is not None + tc = self.training_config self.timestep_shift = float( - self.training_args.pipeline_config.flow_shift + tc.pipeline_config.flow_shift # type: ignore[union-attr] ) self.num_train_timestep = int( self.noise_scheduler.num_train_timesteps ) - self.min_timestep = int( - self.training_args.min_timestep_ratio - * self.num_train_timestep - ) - self.max_timestep = int( - self.training_args.max_timestep_ratio - * self.num_train_timestep - ) + self.min_timestep = 0 + self.max_timestep = self.num_train_timestep def _sample_timesteps( self, batch_size: int, device: torch.device ) -> torch.Tensor: if self.noise_random_generator is None: raise RuntimeError( - "on_train_start() must be called before " - "prepare_batch()" + "on_train_start() must be called " + "before prepare_batch()" ) + assert self.training_config is not None + tc = self.training_config u = compute_density_for_timestep_sampling( - weighting_scheme=self.training_args.weighting_scheme, + weighting_scheme=tc.model.weighting_scheme, batch_size=batch_size, generator=self.noise_random_generator, - logit_mean=self.training_args.logit_mean, - logit_std=self.training_args.logit_std, - mode_scale=self.training_args.mode_scale, + logit_mean=tc.model.logit_mean, + logit_std=tc.model.logit_std, + mode_scale=tc.model.mode_scale, ) indices = ( - u * self.noise_scheduler.config.num_train_timesteps + u + * self.noise_scheduler.config.num_train_timesteps ).long() return self.noise_scheduler.timesteps[indices].to( device=device @@ -593,47 +680,67 @@ def _sample_timesteps( def _build_attention_metadata( self, training_batch: TrainingBatch ) -> TrainingBatch: + assert self.training_config is not None + tc = self.training_config latents_shape = training_batch.raw_latent_shape patch_size = ( - self.training_args.pipeline_config.dit_config.patch_size + tc.pipeline_config.dit_config.patch_size # type: ignore[union-attr] + ) + current_vsa_sparsity = ( + training_batch.current_vsa_sparsity ) - current_vsa_sparsity = training_batch.current_vsa_sparsity assert latents_shape is not None assert training_batch.timesteps is not None - if envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN": + if ( + envs.FASTVIDEO_ATTENTION_BACKEND + == "VIDEO_SPARSE_ATTN" + ): if ( not is_vsa_available() - or VideoSparseAttentionMetadataBuilder is None + or VideoSparseAttentionMetadataBuilder + is None ): raise ImportError( "FASTVIDEO_ATTENTION_BACKEND is " - "VIDEO_SPARSE_ATTN, but fastvideo_kernel " - "is not correctly installed or detected." + "VIDEO_SPARSE_ATTN, but " + "fastvideo_kernel is not correctly " + "installed or detected." ) training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] raw_latent_shape=latents_shape[2:5], - current_timestep=training_batch.timesteps, + current_timestep=( + training_batch.timesteps + ), patch_size=patch_size, VSA_sparsity=current_vsa_sparsity, device=self.device, ) - elif envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN": + elif ( + envs.FASTVIDEO_ATTENTION_BACKEND + == "VMOBA_ATTN" + ): if ( not is_vmoba_available() - or VideoMobaAttentionMetadataBuilder is None + or VideoMobaAttentionMetadataBuilder + is None ): raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is VMOBA_ATTN, " - "but fastvideo_kernel (or flash_attn>=2.7.4) " - "is not correctly installed." + "FASTVIDEO_ATTENTION_BACKEND is " + "VMOBA_ATTN, but fastvideo_kernel " + "(or flash_attn>=2.7.4) is not " + "correctly installed." ) - moba_params = self.training_args.moba_config.copy() + moba_params = tc.model.moba_config.copy() moba_params.update( { - "current_timestep": training_batch.timesteps, + "current_timestep": ( + training_batch.timesteps + ), "raw_latent_shape": ( - training_batch.raw_latent_shape[2:5] + training_batch.raw_latent_shape[ + 2:5 + ] ), "patch_size": patch_size, "device": self.device, @@ -648,14 +755,16 @@ def _build_attention_metadata( def _prepare_dit_inputs( self, training_batch: TrainingBatch ) -> TrainingBatch: + assert self.training_config is not None + tc = self.training_config latents = training_batch.latents assert isinstance(latents, torch.Tensor) batch_size = latents.shape[0] if self.noise_gen_cuda is None: raise RuntimeError( - "on_train_start() must be called before " - "prepare_batch()" + "on_train_start() must be called " + "before prepare_batch()" ) noise = torch.randn( @@ -667,9 +776,7 @@ def _prepare_dit_inputs( timesteps = self._sample_timesteps( batch_size, latents.device ) - if ( - int(getattr(self.training_args, "sp_size", 1) or 1) > 1 - ): + if int(tc.distributed.sp_size or 1) > 1: self.sp_group.broadcast(timesteps, src=0) sigmas = get_sigmas( @@ -679,37 +786,50 @@ def _prepare_dit_inputs( n_dim=latents.ndim, dtype=latents.dtype, ) - noisy_model_input = (1.0 - sigmas) * latents + sigmas * noise + noisy_model_input = ( + (1.0 - sigmas) * latents + sigmas * noise + ) - training_batch.noisy_model_input = noisy_model_input + training_batch.noisy_model_input = ( + noisy_model_input + ) training_batch.timesteps = timesteps training_batch.sigmas = sigmas training_batch.noise = noise training_batch.raw_latent_shape = latents.shape - training_batch.latents = training_batch.latents.permute( - 0, 2, 1, 3, 4 + training_batch.latents = ( + training_batch.latents.permute(0, 2, 1, 3, 4) ) return training_batch def _build_i2v_mask_latents( self, image_latents: torch.Tensor ) -> torch.Tensor: + assert self.training_config is not None + tc = self.training_config temporal_compression_ratio = ( - self.training_args.pipeline_config - .vae_config.arch_config.temporal_compression_ratio + tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio # type: ignore[union-attr] ) num_frames = ( - (self.training_args.num_latent_t - 1) + (tc.data.num_latent_t - 1) * temporal_compression_ratio + 1 ) - batch_size, _num_channels, _t, latent_height, latent_width = ( - image_latents.shape - ) + ( + batch_size, + _num_channels, + _t, + latent_height, + latent_width, + ) = image_latents.shape mask_lat_size = torch.ones( - batch_size, 1, num_frames, latent_height, latent_width + batch_size, + 1, + num_frames, + latent_height, + latent_width, ) mask_lat_size[:, :, 1:] = 0 @@ -720,7 +840,8 @@ def _build_i2v_mask_latents( repeats=temporal_compression_ratio, ) mask_lat_size = torch.cat( - [first_frame_mask, mask_lat_size[:, :, 1:]], dim=2 + [first_frame_mask, mask_lat_size[:, :, 1:]], + dim=2, ) mask_lat_size = mask_lat_size.view( batch_size, @@ -731,18 +852,25 @@ def _build_i2v_mask_latents( ) mask_lat_size = mask_lat_size.transpose(1, 2) return mask_lat_size.to( - device=image_latents.device, dtype=image_latents.dtype + device=image_latents.device, + dtype=image_latents.dtype, ) def _process_actions( self, training_batch: TrainingBatch - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - keyboard_cond = getattr(training_batch, "keyboard_cond", None) - mouse_cond = getattr(training_batch, "mouse_cond", None) + ) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor + ]: + keyboard_cond = getattr( + training_batch, "keyboard_cond", None + ) + mouse_cond = getattr( + training_batch, "mouse_cond", None + ) if keyboard_cond is None or mouse_cond is None: raise ValueError( - "WanGame batch must provide keyboard_cond " - "and mouse_cond" + "WanGame batch must provide " + "keyboard_cond and mouse_cond" ) from fastvideo.models.dits.hyworld.pose import ( @@ -766,19 +894,20 @@ def _process_actions( viewmats = torch.stack(viewmats_list, dim=0).to( device=self.device, dtype=torch.bfloat16 ) - intrinsics = torch.stack(intrinsics_list, dim=0).to( - device=self.device, dtype=torch.bfloat16 - ) - action_labels = torch.stack(action_labels_list, dim=0).to( - device=self.device, dtype=torch.bfloat16 - ) + intrinsics = torch.stack( + intrinsics_list, dim=0 + ).to(device=self.device, dtype=torch.bfloat16) + action_labels = torch.stack( + action_labels_list, dim=0 + ).to(device=self.device, dtype=torch.bfloat16) num_latent_t = int( training_batch.noisy_model_input.shape[2] # type: ignore[union-attr] ) if int(action_labels.shape[1]) != num_latent_t: raise ValueError( - "Action conditioning temporal dim mismatch: " + "Action conditioning temporal dim " + "mismatch: " f"action={tuple(action_labels.shape)} " f"vs latent_t={num_latent_t}" ) @@ -807,7 +936,9 @@ def _build_distill_input_kwargs( ) -> dict[str, Any]: hidden_states = torch.cat( [ - noisy_video_latents.permute(0, 2, 1, 3, 4), + noisy_video_latents.permute( + 0, 2, 1, 3, 4 + ), mask_lat_size, image_latents, ], @@ -840,22 +971,27 @@ def _select_cfg_condition_inputs( mask_lat_size = batch.mask_lat_size if image_embeds is None: raise RuntimeError( - "WanGameModel requires TrainingBatch.image_embeds" + "WanGameModel requires " + "TrainingBatch.image_embeds" ) if image_latents is None: raise RuntimeError( - "WanGameModel requires TrainingBatch.image_latents" + "WanGameModel requires " + "TrainingBatch.image_latents" ) if mask_lat_size is None: raise RuntimeError( - "WanGameModel requires TrainingBatch.mask_lat_size" + "WanGameModel requires " + "TrainingBatch.mask_lat_size" ) viewmats = getattr(batch, "viewmats", None) Ks = getattr(batch, "Ks", None) action = getattr(batch, "action", None) mouse_cond = getattr(batch, "mouse_cond", None) - keyboard_cond = getattr(batch, "keyboard_cond", None) + keyboard_cond = getattr( + batch, "keyboard_cond", None + ) if conditional or cfg_uncond is None: return { @@ -869,18 +1005,20 @@ def _select_cfg_condition_inputs( "keyboard_cond": keyboard_cond, } - on_missing_raw = cfg_uncond.get("on_missing", "error") + on_missing_raw = cfg_uncond.get( + "on_missing", "error" + ) if not isinstance(on_missing_raw, str): raise ValueError( - "method_config.cfg_uncond.on_missing must be " - "a string, got " + "method_config.cfg_uncond.on_missing " + "must be a string, got " f"{type(on_missing_raw).__name__}" ) on_missing = on_missing_raw.strip().lower() if on_missing not in {"error", "ignore"}: raise ValueError( - "method_config.cfg_uncond.on_missing must be " - "one of {error, ignore}, got " + "method_config.cfg_uncond.on_missing " + "must be one of {error, ignore}, got " f"{on_missing_raw!r}" ) @@ -894,9 +1032,10 @@ def _select_cfg_condition_inputs( continue if not isinstance(policy_raw, str): raise ValueError( - "method_config.cfg_uncond values must be " - "strings, got " - f"{channel}={type(policy_raw).__name__}" + "method_config.cfg_uncond values " + "must be strings, got " + f"{channel}=" + f"{type(policy_raw).__name__}" ) policy = policy_raw.strip().lower() if policy == "keep": @@ -904,10 +1043,11 @@ def _select_cfg_condition_inputs( if on_missing == "ignore": continue raise ValueError( - "WanGameModel does not support cfg_uncond " - f"channel {channel!r} (policy={policy!r}). " - "Set cfg_uncond.on_missing=ignore or remove " - "the channel." + "WanGameModel does not support " + "cfg_uncond channel " + f"{channel!r} (policy={policy!r}). " + "Set cfg_uncond.on_missing=ignore or " + "remove the channel." ) def _get_policy(channel: str) -> str: @@ -916,15 +1056,16 @@ def _get_policy(channel: str) -> str: return "keep" if not isinstance(raw, str): raise ValueError( - "method_config.cfg_uncond values must be " - "strings, got " + "method_config.cfg_uncond values " + "must be strings, got " f"{channel}={type(raw).__name__}" ) policy = raw.strip().lower() if policy not in {"keep", "zero", "drop"}: raise ValueError( - "method_config.cfg_uncond values must be " - "one of {keep, zero, drop}, got " + "method_config.cfg_uncond values " + "must be one of " + "{keep, zero, drop}, got " f"{channel}={raw!r}" ) return policy @@ -932,12 +1073,17 @@ def _get_policy(channel: str) -> str: image_policy = _get_policy("image") if image_policy == "zero": image_embeds = torch.zeros_like(image_embeds) - image_latents = torch.zeros_like(image_latents) - mask_lat_size = torch.zeros_like(mask_lat_size) + image_latents = torch.zeros_like( + image_latents + ) + mask_lat_size = torch.zeros_like( + mask_lat_size + ) elif image_policy == "drop": raise ValueError( - "cfg_uncond.image=drop is not supported for " - "WanGame I2V; use cfg_uncond.image=zero or keep." + "cfg_uncond.image=drop is not supported " + "for WanGame I2V; use " + "cfg_uncond.image=zero or keep." ) action_policy = _get_policy("action") @@ -951,9 +1097,10 @@ def _get_policy(channel: str) -> str: pass else: raise ValueError( - "cfg_uncond.action=zero requires action " - "conditioning tensors, but TrainingBatch " - "is missing {viewmats, Ks, action}." + "cfg_uncond.action=zero requires " + "action conditioning tensors, " + "but TrainingBatch is missing " + "{viewmats, Ks, action}." ) else: viewmats = torch.zeros_like(viewmats) @@ -962,7 +1109,9 @@ def _get_policy(channel: str) -> str: if mouse_cond is not None: mouse_cond = torch.zeros_like(mouse_cond) if keyboard_cond is not None: - keyboard_cond = torch.zeros_like(keyboard_cond) + keyboard_cond = torch.zeros_like( + keyboard_cond + ) elif action_policy == "drop": viewmats = None Ks = None diff --git a/fastvideo/distillation/models/wangame/wangame_causal.py b/fastvideo/distillation/models/wangame/wangame_causal.py index 269726c0c..676614743 100644 --- a/fastvideo/distillation/models/wangame/wangame_causal.py +++ b/fastvideo/distillation/models/wangame/wangame_causal.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """WanGame causal model plugin (per-role instance, streaming/cache).""" from __future__ import annotations @@ -31,9 +30,7 @@ class _StreamingCaches: class WanGameCausalModel(WanGameModel, CausalModelBase): """WanGame per-role model with causal/streaming primitives.""" - _transformer_cls_name: str = ( - "CausalWanGameActionTransformer3DModel" - ) + _transformer_cls_name: str = ("CausalWanGameActionTransformer3DModel") def __init__( self, @@ -49,19 +46,13 @@ def __init__( trainable=trainable, disable_custom_init_weights=disable_custom_init_weights, flow_shift=flow_shift, - enable_gradient_checkpointing_type=( - enable_gradient_checkpointing_type - ), + enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), ) - self._streaming_caches: dict[ - tuple[int, str], _StreamingCaches - ] = {} + self._streaming_caches: dict[tuple[int, str], _StreamingCaches] = {} # --- CausalModelBase override: clear_caches --- def clear_caches(self, *, cache_tag: str = "pos") -> None: - self._streaming_caches.pop( - (id(self), str(cache_tag)), None - ) + self._streaming_caches.pop((id(self), str(cache_tag)), None) # --- CausalModelBase override: predict_noise_streaming --- def predict_noise_streaming( @@ -109,10 +100,7 @@ def predict_noise_streaming( kv_cache = caches.kv_cache crossattn_cache = caches.crossattn_cache - if ( - self._should_snapshot_streaming_cache() - and torch.is_grad_enabled() - ): + if (self._should_snapshot_streaming_cache() and torch.is_grad_enabled()): kv_cache = self._snapshot_kv_cache_indices(kv_cache) model_kwargs: dict[str, Any] = { @@ -125,11 +113,9 @@ def predict_noise_streaming( device_type = self.device.type dtype = noisy_latents.dtype - with torch.autocast( - device_type, dtype=dtype - ), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, ): cond_inputs = self._select_cfg_condition_inputs( batch, @@ -154,9 +140,7 @@ def predict_noise_streaming( keyboard_cond=cond_inputs["keyboard_cond"], ) - input_kwargs["timestep"] = timestep_full.to( - device=self.device, dtype=torch.long - ) + input_kwargs["timestep"] = timestep_full.to(device=self.device, dtype=torch.long) input_kwargs.update(model_kwargs) if store_kv: @@ -164,9 +148,7 @@ def predict_noise_streaming( _ = transformer(**input_kwargs) return None - pred_noise = transformer(**input_kwargs).permute( - 0, 2, 1, 3, 4 - ) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) return pred_noise # --- CausalModelBase override: predict_x0_streaming --- @@ -206,8 +188,7 @@ def predict_x0_streaming( batch_size=int(noisy_latents.shape[0]), num_frames=int(noisy_latents.shape[1]), device=noisy_latents.device, - ).flatten() - ), + ).flatten()), scheduler=self.noise_scheduler, ).unflatten(0, pred_noise.shape[:2]) return pred_x0 @@ -223,28 +204,16 @@ def _ensure_per_frame_timestep( device: torch.device, ) -> torch.Tensor: if timestep.ndim == 0: - return ( - timestep.view(1, 1) - .expand(batch_size, num_frames) - .to(device=device) - ) + return (timestep.view(1, 1).expand(batch_size, num_frames).to(device=device)) if timestep.ndim == 1: if int(timestep.shape[0]) == batch_size: - return ( - timestep.view(batch_size, 1) - .expand(batch_size, num_frames) - .to(device=device) - ) - raise ValueError( - "streaming timestep must be scalar, [B], or " - f"[B, T]; got shape={tuple(timestep.shape)}" - ) + return (timestep.view(batch_size, 1).expand(batch_size, num_frames).to(device=device)) + raise ValueError("streaming timestep must be scalar, [B], or " + f"[B, T]; got shape={tuple(timestep.shape)}") if timestep.ndim == 2: return timestep.to(device=device) - raise ValueError( - "streaming timestep must be scalar, [B], or [B, T]; " - f"got ndim={int(timestep.ndim)}" - ) + raise ValueError("streaming timestep must be scalar, [B], or [B, T]; " + f"got ndim={int(timestep.ndim)}") def _slice_cond_inputs_for_streaming( self, @@ -256,13 +225,9 @@ def _slice_cond_inputs_for_streaming( start = int(cur_start_frame) num_frames = int(num_frames) if num_frames <= 0: - raise ValueError( - "num_frames must be positive for streaming" - ) + raise ValueError("num_frames must be positive for streaming") if start < 0: - raise ValueError( - "cur_start_frame must be >= 0 for streaming" - ) + raise ValueError("cur_start_frame must be >= 0 for streaming") end = start + num_frames sliced: dict[str, Any] = dict(cond_inputs) @@ -288,12 +253,8 @@ def _slice_cond_inputs_for_streaming( sliced["action"] = action[:, start:end] temporal_compression_ratio = int( - self.training_args.pipeline_config - .vae_config.arch_config.temporal_compression_ratio - ) - raw_end_frame_idx = ( - 1 + temporal_compression_ratio * max(0, end - 1) - ) + self.training_config.pipeline_config.vae_config.arch_config.temporal_compression_ratio) + raw_end_frame_idx = (1 + temporal_compression_ratio * max(0, end - 1)) mouse_cond = cond_inputs.get("mouse_cond") if isinstance(mouse_cond, torch.Tensor): @@ -301,9 +262,7 @@ def _slice_cond_inputs_for_streaming( keyboard_cond = cond_inputs.get("keyboard_cond") if isinstance(keyboard_cond, torch.Tensor): - sliced["keyboard_cond"] = keyboard_cond[ - :, :raw_end_frame_idx - ] + sliced["keyboard_cond"] = keyboard_cond[:, :raw_end_frame_idx] return sliced @@ -321,13 +280,9 @@ def _get_or_init_streaming_caches( dtype = noisy_latents.dtype device = noisy_latents.device - frame_seq_length = self._compute_frame_seq_length( - transformer, noisy_latents - ) + frame_seq_length = self._compute_frame_seq_length(transformer, noisy_latents) local_attn_size = self._get_local_attn_size(transformer) - sliding_window_num_frames = ( - self._get_sliding_window_num_frames(transformer) - ) + sliding_window_num_frames = (self._get_sliding_window_num_frames(transformer)) meta = ( frame_seq_length, @@ -358,13 +313,9 @@ def _get_or_init_streaming_caches( frame_seq_length=frame_seq_length, local_attn_size=local_attn_size, sliding_window_num_frames=sliding_window_num_frames, - checkpoint_safe=( - self._should_use_checkpoint_safe_kv_cache() - ), - ) - crossattn_cache = self._initialize_crossattn_cache( - transformer=transformer, device=device + checkpoint_safe=(self._should_use_checkpoint_safe_kv_cache()), ) + crossattn_cache = self._initialize_crossattn_cache(transformer=transformer, device=device) caches = _StreamingCaches( kv_cache=kv_cache, @@ -384,9 +335,7 @@ def _compute_frame_seq_length( transformer: torch.nn.Module, noisy_latents: torch.Tensor, ) -> int: - latent_seq_length = int(noisy_latents.shape[-1]) * int( - noisy_latents.shape[-2] - ) + latent_seq_length = int(noisy_latents.shape[-1]) * int(noisy_latents.shape[-2]) patch_size = getattr(transformer, "patch_size", None) if patch_size is None: patch_size = getattr( @@ -396,34 +345,22 @@ def _compute_frame_seq_length( ) patch_size = getattr(patch_size, "patch_size", None) if patch_size is None: - raise ValueError( - "Unable to determine transformer.patch_size " - "for causal streaming" - ) + raise ValueError("Unable to determine transformer.patch_size " + "for causal streaming") patch_ratio = int(patch_size[-1]) * int(patch_size[-2]) if patch_ratio <= 0: - raise ValueError( - "Invalid patch_size for causal streaming" - ) + raise ValueError("Invalid patch_size for causal streaming") return latent_seq_length // patch_ratio - def _get_sliding_window_num_frames( - self, transformer: torch.nn.Module - ) -> int: + def _get_sliding_window_num_frames(self, transformer: torch.nn.Module) -> int: cfg = getattr(transformer, "config", None) arch_cfg = getattr(cfg, "arch_config", None) - value = ( - getattr(arch_cfg, "sliding_window_num_frames", None) - if arch_cfg is not None - else None - ) + value = (getattr(arch_cfg, "sliding_window_num_frames", None) if arch_cfg is not None else None) if value is None: return 15 return int(value) - def _get_local_attn_size( - self, transformer: torch.nn.Module - ) -> int: + def _get_local_attn_size(self, transformer: torch.nn.Module) -> int: try: value = getattr(transformer, "local_attn_size", -1) except Exception: @@ -446,57 +383,39 @@ def _initialize_kv_cache( ) -> list[dict[str, Any]]: num_blocks = len(getattr(transformer, "blocks", [])) if num_blocks <= 0: - raise ValueError( - "Unexpected transformer.blocks for causal " - "streaming" - ) + raise ValueError("Unexpected transformer.blocks for causal " + "streaming") try: - num_attention_heads = int( - transformer.num_attention_heads # type: ignore[attr-defined] - ) + num_attention_heads = int(transformer.num_attention_heads # type: ignore[attr-defined] + ) except AttributeError as e: - raise ValueError( - "Transformer is missing num_attention_heads" - ) from e + raise ValueError("Transformer is missing num_attention_heads") from e try: - attention_head_dim = int( - transformer.attention_head_dim # type: ignore[attr-defined] - ) + attention_head_dim = int(transformer.attention_head_dim # type: ignore[attr-defined] + ) except AttributeError: try: - hidden_size = int( - transformer.hidden_size # type: ignore[attr-defined] - ) + hidden_size = int(transformer.hidden_size # type: ignore[attr-defined] + ) except AttributeError as e: - raise ValueError( - "Transformer is missing attention_head_dim " - "and hidden_size" - ) from e - attention_head_dim = hidden_size // max( - 1, num_attention_heads - ) + raise ValueError("Transformer is missing attention_head_dim " + "and hidden_size") from e + attention_head_dim = hidden_size // max(1, num_attention_heads) if local_attn_size != -1: - kv_cache_size = ( - int(local_attn_size) * int(frame_seq_length) - ) + kv_cache_size = (int(local_attn_size) * int(frame_seq_length)) else: - kv_cache_size = int(frame_seq_length) * int( - sliding_window_num_frames - ) + kv_cache_size = int(frame_seq_length) * int(sliding_window_num_frames) if checkpoint_safe: - total_frames = int( - getattr(self.training_args, "num_frames", 0) or 0 - ) + tc = getattr(self, "training_config", None) + total_frames = int(tc.data.num_frames if tc is not None else 0) if total_frames <= 0: - raise ValueError( - "training.num_frames must be set to enable " - "checkpoint-safe streaming KV cache; " - f"got {total_frames}" - ) + raise ValueError("training.num_frames must be set to enable " + "checkpoint-safe streaming KV cache; " + f"got {total_frames}") kv_cache_size = max( kv_cache_size, int(frame_seq_length) * total_frames, @@ -504,73 +423,61 @@ def _initialize_kv_cache( kv_cache: list[dict[str, Any]] = [] for _ in range(num_blocks): - kv_cache.append( - { - "k": torch.zeros( - [ - batch_size, - kv_cache_size, - num_attention_heads, - attention_head_dim, - ], - dtype=dtype, - device=device, - ), - "v": torch.zeros( - [ - batch_size, - kv_cache_size, - num_attention_heads, - attention_head_dim, - ], - dtype=dtype, - device=device, - ), - "global_end_index": torch.zeros( - (), dtype=torch.long, device=device - ), - "local_end_index": torch.zeros( - (), dtype=torch.long, device=device - ), - } - ) + kv_cache.append({ + "k": + torch.zeros( + [ + batch_size, + kv_cache_size, + num_attention_heads, + attention_head_dim, + ], + dtype=dtype, + device=device, + ), + "v": + torch.zeros( + [ + batch_size, + kv_cache_size, + num_attention_heads, + attention_head_dim, + ], + dtype=dtype, + device=device, + ), + "global_end_index": + torch.zeros((), dtype=torch.long, device=device), + "local_end_index": + torch.zeros((), dtype=torch.long, device=device), + }) return kv_cache def _should_use_checkpoint_safe_kv_cache(self) -> bool: - checkpointing_type = getattr( - self.training_args, - "enable_gradient_checkpointing_type", - None, - ) + tc = getattr(self, "training_config", None) + if tc is not None: + checkpointing_type = tc.model.enable_gradient_checkpointing_type + else: + checkpointing_type = None return bool(checkpointing_type) and bool(self._trainable) def _should_snapshot_streaming_cache(self) -> bool: return self._should_use_checkpoint_safe_kv_cache() - def _snapshot_kv_cache_indices( - self, kv_cache: list[dict[str, Any]] - ) -> list[dict[str, Any]]: + def _snapshot_kv_cache_indices(self, kv_cache: list[dict[str, Any]]) -> list[dict[str, Any]]: snapshot: list[dict[str, Any]] = [] for block_cache in kv_cache: global_end_index = block_cache.get("global_end_index") local_end_index = block_cache.get("local_end_index") - if not isinstance( - global_end_index, torch.Tensor - ) or not isinstance(local_end_index, torch.Tensor): - raise ValueError( - "Unexpected kv_cache index tensors; expected " - "tensors at kv_cache[*].{global_end_index, " - "local_end_index}" - ) + if not isinstance(global_end_index, torch.Tensor) or not isinstance(local_end_index, torch.Tensor): + raise ValueError("Unexpected kv_cache index tensors; expected " + "tensors at kv_cache[*].{global_end_index, " + "local_end_index}") copied = dict(block_cache) - copied["global_end_index"] = ( - global_end_index.detach().clone() - ) - copied["local_end_index"] = ( - local_end_index.detach().clone() - ) + copied["global_end_index"] = (global_end_index.detach().clone()) + copied["local_end_index"] = (local_end_index.detach().clone()) snapshot.append(copied) return snapshot @@ -583,11 +490,8 @@ def _initialize_crossattn_cache( num_blocks = len(getattr(transformer, "blocks", [])) if num_blocks <= 0: return None - return [ - { - "is_init": False, - "k": torch.empty(0, device=device), - "v": torch.empty(0, device=device), - } - for _ in range(num_blocks) - ] + return [{ + "is_init": False, + "k": torch.empty(0, device=device), + "v": torch.empty(0, device=device), + } for _ in range(num_blocks)] diff --git a/fastvideo/distillation/trainer.py b/fastvideo/distillation/trainer.py index 88a8d15b9..d0cd0e190 100644 --- a/fastvideo/distillation/trainer.py +++ b/fastvideo/distillation/trainer.py @@ -5,28 +5,29 @@ import time from collections.abc import Iterator from dataclasses import dataclass -from typing import Any +from typing import Any, TYPE_CHECKING import torch from tqdm.auto import tqdm from fastvideo.distributed import get_sp_group, get_world_group -from fastvideo.fastvideo_args import TrainingArgs from fastvideo.distillation.utils.tracking import build_tracker +if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + DistillTrainingConfig, ) + def _coerce_log_scalar(value: Any, *, where: str) -> float: if isinstance(value, torch.Tensor): if value.numel() != 1: - raise ValueError( - f"Expected scalar tensor at {where}, got shape={tuple(value.shape)}" - ) + raise ValueError(f"Expected scalar tensor at {where}, " + f"got shape={tuple(value.shape)}") return float(value.detach().item()) - if isinstance(value, (float, int)): + if isinstance(value, float | int): return float(value) - raise TypeError( - f"Expected a scalar (float/int/Tensor) at {where}, got {type(value).__name__}" - ) + raise TypeError(f"Expected a scalar (float/int/Tensor) at " + f"{where}, got {type(value).__name__}") @dataclass(slots=True) @@ -37,18 +38,23 @@ class TrainLoopState: class DistillTrainer: + def __init__( self, - training_args: TrainingArgs, + training_config: DistillTrainingConfig, *, config: dict[str, Any] | None = None, ) -> None: - self.training_args = training_args + self.training_config = training_config self.world_group = get_world_group() self.sp_group = get_sp_group() self.global_rank = self.world_group.rank self.local_rank = self.world_group.local_rank - self.tracker = build_tracker(training_args, config=config) + self.tracker = build_tracker( + training_config.tracker, + training_config.checkpoint, + config=config, + ) def _iter_dataloader(self, dataloader: Any) -> Iterator[dict[str, Any]]: data_iter = iter(dataloader) @@ -60,10 +66,10 @@ def _iter_dataloader(self, dataloader: Any) -> Iterator[dict[str, Any]]: yield batch def _get_current_vsa_sparsity(self, step: int) -> float: - # Keep behavior close to existing pipelines. - vsa_sparsity = self.training_args.VSA_sparsity - vsa_decay_rate = self.training_args.VSA_decay_rate - vsa_decay_interval_steps = self.training_args.VSA_decay_interval_steps + tc = self.training_config + vsa_sparsity = tc.vsa.sparsity + vsa_decay_rate = tc.vsa.decay_rate + vsa_decay_interval_steps = (tc.vsa.decay_interval_steps) if vsa_decay_interval_steps > 1: current_decay_times = min( step // vsa_decay_interval_steps, @@ -81,8 +87,11 @@ def run( start_step: int = 0, checkpoint_manager: Any | None = None, ) -> None: - grad_accum = max(1, int(self.training_args.gradient_accumulation_steps - or 1)) + tc = self.training_config + grad_accum = max( + 1, + int(tc.loop.gradient_accumulation_steps or 1), + ) if hasattr(method, "set_tracker"): method.set_tracker(self.tracker) # type: ignore[attr-defined] @@ -90,11 +99,9 @@ def run( if hasattr(method, "on_train_start"): method.on_train_start() # type: ignore[attr-defined] - resume_from_checkpoint = getattr(self.training_args, "resume_from_checkpoint", "") or "" + resume_from_checkpoint = (tc.checkpoint.resume_from_checkpoint or "") if checkpoint_manager is not None: - resumed_step = checkpoint_manager.maybe_resume( - resume_from_checkpoint=resume_from_checkpoint - ) + resumed_step = (checkpoint_manager.maybe_resume(resume_from_checkpoint=(resume_from_checkpoint))) if resumed_step is not None: start_step = int(resumed_step) @@ -113,7 +120,7 @@ def run( ) for step in progress: t0 = time.perf_counter() - current_vsa_sparsity = self._get_current_vsa_sparsity(step) + current_vsa_sparsity = (self._get_current_vsa_sparsity(step)) loss_sums: dict[str, float] = {} metric_sums: dict[str, float] = {} @@ -123,11 +130,11 @@ def run( loss_map, outputs, step_metrics = method.single_train_step( # type: ignore[attr-defined] batch, step, - current_vsa_sparsity=current_vsa_sparsity, + current_vsa_sparsity=(current_vsa_sparsity), ) else: - raise AttributeError( - "method must implement single_train_step()") + raise AttributeError("method must implement " + "single_train_step()") if hasattr(method, "backward"): method.backward( # type: ignore[attr-defined] @@ -136,22 +143,22 @@ def run( grad_accum_rounds=grad_accum, ) else: - total_loss = loss_map["total_loss"] / grad_accum + total_loss = (loss_map["total_loss"] / grad_accum) total_loss.backward() for k, v in loss_map.items(): if isinstance(v, torch.Tensor): - loss_sums[k] = loss_sums.get(k, 0.0) + float( - v.detach().item()) + loss_sums[k] = loss_sums.get(k, 0.0) + float(v.detach().item()) for k, v in step_metrics.items(): if k in loss_sums: - raise ValueError( - f"Metric key {k!r} collides with loss key. " - "Use a different name (e.g. prefix with 'train/')." - ) + raise ValueError(f"Metric key {k!r} collides " + "with loss key. Use a " + "different name (e.g. prefix " + "with 'train/').") metric_sums[k] = metric_sums.get(k, 0.0) + _coerce_log_scalar( v, - where=f"method.single_train_step().metrics[{k!r}]", + where=("method.single_train_step()" + f".metrics[{k!r}]"), ) if hasattr(method, "optimizers_schedulers_step"): @@ -161,7 +168,7 @@ def run( metrics = {k: v / grad_accum for k, v in loss_sums.items()} metrics.update({k: v / grad_accum for k, v in metric_sums.items()}) - metrics["step_time_sec"] = time.perf_counter() - t0 + metrics["step_time_sec"] = (time.perf_counter() - t0) metrics["vsa_sparsity"] = float(current_vsa_sparsity) if self.global_rank == 0 and metrics: self.tracker.log(metrics, step) diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/distillation/utils/config.py index cbea20e81..a02ab5ee6 100644 --- a/fastvideo/distillation/utils/config.py +++ b/fastvideo/distillation/utils/config.py @@ -11,8 +11,20 @@ import yaml +from fastvideo.distillation.utils.distill_config import ( + CheckpointConfig, + DataConfig, + DistillTrainingConfig, + DistributedConfig, + ModelTrainingConfig, + OptimizerConfig, + TrackerConfig, + TrainingLoopConfig, + VSAConfig, +) + if TYPE_CHECKING: - from fastvideo.fastvideo_args import TrainingArgs + pass @dataclass(slots=True) @@ -21,7 +33,7 @@ class RunConfig: models: dict[str, dict[str, Any]] method: dict[str, Any] - training_args: TrainingArgs + training: DistillTrainingConfig validation: dict[str, Any] raw: dict[str, Any] @@ -220,17 +232,275 @@ def require_bool( return raw +def _parse_pipeline_config(cfg: dict[str, Any], ) -> Any: + """Resolve PipelineConfig from top-level YAML keys.""" + from fastvideo.configs.pipelines.base import PipelineConfig + + pipeline_raw = cfg.get("pipeline") + default_pipeline_cfg_raw = cfg.get("default_pipeline_config") + default_pipeline_cfg_path = cfg.get("default_pipeline_config_path") + pipeline_cfg_raw = cfg.get("pipeline_config") + pipeline_cfg_path = cfg.get("pipeline_config_path") + + if pipeline_raw is not None: + if default_pipeline_cfg_raw is not None: + raise ValueError("Provide either 'pipeline:' or " + "'default_pipeline_config:', not both") + default_pipeline_cfg_raw = pipeline_raw + + if (default_pipeline_cfg_raw is not None or default_pipeline_cfg_path + is not None) and (pipeline_cfg_raw is not None or pipeline_cfg_path is not None): + raise ValueError("Provide either default_pipeline_config(_path) " + "or the legacy pipeline_config(_path), not both") + + cfg_raw = (default_pipeline_cfg_raw if default_pipeline_cfg_raw is not None else pipeline_cfg_raw) + cfg_path = (default_pipeline_cfg_path if default_pipeline_cfg_path is not None else pipeline_cfg_path) + + if cfg_path is not None: + cfg_path = _require_str( + cfg_path, + where=("default_pipeline_config_path" if default_pipeline_cfg_path is not None else "pipeline_config_path"), + ) + return PipelineConfig.from_kwargs({ + "pipeline_config": _resolve_existing_file(cfg_path), + }) + if cfg_raw is not None: + if isinstance(cfg_raw, str): + return PipelineConfig.from_kwargs({ + "pipeline_config": _resolve_existing_file(cfg_raw), + }) + if isinstance(cfg_raw, dict): + return PipelineConfig.from_kwargs({"pipeline_config": cfg_raw}) + raise ValueError("default_pipeline_config must be a mapping " + "or a path string") + return None + + +def _is_nested_training_format( + t: dict[str, Any], +) -> bool: + """Detect whether training: uses nested sub-groups.""" + _nested_keys = { + "distributed", + "data", + "optimizer", + "loop", + "checkpoint", + "tracker", + "vsa", + "model", + } + return bool(_nested_keys & set(t)) + + +def _build_training_config_nested( + t: dict[str, Any], + *, + models: dict[str, dict[str, Any]], + pipeline_config: Any, +) -> DistillTrainingConfig: + """Build DistillTrainingConfig from nested training: YAML.""" + d = dict(t.get("distributed", {}) or {}) + da = dict(t.get("data", {}) or {}) + o = dict(t.get("optimizer", {}) or {}) + lo = dict(t.get("loop", {}) or {}) + ck = dict(t.get("checkpoint", {}) or {}) + tr = dict(t.get("tracker", {}) or {}) + vs = dict(t.get("vsa", {}) or {}) + m = dict(t.get("model", {}) or {}) + + num_gpus = int(d.get("num_gpus", 1) or 1) + + betas_raw = o.get("betas", "0.9,0.999") + betas = parse_betas(betas_raw, where="training.optimizer.betas") + + model_path = str(t.get("model_path", "") or "") + if not model_path: + student_cfg = models.get("student") + if student_cfg is not None: + init_from = student_cfg.get("init_from") + if init_from is not None: + model_path = str(init_from) + + return DistillTrainingConfig( + distributed=DistributedConfig( + num_gpus=num_gpus, + tp_size=int(d.get("tp_size", 1) or 1), + sp_size=int(d.get("sp_size", num_gpus) or num_gpus), + hsdp_replicate_dim=int(d.get("hsdp_replicate_dim", 1) or 1), + hsdp_shard_dim=int(d.get("hsdp_shard_dim", num_gpus) or num_gpus), + pin_cpu_memory=bool(d.get("pin_cpu_memory", False)), + ), + data=DataConfig( + data_path=str(da.get("data_path", "") or ""), + train_batch_size=int(da.get("train_batch_size", 1) or 1), + dataloader_num_workers=int(da.get("dataloader_num_workers", 0) or 0), + training_cfg_rate=float(da.get("training_cfg_rate", 0.0) or 0.0), + seed=int(da.get("seed", 0) or 0), + num_height=int(da.get("num_height", 0) or 0), + num_width=int(da.get("num_width", 0) or 0), + num_latent_t=int(da.get("num_latent_t", 0) or 0), + num_frames=int(da.get("num_frames", 0) or 0), + ), + optimizer=OptimizerConfig( + learning_rate=float(o.get("learning_rate", 0.0) or 0.0), + betas=betas, + weight_decay=float(o.get("weight_decay", 0.0) or 0.0), + lr_scheduler=str(o.get("lr_scheduler", "constant") or "constant"), + lr_warmup_steps=int(o.get("lr_warmup_steps", 0) or 0), + lr_num_cycles=int(o.get("lr_num_cycles", 0) or 0), + lr_power=float(o.get("lr_power", 0.0) or 0.0), + min_lr_ratio=float(o.get("min_lr_ratio", 0.5) or 0.5), + max_grad_norm=float(o.get("max_grad_norm", 0.0) or 0.0), + ), + loop=TrainingLoopConfig( + max_train_steps=int(lo.get("max_train_steps", 0) or 0), + gradient_accumulation_steps=int(lo.get("gradient_accumulation_steps", 1) or 1), + ), + checkpoint=CheckpointConfig( + output_dir=str(ck.get("output_dir", "") or ""), + resume_from_checkpoint=str(ck.get("resume_from_checkpoint", "") or ""), + training_state_checkpointing_steps=int(ck.get("training_state_checkpointing_steps", 0) or 0), + checkpoints_total_limit=int(ck.get("checkpoints_total_limit", 0) or 0), + ), + tracker=TrackerConfig( + trackers=list(tr.get("trackers", []) or []), + project_name=str(tr.get("project_name", "fastvideo") or "fastvideo"), + run_name=str(tr.get("run_name", "") or ""), + ), + vsa=VSAConfig( + sparsity=float(vs.get("sparsity", 0.0) or 0.0), + decay_rate=float(vs.get("decay_rate", 0.0) or 0.0), + decay_interval_steps=int(vs.get("decay_interval_steps", 0) or 0), + ), + model=ModelTrainingConfig( + weighting_scheme=str(m.get("weighting_scheme", "uniform") or "uniform"), + logit_mean=float(m.get("logit_mean", 0.0) or 0.0), + logit_std=float(m.get("logit_std", 1.0) or 1.0), + mode_scale=float(m.get("mode_scale", 1.0) or 1.0), + precondition_outputs=bool(m.get("precondition_outputs", False)), + moba_config=dict(m.get("moba_config", {}) or {}), + enable_gradient_checkpointing_type=(m.get("enable_gradient_checkpointing_type")), + ), + pipeline_config=pipeline_config, + model_path=model_path, + dit_precision=str(t.get("dit_precision", "fp32") or "fp32"), + ) + + +def _build_training_config_flat( + t: dict[str, Any], + *, + models: dict[str, dict[str, Any]], + pipeline_config: Any, +) -> DistillTrainingConfig: + """Build DistillTrainingConfig from flat training: YAML.""" + num_gpus = int(t.get("num_gpus", 1) or 1) + + betas_raw = t.get("betas", "0.9,0.999") + betas = parse_betas(betas_raw, where="training.betas") + + # Use the student model path as default model_path. + model_path = str(t.get("model_path", "") or "") + if not model_path: + student_cfg = models.get("student") + if student_cfg is not None: + init_from = student_cfg.get("init_from") + if init_from is not None: + model_path = str(init_from) + + return DistillTrainingConfig( + distributed=DistributedConfig( + num_gpus=num_gpus, + tp_size=int(t.get("tp_size", 1) or 1), + sp_size=int(t.get("sp_size", num_gpus) or num_gpus), + hsdp_replicate_dim=int(t.get("hsdp_replicate_dim", 1) or 1), + hsdp_shard_dim=int(t.get("hsdp_shard_dim", num_gpus) or num_gpus), + pin_cpu_memory=bool(t.get("pin_cpu_memory", False)), + ), + data=DataConfig( + data_path=str(t.get("data_path", "") or ""), + train_batch_size=int(t.get("train_batch_size", 1) or 1), + dataloader_num_workers=int(t.get("dataloader_num_workers", 0) or 0), + training_cfg_rate=float(t.get("training_cfg_rate", 0.0) or 0.0), + seed=int(t.get("seed", 0) or 0), + num_height=int(t.get("num_height", 0) or 0), + num_width=int(t.get("num_width", 0) or 0), + num_latent_t=int(t.get("num_latent_t", 0) or 0), + num_frames=int(t.get("num_frames", 0) or 0), + ), + optimizer=OptimizerConfig( + learning_rate=float(t.get("learning_rate", 0.0) or 0.0), + betas=betas, + weight_decay=float(t.get("weight_decay", 0.0) or 0.0), + lr_scheduler=str(t.get("lr_scheduler", "constant") or "constant"), + lr_warmup_steps=int(t.get("lr_warmup_steps", 0) or 0), + lr_num_cycles=int(t.get("lr_num_cycles", 0) or 0), + lr_power=float(t.get("lr_power", 0.0) or 0.0), + min_lr_ratio=float(t.get("min_lr_ratio", 0.5) or 0.5), + max_grad_norm=float(t.get("max_grad_norm", 0.0) or 0.0), + ), + loop=TrainingLoopConfig( + max_train_steps=int(t.get("max_train_steps", 0) or 0), + gradient_accumulation_steps=int(t.get("gradient_accumulation_steps", 1) or 1), + ), + checkpoint=CheckpointConfig( + output_dir=str(t.get("output_dir", "") or ""), + resume_from_checkpoint=str(t.get("resume_from_checkpoint", "") or ""), + training_state_checkpointing_steps=int(t.get("training_state_checkpointing_steps", 0) or 0), + checkpoints_total_limit=int(t.get("checkpoints_total_limit", 0) or 0), + ), + tracker=TrackerConfig( + trackers=list(t.get("trackers", []) or []), + project_name=str(t.get("tracker_project_name", "fastvideo") or "fastvideo"), + run_name=str(t.get("wandb_run_name", "") or ""), + ), + vsa=VSAConfig( + sparsity=float(t.get("VSA_sparsity", 0.0) or 0.0), + decay_rate=float(t.get("VSA_decay_rate", 0.0) or 0.0), + decay_interval_steps=int(t.get("VSA_decay_interval_steps", 0) or 0), + ), + model=ModelTrainingConfig( + weighting_scheme=str(t.get("weighting_scheme", "uniform") or "uniform"), + logit_mean=float(t.get("logit_mean", 0.0) or 0.0), + logit_std=float(t.get("logit_std", 1.0) or 1.0), + mode_scale=float(t.get("mode_scale", 1.0) or 1.0), + precondition_outputs=bool(t.get("precondition_outputs", False)), + moba_config=dict(t.get("moba_config", {}) or {}), + enable_gradient_checkpointing_type=(t.get("enable_gradient_checkpointing_type")), + ), + pipeline_config=pipeline_config, + model_path=model_path, + dit_precision=str(t.get("dit_precision", "fp32") or "fp32"), + ) + + +def _build_training_config( + training_raw: dict[str, Any], + *, + models: dict[str, dict[str, Any]], + pipeline_config: Any, +) -> DistillTrainingConfig: + """Build DistillTrainingConfig from training: YAML. + + Supports both nested (new) and flat (legacy) formats. + """ + t = dict(training_raw) + t.pop("validation", None) + + if _is_nested_training_format(t): + return _build_training_config_nested( + t, models=models, pipeline_config=pipeline_config) + return _build_training_config_flat( + t, models=models, pipeline_config=pipeline_config) + + def load_run_config(path: str) -> RunConfig: """Load a distillation run config from v3 YAML. V3 format uses ``models:`` with ``_target_`` per role and ``method:`` with ``_target_`` for the algorithm class. """ - from fastvideo.fastvideo_args import ( - ExecutionMode, - TrainingArgs, - ) - path = _resolve_existing_file(path) with open(path, encoding="utf-8") as f: raw = yaml.safe_load(f) @@ -243,7 +513,8 @@ def load_run_config(path: str) -> RunConfig: role_str = _require_str(role, where="models.") model_cfg = _require_mapping(model_cfg_raw, where=f"models.{role_str}") if "_target_" not in model_cfg: - raise ValueError(f"models.{role_str} must have a '_target_' key") + raise ValueError(f"models.{role_str} must have a " + "'_target_' key") models[role_str] = dict(model_cfg) # --- method section --- @@ -252,7 +523,7 @@ def load_run_config(path: str) -> RunConfig: raise ValueError("method must have a '_target_' key") method = dict(method_raw) - # --- backward compat: merge method_config into method --- + # --- backward compat: merge method_config --- method_config_raw = cfg.get("method_config", None) if method_config_raw is not None: warnings.warn( @@ -274,18 +545,15 @@ def load_run_config(path: str) -> RunConfig: ) method.setdefault(k, v) - # --- validation section (top-level or under training) --- + # --- validation section --- validation_raw = cfg.get("validation", None) - - # --- training section --- training_raw = _require_mapping(cfg.get("training"), where="training") - # Support validation under training: for backward compat. training_validation_raw = training_raw.get("validation", None) if training_validation_raw is not None: if validation_raw is not None: - raise ValueError("Provide 'validation:' at top-level or under " - "'training:', not both") + raise ValueError("Provide 'validation:' at top-level or " + "under 'training:', not both") warnings.warn( "Nesting 'validation:' under 'training:' is " "deprecated. Move it to the top level.", @@ -297,87 +565,22 @@ def load_run_config(path: str) -> RunConfig: if validation_raw is None: validation: dict[str, Any] = {} else: - validation = _require_mapping( - validation_raw, - where="validation", - ) - - training_kwargs: dict[str, Any] = dict(training_raw) - training_kwargs.pop("validation", None) - - # Entrypoint invariants. - training_kwargs["mode"] = ExecutionMode.DISTILLATION - training_kwargs["inference_mode"] = False - training_kwargs.setdefault("dit_precision", "fp32") - training_kwargs["dit_cpu_offload"] = False - - num_gpus = int(training_kwargs.get("num_gpus", 1) or 1) - training_kwargs.setdefault("num_gpus", num_gpus) - training_kwargs.setdefault("tp_size", 1) - training_kwargs.setdefault("sp_size", num_gpus) - training_kwargs.setdefault("hsdp_replicate_dim", 1) - training_kwargs.setdefault("hsdp_shard_dim", num_gpus) - - # Use the student model path as default model_path. - student_cfg = models.get("student") - if student_cfg is not None and "model_path" not in training_kwargs: - init_from = student_cfg.get("init_from") - if init_from is not None: - training_kwargs["model_path"] = str(init_from) - - if "pretrained_model_name_or_path" not in training_kwargs: - training_kwargs["pretrained_model_name_or_path"] = (training_kwargs.get("model_path", "")) - - # Pipeline config — support both ``pipeline:`` (new) and - # ``default_pipeline_config:`` / ``pipeline_config:`` - # (legacy). - pipeline_raw = cfg.get("pipeline", None) - default_pipeline_cfg_raw = cfg.get("default_pipeline_config", None) - default_pipeline_cfg_path = cfg.get("default_pipeline_config_path", None) - pipeline_cfg_raw = cfg.get("pipeline_config", None) - pipeline_cfg_path = cfg.get("pipeline_config_path", None) - - # Merge pipeline: into default_pipeline_config: for compat. - if pipeline_raw is not None: - if default_pipeline_cfg_raw is not None: - raise ValueError("Provide either 'pipeline:' or " - "'default_pipeline_config:', not both") - default_pipeline_cfg_raw = pipeline_raw + validation = _require_mapping(validation_raw, where="validation") - if (default_pipeline_cfg_raw is not None or default_pipeline_cfg_path - is not None) and (pipeline_cfg_raw is not None or pipeline_cfg_path is not None): - raise ValueError("Provide either default_pipeline_config(_path) or " - "the legacy pipeline_config(_path), not both") + # --- pipeline config --- + pipeline_config = _parse_pipeline_config(cfg) - cfg_raw = (default_pipeline_cfg_raw if default_pipeline_cfg_raw is not None else pipeline_cfg_raw) - cfg_path = (default_pipeline_cfg_path if default_pipeline_cfg_path is not None else pipeline_cfg_path) - - if cfg_path is not None: - cfg_path = _require_str( - cfg_path, - where=("default_pipeline_config_path" if default_pipeline_cfg_path is not None else "pipeline_config_path"), - ) - training_kwargs["pipeline_config"] = (_resolve_existing_file(cfg_path)) - elif cfg_raw is not None: - if isinstance(cfg_raw, str): - training_kwargs["pipeline_config"] = (_resolve_existing_file(cfg_raw)) - elif isinstance(cfg_raw, dict): - training_kwargs["pipeline_config"] = cfg_raw - else: - raise ValueError("default_pipeline_config must be a mapping " - "or a path string") - - training_args = TrainingArgs.from_kwargs(**training_kwargs) - - # Legacy: models read _validation_cfg from training_args - # during their __init__. Remove once models are updated to - # receive validation config through a different mechanism. - training_args._validation_cfg = validation # type: ignore[attr-defined] + # --- build typed training config --- + training = _build_training_config( + training_raw, + models=models, + pipeline_config=pipeline_config, + ) return RunConfig( models=models, method=method, - training_args=training_args, + training=training, validation=validation, raw=cfg, ) diff --git a/fastvideo/distillation/utils/dataloader.py b/fastvideo/distillation/utils/dataloader.py index b87deb0c9..c70715235 100644 --- a/fastvideo/distillation/utils/dataloader.py +++ b/fastvideo/distillation/utils/dataloader.py @@ -2,52 +2,55 @@ from __future__ import annotations -from typing import Any +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + DataConfig, ) def build_parquet_t2v_train_dataloader( - training_args: Any, + data_config: DataConfig, *, + text_len: int, parquet_schema: Any, ) -> Any: - """Build a parquet map-style dataloader for T2V-style latent datasets.""" + """Build a parquet dataloader for T2V-style datasets.""" - from fastvideo.dataset import build_parquet_map_style_dataloader + from fastvideo.dataset import ( + build_parquet_map_style_dataloader, ) - text_len = training_args.pipeline_config.text_encoder_configs[0].arch_config.text_len # type: ignore[attr-defined] - _dataset, dataloader = build_parquet_map_style_dataloader( - training_args.data_path, - training_args.train_batch_size, - num_data_workers=training_args.dataloader_num_workers, + _dataset, dataloader = (build_parquet_map_style_dataloader( + data_config.data_path, + data_config.train_batch_size, + num_data_workers=(data_config.dataloader_num_workers), parquet_schema=parquet_schema, - cfg_rate=training_args.training_cfg_rate, + cfg_rate=data_config.training_cfg_rate, drop_last=True, text_padding_length=int(text_len), - seed=int(training_args.seed or 0), - ) + seed=int(data_config.seed or 0), + )) return dataloader def build_parquet_wangame_train_dataloader( - training_args: Any, + data_config: DataConfig, *, parquet_schema: Any, ) -> Any: - """Build a parquet map-style dataloader for WanGame (I2V+action) datasets.""" + """Build a parquet dataloader for WanGame datasets.""" - from fastvideo.dataset import build_parquet_map_style_dataloader + from fastvideo.dataset import ( + build_parquet_map_style_dataloader, ) - cfg_rate = float(getattr(training_args, "training_cfg_rate", 0.0) or 0.0) - _dataset, dataloader = build_parquet_map_style_dataloader( - training_args.data_path, - training_args.train_batch_size, - num_data_workers=training_args.dataloader_num_workers, + _dataset, dataloader = (build_parquet_map_style_dataloader( + data_config.data_path, + data_config.train_batch_size, + num_data_workers=(data_config.dataloader_num_workers), parquet_schema=parquet_schema, - cfg_rate=cfg_rate, + cfg_rate=float(data_config.training_cfg_rate or 0.0), drop_last=True, - # WanGame parquet schema does not include text embeddings, but the - # parquet loader expects a padding length parameter. text_padding_length=512, - seed=int(training_args.seed or 0), - ) + seed=int(data_config.seed or 0), + )) return dataloader diff --git a/fastvideo/distillation/utils/distill_config.py b/fastvideo/distillation/utils/distill_config.py new file mode 100644 index 000000000..a45afabff --- /dev/null +++ b/fastvideo/distillation/utils/distill_config.py @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Typed distillation training config — replaces TrainingArgs.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from fastvideo.configs.pipelines.base import PipelineConfig + + +@dataclass(slots=True) +class DistributedConfig: + num_gpus: int = 1 + tp_size: int = 1 + sp_size: int = 1 + hsdp_replicate_dim: int = 1 + hsdp_shard_dim: int = -1 + pin_cpu_memory: bool = False + + +@dataclass(slots=True) +class DataConfig: + data_path: str = "" + train_batch_size: int = 1 + dataloader_num_workers: int = 0 + training_cfg_rate: float = 0.0 + seed: int = 0 + num_height: int = 0 + num_width: int = 0 + num_latent_t: int = 0 + num_frames: int = 0 + + +@dataclass(slots=True) +class OptimizerConfig: + learning_rate: float = 0.0 + betas: tuple[float, float] = (0.9, 0.999) + weight_decay: float = 0.0 + lr_scheduler: str = "constant" + lr_warmup_steps: int = 0 + lr_num_cycles: int = 0 + lr_power: float = 0.0 + min_lr_ratio: float = 0.5 + max_grad_norm: float = 0.0 + + +@dataclass(slots=True) +class TrainingLoopConfig: + max_train_steps: int = 0 + gradient_accumulation_steps: int = 1 + + +@dataclass(slots=True) +class CheckpointConfig: + output_dir: str = "" + resume_from_checkpoint: str = "" + training_state_checkpointing_steps: int = 0 + checkpoints_total_limit: int = 0 + + +@dataclass(slots=True) +class TrackerConfig: + trackers: list[str] = field(default_factory=list) + project_name: str = "fastvideo" + run_name: str = "" + + +@dataclass(slots=True) +class VSAConfig: + sparsity: float = 0.0 + decay_rate: float = 0.0 + decay_interval_steps: int = 0 + + +@dataclass(slots=True) +class ModelTrainingConfig: + weighting_scheme: str = "uniform" + logit_mean: float = 0.0 + logit_std: float = 1.0 + mode_scale: float = 1.0 + precondition_outputs: bool = False + moba_config: dict = field(default_factory=dict) + enable_gradient_checkpointing_type: str | None = None + + +@dataclass(slots=True) +class DistillTrainingConfig: + distributed: DistributedConfig = field(default_factory=DistributedConfig) + data: DataConfig = field(default_factory=DataConfig) + optimizer: OptimizerConfig = field(default_factory=OptimizerConfig) + loop: TrainingLoopConfig = field(default_factory=TrainingLoopConfig) + checkpoint: CheckpointConfig = field(default_factory=CheckpointConfig) + tracker: TrackerConfig = field(default_factory=TrackerConfig) + vsa: VSAConfig = field(default_factory=VSAConfig) + model: ModelTrainingConfig = field(default_factory=ModelTrainingConfig) + pipeline_config: PipelineConfig | None = None + model_path: str = "" + dit_precision: str = "fp32" diff --git a/fastvideo/distillation/utils/loader_args.py b/fastvideo/distillation/utils/loader_args.py new file mode 100644 index 000000000..0ccbb424f --- /dev/null +++ b/fastvideo/distillation/utils/loader_args.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Minimal FastVideoArgs subclass for PipelineComponentLoader.""" + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +from fastvideo.configs.pipelines.base import PipelineConfig +from fastvideo.fastvideo_args import ExecutionMode, FastVideoArgs + +if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + DistillTrainingConfig, ) + + +@dataclasses.dataclass +class DistillLoaderArgs(FastVideoArgs): + """Minimal FastVideoArgs for PipelineComponentLoader.""" + + @classmethod + def from_training_config( + cls, + tc: DistillTrainingConfig, + *, + model_path: str, + ) -> DistillLoaderArgs: + return cls( + model_path=model_path, + mode=ExecutionMode.DISTILLATION, + inference_mode=False, + pipeline_config=(tc.pipeline_config or PipelineConfig()), + num_gpus=tc.distributed.num_gpus, + tp_size=tc.distributed.tp_size, + sp_size=tc.distributed.sp_size, + hsdp_replicate_dim=(tc.distributed.hsdp_replicate_dim), + hsdp_shard_dim=tc.distributed.hsdp_shard_dim, + pin_cpu_memory=tc.distributed.pin_cpu_memory, + dit_cpu_offload=False, + dit_layerwise_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=False, + image_encoder_cpu_offload=False, + use_fsdp_inference=False, + enable_torch_compile=False, + ) + + @classmethod + def for_inference( + cls, + tc: DistillTrainingConfig, + *, + model_path: str, + ) -> DistillLoaderArgs: + args = cls.from_training_config(tc, model_path=model_path) + args.inference_mode = True + args.mode = ExecutionMode.INFERENCE + args.dit_cpu_offload = True + return args diff --git a/fastvideo/distillation/utils/moduleloader.py b/fastvideo/distillation/utils/moduleloader.py index 0be538938..2d6757653 100644 --- a/fastvideo/distillation/utils/moduleloader.py +++ b/fastvideo/distillation/utils/moduleloader.py @@ -7,76 +7,89 @@ import torch -from fastvideo.models.loader.component_loader import PipelineComponentLoader -from fastvideo.utils import maybe_download_model, verify_model_config_and_directory +from fastvideo.models.loader.component_loader import ( + PipelineComponentLoader, ) +from fastvideo.utils import ( + maybe_download_model, + verify_model_config_and_directory, +) def load_module_from_path( *, model_path: str, module_type: str, - training_args: Any, + loader_args: Any = None, disable_custom_init_weights: bool = False, override_transformer_cls_name: str | None = None, + # Legacy alias kept so callers that still pass + # ``training_args=`` don't break during migration. + training_args: Any = None, ) -> torch.nn.Module: - """Load a single pipeline component module from a FastVideo model path. + """Load a single pipeline component module. - This is a thin wrapper over :func:`PipelineComponentLoader.load_module`: - - resolves/downloads ``model_path`` if needed - - reads the per-module config entry to determine transformers/diffusers - - optionally disables custom init weights overrides (legacy flag) + *loader_args* should be a ``DistillLoaderArgs`` or + ``FastVideoArgs``-like object for the + ``PipelineComponentLoader``. When ``None`` a lightweight + stand-in is used. """ + # Support the legacy ``training_args`` kwarg. + if loader_args is None and training_args is not None: + loader_args = training_args + local_model_path = maybe_download_model(model_path) config = verify_model_config_and_directory(local_model_path) if module_type not in config: - raise ValueError(f"Module {module_type!r} not found in config at {local_model_path}") + raise ValueError(f"Module {module_type!r} not found in " + f"config at {local_model_path}") module_info = config[module_type] if module_info is None: - raise ValueError(f"Module {module_type!r} has null value in config at {local_model_path}") + raise ValueError(f"Module {module_type!r} has null value in " + f"config at {local_model_path}") transformers_or_diffusers, _architecture = module_info component_path = os.path.join(local_model_path, module_type) - # When training_args is None (e.g. during model-only loading in - # distillation role construction), create a lightweight stand-in so - # that override_transformer_cls_name and disable_custom_init_weights - # flags can still be forwarded to PipelineComponentLoader. - if training_args is None: + if loader_args is None: from types import SimpleNamespace - training_args = SimpleNamespace() - old_override_transformer_cls_name: str | None = None + loader_args = SimpleNamespace() + + old_override: str | None = None if override_transformer_cls_name is not None: - old_override_transformer_cls_name = getattr( - training_args, "override_transformer_cls_name", None + old_override = getattr( + loader_args, + "override_transformer_cls_name", + None, ) - training_args.override_transformer_cls_name = str(override_transformer_cls_name) + loader_args.override_transformer_cls_name = str(override_transformer_cls_name) if disable_custom_init_weights: - # NOTE: This flag is used by PipelineComponentLoader to skip applying - # `init_weights_from_safetensors*` overrides when loading auxiliary - # roles (teacher/critic/etc). The attribute name is legacy. - training_args._loading_teacher_critic_model = True + loader_args._loading_teacher_critic_model = True try: module = PipelineComponentLoader.load_module( module_name=module_type, component_model_path=component_path, - transformers_or_diffusers=transformers_or_diffusers, - fastvideo_args=training_args, + transformers_or_diffusers=(transformers_or_diffusers), + fastvideo_args=loader_args, ) finally: - if disable_custom_init_weights and hasattr(training_args, "_loading_teacher_critic_model"): - del training_args._loading_teacher_critic_model + if disable_custom_init_weights and hasattr(loader_args, "_loading_teacher_critic_model"): + del loader_args._loading_teacher_critic_model if override_transformer_cls_name is not None: - if old_override_transformer_cls_name is None: - if hasattr(training_args, "override_transformer_cls_name"): - training_args.override_transformer_cls_name = None + if old_override is None: + if hasattr( + loader_args, + "override_transformer_cls_name", + ): + loader_args.override_transformer_cls_name = (None) else: - training_args.override_transformer_cls_name = old_override_transformer_cls_name + loader_args.override_transformer_cls_name = (old_override) if not isinstance(module, torch.nn.Module): - raise TypeError(f"Loaded {module_type!r} is not a torch.nn.Module: {type(module)}") + raise TypeError(f"Loaded {module_type!r} is not a " + f"torch.nn.Module: {type(module)}") return module diff --git a/fastvideo/distillation/utils/optimizer.py b/fastvideo/distillation/utils/optimizer.py index ce707e230..27285796b 100644 --- a/fastvideo/distillation/utils/optimizer.py +++ b/fastvideo/distillation/utils/optimizer.py @@ -12,13 +12,17 @@ ) if TYPE_CHECKING: - from fastvideo.fastvideo_args import TrainingArgs + from fastvideo.distillation.utils.distill_config import ( + OptimizerConfig, + TrainingLoopConfig, + ) def build_optimizer_and_scheduler( *, params: list[torch.nn.Parameter], - training_args: TrainingArgs, + optimizer_config: OptimizerConfig, + loop_config: TrainingLoopConfig, learning_rate: float, betas: tuple[float, float], scheduler_name: str, @@ -29,39 +33,25 @@ def build_optimizer_and_scheduler( as method-level attributes. """ if not params: - raise ValueError( - "No trainable parameters passed to " - "build_optimizer_and_scheduler" - ) + raise ValueError("No trainable parameters passed to " + "build_optimizer_and_scheduler") optimizer = torch.optim.AdamW( params, lr=float(learning_rate), betas=betas, - weight_decay=float( - getattr(training_args, "weight_decay", 0.0) or 0.0 - ), + weight_decay=float(optimizer_config.weight_decay), eps=1e-8, ) scheduler = get_scheduler( str(scheduler_name), optimizer=optimizer, - num_warmup_steps=int( - getattr(training_args, "lr_warmup_steps", 0) or 0 - ), - num_training_steps=int( - getattr(training_args, "max_train_steps", 0) or 0 - ), - num_cycles=int( - getattr(training_args, "lr_num_cycles", 0) or 0 - ), - power=float( - getattr(training_args, "lr_power", 0.0) or 0.0 - ), - min_lr_ratio=float( - getattr(training_args, "min_lr_ratio", 0.5) or 0.5 - ), + num_warmup_steps=int(optimizer_config.lr_warmup_steps), + num_training_steps=int(loop_config.max_train_steps), + num_cycles=int(optimizer_config.lr_num_cycles), + power=float(optimizer_config.lr_power), + min_lr_ratio=float(optimizer_config.min_lr_ratio), last_epoch=-1, ) @@ -69,29 +59,14 @@ def build_optimizer_and_scheduler( def clip_grad_norm_if_needed( - module: torch.nn.Module, training_args: TrainingArgs + module: torch.nn.Module, + max_grad_norm: float, ) -> float: - max_grad_norm_raw = getattr( - training_args, "max_grad_norm", None - ) - if max_grad_norm_raw is None: - return 0.0 - try: - max_grad_norm = float(max_grad_norm_raw) - except (TypeError, ValueError) as e: - raise ValueError( - "training.max_grad_norm must be a number when set, " - f"got {max_grad_norm_raw!r}" - ) from e if max_grad_norm <= 0.0: return 0.0 - grad_norm = ( - clip_grad_norm_while_handling_failing_dtensor_cases( - [p for p in module.parameters()], - max_grad_norm, - foreach=None, - ) - ) - return ( - float(grad_norm.item()) if grad_norm is not None else 0.0 - ) + grad_norm = (clip_grad_norm_while_handling_failing_dtensor_cases( + [p for p in module.parameters()], + max_grad_norm, + foreach=None, + )) + return (float(grad_norm.item()) if grad_norm is not None else 0.0) diff --git a/fastvideo/distillation/utils/tracking.py b/fastvideo/distillation/utils/tracking.py index 3a890f3f5..4e7f23823 100644 --- a/fastvideo/distillation/utils/tracking.py +++ b/fastvideo/distillation/utils/tracking.py @@ -3,40 +3,49 @@ from __future__ import annotations import os -from typing import Any +from typing import Any, TYPE_CHECKING from fastvideo.distributed import get_world_group -from fastvideo.training.trackers import initialize_trackers, Trackers - +from fastvideo.training.trackers import ( + initialize_trackers, + Trackers, +) + +if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + CheckpointConfig, + TrackerConfig, + ) -def build_tracker(training_args: Any, *, config: dict[str, Any] | None) -> Any: - """Build a tracker instance for a distillation run. - Tracker selection is rank0-only; other ranks get a no-op tracker from - ``initialize_trackers([])``. - """ +def build_tracker( + tracker_config: TrackerConfig, + checkpoint_config: CheckpointConfig, + *, + config: dict[str, Any] | None, +) -> Any: + """Build a tracker instance for a distillation run.""" world_group = get_world_group() - trackers = list(training_args.trackers) - if not trackers and str(training_args.tracker_project_name): + trackers = list(tracker_config.trackers) + if not trackers and str(tracker_config.project_name): trackers.append(Trackers.WANDB.value) if world_group.rank != 0: trackers = [] - tracker_log_dir = training_args.output_dir or os.getcwd() + tracker_log_dir = (checkpoint_config.output_dir or os.getcwd()) if trackers: tracker_log_dir = os.path.join(tracker_log_dir, "tracker") - tracker_config = config if trackers else None - tracker_run_name = training_args.wandb_run_name or None - project = training_args.tracker_project_name or "fastvideo" + tracker_config_dict = config if trackers else None + tracker_run_name = tracker_config.run_name or None + project = (tracker_config.project_name or "fastvideo") return initialize_trackers( trackers, experiment_name=project, - config=tracker_config, + config=tracker_config_dict, log_dir=tracker_log_dir, run_name=tracker_run_name, ) - diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 8aa09a657..6984f39ec 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -1,25 +1,28 @@ # SPDX-License-Identifier: Apache-2.0 - """Wan validator (model-family validation backend). Config keys used: -- `training` (selected fields): - - `seed`, `model_path` - - `num_height`, `num_width`, `num_latent_t` - - `tp_size`, `sp_size`, `num_gpus`, `pin_cpu_memory` +- `training` (DistillTrainingConfig): + - `data.seed`, `model_path` + - `data.num_height`, `data.num_width`, `data.num_latent_t` + - `distributed.tp_size`, `distributed.sp_size`, + `distributed.num_gpus`, `distributed.pin_cpu_memory` - `pipeline_config.flow_shift` - - `pipeline_config.vae_config.arch_config.temporal_compression_ratio` - - `VSA_sparsity` -- `training.validation.*` (typically parsed by a method into `ValidationRequest`): + - `pipeline_config.vae_config.arch_config + .temporal_compression_ratio` + - `vsa.sparsity` +- `training.validation.*` (typically parsed by a method into + `ValidationRequest`): - `dataset_file`, `sampling_steps`, `guidance_scale` - - `sampler_kind` (`ode`/`sde`), `ode_solver` (`euler`/`unipc`), `rollout_mode` + - `sampler_kind` (`ode`/`sde`), `ode_solver` (`euler`/`unipc`), + `rollout_mode` """ from __future__ import annotations import os from dataclasses import dataclass -from typing import Any +from typing import Any, TYPE_CHECKING import imageio import numpy as np @@ -29,16 +32,23 @@ from torch.utils.data import DataLoader from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.validation_dataset import ValidationDataset +from fastvideo.dataset.validation_dataset import ( + ValidationDataset, ) from fastvideo.distributed import get_sp_group, get_world_group -from fastvideo.fastvideo_args import ExecutionMode from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch -from fastvideo.distillation.validators.base import ValidationRequest +from fastvideo.distillation.utils.loader_args import ( + DistillLoaderArgs, ) +from fastvideo.distillation.validators.base import ( + ValidationRequest, ) from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline from fastvideo.training.trackers import DummyTracker from fastvideo.utils import shallow_asdict +if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + DistillTrainingConfig, ) + logger = init_logger(__name__) @@ -54,10 +64,10 @@ class WanValidator: def __init__( self, *, - training_args: Any, + training_config: DistillTrainingConfig, tracker: Any | None = None, ) -> None: - self.training_args = training_args + self.training_config = training_config self.tracker = tracker or DummyTracker() self.world_group = get_world_group() @@ -66,14 +76,14 @@ def __init__( self.rank_in_sp_group = self.sp_group.rank_in_group self.sp_world_size = self.sp_group.world_size - seed = getattr(training_args, "seed", None) + seed = training_config.data.seed if seed is None: - raise ValueError("training_args.seed must be set for validation") + raise ValueError("training.data.seed must be set for validation") self.seed = int(seed) - self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) + self.validation_random_generator = (torch.Generator(device="cpu").manual_seed(self.seed)) self._pipeline: WanPipeline | None = None - self._pipeline_key: tuple[int, str, str] | None = None + self._pipeline_key: (tuple[int, str, str] | None) = None self._sampling_param: SamplingParam | None = None def set_tracker(self, tracker: Any) -> None: @@ -81,7 +91,7 @@ def set_tracker(self, tracker: Any) -> None: def _get_sampling_param(self) -> SamplingParam: if self._sampling_param is None: - self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) + self._sampling_param = (SamplingParam.from_pretrained(self.training_config.model_path)) return self._sampling_param def _get_pipeline( @@ -91,22 +101,27 @@ def _get_pipeline( sampler_kind: str, ode_solver: str | None, ) -> WanPipeline: - key = (id(transformer), str(sampler_kind), str(ode_solver)) - if self._pipeline is not None and self._pipeline_key == key: + key = ( + id(transformer), + str(sampler_kind), + str(ode_solver), + ) + if (self._pipeline is not None and self._pipeline_key == key): return self._pipeline - # NOTE: `ComposedPipelineBase.from_pretrained()` ignores `args` when - # `inference_mode=True`, so we must pass pipeline knobs via kwargs. - flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) + tc = self.training_config + flow_shift = getattr(tc.pipeline_config, "flow_shift", None) kwargs: dict[str, Any] = { "inference_mode": True, "sampler_kind": str(sampler_kind), - "loaded_modules": {"transformer": transformer}, - "tp_size": self.training_args.tp_size, - "sp_size": self.training_args.sp_size, - "num_gpus": self.training_args.num_gpus, - "pin_cpu_memory": self.training_args.pin_cpu_memory, + "loaded_modules": { + "transformer": transformer + }, + "tp_size": tc.distributed.tp_size, + "sp_size": tc.distributed.sp_size, + "num_gpus": tc.distributed.num_gpus, + "pin_cpu_memory": tc.distributed.pin_cpu_memory, "dit_cpu_offload": True, } if flow_shift is not None: @@ -114,7 +129,7 @@ def _get_pipeline( if ode_solver is not None: kwargs["ode_solver"] = str(ode_solver) - self._pipeline = WanPipeline.from_pretrained(self.training_args.model_path, **kwargs) + self._pipeline = WanPipeline.from_pretrained(tc.model_path, **kwargs) self._pipeline_key = key return self._pipeline @@ -127,10 +142,12 @@ def _prepare_validation_batch( sampling_timesteps: list[int] | None = None, guidance_scale: float | None = None, ) -> ForwardBatch: + tc = self.training_config + sampling_param.prompt = validation_batch["prompt"] - sampling_param.height = self.training_args.num_height - sampling_param.width = self.training_args.num_width - sampling_param.num_inference_steps = num_inference_steps + sampling_param.height = tc.data.num_height + sampling_param.width = tc.data.num_width + sampling_param.num_inference_steps = (num_inference_steps) sampling_param.data_type = "video" if guidance_scale is not None: sampling_param.guidance_scale = float(guidance_scale) @@ -141,31 +158,33 @@ def _prepare_validation_batch( sampling_param.height // 8, sampling_param.width // 8, ] - n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + n_tokens = (latents_size[0] * latents_size[1] * latents_size[2]) - temporal_compression_factor = ( - self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - ) - num_frames = (self.training_args.num_latent_t - 1) * temporal_compression_factor + 1 + temporal_compression_factor = int(tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio) + num_frames = ((tc.data.num_latent_t - 1) * temporal_compression_factor + 1) sampling_param.num_frames = int(num_frames) - sampling_timesteps_tensor = ( - torch.tensor([int(s) for s in sampling_timesteps], dtype=torch.long) - if sampling_timesteps is not None - else None - ) + sampling_timesteps_tensor = (torch.tensor( + [int(s) for s in sampling_timesteps], + dtype=torch.long, + ) if sampling_timesteps is not None else None) + + # Build DistillLoaderArgs for inference to pass + # to pipeline.forward(). + inference_args = DistillLoaderArgs.for_inference(tc, model_path=tc.model_path) + batch = ForwardBatch( **shallow_asdict(sampling_param), latents=None, generator=self.validation_random_generator, n_tokens=n_tokens, eta=0.0, - VSA_sparsity=self.training_args.VSA_sparsity, - # SDE-style sampling iterates `sampling_timesteps`. Some stages still - # expect `timesteps` to be set, so we mirror the same tensor there. + VSA_sparsity=tc.vsa.sparsity, timesteps=sampling_timesteps_tensor, sampling_timesteps=sampling_timesteps_tensor, ) + # Store inference_args on batch for pipeline access. + batch._inference_args = inference_args # type: ignore[attr-defined] return batch def _run_validation_for_steps( @@ -179,7 +198,7 @@ def _run_validation_for_steps( sampling_timesteps: list[int] | None = None, guidance_scale: float | None = None, ) -> _ValidationStepResult: - training_args = self.training_args + tc = self.training_config pipeline = self._get_pipeline( transformer=transformer, sampler_kind=sampler_kind, @@ -190,6 +209,9 @@ def _run_validation_for_steps( dataset = ValidationDataset(dataset_file) dataloader = DataLoader(dataset, batch_size=None, num_workers=0) + # Build inference args once for this validation run. + inference_args = DistillLoaderArgs.for_inference(tc, model_path=tc.model_path) + videos: list[list[np.ndarray]] = [] captions: list[str] = [] @@ -202,11 +224,11 @@ def _run_validation_for_steps( guidance_scale=guidance_scale, ) - assert batch.prompt is not None and isinstance(batch.prompt, str) + assert (batch.prompt is not None and isinstance(batch.prompt, str)) captions.append(batch.prompt) with torch.no_grad(): - output_batch = pipeline.forward(batch, training_args) + output_batch = pipeline.forward(batch, inference_args) samples = output_batch.output.cpu() if self.rank_in_sp_group != 0: @@ -222,57 +244,59 @@ def _run_validation_for_steps( return _ValidationStepResult(videos=videos, captions=captions) - def log_validation(self, step: int, *, request: ValidationRequest | None = None) -> None: + def log_validation( + self, + step: int, + *, + request: ValidationRequest | None = None, + ) -> None: if request is None: - raise ValueError("WanValidator.log_validation requires a ValidationRequest") + raise ValueError("WanValidator.log_validation requires a " + "ValidationRequest") dataset_file = getattr(request, "dataset_file", None) if not dataset_file: - raise ValueError("ValidationRequest.dataset_file must be provided by the method") + raise ValueError("ValidationRequest.dataset_file must be " + "provided by the method") guidance_scale = getattr(request, "guidance_scale", None) validation_steps = getattr(request, "sampling_steps", None) if not validation_steps: - raise ValueError("ValidationRequest.sampling_steps must be provided by the method") - sampler_kind = getattr(request, "sampler_kind", None) or "ode" + raise ValueError("ValidationRequest.sampling_steps must be " + "provided by the method") + sampler_kind = (getattr(request, "sampler_kind", None) or "ode") rollout_mode = getattr(request, "rollout_mode", None) if rollout_mode not in {None, "parallel"}: - raise ValueError( - "WanValidator only supports rollout_mode='parallel'. " - f"Got rollout_mode={rollout_mode!r}." - ) + raise ValueError("WanValidator only supports " + "rollout_mode='parallel'. " + f"Got rollout_mode={rollout_mode!r}.") ode_solver = getattr(request, "ode_solver", None) sampling_timesteps = getattr(request, "sampling_timesteps", None) if sampling_timesteps is not None: expected = int(len(sampling_timesteps)) for steps in validation_steps: if int(steps) != expected: - raise ValueError( - "validation_sampling_steps must match " - f"len(request.sampling_timesteps)={expected} when " - "sampling_timesteps is provided, got " - f"{validation_steps!r}." - ) + raise ValueError("validation_sampling_steps must " + "match " + "len(request.sampling_timesteps)=" + f"{expected} when " + "sampling_timesteps is provided, " + f"got {validation_steps!r}.") sample_handle = getattr(request, "sample_handle", None) if sample_handle is None: - raise ValueError("ValidationRequest.sample_handle must be provided by the method") + raise ValueError("ValidationRequest.sample_handle must be " + "provided by the method") transformer = sample_handle.require_module("transformer") was_training = bool(getattr(transformer, "training", False)) - training_args = self.training_args - output_dir = getattr(request, "output_dir", None) or training_args.output_dir + tc = self.training_config + output_dir = (getattr(request, "output_dir", None) or tc.checkpoint.output_dir) - old_inference_mode = training_args.inference_mode - old_dit_cpu_offload = training_args.dit_cpu_offload - old_mode = training_args.mode try: - training_args.inference_mode = True - training_args.dit_cpu_offload = True - training_args.mode = ExecutionMode.INFERENCE transformer.eval() - num_sp_groups = self.world_group.world_size // self.sp_group.world_size + num_sp_groups = (self.world_group.world_size // self.sp_group.world_size) for num_inference_steps in validation_steps: result = self._run_validation_for_steps( @@ -280,7 +304,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) dataset_file=str(dataset_file), transformer=transformer, sampler_kind=str(sampler_kind), - ode_solver=str(ode_solver) if ode_solver is not None else None, + ode_solver=(str(ode_solver) if ode_solver is not None else None), sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, ) @@ -292,39 +316,49 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) all_videos = list(result.videos) all_captions = list(result.captions) for sp_group_idx in range(1, num_sp_groups): - src_rank = sp_group_idx * self.sp_world_size - recv_videos = self.world_group.recv_object(src=src_rank) - recv_captions = self.world_group.recv_object(src=src_rank) + src_rank = (sp_group_idx * self.sp_world_size) + recv_videos = (self.world_group.recv_object(src=src_rank)) + recv_captions = (self.world_group.recv_object(src=src_rank)) all_videos.extend(recv_videos) all_captions.extend(recv_captions) os.makedirs(output_dir, exist_ok=True) video_filenames: list[str] = [] - sampling_param = self._get_sampling_param() + sampling_param = (self._get_sampling_param()) for i, video in enumerate(all_videos): filename = os.path.join( output_dir, - f"validation_step_{step}_inference_steps_{num_inference_steps}_video_{i}.mp4", + f"validation_step_{step}" + f"_inference_steps_" + f"{num_inference_steps}" + f"_video_{i}.mp4", + ) + imageio.mimsave( + filename, + video, + fps=sampling_param.fps, ) - imageio.mimsave(filename, video, fps=sampling_param.fps) video_filenames.append(filename) video_logs = [] - for filename, caption in zip(video_filenames, all_captions, strict=True): + for filename, caption in zip( + video_filenames, + all_captions, + strict=True, + ): video_artifact = self.tracker.video(filename, caption=caption) if video_artifact is not None: video_logs.append(video_artifact) if video_logs: - logs = {f"validation_videos_{num_inference_steps}_steps": video_logs} - # Tracker API name uses "artifacts" to mean media/files - # (e.g., videos for W&B). We keep the upstream API name. + logs = { + f"validation_videos_" + f"{num_inference_steps}" + f"_steps": video_logs + } self.tracker.log_artifacts(logs, step) else: self.world_group.send_object(result.videos, dst=0) self.world_group.send_object(result.captions, dst=0) finally: - training_args.inference_mode = old_inference_mode - training_args.dit_cpu_offload = old_dit_cpu_offload - training_args.mode = old_mode if was_training: transformer.train() diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index 4e9363a81..7998a1e1c 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -1,26 +1,29 @@ # SPDX-License-Identifier: Apache-2.0 - """WanGame validator (model-family validation backend). Config keys used: -- `training` (selected fields): - - `seed`, `model_path` - - `num_height`, `num_width`, `num_latent_t` - - `tp_size`, `sp_size`, `num_gpus`, `pin_cpu_memory` +- `training` (DistillTrainingConfig): + - `data.seed`, `model_path` + - `data.num_height`, `data.num_width`, `data.num_latent_t` + - `distributed.tp_size`, `distributed.sp_size`, + `distributed.num_gpus`, `distributed.pin_cpu_memory` - `pipeline_config.flow_shift` - - `pipeline_config.vae_config.arch_config.temporal_compression_ratio` - - `VSA_sparsity`, `VSA_decay_rate`, `VSA_decay_interval_steps` (when applicable) -- `training.validation.*` (typically parsed by a method into `ValidationRequest`): + - `pipeline_config.vae_config.arch_config + .temporal_compression_ratio` + - `vsa.sparsity` +- `training.validation.*` (typically parsed by a method into + `ValidationRequest`): - `dataset_file`, `sampling_steps`, `guidance_scale` - - `sampler_kind` (`ode`/`sde`), `ode_solver` (`euler`/`unipc`) - - `rollout_mode` (`parallel`/`streaming`), `num_frames` (action length) + - `sampler_kind` (`ode`/`sde`), `ode_solver` + (`euler`/`unipc`) + - `rollout_mode` (`parallel`/`streaming`), `num_frames` """ from __future__ import annotations import os from dataclasses import dataclass -from typing import Any +from typing import Any, TYPE_CHECKING import imageio import numpy as np @@ -30,15 +33,22 @@ from torch.utils.data import DataLoader from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.validation_dataset import ValidationDataset +from fastvideo.dataset.validation_dataset import ( + ValidationDataset, ) from fastvideo.distributed import get_sp_group, get_world_group -from fastvideo.fastvideo_args import ExecutionMode from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch -from fastvideo.distillation.validators.base import ValidationRequest +from fastvideo.distillation.utils.loader_args import ( + DistillLoaderArgs, ) +from fastvideo.distillation.validators.base import ( + ValidationRequest, ) from fastvideo.training.trackers import DummyTracker from fastvideo.utils import shallow_asdict +if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + DistillTrainingConfig, ) + logger = init_logger(__name__) @@ -49,15 +59,15 @@ class _ValidationStepResult: class WanGameValidator: - """Standalone validator for WanGame distillation/finetuning.""" + """Standalone validator for WanGame distillation.""" def __init__( self, *, - training_args: Any, + training_config: DistillTrainingConfig, tracker: Any | None = None, ) -> None: - self.training_args = training_args + self.training_config = training_config self.tracker = tracker or DummyTracker() self.world_group = get_world_group() @@ -66,14 +76,15 @@ def __init__( self.rank_in_sp_group = self.sp_group.rank_in_group self.sp_world_size = self.sp_group.world_size - seed = getattr(training_args, "seed", None) + seed = training_config.data.seed if seed is None: - raise ValueError("training_args.seed must be set for validation") + raise ValueError("training.data.seed must be set for " + "validation") self.seed = int(seed) - self.validation_random_generator = torch.Generator(device="cpu").manual_seed(self.seed) + self.validation_random_generator = (torch.Generator(device="cpu").manual_seed(self.seed)) self._pipeline: Any | None = None - self._pipeline_key: tuple[int, str, str, str] | None = None + self._pipeline_key: (tuple[int, str, str, str] | None) = None self._sampling_param: SamplingParam | None = None def set_tracker(self, tracker: Any) -> None: @@ -84,10 +95,7 @@ def _post_process_validation_frames( frames: list[np.ndarray], batch: ForwardBatch, ) -> list[np.ndarray]: - """Optionally overlay action indicators on validation frames. - - Mirrors legacy `WanGameTrainingPipeline._post_process_validation_frames()`. - """ + """Optionally overlay action indicators.""" keyboard_cond = getattr(batch, "keyboard_cond", None) mouse_cond = getattr(batch, "mouse_cond", None) @@ -100,35 +108,39 @@ def _post_process_validation_frames( draw_mouse_on_frame, ) except Exception as e: - logger.warning("WanGame action overlay is unavailable: %s", e) + logger.warning( + "WanGame action overlay is unavailable: %s", + e, + ) return frames - if keyboard_cond is not None and torch.is_tensor(keyboard_cond): - keyboard_np = keyboard_cond.squeeze(0).detach().cpu().float().numpy() + if (keyboard_cond is not None and torch.is_tensor(keyboard_cond)): + keyboard_np = (keyboard_cond.squeeze(0).detach().cpu().float().numpy()) else: keyboard_np = None - if mouse_cond is not None and torch.is_tensor(mouse_cond): - mouse_np = mouse_cond.squeeze(0).detach().cpu().float().numpy() + if (mouse_cond is not None and torch.is_tensor(mouse_cond)): + mouse_np = (mouse_cond.squeeze(0).detach().cpu().float().numpy()) else: mouse_np = None - # MatrixGame convention: keyboard [W, S, A, D, left, right], - # mouse [Pitch, Yaw]. key_names = ["W", "S", "A", "D", "left", "right"] processed_frames: list[np.ndarray] = [] for frame_idx, frame in enumerate(frames): frame = np.ascontiguousarray(frame.copy()) - if keyboard_np is not None and frame_idx < len(keyboard_np): + if (keyboard_np is not None and frame_idx < len(keyboard_np)): keys = { key_names[i]: bool(keyboard_np[frame_idx, i]) - for i in range(min(len(key_names), int(keyboard_np.shape[1]))) + for i in range(min( + len(key_names), + int(keyboard_np.shape[1]), + )) } draw_keys_on_frame(frame, keys, mode="universal") - if mouse_np is not None and frame_idx < len(mouse_np): + if (mouse_np is not None and frame_idx < len(mouse_np)): pitch = float(mouse_np[frame_idx, 0]) yaw = float(mouse_np[frame_idx, 1]) draw_mouse_on_frame(frame, pitch, yaw) @@ -139,7 +151,7 @@ def _post_process_validation_frames( def _get_sampling_param(self) -> SamplingParam: if self._sampling_param is None: - self._sampling_param = SamplingParam.from_pretrained(self.training_args.model_path) + self._sampling_param = (SamplingParam.from_pretrained(self.training_config.model_path)) return self._sampling_param def _get_pipeline( @@ -152,30 +164,37 @@ def _get_pipeline( ) -> Any: rollout_mode = str(rollout_mode).strip().lower() sampler_kind = str(sampler_kind).strip().lower() - key = (id(transformer), rollout_mode, sampler_kind, str(ode_solver)) - if self._pipeline is not None and self._pipeline_key == key: + key = ( + id(transformer), + rollout_mode, + sampler_kind, + str(ode_solver), + ) + if (self._pipeline is not None and self._pipeline_key == key): return self._pipeline + tc = self.training_config + if rollout_mode == "parallel": if sampler_kind not in {"ode", "sde"}: - raise ValueError( - f"Unknown sampler_kind for WanGame validation: {sampler_kind!r}" - ) + raise ValueError("Unknown sampler_kind for WanGame " + f"validation: {sampler_kind!r}") - flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) + flow_shift = getattr(tc.pipeline_config, "flow_shift", None) from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import ( - WanGameActionImageToVideoPipeline, - ) + WanGameActionImageToVideoPipeline, ) kwargs: dict[str, Any] = { "inference_mode": True, "sampler_kind": sampler_kind, - "loaded_modules": {"transformer": transformer}, - "tp_size": self.training_args.tp_size, - "sp_size": self.training_args.sp_size, - "num_gpus": self.training_args.num_gpus, - "pin_cpu_memory": self.training_args.pin_cpu_memory, + "loaded_modules": { + "transformer": transformer, + }, + "tp_size": tc.distributed.tp_size, + "sp_size": tc.distributed.sp_size, + "num_gpus": tc.distributed.num_gpus, + "pin_cpu_memory": (tc.distributed.pin_cpu_memory), "dit_cpu_offload": True, } if flow_shift is not None: @@ -183,67 +202,70 @@ def _get_pipeline( if ode_solver is not None: kwargs["ode_solver"] = str(ode_solver) - self._pipeline = WanGameActionImageToVideoPipeline.from_pretrained( - self.training_args.model_path, + self._pipeline = (WanGameActionImageToVideoPipeline.from_pretrained( + tc.model_path, **kwargs, - ) + )) elif rollout_mode == "streaming": if sampler_kind not in {"ode", "sde"}: - raise ValueError( - f"Unknown sampler_kind for WanGame streaming validation: {sampler_kind!r}" - ) + raise ValueError("Unknown sampler_kind for WanGame " + "streaming validation: " + f"{sampler_kind!r}") - flow_shift = getattr(self.training_args.pipeline_config, "flow_shift", None) + flow_shift = getattr(tc.pipeline_config, "flow_shift", None) if flow_shift is None: - raise ValueError("pipeline_config.flow_shift must be set for WanGame validation") + raise ValueError("pipeline_config.flow_shift must be set " + "for WanGame validation") if sampler_kind == "sde": if ode_solver is not None: - raise ValueError( - "ode_solver is only valid when sampler_kind='ode', got " - f"ode_solver={ode_solver!r}." - ) + raise ValueError("ode_solver is only valid when " + "sampler_kind='ode', got " + f"ode_solver={ode_solver!r}.") from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, - ) + FlowMatchEulerDiscreteScheduler, ) - scheduler = FlowMatchEulerDiscreteScheduler(shift=float(flow_shift)) + scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) else: from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, - ) + FlowMatchEulerDiscreteScheduler, ) from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( - FlowUniPCMultistepScheduler, - ) - - ode_solver_norm = (str(ode_solver).strip().lower() - if ode_solver is not None else "unipc") - if ode_solver_norm in {"unipc", "unipc_multistep", "multistep"}: - scheduler = FlowUniPCMultistepScheduler(shift=float(flow_shift)) - elif ode_solver_norm in {"euler", "flowmatch", "flowmatch_euler"}: - scheduler = FlowMatchEulerDiscreteScheduler(shift=float(flow_shift)) + FlowUniPCMultistepScheduler, ) + + ode_solver_norm = (str(ode_solver).strip().lower() if ode_solver is not None else "unipc") + if ode_solver_norm in { + "unipc", + "unipc_multistep", + "multistep", + }: + scheduler = (FlowUniPCMultistepScheduler(shift=float(flow_shift))) + elif ode_solver_norm in { + "euler", + "flowmatch", + "flowmatch_euler", + }: + scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) else: - raise ValueError( - "Unknown ode_solver for WanGame streaming validation: " - f"{ode_solver!r} (expected 'unipc' or 'euler')." - ) + raise ValueError("Unknown ode_solver for WanGame " + "streaming validation: " + f"{ode_solver!r} (expected 'unipc'" + " or 'euler').") from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( - WanGameCausalDMDPipeline, - ) + WanGameCausalDMDPipeline, ) kwargs = { "inference_mode": True, - "flow_shift": float(flow_shift) if flow_shift is not None else None, + "flow_shift": (float(flow_shift) if flow_shift is not None else None), "sampler_kind": sampler_kind, "loaded_modules": { "transformer": transformer, "scheduler": scheduler, }, - "tp_size": self.training_args.tp_size, - "sp_size": self.training_args.sp_size, - "num_gpus": self.training_args.num_gpus, - "pin_cpu_memory": self.training_args.pin_cpu_memory, + "tp_size": tc.distributed.tp_size, + "sp_size": tc.distributed.sp_size, + "num_gpus": tc.distributed.num_gpus, + "pin_cpu_memory": (tc.distributed.pin_cpu_memory), "dit_cpu_offload": True, } if kwargs["flow_shift"] is None: @@ -251,15 +273,14 @@ def _get_pipeline( if ode_solver is not None: kwargs["ode_solver"] = str(ode_solver) - self._pipeline = WanGameCausalDMDPipeline.from_pretrained( - self.training_args.model_path, + self._pipeline = (WanGameCausalDMDPipeline.from_pretrained( + tc.model_path, **kwargs, - ) + )) else: - raise ValueError( - "Unknown rollout_mode for WanGame validation: " - f"{rollout_mode!r}. Expected 'parallel' or 'streaming'." - ) + raise ValueError("Unknown rollout_mode for WanGame " + f"validation: {rollout_mode!r}. Expected " + "'parallel' or 'streaming'.") self._pipeline_key = key return self._pipeline @@ -274,22 +295,20 @@ def _prepare_validation_batch( guidance_scale: float | None = None, num_frames: int | None = None, ) -> ForwardBatch: - training_args = self.training_args + tc = self.training_config sampling_param.prompt = validation_batch["prompt"] - sampling_param.height = training_args.num_height - sampling_param.width = training_args.num_width - sampling_param.image_path = validation_batch.get("image_path") or validation_batch.get("video_path") + sampling_param.height = tc.data.num_height + sampling_param.width = tc.data.num_width + sampling_param.image_path = (validation_batch.get("image_path") or validation_batch.get("video_path")) sampling_param.num_inference_steps = int(num_inference_steps) sampling_param.data_type = "video" if guidance_scale is not None: sampling_param.guidance_scale = float(guidance_scale) sampling_param.seed = self.seed - temporal_compression_factor = ( - training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - ) - default_num_frames = (training_args.num_latent_t - 1) * temporal_compression_factor + 1 + temporal_compression_factor = int(tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio) + default_num_frames = ((tc.data.num_latent_t - 1) * temporal_compression_factor + 1) if num_frames is not None: sampling_param.num_frames = int(num_frames) else: @@ -300,13 +319,12 @@ def _prepare_validation_batch( sampling_param.height // 8, sampling_param.width // 8, ] - n_tokens = latents_size[0] * latents_size[1] * latents_size[2] + n_tokens = (latents_size[0] * latents_size[1] * latents_size[2]) - sampling_timesteps_tensor = ( - torch.tensor([int(s) for s in sampling_timesteps], dtype=torch.long) - if sampling_timesteps is not None - else None - ) + sampling_timesteps_tensor = (torch.tensor( + [int(s) for s in sampling_timesteps], + dtype=torch.long, + ) if sampling_timesteps is not None else None) batch = ForwardBatch( **shallow_asdict(sampling_param), @@ -314,51 +332,53 @@ def _prepare_validation_batch( generator=self.validation_random_generator, n_tokens=n_tokens, eta=0.0, - VSA_sparsity=training_args.VSA_sparsity, + VSA_sparsity=tc.vsa.sparsity, timesteps=sampling_timesteps_tensor, sampling_timesteps=sampling_timesteps_tensor, ) - if "image" in validation_batch and validation_batch["image"] is not None: + if ("image" in validation_batch and validation_batch["image"] is not None): batch.pil_image = validation_batch["image"] - if "keyboard_cond" in validation_batch and validation_batch["keyboard_cond"] is not None: - keyboard_cond = torch.as_tensor(validation_batch["keyboard_cond"]).to( - dtype=torch.bfloat16 - ) - if keyboard_cond.ndim == 3 and keyboard_cond.shape[0] == 1: + if ("keyboard_cond" in validation_batch and validation_batch["keyboard_cond"] is not None): + keyboard_cond = torch.as_tensor(validation_batch["keyboard_cond"]).to(dtype=torch.bfloat16) + if (keyboard_cond.ndim == 3 and keyboard_cond.shape[0] == 1): keyboard_cond = keyboard_cond.squeeze(0) if keyboard_cond.ndim != 2: - raise ValueError( - "validation keyboard_cond must have shape (T, K) (or (1, T, K)), " - f"got {tuple(keyboard_cond.shape)}" - ) + raise ValueError("validation keyboard_cond must have " + "shape (T, K) (or (1, T, K)), " + f"got {tuple(keyboard_cond.shape)}") target_len = int(sampling_param.num_frames) if keyboard_cond.shape[0] > target_len: keyboard_cond = keyboard_cond[:target_len] elif keyboard_cond.shape[0] < target_len: pad = torch.zeros( - (target_len - keyboard_cond.shape[0], keyboard_cond.shape[1]), + ( + target_len - keyboard_cond.shape[0], + keyboard_cond.shape[1], + ), dtype=keyboard_cond.dtype, device=keyboard_cond.device, ) keyboard_cond = torch.cat([keyboard_cond, pad], dim=0) batch.keyboard_cond = keyboard_cond.unsqueeze(0) - if "mouse_cond" in validation_batch and validation_batch["mouse_cond"] is not None: + if ("mouse_cond" in validation_batch and validation_batch["mouse_cond"] is not None): mouse_cond = torch.as_tensor(validation_batch["mouse_cond"]).to(dtype=torch.bfloat16) - if mouse_cond.ndim == 3 and mouse_cond.shape[0] == 1: + if (mouse_cond.ndim == 3 and mouse_cond.shape[0] == 1): mouse_cond = mouse_cond.squeeze(0) if mouse_cond.ndim != 2: - raise ValueError( - "validation mouse_cond must have shape (T, 2) (or (1, T, 2)), " - f"got {tuple(mouse_cond.shape)}" - ) + raise ValueError("validation mouse_cond must have shape" + " (T, 2) (or (1, T, 2)), " + f"got {tuple(mouse_cond.shape)}") target_len = int(sampling_param.num_frames) if mouse_cond.shape[0] > target_len: mouse_cond = mouse_cond[:target_len] elif mouse_cond.shape[0] < target_len: pad = torch.zeros( - (target_len - mouse_cond.shape[0], mouse_cond.shape[1]), + ( + target_len - mouse_cond.shape[0], + mouse_cond.shape[1], + ), dtype=mouse_cond.dtype, device=mouse_cond.device, ) @@ -380,7 +400,7 @@ def _run_validation_for_steps( guidance_scale: float | None = None, num_frames: int | None = None, ) -> _ValidationStepResult: - training_args = self.training_args + tc = self.training_config pipeline = self._get_pipeline( transformer=transformer, rollout_mode=rollout_mode, @@ -392,6 +412,9 @@ def _run_validation_for_steps( dataset = ValidationDataset(dataset_file) dataloader = DataLoader(dataset, batch_size=None, num_workers=0) + # Build inference args once for this validation run. + inference_args = DistillLoaderArgs.for_inference(tc, model_path=tc.model_path) + videos: list[list[np.ndarray]] = [] captions: list[str] = [] @@ -405,11 +428,11 @@ def _run_validation_for_steps( num_frames=num_frames, ) - assert batch.prompt is not None and isinstance(batch.prompt, str) + assert (batch.prompt is not None and isinstance(batch.prompt, str)) captions.append(batch.prompt) with torch.no_grad(): - output_batch = pipeline.forward(batch, training_args) + output_batch = pipeline.forward(batch, inference_args) samples = output_batch.output.cpu() if self.rank_in_sp_group != 0: @@ -421,106 +444,119 @@ def _run_validation_for_steps( x = torchvision.utils.make_grid(x, nrow=6) x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) frames.append((x * 255).numpy().astype(np.uint8)) - frames = self._post_process_validation_frames(frames, batch) + frames = (self._post_process_validation_frames(frames, batch)) videos.append(frames) return _ValidationStepResult(videos=videos, captions=captions) - def log_validation(self, step: int, *, request: ValidationRequest | None = None) -> None: + def log_validation( + self, + step: int, + *, + request: ValidationRequest | None = None, + ) -> None: if request is None: - raise ValueError("WanGameValidator.log_validation requires a ValidationRequest") + raise ValueError("WanGameValidator.log_validation requires " + "a ValidationRequest") dataset_file = getattr(request, "dataset_file", None) if not dataset_file: - raise ValueError("ValidationRequest.dataset_file must be provided by the method") + raise ValueError("ValidationRequest.dataset_file must be " + "provided by the method") guidance_scale = getattr(request, "guidance_scale", None) validation_steps = getattr(request, "sampling_steps", None) if not validation_steps: - raise ValueError("ValidationRequest.sampling_steps must be provided by the method") - sampler_kind = getattr(request, "sampler_kind", None) or "ode" - rollout_mode_raw = getattr(request, "rollout_mode", None) or "parallel" + raise ValueError("ValidationRequest.sampling_steps must be " + "provided by the method") + sampler_kind = (getattr(request, "sampler_kind", None) or "ode") + rollout_mode_raw = (getattr(request, "rollout_mode", None) or "parallel") if not isinstance(rollout_mode_raw, str): - raise ValueError( - "ValidationRequest.rollout_mode must be a string when set, got " - f"{type(rollout_mode_raw).__name__}" - ) + raise ValueError("ValidationRequest.rollout_mode must be a " + "string when set, got " + f"{type(rollout_mode_raw).__name__}") rollout_mode = rollout_mode_raw.strip().lower() if rollout_mode not in {"parallel", "streaming"}: - raise ValueError( - "ValidationRequest.rollout_mode must be one of {parallel, streaming}, got " - f"{rollout_mode_raw!r}" - ) + raise ValueError("ValidationRequest.rollout_mode must be " + "one of {parallel, streaming}, got " + f"{rollout_mode_raw!r}") ode_solver = getattr(request, "ode_solver", None) num_frames_raw = getattr(request, "num_frames", None) if num_frames_raw is None: num_frames = None elif isinstance(num_frames_raw, bool): - raise ValueError("ValidationRequest.num_frames must be an int when set") + raise ValueError("ValidationRequest.num_frames must be an " + "int when set") elif isinstance(num_frames_raw, int): num_frames = int(num_frames_raw) else: - raise ValueError( - "ValidationRequest.num_frames must be an int when set, got " - f"{type(num_frames_raw).__name__}" - ) + raise ValueError("ValidationRequest.num_frames must be an " + "int when set, got " + f"{type(num_frames_raw).__name__}") if num_frames is not None and num_frames <= 0: - raise ValueError("ValidationRequest.num_frames must be > 0 when set") + raise ValueError("ValidationRequest.num_frames must be > 0 " + "when set") if rollout_mode == "streaming": - sampler_kind_norm = str(sampler_kind).strip().lower() + sampler_kind_norm = (str(sampler_kind).strip().lower()) if sampler_kind_norm not in {"ode", "sde"}: - raise ValueError( - "WanGame validation rollout_mode='streaming' requires " - "sampler_kind to be one of {'ode', 'sde'}, got " - f"{sampler_kind!r}." - ) - if sampler_kind_norm == "sde" and ode_solver is not None: - raise ValueError( - "WanGame validation rollout_mode='streaming' only supports " - f"ode_solver when sampler_kind='ode', got ode_solver={ode_solver!r}." - ) + raise ValueError("WanGame validation " + "rollout_mode='streaming' requires " + "sampler_kind to be one of " + "{'ode', 'sde'}, got " + f"{sampler_kind!r}.") + if (sampler_kind_norm == "sde" and ode_solver is not None): + raise ValueError("WanGame validation " + "rollout_mode='streaming' only " + "supports ode_solver when " + "sampler_kind='ode', got " + f"ode_solver={ode_solver!r}.") sampling_timesteps = getattr(request, "sampling_timesteps", None) if sampling_timesteps is not None: expected = int(len(sampling_timesteps)) for steps in validation_steps: if int(steps) != expected: - raise ValueError( - "validation_sampling_steps must match " - f"len(request.sampling_timesteps)={expected} when " - "sampling_timesteps is provided, got " - f"{validation_steps!r}." - ) + raise ValueError("validation_sampling_steps must " + "match " + "len(request.sampling_timesteps)=" + f"{expected} when " + "sampling_timesteps is provided, " + f"got {validation_steps!r}.") sample_handle = getattr(request, "sample_handle", None) if sample_handle is None: - raise ValueError("ValidationRequest.sample_handle must be provided by the method") + raise ValueError("ValidationRequest.sample_handle must be " + "provided by the method") transformer = sample_handle.require_module("transformer") was_training = bool(getattr(transformer, "training", False)) - training_args = self.training_args - output_dir = getattr(request, "output_dir", None) or training_args.output_dir + tc = self.training_config + output_dir = (getattr(request, "output_dir", None) or tc.checkpoint.output_dir) - old_inference_mode = training_args.inference_mode - old_dit_cpu_offload = training_args.dit_cpu_offload - old_mode = training_args.mode - old_dmd_denoising_steps = getattr(training_args.pipeline_config, "dmd_denoising_steps", None) + # For streaming SDE, we need to set + # dmd_denoising_steps on pipeline_config. + old_dmd_denoising_steps = getattr( + tc.pipeline_config, + "dmd_denoising_steps", + None, + ) try: - training_args.inference_mode = True - training_args.dit_cpu_offload = True - training_args.mode = ExecutionMode.INFERENCE transformer.eval() - num_sp_groups = self.world_group.world_size // self.sp_group.world_size + num_sp_groups = (self.world_group.world_size // self.sp_group.world_size) for num_inference_steps in validation_steps: - if rollout_mode == "streaming" and str(sampler_kind).strip().lower() == "sde": + if (rollout_mode == "streaming" and str(sampler_kind).strip().lower() == "sde"): if sampling_timesteps is not None: - training_args.pipeline_config.dmd_denoising_steps = list(sampling_timesteps) + tc.pipeline_config.dmd_denoising_steps = list(sampling_timesteps) else: - timesteps = np.linspace(1000, 0, int(num_inference_steps)) - training_args.pipeline_config.dmd_denoising_steps = [ - int(max(0, min(1000, round(t)))) for t in timesteps - ] + import numpy as np + + timesteps = np.linspace( + 1000, + 0, + int(num_inference_steps), + ) + tc.pipeline_config.dmd_denoising_steps = [int(max(0, min(1000, round(t)))) for t in timesteps] result = self._run_validation_for_steps( num_inference_steps, @@ -528,7 +564,7 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) transformer=transformer, rollout_mode=rollout_mode, sampler_kind=str(sampler_kind), - ode_solver=str(ode_solver) if ode_solver is not None else None, + ode_solver=(str(ode_solver) if ode_solver is not None else None), sampling_timesteps=sampling_timesteps, guidance_scale=guidance_scale, num_frames=num_frames, @@ -541,38 +577,50 @@ def log_validation(self, step: int, *, request: ValidationRequest | None = None) all_videos = list(result.videos) all_captions = list(result.captions) for sp_group_idx in range(1, num_sp_groups): - src_rank = sp_group_idx * self.sp_world_size - recv_videos = self.world_group.recv_object(src=src_rank) - recv_captions = self.world_group.recv_object(src=src_rank) + src_rank = (sp_group_idx * self.sp_world_size) + recv_videos = (self.world_group.recv_object(src=src_rank)) + recv_captions = (self.world_group.recv_object(src=src_rank)) all_videos.extend(recv_videos) all_captions.extend(recv_captions) os.makedirs(output_dir, exist_ok=True) video_filenames: list[str] = [] - sampling_param = self._get_sampling_param() + sampling_param = (self._get_sampling_param()) for i, video in enumerate(all_videos): filename = os.path.join( output_dir, - f"validation_step_{step}_inference_steps_{num_inference_steps}_video_{i}.mp4", + f"validation_step_{step}" + f"_inference_steps_" + f"{num_inference_steps}" + f"_video_{i}.mp4", + ) + imageio.mimsave( + filename, + video, + fps=sampling_param.fps, ) - imageio.mimsave(filename, video, fps=sampling_param.fps) video_filenames.append(filename) video_logs = [] - for filename, caption in zip(video_filenames, all_captions, strict=True): + for filename, caption in zip( + video_filenames, + all_captions, + strict=True, + ): video_artifact = self.tracker.video(filename, caption=caption) if video_artifact is not None: video_logs.append(video_artifact) if video_logs: - logs = {f"validation_videos_{num_inference_steps}_steps": video_logs} + logs = { + f"validation_videos_" + f"{num_inference_steps}" + f"_steps": video_logs + } self.tracker.log_artifacts(logs, step) else: self.world_group.send_object(result.videos, dst=0) self.world_group.send_object(result.captions, dst=0) finally: - training_args.inference_mode = old_inference_mode - training_args.dit_cpu_offload = old_dit_cpu_offload - training_args.mode = old_mode - training_args.pipeline_config.dmd_denoising_steps = old_dmd_denoising_steps + tc.pipeline_config.dmd_denoising_steps = (old_dmd_denoising_steps) if was_training: transformer.train() diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 912b58c94..f82a77335 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -21,43 +21,50 @@ def run_distillation_from_config( ) -> None: """YAML-only distillation entrypoint (schema v2).""" - from fastvideo.distributed import maybe_init_distributed_environment_and_model_parallel + from fastvideo.distributed import ( + maybe_init_distributed_environment_and_model_parallel, ) from fastvideo.distillation import DistillTrainer from fastvideo.distillation.utils.checkpoint import ( DistillCheckpointConfig, DistillCheckpointManager, ) - from fastvideo.distillation.dispatch import build_from_config - from fastvideo.distillation.utils.config import load_run_config + from fastvideo.distillation.dispatch import ( + build_from_config, ) + from fastvideo.distillation.utils.config import ( + load_run_config, ) cfg = load_run_config(config_path) - training_args = cfg.training_args + tc = cfg.training if resume_from_checkpoint is not None: - training_args.resume_from_checkpoint = str(resume_from_checkpoint) + tc.checkpoint.resume_from_checkpoint = str(resume_from_checkpoint) if override_output_dir is not None: - training_args.output_dir = str(override_output_dir) + tc.checkpoint.output_dir = str(override_output_dir) maybe_init_distributed_environment_and_model_parallel( - training_args.tp_size, - training_args.sp_size, + tc.distributed.tp_size, + tc.distributed.sp_size, ) _, method, dataloader, start_step = build_from_config(cfg) if dry_run: - logger.info("Dry-run: config parsed and build_from_config succeeded.") + logger.info("Dry-run: config parsed and " + "build_from_config succeeded.") return - trainer = DistillTrainer(training_args, config=cfg.raw) + trainer = DistillTrainer(tc, config=cfg.raw) - # Attach the exact YAML used for this run to the tracker (e.g., W&B Files). - # This helps reproducibility and makes runs easy to inspect later. - trainer.tracker.log_file(os.path.abspath(os.path.expanduser(config_path)), name="run.yaml") + # Attach the exact YAML used for this run to the + # tracker (e.g., W&B Files). + trainer.tracker.log_file( + os.path.abspath(os.path.expanduser(config_path)), + name="run.yaml", + ) ckpt_config = DistillCheckpointConfig( - save_steps=int(getattr(training_args, "training_state_checkpointing_steps", 0) or 0), - keep_last=int(getattr(training_args, "checkpoints_total_limit", 0) or 0), + save_steps=int(tc.checkpoint.training_state_checkpointing_steps or 0), + keep_last=int(tc.checkpoint.checkpoints_total_limit or 0), ) get_rng_generators = getattr(method, "get_rng_generators", None) @@ -68,13 +75,11 @@ def run_distillation_from_config( get_rng_generators = None checkpoint_manager = DistillCheckpointManager( - role_models=getattr(method, '_role_models', None) or {}, - optimizers=getattr(method, '_optimizer_dict', None) or {}, - lr_schedulers=getattr( - method, '_lr_scheduler_dict', None - ) or {}, + role_models=(getattr(method, '_role_models', None) or {}), + optimizers=(getattr(method, '_optimizer_dict', None) or {}), + lr_schedulers=(getattr(method, '_lr_scheduler_dict', None) or {}), dataloader=dataloader, - output_dir=training_args.output_dir, + output_dir=tc.checkpoint.output_dir, config=ckpt_config, get_rng_generators=get_rng_generators, ) @@ -82,7 +87,7 @@ def run_distillation_from_config( trainer.run( method, dataloader=dataloader, - max_steps=training_args.max_train_steps, + max_steps=tc.loop.max_train_steps, start_step=start_step, checkpoint_manager=checkpoint_manager, ) @@ -93,7 +98,10 @@ def main(args: Any) -> None: dry_run = bool(args.dry_run) resume_from_checkpoint = getattr(args, "resume_from_checkpoint", None) override_output_dir = getattr(args, "override_output_dir", None) - logger.info("Starting distillation from config=%s", config_path) + logger.info( + "Starting distillation from config=%s", + config_path, + ) run_distillation_from_config( config_path, dry_run=dry_run, @@ -105,36 +113,35 @@ def main(args: Any) -> None: if __name__ == "__main__": argv = sys.argv - # NOTE: do not use `FlexibleArgumentParser` here. - # It treats `--config` specially (loads and inlines CLI args from YAML), - # which conflicts with Phase 2 distillation where `--config` points to the - # distillation run YAML itself. parser = argparse.ArgumentParser() parser.add_argument( "--config", type=str, required=True, - help="Path to distillation YAML config (schema v2).", + help=("Path to distillation YAML config " + "(schema v2)."), ) parser.add_argument( "--dry-run", action="store_true", - help="Parse config and build runtime, but do not start training.", + help=("Parse config and build runtime, but do not " + "start training."), ) parser.add_argument( "--resume-from-checkpoint", type=str, default=None, - help=( - "Path to a checkpoint directory (checkpoint-), its 'dcp/' subdir, " - "or an output_dir containing checkpoints (auto-picks latest)." - ), + help=("Path to a checkpoint directory " + "(checkpoint-), its 'dcp/' subdir, " + "or an output_dir containing checkpoints " + "(auto-picks latest)."), ) parser.add_argument( "--override-output-dir", type=str, default=None, - help="Override training.output_dir from YAML (useful for repeated runs).", + help=("Override training.output_dir from YAML " + "(useful for repeated runs)."), ) args = parser.parse_args(argv[1:]) main(args) From 42fa1ba23ed349ecbb75c761b6edd82f02d6c63f Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 22:25:09 +0000 Subject: [PATCH 176/214] remove redundant loader_args --- fastvideo/distillation/models/wan/wan.py | 22 +--- .../distillation/models/wangame/wangame.py | 12 +- fastvideo/distillation/utils/loader_args.py | 59 ---------- fastvideo/distillation/utils/moduleloader.py | 107 +++++++++++++----- fastvideo/distillation/validators/wan.py | 10 +- fastvideo/distillation/validators/wangame.py | 6 +- 6 files changed, 94 insertions(+), 122 deletions(-) delete mode 100644 fastvideo/distillation/utils/loader_args.py diff --git a/fastvideo/distillation/models/wan/wan.py b/fastvideo/distillation/models/wan/wan.py index ab77764c9..48768edc4 100644 --- a/fastvideo/distillation/models/wan/wan.py +++ b/fastvideo/distillation/models/wan/wan.py @@ -88,7 +88,6 @@ def __init__( transformer = load_module_from_path( model_path=self._init_from, module_type="transformer", - loader_args=None, disable_custom_init_weights=( disable_custom_init_weights ), @@ -159,19 +158,10 @@ def init_preprocessors( ) -> None: self.training_config = training_config - from fastvideo.distillation.utils.loader_args import ( - DistillLoaderArgs, - ) - - loader_args = DistillLoaderArgs.from_training_config( - training_config, - model_path=training_config.model_path, - ) - self.vae = load_module_from_path( model_path=str(training_config.model_path), module_type="vae", - loader_args=loader_args, + training_config=training_config, ) self.world_group = get_world_group() @@ -546,8 +536,8 @@ def ensure_negative_conditioning(self) -> None: device = self.device dtype = self._get_training_dtype() - from fastvideo.distillation.utils.loader_args import ( - DistillLoaderArgs, + from fastvideo.distillation.utils.moduleloader import ( + make_inference_args, ) neg_embeds: torch.Tensor | None = None @@ -559,10 +549,8 @@ def ensure_negative_conditioning(self) -> None: ) negative_prompt = sampling_param.negative_prompt - inference_args = ( - DistillLoaderArgs.for_inference( - tc, model_path=tc.model_path - ) + inference_args = make_inference_args( + tc, model_path=tc.model_path ) prompt_pipeline = WanPipeline.from_pretrained( diff --git a/fastvideo/distillation/models/wangame/wangame.py b/fastvideo/distillation/models/wangame/wangame.py index 842d40522..b330e70e0 100644 --- a/fastvideo/distillation/models/wangame/wangame.py +++ b/fastvideo/distillation/models/wangame/wangame.py @@ -134,7 +134,6 @@ def _load_transformer( transformer = load_module_from_path( model_path=init_from, module_type="transformer", - loader_args=None, disable_custom_init_weights=( disable_custom_init_weights ), @@ -167,19 +166,10 @@ def init_preprocessors( """Load VAE, build dataloader, seed RNGs.""" self.training_config = training_config - from fastvideo.distillation.utils.loader_args import ( - DistillLoaderArgs, - ) - - loader_args = DistillLoaderArgs.from_training_config( - training_config, - model_path=training_config.model_path, - ) - self.vae = load_module_from_path( model_path=str(training_config.model_path), module_type="vae", - loader_args=loader_args, + training_config=training_config, ) self.world_group = get_world_group() diff --git a/fastvideo/distillation/utils/loader_args.py b/fastvideo/distillation/utils/loader_args.py deleted file mode 100644 index 0ccbb424f..000000000 --- a/fastvideo/distillation/utils/loader_args.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Minimal FastVideoArgs subclass for PipelineComponentLoader.""" - -from __future__ import annotations - -import dataclasses -from typing import TYPE_CHECKING - -from fastvideo.configs.pipelines.base import PipelineConfig -from fastvideo.fastvideo_args import ExecutionMode, FastVideoArgs - -if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( - DistillTrainingConfig, ) - - -@dataclasses.dataclass -class DistillLoaderArgs(FastVideoArgs): - """Minimal FastVideoArgs for PipelineComponentLoader.""" - - @classmethod - def from_training_config( - cls, - tc: DistillTrainingConfig, - *, - model_path: str, - ) -> DistillLoaderArgs: - return cls( - model_path=model_path, - mode=ExecutionMode.DISTILLATION, - inference_mode=False, - pipeline_config=(tc.pipeline_config or PipelineConfig()), - num_gpus=tc.distributed.num_gpus, - tp_size=tc.distributed.tp_size, - sp_size=tc.distributed.sp_size, - hsdp_replicate_dim=(tc.distributed.hsdp_replicate_dim), - hsdp_shard_dim=tc.distributed.hsdp_shard_dim, - pin_cpu_memory=tc.distributed.pin_cpu_memory, - dit_cpu_offload=False, - dit_layerwise_offload=False, - vae_cpu_offload=False, - text_encoder_cpu_offload=False, - image_encoder_cpu_offload=False, - use_fsdp_inference=False, - enable_torch_compile=False, - ) - - @classmethod - def for_inference( - cls, - tc: DistillTrainingConfig, - *, - model_path: str, - ) -> DistillLoaderArgs: - args = cls.from_training_config(tc, model_path=model_path) - args.inference_mode = True - args.mode = ExecutionMode.INFERENCE - args.dit_cpu_offload = True - return args diff --git a/fastvideo/distillation/utils/moduleloader.py b/fastvideo/distillation/utils/moduleloader.py index 2d6757653..e2b731b96 100644 --- a/fastvideo/distillation/utils/moduleloader.py +++ b/fastvideo/distillation/utils/moduleloader.py @@ -3,10 +3,12 @@ from __future__ import annotations import os -from typing import Any +from typing import Any, TYPE_CHECKING import torch +from fastvideo.configs.pipelines.base import PipelineConfig +from fastvideo.fastvideo_args import ExecutionMode, TrainingArgs from fastvideo.models.loader.component_loader import ( PipelineComponentLoader, ) from fastvideo.utils import ( @@ -14,29 +16,80 @@ verify_model_config_and_directory, ) +if TYPE_CHECKING: + from fastvideo.distillation.utils.distill_config import ( + DistillTrainingConfig, ) + + +# ------------------------------------------------------------------ +# TrainingArgs builders (only place that creates FastVideoArgs) +# ------------------------------------------------------------------ + + +def _make_training_args( + tc: DistillTrainingConfig, + *, + model_path: str, +) -> TrainingArgs: + """Build a TrainingArgs for PipelineComponentLoader.""" + return TrainingArgs( + model_path=model_path, + mode=ExecutionMode.DISTILLATION, + inference_mode=False, + pipeline_config=(tc.pipeline_config or PipelineConfig()), + num_gpus=tc.distributed.num_gpus, + tp_size=tc.distributed.tp_size, + sp_size=tc.distributed.sp_size, + hsdp_replicate_dim=tc.distributed.hsdp_replicate_dim, + hsdp_shard_dim=tc.distributed.hsdp_shard_dim, + pin_cpu_memory=tc.distributed.pin_cpu_memory, + dit_cpu_offload=False, + dit_layerwise_offload=False, + vae_cpu_offload=False, + text_encoder_cpu_offload=False, + image_encoder_cpu_offload=False, + use_fsdp_inference=False, + enable_torch_compile=False, + ) + + +def make_inference_args( + tc: DistillTrainingConfig, + *, + model_path: str, +) -> TrainingArgs: + """Build a TrainingArgs for inference (validation / pipelines).""" + args = _make_training_args(tc, model_path=model_path) + args.inference_mode = True + args.mode = ExecutionMode.INFERENCE + args.dit_cpu_offload = True + return args + + +# ------------------------------------------------------------------ +# Module loading +# ------------------------------------------------------------------ + def load_module_from_path( *, model_path: str, module_type: str, - loader_args: Any = None, + training_config: DistillTrainingConfig | None = None, disable_custom_init_weights: bool = False, override_transformer_cls_name: str | None = None, - # Legacy alias kept so callers that still pass - # ``training_args=`` don't break during migration. - training_args: Any = None, ) -> torch.nn.Module: """Load a single pipeline component module. - *loader_args* should be a ``DistillLoaderArgs`` or - ``FastVideoArgs``-like object for the - ``PipelineComponentLoader``. When ``None`` a lightweight - stand-in is used. + Accepts a ``DistillTrainingConfig`` and internally builds the + ``TrainingArgs`` needed by ``PipelineComponentLoader``. """ - - # Support the legacy ``training_args`` kwarg. - if loader_args is None and training_args is not None: - loader_args = training_args + if training_config is not None: + fastvideo_args: Any = _make_training_args( + training_config, model_path=model_path) + else: + from types import SimpleNamespace + fastvideo_args = SimpleNamespace() local_model_path = maybe_download_model(model_path) config = verify_model_config_and_directory(local_model_path) @@ -53,41 +106,41 @@ def load_module_from_path( transformers_or_diffusers, _architecture = module_info component_path = os.path.join(local_model_path, module_type) - if loader_args is None: - from types import SimpleNamespace - - loader_args = SimpleNamespace() - old_override: str | None = None if override_transformer_cls_name is not None: old_override = getattr( - loader_args, + fastvideo_args, "override_transformer_cls_name", None, ) - loader_args.override_transformer_cls_name = str(override_transformer_cls_name) + fastvideo_args.override_transformer_cls_name = str( + override_transformer_cls_name) if disable_custom_init_weights: - loader_args._loading_teacher_critic_model = True + fastvideo_args._loading_teacher_critic_model = True try: module = PipelineComponentLoader.load_module( module_name=module_type, component_model_path=component_path, transformers_or_diffusers=(transformers_or_diffusers), - fastvideo_args=loader_args, + fastvideo_args=fastvideo_args, ) finally: - if disable_custom_init_weights and hasattr(loader_args, "_loading_teacher_critic_model"): - del loader_args._loading_teacher_critic_model + if disable_custom_init_weights and hasattr( + fastvideo_args, + "_loading_teacher_critic_model"): + del fastvideo_args._loading_teacher_critic_model if override_transformer_cls_name is not None: if old_override is None: if hasattr( - loader_args, + fastvideo_args, "override_transformer_cls_name", ): - loader_args.override_transformer_cls_name = (None) + fastvideo_args.override_transformer_cls_name = ( + None) else: - loader_args.override_transformer_cls_name = (old_override) + fastvideo_args.override_transformer_cls_name = ( + old_override) if not isinstance(module, torch.nn.Module): raise TypeError(f"Loaded {module_type!r} is not a " diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/distillation/validators/wan.py index 6984f39ec..dcdbd4765 100644 --- a/fastvideo/distillation/validators/wan.py +++ b/fastvideo/distillation/validators/wan.py @@ -37,8 +37,8 @@ from fastvideo.distributed import get_sp_group, get_world_group from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch -from fastvideo.distillation.utils.loader_args import ( - DistillLoaderArgs, ) +from fastvideo.distillation.utils.moduleloader import ( + make_inference_args, ) from fastvideo.distillation.validators.base import ( ValidationRequest, ) from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline @@ -169,9 +169,9 @@ def _prepare_validation_batch( dtype=torch.long, ) if sampling_timesteps is not None else None) - # Build DistillLoaderArgs for inference to pass + # Build TrainingArgs for inference to pass # to pipeline.forward(). - inference_args = DistillLoaderArgs.for_inference(tc, model_path=tc.model_path) + inference_args = make_inference_args(tc, model_path=tc.model_path) batch = ForwardBatch( **shallow_asdict(sampling_param), @@ -210,7 +210,7 @@ def _run_validation_for_steps( dataloader = DataLoader(dataset, batch_size=None, num_workers=0) # Build inference args once for this validation run. - inference_args = DistillLoaderArgs.for_inference(tc, model_path=tc.model_path) + inference_args = make_inference_args(tc, model_path=tc.model_path) videos: list[list[np.ndarray]] = [] captions: list[str] = [] diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/distillation/validators/wangame.py index 7998a1e1c..c87228b48 100644 --- a/fastvideo/distillation/validators/wangame.py +++ b/fastvideo/distillation/validators/wangame.py @@ -38,8 +38,8 @@ from fastvideo.distributed import get_sp_group, get_world_group from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch -from fastvideo.distillation.utils.loader_args import ( - DistillLoaderArgs, ) +from fastvideo.distillation.utils.moduleloader import ( + make_inference_args, ) from fastvideo.distillation.validators.base import ( ValidationRequest, ) from fastvideo.training.trackers import DummyTracker @@ -413,7 +413,7 @@ def _run_validation_for_steps( dataloader = DataLoader(dataset, batch_size=None, num_workers=0) # Build inference args once for this validation run. - inference_args = DistillLoaderArgs.for_inference(tc, model_path=tc.model_path) + inference_args = make_inference_args(tc, model_path=tc.model_path) videos: list[list[np.ndarray]] = [] captions: list[str] = [] From b6af8ebe7db97b802767ae5af0e3a9f8b83a9134 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 22:40:50 +0000 Subject: [PATCH 177/214] distill->train --- fastvideo/{distillation => train}/.style.yapf | 0 fastvideo/{distillation => train}/__init__.py | 0 fastvideo/{distillation => train}/dispatch.py | 0 fastvideo/{distillation => train}/doc/README.md | 0 fastvideo/{distillation => train}/doc/RFC-v1-en.md | 0 fastvideo/{distillation => train}/doc/RFC-v1.md | 0 fastvideo/{distillation => train}/doc/__init__.md | 0 fastvideo/{distillation => train}/doc/dispatch.md | 0 fastvideo/{distillation => train}/doc/methods/__init__.md | 0 fastvideo/{distillation => train}/doc/methods/base.md | 0 .../doc/methods/consistency_model/__init__.md | 0 .../doc/methods/distribution_matching/__init__.md | 0 .../doc/methods/distribution_matching/dmd2.md | 0 .../{distillation => train}/doc/methods/fine_tuning/__init__.md | 0 .../{distillation => train}/doc/methods/fine_tuning/dfsft.md | 0 .../{distillation => train}/doc/methods/fine_tuning/finetune.md | 0 .../doc/methods/knowledge_distillation/__init__.md | 0 fastvideo/{distillation => train}/doc/models/__init__.md | 0 fastvideo/{distillation => train}/doc/models/base.md | 0 fastvideo/{distillation => train}/doc/models/wan.md | 0 fastvideo/{distillation => train}/doc/models/wangame.md | 0 fastvideo/{distillation => train}/doc/roles.md | 0 fastvideo/{distillation => train}/doc/trainer.md | 0 fastvideo/{distillation => train}/doc/utils/__init__.md | 0 fastvideo/{distillation => train}/doc/utils/checkpoint.md | 0 fastvideo/{distillation => train}/doc/utils/config.md | 0 fastvideo/{distillation => train}/doc/utils/dataloader.md | 0 fastvideo/{distillation => train}/doc/utils/module_state.md | 0 fastvideo/{distillation => train}/doc/utils/moduleloader.md | 0 fastvideo/{distillation => train}/doc/utils/tracking.md | 0 fastvideo/{distillation => train}/doc/validators/__init__.md | 0 fastvideo/{distillation => train}/doc/validators/base.md | 0 fastvideo/{distillation => train}/doc/validators/wan.md | 0 fastvideo/{distillation => train}/doc/validators/wangame.md | 0 fastvideo/{distillation => train}/methods/__init__.py | 0 fastvideo/{distillation => train}/methods/base.py | 0 .../{distillation => train}/methods/consistency_model/__init__.py | 0 .../methods/distribution_matching/__init__.py | 0 .../{distillation => train}/methods/distribution_matching/dmd2.py | 0 .../methods/distribution_matching/self_forcing.py | 0 fastvideo/{distillation => train}/methods/fine_tuning/__init__.py | 0 fastvideo/{distillation => train}/methods/fine_tuning/dfsft.py | 0 fastvideo/{distillation => train}/methods/fine_tuning/finetune.py | 0 .../methods/knowledge_distillation/__init__.py | 0 fastvideo/{distillation => train}/models/__init__.py | 0 fastvideo/{distillation => train}/models/base.py | 0 fastvideo/{distillation => train}/models/wan/__init__.py | 0 fastvideo/{distillation => train}/models/wan/wan.py | 0 fastvideo/{distillation => train}/models/wangame/__init__.py | 0 fastvideo/{distillation => train}/models/wangame/wangame.py | 0 .../{distillation => train}/models/wangame/wangame_causal.py | 0 fastvideo/{distillation => train}/trainer.py | 0 fastvideo/{distillation => train}/utils/__init__.py | 0 fastvideo/{distillation => train}/utils/checkpoint.py | 0 fastvideo/{distillation => train}/utils/config.py | 0 fastvideo/{distillation => train}/utils/dataloader.py | 0 fastvideo/{distillation => train}/utils/instantiate.py | 0 fastvideo/{distillation => train}/utils/module_state.py | 0 fastvideo/{distillation => train}/utils/moduleloader.py | 0 fastvideo/{distillation => train}/utils/optimizer.py | 0 fastvideo/{distillation => train}/utils/tracking.py | 0 .../utils/distill_config.py => train/utils/training_config.py} | 0 fastvideo/{distillation => train}/utils/validation.py | 0 fastvideo/{distillation => train}/validators/__init__.py | 0 fastvideo/{distillation => train}/validators/base.py | 0 fastvideo/{distillation => train}/validators/wan.py | 0 fastvideo/{distillation => train}/validators/wangame.py | 0 67 files changed, 0 insertions(+), 0 deletions(-) rename fastvideo/{distillation => train}/.style.yapf (100%) rename fastvideo/{distillation => train}/__init__.py (100%) rename fastvideo/{distillation => train}/dispatch.py (100%) rename fastvideo/{distillation => train}/doc/README.md (100%) rename fastvideo/{distillation => train}/doc/RFC-v1-en.md (100%) rename fastvideo/{distillation => train}/doc/RFC-v1.md (100%) rename fastvideo/{distillation => train}/doc/__init__.md (100%) rename fastvideo/{distillation => train}/doc/dispatch.md (100%) rename fastvideo/{distillation => train}/doc/methods/__init__.md (100%) rename fastvideo/{distillation => train}/doc/methods/base.md (100%) rename fastvideo/{distillation => train}/doc/methods/consistency_model/__init__.md (100%) rename fastvideo/{distillation => train}/doc/methods/distribution_matching/__init__.md (100%) rename fastvideo/{distillation => train}/doc/methods/distribution_matching/dmd2.md (100%) rename fastvideo/{distillation => train}/doc/methods/fine_tuning/__init__.md (100%) rename fastvideo/{distillation => train}/doc/methods/fine_tuning/dfsft.md (100%) rename fastvideo/{distillation => train}/doc/methods/fine_tuning/finetune.md (100%) rename fastvideo/{distillation => train}/doc/methods/knowledge_distillation/__init__.md (100%) rename fastvideo/{distillation => train}/doc/models/__init__.md (100%) rename fastvideo/{distillation => train}/doc/models/base.md (100%) rename fastvideo/{distillation => train}/doc/models/wan.md (100%) rename fastvideo/{distillation => train}/doc/models/wangame.md (100%) rename fastvideo/{distillation => train}/doc/roles.md (100%) rename fastvideo/{distillation => train}/doc/trainer.md (100%) rename fastvideo/{distillation => train}/doc/utils/__init__.md (100%) rename fastvideo/{distillation => train}/doc/utils/checkpoint.md (100%) rename fastvideo/{distillation => train}/doc/utils/config.md (100%) rename fastvideo/{distillation => train}/doc/utils/dataloader.md (100%) rename fastvideo/{distillation => train}/doc/utils/module_state.md (100%) rename fastvideo/{distillation => train}/doc/utils/moduleloader.md (100%) rename fastvideo/{distillation => train}/doc/utils/tracking.md (100%) rename fastvideo/{distillation => train}/doc/validators/__init__.md (100%) rename fastvideo/{distillation => train}/doc/validators/base.md (100%) rename fastvideo/{distillation => train}/doc/validators/wan.md (100%) rename fastvideo/{distillation => train}/doc/validators/wangame.md (100%) rename fastvideo/{distillation => train}/methods/__init__.py (100%) rename fastvideo/{distillation => train}/methods/base.py (100%) rename fastvideo/{distillation => train}/methods/consistency_model/__init__.py (100%) rename fastvideo/{distillation => train}/methods/distribution_matching/__init__.py (100%) rename fastvideo/{distillation => train}/methods/distribution_matching/dmd2.py (100%) rename fastvideo/{distillation => train}/methods/distribution_matching/self_forcing.py (100%) rename fastvideo/{distillation => train}/methods/fine_tuning/__init__.py (100%) rename fastvideo/{distillation => train}/methods/fine_tuning/dfsft.py (100%) rename fastvideo/{distillation => train}/methods/fine_tuning/finetune.py (100%) rename fastvideo/{distillation => train}/methods/knowledge_distillation/__init__.py (100%) rename fastvideo/{distillation => train}/models/__init__.py (100%) rename fastvideo/{distillation => train}/models/base.py (100%) rename fastvideo/{distillation => train}/models/wan/__init__.py (100%) rename fastvideo/{distillation => train}/models/wan/wan.py (100%) rename fastvideo/{distillation => train}/models/wangame/__init__.py (100%) rename fastvideo/{distillation => train}/models/wangame/wangame.py (100%) rename fastvideo/{distillation => train}/models/wangame/wangame_causal.py (100%) rename fastvideo/{distillation => train}/trainer.py (100%) rename fastvideo/{distillation => train}/utils/__init__.py (100%) rename fastvideo/{distillation => train}/utils/checkpoint.py (100%) rename fastvideo/{distillation => train}/utils/config.py (100%) rename fastvideo/{distillation => train}/utils/dataloader.py (100%) rename fastvideo/{distillation => train}/utils/instantiate.py (100%) rename fastvideo/{distillation => train}/utils/module_state.py (100%) rename fastvideo/{distillation => train}/utils/moduleloader.py (100%) rename fastvideo/{distillation => train}/utils/optimizer.py (100%) rename fastvideo/{distillation => train}/utils/tracking.py (100%) rename fastvideo/{distillation/utils/distill_config.py => train/utils/training_config.py} (100%) rename fastvideo/{distillation => train}/utils/validation.py (100%) rename fastvideo/{distillation => train}/validators/__init__.py (100%) rename fastvideo/{distillation => train}/validators/base.py (100%) rename fastvideo/{distillation => train}/validators/wan.py (100%) rename fastvideo/{distillation => train}/validators/wangame.py (100%) diff --git a/fastvideo/distillation/.style.yapf b/fastvideo/train/.style.yapf similarity index 100% rename from fastvideo/distillation/.style.yapf rename to fastvideo/train/.style.yapf diff --git a/fastvideo/distillation/__init__.py b/fastvideo/train/__init__.py similarity index 100% rename from fastvideo/distillation/__init__.py rename to fastvideo/train/__init__.py diff --git a/fastvideo/distillation/dispatch.py b/fastvideo/train/dispatch.py similarity index 100% rename from fastvideo/distillation/dispatch.py rename to fastvideo/train/dispatch.py diff --git a/fastvideo/distillation/doc/README.md b/fastvideo/train/doc/README.md similarity index 100% rename from fastvideo/distillation/doc/README.md rename to fastvideo/train/doc/README.md diff --git a/fastvideo/distillation/doc/RFC-v1-en.md b/fastvideo/train/doc/RFC-v1-en.md similarity index 100% rename from fastvideo/distillation/doc/RFC-v1-en.md rename to fastvideo/train/doc/RFC-v1-en.md diff --git a/fastvideo/distillation/doc/RFC-v1.md b/fastvideo/train/doc/RFC-v1.md similarity index 100% rename from fastvideo/distillation/doc/RFC-v1.md rename to fastvideo/train/doc/RFC-v1.md diff --git a/fastvideo/distillation/doc/__init__.md b/fastvideo/train/doc/__init__.md similarity index 100% rename from fastvideo/distillation/doc/__init__.md rename to fastvideo/train/doc/__init__.md diff --git a/fastvideo/distillation/doc/dispatch.md b/fastvideo/train/doc/dispatch.md similarity index 100% rename from fastvideo/distillation/doc/dispatch.md rename to fastvideo/train/doc/dispatch.md diff --git a/fastvideo/distillation/doc/methods/__init__.md b/fastvideo/train/doc/methods/__init__.md similarity index 100% rename from fastvideo/distillation/doc/methods/__init__.md rename to fastvideo/train/doc/methods/__init__.md diff --git a/fastvideo/distillation/doc/methods/base.md b/fastvideo/train/doc/methods/base.md similarity index 100% rename from fastvideo/distillation/doc/methods/base.md rename to fastvideo/train/doc/methods/base.md diff --git a/fastvideo/distillation/doc/methods/consistency_model/__init__.md b/fastvideo/train/doc/methods/consistency_model/__init__.md similarity index 100% rename from fastvideo/distillation/doc/methods/consistency_model/__init__.md rename to fastvideo/train/doc/methods/consistency_model/__init__.md diff --git a/fastvideo/distillation/doc/methods/distribution_matching/__init__.md b/fastvideo/train/doc/methods/distribution_matching/__init__.md similarity index 100% rename from fastvideo/distillation/doc/methods/distribution_matching/__init__.md rename to fastvideo/train/doc/methods/distribution_matching/__init__.md diff --git a/fastvideo/distillation/doc/methods/distribution_matching/dmd2.md b/fastvideo/train/doc/methods/distribution_matching/dmd2.md similarity index 100% rename from fastvideo/distillation/doc/methods/distribution_matching/dmd2.md rename to fastvideo/train/doc/methods/distribution_matching/dmd2.md diff --git a/fastvideo/distillation/doc/methods/fine_tuning/__init__.md b/fastvideo/train/doc/methods/fine_tuning/__init__.md similarity index 100% rename from fastvideo/distillation/doc/methods/fine_tuning/__init__.md rename to fastvideo/train/doc/methods/fine_tuning/__init__.md diff --git a/fastvideo/distillation/doc/methods/fine_tuning/dfsft.md b/fastvideo/train/doc/methods/fine_tuning/dfsft.md similarity index 100% rename from fastvideo/distillation/doc/methods/fine_tuning/dfsft.md rename to fastvideo/train/doc/methods/fine_tuning/dfsft.md diff --git a/fastvideo/distillation/doc/methods/fine_tuning/finetune.md b/fastvideo/train/doc/methods/fine_tuning/finetune.md similarity index 100% rename from fastvideo/distillation/doc/methods/fine_tuning/finetune.md rename to fastvideo/train/doc/methods/fine_tuning/finetune.md diff --git a/fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md b/fastvideo/train/doc/methods/knowledge_distillation/__init__.md similarity index 100% rename from fastvideo/distillation/doc/methods/knowledge_distillation/__init__.md rename to fastvideo/train/doc/methods/knowledge_distillation/__init__.md diff --git a/fastvideo/distillation/doc/models/__init__.md b/fastvideo/train/doc/models/__init__.md similarity index 100% rename from fastvideo/distillation/doc/models/__init__.md rename to fastvideo/train/doc/models/__init__.md diff --git a/fastvideo/distillation/doc/models/base.md b/fastvideo/train/doc/models/base.md similarity index 100% rename from fastvideo/distillation/doc/models/base.md rename to fastvideo/train/doc/models/base.md diff --git a/fastvideo/distillation/doc/models/wan.md b/fastvideo/train/doc/models/wan.md similarity index 100% rename from fastvideo/distillation/doc/models/wan.md rename to fastvideo/train/doc/models/wan.md diff --git a/fastvideo/distillation/doc/models/wangame.md b/fastvideo/train/doc/models/wangame.md similarity index 100% rename from fastvideo/distillation/doc/models/wangame.md rename to fastvideo/train/doc/models/wangame.md diff --git a/fastvideo/distillation/doc/roles.md b/fastvideo/train/doc/roles.md similarity index 100% rename from fastvideo/distillation/doc/roles.md rename to fastvideo/train/doc/roles.md diff --git a/fastvideo/distillation/doc/trainer.md b/fastvideo/train/doc/trainer.md similarity index 100% rename from fastvideo/distillation/doc/trainer.md rename to fastvideo/train/doc/trainer.md diff --git a/fastvideo/distillation/doc/utils/__init__.md b/fastvideo/train/doc/utils/__init__.md similarity index 100% rename from fastvideo/distillation/doc/utils/__init__.md rename to fastvideo/train/doc/utils/__init__.md diff --git a/fastvideo/distillation/doc/utils/checkpoint.md b/fastvideo/train/doc/utils/checkpoint.md similarity index 100% rename from fastvideo/distillation/doc/utils/checkpoint.md rename to fastvideo/train/doc/utils/checkpoint.md diff --git a/fastvideo/distillation/doc/utils/config.md b/fastvideo/train/doc/utils/config.md similarity index 100% rename from fastvideo/distillation/doc/utils/config.md rename to fastvideo/train/doc/utils/config.md diff --git a/fastvideo/distillation/doc/utils/dataloader.md b/fastvideo/train/doc/utils/dataloader.md similarity index 100% rename from fastvideo/distillation/doc/utils/dataloader.md rename to fastvideo/train/doc/utils/dataloader.md diff --git a/fastvideo/distillation/doc/utils/module_state.md b/fastvideo/train/doc/utils/module_state.md similarity index 100% rename from fastvideo/distillation/doc/utils/module_state.md rename to fastvideo/train/doc/utils/module_state.md diff --git a/fastvideo/distillation/doc/utils/moduleloader.md b/fastvideo/train/doc/utils/moduleloader.md similarity index 100% rename from fastvideo/distillation/doc/utils/moduleloader.md rename to fastvideo/train/doc/utils/moduleloader.md diff --git a/fastvideo/distillation/doc/utils/tracking.md b/fastvideo/train/doc/utils/tracking.md similarity index 100% rename from fastvideo/distillation/doc/utils/tracking.md rename to fastvideo/train/doc/utils/tracking.md diff --git a/fastvideo/distillation/doc/validators/__init__.md b/fastvideo/train/doc/validators/__init__.md similarity index 100% rename from fastvideo/distillation/doc/validators/__init__.md rename to fastvideo/train/doc/validators/__init__.md diff --git a/fastvideo/distillation/doc/validators/base.md b/fastvideo/train/doc/validators/base.md similarity index 100% rename from fastvideo/distillation/doc/validators/base.md rename to fastvideo/train/doc/validators/base.md diff --git a/fastvideo/distillation/doc/validators/wan.md b/fastvideo/train/doc/validators/wan.md similarity index 100% rename from fastvideo/distillation/doc/validators/wan.md rename to fastvideo/train/doc/validators/wan.md diff --git a/fastvideo/distillation/doc/validators/wangame.md b/fastvideo/train/doc/validators/wangame.md similarity index 100% rename from fastvideo/distillation/doc/validators/wangame.md rename to fastvideo/train/doc/validators/wangame.md diff --git a/fastvideo/distillation/methods/__init__.py b/fastvideo/train/methods/__init__.py similarity index 100% rename from fastvideo/distillation/methods/__init__.py rename to fastvideo/train/methods/__init__.py diff --git a/fastvideo/distillation/methods/base.py b/fastvideo/train/methods/base.py similarity index 100% rename from fastvideo/distillation/methods/base.py rename to fastvideo/train/methods/base.py diff --git a/fastvideo/distillation/methods/consistency_model/__init__.py b/fastvideo/train/methods/consistency_model/__init__.py similarity index 100% rename from fastvideo/distillation/methods/consistency_model/__init__.py rename to fastvideo/train/methods/consistency_model/__init__.py diff --git a/fastvideo/distillation/methods/distribution_matching/__init__.py b/fastvideo/train/methods/distribution_matching/__init__.py similarity index 100% rename from fastvideo/distillation/methods/distribution_matching/__init__.py rename to fastvideo/train/methods/distribution_matching/__init__.py diff --git a/fastvideo/distillation/methods/distribution_matching/dmd2.py b/fastvideo/train/methods/distribution_matching/dmd2.py similarity index 100% rename from fastvideo/distillation/methods/distribution_matching/dmd2.py rename to fastvideo/train/methods/distribution_matching/dmd2.py diff --git a/fastvideo/distillation/methods/distribution_matching/self_forcing.py b/fastvideo/train/methods/distribution_matching/self_forcing.py similarity index 100% rename from fastvideo/distillation/methods/distribution_matching/self_forcing.py rename to fastvideo/train/methods/distribution_matching/self_forcing.py diff --git a/fastvideo/distillation/methods/fine_tuning/__init__.py b/fastvideo/train/methods/fine_tuning/__init__.py similarity index 100% rename from fastvideo/distillation/methods/fine_tuning/__init__.py rename to fastvideo/train/methods/fine_tuning/__init__.py diff --git a/fastvideo/distillation/methods/fine_tuning/dfsft.py b/fastvideo/train/methods/fine_tuning/dfsft.py similarity index 100% rename from fastvideo/distillation/methods/fine_tuning/dfsft.py rename to fastvideo/train/methods/fine_tuning/dfsft.py diff --git a/fastvideo/distillation/methods/fine_tuning/finetune.py b/fastvideo/train/methods/fine_tuning/finetune.py similarity index 100% rename from fastvideo/distillation/methods/fine_tuning/finetune.py rename to fastvideo/train/methods/fine_tuning/finetune.py diff --git a/fastvideo/distillation/methods/knowledge_distillation/__init__.py b/fastvideo/train/methods/knowledge_distillation/__init__.py similarity index 100% rename from fastvideo/distillation/methods/knowledge_distillation/__init__.py rename to fastvideo/train/methods/knowledge_distillation/__init__.py diff --git a/fastvideo/distillation/models/__init__.py b/fastvideo/train/models/__init__.py similarity index 100% rename from fastvideo/distillation/models/__init__.py rename to fastvideo/train/models/__init__.py diff --git a/fastvideo/distillation/models/base.py b/fastvideo/train/models/base.py similarity index 100% rename from fastvideo/distillation/models/base.py rename to fastvideo/train/models/base.py diff --git a/fastvideo/distillation/models/wan/__init__.py b/fastvideo/train/models/wan/__init__.py similarity index 100% rename from fastvideo/distillation/models/wan/__init__.py rename to fastvideo/train/models/wan/__init__.py diff --git a/fastvideo/distillation/models/wan/wan.py b/fastvideo/train/models/wan/wan.py similarity index 100% rename from fastvideo/distillation/models/wan/wan.py rename to fastvideo/train/models/wan/wan.py diff --git a/fastvideo/distillation/models/wangame/__init__.py b/fastvideo/train/models/wangame/__init__.py similarity index 100% rename from fastvideo/distillation/models/wangame/__init__.py rename to fastvideo/train/models/wangame/__init__.py diff --git a/fastvideo/distillation/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py similarity index 100% rename from fastvideo/distillation/models/wangame/wangame.py rename to fastvideo/train/models/wangame/wangame.py diff --git a/fastvideo/distillation/models/wangame/wangame_causal.py b/fastvideo/train/models/wangame/wangame_causal.py similarity index 100% rename from fastvideo/distillation/models/wangame/wangame_causal.py rename to fastvideo/train/models/wangame/wangame_causal.py diff --git a/fastvideo/distillation/trainer.py b/fastvideo/train/trainer.py similarity index 100% rename from fastvideo/distillation/trainer.py rename to fastvideo/train/trainer.py diff --git a/fastvideo/distillation/utils/__init__.py b/fastvideo/train/utils/__init__.py similarity index 100% rename from fastvideo/distillation/utils/__init__.py rename to fastvideo/train/utils/__init__.py diff --git a/fastvideo/distillation/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py similarity index 100% rename from fastvideo/distillation/utils/checkpoint.py rename to fastvideo/train/utils/checkpoint.py diff --git a/fastvideo/distillation/utils/config.py b/fastvideo/train/utils/config.py similarity index 100% rename from fastvideo/distillation/utils/config.py rename to fastvideo/train/utils/config.py diff --git a/fastvideo/distillation/utils/dataloader.py b/fastvideo/train/utils/dataloader.py similarity index 100% rename from fastvideo/distillation/utils/dataloader.py rename to fastvideo/train/utils/dataloader.py diff --git a/fastvideo/distillation/utils/instantiate.py b/fastvideo/train/utils/instantiate.py similarity index 100% rename from fastvideo/distillation/utils/instantiate.py rename to fastvideo/train/utils/instantiate.py diff --git a/fastvideo/distillation/utils/module_state.py b/fastvideo/train/utils/module_state.py similarity index 100% rename from fastvideo/distillation/utils/module_state.py rename to fastvideo/train/utils/module_state.py diff --git a/fastvideo/distillation/utils/moduleloader.py b/fastvideo/train/utils/moduleloader.py similarity index 100% rename from fastvideo/distillation/utils/moduleloader.py rename to fastvideo/train/utils/moduleloader.py diff --git a/fastvideo/distillation/utils/optimizer.py b/fastvideo/train/utils/optimizer.py similarity index 100% rename from fastvideo/distillation/utils/optimizer.py rename to fastvideo/train/utils/optimizer.py diff --git a/fastvideo/distillation/utils/tracking.py b/fastvideo/train/utils/tracking.py similarity index 100% rename from fastvideo/distillation/utils/tracking.py rename to fastvideo/train/utils/tracking.py diff --git a/fastvideo/distillation/utils/distill_config.py b/fastvideo/train/utils/training_config.py similarity index 100% rename from fastvideo/distillation/utils/distill_config.py rename to fastvideo/train/utils/training_config.py diff --git a/fastvideo/distillation/utils/validation.py b/fastvideo/train/utils/validation.py similarity index 100% rename from fastvideo/distillation/utils/validation.py rename to fastvideo/train/utils/validation.py diff --git a/fastvideo/distillation/validators/__init__.py b/fastvideo/train/validators/__init__.py similarity index 100% rename from fastvideo/distillation/validators/__init__.py rename to fastvideo/train/validators/__init__.py diff --git a/fastvideo/distillation/validators/base.py b/fastvideo/train/validators/base.py similarity index 100% rename from fastvideo/distillation/validators/base.py rename to fastvideo/train/validators/base.py diff --git a/fastvideo/distillation/validators/wan.py b/fastvideo/train/validators/wan.py similarity index 100% rename from fastvideo/distillation/validators/wan.py rename to fastvideo/train/validators/wan.py diff --git a/fastvideo/distillation/validators/wangame.py b/fastvideo/train/validators/wangame.py similarity index 100% rename from fastvideo/distillation/validators/wangame.py rename to fastvideo/train/validators/wangame.py From d72ea2bd592683ffca59b63d53c738b49ce58680 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 22:41:18 +0000 Subject: [PATCH 178/214] distill->train --- dev/config.md | 6 +- dev/design.md | 1 + dev/fastgen.md | 1 + dev/phase_add_causal_wangame_dfsft.md | 2 +- dev/phase_mode.md | 5 +- dev/phases/phase_3.md | 7 +- docs/wangame/zero_init_fixes.md | 3 +- .../refactor/dfsft_wangame_causal_v3.yaml | 4 +- .../self_forcing_wangame_causal_v3.yaml | 8 +- fastvideo/configs/models/dits/__init__.py | 3 +- fastvideo/configs/pipelines/__init__.py | 4 +- .../pipelines/basic/wan/wan_dmd_pipeline.py | 1 - .../basic/wan/wan_i2v_dmd_pipeline.py | 10 +- fastvideo/pipelines/basic/wan/wan_pipeline.py | 9 +- .../basic/wan/wangame_causal_dmd_pipeline.py | 21 +- .../basic/wan/wangame_i2v_pipeline.py | 3 +- ...game_preprocess_pipeline_ode_trajectory.py | 3 +- fastvideo/pipelines/samplers/__init__.py | 1 - fastvideo/pipelines/samplers/base.py | 4 +- fastvideo/pipelines/samplers/wan.py | 18 +- fastvideo/pipelines/stages/denoising.py | 51 +- .../pipelines/stages/matrixgame_denoising.py | 43 +- fastvideo/registry.py | 21 +- .../test_optimizer_scheduler_alignment.py | 4 +- fastvideo/train/__init__.py | 4 +- fastvideo/train/dispatch.py | 16 +- fastvideo/train/doc/utils/module_state.md | 1 - fastvideo/train/methods/__init__.py | 12 +- fastvideo/train/methods/base.py | 14 +- .../methods/consistency_model/__init__.py | 1 - .../methods/distribution_matching/__init__.py | 7 +- .../methods/distribution_matching/dmd2.py | 28 +- .../distribution_matching/self_forcing.py | 6 +- .../train/methods/fine_tuning/__init__.py | 11 +- fastvideo/train/methods/fine_tuning/dfsft.py | 26 +- .../train/methods/fine_tuning/finetune.py | 26 +- .../knowledge_distillation/__init__.py | 1 - fastvideo/train/models/__init__.py | 1 - fastvideo/train/models/base.py | 6 +- fastvideo/train/models/wan/__init__.py | 6 +- fastvideo/train/models/wan/wan.py | 676 +++++---------- fastvideo/train/models/wangame/__init__.py | 11 +- fastvideo/train/models/wangame/wangame.py | 776 ++++++------------ .../train/models/wangame/wangame_causal.py | 4 +- fastvideo/train/trainer.py | 10 +- fastvideo/train/utils/__init__.py | 2 - fastvideo/train/utils/checkpoint.py | 97 +-- fastvideo/train/utils/config.py | 32 +- fastvideo/train/utils/dataloader.py | 2 +- fastvideo/train/utils/instantiate.py | 46 +- fastvideo/train/utils/module_state.py | 1 - fastvideo/train/utils/moduleloader.py | 29 +- fastvideo/train/utils/optimizer.py | 2 +- fastvideo/train/utils/tracking.py | 2 +- fastvideo/train/utils/training_config.py | 2 +- fastvideo/train/utils/validation.py | 74 +- fastvideo/train/validators/__init__.py | 6 +- fastvideo/train/validators/base.py | 5 +- fastvideo/train/validators/wan.py | 12 +- fastvideo/train/validators/wangame.py | 12 +- fastvideo/training/distillation.py | 18 +- visualize_trajectory.py | 90 +- 62 files changed, 826 insertions(+), 1482 deletions(-) diff --git a/dev/config.md b/dev/config.md index 4089e7f2a..d1e61d0cf 100644 --- a/dev/config.md +++ b/dev/config.md @@ -14,6 +14,7 @@ distillation 入口 **只接受一个真实存在的 YAML 文件路径**（不 m 也不做路径补全/overlay）。YAML 是 single source of truth。 运行方式（示意）： + ```bash python fastvideo/training/distillation.py \ --config /abs/path/to/examples/distillation//xxx.yaml @@ -117,12 +118,14 @@ loader 会注入/补全的 invariants（见 `fastvideo/distillation/utils/config 两种写法（二选一）： 1) inline（适合少量 override）： + ```yaml default_pipeline_config: flow_shift: 8 ``` -2) path（适合复用大型 config 文件）： +1) path（适合复用大型 config 文件）： + ```yaml default_pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json ``` @@ -139,6 +142,7 @@ default_pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json ## 7) `method_config`: method/algorithm 专属超参 `method_config` 由 method 自己解释。以 DMD2 为例： + ```yaml method_config: rollout_mode: simulate # {simulate|data_latent} diff --git a/dev/design.md b/dev/design.md index 9be6b13e4..111729900 100644 --- a/dev/design.md +++ b/dev/design.md @@ -664,6 +664,7 @@ Phase 3.1 的 `recipe/method_config` 对齐。 - Wan 侧由 `WanAdapter` 实现该 contract；若未来出现 family-specific 分支，优先在 adapter 内部消化而不是膨胀 method Finetune 的 config（示意，schema v2）： + ```yaml recipe: {family: wan, method: finetune} roles: diff --git a/dev/fastgen.md b/dev/fastgen.md index 295d296ad..e5b25b951 100644 --- a/dev/fastgen.md +++ b/dev/fastgen.md @@ -373,6 +373,7 @@ def gan_loss_discriminator(real_logits, fake_logits): ### Three-level config hierarchy 1. **Base config** (`fastgen/configs/config.py`): + ```python @attrs.define class BaseModelConfig: diff --git a/dev/phase_add_causal_wangame_dfsft.md b/dev/phase_add_causal_wangame_dfsft.md index f621b4eeb..11c379fb7 100644 --- a/dev/phase_add_causal_wangame_dfsft.md +++ b/dev/phase_add_causal_wangame_dfsft.md @@ -299,7 +299,7 @@ default_pipeline_config: - 方案 A（最小改动）：validator 在构建 validation pipeline 时，把 `training.validation.*` 写入 `args_copy.pipeline_config.*`（validation-only）； - 方案 B（更干净）：把 streaming pipeline 改为优先读 `ForwardBatch.sampling_timesteps` - + `ValidationRequest.context_noise`，彻底摆脱 pipeline_config 依赖（工程量更大）。 + - `ValidationRequest.context_noise`，彻底摆脱 pipeline_config 依赖（工程量更大）。 7) **context noise 的“语义一致性”**（潜在坑）： - legacy self-forcing 里 context noise 是： diff --git a/dev/phase_mode.md b/dev/phase_mode.md index b4750ecd1..142b33c1d 100644 --- a/dev/phase_mode.md +++ b/dev/phase_mode.md @@ -224,11 +224,11 @@ return DistillRuntime( - 先在 models 内部把 `WanAdapter` 改名 `WanModel`（类名、注释、引用） - `ModelAdapter` 改名 `ModelBase` -2) **去掉 ModelComponents** +1) **去掉 ModelComponents** - model plugin 不再返回 dataclass，而是返回模型对象本身 - dispatch 改为实例化模型对象 -3) **methods 参数名收敛** +1) **methods 参数名收敛** - `adapter` 全部改为 `model` 做到这一步就能得到一个更直觉的结构： @@ -236,4 +236,3 @@ return DistillRuntime( - `models/` 里就是“模型插件对象” - `methods/` 只看 `model`（operation-centric）+ `bundle`（role-centric） - dispatch 就是 “cfg -> model -> method -> trainer” - diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md index 0d22acf26..f922cb687 100644 --- a/dev/phases/phase_3.md +++ b/dev/phases/phase_3.md @@ -28,6 +28,7 @@ Phase 2.9 已完成三件关键事情（为 Phase 3 铺路）： - `WanAdapter.prepare_batch()` 不再读取 `training_args.simulate_generator_forward`。 ### Schema v2（示意） + ```yaml recipe: family: wan @@ -186,16 +187,16 @@ Phase 3.1~3.3 已经把训练端到端跑通；但目前 `fastvideo/distillation - `fastvideo/distillation/families/` → `fastvideo/distillation/models/` - 语义：这里的 “models” 指 **模型家族/管线 contract 的集成插件**（不是 YAML 的 `roles:`）。 -2) **roles 容器命名统一** +1) **roles 容器命名统一** - `fastvideo/distillation/bundle.py` → `fastvideo/distillation/roles.py` - `ModelBundle` → `RoleManager` -3) **把 infra 从 models(原 families) 中解耦合** +1) **把 infra 从 models(原 families) 中解耦合** - dataloader 构建逻辑从 `models/*` 抽到 `fastvideo/distillation/utils/`（或 `infra/`） - tracker 初始化从 `models/*` 抽到 `trainer/entrypoint`（更符合“infra 归 infra”） - checkpointing 相关（`fastvideo/distillation/utils/checkpoint.py`）统一放在 `utils/`（或 `infra/`） -4) **减少“文件级概念数量”** +1) **减少“文件级概念数量”** - 已将纯 dataclass（原 `specs.py/runtime.py`）合并到 `utils/config.py`，减少“文件级概念数量” - 已将 YAML loader（原 `yaml_config.py`）合并到 `utils/config.py`（schema+解析逻辑同处） - `registry.py + builder.py` 可以合并/重命名为更直觉的 `dispatch.py`（保留注册表与 build_runtime 的入口） diff --git a/docs/wangame/zero_init_fixes.md b/docs/wangame/zero_init_fixes.md index b418cb5dd..3a397657f 100644 --- a/docs/wangame/zero_init_fixes.md +++ b/docs/wangame/zero_init_fixes.md @@ -25,7 +25,7 @@ for new_param_name in unused_keys: torch.zeros_like(...) # Zero for output projections (residual behavior) ``` -**Why:** +**Why:** - Input projections (`fc_in.weight`) need non-zero weights for gradients to flow - Output projections (`fc_out.weight`) should be zero-initialized for stable residual learning (ControlNet/adapter pattern) @@ -36,6 +36,7 @@ for new_param_name in unused_keys: **Problem:** Attention mask had shape `[B, L]` but query tensor had shape `[2*B, L, ...]` (rope + prope concatenated). The prope batch (second half) had no mask coverage → output was zeros. **Fix:** + ```python # Before (wrong): attention_mask = torch.ones(batch_size, seq_len, ...) # [B, L] diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index c5186e2a4..64353998e 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -5,12 +5,12 @@ models: student: - _target_: fastvideo.distillation.models.wangame.WanGameCausalModel + _target_: fastvideo.train.models.wangame.WanGameCausalModel init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true method: - _target_: fastvideo.distillation.methods.fine_tuning.dfsft.DiffusionForcingSFTMethod + _target_: fastvideo.train.methods.fine_tuning.dfsft.DiffusionForcingSFTMethod attn_kind: dense chunk_size: 3 min_timestep_ratio: 0.02 diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index 3565f2831..2acdf842e 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -8,19 +8,19 @@ models: student: - _target_: fastvideo.distillation.models.wangame.WanGameCausalModel + _target_: fastvideo.train.models.wangame.WanGameCausalModel init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true init_from_checkpoint: outputs/wangame_dfsft_causal_v3/persistent/checkpoint-22000 teacher: - _target_: fastvideo.distillation.models.wangame.WanGameCausalModel + _target_: fastvideo.train.models.wangame.WanGameCausalModel init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: false disable_custom_init_weights: true init_from_checkpoint: outputs/wangame_dfsft_causal_v3/persistent/checkpoint-22000 init_from_checkpoint_role: student critic: - _target_: fastvideo.distillation.models.wangame.WanGameCausalModel + _target_: fastvideo.train.models.wangame.WanGameCausalModel init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true disable_custom_init_weights: true @@ -28,7 +28,7 @@ models: init_from_checkpoint_role: student method: - _target_: fastvideo.distillation.methods.distribution_matching.self_forcing.SelfForcingMethod + _target_: fastvideo.train.methods.distribution_matching.self_forcing.SelfForcingMethod rollout_mode: simulate generator_update_interval: 5 real_score_guidance_scale: 3.5 diff --git a/fastvideo/configs/models/dits/__init__.py b/fastvideo/configs/models/dits/__init__.py index c4abf0a12..0af21d5c0 100644 --- a/fastvideo/configs/models/dits/__init__.py +++ b/fastvideo/configs/models/dits/__init__.py @@ -14,7 +14,6 @@ "HunyuanVideoConfig", "HunyuanVideo15Config", "WanVideoConfig", "StepVideoConfig", "CosmosVideoConfig", "Cosmos25VideoConfig", "LongCatVideoConfig", "LTX2VideoConfig", "HYWorldConfig", - "LingBotWorldVideoConfig", - "WanGameVideoConfig", "WanLingBotVideoConfig" + "LingBotWorldVideoConfig", "WanGameVideoConfig", "WanLingBotVideoConfig" "LingBotWorldVideoConfig", "HunyuanGameCraftConfig", "WanVideoConfig" ] diff --git a/fastvideo/configs/pipelines/__init__.py b/fastvideo/configs/pipelines/__init__.py index 075e97861..b87c23102 100644 --- a/fastvideo/configs/pipelines/__init__.py +++ b/fastvideo/configs/pipelines/__init__.py @@ -20,7 +20,7 @@ "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig", "StepVideoT2VConfig", "SelfForcingWanT2V480PConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig", "HYWorldConfig", - "SD35Config", "LingBotWorldI2V480PConfig", - "WanGameI2V480PConfig", "WanLingBotI2V480PConfig", "SelfForcingWanGameI2V480PConfig", + "SD35Config", "LingBotWorldI2V480PConfig", "WanGameI2V480PConfig", + "WanLingBotI2V480PConfig", "SelfForcingWanGameI2V480PConfig", "get_pipeline_config_cls_from_name" ] diff --git a/fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py index 2729d3988..7a5106126 100644 --- a/fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py @@ -25,4 +25,3 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: EntryClass = WanDMDPipeline - diff --git a/fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py index d092e7c65..dd3ff1538 100644 --- a/fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py @@ -12,10 +12,12 @@ from fastvideo.pipelines.lora_pipeline import LoRAPipeline # isort: off -from fastvideo.pipelines.stages import ( - ImageEncodingStage, ConditioningStage, DecodingStage, DmdDenoisingStage, - ImageVAEEncodingStage, InputValidationStage, LatentPreparationStage, - TextEncodingStage) +from fastvideo.pipelines.stages import (ImageEncodingStage, ConditioningStage, + DecodingStage, DmdDenoisingStage, + ImageVAEEncodingStage, + InputValidationStage, + LatentPreparationStage, + TextEncodingStage) # isort: on from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( FlowMatchEulerDiscreteScheduler) diff --git a/fastvideo/pipelines/basic/wan/wan_pipeline.py b/fastvideo/pipelines/basic/wan/wan_pipeline.py index 0f0c5e9c1..78c2e4dad 100644 --- a/fastvideo/pipelines/basic/wan/wan_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wan_pipeline.py @@ -17,8 +17,7 @@ from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, DenoisingStage, InputValidationStage, LatentPreparationStage, - SdeDenoisingStage, - TextEncodingStage, + SdeDenoisingStage, TextEncodingStage, TimestepPreparationStage) logger = init_logger(__name__) @@ -35,7 +34,8 @@ class WanPipeline(LoRAPipeline, ComposedPipelineBase): def initialize_pipeline(self, fastvideo_args: FastVideoArgs): sampler_kind = get_wan_sampler_kind(fastvideo_args) - self.modules["scheduler"] = build_wan_scheduler(fastvideo_args, sampler_kind) + self.modules["scheduler"] = build_wan_scheduler(fastvideo_args, + sampler_kind) def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: """Set up pipeline stages with proper dependency injection.""" @@ -75,7 +75,8 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: self.add_stage(stage_name="denoising_stage", stage=DenoisingStage( transformer=self.get_module("transformer"), - transformer_2=self.get_module("transformer_2", None), + transformer_2=self.get_module( + "transformer_2", None), scheduler=self.get_module("scheduler"), vae=self.get_module("vae"), pipeline=self)) diff --git a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py index ecd05fba8..af6191b4c 100644 --- a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py @@ -6,15 +6,13 @@ from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline from fastvideo.pipelines.samplers.wan import get_wan_sampler_kind -from fastvideo.pipelines.stages import (ConditioningStage, DecodingStage, - MatrixGameCausalDenoisingStage, - MatrixGameCausalOdeDenoisingStage, - MatrixGameImageEncodingStage, - InputValidationStage, - LatentPreparationStage, - TextEncodingStage, - TimestepPreparationStage) -from fastvideo.pipelines.stages.image_encoding import (MatrixGameImageVAEEncodingStage) +from fastvideo.pipelines.stages import ( + ConditioningStage, DecodingStage, MatrixGameCausalDenoisingStage, + MatrixGameCausalOdeDenoisingStage, MatrixGameImageEncodingStage, + InputValidationStage, LatentPreparationStage, TextEncodingStage, + TimestepPreparationStage) +from fastvideo.pipelines.stages.image_encoding import ( + MatrixGameImageVAEEncodingStage) logger = init_logger(__name__) @@ -59,8 +57,9 @@ def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: scheduler=self.get_module("scheduler"), transformer=self.get_module("transformer", None))) - self.add_stage(stage_name="image_latent_preparation_stage", - stage=MatrixGameImageVAEEncodingStage(vae=self.get_module("vae"))) + self.add_stage( + stage_name="image_latent_preparation_stage", + stage=MatrixGameImageVAEEncodingStage(vae=self.get_module("vae"))) if sampler_kind == "ode": denoising_stage = MatrixGameCausalOdeDenoisingStage( diff --git a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py index fd77540a6..307bf48f9 100644 --- a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py +++ b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py @@ -45,7 +45,8 @@ class WanGameActionImageToVideoPipeline(LoRAPipeline, ComposedPipelineBase): def initialize_pipeline(self, fastvideo_args: FastVideoArgs): sampler_kind = get_wan_sampler_kind(fastvideo_args) - self.modules["scheduler"] = build_wan_scheduler(fastvideo_args, sampler_kind) + self.modules["scheduler"] = build_wan_scheduler(fastvideo_args, + sampler_kind) def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): """Set up pipeline stages with proper dependency injection.""" diff --git a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py index def0dab3f..6335d187e 100644 --- a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py +++ b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py @@ -328,7 +328,8 @@ def preprocess_action_and_trajectory(self, fastvideo_args: FastVideoArgs, batch.image_latent = image_latents[i].unsqueeze(0) sample_keyboard = keyboard_cond[ i] if keyboard_cond is not None else None - sample_mouse = mouse_cond[i] if mouse_cond is not None else None + sample_mouse = mouse_cond[ + i] if mouse_cond is not None else None if sample_keyboard is not None and sample_mouse is not None: batch.keyboard_cond = torch.from_numpy( sample_keyboard).unsqueeze(0).to(device) diff --git a/fastvideo/pipelines/samplers/__init__.py b/fastvideo/pipelines/samplers/__init__.py index 3ae85ae51..638a9e532 100644 --- a/fastvideo/pipelines/samplers/__init__.py +++ b/fastvideo/pipelines/samplers/__init__.py @@ -5,4 +5,3 @@ __all__ = [ "SamplerKind", ] - diff --git a/fastvideo/pipelines/samplers/base.py b/fastvideo/pipelines/samplers/base.py index 571713295..da4ef502c 100644 --- a/fastvideo/pipelines/samplers/base.py +++ b/fastvideo/pipelines/samplers/base.py @@ -22,5 +22,5 @@ def normalize_sampler_kind( if kind == "sde": return "sde" - raise ValueError(f"Unknown sampler kind at {where}: {raw!r} (expected ode|sde)") - + raise ValueError( + f"Unknown sampler kind at {where}: {raw!r} (expected ode|sde)") diff --git a/fastvideo/pipelines/samplers/wan.py b/fastvideo/pipelines/samplers/wan.py index 92c868c05..22fd9d3f3 100644 --- a/fastvideo/pipelines/samplers/wan.py +++ b/fastvideo/pipelines/samplers/wan.py @@ -4,11 +4,9 @@ from fastvideo.fastvideo_args import FastVideoArgs from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, -) + FlowMatchEulerDiscreteScheduler, ) from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( - FlowUniPCMultistepScheduler, -) + FlowUniPCMultistepScheduler, ) from fastvideo.pipelines.samplers.base import SamplerKind, normalize_sampler_kind @@ -22,17 +20,17 @@ def build_wan_scheduler(fastvideo_args: FastVideoArgs, kind: SamplerKind): if kind == "sde": return FlowMatchEulerDiscreteScheduler(shift=shift) - ode_solver_raw = getattr(fastvideo_args.pipeline_config, "ode_solver", "unipc") - ode_solver = str(ode_solver_raw).strip().lower() if ode_solver_raw is not None else "unipc" + ode_solver_raw = getattr(fastvideo_args.pipeline_config, "ode_solver", + "unipc") + ode_solver = str(ode_solver_raw).strip().lower( + ) if ode_solver_raw is not None else "unipc" if ode_solver in {"unipc", "unipc_multistep", "multistep"}: return FlowUniPCMultistepScheduler(shift=shift) if ode_solver in {"euler", "flowmatch", "flowmatch_euler"}: return FlowMatchEulerDiscreteScheduler(shift=shift) - raise ValueError( - "Unknown pipeline_config.ode_solver for wan pipelines: " - f"{ode_solver_raw!r} (expected 'unipc' or 'euler')." - ) + raise ValueError("Unknown pipeline_config.ode_solver for wan pipelines: " + f"{ode_solver_raw!r} (expected 'unipc' or 'euler').") def wan_use_btchw_layout(kind: SamplerKind) -> bool: diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index 6ffd7b46c..38cf77f14 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -554,10 +554,8 @@ def prepare_extra_func_kwargs(self, func, kwargs) -> dict[str, Any]: The prepared kwargs. """ signature = inspect.signature(func) - if any( - p.kind == inspect.Parameter.VAR_KEYWORD - for p in signature.parameters.values() - ): + if any(p.kind == inspect.Parameter.VAR_KEYWORD + for p in signature.parameters.values()): # If the callee accepts `**kwargs`, do not filter by signature. # This is important for models that route parameters internally via # `forward(*args, **kwargs)` (e.g. causal Wangame), where filtering @@ -1270,20 +1268,19 @@ def forward( from fastvideo.models.dits.hyworld.pose import process_custom_actions viewmats, intrinsics, action_labels = process_custom_actions( - batch.keyboard_cond, batch.mouse_cond - ) + batch.keyboard_cond, batch.mouse_cond) camera_action_kwargs = self.prepare_extra_func_kwargs( self.transformer.forward, { - "viewmats": viewmats.unsqueeze(0).to( - get_local_torch_device(), dtype=target_dtype - ), - "Ks": intrinsics.unsqueeze(0).to( - get_local_torch_device(), dtype=target_dtype - ), - "action": action_labels.unsqueeze(0).to( - get_local_torch_device(), dtype=target_dtype - ), + "viewmats": + viewmats.unsqueeze(0).to(get_local_torch_device(), + dtype=target_dtype), + "Ks": + intrinsics.unsqueeze(0).to(get_local_torch_device(), + dtype=target_dtype), + "action": + action_labels.unsqueeze(0).to(get_local_torch_device(), + dtype=target_dtype), }, ) else: @@ -1298,7 +1295,6 @@ def forward( }, ) - # Get latents and embeddings assert batch.latents is not None, "latents must be provided" latents = batch.latents @@ -1309,7 +1305,8 @@ def forward( prompt_embeds[0]).any(), "prompt_embeds contains nan" loop_timesteps = batch.sampling_timesteps if loop_timesteps is None: - legacy = getattr(fastvideo_args.pipeline_config, "dmd_denoising_steps", None) + legacy = getattr(fastvideo_args.pipeline_config, + "dmd_denoising_steps", None) if legacy is not None: loop_timesteps = torch.tensor(legacy, dtype=torch.long) else: @@ -1318,15 +1315,12 @@ def forward( if loop_timesteps is None: raise ValueError( "SDE sampling requires `batch.sampling_timesteps` (preferred) " - "or `pipeline_config.dmd_denoising_steps`." - ) + "or `pipeline_config.dmd_denoising_steps`.") if not isinstance(loop_timesteps, torch.Tensor): loop_timesteps = torch.tensor(loop_timesteps, dtype=torch.long) if loop_timesteps.ndim != 1: - raise ValueError( - "Expected 1D `sampling_timesteps`, got shape " - f"{tuple(loop_timesteps.shape)}" - ) + raise ValueError("Expected 1D `sampling_timesteps`, got shape " + f"{tuple(loop_timesteps.shape)}") loop_timesteps = loop_timesteps.to(get_local_torch_device()) # Run denoising loop @@ -1419,11 +1413,12 @@ def forward( if i < len(loop_timesteps) - 1: next_timestep = loop_timesteps[i + 1] * torch.ones( [1], dtype=torch.long, device=pred_video.device) - noise = torch.randn(video_raw_latent_shape, - dtype=pred_video.dtype, - generator=batch.generator[0] - if isinstance(batch.generator, list) - else batch.generator).to(self.device) + noise = torch.randn( + video_raw_latent_shape, + dtype=pred_video.dtype, + generator=batch.generator[0] if isinstance( + batch.generator, list) else batch.generator).to( + self.device) latents = self.scheduler.add_noise( pred_video.flatten(0, 1), noise.flatten(0, 1), next_timestep).unflatten(0, pred_video.shape[:2]) diff --git a/fastvideo/pipelines/stages/matrixgame_denoising.py b/fastvideo/pipelines/stages/matrixgame_denoising.py index 4f4b5c89a..85f42f6b9 100644 --- a/fastvideo/pipelines/stages/matrixgame_denoising.py +++ b/fastvideo/pipelines/stages/matrixgame_denoising.py @@ -661,8 +661,8 @@ def _update_context_cache( independent_first_frame = getattr(self.transformer, "independent_first_frame", False) if batch.image_latent is not None and not independent_first_frame: - image_context_chunk = batch.image_latent[:, :, start_index: - start_index + + image_context_chunk = batch.image_latent[:, :, + start_index:start_index + current_num_frames, :, :] context_input = torch.cat( [context_input, @@ -707,7 +707,8 @@ def _update_context_cache( **ctx.image_kwargs, **ctx.pos_cond_kwargs, ) - if isinstance(cache_update_ret_2, list) and len(cache_update_ret_2) > 0: + if isinstance(cache_update_ret_2, + list) and len(cache_update_ret_2) > 0: ctx.kv_cache2 = cache_update_ret_2 cache_update_ret = self.transformer( @@ -740,8 +741,7 @@ def forward( if timesteps is None: raise ValueError( "MatrixGameCausalOdeDenoisingStage requires batch.timesteps. " - "Make sure TimestepPreparationStage runs before this stage." - ) + "Make sure TimestepPreparationStage runs before this stage.") target_dtype = torch.bfloat16 autocast_enabled = (target_dtype != torch.float32 @@ -858,8 +858,8 @@ def forward( autocast_enabled=autocast_enabled, boundary_timestep=boundary_timestep, high_noise_timesteps=None, - context_noise=getattr(fastvideo_args.pipeline_config, "context_noise", - 0), + context_noise=getattr(fastvideo_args.pipeline_config, + "context_noise", 0), image_kwargs=image_kwargs, pos_cond_kwargs=pos_cond_kwargs, viewmats_full=viewmats_full, @@ -939,18 +939,24 @@ def _reset_scheduler_state_for_new_rollout(self) -> None: # between independent trajectories. if hasattr(scheduler, "model_outputs") and hasattr(scheduler, "config"): try: - solver_order = int(getattr(scheduler.config, "solver_order", 0) or 0) + solver_order = int( + getattr(scheduler.config, "solver_order", 0) or 0) except Exception: solver_order = 0 if solver_order > 0: - scheduler.model_outputs = [None] * solver_order # type: ignore[attr-defined] + scheduler.model_outputs = [ + None + ] * solver_order # type: ignore[attr-defined] if hasattr(scheduler, "timestep_list") and hasattr(scheduler, "config"): try: - solver_order = int(getattr(scheduler.config, "solver_order", 0) or 0) + solver_order = int( + getattr(scheduler.config, "solver_order", 0) or 0) except Exception: solver_order = 0 if solver_order > 0: - scheduler.timestep_list = [None] * solver_order # type: ignore[attr-defined] + scheduler.timestep_list = [ + None + ] * solver_order # type: ignore[attr-defined] if hasattr(scheduler, "lower_order_nums"): scheduler.lower_order_nums = 0 # type: ignore[attr-defined] if hasattr(scheduler, "last_sample"): @@ -987,8 +993,7 @@ def _process_single_block_ode( latent_model_input = current_latents.to(ctx.target_dtype) independent_first_frame = getattr(self.transformer, - "independent_first_frame", - False) + "independent_first_frame", False) if batch.image_latent is not None and not independent_first_frame: image_latent_chunk = batch.image_latent[:, :, start_index: start_index + @@ -998,10 +1003,8 @@ def _process_single_block_ode( image_latent_chunk.to(ctx.target_dtype) ], dim=1) - elif ( - batch.image_latent is not None and independent_first_frame - and start_index == 0 - ): + elif (batch.image_latent is not None and independent_first_frame + and start_index == 0): latent_model_input = torch.cat([ latent_model_input, batch.image_latent.to(ctx.target_dtype) @@ -1054,8 +1057,10 @@ def _process_single_block_ode( if self.use_action_module and current_model == self.transformer: model_kwargs.update({ - "kv_cache_mouse": ctx.kv_cache_mouse, - "kv_cache_keyboard": ctx.kv_cache_keyboard, + "kv_cache_mouse": + ctx.kv_cache_mouse, + "kv_cache_keyboard": + ctx.kv_cache_keyboard, }) model_kwargs.update(action_kwargs) diff --git a/fastvideo/registry.py b/fastvideo/registry.py index b3cb5a80c..2d8c1f29f 100644 --- a/fastvideo/registry.py +++ b/fastvideo/registry.py @@ -32,22 +32,11 @@ TurboDiffusionT2V_1_3B_Config, ) from fastvideo.configs.pipelines.wan import ( - FastWan2_1_T2V_480P_Config, - FastWan2_2_TI2V_5B_Config, - MatrixGameI2V480PConfig, - SelfForcingWan2_2_T2V480PConfig, - SelfForcingWanT2V480PConfig, - WANV2VConfig, - Wan2_2_I2V_A14B_Config, - Wan2_2_T2V_A14B_Config, - Wan2_2_TI2V_5B_Config, - WanI2V480PConfig, - WanI2V720PConfig, - WanT2V480PConfig, - WanT2V720PConfig, - WanGameI2V480PConfig, - WanLingBotI2V480PConfig -) + FastWan2_1_T2V_480P_Config, FastWan2_2_TI2V_5B_Config, + MatrixGameI2V480PConfig, SelfForcingWan2_2_T2V480PConfig, + SelfForcingWanT2V480PConfig, WANV2VConfig, Wan2_2_I2V_A14B_Config, + Wan2_2_T2V_A14B_Config, Wan2_2_TI2V_5B_Config, WanI2V480PConfig, + WanI2V720PConfig, WanT2V480PConfig, WanT2V720PConfig, WanGameI2V480PConfig) from fastvideo.configs.pipelines.sd35 import SD35Config from fastvideo.configs.sample.base import SamplingParam from fastvideo.configs.sample.cosmos import ( diff --git a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py index 745e25729..d7e592a40 100644 --- a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py +++ b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py @@ -1,6 +1,6 @@ import torch -from fastvideo.distillation.methods.base import DistillMethod +from fastvideo.train.methods.base import TrainingMethod class _FakeScheduler: @@ -26,7 +26,7 @@ def zero_grad(self, *args, **kwargs): # noqa: ANN002, ANN003, ANN201 self.zero_grad_calls += 1 -class _ScheduleMethod(DistillMethod): +class _ScheduleMethod(TrainingMethod): def __init__(self, interval: int) -> None: self.student_opt = _FakeOptimizer() self.critic_opt = _FakeOptimizer() diff --git a/fastvideo/train/__init__.py b/fastvideo/train/__init__.py index 1c6599375..fed6b183c 100644 --- a/fastvideo/train/__init__.py +++ b/fastvideo/train/__init__.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -from fastvideo.distillation.trainer import DistillTrainer +from fastvideo.train.trainer import Trainer __all__ = [ - "DistillTrainer", + "Trainer", ] diff --git a/fastvideo/train/dispatch.py b/fastvideo/train/dispatch.py index f465abe81..5a75e627a 100644 --- a/fastvideo/train/dispatch.py +++ b/fastvideo/train/dispatch.py @@ -5,19 +5,19 @@ from typing import Any, TYPE_CHECKING -from fastvideo.distillation.utils.instantiate import ( +from fastvideo.train.utils.instantiate import ( instantiate, resolve_target, ) -from fastvideo.distillation.utils.config import RunConfig +from fastvideo.train.utils.config import RunConfig if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( - DistillTrainingConfig, ) - from fastvideo.distillation.methods.base import DistillMethod + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) + from fastvideo.train.methods.base import TrainingMethod -def build_from_config(cfg: RunConfig, ) -> tuple[DistillTrainingConfig, DistillMethod, Any, int]: +def build_from_config(cfg: RunConfig, ) -> tuple[TrainingConfig, TrainingMethod, Any, int]: """Build method + dataloader from a v3 run config. 1. Instantiate each model in ``cfg.models`` via ``_target_``. @@ -26,7 +26,7 @@ def build_from_config(cfg: RunConfig, ) -> tuple[DistillTrainingConfig, DistillM validator=...)``. 4. Return ``(training_args, method, dataloader, start_step)``. """ - from fastvideo.distillation.models.base import ModelBase + from fastvideo.train.models.base import ModelBase # --- 1. Build role model instances --- role_models: dict[str, ModelBase] = {} @@ -38,7 +38,7 @@ def build_from_config(cfg: RunConfig, ) -> tuple[DistillTrainingConfig, DistillM role_models[role] = model # --- 2. Warm-start from checkpoint if needed --- - from fastvideo.distillation.utils.checkpoint import ( + from fastvideo.train.utils.checkpoint import ( maybe_warmstart_role_modules, ) for role, model_cfg in cfg.models.items(): diff --git a/fastvideo/train/doc/utils/module_state.md b/fastvideo/train/doc/utils/module_state.md index 2c4ce9629..51d2bc685 100644 --- a/fastvideo/train/doc/utils/module_state.md +++ b/fastvideo/train/doc/utils/module_state.md @@ -11,4 +11,3 @@ **边界** - ✅ 不涉及 optimizer/scheduler（由 method 管理）。 - ✅ 不涉及激活检查点策略（由 model plugin 在加载后按需启用）。 - diff --git a/fastvideo/train/methods/__init__.py b/fastvideo/train/methods/__init__.py index 4dc7bb63a..61fd6ef2e 100644 --- a/fastvideo/train/methods/__init__.py +++ b/fastvideo/train/methods/__init__.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 -from fastvideo.distillation.methods.base import DistillMethod +from fastvideo.train.methods.base import TrainingMethod __all__ = [ - "DistillMethod", + "TrainingMethod", "DMD2Method", "FineTuneMethod", "SelfForcingMethod", @@ -13,15 +13,15 @@ def __getattr__(name: str) -> object: if name == "DMD2Method": - from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method + from fastvideo.train.methods.distribution_matching.dmd2 import DMD2Method return DMD2Method if name == "FineTuneMethod": - from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod + from fastvideo.train.methods.fine_tuning.finetune import FineTuneMethod return FineTuneMethod if name == "SelfForcingMethod": - from fastvideo.distillation.methods.distribution_matching.self_forcing import SelfForcingMethod + from fastvideo.train.methods.distribution_matching.self_forcing import SelfForcingMethod return SelfForcingMethod if name == "DiffusionForcingSFTMethod": - from fastvideo.distillation.methods.fine_tuning.dfsft import DiffusionForcingSFTMethod + from fastvideo.train.methods.fine_tuning.dfsft import DiffusionForcingSFTMethod return DiffusionForcingSFTMethod raise AttributeError(name) diff --git a/fastvideo/train/methods/base.py b/fastvideo/train/methods/base.py index 6c08e0e5f..2f81fdad3 100644 --- a/fastvideo/train/methods/base.py +++ b/fastvideo/train/methods/base.py @@ -8,12 +8,12 @@ import torch -from fastvideo.distillation.models.base import ModelBase +from fastvideo.train.models.base import ModelBase LogScalar = float | int | torch.Tensor -class DistillMethod(torch.nn.Module, ABC): +class TrainingMethod(torch.nn.Module, ABC): """Base distillation method (algorithm layer). Subclasses own their role models (student, teacher, critic, …) as @@ -63,16 +63,14 @@ def single_train_step( *, current_vsa_sparsity: float = 0.0, ) -> tuple[ - dict[str, torch.Tensor], - dict[str, Any], - dict[str, LogScalar], + dict[str, torch.Tensor], + dict[str, Any], + dict[str, LogScalar], ]: raise NotImplementedError @abstractmethod - def get_optimizers( - self, iteration: int - ) -> Sequence[torch.optim.Optimizer]: + def get_optimizers(self, iteration: int) -> Sequence[torch.optim.Optimizer]: raise NotImplementedError @abstractmethod diff --git a/fastvideo/train/methods/consistency_model/__init__.py b/fastvideo/train/methods/consistency_model/__init__.py index 57c85a008..324710b84 100644 --- a/fastvideo/train/methods/consistency_model/__init__.py +++ b/fastvideo/train/methods/consistency_model/__init__.py @@ -1,4 +1,3 @@ # SPDX-License-Identifier: Apache-2.0 __all__: list[str] = [] - diff --git a/fastvideo/train/methods/distribution_matching/__init__.py b/fastvideo/train/methods/distribution_matching/__init__.py index 1f5c53781..4edb43cf7 100644 --- a/fastvideo/train/methods/distribution_matching/__init__.py +++ b/fastvideo/train/methods/distribution_matching/__init__.py @@ -1,9 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 -from fastvideo.distillation.methods.distribution_matching.dmd2 import DMD2Method -from fastvideo.distillation.methods.distribution_matching.self_forcing import ( - SelfForcingMethod, -) +from fastvideo.train.methods.distribution_matching.dmd2 import DMD2Method +from fastvideo.train.methods.distribution_matching.self_forcing import ( + SelfForcingMethod, ) __all__ = [ "DMD2Method", diff --git a/fastvideo/train/methods/distribution_matching/dmd2.py b/fastvideo/train/methods/distribution_matching/dmd2.py index 63dba055e..696eff501 100644 --- a/fastvideo/train/methods/distribution_matching/dmd2.py +++ b/fastvideo/train/methods/distribution_matching/dmd2.py @@ -8,13 +8,13 @@ import torch import torch.nn.functional as F -from fastvideo.distillation.methods.base import DistillMethod, LogScalar -from fastvideo.distillation.models.base import ModelBase -from fastvideo.distillation.utils.optimizer import ( +from fastvideo.train.methods.base import TrainingMethod, LogScalar +from fastvideo.train.models.base import ModelBase +from fastvideo.train.utils.optimizer import ( build_optimizer_and_scheduler, clip_grad_norm_if_needed, ) -from fastvideo.distillation.utils.validation import ( +from fastvideo.train.utils.validation import ( is_validation_enabled, parse_validation_dataset_file, parse_validation_every_steps, @@ -26,8 +26,8 @@ parse_validation_sampler_kind, parse_validation_sampling_steps, ) -from fastvideo.distillation.validators.base import ValidationRequest -from fastvideo.distillation.utils.config import ( +from fastvideo.train.validators.base import ValidationRequest +from fastvideo.train.utils.config import ( get_optional_float, get_optional_int, parse_betas, @@ -37,7 +37,7 @@ pass -class DMD2Method(DistillMethod): +class DMD2Method(TrainingMethod): """DMD2 distillation algorithm (method layer). Owns role model instances directly: @@ -100,7 +100,7 @@ def _lr_scheduler_dict(self) -> dict[str, Any]: "critic": self._critic_lr_scheduler, } - # DistillMethod override: single_train_step + # TrainingMethod override: single_train_step def single_train_step( self, batch: dict[str, Any], @@ -160,7 +160,7 @@ def single_train_step( metrics: dict[str, LogScalar] = {"update_student": float(update_student)} return loss_map, outputs, metrics - # DistillMethod override: backward + # TrainingMethod override: backward def backward( self, loss_map: dict[str, torch.Tensor], @@ -198,7 +198,7 @@ def backward( grad_accum_rounds=grad_accum_rounds, ) - # DistillMethod override: get_optimizers + # TrainingMethod override: get_optimizers def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: optimizers: list[torch.optim.Optimizer] = [] optimizers.append(self._critic_optimizer) @@ -206,7 +206,7 @@ def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: optimizers.append(self._student_optimizer) return optimizers - # DistillMethod override: get_lr_schedulers + # TrainingMethod override: get_lr_schedulers def get_lr_schedulers(self, iteration: int) -> list[Any]: schedulers: list[Any] = [] schedulers.append(self._critic_lr_scheduler) @@ -214,7 +214,7 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: schedulers.append(self._student_lr_scheduler) return schedulers - # DistillMethod override: optimizers_schedulers_step + # TrainingMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: max_grad_norm = self.training_config.optimizer.max_grad_norm if self._should_update_student(iteration): @@ -222,11 +222,11 @@ def optimizers_schedulers_step(self, iteration: int) -> None: clip_grad_norm_if_needed(self.critic.transformer, max_grad_norm) super().optimizers_schedulers_step(iteration) - # DistillTrainer hook: on_train_start + # Trainer hook: on_train_start def on_train_start(self) -> None: self.student.on_train_start() - # DistillTrainer hook: log_validation + # Trainer hook: log_validation def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: diff --git a/fastvideo/train/methods/distribution_matching/self_forcing.py b/fastvideo/train/methods/distribution_matching/self_forcing.py index de6d42b4a..a0ee28b2c 100644 --- a/fastvideo/train/methods/distribution_matching/self_forcing.py +++ b/fastvideo/train/methods/distribution_matching/self_forcing.py @@ -8,13 +8,13 @@ import torch import torch.distributed as dist -from fastvideo.distillation.models.base import ( +from fastvideo.train.models.base import ( CausalModelBase, ModelBase, ) -from fastvideo.distillation.methods.distribution_matching.dmd2 import ( +from fastvideo.train.methods.distribution_matching.dmd2 import ( DMD2Method, ) -from fastvideo.distillation.utils.config import ( +from fastvideo.train.utils.config import ( get_optional_float, get_optional_int, ) diff --git a/fastvideo/train/methods/fine_tuning/__init__.py b/fastvideo/train/methods/fine_tuning/__init__.py index edce4245a..6f862df4e 100644 --- a/fastvideo/train/methods/fine_tuning/__init__.py +++ b/fastvideo/train/methods/fine_tuning/__init__.py @@ -5,8 +5,8 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from fastvideo.distillation.methods.fine_tuning.dfsft import DiffusionForcingSFTMethod - from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod + from fastvideo.train.methods.fine_tuning.dfsft import DiffusionForcingSFTMethod + from fastvideo.train.methods.fine_tuning.finetune import FineTuneMethod __all__ = [ "DiffusionForcingSFTMethod", @@ -17,13 +17,12 @@ def __getattr__(name: str) -> object: # Lazy import to avoid circular imports during registry bring-up. if name == "DiffusionForcingSFTMethod": - from fastvideo.distillation.methods.fine_tuning.dfsft import ( - DiffusionForcingSFTMethod, - ) + from fastvideo.train.methods.fine_tuning.dfsft import ( + DiffusionForcingSFTMethod, ) return DiffusionForcingSFTMethod if name == "FineTuneMethod": - from fastvideo.distillation.methods.fine_tuning.finetune import FineTuneMethod + from fastvideo.train.methods.fine_tuning.finetune import FineTuneMethod return FineTuneMethod raise AttributeError(name) diff --git a/fastvideo/train/methods/fine_tuning/dfsft.py b/fastvideo/train/methods/fine_tuning/dfsft.py index f91a28f0a..510682211 100644 --- a/fastvideo/train/methods/fine_tuning/dfsft.py +++ b/fastvideo/train/methods/fine_tuning/dfsft.py @@ -8,13 +8,13 @@ import torch import torch.nn.functional as F -from fastvideo.distillation.methods.base import DistillMethod, LogScalar -from fastvideo.distillation.models.base import ModelBase -from fastvideo.distillation.utils.optimizer import ( +from fastvideo.train.methods.base import TrainingMethod, LogScalar +from fastvideo.train.models.base import ModelBase +from fastvideo.train.utils.optimizer import ( build_optimizer_and_scheduler, clip_grad_norm_if_needed, ) -from fastvideo.distillation.utils.validation import ( +from fastvideo.train.utils.validation import ( is_validation_enabled, parse_validation_dataset_file, parse_validation_every_steps, @@ -26,13 +26,13 @@ parse_validation_sampler_kind, parse_validation_sampling_steps, ) -from fastvideo.distillation.validators.base import ValidationRequest +from fastvideo.train.validators.base import ValidationRequest if TYPE_CHECKING: pass -class DiffusionForcingSFTMethod(DistillMethod): +class DiffusionForcingSFTMethod(TrainingMethod): """Diffusion-forcing SFT (DFSFT): train only ``student`` with inhomogeneous timesteps. """ @@ -74,7 +74,7 @@ def _optimizer_dict(self) -> dict[str, Any]: def _lr_scheduler_dict(self) -> dict[str, Any]: return {"student": self._student_lr_scheduler} - # DistillMethod override: single_train_step + # TrainingMethod override: single_train_step def single_train_step( self, batch: dict[str, Any], @@ -185,7 +185,7 @@ def single_train_step( metrics: dict[str, LogScalar] = {} return loss_map, outputs, metrics - # DistillMethod override: backward + # TrainingMethod override: backward def backward( self, loss_map: dict[str, torch.Tensor], @@ -208,26 +208,26 @@ def backward( grad_accum_rounds=grad_accum_rounds, ) - # DistillMethod override: get_optimizers + # TrainingMethod override: get_optimizers def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: del iteration return [self._student_optimizer] - # DistillMethod override: get_lr_schedulers + # TrainingMethod override: get_lr_schedulers def get_lr_schedulers(self, iteration: int) -> list[Any]: del iteration return [self._student_lr_scheduler] - # DistillMethod override: optimizers_schedulers_step + # TrainingMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: clip_grad_norm_if_needed(self.student.transformer, self.training_config.optimizer.max_grad_norm) super().optimizers_schedulers_step(iteration) - # DistillTrainer hook: on_train_start + # Trainer hook: on_train_start def on_train_start(self) -> None: self.student.on_train_start() - # DistillTrainer hook: log_validation + # Trainer hook: log_validation def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: diff --git a/fastvideo/train/methods/fine_tuning/finetune.py b/fastvideo/train/methods/fine_tuning/finetune.py index c2f274be7..3a314b84c 100644 --- a/fastvideo/train/methods/fine_tuning/finetune.py +++ b/fastvideo/train/methods/fine_tuning/finetune.py @@ -8,13 +8,13 @@ import torch import torch.nn.functional as F -from fastvideo.distillation.methods.base import DistillMethod, LogScalar -from fastvideo.distillation.models.base import ModelBase -from fastvideo.distillation.utils.optimizer import ( +from fastvideo.train.methods.base import TrainingMethod, LogScalar +from fastvideo.train.models.base import ModelBase +from fastvideo.train.utils.optimizer import ( build_optimizer_and_scheduler, clip_grad_norm_if_needed, ) -from fastvideo.distillation.utils.validation import ( +from fastvideo.train.utils.validation import ( is_validation_enabled, parse_validation_dataset_file, parse_validation_every_steps, @@ -26,13 +26,13 @@ parse_validation_sampler_kind, parse_validation_sampling_steps, ) -from fastvideo.distillation.validators.base import ValidationRequest +from fastvideo.train.validators.base import ValidationRequest if TYPE_CHECKING: pass -class FineTuneMethod(DistillMethod): +class FineTuneMethod(TrainingMethod): """Supervised finetuning: only ``student`` participates.""" def __init__( @@ -69,7 +69,7 @@ def _optimizer_dict(self) -> dict[str, Any]: def _lr_scheduler_dict(self) -> dict[str, Any]: return {"student": self._student_lr_scheduler} - # DistillMethod override: single_train_step + # TrainingMethod override: single_train_step def single_train_step( self, batch: dict[str, Any], @@ -137,7 +137,7 @@ def single_train_step( metrics: dict[str, LogScalar] = {} return loss_map, outputs, metrics - # DistillMethod override: backward + # TrainingMethod override: backward def backward( self, loss_map: dict[str, torch.Tensor], @@ -160,26 +160,26 @@ def backward( grad_accum_rounds=grad_accum_rounds, ) - # DistillMethod override: get_optimizers + # TrainingMethod override: get_optimizers def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: del iteration return [self._student_optimizer] - # DistillMethod override: get_lr_schedulers + # TrainingMethod override: get_lr_schedulers def get_lr_schedulers(self, iteration: int) -> list[Any]: del iteration return [self._student_lr_scheduler] - # DistillMethod override: optimizers_schedulers_step + # TrainingMethod override: optimizers_schedulers_step def optimizers_schedulers_step(self, iteration: int) -> None: clip_grad_norm_if_needed(self.student.transformer, self.training_config.optimizer.max_grad_norm) super().optimizers_schedulers_step(iteration) - # DistillTrainer hook: on_train_start + # Trainer hook: on_train_start def on_train_start(self) -> None: self.student.on_train_start() - # DistillTrainer hook: log_validation + # Trainer hook: log_validation def log_validation(self, iteration: int) -> None: validator = getattr(self, "validator", None) if validator is None: diff --git a/fastvideo/train/methods/knowledge_distillation/__init__.py b/fastvideo/train/methods/knowledge_distillation/__init__.py index 57c85a008..324710b84 100644 --- a/fastvideo/train/methods/knowledge_distillation/__init__.py +++ b/fastvideo/train/methods/knowledge_distillation/__init__.py @@ -1,4 +1,3 @@ # SPDX-License-Identifier: Apache-2.0 __all__: list[str] = [] - diff --git a/fastvideo/train/models/__init__.py b/fastvideo/train/models/__init__.py index dd027ea6e..56b47b1af 100644 --- a/fastvideo/train/models/__init__.py +++ b/fastvideo/train/models/__init__.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """Model build plugins for Phase 2/2.9 distillation. These are "model plugins" selected by ``recipe.family`` / ``roles..family``. diff --git a/fastvideo/train/models/base.py b/fastvideo/train/models/base.py index 00349922a..dff3c26cc 100644 --- a/fastvideo/train/models/base.py +++ b/fastvideo/train/models/base.py @@ -8,8 +8,8 @@ import torch if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( - DistillTrainingConfig, ) + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) from fastvideo.pipelines import TrainingBatch @@ -30,7 +30,7 @@ class ModelBase(ABC): # Lifecycle # ------------------------------------------------------------------ - def init_preprocessors(self, training_config: DistillTrainingConfig) -> None: + def init_preprocessors(self, training_config: TrainingConfig) -> None: """Load VAE, build dataloader, seed RNGs. Called only on the student by the method's ``__init__``. diff --git a/fastvideo/train/models/wan/__init__.py b/fastvideo/train/models/wan/__init__.py index 656b116f1..d79381246 100644 --- a/fastvideo/train/models/wan/__init__.py +++ b/fastvideo/train/models/wan/__init__.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 - """Wan model plugin package.""" -from fastvideo.distillation.models.wan.wan import ( - WanModel as WanModel, -) +from fastvideo.train.models.wan.wan import ( + WanModel as WanModel, ) diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index 48768edc4..38a93db9f 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """Wan model plugin (per-role instance).""" from __future__ import annotations @@ -19,19 +18,15 @@ ) from fastvideo.forward_context import set_forward_context from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, -) + FlowMatchEulerDiscreteScheduler, ) from fastvideo.models.utils import pred_noise_to_pred_video from fastvideo.pipelines import TrainingBatch from fastvideo.pipelines.basic.wan.wan_pipeline import ( - WanPipeline, -) + WanPipeline, ) from fastvideo.pipelines.pipeline_batch_info import ( - ForwardBatch, -) + ForwardBatch, ) from fastvideo.training.activation_checkpoint import ( - apply_activation_checkpointing, -) + apply_activation_checkpointing, ) from fastvideo.training.training_utils import ( compute_density_for_timestep_sampling, get_sigmas, @@ -44,26 +39,21 @@ set_random_seed, ) -from fastvideo.distillation.models.base import ModelBase -from fastvideo.distillation.utils.module_state import ( - apply_trainable, -) -from fastvideo.distillation.utils.moduleloader import ( - load_module_from_path, -) +from fastvideo.train.models.base import ModelBase +from fastvideo.train.utils.module_state import ( + apply_trainable, ) +from fastvideo.train.utils.moduleloader import ( + load_module_from_path, ) if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( - DistillTrainingConfig, - ) + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) try: from fastvideo.attention.backends.video_sparse_attn import ( - VideoSparseAttentionMetadataBuilder, - ) + VideoSparseAttentionMetadataBuilder, ) from fastvideo.attention.backends.vmoba import ( - VideoMobaAttentionMetadataBuilder, - ) + VideoMobaAttentionMetadataBuilder, ) except Exception: VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] @@ -88,39 +78,22 @@ def __init__( transformer = load_module_from_path( model_path=self._init_from, module_type="transformer", - disable_custom_init_weights=( - disable_custom_init_weights - ), - override_transformer_cls_name=( - "WanTransformer3DModel" - ), - ) - transformer = apply_trainable( - transformer, trainable=self._trainable + disable_custom_init_weights=(disable_custom_init_weights), + override_transformer_cls_name=("WanTransformer3DModel"), ) - if ( - self._trainable - and enable_gradient_checkpointing_type - ): + transformer = apply_trainable(transformer, trainable=self._trainable) + if (self._trainable and enable_gradient_checkpointing_type): transformer = apply_activation_checkpointing( transformer, - checkpointing_type=( - enable_gradient_checkpointing_type - ), + checkpointing_type=(enable_gradient_checkpointing_type), ) self.transformer = transformer - self.noise_scheduler = ( - FlowMatchEulerDiscreteScheduler( - shift=float(flow_shift) - ) - ) + self.noise_scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) # Filled by init_preprocessors (student only). self.vae: Any = None - self.training_config: DistillTrainingConfig | None = ( - None - ) + self.training_config: TrainingConfig | None = (None) self.dataloader: Any = None self.validator: Any = None self.start_step: int = 0 @@ -129,23 +102,15 @@ def __init__( self.sp_group: Any = None self.device: Any = None - self.noise_random_generator: ( - torch.Generator | None - ) = None + self.noise_random_generator: (torch.Generator | None) = None self.noise_gen_cuda: torch.Generator | None = None - self.negative_prompt_embeds: ( - torch.Tensor | None - ) = None - self.negative_prompt_attention_mask: ( - torch.Tensor | None - ) = None + self.negative_prompt_embeds: (torch.Tensor | None) = None + self.negative_prompt_attention_mask: (torch.Tensor | None) = None # Timestep mechanics. self.timestep_shift: float = float(flow_shift) - self.num_train_timestep: int = int( - self.noise_scheduler.num_train_timesteps - ) + self.num_train_timestep: int = int(self.noise_scheduler.num_train_timesteps) self.min_timestep: int = 0 self.max_timestep: int = self.num_train_timestep @@ -153,9 +118,7 @@ def __init__( # Lifecycle # ------------------------------------------------------------------ - def init_preprocessors( - self, training_config: DistillTrainingConfig - ) -> None: + def init_preprocessors(self, training_config: TrainingConfig) -> None: self.training_config = training_config self.vae = load_module_from_path( @@ -171,36 +134,23 @@ def init_preprocessors( self._init_timestep_mechanics() # Optional validator. - validation_cfg = getattr( - training_config, "_validation_cfg", None - ) + validation_cfg = getattr(training_config, "_validation_cfg", None) if validation_cfg: - validation_enabled = bool( - validation_cfg.get( - "enabled", bool(validation_cfg) - ) - ) + validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) if validation_enabled: - from fastvideo.distillation.validators.wan import ( - WanValidator, - ) + from fastvideo.train.validators.wan import ( + WanValidator, ) - self.validator = WanValidator( - training_config=training_config - ) + self.validator = WanValidator(training_config=training_config) from fastvideo.dataset.dataloader.schema import ( - pyarrow_schema_t2v, - ) - from fastvideo.distillation.utils.dataloader import ( - build_parquet_t2v_train_dataloader, - ) + pyarrow_schema_t2v, ) + from fastvideo.train.utils.dataloader import ( + build_parquet_t2v_train_dataloader, ) text_len = ( training_config.pipeline_config.text_encoder_configs[ # type: ignore[union-attr] - 0 - ].arch_config.text_len - ) + 0].arch_config.text_len) self.dataloader = build_parquet_t2v_train_dataloader( training_config.data, text_len=int(text_len), @@ -212,58 +162,38 @@ def init_preprocessors( def num_train_timesteps(self) -> int: return int(self.num_train_timestep) - def shift_and_clamp_timestep( - self, timestep: torch.Tensor - ) -> torch.Tensor: + def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: timestep = shift_timestep( timestep, self.timestep_shift, self.num_train_timestep, ) - return timestep.clamp( - self.min_timestep, self.max_timestep - ) + return timestep.clamp(self.min_timestep, self.max_timestep) def on_train_start(self) -> None: assert self.training_config is not None seed = self.training_config.data.seed if seed is None: - raise ValueError( - "training.data.seed must be set " - "for distillation" - ) + raise ValueError("training.data.seed must be set " + "for distillation") - global_rank = int( - getattr(self.world_group, "rank", 0) - ) - sp_world_size = int( - self.training_config.distributed.sp_size or 1 - ) + global_rank = int(getattr(self.world_group, "rank", 0)) + sp_world_size = int(self.training_config.distributed.sp_size or 1) if sp_world_size > 1: - sp_group_seed = int(seed) + ( - global_rank // sp_world_size - ) + sp_group_seed = int(seed) + (global_rank // sp_world_size) set_random_seed(sp_group_seed) else: set_random_seed(int(seed) + global_rank) - self.noise_random_generator = torch.Generator( - device="cpu" - ).manual_seed(int(seed)) - self.noise_gen_cuda = torch.Generator( - device=self.device - ).manual_seed(int(seed)) + self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) + self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) self.ensure_negative_conditioning() - def get_rng_generators( - self, - ) -> dict[str, torch.Generator]: + def get_rng_generators(self, ) -> dict[str, torch.Generator]: generators: dict[str, torch.Generator] = {} if self.noise_random_generator is not None: - generators["noise_cpu"] = ( - self.noise_random_generator - ) + generators["noise_cpu"] = (self.noise_random_generator) if self.noise_gen_cuda is not None: generators["noise_cuda"] = self.noise_gen_cuda return generators @@ -286,15 +216,9 @@ def prepare_batch( dtype = self._get_training_dtype() device = self.device - training_batch = TrainingBatch( - current_vsa_sparsity=current_vsa_sparsity - ) - encoder_hidden_states = raw_batch[ - "text_embedding" - ] - encoder_attention_mask = raw_batch[ - "text_attention_mask" - ] + training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) + encoder_hidden_states = raw_batch["text_embedding"] + encoder_attention_mask = raw_batch["text_attention_mask"] infos = raw_batch.get("info_list") if latents_source == "zeros": @@ -303,17 +227,9 @@ def prepare_batch( tc.pipeline_config.vae_config.arch_config # type: ignore[union-attr] ) num_channels = vae_config.z_dim - spatial_compression_ratio = ( - vae_config.spatial_compression_ratio - ) - latent_height = ( - tc.data.num_height - // spatial_compression_ratio - ) - latent_width = ( - tc.data.num_width - // spatial_compression_ratio - ) + spatial_compression_ratio = (vae_config.spatial_compression_ratio) + latent_height = (tc.data.num_height // spatial_compression_ratio) + latent_width = (tc.data.num_width // spatial_compression_ratio) latents = torch.zeros( batch_size, num_channels, @@ -325,43 +241,25 @@ def prepare_batch( ) elif latents_source == "data": if "vae_latent" not in raw_batch: - raise ValueError( - "vae_latent not found in batch " - "and latents_source='data'" - ) + raise ValueError("vae_latent not found in batch " + "and latents_source='data'") latents = raw_batch["vae_latent"] - latents = latents[ - :, :, : tc.data.num_latent_t - ] + latents = latents[:, :, :tc.data.num_latent_t] latents = latents.to(device, dtype=dtype) else: - raise ValueError( - f"Unknown latents_source: " - f"{latents_source!r}" - ) + raise ValueError(f"Unknown latents_source: " + f"{latents_source!r}") training_batch.latents = latents - training_batch.encoder_hidden_states = ( - encoder_hidden_states.to(device, dtype=dtype) - ) - training_batch.encoder_attention_mask = ( - encoder_attention_mask.to(device, dtype=dtype) - ) + training_batch.encoder_hidden_states = (encoder_hidden_states.to(device, dtype=dtype)) + training_batch.encoder_attention_mask = (encoder_attention_mask.to(device, dtype=dtype)) training_batch.infos = infos - training_batch.latents = normalize_dit_input( - "wan", training_batch.latents, self.vae - ) - training_batch = self._prepare_dit_inputs( - training_batch - ) - training_batch = self._build_attention_metadata( - training_batch - ) + training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch = self._prepare_dit_inputs(training_batch) + training_batch = self._build_attention_metadata(training_batch) - training_batch.attn_metadata_vsa = copy.deepcopy( - training_batch.attn_metadata - ) + training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) if training_batch.attn_metadata is not None: training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] @@ -396,44 +294,28 @@ def predict_x0( if conditional: text_dict = batch.conditional_dict if text_dict is None: - raise RuntimeError( - "Missing conditional_dict in " - "TrainingBatch" - ) + raise RuntimeError("Missing conditional_dict in " + "TrainingBatch") else: - text_dict = self._get_uncond_text_dict( - batch, cfg_uncond=cfg_uncond - ) + text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) if attn_kind == "dense": attn_metadata = batch.attn_metadata elif attn_kind == "vsa": attn_metadata = batch.attn_metadata_vsa else: - raise ValueError( - f"Unknown attn_kind: {attn_kind!r}" - ) + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - with torch.autocast( - device_type, dtype=dtype - ), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, ): - input_kwargs = ( - self._build_distill_input_kwargs( - noisy_latents, timestep, text_dict - ) - ) + input_kwargs = (self._build_distill_input_kwargs(noisy_latents, timestep, text_dict)) transformer = self._get_transformer(timestep) - pred_noise = transformer( - **input_kwargs - ).permute(0, 2, 1, 3, 4) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) pred_x0 = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten( - 0, 1 - ), + noise_input_latent=noisy_latents.flatten(0, 1), timestep=timestep, scheduler=self.noise_scheduler, ).unflatten(0, pred_noise.shape[:2]) @@ -454,39 +336,25 @@ def predict_noise( if conditional: text_dict = batch.conditional_dict if text_dict is None: - raise RuntimeError( - "Missing conditional_dict in " - "TrainingBatch" - ) + raise RuntimeError("Missing conditional_dict in " + "TrainingBatch") else: - text_dict = self._get_uncond_text_dict( - batch, cfg_uncond=cfg_uncond - ) + text_dict = self._get_uncond_text_dict(batch, cfg_uncond=cfg_uncond) if attn_kind == "dense": attn_metadata = batch.attn_metadata elif attn_kind == "vsa": attn_metadata = batch.attn_metadata_vsa else: - raise ValueError( - f"Unknown attn_kind: {attn_kind!r}" - ) + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - with torch.autocast( - device_type, dtype=dtype - ), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, ): - input_kwargs = ( - self._build_distill_input_kwargs( - noisy_latents, timestep, text_dict - ) - ) + input_kwargs = (self._build_distill_input_kwargs(noisy_latents, timestep, text_dict)) transformer = self._get_transformer(timestep) - pred_noise = transformer( - **input_kwargs - ).permute(0, 2, 1, 3, 4) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) return pred_noise def backward( @@ -498,12 +366,10 @@ def backward( ) -> None: timesteps, attn_metadata = ctx with set_forward_context( - current_timestep=timesteps, - attn_metadata=attn_metadata, + current_timestep=timesteps, + attn_metadata=attn_metadata, ): - ( - loss / max(1, int(grad_accum_rounds)) - ).backward() + (loss / max(1, int(grad_accum_rounds))).backward() # ------------------------------------------------------------------ # Internal helpers @@ -515,12 +381,9 @@ def _get_training_dtype(self) -> torch.dtype: def _init_timestep_mechanics(self) -> None: assert self.training_config is not None tc = self.training_config - self.timestep_shift = float( - tc.pipeline_config.flow_shift # type: ignore[union-attr] - ) - self.num_train_timestep = int( - self.noise_scheduler.num_train_timesteps - ) + self.timestep_shift = float(tc.pipeline_config.flow_shift # type: ignore[union-attr] + ) + self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) # min/max timestep ratios now come from method_config; # default to full range. self.min_timestep = 0 @@ -536,36 +399,27 @@ def ensure_negative_conditioning(self) -> None: device = self.device dtype = self._get_training_dtype() - from fastvideo.distillation.utils.moduleloader import ( - make_inference_args, - ) + from fastvideo.train.utils.moduleloader import ( + make_inference_args, ) neg_embeds: torch.Tensor | None = None neg_mask: torch.Tensor | None = None if world_group.rank_in_group == 0: - sampling_param = SamplingParam.from_pretrained( - tc.model_path - ) + sampling_param = SamplingParam.from_pretrained(tc.model_path) negative_prompt = sampling_param.negative_prompt - inference_args = make_inference_args( - tc, model_path=tc.model_path - ) + inference_args = make_inference_args(tc, model_path=tc.model_path) prompt_pipeline = WanPipeline.from_pretrained( tc.model_path, args=inference_args, inference_mode=True, - loaded_modules={ - "transformer": self.transformer - }, + loaded_modules={"transformer": self.transformer}, tp_size=tc.distributed.tp_size, sp_size=tc.distributed.sp_size, num_gpus=tc.distributed.num_gpus, - pin_cpu_memory=( - tc.distributed.pin_cpu_memory - ), + pin_cpu_memory=(tc.distributed.pin_cpu_memory), dit_cpu_offload=True, ) @@ -580,23 +434,15 @@ def ensure_negative_conditioning(self) -> None: inference_args, ) - neg_embeds = result_batch.prompt_embeds[0].to( - device=device, dtype=dtype - ) - neg_mask = ( - result_batch.prompt_attention_mask[0].to( - device=device, dtype=dtype - ) - ) + neg_embeds = result_batch.prompt_embeds[0].to(device=device, dtype=dtype) + neg_mask = (result_batch.prompt_attention_mask[0].to(device=device, dtype=dtype)) del prompt_pipeline gc.collect() if torch.cuda.is_available(): torch.cuda.empty_cache() - meta = torch.zeros( - (2,), device=device, dtype=torch.int64 - ) + meta = torch.zeros((2, ), device=device, dtype=torch.int64) if world_group.rank_in_group == 0: assert neg_embeds is not None assert neg_mask is not None @@ -609,12 +455,8 @@ def ensure_negative_conditioning(self) -> None: ) max_ndim = 8 - embed_shape = torch.full( - (max_ndim,), -1, device=device, dtype=torch.int64 - ) - mask_shape = torch.full( - (max_ndim,), -1, device=device, dtype=torch.int64 - ) + embed_shape = torch.full((max_ndim, ), -1, device=device, dtype=torch.int64) + mask_shape = torch.full((max_ndim, ), -1, device=device, dtype=torch.int64) if world_group.rank_in_group == 0: assert neg_embeds is not None assert neg_mask is not None @@ -631,22 +473,12 @@ def ensure_negative_conditioning(self) -> None: world_group.broadcast(embed_shape, src=0) world_group.broadcast(mask_shape, src=0) - embed_sizes = tuple( - int(x) - for x in embed_shape[:embed_ndim].tolist() - ) - mask_sizes = tuple( - int(x) - for x in mask_shape[:mask_ndim].tolist() - ) + embed_sizes = tuple(int(x) for x in embed_shape[:embed_ndim].tolist()) + mask_sizes = tuple(int(x) for x in mask_shape[:mask_ndim].tolist()) if world_group.rank_in_group != 0: - neg_embeds = torch.empty( - embed_sizes, device=device, dtype=dtype - ) - neg_mask = torch.empty( - mask_sizes, device=device, dtype=dtype - ) + neg_embeds = torch.empty(embed_sizes, device=device, dtype=dtype) + neg_mask = torch.empty(mask_sizes, device=device, dtype=dtype) assert neg_embeds is not None assert neg_mask is not None @@ -656,14 +488,10 @@ def ensure_negative_conditioning(self) -> None: self.negative_prompt_embeds = neg_embeds self.negative_prompt_attention_mask = neg_mask - def _sample_timesteps( - self, batch_size: int, device: torch.device - ) -> torch.Tensor: + def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: if self.noise_random_generator is None: - raise RuntimeError( - "on_train_start() must be called before " - "prepare_batch()" - ) + raise RuntimeError("on_train_start() must be called before " + "prepare_batch()") assert self.training_config is not None tc = self.training_config @@ -675,92 +503,54 @@ def _sample_timesteps( logit_std=tc.model.logit_std, mode_scale=tc.model.mode_scale, ) - indices = ( - u - * self.noise_scheduler.config.num_train_timesteps - ).long() - return self.noise_scheduler.timesteps[indices].to( - device=device - ) + indices = (u * self.noise_scheduler.config.num_train_timesteps).long() + return self.noise_scheduler.timesteps[indices].to(device=device) - def _build_attention_metadata( - self, training_batch: TrainingBatch - ) -> TrainingBatch: + def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: assert self.training_config is not None tc = self.training_config latents_shape = training_batch.raw_latent_shape patch_size = ( tc.pipeline_config.dit_config.patch_size # type: ignore[union-attr] ) - current_vsa_sparsity = ( - training_batch.current_vsa_sparsity - ) + current_vsa_sparsity = (training_batch.current_vsa_sparsity) assert latents_shape is not None assert training_batch.timesteps is not None - if ( - envs.FASTVIDEO_ATTENTION_BACKEND - == "VIDEO_SPARSE_ATTN" - ): - if ( - not is_vsa_available() - or VideoSparseAttentionMetadataBuilder - is None - ): - raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is " - "VIDEO_SPARSE_ATTN, but " - "fastvideo_kernel is not correctly " - "installed or detected." - ) + if (envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN"): + if (not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None): + raise ImportError("FASTVIDEO_ATTENTION_BACKEND is " + "VIDEO_SPARSE_ATTN, but " + "fastvideo_kernel is not correctly " + "installed or detected.") training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] raw_latent_shape=latents_shape[2:5], - current_timestep=( - training_batch.timesteps - ), + current_timestep=(training_batch.timesteps), patch_size=patch_size, VSA_sparsity=current_vsa_sparsity, device=self.device, ) - elif ( - envs.FASTVIDEO_ATTENTION_BACKEND - == "VMOBA_ATTN" - ): - if ( - not is_vmoba_available() - or VideoMobaAttentionMetadataBuilder - is None - ): - raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is " - "VMOBA_ATTN, but fastvideo_kernel " - "(or flash_attn>=2.7.4) is not " - "correctly installed." - ) + elif (envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN"): + if (not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None): + raise ImportError("FASTVIDEO_ATTENTION_BACKEND is " + "VMOBA_ATTN, but fastvideo_kernel " + "(or flash_attn>=2.7.4) is not " + "correctly installed.") moba_params = tc.model.moba_config.copy() - moba_params.update( - { - "current_timestep": ( - training_batch.timesteps - ), - "raw_latent_shape": ( - training_batch.raw_latent_shape[ - 2:5 - ] - ), - "patch_size": patch_size, - "device": self.device, - } - ) - training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(**moba_params) # type: ignore[misc] + moba_params.update({ + "current_timestep": (training_batch.timesteps), + "raw_latent_shape": (training_batch.raw_latent_shape[2:5]), + "patch_size": patch_size, + "device": self.device, + }) + training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(** + moba_params) # type: ignore[misc] else: training_batch.attn_metadata = None return training_batch - def _prepare_dit_inputs( - self, training_batch: TrainingBatch - ) -> TrainingBatch: + def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: assert self.training_config is not None tc = self.training_config latents = training_batch.latents @@ -768,10 +558,8 @@ def _prepare_dit_inputs( batch_size = latents.shape[0] if self.noise_gen_cuda is None: - raise RuntimeError( - "on_train_start() must be called before " - "prepare_batch()" - ) + raise RuntimeError("on_train_start() must be called before " + "prepare_batch()") noise = torch.randn( latents.shape, @@ -779,9 +567,7 @@ def _prepare_dit_inputs( device=latents.device, dtype=latents.dtype, ) - timesteps = self._sample_timesteps( - batch_size, latents.device - ) + timesteps = self._sample_timesteps(batch_size, latents.device) if int(tc.distributed.sp_size or 1) > 1: self.sp_group.broadcast(timesteps, src=0) @@ -792,58 +578,32 @@ def _prepare_dit_inputs( n_dim=latents.ndim, dtype=latents.dtype, ) - noisy_model_input = ( - (1.0 - sigmas) * latents + sigmas * noise - ) + noisy_model_input = ((1.0 - sigmas) * latents + sigmas * noise) - training_batch.noisy_model_input = ( - noisy_model_input - ) + training_batch.noisy_model_input = (noisy_model_input) training_batch.timesteps = timesteps training_batch.sigmas = sigmas training_batch.noise = noise training_batch.raw_latent_shape = latents.shape training_batch.conditional_dict = { - "encoder_hidden_states": ( - training_batch.encoder_hidden_states - ), - "encoder_attention_mask": ( - training_batch.encoder_attention_mask - ), + "encoder_hidden_states": (training_batch.encoder_hidden_states), + "encoder_attention_mask": (training_batch.encoder_attention_mask), } - if ( - self.negative_prompt_embeds is not None - and self.negative_prompt_attention_mask - is not None - ): + if (self.negative_prompt_embeds is not None and self.negative_prompt_attention_mask is not None): neg_embeds = self.negative_prompt_embeds - neg_mask = ( - self.negative_prompt_attention_mask - ) - if ( - neg_embeds.shape[0] == 1 - and batch_size > 1 - ): - neg_embeds = neg_embeds.expand( - batch_size, *neg_embeds.shape[1:] - ).contiguous() - if ( - neg_mask.shape[0] == 1 - and batch_size > 1 - ): - neg_mask = neg_mask.expand( - batch_size, *neg_mask.shape[1:] - ).contiguous() + neg_mask = (self.negative_prompt_attention_mask) + if (neg_embeds.shape[0] == 1 and batch_size > 1): + neg_embeds = neg_embeds.expand(batch_size, *neg_embeds.shape[1:]).contiguous() + if (neg_mask.shape[0] == 1 and batch_size > 1): + neg_mask = neg_mask.expand(batch_size, *neg_mask.shape[1:]).contiguous() training_batch.unconditional_dict = { "encoder_hidden_states": neg_embeds, "encoder_attention_mask": neg_mask, } - training_batch.latents = ( - training_batch.latents.permute(0, 2, 1, 3, 4) - ) + training_batch.latents = (training_batch.latents.permute(0, 2, 1, 3, 4)) return training_batch def _build_distill_input_kwargs( @@ -853,27 +613,17 @@ def _build_distill_input_kwargs( text_dict: dict[str, torch.Tensor] | None, ) -> dict[str, Any]: if text_dict is None: - raise ValueError( - "text_dict cannot be None for " - "Wan distillation" - ) + raise ValueError("text_dict cannot be None for " + "Wan distillation") return { - "hidden_states": noise_input.permute( - 0, 2, 1, 3, 4 - ), - "encoder_hidden_states": text_dict[ - "encoder_hidden_states" - ], - "encoder_attention_mask": text_dict[ - "encoder_attention_mask" - ], + "hidden_states": noise_input.permute(0, 2, 1, 3, 4), + "encoder_hidden_states": text_dict["encoder_hidden_states"], + "encoder_attention_mask": text_dict["encoder_attention_mask"], "timestep": timestep, "return_dict": False, } - def _get_transformer( - self, timestep: torch.Tensor - ) -> torch.nn.Module: + def _get_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: return self.transformer def _get_uncond_text_dict( @@ -883,33 +633,23 @@ def _get_uncond_text_dict( cfg_uncond: dict[str, Any] | None, ) -> dict[str, torch.Tensor]: if cfg_uncond is None: - text_dict = getattr( - batch, "unconditional_dict", None - ) + text_dict = getattr(batch, "unconditional_dict", None) if text_dict is None: - raise RuntimeError( - "Missing unconditional_dict; " - "ensure_negative_conditioning() " - "may have failed" - ) + raise RuntimeError("Missing unconditional_dict; " + "ensure_negative_conditioning() " + "may have failed") return text_dict - on_missing_raw = cfg_uncond.get( - "on_missing", "error" - ) + on_missing_raw = cfg_uncond.get("on_missing", "error") if not isinstance(on_missing_raw, str): - raise ValueError( - "method_config.cfg_uncond.on_missing " - "must be a string, got " - f"{type(on_missing_raw).__name__}" - ) + raise ValueError("method_config.cfg_uncond.on_missing " + "must be a string, got " + f"{type(on_missing_raw).__name__}") on_missing = on_missing_raw.strip().lower() if on_missing not in {"error", "ignore"}: - raise ValueError( - "method_config.cfg_uncond.on_missing " - "must be one of {error, ignore}, got " - f"{on_missing_raw!r}" - ) + raise ValueError("method_config.cfg_uncond.on_missing " + "must be one of {error, ignore}, got " + f"{on_missing_raw!r}") for channel, policy_raw in cfg_uncond.items(): if channel in {"on_missing", "text"}: @@ -917,89 +657,61 @@ def _get_uncond_text_dict( if policy_raw is None: continue if not isinstance(policy_raw, str): - raise ValueError( - "method_config.cfg_uncond values " - "must be strings, got " - f"{channel}=" - f"{type(policy_raw).__name__}" - ) + raise ValueError("method_config.cfg_uncond values " + "must be strings, got " + f"{channel}=" + f"{type(policy_raw).__name__}") policy = policy_raw.strip().lower() if policy == "keep": continue if on_missing == "ignore": continue - raise ValueError( - "WanModel does not support " - "cfg_uncond channel " - f"{channel!r} (policy={policy!r}). " - "Set cfg_uncond.on_missing=ignore or " - "remove the channel." - ) + raise ValueError("WanModel does not support " + "cfg_uncond channel " + f"{channel!r} (policy={policy!r}). " + "Set cfg_uncond.on_missing=ignore or " + "remove the channel.") text_policy_raw = cfg_uncond.get("text", None) if text_policy_raw is None: text_policy = "negative_prompt" elif not isinstance(text_policy_raw, str): - raise ValueError( - "method_config.cfg_uncond.text must be " - "a string, got " - f"{type(text_policy_raw).__name__}" - ) + raise ValueError("method_config.cfg_uncond.text must be " + "a string, got " + f"{type(text_policy_raw).__name__}") else: - text_policy = ( - text_policy_raw.strip().lower() - ) + text_policy = (text_policy_raw.strip().lower()) if text_policy in {"negative_prompt"}: - text_dict = getattr( - batch, "unconditional_dict", None - ) + text_dict = getattr(batch, "unconditional_dict", None) if text_dict is None: - raise RuntimeError( - "Missing unconditional_dict; " - "ensure_negative_conditioning() " - "may have failed" - ) + raise RuntimeError("Missing unconditional_dict; " + "ensure_negative_conditioning() " + "may have failed") return text_dict if text_policy == "keep": if batch.conditional_dict is None: - raise RuntimeError( - "Missing conditional_dict in " - "TrainingBatch" - ) + raise RuntimeError("Missing conditional_dict in " + "TrainingBatch") return batch.conditional_dict if text_policy == "zero": if batch.conditional_dict is None: - raise RuntimeError( - "Missing conditional_dict in " - "TrainingBatch" - ) + raise RuntimeError("Missing conditional_dict in " + "TrainingBatch") cond = batch.conditional_dict enc = cond["encoder_hidden_states"] mask = cond["encoder_attention_mask"] - if not torch.is_tensor( - enc - ) or not torch.is_tensor(mask): - raise TypeError( - "conditional_dict must contain " - "tensor text inputs" - ) + if not torch.is_tensor(enc) or not torch.is_tensor(mask): + raise TypeError("conditional_dict must contain " + "tensor text inputs") return { - "encoder_hidden_states": ( - torch.zeros_like(enc) - ), - "encoder_attention_mask": ( - torch.zeros_like(mask) - ), + "encoder_hidden_states": (torch.zeros_like(enc)), + "encoder_attention_mask": (torch.zeros_like(mask)), } if text_policy == "drop": - raise ValueError( - "cfg_uncond.text=drop is not supported " - "for Wan. Use " - "{negative_prompt, keep, zero}." - ) - raise ValueError( - "cfg_uncond.text must be one of " - "{negative_prompt, keep, zero, drop}, got " - f"{text_policy_raw!r}" - ) + raise ValueError("cfg_uncond.text=drop is not supported " + "for Wan. Use " + "{negative_prompt, keep, zero}.") + raise ValueError("cfg_uncond.text must be one of " + "{negative_prompt, keep, zero, drop}, got " + f"{text_policy_raw!r}") diff --git a/fastvideo/train/models/wangame/__init__.py b/fastvideo/train/models/wangame/__init__.py index bcbd06ad4..101e3e7a8 100644 --- a/fastvideo/train/models/wangame/__init__.py +++ b/fastvideo/train/models/wangame/__init__.py @@ -1,10 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 - """WanGame model plugin package.""" -from fastvideo.distillation.models.wangame.wangame import ( - WanGameModel as WanGameModel, -) -from fastvideo.distillation.models.wangame.wangame_causal import ( - WanGameCausalModel as WanGameCausalModel, -) +from fastvideo.train.models.wangame.wangame import ( + WanGameModel as WanGameModel, ) +from fastvideo.train.models.wangame.wangame_causal import ( + WanGameCausalModel as WanGameCausalModel, ) diff --git a/fastvideo/train/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py index b330e70e0..181214b5f 100644 --- a/fastvideo/train/models/wangame/wangame.py +++ b/fastvideo/train/models/wangame/wangame.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """WanGame bidirectional model plugin (per-role instance).""" from __future__ import annotations @@ -17,13 +16,11 @@ ) from fastvideo.forward_context import set_forward_context from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, -) + FlowMatchEulerDiscreteScheduler, ) from fastvideo.models.utils import pred_noise_to_pred_video from fastvideo.pipelines import TrainingBatch from fastvideo.training.activation_checkpoint import ( - apply_activation_checkpointing, -) + apply_activation_checkpointing, ) from fastvideo.training.training_utils import ( compute_density_for_timestep_sampling, get_sigmas, @@ -36,26 +33,21 @@ set_random_seed, ) -from fastvideo.distillation.models.base import ModelBase -from fastvideo.distillation.utils.module_state import ( - apply_trainable, -) -from fastvideo.distillation.utils.moduleloader import ( - load_module_from_path, -) +from fastvideo.train.models.base import ModelBase +from fastvideo.train.utils.module_state import ( + apply_trainable, ) +from fastvideo.train.utils.moduleloader import ( + load_module_from_path, ) if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( - DistillTrainingConfig, - ) + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) try: from fastvideo.attention.backends.video_sparse_attn import ( - VideoSparseAttentionMetadataBuilder, - ) + VideoSparseAttentionMetadataBuilder, ) from fastvideo.attention.backends.vmoba import ( - VideoMobaAttentionMetadataBuilder, - ) + VideoMobaAttentionMetadataBuilder, ) except Exception: VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] @@ -64,9 +56,7 @@ class WanGameModel(ModelBase): """WanGame per-role model: owns transformer + noise_scheduler.""" - _transformer_cls_name: str = ( - "WanGameActionTransformer3DModel" - ) + _transformer_cls_name: str = ("WanGameActionTransformer3DModel") def __init__( self, @@ -84,25 +74,15 @@ def __init__( self.transformer = self._load_transformer( init_from=self._init_from, trainable=self._trainable, - disable_custom_init_weights=( - disable_custom_init_weights - ), - enable_gradient_checkpointing_type=( - enable_gradient_checkpointing_type - ), + disable_custom_init_weights=(disable_custom_init_weights), + enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), ) - self.noise_scheduler = ( - FlowMatchEulerDiscreteScheduler( - shift=float(flow_shift) - ) - ) + self.noise_scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) # Filled by init_preprocessors (student only). self.vae: Any = None - self.training_config: DistillTrainingConfig | None = ( - None - ) + self.training_config: TrainingConfig | None = (None) self.dataloader: Any = None self.validator: Any = None self.start_step: int = 0 @@ -111,15 +91,11 @@ def __init__( self.sp_group: Any = None self.device: Any = None - self.noise_random_generator: ( - torch.Generator | None - ) = None + self.noise_random_generator: (torch.Generator | None) = None self.noise_gen_cuda: torch.Generator | None = None self.timestep_shift: float = float(flow_shift) - self.num_train_timestep: int = int( - self.noise_scheduler.num_train_timesteps - ) + self.num_train_timestep: int = int(self.noise_scheduler.num_train_timesteps) self.min_timestep: int = 0 self.max_timestep: int = self.num_train_timestep @@ -134,25 +110,14 @@ def _load_transformer( transformer = load_module_from_path( model_path=init_from, module_type="transformer", - disable_custom_init_weights=( - disable_custom_init_weights - ), - override_transformer_cls_name=( - self._transformer_cls_name - ), - ) - transformer = apply_trainable( - transformer, trainable=trainable + disable_custom_init_weights=(disable_custom_init_weights), + override_transformer_cls_name=(self._transformer_cls_name), ) - if ( - trainable - and enable_gradient_checkpointing_type - ): + transformer = apply_trainable(transformer, trainable=trainable) + if (trainable and enable_gradient_checkpointing_type): transformer = apply_activation_checkpointing( transformer, - checkpointing_type=( - enable_gradient_checkpointing_type - ), + checkpointing_type=(enable_gradient_checkpointing_type), ) return transformer @@ -160,9 +125,7 @@ def _load_transformer( # Lifecycle # ------------------------------------------------------------------ - def init_preprocessors( - self, training_config: DistillTrainingConfig - ) -> None: + def init_preprocessors(self, training_config: TrainingConfig) -> None: """Load VAE, build dataloader, seed RNGs.""" self.training_config = training_config @@ -179,37 +142,24 @@ def init_preprocessors( self._init_timestep_mechanics() # Optional validator. - validation_cfg = getattr( - training_config, "_validation_cfg", None - ) + validation_cfg = getattr(training_config, "_validation_cfg", None) if validation_cfg: - validation_enabled = bool( - validation_cfg.get( - "enabled", bool(validation_cfg) - ) - ) + validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) if validation_enabled: - from fastvideo.distillation.validators.wangame import ( - WanGameValidator, - ) + from fastvideo.train.validators.wangame import ( + WanGameValidator, ) - self.validator = WanGameValidator( - training_config=training_config - ) + self.validator = WanGameValidator(training_config=training_config) from fastvideo.dataset.dataloader.schema import ( - pyarrow_schema_wangame, - ) - from fastvideo.distillation.utils.dataloader import ( - build_parquet_wangame_train_dataloader, - ) - - self.dataloader = ( - build_parquet_wangame_train_dataloader( - training_config.data, - parquet_schema=pyarrow_schema_wangame, - ) - ) + pyarrow_schema_wangame, ) + from fastvideo.train.utils.dataloader import ( + build_parquet_wangame_train_dataloader, ) + + self.dataloader = (build_parquet_wangame_train_dataloader( + training_config.data, + parquet_schema=pyarrow_schema_wangame, + )) self.start_step = 0 # ------------------------------------------------------------------ @@ -220,17 +170,13 @@ def init_preprocessors( def num_train_timesteps(self) -> int: return int(self.num_train_timestep) - def shift_and_clamp_timestep( - self, timestep: torch.Tensor - ) -> torch.Tensor: + def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: timestep = shift_timestep( timestep, self.timestep_shift, self.num_train_timestep, ) - return timestep.clamp( - self.min_timestep, self.max_timestep - ) + return timestep.clamp(self.min_timestep, self.max_timestep) # ------------------------------------------------------------------ # ModelBase overrides: lifecycle hooks @@ -241,38 +187,24 @@ def on_train_start(self) -> None: tc = self.training_config seed = tc.data.seed if seed is None: - raise ValueError( - "training.data.seed must be set " - "for distillation" - ) + raise ValueError("training.data.seed must be set " + "for distillation") - global_rank = int( - getattr(self.world_group, "rank", 0) - ) + global_rank = int(getattr(self.world_group, "rank", 0)) sp_world_size = int(tc.distributed.sp_size or 1) if sp_world_size > 1: - sp_group_seed = int(seed) + ( - global_rank // sp_world_size - ) + sp_group_seed = int(seed) + (global_rank // sp_world_size) set_random_seed(sp_group_seed) else: set_random_seed(int(seed) + global_rank) - self.noise_random_generator = torch.Generator( - device="cpu" - ).manual_seed(int(seed)) - self.noise_gen_cuda = torch.Generator( - device=self.device - ).manual_seed(int(seed)) + self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) + self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) - def get_rng_generators( - self, - ) -> dict[str, torch.Generator]: + def get_rng_generators(self, ) -> dict[str, torch.Generator]: generators: dict[str, torch.Generator] = {} if self.noise_random_generator is not None: - generators["noise_cpu"] = ( - self.noise_random_generator - ) + generators["noise_cpu"] = (self.noise_random_generator) if self.noise_gen_cuda is not None: generators["noise_cuda"] = self.noise_gen_cuda return generators @@ -293,9 +225,7 @@ def prepare_batch( dtype = self._get_training_dtype() device = self.device - training_batch = TrainingBatch( - current_vsa_sparsity=current_vsa_sparsity - ) + training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) infos = raw_batch.get("info_list") if latents_source == "zeros": @@ -305,17 +235,9 @@ def prepare_batch( tc.pipeline_config.vae_config.arch_config # type: ignore[union-attr] ) num_channels = int(vae_config.z_dim) - spatial_compression_ratio = int( - vae_config.spatial_compression_ratio - ) - latent_height = ( - int(tc.data.num_height) - // spatial_compression_ratio - ) - latent_width = ( - int(tc.data.num_width) - // spatial_compression_ratio - ) + spatial_compression_ratio = int(vae_config.spatial_compression_ratio) + latent_height = (int(tc.data.num_height) // spatial_compression_ratio) + latent_width = (int(tc.data.num_width) // spatial_compression_ratio) latents = torch.zeros( batch_size, num_channels, @@ -327,100 +249,58 @@ def prepare_batch( ) elif latents_source == "data": if "vae_latent" not in raw_batch: - raise ValueError( - "vae_latent not found in batch " - "and latents_source='data'" - ) + raise ValueError("vae_latent not found in batch " + "and latents_source='data'") latents = raw_batch["vae_latent"] - latents = latents[ - :, :, : tc.data.num_latent_t - ] + latents = latents[:, :, :tc.data.num_latent_t] latents = latents.to(device, dtype=dtype) else: - raise ValueError( - f"Unknown latents_source: " - f"{latents_source!r}" - ) + raise ValueError(f"Unknown latents_source: " + f"{latents_source!r}") if "clip_feature" not in raw_batch: - raise ValueError( - "clip_feature must be present for WanGame" - ) - image_embeds = raw_batch["clip_feature"].to( - device, dtype=dtype - ) + raise ValueError("clip_feature must be present for WanGame") + image_embeds = raw_batch["clip_feature"].to(device, dtype=dtype) if "first_frame_latent" not in raw_batch: - raise ValueError( - "first_frame_latent must be present " - "for WanGame" - ) + raise ValueError("first_frame_latent must be present " + "for WanGame") image_latents = raw_batch["first_frame_latent"] - image_latents = image_latents[ - :, :, : tc.data.num_latent_t - ] - image_latents = image_latents.to( - device, dtype=dtype - ) + image_latents = image_latents[:, :, :tc.data.num_latent_t] + image_latents = image_latents.to(device, dtype=dtype) pil_image = raw_batch.get("pil_image") if isinstance(pil_image, torch.Tensor): - training_batch.preprocessed_image = ( - pil_image.to(device=device) - ) + training_batch.preprocessed_image = (pil_image.to(device=device)) else: training_batch.preprocessed_image = pil_image keyboard_cond = raw_batch.get("keyboard_cond") - if ( - isinstance(keyboard_cond, torch.Tensor) - and keyboard_cond.numel() > 0 - ): - training_batch.keyboard_cond = ( - keyboard_cond.to(device, dtype=dtype) - ) + if (isinstance(keyboard_cond, torch.Tensor) and keyboard_cond.numel() > 0): + training_batch.keyboard_cond = (keyboard_cond.to(device, dtype=dtype)) else: training_batch.keyboard_cond = None mouse_cond = raw_batch.get("mouse_cond") - if ( - isinstance(mouse_cond, torch.Tensor) - and mouse_cond.numel() > 0 - ): - training_batch.mouse_cond = mouse_cond.to( - device, dtype=dtype - ) + if (isinstance(mouse_cond, torch.Tensor) and mouse_cond.numel() > 0): + training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) else: training_batch.mouse_cond = None temporal_compression_ratio = ( tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio # type: ignore[union-attr] ) - expected_num_frames = ( - (tc.data.num_latent_t - 1) - * temporal_compression_ratio - + 1 - ) - if ( - training_batch.keyboard_cond is not None - and int(training_batch.keyboard_cond.shape[1]) - != int(expected_num_frames) - ): - raise ValueError( - "keyboard_cond temporal dim mismatch: " - f"got {int(training_batch.keyboard_cond.shape[1])}, " - f"expected {int(expected_num_frames)}" - ) - if ( - training_batch.mouse_cond is not None - and int(training_batch.mouse_cond.shape[1]) - != int(expected_num_frames) - ): - raise ValueError( - "mouse_cond temporal dim mismatch: " - f"got {int(training_batch.mouse_cond.shape[1])}, " - f"expected {int(expected_num_frames)}" - ) + expected_num_frames = ((tc.data.num_latent_t - 1) * temporal_compression_ratio + 1) + if (training_batch.keyboard_cond is not None + and int(training_batch.keyboard_cond.shape[1]) != int(expected_num_frames)): + raise ValueError("keyboard_cond temporal dim mismatch: " + f"got {int(training_batch.keyboard_cond.shape[1])}, " + f"expected {int(expected_num_frames)}") + if (training_batch.mouse_cond is not None + and int(training_batch.mouse_cond.shape[1]) != int(expected_num_frames)): + raise ValueError("mouse_cond temporal dim mismatch: " + f"got {int(training_batch.mouse_cond.shape[1])}, " + f"expected {int(expected_num_frames)}") training_batch.latents = latents training_batch.encoder_hidden_states = None @@ -429,28 +309,16 @@ def prepare_batch( training_batch.image_latents = image_latents training_batch.infos = infos - training_batch.latents = normalize_dit_input( - "wan", training_batch.latents, self.vae - ) - training_batch = self._prepare_dit_inputs( - training_batch - ) - training_batch = self._build_attention_metadata( - training_batch - ) + training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) + training_batch = self._prepare_dit_inputs(training_batch) + training_batch = self._build_attention_metadata(training_batch) - training_batch.attn_metadata_vsa = copy.deepcopy( - training_batch.attn_metadata - ) + training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) if training_batch.attn_metadata is not None: training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] - training_batch.mask_lat_size = ( - self._build_i2v_mask_latents(image_latents) - ) - viewmats, intrinsics, action_labels = ( - self._process_actions(training_batch) - ) + training_batch.mask_lat_size = (self._build_i2v_mask_latents(image_latents)) + viewmats, intrinsics, action_labels = (self._process_actions(training_batch)) training_batch.viewmats = viewmats training_batch.Ks = intrinsics training_batch.action = action_labels @@ -489,54 +357,34 @@ def predict_x0( elif attn_kind == "vsa": attn_metadata = batch.attn_metadata_vsa else: - raise ValueError( - f"Unknown attn_kind: {attn_kind!r}" - ) + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - with torch.autocast( - device_type, dtype=dtype - ), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, ): - cond_inputs = ( - self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - ) - ) - input_kwargs = ( - self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs[ - "image_embeds" - ], - image_latents=cond_inputs[ - "image_latents" - ], - mask_lat_size=cond_inputs[ - "mask_lat_size" - ], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs[ - "keyboard_cond" - ], - ) - ) + cond_inputs = (self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + )) + input_kwargs = (self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs["image_embeds"], + image_latents=cond_inputs["image_latents"], + mask_lat_size=cond_inputs["mask_lat_size"], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs["keyboard_cond"], + )) transformer = self._get_transformer(timestep) - pred_noise = transformer( - **input_kwargs - ).permute(0, 2, 1, 3, 4) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) pred_x0 = pred_noise_to_pred_video( pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten( - 0, 1 - ), + noise_input_latent=noisy_latents.flatten(0, 1), timestep=timestep, scheduler=self.noise_scheduler, ).unflatten(0, pred_noise.shape[:2]) @@ -560,49 +408,31 @@ def predict_noise( elif attn_kind == "vsa": attn_metadata = batch.attn_metadata_vsa else: - raise ValueError( - f"Unknown attn_kind: {attn_kind!r}" - ) + raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - with torch.autocast( - device_type, dtype=dtype - ), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, + with torch.autocast(device_type, dtype=dtype), set_forward_context( + current_timestep=batch.timesteps, + attn_metadata=attn_metadata, ): - cond_inputs = ( - self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - ) - ) - input_kwargs = ( - self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs[ - "image_embeds" - ], - image_latents=cond_inputs[ - "image_latents" - ], - mask_lat_size=cond_inputs[ - "mask_lat_size" - ], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs[ - "keyboard_cond" - ], - ) - ) + cond_inputs = (self._select_cfg_condition_inputs( + batch, + conditional=conditional, + cfg_uncond=cfg_uncond, + )) + input_kwargs = (self._build_distill_input_kwargs( + noisy_latents, + timestep, + image_embeds=cond_inputs["image_embeds"], + image_latents=cond_inputs["image_latents"], + mask_lat_size=cond_inputs["mask_lat_size"], + viewmats=cond_inputs["viewmats"], + Ks=cond_inputs["Ks"], + action=cond_inputs["action"], + mouse_cond=cond_inputs["mouse_cond"], + keyboard_cond=cond_inputs["keyboard_cond"], + )) transformer = self._get_transformer(timestep) - pred_noise = transformer( - **input_kwargs - ).permute(0, 2, 1, 3, 4) + pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) return pred_noise def backward( @@ -614,12 +444,10 @@ def backward( ) -> None: timesteps, attn_metadata = ctx with set_forward_context( - current_timestep=timesteps, - attn_metadata=attn_metadata, + current_timestep=timesteps, + attn_metadata=attn_metadata, ): - ( - loss / max(1, int(grad_accum_rounds)) - ).backward() + (loss / max(1, int(grad_accum_rounds))).backward() # ------------------------------------------------------------------ # Internal helpers @@ -631,23 +459,16 @@ def _get_training_dtype(self) -> torch.dtype: def _init_timestep_mechanics(self) -> None: assert self.training_config is not None tc = self.training_config - self.timestep_shift = float( - tc.pipeline_config.flow_shift # type: ignore[union-attr] - ) - self.num_train_timestep = int( - self.noise_scheduler.num_train_timesteps - ) + self.timestep_shift = float(tc.pipeline_config.flow_shift # type: ignore[union-attr] + ) + self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) self.min_timestep = 0 self.max_timestep = self.num_train_timestep - def _sample_timesteps( - self, batch_size: int, device: torch.device - ) -> torch.Tensor: + def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: if self.noise_random_generator is None: - raise RuntimeError( - "on_train_start() must be called " - "before prepare_batch()" - ) + raise RuntimeError("on_train_start() must be called " + "before prepare_batch()") assert self.training_config is not None tc = self.training_config @@ -659,92 +480,54 @@ def _sample_timesteps( logit_std=tc.model.logit_std, mode_scale=tc.model.mode_scale, ) - indices = ( - u - * self.noise_scheduler.config.num_train_timesteps - ).long() - return self.noise_scheduler.timesteps[indices].to( - device=device - ) + indices = (u * self.noise_scheduler.config.num_train_timesteps).long() + return self.noise_scheduler.timesteps[indices].to(device=device) - def _build_attention_metadata( - self, training_batch: TrainingBatch - ) -> TrainingBatch: + def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: assert self.training_config is not None tc = self.training_config latents_shape = training_batch.raw_latent_shape patch_size = ( tc.pipeline_config.dit_config.patch_size # type: ignore[union-attr] ) - current_vsa_sparsity = ( - training_batch.current_vsa_sparsity - ) + current_vsa_sparsity = (training_batch.current_vsa_sparsity) assert latents_shape is not None assert training_batch.timesteps is not None - if ( - envs.FASTVIDEO_ATTENTION_BACKEND - == "VIDEO_SPARSE_ATTN" - ): - if ( - not is_vsa_available() - or VideoSparseAttentionMetadataBuilder - is None - ): - raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is " - "VIDEO_SPARSE_ATTN, but " - "fastvideo_kernel is not correctly " - "installed or detected." - ) + if (envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN"): + if (not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None): + raise ImportError("FASTVIDEO_ATTENTION_BACKEND is " + "VIDEO_SPARSE_ATTN, but " + "fastvideo_kernel is not correctly " + "installed or detected.") training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] raw_latent_shape=latents_shape[2:5], - current_timestep=( - training_batch.timesteps - ), + current_timestep=(training_batch.timesteps), patch_size=patch_size, VSA_sparsity=current_vsa_sparsity, device=self.device, ) - elif ( - envs.FASTVIDEO_ATTENTION_BACKEND - == "VMOBA_ATTN" - ): - if ( - not is_vmoba_available() - or VideoMobaAttentionMetadataBuilder - is None - ): - raise ImportError( - "FASTVIDEO_ATTENTION_BACKEND is " - "VMOBA_ATTN, but fastvideo_kernel " - "(or flash_attn>=2.7.4) is not " - "correctly installed." - ) + elif (envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN"): + if (not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None): + raise ImportError("FASTVIDEO_ATTENTION_BACKEND is " + "VMOBA_ATTN, but fastvideo_kernel " + "(or flash_attn>=2.7.4) is not " + "correctly installed.") moba_params = tc.model.moba_config.copy() - moba_params.update( - { - "current_timestep": ( - training_batch.timesteps - ), - "raw_latent_shape": ( - training_batch.raw_latent_shape[ - 2:5 - ] - ), - "patch_size": patch_size, - "device": self.device, - } - ) - training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(**moba_params) # type: ignore[misc] + moba_params.update({ + "current_timestep": (training_batch.timesteps), + "raw_latent_shape": (training_batch.raw_latent_shape[2:5]), + "patch_size": patch_size, + "device": self.device, + }) + training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(** + moba_params) # type: ignore[misc] else: training_batch.attn_metadata = None return training_batch - def _prepare_dit_inputs( - self, training_batch: TrainingBatch - ) -> TrainingBatch: + def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: assert self.training_config is not None tc = self.training_config latents = training_batch.latents @@ -752,10 +535,8 @@ def _prepare_dit_inputs( batch_size = latents.shape[0] if self.noise_gen_cuda is None: - raise RuntimeError( - "on_train_start() must be called " - "before prepare_batch()" - ) + raise RuntimeError("on_train_start() must be called " + "before prepare_batch()") noise = torch.randn( latents.shape, @@ -763,9 +544,7 @@ def _prepare_dit_inputs( device=latents.device, dtype=latents.dtype, ) - timesteps = self._sample_timesteps( - batch_size, latents.device - ) + timesteps = self._sample_timesteps(batch_size, latents.device) if int(tc.distributed.sp_size or 1) > 1: self.sp_group.broadcast(timesteps, src=0) @@ -776,36 +555,24 @@ def _prepare_dit_inputs( n_dim=latents.ndim, dtype=latents.dtype, ) - noisy_model_input = ( - (1.0 - sigmas) * latents + sigmas * noise - ) + noisy_model_input = ((1.0 - sigmas) * latents + sigmas * noise) - training_batch.noisy_model_input = ( - noisy_model_input - ) + training_batch.noisy_model_input = (noisy_model_input) training_batch.timesteps = timesteps training_batch.sigmas = sigmas training_batch.noise = noise training_batch.raw_latent_shape = latents.shape - training_batch.latents = ( - training_batch.latents.permute(0, 2, 1, 3, 4) - ) + training_batch.latents = (training_batch.latents.permute(0, 2, 1, 3, 4)) return training_batch - def _build_i2v_mask_latents( - self, image_latents: torch.Tensor - ) -> torch.Tensor: + def _build_i2v_mask_latents(self, image_latents: torch.Tensor) -> torch.Tensor: assert self.training_config is not None tc = self.training_config temporal_compression_ratio = ( tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio # type: ignore[union-attr] ) - num_frames = ( - (tc.data.num_latent_t - 1) - * temporal_compression_ratio - + 1 - ) + num_frames = ((tc.data.num_latent_t - 1) * temporal_compression_ratio + 1) ( batch_size, @@ -846,67 +613,42 @@ def _build_i2v_mask_latents( dtype=image_latents.dtype, ) - def _process_actions( - self, training_batch: TrainingBatch - ) -> tuple[ - torch.Tensor, torch.Tensor, torch.Tensor - ]: - keyboard_cond = getattr( - training_batch, "keyboard_cond", None - ) - mouse_cond = getattr( - training_batch, "mouse_cond", None - ) + def _process_actions(self, training_batch: TrainingBatch) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + keyboard_cond = getattr(training_batch, "keyboard_cond", None) + mouse_cond = getattr(training_batch, "mouse_cond", None) if keyboard_cond is None or mouse_cond is None: - raise ValueError( - "WanGame batch must provide " - "keyboard_cond and mouse_cond" - ) + raise ValueError("WanGame batch must provide " + "keyboard_cond and mouse_cond") from fastvideo.models.dits.hyworld.pose import ( - process_custom_actions, - ) + process_custom_actions, ) - batch_size = int( - training_batch.noisy_model_input.shape[0] # type: ignore[union-attr] - ) + batch_size = int(training_batch.noisy_model_input.shape[0] # type: ignore[union-attr] + ) viewmats_list: list[torch.Tensor] = [] intrinsics_list: list[torch.Tensor] = [] action_labels_list: list[torch.Tensor] = [] for b in range(batch_size): - v, i, a = process_custom_actions( - keyboard_cond[b], mouse_cond[b] - ) + v, i, a = process_custom_actions(keyboard_cond[b], mouse_cond[b]) viewmats_list.append(v) intrinsics_list.append(i) action_labels_list.append(a) - viewmats = torch.stack(viewmats_list, dim=0).to( - device=self.device, dtype=torch.bfloat16 - ) - intrinsics = torch.stack( - intrinsics_list, dim=0 - ).to(device=self.device, dtype=torch.bfloat16) - action_labels = torch.stack( - action_labels_list, dim=0 - ).to(device=self.device, dtype=torch.bfloat16) - - num_latent_t = int( - training_batch.noisy_model_input.shape[2] # type: ignore[union-attr] - ) + viewmats = torch.stack(viewmats_list, dim=0).to(device=self.device, dtype=torch.bfloat16) + intrinsics = torch.stack(intrinsics_list, dim=0).to(device=self.device, dtype=torch.bfloat16) + action_labels = torch.stack(action_labels_list, dim=0).to(device=self.device, dtype=torch.bfloat16) + + num_latent_t = int(training_batch.noisy_model_input.shape[2] # type: ignore[union-attr] + ) if int(action_labels.shape[1]) != num_latent_t: - raise ValueError( - "Action conditioning temporal dim " - "mismatch: " - f"action={tuple(action_labels.shape)} " - f"vs latent_t={num_latent_t}" - ) + raise ValueError("Action conditioning temporal dim " + "mismatch: " + f"action={tuple(action_labels.shape)} " + f"vs latent_t={num_latent_t}") if int(viewmats.shape[1]) != num_latent_t: - raise ValueError( - "Viewmats temporal dim mismatch: " - f"viewmats={tuple(viewmats.shape)} " - f"vs latent_t={num_latent_t}" - ) + raise ValueError("Viewmats temporal dim mismatch: " + f"viewmats={tuple(viewmats.shape)} " + f"vs latent_t={num_latent_t}") return viewmats, intrinsics, action_labels @@ -926,9 +668,7 @@ def _build_distill_input_kwargs( ) -> dict[str, Any]: hidden_states = torch.cat( [ - noisy_video_latents.permute( - 0, 2, 1, 3, 4 - ), + noisy_video_latents.permute(0, 2, 1, 3, 4), mask_lat_size, image_latents, ], @@ -937,9 +677,7 @@ def _build_distill_input_kwargs( return { "hidden_states": hidden_states, "encoder_hidden_states": None, - "timestep": timestep.to( - device=self.device, dtype=torch.bfloat16 - ), + "timestep": timestep.to(device=self.device, dtype=torch.bfloat16), "encoder_hidden_states_image": image_embeds, "viewmats": viewmats, "Ks": Ks, @@ -960,28 +698,20 @@ def _select_cfg_condition_inputs( image_latents = batch.image_latents mask_lat_size = batch.mask_lat_size if image_embeds is None: - raise RuntimeError( - "WanGameModel requires " - "TrainingBatch.image_embeds" - ) + raise RuntimeError("WanGameModel requires " + "TrainingBatch.image_embeds") if image_latents is None: - raise RuntimeError( - "WanGameModel requires " - "TrainingBatch.image_latents" - ) + raise RuntimeError("WanGameModel requires " + "TrainingBatch.image_latents") if mask_lat_size is None: - raise RuntimeError( - "WanGameModel requires " - "TrainingBatch.mask_lat_size" - ) + raise RuntimeError("WanGameModel requires " + "TrainingBatch.mask_lat_size") viewmats = getattr(batch, "viewmats", None) Ks = getattr(batch, "Ks", None) action = getattr(batch, "action", None) mouse_cond = getattr(batch, "mouse_cond", None) - keyboard_cond = getattr( - batch, "keyboard_cond", None - ) + keyboard_cond = getattr(batch, "keyboard_cond", None) if conditional or cfg_uncond is None: return { @@ -995,22 +725,16 @@ def _select_cfg_condition_inputs( "keyboard_cond": keyboard_cond, } - on_missing_raw = cfg_uncond.get( - "on_missing", "error" - ) + on_missing_raw = cfg_uncond.get("on_missing", "error") if not isinstance(on_missing_raw, str): - raise ValueError( - "method_config.cfg_uncond.on_missing " - "must be a string, got " - f"{type(on_missing_raw).__name__}" - ) + raise ValueError("method_config.cfg_uncond.on_missing " + "must be a string, got " + f"{type(on_missing_raw).__name__}") on_missing = on_missing_raw.strip().lower() if on_missing not in {"error", "ignore"}: - raise ValueError( - "method_config.cfg_uncond.on_missing " - "must be one of {error, ignore}, got " - f"{on_missing_raw!r}" - ) + raise ValueError("method_config.cfg_uncond.on_missing " + "must be one of {error, ignore}, got " + f"{on_missing_raw!r}") supported_channels = {"image", "action"} for channel, policy_raw in cfg_uncond.items(): @@ -1021,77 +745,57 @@ def _select_cfg_condition_inputs( if policy_raw is None: continue if not isinstance(policy_raw, str): - raise ValueError( - "method_config.cfg_uncond values " - "must be strings, got " - f"{channel}=" - f"{type(policy_raw).__name__}" - ) + raise ValueError("method_config.cfg_uncond values " + "must be strings, got " + f"{channel}=" + f"{type(policy_raw).__name__}") policy = policy_raw.strip().lower() if policy == "keep": continue if on_missing == "ignore": continue - raise ValueError( - "WanGameModel does not support " - "cfg_uncond channel " - f"{channel!r} (policy={policy!r}). " - "Set cfg_uncond.on_missing=ignore or " - "remove the channel." - ) + raise ValueError("WanGameModel does not support " + "cfg_uncond channel " + f"{channel!r} (policy={policy!r}). " + "Set cfg_uncond.on_missing=ignore or " + "remove the channel.") def _get_policy(channel: str) -> str: raw = cfg_uncond.get(channel, "keep") if raw is None: return "keep" if not isinstance(raw, str): - raise ValueError( - "method_config.cfg_uncond values " - "must be strings, got " - f"{channel}={type(raw).__name__}" - ) + raise ValueError("method_config.cfg_uncond values " + "must be strings, got " + f"{channel}={type(raw).__name__}") policy = raw.strip().lower() if policy not in {"keep", "zero", "drop"}: - raise ValueError( - "method_config.cfg_uncond values " - "must be one of " - "{keep, zero, drop}, got " - f"{channel}={raw!r}" - ) + raise ValueError("method_config.cfg_uncond values " + "must be one of " + "{keep, zero, drop}, got " + f"{channel}={raw!r}") return policy image_policy = _get_policy("image") if image_policy == "zero": image_embeds = torch.zeros_like(image_embeds) - image_latents = torch.zeros_like( - image_latents - ) - mask_lat_size = torch.zeros_like( - mask_lat_size - ) + image_latents = torch.zeros_like(image_latents) + mask_lat_size = torch.zeros_like(mask_lat_size) elif image_policy == "drop": - raise ValueError( - "cfg_uncond.image=drop is not supported " - "for WanGame I2V; use " - "cfg_uncond.image=zero or keep." - ) + raise ValueError("cfg_uncond.image=drop is not supported " + "for WanGame I2V; use " + "cfg_uncond.image=zero or keep.") action_policy = _get_policy("action") if action_policy == "zero": - if ( - viewmats is None - or Ks is None - or action is None - ): + if (viewmats is None or Ks is None or action is None): if on_missing == "ignore": pass else: - raise ValueError( - "cfg_uncond.action=zero requires " - "action conditioning tensors, " - "but TrainingBatch is missing " - "{viewmats, Ks, action}." - ) + raise ValueError("cfg_uncond.action=zero requires " + "action conditioning tensors, " + "but TrainingBatch is missing " + "{viewmats, Ks, action}.") else: viewmats = torch.zeros_like(viewmats) Ks = torch.zeros_like(Ks) @@ -1099,9 +803,7 @@ def _get_policy(channel: str) -> str: if mouse_cond is not None: mouse_cond = torch.zeros_like(mouse_cond) if keyboard_cond is not None: - keyboard_cond = torch.zeros_like( - keyboard_cond - ) + keyboard_cond = torch.zeros_like(keyboard_cond) elif action_policy == "drop": viewmats = None Ks = None @@ -1120,7 +822,5 @@ def _get_policy(channel: str) -> str: "keyboard_cond": keyboard_cond, } - def _get_transformer( - self, timestep: torch.Tensor - ) -> torch.nn.Module: + def _get_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: return self.transformer diff --git a/fastvideo/train/models/wangame/wangame_causal.py b/fastvideo/train/models/wangame/wangame_causal.py index 676614743..ac92362b9 100644 --- a/fastvideo/train/models/wangame/wangame_causal.py +++ b/fastvideo/train/models/wangame/wangame_causal.py @@ -11,8 +11,8 @@ from fastvideo.forward_context import set_forward_context from fastvideo.models.utils import pred_noise_to_pred_video -from fastvideo.distillation.models.base import CausalModelBase -from fastvideo.distillation.models.wangame.wangame import WanGameModel +from fastvideo.train.models.base import CausalModelBase +from fastvideo.train.models.wangame.wangame import WanGameModel @dataclass(slots=True) diff --git a/fastvideo/train/trainer.py b/fastvideo/train/trainer.py index d0cd0e190..46a638cf4 100644 --- a/fastvideo/train/trainer.py +++ b/fastvideo/train/trainer.py @@ -11,11 +11,11 @@ from tqdm.auto import tqdm from fastvideo.distributed import get_sp_group, get_world_group -from fastvideo.distillation.utils.tracking import build_tracker +from fastvideo.train.utils.tracking import build_tracker if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( - DistillTrainingConfig, ) + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) def _coerce_log_scalar(value: Any, *, where: str) -> float: @@ -37,11 +37,11 @@ class TrainLoopState: current_vsa_sparsity: float -class DistillTrainer: +class Trainer: def __init__( self, - training_config: DistillTrainingConfig, + training_config: TrainingConfig, *, config: dict[str, Any] | None = None, ) -> None: diff --git a/fastvideo/train/utils/__init__.py b/fastvideo/train/utils/__init__.py index 21b2ccb44..d7eba4033 100644 --- a/fastvideo/train/utils/__init__.py +++ b/fastvideo/train/utils/__init__.py @@ -1,4 +1,2 @@ # SPDX-License-Identifier: Apache-2.0 - """Distillation utilities shared across families/methods/entrypoints.""" - diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index 42239ef63..2802a7838 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -19,7 +19,7 @@ set_model_state_dict, ) -from fastvideo.distillation.models.base import ModelBase +from fastvideo.train.models.base import ModelBase from fastvideo.logger import init_logger from fastvideo.training.checkpointing_utils import ( ModelWrapper, @@ -30,14 +30,11 @@ logger = init_logger(__name__) - _CHECKPOINT_DIR_RE = re.compile(r"^checkpoint-(\d+)$") def _is_stateful(obj: Any) -> bool: - return callable(getattr(obj, "state_dict", None)) and callable( - getattr(obj, "load_state_dict", None) - ) + return callable(getattr(obj, "state_dict", None)) and callable(getattr(obj, "load_state_dict", None)) def _rank() -> int: @@ -54,10 +51,8 @@ def _barrier() -> None: def _parse_step_from_dir(checkpoint_dir: Path) -> int: match = _CHECKPOINT_DIR_RE.match(checkpoint_dir.name) if not match: - raise ValueError( - f"Invalid checkpoint directory name {checkpoint_dir.name!r}; " - "expected 'checkpoint-'" - ) + raise ValueError(f"Invalid checkpoint directory name {checkpoint_dir.name!r}; " + "expected 'checkpoint-'") return int(match.group(1)) @@ -114,11 +109,9 @@ def _resolve_resume_checkpoint(resume_from_checkpoint: str, *, output_dir: str) # Give a clearer error message. out = Path(os.path.expanduser(str(output_dir))).resolve() - raise ValueError( - "Could not resolve resume checkpoint. Expected a checkpoint directory " - f"named 'checkpoint-' (with 'dcp/' inside), or an output_dir " - f"containing such checkpoints. Got: {path} (output_dir={out})." - ) + raise ValueError("Could not resolve resume checkpoint. Expected a checkpoint directory " + f"named 'checkpoint-' (with 'dcp/' inside), or an output_dir " + f"containing such checkpoints. Got: {path} (output_dir={out}).") def _get_dcp_role_module_names(dcp_dir: Path, role: str) -> set[str]: @@ -147,9 +140,7 @@ def _get_dcp_role_module_names(dcp_dir: Path, role: str) -> set[str]: return set(str(name) for name in packed) -def _get_dcp_role_module_param_keys( - dcp_dir: Path, *, role: str, module_name: str -) -> set[str]: +def _get_dcp_role_module_param_keys(dcp_dir: Path, *, role: str, module_name: str) -> set[str]: """Return inner param keys present under `roles...*` in a DCP checkpoint. Example checkpoint key: @@ -170,7 +161,7 @@ def _get_dcp_role_module_param_keys( for key in metadata.state_dict_metadata: if not key.startswith(prefix): continue - inner = key[len(prefix) :] + inner = key[len(prefix):] if inner: keys.add(inner) packed: list[str] = sorted(keys) @@ -253,10 +244,8 @@ def maybe_warmstart_role_modules( handle = bundle.role(str(role)) modules = dict(handle.modules) else: - raise ValueError( - "maybe_warmstart_role_modules requires either " - "'model' or 'bundle'" - ) + raise ValueError("maybe_warmstart_role_modules requires either " + "'model' or 'bundle'") available_modules = _get_dcp_role_module_names(dcp_dir, role=str(checkpoint_role)) @@ -277,11 +266,9 @@ def maybe_warmstart_role_modules( ) if not states: - raise ValueError( - f"init_from_checkpoint={resolved} does not contain any saved modules for " - f"checkpoint_role={checkpoint_role!r}. Available modules in checkpoint: " - f"{sorted(available_modules)}" - ) + raise ValueError(f"init_from_checkpoint={resolved} does not contain any saved modules for " + f"checkpoint_role={checkpoint_role!r}. Available modules in checkpoint: " + f"{sorted(available_modules)}") if _rank() == 0: logger.info( @@ -327,10 +314,8 @@ def save_role_pretrained( if overwrite: shutil.rmtree(dst, ignore_errors=True) else: - raise FileExistsError( - f"Refusing to overwrite existing directory: {dst}. " - "Pass overwrite=True to replace it." - ) + raise FileExistsError(f"Refusing to overwrite existing directory: {dst}. " + "Pass overwrite=True to replace it.") def _copy_or_link(src: str, dest: str) -> None: try: @@ -352,26 +337,20 @@ def _copy_or_link(src: str, dest: str) -> None: handle = bundle.role(str(role)) modules = dict(handle.modules) else: - raise ValueError( - "save_role_pretrained requires either " - "'model' or 'bundle'" - ) + raise ValueError("save_role_pretrained requires either " + "'model' or 'bundle'") if module_names is None: module_names = sorted(modules.keys()) for module_name in module_names: if module_name not in modules: - raise KeyError( - f"Role {role!r} does not have module {module_name!r}. " - f"Available: {sorted(modules.keys())}" - ) + raise KeyError(f"Role {role!r} does not have module {module_name!r}. " + f"Available: {sorted(modules.keys())}") module_dir = dst / module_name if not module_dir.is_dir(): - raise FileNotFoundError( - f"Export directory missing component dir {module_name!r}: {module_dir}" - ) + raise FileNotFoundError(f"Export directory missing component dir {module_name!r}: {module_dir}") options = StateDictOptions(full_state_dict=True, cpu_offload=True) state_dict = get_model_state_dict(modules[module_name], options=options) @@ -384,10 +363,8 @@ def _copy_or_link(src: str, dest: str) -> None: tensor_state: dict[str, torch.Tensor] = {} for key, value in state_dict.items(): if not isinstance(value, torch.Tensor): - raise TypeError( - f"Expected tensor in state_dict for {module_name}.{key}, " - f"got {type(value).__name__}" - ) + raise TypeError(f"Expected tensor in state_dict for {module_name}.{key}, " + f"got {type(value).__name__}") tensor_state[key] = value.detach().cpu() from safetensors.torch import save_file @@ -420,12 +397,12 @@ def __init__(self, modules: dict[str, torch.nn.Module]) -> None: @dataclass(slots=True) -class DistillCheckpointConfig: +class CheckpointConfig: save_steps: int keep_last: int -class DistillCheckpointManager: +class CheckpointManager: """Role-based checkpoint manager for distillation runtime. - Checkpoint policy lives in YAML (via TrainingArgs fields). @@ -444,7 +421,7 @@ def __init__( lr_schedulers: dict[str, Any] | None = None, dataloader: Any, output_dir: str, - config: DistillCheckpointConfig, + config: CheckpointConfig, get_rng_generators: Callable[[], dict[str, torch.Generator]] | None = None, ) -> None: self.bundle = bundle @@ -458,7 +435,9 @@ def __init__( self._last_saved_step: int | None = None def _build_role_states_from_model( - self, role: str, model: ModelBase, + self, + role: str, + model: ModelBase, ) -> dict[str, Any]: if not getattr(model, "_trainable", False): return {} @@ -477,7 +456,8 @@ def _build_role_states_from_model( for name, optimizer in self.optimizers.items(): if name.startswith(f"{role}.") or name == role: states[f"optimizers.{name}"] = OptimizerWrapper( - container, optimizer, + container, + optimizer, ) for name, scheduler in self.lr_schedulers.items(): @@ -487,7 +467,9 @@ def _build_role_states_from_model( return states def _build_role_states_from_handle( - self, role: str, handle: Any, + self, + role: str, + handle: Any, ) -> dict[str, Any]: if not handle.trainable: return {} @@ -500,7 +482,8 @@ def _build_role_states_from_handle( for name, optimizer in handle.optimizers.items(): states[f"optimizers.{role}.{name}"] = OptimizerWrapper( - container, optimizer, + container, + optimizer, ) for name, scheduler in handle.lr_schedulers.items(): @@ -514,14 +497,10 @@ def _build_states(self) -> dict[str, Any]: # Models/opts/schedulers are role-scoped. if self.role_models: for role, model in self.role_models.items(): - states.update( - self._build_role_states_from_model(role, model) - ) + states.update(self._build_role_states_from_model(role, model)) elif self.bundle is not None: for role, handle in self.bundle.roles.items(): - states.update( - self._build_role_states_from_handle(role, handle) - ) + states.update(self._build_role_states_from_handle(role, handle)) # Dataloader (optional but recommended for exact resume). if _is_stateful(self.dataloader): diff --git a/fastvideo/train/utils/config.py b/fastvideo/train/utils/config.py index a02ab5ee6..f8c18ae19 100644 --- a/fastvideo/train/utils/config.py +++ b/fastvideo/train/utils/config.py @@ -11,10 +11,10 @@ import yaml -from fastvideo.distillation.utils.distill_config import ( +from fastvideo.train.utils.training_config import ( CheckpointConfig, DataConfig, - DistillTrainingConfig, + TrainingConfig, DistributedConfig, ModelTrainingConfig, OptimizerConfig, @@ -33,7 +33,7 @@ class RunConfig: models: dict[str, dict[str, Any]] method: dict[str, Any] - training: DistillTrainingConfig + training: TrainingConfig validation: dict[str, Any] raw: dict[str, Any] @@ -276,9 +276,7 @@ def _parse_pipeline_config(cfg: dict[str, Any], ) -> Any: return None -def _is_nested_training_format( - t: dict[str, Any], -) -> bool: +def _is_nested_training_format(t: dict[str, Any], ) -> bool: """Detect whether training: uses nested sub-groups.""" _nested_keys = { "distributed", @@ -298,8 +296,8 @@ def _build_training_config_nested( *, models: dict[str, dict[str, Any]], pipeline_config: Any, -) -> DistillTrainingConfig: - """Build DistillTrainingConfig from nested training: YAML.""" +) -> TrainingConfig: + """Build TrainingConfig from nested training: YAML.""" d = dict(t.get("distributed", {}) or {}) da = dict(t.get("data", {}) or {}) o = dict(t.get("optimizer", {}) or {}) @@ -322,7 +320,7 @@ def _build_training_config_nested( if init_from is not None: model_path = str(init_from) - return DistillTrainingConfig( + return TrainingConfig( distributed=DistributedConfig( num_gpus=num_gpus, tp_size=int(d.get("tp_size", 1) or 1), @@ -393,8 +391,8 @@ def _build_training_config_flat( *, models: dict[str, dict[str, Any]], pipeline_config: Any, -) -> DistillTrainingConfig: - """Build DistillTrainingConfig from flat training: YAML.""" +) -> TrainingConfig: + """Build TrainingConfig from flat training: YAML.""" num_gpus = int(t.get("num_gpus", 1) or 1) betas_raw = t.get("betas", "0.9,0.999") @@ -409,7 +407,7 @@ def _build_training_config_flat( if init_from is not None: model_path = str(init_from) - return DistillTrainingConfig( + return TrainingConfig( distributed=DistributedConfig( num_gpus=num_gpus, tp_size=int(t.get("tp_size", 1) or 1), @@ -480,8 +478,8 @@ def _build_training_config( *, models: dict[str, dict[str, Any]], pipeline_config: Any, -) -> DistillTrainingConfig: - """Build DistillTrainingConfig from training: YAML. +) -> TrainingConfig: + """Build TrainingConfig from training: YAML. Supports both nested (new) and flat (legacy) formats. """ @@ -489,10 +487,8 @@ def _build_training_config( t.pop("validation", None) if _is_nested_training_format(t): - return _build_training_config_nested( - t, models=models, pipeline_config=pipeline_config) - return _build_training_config_flat( - t, models=models, pipeline_config=pipeline_config) + return _build_training_config_nested(t, models=models, pipeline_config=pipeline_config) + return _build_training_config_flat(t, models=models, pipeline_config=pipeline_config) def load_run_config(path: str) -> RunConfig: diff --git a/fastvideo/train/utils/dataloader.py b/fastvideo/train/utils/dataloader.py index c70715235..9771db6d2 100644 --- a/fastvideo/train/utils/dataloader.py +++ b/fastvideo/train/utils/dataloader.py @@ -5,7 +5,7 @@ from typing import Any, TYPE_CHECKING if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( + from fastvideo.train.utils.training_config import ( DataConfig, ) diff --git a/fastvideo/train/utils/instantiate.py b/fastvideo/train/utils/instantiate.py index b99ca16f0..ed43122ee 100644 --- a/fastvideo/train/utils/instantiate.py +++ b/fastvideo/train/utils/instantiate.py @@ -1,5 +1,4 @@ # SPDX-License-Identifier: Apache-2.0 - """``_target_``-based instantiation utilities. These helpers resolve a dotted Python path to a class and instantiate it, @@ -21,35 +20,27 @@ def resolve_target(target: str) -> type: """Import and return the class (or callable) at *target*. *target* must be a fully-qualified dotted path, e.g. - ``"fastvideo.distillation.models.wangame.wangame.WanGameModel"``. + ``"fastvideo.train.models.wangame.wangame.WanGameModel"``. """ if not isinstance(target, str) or not target.strip(): - raise ValueError( - f"_target_ must be a non-empty dotted path string, " - f"got {target!r}" - ) + raise ValueError(f"_target_ must be a non-empty dotted path string, " + f"got {target!r}") target = target.strip() parts = target.rsplit(".", 1) if len(parts) != 2: - raise ValueError( - f"_target_ must contain at least one dot " - f"(module.ClassName), got {target!r}" - ) + raise ValueError(f"_target_ must contain at least one dot " + f"(module.ClassName), got {target!r}") module_path, attr_name = parts try: module = importlib.import_module(module_path) except ModuleNotFoundError as exc: - raise ImportError( - f"Cannot import module {module_path!r} " - f"(from _target_={target!r})" - ) from exc + raise ImportError(f"Cannot import module {module_path!r} " + f"(from _target_={target!r})") from exc try: cls = getattr(module, attr_name) except AttributeError as exc: - raise ImportError( - f"Module {module_path!r} has no attribute " - f"{attr_name!r} (from _target_={target!r})" - ) from exc + raise ImportError(f"Module {module_path!r} has no attribute " + f"{attr_name!r} (from _target_={target!r})") from exc return cls @@ -62,33 +53,24 @@ def instantiate(cfg: dict[str, Any], **extra: Any) -> Any: dropped, so callers can safely pass a superset. """ if not isinstance(cfg, dict): - raise TypeError( - f"instantiate() expects a dict with '_target_', " - f"got {type(cfg).__name__}" - ) + raise TypeError(f"instantiate() expects a dict with '_target_', " + f"got {type(cfg).__name__}") target_str = cfg.get("_target_") if target_str is None: raise KeyError("Config dict is missing '_target_' key") cls = resolve_target(str(target_str)) - kwargs: dict[str, Any] = { - k: v for k, v in cfg.items() if k != "_target_" - } + kwargs: dict[str, Any] = {k: v for k, v in cfg.items() if k != "_target_"} kwargs.update(extra) sig = inspect.signature(cls.__init__) params = sig.parameters - has_var_keyword = any( - p.kind == inspect.Parameter.VAR_KEYWORD - for p in params.values() - ) + has_var_keyword = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) if not has_var_keyword: valid_names = { name - for name, p in params.items() - if p.kind - in ( + for name, p in params.items() if p.kind in ( inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY, ) diff --git a/fastvideo/train/utils/module_state.py b/fastvideo/train/utils/module_state.py index 2b86085bb..6d28a005f 100644 --- a/fastvideo/train/utils/module_state.py +++ b/fastvideo/train/utils/module_state.py @@ -14,4 +14,3 @@ def apply_trainable(module: torch.nn.Module, *, trainable: bool) -> torch.nn.Mod else: module.eval() return module - diff --git a/fastvideo/train/utils/moduleloader.py b/fastvideo/train/utils/moduleloader.py index e2b731b96..3b0854ed1 100644 --- a/fastvideo/train/utils/moduleloader.py +++ b/fastvideo/train/utils/moduleloader.py @@ -17,9 +17,8 @@ ) if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( - DistillTrainingConfig, ) - + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) # ------------------------------------------------------------------ # TrainingArgs builders (only place that creates FastVideoArgs) @@ -27,7 +26,7 @@ def _make_training_args( - tc: DistillTrainingConfig, + tc: TrainingConfig, *, model_path: str, ) -> TrainingArgs: @@ -54,7 +53,7 @@ def _make_training_args( def make_inference_args( - tc: DistillTrainingConfig, + tc: TrainingConfig, *, model_path: str, ) -> TrainingArgs: @@ -75,18 +74,17 @@ def load_module_from_path( *, model_path: str, module_type: str, - training_config: DistillTrainingConfig | None = None, + training_config: TrainingConfig | None = None, disable_custom_init_weights: bool = False, override_transformer_cls_name: str | None = None, ) -> torch.nn.Module: """Load a single pipeline component module. - Accepts a ``DistillTrainingConfig`` and internally builds the + Accepts a ``TrainingConfig`` and internally builds the ``TrainingArgs`` needed by ``PipelineComponentLoader``. """ if training_config is not None: - fastvideo_args: Any = _make_training_args( - training_config, model_path=model_path) + fastvideo_args: Any = _make_training_args(training_config, model_path=model_path) else: from types import SimpleNamespace fastvideo_args = SimpleNamespace() @@ -113,8 +111,7 @@ def load_module_from_path( "override_transformer_cls_name", None, ) - fastvideo_args.override_transformer_cls_name = str( - override_transformer_cls_name) + fastvideo_args.override_transformer_cls_name = str(override_transformer_cls_name) if disable_custom_init_weights: fastvideo_args._loading_teacher_critic_model = True @@ -126,9 +123,7 @@ def load_module_from_path( fastvideo_args=fastvideo_args, ) finally: - if disable_custom_init_weights and hasattr( - fastvideo_args, - "_loading_teacher_critic_model"): + if disable_custom_init_weights and hasattr(fastvideo_args, "_loading_teacher_critic_model"): del fastvideo_args._loading_teacher_critic_model if override_transformer_cls_name is not None: if old_override is None: @@ -136,11 +131,9 @@ def load_module_from_path( fastvideo_args, "override_transformer_cls_name", ): - fastvideo_args.override_transformer_cls_name = ( - None) + fastvideo_args.override_transformer_cls_name = (None) else: - fastvideo_args.override_transformer_cls_name = ( - old_override) + fastvideo_args.override_transformer_cls_name = (old_override) if not isinstance(module, torch.nn.Module): raise TypeError(f"Loaded {module_type!r} is not a " diff --git a/fastvideo/train/utils/optimizer.py b/fastvideo/train/utils/optimizer.py index 27285796b..43a79d98d 100644 --- a/fastvideo/train/utils/optimizer.py +++ b/fastvideo/train/utils/optimizer.py @@ -12,7 +12,7 @@ ) if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( + from fastvideo.train.utils.training_config import ( OptimizerConfig, TrainingLoopConfig, ) diff --git a/fastvideo/train/utils/tracking.py b/fastvideo/train/utils/tracking.py index 4e7f23823..7ad28a2e4 100644 --- a/fastvideo/train/utils/tracking.py +++ b/fastvideo/train/utils/tracking.py @@ -12,7 +12,7 @@ ) if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( + from fastvideo.train.utils.training_config import ( CheckpointConfig, TrackerConfig, ) diff --git a/fastvideo/train/utils/training_config.py b/fastvideo/train/utils/training_config.py index a45afabff..da8bd9783 100644 --- a/fastvideo/train/utils/training_config.py +++ b/fastvideo/train/utils/training_config.py @@ -86,7 +86,7 @@ class ModelTrainingConfig: @dataclass(slots=True) -class DistillTrainingConfig: +class TrainingConfig: distributed: DistributedConfig = field(default_factory=DistributedConfig) data: DataConfig = field(default_factory=DataConfig) optimizer: OptimizerConfig = field(default_factory=OptimizerConfig) diff --git a/fastvideo/train/utils/validation.py b/fastvideo/train/utils/validation.py index 6f7cc4f0d..5d7722d97 100644 --- a/fastvideo/train/utils/validation.py +++ b/fastvideo/train/utils/validation.py @@ -4,7 +4,7 @@ from typing import Any, Literal, cast -from fastvideo.distillation.utils.config import get_optional_int +from fastvideo.train.utils.config import get_optional_int def is_validation_enabled(cfg: dict[str, Any]) -> bool: @@ -15,10 +15,8 @@ def is_validation_enabled(cfg: dict[str, Any]) -> bool: return True if isinstance(enabled, bool): return bool(enabled) - raise ValueError( - "training.validation.enabled must be a bool when set, got " - f"{type(enabled).__name__}" - ) + raise ValueError("training.validation.enabled must be a bool when set, got " + f"{type(enabled).__name__}") def parse_validation_every_steps(cfg: dict[str, Any]) -> int: @@ -33,10 +31,8 @@ def parse_validation_every_steps(cfg: dict[str, Any]) -> int: return int(raw) if isinstance(raw, str) and raw.strip(): return int(raw) - raise ValueError( - "training.validation.every_steps must be an int, got " - f"{type(raw).__name__}" - ) + raise ValueError("training.validation.every_steps must be an int, got " + f"{type(raw).__name__}") def parse_validation_dataset_file(cfg: dict[str, Any]) -> str: @@ -60,10 +56,8 @@ def parse_validation_sampling_steps(cfg: dict[str, Any]) -> list[int]: elif isinstance(raw, list): steps = [int(s) for s in raw] else: - raise ValueError( - "validation sampling_steps must be an int/list/str, got " - f"{type(raw).__name__}" - ) + raise ValueError("validation sampling_steps must be an int/list/str, got " + f"{type(raw).__name__}") return [s for s in steps if int(s) > 0] @@ -77,10 +71,8 @@ def parse_validation_guidance_scale(cfg: dict[str, Any]) -> float | None: return float(raw) if isinstance(raw, str) and raw.strip(): return float(raw) - raise ValueError( - "validation guidance_scale must be a number/string, got " - f"{type(raw).__name__}" - ) + raise ValueError("validation guidance_scale must be a number/string, got " + f"{type(raw).__name__}") def parse_validation_sampler_kind( @@ -92,16 +84,12 @@ def parse_validation_sampler_kind( if raw is None: raw = default if not isinstance(raw, str): - raise ValueError( - "training.validation.sampler_kind must be a string when set, got " - f"{type(raw).__name__}" - ) + raise ValueError("training.validation.sampler_kind must be a string when set, got " + f"{type(raw).__name__}") kind = raw.strip().lower() if kind not in {"ode", "sde"}: - raise ValueError( - "training.validation.sampler_kind must be one of {ode, sde}, got " - f"{raw!r}" - ) + raise ValueError("training.validation.sampler_kind must be one of {ode, sde}, got " + f"{raw!r}") return cast(Literal["ode", "sde"], kind) @@ -114,16 +102,12 @@ def parse_validation_rollout_mode( if raw is None: raw = default if not isinstance(raw, str): - raise ValueError( - "training.validation.rollout_mode must be a string when set, got " - f"{type(raw).__name__}" - ) + raise ValueError("training.validation.rollout_mode must be a string when set, got " + f"{type(raw).__name__}") mode = raw.strip().lower() if mode not in {"parallel", "streaming"}: - raise ValueError( - "training.validation.rollout_mode must be one of {parallel, streaming}, " - f"got {raw!r}" - ) + raise ValueError("training.validation.rollout_mode must be one of {parallel, streaming}, " + f"got {raw!r}") return cast(Literal["parallel", "streaming"], mode) @@ -136,24 +120,18 @@ def parse_validation_ode_solver( if raw in (None, ""): return None if sampler_kind != "ode": - raise ValueError( - "training.validation.ode_solver is only valid when " - "training.validation.sampler_kind='ode'" - ) + raise ValueError("training.validation.ode_solver is only valid when " + "training.validation.sampler_kind='ode'") if not isinstance(raw, str): - raise ValueError( - "training.validation.ode_solver must be a string when set, got " - f"{type(raw).__name__}" - ) + raise ValueError("training.validation.ode_solver must be a string when set, got " + f"{type(raw).__name__}") solver = raw.strip().lower() if solver in {"unipc", "unipc_multistep", "multistep"}: return "unipc" if solver in {"euler", "flowmatch", "flowmatch_euler"}: return "euler" - raise ValueError( - "training.validation.ode_solver must be one of {unipc, euler}, got " - f"{raw!r}" - ) + raise ValueError("training.validation.ode_solver must be one of {unipc, euler}, got " + f"{raw!r}") def parse_validation_output_dir(cfg: dict[str, Any]) -> str | None: @@ -161,10 +139,8 @@ def parse_validation_output_dir(cfg: dict[str, Any]) -> str | None: if raw is None: return None if not isinstance(raw, str): - raise ValueError( - "training.validation.output_dir must be a string when set, got " - f"{type(raw).__name__}" - ) + raise ValueError("training.validation.output_dir must be a string when set, got " + f"{type(raw).__name__}") return raw diff --git a/fastvideo/train/validators/__init__.py b/fastvideo/train/validators/__init__.py index 84dcab5c6..b50574b29 100644 --- a/fastvideo/train/validators/__init__.py +++ b/fastvideo/train/validators/__init__.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 -from fastvideo.distillation.validators.base import DistillValidator, ValidationRequest -from fastvideo.distillation.validators.wan import WanValidator +from fastvideo.train.validators.base import Validator, ValidationRequest +from fastvideo.train.validators.wan import WanValidator __all__ = [ - "DistillValidator", + "Validator", "ValidationRequest", "WanValidator", ] diff --git a/fastvideo/train/validators/base.py b/fastvideo/train/validators/base.py index ffd9797d4..dd89b9de1 100644 --- a/fastvideo/train/validators/base.py +++ b/fastvideo/train/validators/base.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from typing import Literal -from fastvideo.distillation.models.base import ModelBase +from fastvideo.train.models.base import ModelBase @dataclass(slots=True) @@ -32,7 +32,8 @@ class ValidationRequest: output_dir: str | None = None -class DistillValidator(ABC): +class Validator(ABC): + @abstractmethod def log_validation(self, step: int, *, request: ValidationRequest | None = None) -> None: raise NotImplementedError diff --git a/fastvideo/train/validators/wan.py b/fastvideo/train/validators/wan.py index dcdbd4765..b4cd502a7 100644 --- a/fastvideo/train/validators/wan.py +++ b/fastvideo/train/validators/wan.py @@ -2,7 +2,7 @@ """Wan validator (model-family validation backend). Config keys used: -- `training` (DistillTrainingConfig): +- `training` (TrainingConfig): - `data.seed`, `model_path` - `data.num_height`, `data.num_width`, `data.num_latent_t` - `distributed.tp_size`, `distributed.sp_size`, @@ -37,17 +37,17 @@ from fastvideo.distributed import get_sp_group, get_world_group from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch -from fastvideo.distillation.utils.moduleloader import ( +from fastvideo.train.utils.moduleloader import ( make_inference_args, ) -from fastvideo.distillation.validators.base import ( +from fastvideo.train.validators.base import ( ValidationRequest, ) from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline from fastvideo.training.trackers import DummyTracker from fastvideo.utils import shallow_asdict if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( - DistillTrainingConfig, ) + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) logger = init_logger(__name__) @@ -64,7 +64,7 @@ class WanValidator: def __init__( self, *, - training_config: DistillTrainingConfig, + training_config: TrainingConfig, tracker: Any | None = None, ) -> None: self.training_config = training_config diff --git a/fastvideo/train/validators/wangame.py b/fastvideo/train/validators/wangame.py index c87228b48..996cb8a0f 100644 --- a/fastvideo/train/validators/wangame.py +++ b/fastvideo/train/validators/wangame.py @@ -2,7 +2,7 @@ """WanGame validator (model-family validation backend). Config keys used: -- `training` (DistillTrainingConfig): +- `training` (TrainingConfig): - `data.seed`, `model_path` - `data.num_height`, `data.num_width`, `data.num_latent_t` - `distributed.tp_size`, `distributed.sp_size`, @@ -38,16 +38,16 @@ from fastvideo.distributed import get_sp_group, get_world_group from fastvideo.logger import init_logger from fastvideo.pipelines import ForwardBatch -from fastvideo.distillation.utils.moduleloader import ( +from fastvideo.train.utils.moduleloader import ( make_inference_args, ) -from fastvideo.distillation.validators.base import ( +from fastvideo.train.validators.base import ( ValidationRequest, ) from fastvideo.training.trackers import DummyTracker from fastvideo.utils import shallow_asdict if TYPE_CHECKING: - from fastvideo.distillation.utils.distill_config import ( - DistillTrainingConfig, ) + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) logger = init_logger(__name__) @@ -64,7 +64,7 @@ class WanGameValidator: def __init__( self, *, - training_config: DistillTrainingConfig, + training_config: TrainingConfig, tracker: Any | None = None, ) -> None: self.training_config = training_config diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index f82a77335..e2f5b8dc7 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -23,14 +23,14 @@ def run_distillation_from_config( from fastvideo.distributed import ( maybe_init_distributed_environment_and_model_parallel, ) - from fastvideo.distillation import DistillTrainer - from fastvideo.distillation.utils.checkpoint import ( - DistillCheckpointConfig, - DistillCheckpointManager, + from fastvideo.train import Trainer + from fastvideo.train.utils.checkpoint import ( + CheckpointConfig, + CheckpointManager, ) - from fastvideo.distillation.dispatch import ( + from fastvideo.train.dispatch import ( build_from_config, ) - from fastvideo.distillation.utils.config import ( + from fastvideo.train.utils.config import ( load_run_config, ) cfg = load_run_config(config_path) @@ -53,7 +53,7 @@ def run_distillation_from_config( "build_from_config succeeded.") return - trainer = DistillTrainer(tc, config=cfg.raw) + trainer = Trainer(tc, config=cfg.raw) # Attach the exact YAML used for this run to the # tracker (e.g., W&B Files). @@ -62,7 +62,7 @@ def run_distillation_from_config( name="run.yaml", ) - ckpt_config = DistillCheckpointConfig( + ckpt_config = CheckpointConfig( save_steps=int(tc.checkpoint.training_state_checkpointing_steps or 0), keep_last=int(tc.checkpoint.checkpoints_total_limit or 0), ) @@ -74,7 +74,7 @@ def run_distillation_from_config( if not callable(get_rng_generators): get_rng_generators = None - checkpoint_manager = DistillCheckpointManager( + checkpoint_manager = CheckpointManager( role_models=(getattr(method, '_role_models', None) or {}), optimizers=(getattr(method, '_optimizer_dict', None) or {}), lr_schedulers=(getattr(method, '_lr_scheduler_dict', None) or {}), diff --git a/visualize_trajectory.py b/visualize_trajectory.py index 505303190..a0eafd048 100644 --- a/visualize_trajectory.py +++ b/visualize_trajectory.py @@ -73,13 +73,29 @@ def _decode_with_vae(vae, latents: torch.Tensor, *, device: torch.device, return (decoded / 2 + 0.5).clamp(0, 1) + def main(): - parser = argparse.ArgumentParser(description="Visualize Trajectory from Parquet file") - parser.add_argument("--parquet_path", type=str, required=True, help="Path to the input parquet file") - parser.add_argument("--model_path", type=str, required=True, help="Path to the model directory") - parser.add_argument("--output_dir", type=str, default="visualizations", help="Directory to save output videos") - parser.add_argument("--num_samples", type=int, default=1, help="Number of samples to visualize") - parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu") + parser = argparse.ArgumentParser( + description="Visualize Trajectory from Parquet file") + parser.add_argument("--parquet_path", + type=str, + required=True, + help="Path to the input parquet file") + parser.add_argument("--model_path", + type=str, + required=True, + help="Path to the model directory") + parser.add_argument("--output_dir", + type=str, + default="visualizations", + help="Directory to save output videos") + parser.add_argument("--num_samples", + type=int, + default=1, + help="Number of samples to visualize") + parser.add_argument("--device", + type=str, + default="cuda" if torch.cuda.is_available() else "cpu") parser.add_argument("--vae_precision", type=str, default="fp32", @@ -97,21 +113,23 @@ def main(): help= "Which trajectory steps to decode: 'last', 'all', or comma-separated indices (e.g. '0,10,20')", ) - + args = parser.parse_args() device = torch.device(args.device) print(f"Using device: {device}, vae_precision: {args.vae_precision}") - + os.makedirs(args.output_dir, exist_ok=True) - + # Load VAE (must load weights; creating AutoencoderKLWan(config) alone leaves random weights) print(f"Loading model from {args.model_path}...") model_root = maybe_download_model(args.model_path) pipeline_config = PipelineConfig.from_pretrained(model_root) pipeline_config.update_config_from_dict({ - "vae_precision": args.vae_precision, - "vae_config": WanVAEConfig(load_encoder=False, load_decoder=True), + "vae_precision": + args.vae_precision, + "vae_config": + WanVAEConfig(load_encoder=False, load_decoder=True), }) fastvideo_args = FastVideoArgs( model_path=model_root, @@ -125,44 +143,45 @@ def main(): vae_path = os.path.join(model_root, args.vae_subfolder) vae = VAELoader().load(vae_path, fastvideo_args) vae.to(device) - + # Read Parquet print(f"Reading parquet file: {args.parquet_path}") table = pq.read_table(args.parquet_path) - + # Iterate over rows num_visualized = 0 - + pbar = tqdm(total=min(args.num_samples, table.num_rows)) - + for i in range(table.num_rows): if num_visualized >= args.num_samples: break - + row = table.slice(i, length=1) record = row.to_pydict() - + video_id = record["id"][0] - + # Parse Latents shape = record["trajectory_latents_shape"][0] dtype = record["trajectory_latents_dtype"][0] dtype = np.dtype(dtype) - + latents_bytes = record["trajectory_latents_bytes"][0] # Copy to avoid read-only warning - latents_np = np.copy(np.frombuffer(latents_bytes, dtype=dtype).reshape(shape)) - + latents_np = np.copy( + np.frombuffer(latents_bytes, dtype=dtype).reshape(shape)) + latents_tensor = torch.from_numpy(latents_np) if latents_tensor.ndim == 6 and latents_tensor.shape[0] == 1: latents_tensor = latents_tensor.squeeze(0) - + print(f"Decoding video {video_id} with shape {latents_tensor.shape}...") - + # create subfolder vid_output_dir = os.path.join(args.output_dir, str(video_id)) os.makedirs(vid_output_dir, exist_ok=True) - + # Pick steps to decode steps = latents_tensor.shape[0] if args.decode_steps == "last": @@ -170,18 +189,24 @@ def main(): elif args.decode_steps == "all": indices_to_decode = list(range(steps)) else: - indices_to_decode = [int(x) for x in args.decode_steps.split(",") if x.strip() != ""] + indices_to_decode = [ + int(x) for x in args.decode_steps.split(",") if x.strip() != "" + ] indices_to_decode = [i for i in indices_to_decode if 0 <= i < steps] if not indices_to_decode: - raise ValueError(f"No valid indices selected for decode_steps='{args.decode_steps}' with steps={steps}") - - for step in tqdm(indices_to_decode, desc=f"Decoding {video_id}", leave=False): + raise ValueError( + f"No valid indices selected for decode_steps='{args.decode_steps}' with steps={steps}" + ) + + for step in tqdm(indices_to_decode, + desc=f"Decoding {video_id}", + leave=False): latent_step = latents_tensor[step].unsqueeze(0) # [1, C, T, H, W] decoded_video = _decode_with_vae(vae, - latent_step, - device=device, - precision=args.vae_precision) + latent_step, + device=device, + precision=args.vae_precision) save_path = os.path.join(vid_output_dir, f"step_{step:03d}.mp4") save_decoded_latents_as_video(decoded_video.float(), @@ -191,8 +216,9 @@ def main(): print(f"Saved {len(indices_to_decode)} step(s) to {vid_output_dir}") num_visualized += 1 pbar.update(1) - + pbar.close() + if __name__ == "__main__": main() From d6a101f1c49ffba1415cad87fa8282c96d356011 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 22:55:44 +0000 Subject: [PATCH 179/214] simplify. only nested config is allowed --- fastvideo/train/utils/config.py | 423 +++++++++++--------------------- 1 file changed, 150 insertions(+), 273 deletions(-) diff --git a/fastvideo/train/utils/config.py b/fastvideo/train/utils/config.py index f8c18ae19..11c79a5ec 100644 --- a/fastvideo/train/utils/config.py +++ b/fastvideo/train/utils/config.py @@ -1,13 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 -"""Distillation run config (v3 — ``_target_`` based).""" +"""Training run config (``_target_`` based YAML).""" from __future__ import annotations import os -import warnings from dataclasses import dataclass from pathlib import Path -from typing import Any, TYPE_CHECKING +from typing import Any import yaml @@ -23,13 +22,10 @@ VSAConfig, ) -if TYPE_CHECKING: - pass - @dataclass(slots=True) class RunConfig: - """Parsed distillation run config loaded from v3 YAML.""" + """Parsed run config loaded from YAML.""" models: dict[str, dict[str, Any]] method: dict[str, Any] @@ -66,15 +62,6 @@ def _require_str(raw: Any, *, where: str) -> str: return raw -def _get_bool(raw: Any, *, where: str, default: bool) -> bool: - if raw is None: - return default - if isinstance(raw, bool): - return raw - raise ValueError(f"Expected bool at {where}, " - f"got {type(raw).__name__}") - - def get_optional_int(mapping: dict[str, Any], key: str, *, where: str) -> int | None: raw = mapping.get(key) if raw is None: @@ -232,66 +219,39 @@ def require_bool( return raw -def _parse_pipeline_config(cfg: dict[str, Any], ) -> Any: - """Resolve PipelineConfig from top-level YAML keys.""" +def _parse_pipeline_config( + cfg: dict[str, Any], + *, + models: dict[str, dict[str, Any]], +) -> Any: + """Resolve PipelineConfig from the ``pipeline:`` YAML key.""" from fastvideo.configs.pipelines.base import PipelineConfig pipeline_raw = cfg.get("pipeline") - default_pipeline_cfg_raw = cfg.get("default_pipeline_config") - default_pipeline_cfg_path = cfg.get("default_pipeline_config_path") - pipeline_cfg_raw = cfg.get("pipeline_config") - pipeline_cfg_path = cfg.get("pipeline_config_path") - - if pipeline_raw is not None: - if default_pipeline_cfg_raw is not None: - raise ValueError("Provide either 'pipeline:' or " - "'default_pipeline_config:', not both") - default_pipeline_cfg_raw = pipeline_raw - - if (default_pipeline_cfg_raw is not None or default_pipeline_cfg_path - is not None) and (pipeline_cfg_raw is not None or pipeline_cfg_path is not None): - raise ValueError("Provide either default_pipeline_config(_path) " - "or the legacy pipeline_config(_path), not both") - - cfg_raw = (default_pipeline_cfg_raw if default_pipeline_cfg_raw is not None else pipeline_cfg_raw) - cfg_path = (default_pipeline_cfg_path if default_pipeline_cfg_path is not None else pipeline_cfg_path) - - if cfg_path is not None: - cfg_path = _require_str( - cfg_path, - where=("default_pipeline_config_path" if default_pipeline_cfg_path is not None else "pipeline_config_path"), - ) - return PipelineConfig.from_kwargs({ - "pipeline_config": _resolve_existing_file(cfg_path), - }) - if cfg_raw is not None: - if isinstance(cfg_raw, str): - return PipelineConfig.from_kwargs({ - "pipeline_config": _resolve_existing_file(cfg_raw), - }) - if isinstance(cfg_raw, dict): - return PipelineConfig.from_kwargs({"pipeline_config": cfg_raw}) - raise ValueError("default_pipeline_config must be a mapping " - "or a path string") - return None - - -def _is_nested_training_format(t: dict[str, Any], ) -> bool: - """Detect whether training: uses nested sub-groups.""" - _nested_keys = { - "distributed", - "data", - "optimizer", - "loop", - "checkpoint", - "tracker", - "vsa", - "model", - } - return bool(_nested_keys & set(t)) - - -def _build_training_config_nested( + if pipeline_raw is None: + return None + + # Derive model_path from models.student.init_from — + # needed by PipelineConfig.from_kwargs. + model_path: str | None = None + student_cfg = models.get("student") + if student_cfg is not None: + init_from = student_cfg.get("init_from") + if init_from is not None: + model_path = str(init_from) + + kwargs: dict[str, Any] = {"pipeline_config": pipeline_raw} + if model_path is not None: + kwargs["model_path"] = model_path + + if isinstance(pipeline_raw, str): + kwargs["pipeline_config"] = _resolve_existing_file( + pipeline_raw) + + return PipelineConfig.from_kwargs(kwargs) + + +def _build_training_config( t: dict[str, Any], *, models: dict[str, dict[str, Any]], @@ -310,7 +270,8 @@ def _build_training_config_nested( num_gpus = int(d.get("num_gpus", 1) or 1) betas_raw = o.get("betas", "0.9,0.999") - betas = parse_betas(betas_raw, where="training.optimizer.betas") + betas = parse_betas(betas_raw, + where="training.optimizer.betas") model_path = str(t.get("model_path", "") or "") if not model_path: @@ -324,254 +285,170 @@ def _build_training_config_nested( distributed=DistributedConfig( num_gpus=num_gpus, tp_size=int(d.get("tp_size", 1) or 1), - sp_size=int(d.get("sp_size", num_gpus) or num_gpus), - hsdp_replicate_dim=int(d.get("hsdp_replicate_dim", 1) or 1), - hsdp_shard_dim=int(d.get("hsdp_shard_dim", num_gpus) or num_gpus), - pin_cpu_memory=bool(d.get("pin_cpu_memory", False)), + sp_size=int( + d.get("sp_size", num_gpus) or num_gpus), + hsdp_replicate_dim=int( + d.get("hsdp_replicate_dim", 1) or 1), + hsdp_shard_dim=int( + d.get("hsdp_shard_dim", num_gpus) + or num_gpus), + pin_cpu_memory=bool( + d.get("pin_cpu_memory", False)), ), data=DataConfig( data_path=str(da.get("data_path", "") or ""), - train_batch_size=int(da.get("train_batch_size", 1) or 1), - dataloader_num_workers=int(da.get("dataloader_num_workers", 0) or 0), - training_cfg_rate=float(da.get("training_cfg_rate", 0.0) or 0.0), + train_batch_size=int( + da.get("train_batch_size", 1) or 1), + dataloader_num_workers=int( + da.get("dataloader_num_workers", 0) or 0), + training_cfg_rate=float( + da.get("training_cfg_rate", 0.0) or 0.0), seed=int(da.get("seed", 0) or 0), - num_height=int(da.get("num_height", 0) or 0), + num_height=int( + da.get("num_height", 0) or 0), num_width=int(da.get("num_width", 0) or 0), - num_latent_t=int(da.get("num_latent_t", 0) or 0), - num_frames=int(da.get("num_frames", 0) or 0), + num_latent_t=int( + da.get("num_latent_t", 0) or 0), + num_frames=int( + da.get("num_frames", 0) or 0), ), optimizer=OptimizerConfig( - learning_rate=float(o.get("learning_rate", 0.0) or 0.0), + learning_rate=float( + o.get("learning_rate", 0.0) or 0.0), betas=betas, - weight_decay=float(o.get("weight_decay", 0.0) or 0.0), - lr_scheduler=str(o.get("lr_scheduler", "constant") or "constant"), - lr_warmup_steps=int(o.get("lr_warmup_steps", 0) or 0), - lr_num_cycles=int(o.get("lr_num_cycles", 0) or 0), - lr_power=float(o.get("lr_power", 0.0) or 0.0), - min_lr_ratio=float(o.get("min_lr_ratio", 0.5) or 0.5), - max_grad_norm=float(o.get("max_grad_norm", 0.0) or 0.0), + weight_decay=float( + o.get("weight_decay", 0.0) or 0.0), + lr_scheduler=str( + o.get("lr_scheduler", "constant") + or "constant"), + lr_warmup_steps=int( + o.get("lr_warmup_steps", 0) or 0), + lr_num_cycles=int( + o.get("lr_num_cycles", 0) or 0), + lr_power=float( + o.get("lr_power", 0.0) or 0.0), + min_lr_ratio=float( + o.get("min_lr_ratio", 0.5) or 0.5), + max_grad_norm=float( + o.get("max_grad_norm", 0.0) or 0.0), ), loop=TrainingLoopConfig( - max_train_steps=int(lo.get("max_train_steps", 0) or 0), - gradient_accumulation_steps=int(lo.get("gradient_accumulation_steps", 1) or 1), + max_train_steps=int( + lo.get("max_train_steps", 0) or 0), + gradient_accumulation_steps=int( + lo.get("gradient_accumulation_steps", 1) + or 1), ), checkpoint=CheckpointConfig( - output_dir=str(ck.get("output_dir", "") or ""), - resume_from_checkpoint=str(ck.get("resume_from_checkpoint", "") or ""), - training_state_checkpointing_steps=int(ck.get("training_state_checkpointing_steps", 0) or 0), - checkpoints_total_limit=int(ck.get("checkpoints_total_limit", 0) or 0), + output_dir=str( + ck.get("output_dir", "") or ""), + resume_from_checkpoint=str( + ck.get("resume_from_checkpoint", "") + or ""), + training_state_checkpointing_steps=int( + ck.get( + "training_state_checkpointing_steps", + 0) or 0), + checkpoints_total_limit=int( + ck.get("checkpoints_total_limit", 0) + or 0), ), tracker=TrackerConfig( - trackers=list(tr.get("trackers", []) or []), - project_name=str(tr.get("project_name", "fastvideo") or "fastvideo"), + trackers=list( + tr.get("trackers", []) or []), + project_name=str( + tr.get("project_name", "fastvideo") + or "fastvideo"), run_name=str(tr.get("run_name", "") or ""), ), vsa=VSAConfig( - sparsity=float(vs.get("sparsity", 0.0) or 0.0), - decay_rate=float(vs.get("decay_rate", 0.0) or 0.0), - decay_interval_steps=int(vs.get("decay_interval_steps", 0) or 0), + sparsity=float( + vs.get("sparsity", 0.0) or 0.0), + decay_rate=float( + vs.get("decay_rate", 0.0) or 0.0), + decay_interval_steps=int( + vs.get("decay_interval_steps", 0) or 0), ), model=ModelTrainingConfig( - weighting_scheme=str(m.get("weighting_scheme", "uniform") or "uniform"), - logit_mean=float(m.get("logit_mean", 0.0) or 0.0), - logit_std=float(m.get("logit_std", 1.0) or 1.0), - mode_scale=float(m.get("mode_scale", 1.0) or 1.0), - precondition_outputs=bool(m.get("precondition_outputs", False)), - moba_config=dict(m.get("moba_config", {}) or {}), - enable_gradient_checkpointing_type=(m.get("enable_gradient_checkpointing_type")), + weighting_scheme=str( + m.get("weighting_scheme", "uniform") + or "uniform"), + logit_mean=float( + m.get("logit_mean", 0.0) or 0.0), + logit_std=float( + m.get("logit_std", 1.0) or 1.0), + mode_scale=float( + m.get("mode_scale", 1.0) or 1.0), + precondition_outputs=bool( + m.get("precondition_outputs", False)), + moba_config=dict( + m.get("moba_config", {}) or {}), + enable_gradient_checkpointing_type=( + m.get( + "enable_gradient_checkpointing_type" + )), ), pipeline_config=pipeline_config, model_path=model_path, - dit_precision=str(t.get("dit_precision", "fp32") or "fp32"), + dit_precision=str( + t.get("dit_precision", "fp32") or "fp32"), ) -def _build_training_config_flat( - t: dict[str, Any], - *, - models: dict[str, dict[str, Any]], - pipeline_config: Any, -) -> TrainingConfig: - """Build TrainingConfig from flat training: YAML.""" - num_gpus = int(t.get("num_gpus", 1) or 1) - - betas_raw = t.get("betas", "0.9,0.999") - betas = parse_betas(betas_raw, where="training.betas") - - # Use the student model path as default model_path. - model_path = str(t.get("model_path", "") or "") - if not model_path: - student_cfg = models.get("student") - if student_cfg is not None: - init_from = student_cfg.get("init_from") - if init_from is not None: - model_path = str(init_from) - - return TrainingConfig( - distributed=DistributedConfig( - num_gpus=num_gpus, - tp_size=int(t.get("tp_size", 1) or 1), - sp_size=int(t.get("sp_size", num_gpus) or num_gpus), - hsdp_replicate_dim=int(t.get("hsdp_replicate_dim", 1) or 1), - hsdp_shard_dim=int(t.get("hsdp_shard_dim", num_gpus) or num_gpus), - pin_cpu_memory=bool(t.get("pin_cpu_memory", False)), - ), - data=DataConfig( - data_path=str(t.get("data_path", "") or ""), - train_batch_size=int(t.get("train_batch_size", 1) or 1), - dataloader_num_workers=int(t.get("dataloader_num_workers", 0) or 0), - training_cfg_rate=float(t.get("training_cfg_rate", 0.0) or 0.0), - seed=int(t.get("seed", 0) or 0), - num_height=int(t.get("num_height", 0) or 0), - num_width=int(t.get("num_width", 0) or 0), - num_latent_t=int(t.get("num_latent_t", 0) or 0), - num_frames=int(t.get("num_frames", 0) or 0), - ), - optimizer=OptimizerConfig( - learning_rate=float(t.get("learning_rate", 0.0) or 0.0), - betas=betas, - weight_decay=float(t.get("weight_decay", 0.0) or 0.0), - lr_scheduler=str(t.get("lr_scheduler", "constant") or "constant"), - lr_warmup_steps=int(t.get("lr_warmup_steps", 0) or 0), - lr_num_cycles=int(t.get("lr_num_cycles", 0) or 0), - lr_power=float(t.get("lr_power", 0.0) or 0.0), - min_lr_ratio=float(t.get("min_lr_ratio", 0.5) or 0.5), - max_grad_norm=float(t.get("max_grad_norm", 0.0) or 0.0), - ), - loop=TrainingLoopConfig( - max_train_steps=int(t.get("max_train_steps", 0) or 0), - gradient_accumulation_steps=int(t.get("gradient_accumulation_steps", 1) or 1), - ), - checkpoint=CheckpointConfig( - output_dir=str(t.get("output_dir", "") or ""), - resume_from_checkpoint=str(t.get("resume_from_checkpoint", "") or ""), - training_state_checkpointing_steps=int(t.get("training_state_checkpointing_steps", 0) or 0), - checkpoints_total_limit=int(t.get("checkpoints_total_limit", 0) or 0), - ), - tracker=TrackerConfig( - trackers=list(t.get("trackers", []) or []), - project_name=str(t.get("tracker_project_name", "fastvideo") or "fastvideo"), - run_name=str(t.get("wandb_run_name", "") or ""), - ), - vsa=VSAConfig( - sparsity=float(t.get("VSA_sparsity", 0.0) or 0.0), - decay_rate=float(t.get("VSA_decay_rate", 0.0) or 0.0), - decay_interval_steps=int(t.get("VSA_decay_interval_steps", 0) or 0), - ), - model=ModelTrainingConfig( - weighting_scheme=str(t.get("weighting_scheme", "uniform") or "uniform"), - logit_mean=float(t.get("logit_mean", 0.0) or 0.0), - logit_std=float(t.get("logit_std", 1.0) or 1.0), - mode_scale=float(t.get("mode_scale", 1.0) or 1.0), - precondition_outputs=bool(t.get("precondition_outputs", False)), - moba_config=dict(t.get("moba_config", {}) or {}), - enable_gradient_checkpointing_type=(t.get("enable_gradient_checkpointing_type")), - ), - pipeline_config=pipeline_config, - model_path=model_path, - dit_precision=str(t.get("dit_precision", "fp32") or "fp32"), - ) - - -def _build_training_config( - training_raw: dict[str, Any], - *, - models: dict[str, dict[str, Any]], - pipeline_config: Any, -) -> TrainingConfig: - """Build TrainingConfig from training: YAML. - - Supports both nested (new) and flat (legacy) formats. - """ - t = dict(training_raw) - t.pop("validation", None) - - if _is_nested_training_format(t): - return _build_training_config_nested(t, models=models, pipeline_config=pipeline_config) - return _build_training_config_flat(t, models=models, pipeline_config=pipeline_config) - - def load_run_config(path: str) -> RunConfig: - """Load a distillation run config from v3 YAML. + """Load a run config from YAML. - V3 format uses ``models:`` with ``_target_`` per role and - ``method:`` with ``_target_`` for the algorithm class. + Expected top-level keys: ``models``, ``method``, + ``training`` (nested), and optionally ``validation`` + and ``pipeline``. """ path = _resolve_existing_file(path) with open(path, encoding="utf-8") as f: raw = yaml.safe_load(f) cfg = _require_mapping(raw, where=path) - # --- models section --- - models_raw = _require_mapping(cfg.get("models"), where="models") + # --- models --- + models_raw = _require_mapping( + cfg.get("models"), where="models") models: dict[str, dict[str, Any]] = {} for role, model_cfg_raw in models_raw.items(): - role_str = _require_str(role, where="models.") - model_cfg = _require_mapping(model_cfg_raw, where=f"models.{role_str}") + role_str = _require_str( + role, where="models.") + model_cfg = _require_mapping( + model_cfg_raw, where=f"models.{role_str}") if "_target_" not in model_cfg: - raise ValueError(f"models.{role_str} must have a " - "'_target_' key") + raise ValueError( + f"models.{role_str} must have a " + "'_target_' key") models[role_str] = dict(model_cfg) - # --- method section --- - method_raw = _require_mapping(cfg.get("method"), where="method") + # --- method --- + method_raw = _require_mapping( + cfg.get("method"), where="method") if "_target_" not in method_raw: - raise ValueError("method must have a '_target_' key") + raise ValueError( + "method must have a '_target_' key") method = dict(method_raw) - # --- backward compat: merge method_config --- - method_config_raw = cfg.get("method_config", None) - if method_config_raw is not None: - warnings.warn( - "The top-level 'method_config:' section is " - "deprecated. Move its keys into 'method:' " - "directly.", - DeprecationWarning, - stacklevel=2, - ) - mc = _require_mapping(method_config_raw, where="method_config") - for k, v in mc.items(): - if k in method and k != "_target_": - warnings.warn( - f"method_config.{k} overrides " - f"method.{k} — prefer using " - "method: only", - DeprecationWarning, - stacklevel=2, - ) - method.setdefault(k, v) - - # --- validation section --- + # --- validation --- validation_raw = cfg.get("validation", None) - training_raw = _require_mapping(cfg.get("training"), where="training") - - training_validation_raw = training_raw.get("validation", None) - if training_validation_raw is not None: - if validation_raw is not None: - raise ValueError("Provide 'validation:' at top-level or " - "under 'training:', not both") - warnings.warn( - "Nesting 'validation:' under 'training:' is " - "deprecated. Move it to the top level.", - DeprecationWarning, - stacklevel=2, - ) - validation_raw = training_validation_raw - if validation_raw is None: validation: dict[str, Any] = {} else: - validation = _require_mapping(validation_raw, where="validation") + validation = _require_mapping( + validation_raw, where="validation") # --- pipeline config --- - pipeline_config = _parse_pipeline_config(cfg) + pipeline_config = _parse_pipeline_config( + cfg, models=models) - # --- build typed training config --- + # --- training config --- + training_raw = _require_mapping( + cfg.get("training"), where="training") + t = dict(training_raw) + t.pop("validation", None) training = _build_training_config( - training_raw, - models=models, - pipeline_config=pipeline_config, - ) + t, models=models, pipeline_config=pipeline_config) return RunConfig( models=models, From 37b001dd386d349da1345c4df2ba0c36fa90bb36 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 23:07:54 +0000 Subject: [PATCH 180/214] trainconfig should not be none during init --- fastvideo/train/dispatch.py | 3 ++- fastvideo/train/models/wan/wan.py | 6 +++--- fastvideo/train/models/wangame/wangame.py | 11 ++++++----- fastvideo/train/models/wangame/wangame_causal.py | 8 +++++++- fastvideo/train/utils/moduleloader.py | 9 +++------ 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/fastvideo/train/dispatch.py b/fastvideo/train/dispatch.py index 5a75e627a..5aa8f45d7 100644 --- a/fastvideo/train/dispatch.py +++ b/fastvideo/train/dispatch.py @@ -31,7 +31,8 @@ def build_from_config(cfg: RunConfig, ) -> tuple[TrainingConfig, TrainingMethod, # --- 1. Build role model instances --- role_models: dict[str, ModelBase] = {} for role, model_cfg in cfg.models.items(): - model = instantiate(model_cfg) + model = instantiate( + model_cfg, training_config=cfg.training) if not isinstance(model, ModelBase): raise TypeError(f"models.{role}._target_ must resolve to a " f"ModelBase subclass, got {type(model).__name__}") diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index 38a93db9f..cb96151c8 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -66,6 +66,7 @@ def __init__( self, *, init_from: str, + training_config: TrainingConfig, trainable: bool = True, disable_custom_init_weights: bool = False, flow_shift: float = 3.0, @@ -78,6 +79,7 @@ def __init__( transformer = load_module_from_path( model_path=self._init_from, module_type="transformer", + training_config=training_config, disable_custom_init_weights=(disable_custom_init_weights), override_transformer_cls_name=("WanTransformer3DModel"), ) @@ -93,7 +95,7 @@ def __init__( # Filled by init_preprocessors (student only). self.vae: Any = None - self.training_config: TrainingConfig | None = (None) + self.training_config: TrainingConfig = training_config self.dataloader: Any = None self.validator: Any = None self.start_step: int = 0 @@ -119,8 +121,6 @@ def __init__( # ------------------------------------------------------------------ def init_preprocessors(self, training_config: TrainingConfig) -> None: - self.training_config = training_config - self.vae = load_module_from_path( model_path=str(training_config.model_path), module_type="vae", diff --git a/fastvideo/train/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py index 181214b5f..c63a19e84 100644 --- a/fastvideo/train/models/wangame/wangame.py +++ b/fastvideo/train/models/wangame/wangame.py @@ -62,11 +62,11 @@ def __init__( self, *, init_from: str, + training_config: TrainingConfig, trainable: bool = True, disable_custom_init_weights: bool = False, flow_shift: float = 3.0, - enable_gradient_checkpointing_type: str - | None = None, + enable_gradient_checkpointing_type: str | None = None, ) -> None: self._init_from = str(init_from) self._trainable = bool(trainable) @@ -76,13 +76,14 @@ def __init__( trainable=self._trainable, disable_custom_init_weights=(disable_custom_init_weights), enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), + training_config=training_config, ) self.noise_scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) # Filled by init_preprocessors (student only). self.vae: Any = None - self.training_config: TrainingConfig | None = (None) + self.training_config: TrainingConfig = training_config self.dataloader: Any = None self.validator: Any = None self.start_step: int = 0 @@ -106,10 +107,12 @@ def _load_transformer( trainable: bool, disable_custom_init_weights: bool, enable_gradient_checkpointing_type: str | None, + training_config: TrainingConfig, ) -> torch.nn.Module: transformer = load_module_from_path( model_path=init_from, module_type="transformer", + training_config=training_config, disable_custom_init_weights=(disable_custom_init_weights), override_transformer_cls_name=(self._transformer_cls_name), ) @@ -127,8 +130,6 @@ def _load_transformer( def init_preprocessors(self, training_config: TrainingConfig) -> None: """Load VAE, build dataloader, seed RNGs.""" - self.training_config = training_config - self.vae = load_module_from_path( model_path=str(training_config.model_path), module_type="vae", diff --git a/fastvideo/train/models/wangame/wangame_causal.py b/fastvideo/train/models/wangame/wangame_causal.py index ac92362b9..902ed7824 100644 --- a/fastvideo/train/models/wangame/wangame_causal.py +++ b/fastvideo/train/models/wangame/wangame_causal.py @@ -4,7 +4,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, Literal +from typing import Any, Literal, TYPE_CHECKING import torch @@ -14,6 +14,10 @@ from fastvideo.train.models.base import CausalModelBase from fastvideo.train.models.wangame.wangame import WanGameModel +if TYPE_CHECKING: + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) + @dataclass(slots=True) class _StreamingCaches: @@ -36,6 +40,7 @@ def __init__( self, *, init_from: str, + training_config: TrainingConfig, trainable: bool = True, disable_custom_init_weights: bool = False, flow_shift: float = 3.0, @@ -47,6 +52,7 @@ def __init__( disable_custom_init_weights=disable_custom_init_weights, flow_shift=flow_shift, enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), + training_config=training_config, ) self._streaming_caches: dict[tuple[int, str], _StreamingCaches] = {} diff --git a/fastvideo/train/utils/moduleloader.py b/fastvideo/train/utils/moduleloader.py index 3b0854ed1..845adc0cd 100644 --- a/fastvideo/train/utils/moduleloader.py +++ b/fastvideo/train/utils/moduleloader.py @@ -74,7 +74,7 @@ def load_module_from_path( *, model_path: str, module_type: str, - training_config: TrainingConfig | None = None, + training_config: TrainingConfig, disable_custom_init_weights: bool = False, override_transformer_cls_name: str | None = None, ) -> torch.nn.Module: @@ -83,11 +83,8 @@ def load_module_from_path( Accepts a ``TrainingConfig`` and internally builds the ``TrainingArgs`` needed by ``PipelineComponentLoader``. """ - if training_config is not None: - fastvideo_args: Any = _make_training_args(training_config, model_path=model_path) - else: - from types import SimpleNamespace - fastvideo_args = SimpleNamespace() + fastvideo_args: Any = _make_training_args( + training_config, model_path=model_path) local_model_path = maybe_download_model(model_path) config = verify_model_config_and_directory(local_model_path) From dd7fbb56a50621c7c9df5f5f547d51b12476dcd8 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 23:18:48 +0000 Subject: [PATCH 181/214] validation config will be included in traning config --- fastvideo/train/models/wan/wan.py | 2 +- fastvideo/train/models/wangame/wangame.py | 2 +- fastvideo/train/utils/config.py | 5 ++++- fastvideo/train/utils/training_config.py | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index cb96151c8..a290e4a78 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -134,7 +134,7 @@ def init_preprocessors(self, training_config: TrainingConfig) -> None: self._init_timestep_mechanics() # Optional validator. - validation_cfg = getattr(training_config, "_validation_cfg", None) + validation_cfg = training_config.validation if validation_cfg: validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) if validation_enabled: diff --git a/fastvideo/train/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py index c63a19e84..e30a4bbc6 100644 --- a/fastvideo/train/models/wangame/wangame.py +++ b/fastvideo/train/models/wangame/wangame.py @@ -143,7 +143,7 @@ def init_preprocessors(self, training_config: TrainingConfig) -> None: self._init_timestep_mechanics() # Optional validator. - validation_cfg = getattr(training_config, "_validation_cfg", None) + validation_cfg = training_config.validation if validation_cfg: validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) if validation_enabled: diff --git a/fastvideo/train/utils/config.py b/fastvideo/train/utils/config.py index 11c79a5ec..a576ede26 100644 --- a/fastvideo/train/utils/config.py +++ b/fastvideo/train/utils/config.py @@ -256,6 +256,7 @@ def _build_training_config( *, models: dict[str, dict[str, Any]], pipeline_config: Any, + validation: dict[str, Any], ) -> TrainingConfig: """Build TrainingConfig from nested training: YAML.""" d = dict(t.get("distributed", {}) or {}) @@ -388,6 +389,7 @@ def _build_training_config( "enable_gradient_checkpointing_type" )), ), + validation=validation, pipeline_config=pipeline_config, model_path=model_path, dit_precision=str( @@ -448,7 +450,8 @@ def load_run_config(path: str) -> RunConfig: t = dict(training_raw) t.pop("validation", None) training = _build_training_config( - t, models=models, pipeline_config=pipeline_config) + t, models=models, pipeline_config=pipeline_config, + validation=validation) return RunConfig( models=models, diff --git a/fastvideo/train/utils/training_config.py b/fastvideo/train/utils/training_config.py index da8bd9783..a4fc15f4a 100644 --- a/fastvideo/train/utils/training_config.py +++ b/fastvideo/train/utils/training_config.py @@ -95,6 +95,7 @@ class TrainingConfig: tracker: TrackerConfig = field(default_factory=TrackerConfig) vsa: VSAConfig = field(default_factory=VSAConfig) model: ModelTrainingConfig = field(default_factory=ModelTrainingConfig) + validation: dict = field(default_factory=dict) pipeline_config: PipelineConfig | None = None model_path: str = "" dit_precision: str = "fp32" From b753c1765e5636d9755521c183958a11fe59bbb6 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 23:25:37 +0000 Subject: [PATCH 182/214] self.student.validator is guaranteed exist --- fastvideo/train/dispatch.py | 4 +--- fastvideo/train/methods/base.py | 13 +++++++------ .../train/methods/distribution_matching/dmd2.py | 16 +++++----------- .../distribution_matching/self_forcing.py | 2 -- fastvideo/train/methods/fine_tuning/dfsft.py | 16 +++++----------- fastvideo/train/methods/fine_tuning/finetune.py | 16 +++++----------- 6 files changed, 23 insertions(+), 44 deletions(-) diff --git a/fastvideo/train/dispatch.py b/fastvideo/train/dispatch.py index 5aa8f45d7..df16c9614 100644 --- a/fastvideo/train/dispatch.py +++ b/fastvideo/train/dispatch.py @@ -66,14 +66,12 @@ def build_from_config(cfg: RunConfig, ) -> tuple[TrainingConfig, TrainingMethod, method_target = str(method_cfg.pop("_target_")) method_cls = resolve_target(method_target) - # The student model provides the validator and dataloader. + # The student model provides the dataloader. student = role_models.get("student") - validator = (getattr(student, "validator", None) if student is not None else None) method = method_cls( cfg=cfg, role_models=role_models, - validator=validator, ) # --- 4. Gather dataloader and start_step --- diff --git a/fastvideo/train/methods/base.py b/fastvideo/train/methods/base.py index 2f81fdad3..ea6c795d4 100644 --- a/fastvideo/train/methods/base.py +++ b/fastvideo/train/methods/base.py @@ -45,14 +45,15 @@ def __init__( def set_tracker(self, tracker: Any) -> None: self.tracker = tracker - validator = getattr(self, "validator", None) - if validator is None: + student = self._role_models.get("student") + if student is None: return - set_tracker = getattr(validator, "set_tracker", None) - if callable(set_tracker): - set_tracker(tracker) + validator = getattr(student, "validator", None) + if validator is None: return - if hasattr(validator, "tracker"): + if hasattr(validator, "set_tracker"): + validator.set_tracker(tracker) + elif hasattr(validator, "tracker"): validator.tracker = tracker # type: ignore[attr-defined] @abstractmethod diff --git a/fastvideo/train/methods/distribution_matching/dmd2.py b/fastvideo/train/methods/distribution_matching/dmd2.py index 696eff501..77d942ab0 100644 --- a/fastvideo/train/methods/distribution_matching/dmd2.py +++ b/fastvideo/train/methods/distribution_matching/dmd2.py @@ -51,7 +51,6 @@ def __init__( *, cfg: Any, role_models: dict[str, ModelBase], - validator: Any | None = None, ) -> None: super().__init__(role_models=role_models) @@ -72,8 +71,6 @@ def __init__( raise ValueError("DMD2Method requires teacher to be non-trainable") if not getattr(self.critic, "_trainable", True): raise ValueError("DMD2Method requires critic to be trainable") - - self.validator = validator self.training_config = cfg.training self.method_config: dict[str, Any] = dict(cfg.method) self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) @@ -228,9 +225,6 @@ def on_train_start(self) -> None: # Trainer hook: log_validation def log_validation(self, iteration: int) -> None: - validator = getattr(self, "validator", None) - if validator is None: - return if not is_validation_enabled(self.validation_config): return @@ -278,7 +272,7 @@ def log_validation(self, iteration: int) -> None: num_frames=num_actions, output_dir=output_dir, ) - validator.log_validation(iteration, request=request) + self.student.validator.log_validation(iteration, request=request) # Checkpoint hook: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: @@ -287,10 +281,10 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: student_gens = self.student.get_rng_generators() generators.update(student_gens) - validator = getattr(self, "validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen + if is_validation_enabled(self.validation_config): + validation_gen = self.student.validator.validation_random_generator + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen return generators diff --git a/fastvideo/train/methods/distribution_matching/self_forcing.py b/fastvideo/train/methods/distribution_matching/self_forcing.py index a0ee28b2c..3a8c8c083 100644 --- a/fastvideo/train/methods/distribution_matching/self_forcing.py +++ b/fastvideo/train/methods/distribution_matching/self_forcing.py @@ -49,12 +49,10 @@ def __init__( *, cfg: Any, role_models: dict[str, ModelBase], - validator: Any | None = None, ) -> None: super().__init__( cfg=cfg, role_models=role_models, - validator=validator, ) # Validate causal student. diff --git a/fastvideo/train/methods/fine_tuning/dfsft.py b/fastvideo/train/methods/fine_tuning/dfsft.py index 510682211..f7a44290b 100644 --- a/fastvideo/train/methods/fine_tuning/dfsft.py +++ b/fastvideo/train/methods/fine_tuning/dfsft.py @@ -42,7 +42,6 @@ def __init__( *, cfg: Any, role_models: dict[str, ModelBase], - validator: Any | None = None, ) -> None: super().__init__(role_models=role_models) @@ -51,8 +50,6 @@ def __init__( self.student = role_models["student"] if not getattr(self.student, "_trainable", True): raise ValueError("DFSFT requires student to be trainable") - - self.validator = validator self.training_config = cfg.training self.method_config: dict[str, Any] = dict(cfg.method) self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) @@ -229,9 +226,6 @@ def on_train_start(self) -> None: # Trainer hook: log_validation def log_validation(self, iteration: int) -> None: - validator = getattr(self, "validator", None) - if validator is None: - return if not is_validation_enabled(self.validation_config): return @@ -262,7 +256,7 @@ def log_validation(self, iteration: int) -> None: num_frames=num_actions, output_dir=output_dir, ) - validator.log_validation(iteration, request=request) + self.student.validator.log_validation(iteration, request=request) # Checkpoint hook: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: @@ -271,10 +265,10 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: student_gens = self.student.get_rng_generators() generators.update(student_gens) - validator = getattr(self, "validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen + if is_validation_enabled(self.validation_config): + validation_gen = self.student.validator.validation_random_generator + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen return generators diff --git a/fastvideo/train/methods/fine_tuning/finetune.py b/fastvideo/train/methods/fine_tuning/finetune.py index 3a314b84c..da05f79d3 100644 --- a/fastvideo/train/methods/fine_tuning/finetune.py +++ b/fastvideo/train/methods/fine_tuning/finetune.py @@ -40,7 +40,6 @@ def __init__( *, cfg: Any, role_models: dict[str, ModelBase], - validator: Any | None = None, ) -> None: super().__init__(role_models=role_models) @@ -49,8 +48,6 @@ def __init__( self.student = role_models["student"] if not getattr(self.student, "_trainable", True): raise ValueError("FineTuneMethod requires student to be trainable") - - self.validator = validator self.training_config = cfg.training self.method_config: dict[str, Any] = dict(cfg.method) self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) @@ -181,9 +178,6 @@ def on_train_start(self) -> None: # Trainer hook: log_validation def log_validation(self, iteration: int) -> None: - validator = getattr(self, "validator", None) - if validator is None: - return if not is_validation_enabled(self.validation_config): return @@ -214,7 +208,7 @@ def log_validation(self, iteration: int) -> None: num_frames=num_actions, output_dir=output_dir, ) - validator.log_validation(iteration, request=request) + self.student.validator.log_validation(iteration, request=request) # Checkpoint hook: get_rng_generators def get_rng_generators(self) -> dict[str, torch.Generator]: @@ -223,10 +217,10 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: student_gens = self.student.get_rng_generators() generators.update(student_gens) - validator = getattr(self, "validator", None) - validation_gen = getattr(validator, "validation_random_generator", None) - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen + if is_validation_enabled(self.validation_config): + validation_gen = self.student.validator.validation_random_generator + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen return generators From eacb5f6d6df285154b3ae529eddfc6661e3e37c0 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 23:41:44 +0000 Subject: [PATCH 183/214] ~/.claude/plans/wise-mixing-pie.md --- examples/distillation/refactor/dfsft_wangame_causal_v3.yaml | 2 +- examples/distillation/refactor/run.sh | 2 +- fastvideo/train/validators/wan.py | 2 +- fastvideo/train/validators/wangame.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index 64353998e..8360750e8 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -76,7 +76,7 @@ validation: sampler_kind: ode ode_solver: euler guidance_scale: 1.0 - num_frames: 75 + num_frames: 69 pipeline: flow_shift: 3 diff --git a/examples/distillation/refactor/run.sh b/examples/distillation/refactor/run.sh index b43004fb8..7c77256f7 100755 --- a/examples/distillation/refactor/run.sh +++ b/examples/distillation/refactor/run.sh @@ -23,7 +23,7 @@ NUM_GPUS="${NUM_GPUS:-1}" NNODES="${NNODES:-1}" NODE_RANK="${NODE_RANK:-0}" MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" -MASTER_PORT="${MASTER_PORT:-29500}" +MASTER_PORT="${MASTER_PORT:-29501}" # ── W&B ────────────────────────────────────────────────────────────── export WANDB_API_KEY="${WANDB_API_KEY:-}" diff --git a/fastvideo/train/validators/wan.py b/fastvideo/train/validators/wan.py index b4cd502a7..9f736ef33 100644 --- a/fastvideo/train/validators/wan.py +++ b/fastvideo/train/validators/wan.py @@ -287,7 +287,7 @@ def log_validation( if sample_handle is None: raise ValueError("ValidationRequest.sample_handle must be " "provided by the method") - transformer = sample_handle.require_module("transformer") + transformer = sample_handle.transformer was_training = bool(getattr(transformer, "training", False)) tc = self.training_config diff --git a/fastvideo/train/validators/wangame.py b/fastvideo/train/validators/wangame.py index 996cb8a0f..3dee07759 100644 --- a/fastvideo/train/validators/wangame.py +++ b/fastvideo/train/validators/wangame.py @@ -526,7 +526,7 @@ def log_validation( if sample_handle is None: raise ValueError("ValidationRequest.sample_handle must be " "provided by the method") - transformer = sample_handle.require_module("transformer") + transformer = sample_handle.transformer was_training = bool(getattr(transformer, "training", False)) tc = self.training_config From 08ed16e84085bc69992960143aeb14e66938e095 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 23:50:58 +0000 Subject: [PATCH 184/214] simplifying code --- .../test_optimizer_scheduler_alignment.py | 32 +- fastvideo/train/methods/base.py | 133 ++++- .../methods/distribution_matching/dmd2.py | 458 ++++++++++++------ fastvideo/train/methods/fine_tuning/dfsft.py | 304 ++++++------ .../train/methods/fine_tuning/finetune.py | 182 +++---- fastvideo/train/models/wan/wan.py | 2 +- fastvideo/train/models/wangame/wangame.py | 2 +- fastvideo/train/trainer.py | 56 +-- fastvideo/train/utils/checkpoint.py | 2 +- fastvideo/train/utils/training_config.py | 2 +- fastvideo/train/validators/wan.py | 2 +- fastvideo/train/validators/wangame.py | 2 +- fastvideo/training/distillation.py | 27 +- 13 files changed, 741 insertions(+), 463 deletions(-) diff --git a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py index d7e592a40..98b2174b9 100644 --- a/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py +++ b/fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py @@ -26,15 +26,45 @@ def zero_grad(self, *args, **kwargs): # noqa: ANN002, ANN003, ANN201 self.zero_grad_calls += 1 +class _FakeModel: + transformer = None + + def on_train_start(self) -> None: + pass + + def get_rng_generators(self) -> dict: + return {} + + +class _FakeCfg: + class training: + pass + + method: dict = {} + validation: dict = {} + + class _ScheduleMethod(TrainingMethod): def __init__(self, interval: int) -> None: self.student_opt = _FakeOptimizer() self.critic_opt = _FakeOptimizer() self.student_sched = _FakeScheduler() self.critic_sched = _FakeScheduler() - super().__init__(role_models={}) + cfg = _FakeCfg() + cfg.method = {} + cfg.validation = {} + role_models = {"student": _FakeModel()} # type: ignore[dict-item] + super().__init__(cfg=cfg, role_models=role_models) self.interval = interval + @property + def _optimizer_dict(self): # noqa: ANN201 + return {"student": self.student_opt, "critic": self.critic_opt} + + @property + def _lr_scheduler_dict(self): # noqa: ANN201 + return {"student": self.student_sched, "critic": self.critic_sched} + def _update_student(self, iteration: int) -> bool: return iteration % self.interval == 0 diff --git a/fastvideo/train/methods/base.py b/fastvideo/train/methods/base.py index ea6c795d4..30e02a862 100644 --- a/fastvideo/train/methods/base.py +++ b/fastvideo/train/methods/base.py @@ -4,17 +4,30 @@ from abc import ABC, abstractmethod from collections.abc import Sequence -from typing import Any +from typing import Any, Literal, cast import torch from fastvideo.train.models.base import ModelBase +from fastvideo.train.utils.validation import ( + is_validation_enabled, + parse_validation_dataset_file, + parse_validation_every_steps, + parse_validation_guidance_scale, + parse_validation_num_frames, + parse_validation_ode_solver, + parse_validation_output_dir, + parse_validation_rollout_mode, + parse_validation_sampler_kind, + parse_validation_sampling_steps, +) +from fastvideo.train.validators.base import ValidationRequest LogScalar = float | int | torch.Tensor class TrainingMethod(torch.nn.Module, ABC): - """Base distillation method (algorithm layer). + """Base training method (algorithm layer). Subclasses own their role models (student, teacher, critic, …) as plain attributes and manage optimizers directly — no ``RoleManager`` @@ -28,11 +41,20 @@ class TrainingMethod(torch.nn.Module, ABC): def __init__( self, *, + cfg: Any, role_models: dict[str, ModelBase], ) -> None: super().__init__() self.tracker: Any | None = None self._role_models: dict[str, ModelBase] = dict(role_models) + + self.student = role_models["student"] + self.training_config = cfg.training + self.method_config: dict[str, Any] = dict(cfg.method) + self.validation_config: dict[str, Any] = dict( + getattr(cfg, "validation", {}) or {} + ) + # Build nn.ModuleDict for FSDP / checkpoint visibility. self.role_modules = torch.nn.ModuleDict() for role, model in role_models.items(): @@ -71,13 +93,27 @@ def single_train_step( raise NotImplementedError @abstractmethod - def get_optimizers(self, iteration: int) -> Sequence[torch.optim.Optimizer]: + def get_optimizers( + self, iteration: int, + ) -> Sequence[torch.optim.Optimizer]: raise NotImplementedError @abstractmethod - def get_lr_schedulers(self, iteration: int) -> Sequence[Any]: + def get_lr_schedulers( + self, iteration: int, + ) -> Sequence[Any]: raise NotImplementedError + @property + @abstractmethod + def _optimizer_dict(self) -> dict[str, Any]: + ... + + @property + @abstractmethod + def _lr_scheduler_dict(self) -> dict[str, Any]: + ... + def backward( self, loss_map: dict[str, torch.Tensor], @@ -89,15 +125,100 @@ def backward( grad_accum_rounds = max(1, int(grad_accum_rounds)) (loss_map["total_loss"] / grad_accum_rounds).backward() - def optimizers_schedulers_step(self, iteration: int) -> None: + def optimizers_schedulers_step( + self, iteration: int, + ) -> None: for optimizer in self.get_optimizers(iteration): optimizer.step() for scheduler in self.get_lr_schedulers(iteration): scheduler.step() - def optimizers_zero_grad(self, iteration: int) -> None: + def optimizers_zero_grad( + self, iteration: int, + ) -> None: for optimizer in self.get_optimizers(iteration): try: optimizer.zero_grad(set_to_none=True) except TypeError: optimizer.zero_grad() + + # -- Shared hooks (override in subclasses as needed) -- + + def on_train_start(self) -> None: + self.student.on_train_start() + + def get_rng_generators( + self, + ) -> dict[str, torch.Generator]: + generators: dict[str, torch.Generator] = {} + + student_gens = self.student.get_rng_generators() + generators.update(student_gens) + + if is_validation_enabled(self.validation_config): + validation_gen = ( + self.student.validator.validation_random_generator + ) + if isinstance(validation_gen, torch.Generator): + generators["validation_cpu"] = validation_gen + + return generators + + def log_validation(self, iteration: int) -> None: + if not is_validation_enabled(self.validation_config): + return + + every_steps = parse_validation_every_steps( + self.validation_config, + ) + if every_steps <= 0: + return + if iteration % every_steps != 0: + return + + request = self._build_validation_request() + self.student.validator.log_validation( + iteration, request=request, + ) + + def _build_validation_request(self) -> ValidationRequest: + """Build the ``ValidationRequest`` for validation. + + Override in subclasses that need custom parameters (e.g. + ``sampling_timesteps`` for DMD2). + """ + vc = self.validation_config + sampling_steps = parse_validation_sampling_steps(vc) + guidance_scale = parse_validation_guidance_scale(vc) + sampler_kind = parse_validation_sampler_kind( + vc, default="ode", + ) + ode_solver = parse_validation_ode_solver( + vc, sampler_kind=sampler_kind, + ) + return ValidationRequest( + sample_handle=self.student, + dataset_file=parse_validation_dataset_file(vc), + sampling_steps=sampling_steps, + sampler_kind=sampler_kind, + rollout_mode=parse_validation_rollout_mode(vc), + ode_solver=ode_solver, + sampling_timesteps=None, + guidance_scale=guidance_scale, + num_frames=parse_validation_num_frames(vc), + output_dir=parse_validation_output_dir(vc), + ) + + @staticmethod + def _parse_attn_kind( + raw: Any, + ) -> Literal["dense", "vsa"]: + if raw in (None, ""): + return "dense" + kind = str(raw).strip().lower() + if kind not in {"dense", "vsa"}: + raise ValueError( + "method_config.attn_kind must be one of " + f"{{'dense', 'vsa'}}, got {raw!r}." + ) + return cast(Literal["dense", "vsa"], kind) diff --git a/fastvideo/train/methods/distribution_matching/dmd2.py b/fastvideo/train/methods/distribution_matching/dmd2.py index 77d942ab0..e443b0256 100644 --- a/fastvideo/train/methods/distribution_matching/dmd2.py +++ b/fastvideo/train/methods/distribution_matching/dmd2.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import Any, Literal, TYPE_CHECKING +from typing import Any, Literal import torch import torch.nn.functional as F @@ -15,9 +15,7 @@ clip_grad_norm_if_needed, ) from fastvideo.train.utils.validation import ( - is_validation_enabled, parse_validation_dataset_file, - parse_validation_every_steps, parse_validation_guidance_scale, parse_validation_num_frames, parse_validation_ode_solver, @@ -33,9 +31,6 @@ parse_betas, ) -if TYPE_CHECKING: - pass - class DMD2Method(TrainingMethod): """DMD2 distillation algorithm (method layer). @@ -52,31 +47,42 @@ def __init__( cfg: Any, role_models: dict[str, ModelBase], ) -> None: - super().__init__(role_models=role_models) + super().__init__(cfg=cfg, role_models=role_models) if "student" not in role_models: - raise ValueError("DMD2Method requires role 'student'") + raise ValueError( + "DMD2Method requires role 'student'" + ) if "teacher" not in role_models: - raise ValueError("DMD2Method requires role 'teacher'") + raise ValueError( + "DMD2Method requires role 'teacher'" + ) if "critic" not in role_models: - raise ValueError("DMD2Method requires role 'critic'") + raise ValueError( + "DMD2Method requires role 'critic'" + ) - self.student = role_models["student"] self.teacher = role_models["teacher"] self.critic = role_models["critic"] if not getattr(self.student, "_trainable", True): - raise ValueError("DMD2Method requires student to be trainable") + raise ValueError( + "DMD2Method requires student to be trainable" + ) if getattr(self.teacher, "_trainable", True): - raise ValueError("DMD2Method requires teacher to be non-trainable") + raise ValueError( + "DMD2Method requires teacher to be " + "non-trainable" + ) if not getattr(self.critic, "_trainable", True): - raise ValueError("DMD2Method requires critic to be trainable") - self.training_config = cfg.training - self.method_config: dict[str, Any] = dict(cfg.method) - self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) + raise ValueError( + "DMD2Method requires critic to be trainable" + ) self._cfg_uncond = self._parse_cfg_uncond() self._rollout_mode = self._parse_rollout_mode() - self._denoising_step_list: torch.Tensor | None = None + self._denoising_step_list: torch.Tensor | None = ( + None + ) # Initialize preprocessors on student. self.student.init_preprocessors(self.training_config) @@ -84,7 +90,9 @@ def __init__( self._init_optimizers_and_schedulers() @property - def _optimizer_dict(self) -> dict[str, torch.optim.Optimizer]: + def _optimizer_dict( + self, + ) -> dict[str, torch.optim.Optimizer]: return { "student": self._student_optimizer, "critic": self._critic_optimizer, @@ -119,7 +127,9 @@ def single_train_step( latents_source=latents_source, ) - update_student = self._should_update_student(iteration) + update_student = self._should_update_student( + iteration + ) generator_loss = torch.zeros( (), @@ -128,12 +138,16 @@ def single_train_step( ) student_ctx = None if update_student: - generator_pred_x0 = self._student_rollout(training_batch, with_grad=True) + generator_pred_x0 = self._student_rollout( + training_batch, with_grad=True + ) student_ctx = ( training_batch.timesteps, training_batch.attn_metadata_vsa, ) - generator_loss = self._dmd_loss(generator_pred_x0, training_batch) + generator_loss = self._dmd_loss( + generator_pred_x0, training_batch + ) ( fake_score_loss, @@ -154,7 +168,9 @@ def single_train_step( "student_ctx": student_ctx, "critic_ctx": critic_ctx, } - metrics: dict[str, LogScalar] = {"update_student": float(update_student)} + metrics: dict[str, LogScalar] = { + "update_student": float(update_student) + } return loss_map, outputs, metrics # TrainingMethod override: backward @@ -175,11 +191,15 @@ def backward( ) return - update_student = bool(backward_ctx.get("update_student", False)) + update_student = bool( + backward_ctx.get("update_student", False) + ) if update_student: student_ctx = backward_ctx.get("student_ctx") if student_ctx is None: - raise RuntimeError("Missing student backward context") + raise RuntimeError( + "Missing student backward context" + ) self.student.backward( loss_map["generator_loss"], student_ctx, @@ -188,7 +208,9 @@ def backward( critic_ctx = backward_ctx.get("critic_ctx") if critic_ctx is None: - raise RuntimeError("Missing critic backward context") + raise RuntimeError( + "Missing critic backward context" + ) self.critic.backward( loss_map["fake_score_loss"], critic_ctx, @@ -196,7 +218,9 @@ def backward( ) # TrainingMethod override: get_optimizers - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + def get_optimizers( + self, iteration: int, + ) -> list[torch.optim.Optimizer]: optimizers: list[torch.optim.Optimizer] = [] optimizers.append(self._critic_optimizer) if self._should_update_student(iteration): @@ -204,7 +228,9 @@ def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: return optimizers # TrainingMethod override: get_lr_schedulers - def get_lr_schedulers(self, iteration: int) -> list[Any]: + def get_lr_schedulers( + self, iteration: int, + ) -> list[Any]: schedulers: list[Any] = [] schedulers.append(self._critic_lr_scheduler) if self._should_update_student(iteration): @@ -212,105 +238,119 @@ def get_lr_schedulers(self, iteration: int) -> list[Any]: return schedulers # TrainingMethod override: optimizers_schedulers_step - def optimizers_schedulers_step(self, iteration: int) -> None: - max_grad_norm = self.training_config.optimizer.max_grad_norm + def optimizers_schedulers_step( + self, iteration: int, + ) -> None: + max_grad_norm = ( + self.training_config.optimizer.max_grad_norm + ) if self._should_update_student(iteration): - clip_grad_norm_if_needed(self.student.transformer, max_grad_norm) - clip_grad_norm_if_needed(self.critic.transformer, max_grad_norm) + clip_grad_norm_if_needed( + self.student.transformer, max_grad_norm + ) + clip_grad_norm_if_needed( + self.critic.transformer, max_grad_norm + ) super().optimizers_schedulers_step(iteration) - # Trainer hook: on_train_start - def on_train_start(self) -> None: - self.student.on_train_start() - - # Trainer hook: log_validation - def log_validation(self, iteration: int) -> None: - if not is_validation_enabled(self.validation_config): - return - - every_steps = parse_validation_every_steps(self.validation_config) - if every_steps <= 0: - return - if iteration % every_steps != 0: - return - - dataset_file = parse_validation_dataset_file(self.validation_config) - sampling_steps = parse_validation_sampling_steps(self.validation_config) + # Override: DMD2-specific validation request with + # sampling_timesteps and default="sde". + def _build_validation_request( + self, + ) -> ValidationRequest: + vc = self.validation_config + sampling_steps = parse_validation_sampling_steps(vc) sampling_timesteps: list[int] | None = None - raw_timesteps = self.validation_config.get("sampling_timesteps", None) + raw_timesteps = vc.get( + "sampling_timesteps", None + ) if raw_timesteps is None: - raw_timesteps = self.method_config.get("dmd_denoising_steps", None) + raw_timesteps = self.method_config.get( + "dmd_denoising_steps", None + ) if isinstance(raw_timesteps, list) and raw_timesteps: - sampling_timesteps = [int(s) for s in raw_timesteps] + sampling_timesteps = [ + int(s) for s in raw_timesteps + ] if not sampling_steps: if sampling_timesteps is None: - return + return ValidationRequest( + sample_handle=self.student + ) sampling_steps = [int(len(sampling_timesteps))] - sampler_kind = parse_validation_sampler_kind(self.validation_config, default="sde") - ode_solver = parse_validation_ode_solver(self.validation_config, sampler_kind=sampler_kind) - if (sampling_timesteps is not None and sampler_kind != "sde"): - raise ValueError("method_config.validation.sampling_timesteps is " - "only valid when sampler_kind='sde'") - - rollout_mode = parse_validation_rollout_mode(self.validation_config) - guidance_scale = parse_validation_guidance_scale(self.validation_config) - output_dir = parse_validation_output_dir(self.validation_config) - num_actions = parse_validation_num_frames(self.validation_config) + sampler_kind = parse_validation_sampler_kind( + vc, default="sde" + ) + ode_solver = parse_validation_ode_solver( + vc, sampler_kind=sampler_kind + ) + if ( + sampling_timesteps is not None + and sampler_kind != "sde" + ): + raise ValueError( + "method_config.validation." + "sampling_timesteps is " + "only valid when sampler_kind='sde'" + ) - request = ValidationRequest( + return ValidationRequest( sample_handle=self.student, - dataset_file=dataset_file, + dataset_file=parse_validation_dataset_file(vc), sampling_steps=sampling_steps, sampler_kind=sampler_kind, - rollout_mode=rollout_mode, + rollout_mode=parse_validation_rollout_mode(vc), ode_solver=ode_solver, sampling_timesteps=sampling_timesteps, - guidance_scale=guidance_scale, - num_frames=num_actions, - output_dir=output_dir, + guidance_scale=parse_validation_guidance_scale( + vc + ), + num_frames=parse_validation_num_frames(vc), + output_dir=parse_validation_output_dir(vc), ) - self.student.validator.log_validation(iteration, request=request) - - # Checkpoint hook: get_rng_generators - def get_rng_generators(self) -> dict[str, torch.Generator]: - generators: dict[str, torch.Generator] = {} - - student_gens = self.student.get_rng_generators() - generators.update(student_gens) - if is_validation_enabled(self.validation_config): - validation_gen = self.student.validator.validation_random_generator - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen - - return generators - - def _parse_rollout_mode(self, ) -> Literal["simulate", "data_latent"]: - raw = self.method_config.get("rollout_mode", None) + def _parse_rollout_mode( + self, + ) -> Literal["simulate", "data_latent"]: + raw = self.method_config.get( + "rollout_mode", None + ) if raw is None: - raise ValueError("method_config.rollout_mode must be set for DMD2") + raise ValueError( + "method_config.rollout_mode must be set " + "for DMD2" + ) if not isinstance(raw, str): - raise ValueError("method_config.rollout_mode must be a string, " - f"got {type(raw).__name__}") + raise ValueError( + "method_config.rollout_mode must be a " + "string, " + f"got {type(raw).__name__}" + ) mode = raw.strip().lower() if mode in ("simulate", "sim"): return "simulate" if mode in ("data_latent", "data", "vae_latent"): return "data_latent" - raise ValueError("method_config.rollout_mode must be one of " - "{simulate, data_latent}, got " - f"{raw!r}") + raise ValueError( + "method_config.rollout_mode must be one of " + "{simulate, data_latent}, got " + f"{raw!r}" + ) - def _parse_cfg_uncond(self) -> dict[str, Any] | None: + def _parse_cfg_uncond( + self, + ) -> dict[str, Any] | None: raw = self.method_config.get("cfg_uncond", None) if raw is None: return None if not isinstance(raw, dict): - raise ValueError("method_config.cfg_uncond must be a dict when " - f"set, got {type(raw).__name__}") + raise ValueError( + "method_config.cfg_uncond must be a dict " + f"when set, got {type(raw).__name__}" + ) cfg: dict[str, Any] = dict(raw) @@ -318,13 +358,18 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: if on_missing_raw is None: on_missing_raw = "error" if not isinstance(on_missing_raw, str): - raise ValueError("method_config.cfg_uncond.on_missing must be a " - f"string, got {type(on_missing_raw).__name__}") + raise ValueError( + "method_config.cfg_uncond.on_missing must " + "be a string, got " + f"{type(on_missing_raw).__name__}" + ) on_missing = on_missing_raw.strip().lower() if on_missing not in {"error", "ignore"}: - raise ValueError("method_config.cfg_uncond.on_missing must be one " - "of {error, ignore}, got " - f"{on_missing_raw!r}") + raise ValueError( + "method_config.cfg_uncond.on_missing must " + "be one of {error, ignore}, got " + f"{on_missing_raw!r}" + ) cfg["on_missing"] = on_missing for channel, policy_raw in list(cfg.items()): @@ -333,17 +378,23 @@ def _parse_cfg_uncond(self) -> dict[str, Any] | None: if policy_raw is None: continue if not isinstance(policy_raw, str): - raise ValueError("method_config.cfg_uncond values must be " - "strings, got " - f"{channel}={type(policy_raw).__name__}") + raise ValueError( + "method_config.cfg_uncond values must " + "be strings, got " + f"{channel}=" + f"{type(policy_raw).__name__}" + ) policy = policy_raw.strip().lower() allowed = {"keep", "zero", "drop"} if channel == "text": allowed = {*allowed, "negative_prompt"} if policy not in allowed: - raise ValueError("method_config.cfg_uncond values must be one " - f"of {sorted(allowed)}, got " - f"{channel}={policy_raw!r}") + raise ValueError( + "method_config.cfg_uncond values must " + "be one of " + f"{sorted(allowed)}, got " + f"{channel}={policy_raw!r}" + ) cfg[channel] = policy return cfg @@ -355,7 +406,11 @@ def _init_optimizers_and_schedulers(self) -> None: student_lr = float(tc.optimizer.learning_rate) student_betas = tc.optimizer.betas student_sched = str(tc.optimizer.lr_scheduler) - student_params = [p for p in self.student.transformer.parameters() if p.requires_grad] + student_params = [ + p + for p in self.student.transformer.parameters() + if p.requires_grad + ] ( self._student_optimizer, self._student_lr_scheduler, @@ -368,7 +423,8 @@ def _init_optimizers_and_schedulers(self) -> None: scheduler_name=student_sched, ) - # Critic optimizer/scheduler — read from method config. + # Critic optimizer/scheduler — read from method + # config. critic_lr_raw = get_optional_float( self.method_config, "fake_score_learning_rate", @@ -378,14 +434,27 @@ def _init_optimizers_and_schedulers(self) -> None: if critic_lr == 0.0: critic_lr = student_lr - critic_betas_raw = self.method_config.get("fake_score_betas", None) + critic_betas_raw = self.method_config.get( + "fake_score_betas", None + ) if critic_betas_raw is None: critic_betas_raw = tc.optimizer.betas - critic_betas = parse_betas(critic_betas_raw, where="method.fake_score_betas") + critic_betas = parse_betas( + critic_betas_raw, + where="method.fake_score_betas", + ) - critic_sched_raw = self.method_config.get("fake_score_lr_scheduler", None) - critic_sched = str(critic_sched_raw or student_sched) - critic_params = [p for p in self.critic.transformer.parameters() if p.requires_grad] + critic_sched_raw = self.method_config.get( + "fake_score_lr_scheduler", None + ) + critic_sched = str( + critic_sched_raw or student_sched + ) + critic_params = [ + p + for p in self.critic.transformer.parameters() + if p.requires_grad + ] ( self._critic_optimizer, self._critic_lr_scheduler, @@ -398,7 +467,9 @@ def _init_optimizers_and_schedulers(self) -> None: scheduler_name=critic_sched, ) - def _should_update_student(self, iteration: int) -> bool: + def _should_update_student( + self, iteration: int, + ) -> bool: interval = get_optional_int( self.method_config, "generator_update_interval", @@ -410,14 +481,23 @@ def _should_update_student(self, iteration: int) -> bool: return True return iteration % interval == 0 - def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: - if (self._denoising_step_list is not None and self._denoising_step_list.device == device): + def _get_denoising_step_list( + self, device: torch.device, + ) -> torch.Tensor: + if ( + self._denoising_step_list is not None + and self._denoising_step_list.device == device + ): return self._denoising_step_list - raw = self.method_config.get("dmd_denoising_steps", None) + raw = self.method_config.get( + "dmd_denoising_steps", None + ) if not isinstance(raw, list) or not raw: - raise ValueError("method_config.dmd_denoising_steps must be set " - "for DMD2 distillation") + raise ValueError( + "method_config.dmd_denoising_steps must " + "be set for DMD2 distillation" + ) steps = torch.tensor( [int(s) for s in raw], @@ -425,20 +505,28 @@ def _get_denoising_step_list(self, device: torch.device) -> torch.Tensor: device=device, ) - warp = self.method_config.get("warp_denoising_step", None) + warp = self.method_config.get( + "warp_denoising_step", None + ) if warp is None: warp = False if bool(warp): timesteps = torch.cat(( - self.student.noise_scheduler.timesteps.to("cpu"), - torch.tensor([0], dtype=torch.float32), + self.student.noise_scheduler.timesteps.to( + "cpu" + ), + torch.tensor( + [0], dtype=torch.float32 + ), )).to(device) steps = timesteps[1000 - steps] self._denoising_step_list = steps return steps - def _sample_rollout_timestep(self, device: torch.device) -> torch.Tensor: + def _sample_rollout_timestep( + self, device: torch.device, + ) -> torch.Tensor: step_list = self._get_denoising_step_list(device) index = torch.randint( 0, @@ -449,16 +537,24 @@ def _sample_rollout_timestep(self, device: torch.device) -> torch.Tensor: ) return step_list[index] - def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: + def _student_rollout( + self, batch: Any, *, with_grad: bool, + ) -> torch.Tensor: latents = batch.latents device = latents.device dtype = latents.dtype step_list = self._get_denoising_step_list(device) if self._rollout_mode != "simulate": - timestep = self._sample_rollout_timestep(device) - noise = torch.randn(latents.shape, device=device, dtype=dtype) - noisy_latents = self.student.add_noise(latents, noise, timestep) + timestep = self._sample_rollout_timestep( + device + ) + noise = torch.randn( + latents.shape, device=device, dtype=dtype + ) + noisy_latents = self.student.add_noise( + latents, noise, timestep + ) pred_x0 = self.student.predict_x0( noisy_latents, timestep, @@ -467,7 +563,9 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: cfg_uncond=self._cfg_uncond, attn_kind="vsa", ) - batch.dmd_latent_vis_dict["generator_timestep"] = timestep + batch.dmd_latent_vis_dict[ + "generator_timestep" + ] = timestep return pred_x0 target_timestep_idx = torch.randint( @@ -477,11 +575,17 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: device=device, dtype=torch.long, ) - target_timestep_idx_int = int(target_timestep_idx.item()) + target_timestep_idx_int = int( + target_timestep_idx.item() + ) target_timestep = step_list[target_timestep_idx] - current_noise_latents = torch.randn(latents.shape, device=device, dtype=dtype) - current_noise_latents_copy = current_noise_latents.clone() + current_noise_latents = torch.randn( + latents.shape, device=device, dtype=dtype + ) + current_noise_latents_copy = ( + current_noise_latents.clone() + ) max_target_idx = len(step_list) - 1 noise_latents: list[torch.Tensor] = [] @@ -491,7 +595,14 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: with torch.no_grad(): for step_idx in range(max_target_idx): current_timestep = step_list[step_idx] - current_timestep_tensor = (current_timestep * torch.ones(1, device=device, dtype=torch.long)) + current_timestep_tensor = ( + current_timestep + * torch.ones( + 1, + device=device, + dtype=torch.long, + ) + ) pred_clean = self.student.predict_x0( current_noise_latents, @@ -503,22 +614,35 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: ) next_timestep = step_list[step_idx + 1] - next_timestep_tensor = (next_timestep * torch.ones(1, device=device, dtype=torch.long)) + next_timestep_tensor = ( + next_timestep + * torch.ones( + 1, + device=device, + dtype=torch.long, + ) + ) noise = torch.randn( latents.shape, device=device, dtype=pred_clean.dtype, ) - current_noise_latents = (self.student.add_noise( - pred_clean, - noise, - next_timestep_tensor, - )) - noise_latents.append(current_noise_latents.clone()) + current_noise_latents = ( + self.student.add_noise( + pred_clean, + noise, + next_timestep_tensor, + ) + ) + noise_latents.append( + current_noise_latents.clone() + ) if noise_latent_index >= 0: if noise_latent_index >= len(noise_latents): - raise RuntimeError("noise_latent_index is out of bounds") + raise RuntimeError( + "noise_latent_index is out of bounds" + ) noisy_input = noise_latents[noise_latent_index] else: noisy_input = current_noise_latents_copy @@ -543,12 +667,18 @@ def _student_rollout(self, batch: Any, *, with_grad: bool) -> torch.Tensor: attn_kind="vsa", ) - batch.dmd_latent_vis_dict["generator_timestep"] = target_timestep.float().detach() + batch.dmd_latent_vis_dict[ + "generator_timestep" + ] = target_timestep.float().detach() return pred_x0 - def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dict[str, Any]]: + def _critic_flow_matching_loss( + self, batch: Any, + ) -> tuple[torch.Tensor, Any, dict[str, Any]]: with torch.no_grad(): - generator_pred_x0 = self._student_rollout(batch, with_grad=False) + generator_pred_x0 = self._student_rollout( + batch, with_grad=False + ) device = generator_pred_x0.device fake_score_timestep = torch.randint( @@ -558,14 +688,20 @@ def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dic device=device, dtype=torch.long, ) - fake_score_timestep = (self.student.shift_and_clamp_timestep(fake_score_timestep)) + fake_score_timestep = ( + self.student.shift_and_clamp_timestep( + fake_score_timestep + ) + ) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_x0 = self.student.add_noise(generator_pred_x0, noise, fake_score_timestep) + noisy_x0 = self.student.add_noise( + generator_pred_x0, noise, fake_score_timestep + ) pred_noise = self.critic.predict_noise( noisy_x0, @@ -576,13 +712,19 @@ def _critic_flow_matching_loss(self, batch: Any) -> tuple[torch.Tensor, Any, dic attn_kind="dense", ) target = noise - generator_pred_x0 - flow_matching_loss = torch.mean((pred_noise - target)**2) + flow_matching_loss = torch.mean( + (pred_noise - target)**2 + ) batch.fake_score_latent_vis_dict = { "generator_pred_video": generator_pred_x0, "fake_score_timestep": fake_score_timestep, } - outputs = {"fake_score_latent_vis_dict": (batch.fake_score_latent_vis_dict)} + outputs = { + "fake_score_latent_vis_dict": ( + batch.fake_score_latent_vis_dict + ) + } return ( flow_matching_loss, (batch.timesteps, batch.attn_metadata), @@ -611,14 +753,20 @@ def _dmd_loss( device=device, dtype=torch.long, ) - timestep = self.student.shift_and_clamp_timestep(timestep) + timestep = ( + self.student.shift_and_clamp_timestep( + timestep + ) + ) noise = torch.randn( generator_pred_x0.shape, device=device, dtype=generator_pred_x0.dtype, ) - noisy_latents = self.student.add_noise(generator_pred_x0, noise, timestep) + noisy_latents = self.student.add_noise( + generator_pred_x0, noise, timestep + ) faker_x0 = self.critic.predict_x0( noisy_latents, @@ -644,14 +792,20 @@ def _dmd_loss( cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * (guidance_scale - 1) + real_cfg_x0 = real_cond_x0 + ( + real_cond_x0 - real_uncond_x0 + ) * (guidance_scale - 1) - denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() + denom = torch.abs( + generator_pred_x0 - real_cfg_x0 + ).mean() grad = (faker_x0 - real_cfg_x0) / denom grad = torch.nan_to_num(grad) loss = 0.5 * F.mse_loss( generator_pred_x0.float(), - (generator_pred_x0.float() - grad.float()).detach(), + ( + generator_pred_x0.float() - grad.float() + ).detach(), ) return loss diff --git a/fastvideo/train/methods/fine_tuning/dfsft.py b/fastvideo/train/methods/fine_tuning/dfsft.py index f7a44290b..d59ad9488 100644 --- a/fastvideo/train/methods/fine_tuning/dfsft.py +++ b/fastvideo/train/methods/fine_tuning/dfsft.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import Any, Literal, TYPE_CHECKING, cast +from typing import Any, Literal import torch import torch.nn.functional as F @@ -14,22 +14,6 @@ build_optimizer_and_scheduler, clip_grad_norm_if_needed, ) -from fastvideo.train.utils.validation import ( - is_validation_enabled, - parse_validation_dataset_file, - parse_validation_every_steps, - parse_validation_guidance_scale, - parse_validation_num_frames, - parse_validation_ode_solver, - parse_validation_output_dir, - parse_validation_rollout_mode, - parse_validation_sampler_kind, - parse_validation_sampling_steps, -) -from fastvideo.train.validators.base import ValidationRequest - -if TYPE_CHECKING: - pass class DiffusionForcingSFTMethod(TrainingMethod): @@ -43,20 +27,26 @@ def __init__( cfg: Any, role_models: dict[str, ModelBase], ) -> None: - super().__init__(role_models=role_models) + super().__init__(cfg=cfg, role_models=role_models) if "student" not in role_models: raise ValueError("DFSFT requires role 'student'") - self.student = role_models["student"] if not getattr(self.student, "_trainable", True): - raise ValueError("DFSFT requires student to be trainable") - self.training_config = cfg.training - self.method_config: dict[str, Any] = dict(cfg.method) - self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) - self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind(self.method_config.get("attn_kind", None))) + raise ValueError( + "DFSFT requires student to be trainable" + ) + self._attn_kind: Literal["dense", "vsa"] = ( + self._parse_attn_kind( + self.method_config.get("attn_kind", None) + ) + ) - self._chunk_size = self._parse_chunk_size(self.method_config.get("chunk_size", None)) - self._timestep_index_range = (self._parse_timestep_index_range()) + self._chunk_size = self._parse_chunk_size( + self.method_config.get("chunk_size", None) + ) + self._timestep_index_range = ( + self._parse_timestep_index_range() + ) # Initialize preprocessors on student. self.student.init_preprocessors(self.training_config) @@ -91,44 +81,68 @@ def single_train_step( ) if training_batch.latents is None: - raise RuntimeError("prepare_batch() must set TrainingBatch.latents") + raise RuntimeError( + "prepare_batch() must set TrainingBatch.latents" + ) clean_latents = training_batch.latents if not torch.is_tensor(clean_latents): - raise TypeError("TrainingBatch.latents must be a torch.Tensor") + raise TypeError( + "TrainingBatch.latents must be a torch.Tensor" + ) if clean_latents.ndim != 5: - raise ValueError("TrainingBatch.latents must be " - "[B, T, C, H, W], got " - f"shape={tuple(clean_latents.shape)}") + raise ValueError( + "TrainingBatch.latents must be " + "[B, T, C, H, W], got " + f"shape={tuple(clean_latents.shape)}" + ) - batch_size, num_latents = int(clean_latents.shape[0]), int(clean_latents.shape[1]) + batch_size, num_latents = ( + int(clean_latents.shape[0]), + int(clean_latents.shape[1]), + ) expected_chunk = getattr( self.student.transformer, "num_frame_per_block", None, ) - if (expected_chunk is not None and int(expected_chunk) != int(self._chunk_size)): - raise ValueError("DFSFT chunk_size must match " - "transformer.num_frame_per_block for " - f"causal training (got {self._chunk_size}, " - f"expected {expected_chunk}).") + if ( + expected_chunk is not None + and int(expected_chunk) != int(self._chunk_size) + ): + raise ValueError( + "DFSFT chunk_size must match " + "transformer.num_frame_per_block for " + f"causal training (got {self._chunk_size}, " + f"expected {expected_chunk})." + ) timestep_indices = self._sample_t_inhom_indices( batch_size=batch_size, num_latents=num_latents, device=clean_latents.device, ) - sp_size = int(self.training_config.distributed.sp_size) + sp_size = int( + self.training_config.distributed.sp_size + ) sp_group = getattr(self.student, "sp_group", None) - if (sp_size > 1 and sp_group is not None and hasattr(sp_group, "broadcast")): + if ( + sp_size > 1 + and sp_group is not None + and hasattr(sp_group, "broadcast") + ): sp_group.broadcast(timestep_indices, src=0) scheduler = self.student.noise_scheduler if scheduler is None: - raise ValueError("DFSFT requires student.noise_scheduler") + raise ValueError( + "DFSFT requires student.noise_scheduler" + ) - schedule_timesteps = scheduler.timesteps.to(device=clean_latents.device, dtype=torch.float32) + schedule_timesteps = scheduler.timesteps.to( + device=clean_latents.device, dtype=torch.float32 + ) schedule_sigmas = scheduler.sigmas.to( device=clean_latents.device, dtype=clean_latents.dtype, @@ -140,9 +154,13 @@ def single_train_step( noise = torch.randn_like(clean_latents) else: if not torch.is_tensor(noise): - raise TypeError("TrainingBatch.noise must be a " - "torch.Tensor when set") - noise = noise.permute(0, 2, 1, 3, 4).to(dtype=clean_latents.dtype) + raise TypeError( + "TrainingBatch.noise must be a " + "torch.Tensor when set" + ) + noise = noise.permute(0, 2, 1, 3, 4).to( + dtype=clean_latents.dtype + ) noisy_latents = self.student.add_noise( clean_latents, @@ -158,14 +176,22 @@ def single_train_step( attn_kind=self._attn_kind, ) - if bool(self.training_config.model.precondition_outputs): + if bool( + self.training_config.model.precondition_outputs + ): sigmas = schedule_sigmas[timestep_indices] - sigmas = sigmas.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) + sigmas = sigmas.unsqueeze(-1).unsqueeze( + -1 + ).unsqueeze(-1) pred_x0 = noisy_latents - pred * sigmas - loss = F.mse_loss(pred_x0.float(), clean_latents.float()) + loss = F.mse_loss( + pred_x0.float(), clean_latents.float() + ) else: target = noise - clean_latents - loss = F.mse_loss(pred.float(), target.float()) + loss = F.mse_loss( + pred.float(), target.float() + ) if self._attn_kind == "vsa": attn_metadata = training_batch.attn_metadata_vsa @@ -206,137 +232,124 @@ def backward( ) # TrainingMethod override: get_optimizers - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + def get_optimizers( + self, iteration: int, + ) -> list[torch.optim.Optimizer]: del iteration return [self._student_optimizer] # TrainingMethod override: get_lr_schedulers - def get_lr_schedulers(self, iteration: int) -> list[Any]: + def get_lr_schedulers( + self, iteration: int, + ) -> list[Any]: del iteration return [self._student_lr_scheduler] # TrainingMethod override: optimizers_schedulers_step - def optimizers_schedulers_step(self, iteration: int) -> None: - clip_grad_norm_if_needed(self.student.transformer, self.training_config.optimizer.max_grad_norm) - super().optimizers_schedulers_step(iteration) - - # Trainer hook: on_train_start - def on_train_start(self) -> None: - self.student.on_train_start() - - # Trainer hook: log_validation - def log_validation(self, iteration: int) -> None: - if not is_validation_enabled(self.validation_config): - return - - every_steps = parse_validation_every_steps(self.validation_config) - if every_steps <= 0: - return - if iteration % every_steps != 0: - return - - dataset_file = parse_validation_dataset_file(self.validation_config) - sampling_steps = parse_validation_sampling_steps(self.validation_config) - guidance_scale = parse_validation_guidance_scale(self.validation_config) - sampler_kind = parse_validation_sampler_kind(self.validation_config, default="ode") - rollout_mode = parse_validation_rollout_mode(self.validation_config) - output_dir = parse_validation_output_dir(self.validation_config) - num_actions = parse_validation_num_frames(self.validation_config) - ode_solver = parse_validation_ode_solver(self.validation_config, sampler_kind=sampler_kind) - - request = ValidationRequest( - sample_handle=self.student, - dataset_file=dataset_file, - sampling_steps=sampling_steps, - sampler_kind=sampler_kind, - rollout_mode=rollout_mode, - ode_solver=ode_solver, - sampling_timesteps=None, - guidance_scale=guidance_scale, - num_frames=num_actions, - output_dir=output_dir, + def optimizers_schedulers_step( + self, iteration: int, + ) -> None: + clip_grad_norm_if_needed( + self.student.transformer, + self.training_config.optimizer.max_grad_norm, ) - self.student.validator.log_validation(iteration, request=request) - - # Checkpoint hook: get_rng_generators - def get_rng_generators(self) -> dict[str, torch.Generator]: - generators: dict[str, torch.Generator] = {} - - student_gens = self.student.get_rng_generators() - generators.update(student_gens) - - if is_validation_enabled(self.validation_config): - validation_gen = self.student.validator.validation_random_generator - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen - - return generators - - def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: - if raw in (None, ""): - return "dense" - kind = str(raw).strip().lower() - if kind not in {"dense", "vsa"}: - raise ValueError("method_config.attn_kind must be one of " - f"{{'dense', 'vsa'}}, got {raw!r}.") - return cast(Literal["dense", "vsa"], kind) + super().optimizers_schedulers_step(iteration) def _parse_chunk_size(self, raw: Any) -> int: if raw in (None, ""): return 3 if isinstance(raw, bool): - raise ValueError("method_config.chunk_size must be an int, " - "got bool") + raise ValueError( + "method_config.chunk_size must be an int, " + "got bool" + ) if isinstance(raw, float) and not raw.is_integer(): - raise ValueError("method_config.chunk_size must be an int, " - "got float") + raise ValueError( + "method_config.chunk_size must be an int, " + "got float" + ) if isinstance(raw, str) and not raw.strip(): - raise ValueError("method_config.chunk_size must be an int, " - "got empty string") + raise ValueError( + "method_config.chunk_size must be an int, " + "got empty string" + ) try: value = int(raw) except (TypeError, ValueError) as e: - raise ValueError("method_config.chunk_size must be an int, " - f"got {type(raw).__name__}") from e + raise ValueError( + "method_config.chunk_size must be an int, " + f"got {type(raw).__name__}" + ) from e if value <= 0: - raise ValueError("method_config.chunk_size must be > 0") + raise ValueError( + "method_config.chunk_size must be > 0" + ) return value - def _parse_ratio(self, raw: Any, *, where: str, default: float) -> float: + def _parse_ratio( + self, + raw: Any, + *, + where: str, + default: float, + ) -> float: if raw in (None, ""): return float(default) if isinstance(raw, bool): - raise ValueError(f"{where} must be a number/string, got bool") + raise ValueError( + f"{where} must be a number/string, got bool" + ) if isinstance(raw, int | float): return float(raw) if isinstance(raw, str) and raw.strip(): return float(raw) - raise ValueError(f"{where} must be a number/string, " - f"got {type(raw).__name__}") + raise ValueError( + f"{where} must be a number/string, " + f"got {type(raw).__name__}" + ) - def _parse_timestep_index_range(self) -> tuple[int, int]: + def _parse_timestep_index_range( + self, + ) -> tuple[int, int]: scheduler = self.student.noise_scheduler if scheduler is None: - raise ValueError("DFSFT requires student.noise_scheduler") - num_steps = int(getattr(scheduler, "config", scheduler).num_train_timesteps) + raise ValueError( + "DFSFT requires student.noise_scheduler" + ) + num_steps = int( + getattr( + scheduler, "config", scheduler + ).num_train_timesteps + ) min_ratio = self._parse_ratio( - self.method_config.get("min_timestep_ratio", None), + self.method_config.get( + "min_timestep_ratio", None + ), where="method.min_timestep_ratio", default=0.0, ) max_ratio = self._parse_ratio( - self.method_config.get("max_timestep_ratio", None), + self.method_config.get( + "max_timestep_ratio", None + ), where="method.max_timestep_ratio", default=1.0, ) - if not (0.0 <= min_ratio <= 1.0 and 0.0 <= max_ratio <= 1.0): - raise ValueError("DFSFT timestep ratios must be in [0,1], " - f"got min={min_ratio}, max={max_ratio}") + if not ( + 0.0 <= min_ratio <= 1.0 + and 0.0 <= max_ratio <= 1.0 + ): + raise ValueError( + "DFSFT timestep ratios must be in [0,1], " + f"got min={min_ratio}, max={max_ratio}" + ) if max_ratio < min_ratio: - raise ValueError("method_config.max_timestep_ratio must be " - ">= min_timestep_ratio") + raise ValueError( + "method_config.max_timestep_ratio must be " + ">= min_timestep_ratio" + ) min_index = int(min_ratio * num_steps) max_index = int(max_ratio * num_steps) @@ -352,11 +365,18 @@ def _init_optimizers_and_schedulers(self) -> None: tc = self.training_config student_lr = float(tc.optimizer.learning_rate) if student_lr <= 0.0: - raise ValueError("training.learning_rate must be > 0 for dfsft") + raise ValueError( + "training.learning_rate must be > 0 " + "for dfsft" + ) student_betas = tc.optimizer.betas student_sched = str(tc.optimizer.lr_scheduler) - student_params = [p for p in self.student.transformer.parameters() if p.requires_grad] + student_params = [ + p + for p in self.student.transformer.parameters() + if p.requires_grad + ] ( self._student_optimizer, self._student_lr_scheduler, @@ -377,7 +397,9 @@ def _sample_t_inhom_indices( device: torch.device, ) -> torch.Tensor: chunk_size = self._chunk_size - num_chunks = (num_latents + chunk_size - 1) // chunk_size + num_chunks = ( + (num_latents + chunk_size - 1) // chunk_size + ) low, high = self._timestep_index_range chunk_indices = torch.randint( low=low, @@ -386,5 +408,7 @@ def _sample_t_inhom_indices( device=device, dtype=torch.long, ) - expanded = chunk_indices.repeat_interleave(chunk_size, dim=1) + expanded = chunk_indices.repeat_interleave( + chunk_size, dim=1 + ) return expanded[:, :num_latents] diff --git a/fastvideo/train/methods/fine_tuning/finetune.py b/fastvideo/train/methods/fine_tuning/finetune.py index da05f79d3..2c10339da 100644 --- a/fastvideo/train/methods/fine_tuning/finetune.py +++ b/fastvideo/train/methods/fine_tuning/finetune.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import Any, Literal, TYPE_CHECKING, cast +from typing import Any, Literal import torch import torch.nn.functional as F @@ -14,22 +14,6 @@ build_optimizer_and_scheduler, clip_grad_norm_if_needed, ) -from fastvideo.train.utils.validation import ( - is_validation_enabled, - parse_validation_dataset_file, - parse_validation_every_steps, - parse_validation_guidance_scale, - parse_validation_num_frames, - parse_validation_ode_solver, - parse_validation_output_dir, - parse_validation_rollout_mode, - parse_validation_sampler_kind, - parse_validation_sampling_steps, -) -from fastvideo.train.validators.base import ValidationRequest - -if TYPE_CHECKING: - pass class FineTuneMethod(TrainingMethod): @@ -41,17 +25,22 @@ def __init__( cfg: Any, role_models: dict[str, ModelBase], ) -> None: - super().__init__(role_models=role_models) + super().__init__(cfg=cfg, role_models=role_models) if "student" not in role_models: - raise ValueError("FineTuneMethod requires role 'student'") - self.student = role_models["student"] + raise ValueError( + "FineTuneMethod requires role 'student'" + ) if not getattr(self.student, "_trainable", True): - raise ValueError("FineTuneMethod requires student to be trainable") - self.training_config = cfg.training - self.method_config: dict[str, Any] = dict(cfg.method) - self.validation_config: dict[str, Any] = dict(getattr(cfg, "validation", {}) or {}) - self._attn_kind: Literal["dense", "vsa"] = (self._parse_attn_kind(self.method_config.get("attn_kind", None))) + raise ValueError( + "FineTuneMethod requires student to be " + "trainable" + ) + self._attn_kind: Literal["dense", "vsa"] = ( + self._parse_attn_kind( + self.method_config.get("attn_kind", None) + ) + ) # Initialize preprocessors on student. self.student.init_preprocessors(self.training_config) @@ -86,21 +75,40 @@ def single_train_step( ) if training_batch.latents is None: - raise RuntimeError("prepare_batch() must set TrainingBatch.latents") + raise RuntimeError( + "prepare_batch() must set " + "TrainingBatch.latents" + ) if training_batch.noisy_model_input is None: - raise RuntimeError("prepare_batch() must set " - "TrainingBatch.noisy_model_input") + raise RuntimeError( + "prepare_batch() must set " + "TrainingBatch.noisy_model_input" + ) if training_batch.noise is None: - raise RuntimeError("prepare_batch() must set TrainingBatch.noise") + raise RuntimeError( + "prepare_batch() must set " + "TrainingBatch.noise" + ) if training_batch.sigmas is None: - raise RuntimeError("prepare_batch() must set TrainingBatch.sigmas") + raise RuntimeError( + "prepare_batch() must set " + "TrainingBatch.sigmas" + ) if training_batch.timesteps is None: - raise RuntimeError("prepare_batch() must set " - "TrainingBatch.timesteps") + raise RuntimeError( + "prepare_batch() must set " + "TrainingBatch.timesteps" + ) clean_latents = training_batch.latents - noisy_latents = training_batch.noisy_model_input.permute(0, 2, 1, 3, 4) - noise = training_batch.noise.permute(0, 2, 1, 3, 4) + noisy_latents = ( + training_batch.noisy_model_input.permute( + 0, 2, 1, 3, 4 + ) + ) + noise = training_batch.noise.permute( + 0, 2, 1, 3, 4 + ) sigmas = training_batch.sigmas timesteps = training_batch.timesteps @@ -112,19 +120,28 @@ def single_train_step( attn_kind=self._attn_kind, ) - if bool(self.training_config.model.precondition_outputs): + if bool( + self.training_config.model.precondition_outputs + ): pred_x0 = noisy_latents - pred * sigmas - loss = F.mse_loss(pred_x0.float(), clean_latents.float()) + loss = F.mse_loss( + pred_x0.float(), clean_latents.float() + ) else: target = noise - clean_latents - loss = F.mse_loss(pred.float(), target.float()) + loss = F.mse_loss( + pred.float(), target.float() + ) if self._attn_kind == "vsa": attn_metadata = training_batch.attn_metadata_vsa else: attn_metadata = training_batch.attn_metadata - loss_map = {"total_loss": loss, "finetune_loss": loss} + loss_map = { + "total_loss": loss, + "finetune_loss": loss, + } outputs: dict[str, Any] = { "_fv_backward": ( training_batch.timesteps, @@ -158,91 +175,46 @@ def backward( ) # TrainingMethod override: get_optimizers - def get_optimizers(self, iteration: int) -> list[torch.optim.Optimizer]: + def get_optimizers( + self, iteration: int, + ) -> list[torch.optim.Optimizer]: del iteration return [self._student_optimizer] # TrainingMethod override: get_lr_schedulers - def get_lr_schedulers(self, iteration: int) -> list[Any]: + def get_lr_schedulers( + self, iteration: int, + ) -> list[Any]: del iteration return [self._student_lr_scheduler] # TrainingMethod override: optimizers_schedulers_step - def optimizers_schedulers_step(self, iteration: int) -> None: - clip_grad_norm_if_needed(self.student.transformer, self.training_config.optimizer.max_grad_norm) - super().optimizers_schedulers_step(iteration) - - # Trainer hook: on_train_start - def on_train_start(self) -> None: - self.student.on_train_start() - - # Trainer hook: log_validation - def log_validation(self, iteration: int) -> None: - if not is_validation_enabled(self.validation_config): - return - - every_steps = parse_validation_every_steps(self.validation_config) - if every_steps <= 0: - return - if iteration % every_steps != 0: - return - - dataset_file = parse_validation_dataset_file(self.validation_config) - sampling_steps = parse_validation_sampling_steps(self.validation_config) - guidance_scale = parse_validation_guidance_scale(self.validation_config) - sampler_kind = parse_validation_sampler_kind(self.validation_config, default="ode") - rollout_mode = parse_validation_rollout_mode(self.validation_config) - output_dir = parse_validation_output_dir(self.validation_config) - num_actions = parse_validation_num_frames(self.validation_config) - ode_solver = parse_validation_ode_solver(self.validation_config, sampler_kind=sampler_kind) - - request = ValidationRequest( - sample_handle=self.student, - dataset_file=dataset_file, - sampling_steps=sampling_steps, - sampler_kind=sampler_kind, - rollout_mode=rollout_mode, - ode_solver=ode_solver, - sampling_timesteps=None, - guidance_scale=guidance_scale, - num_frames=num_actions, - output_dir=output_dir, + def optimizers_schedulers_step( + self, iteration: int, + ) -> None: + clip_grad_norm_if_needed( + self.student.transformer, + self.training_config.optimizer.max_grad_norm, ) - self.student.validator.log_validation(iteration, request=request) - - # Checkpoint hook: get_rng_generators - def get_rng_generators(self) -> dict[str, torch.Generator]: - generators: dict[str, torch.Generator] = {} - - student_gens = self.student.get_rng_generators() - generators.update(student_gens) - - if is_validation_enabled(self.validation_config): - validation_gen = self.student.validator.validation_random_generator - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen - - return generators - - def _parse_attn_kind(self, raw: Any) -> Literal["dense", "vsa"]: - if raw in (None, ""): - return "dense" - kind = str(raw).strip().lower() - if kind not in {"dense", "vsa"}: - raise ValueError("method_config.attn_kind must be one of " - f"{{'dense', 'vsa'}}, got {raw!r}.") - return cast(Literal["dense", "vsa"], kind) + super().optimizers_schedulers_step(iteration) def _init_optimizers_and_schedulers(self) -> None: tc = self.training_config student_lr = float(tc.optimizer.learning_rate) if student_lr <= 0.0: - raise ValueError("training.learning_rate must be > 0 for finetune") + raise ValueError( + "training.learning_rate must be > 0 " + "for finetune" + ) student_betas = tc.optimizer.betas student_sched = str(tc.optimizer.lr_scheduler) - student_params = [p for p in self.student.transformer.parameters() if p.requires_grad] + student_params = [ + p + for p in self.student.transformer.parameters() + if p.requires_grad + ] ( self._student_optimizer, self._student_lr_scheduler, diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index a290e4a78..7ef411146 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -175,7 +175,7 @@ def on_train_start(self) -> None: seed = self.training_config.data.seed if seed is None: raise ValueError("training.data.seed must be set " - "for distillation") + "for training") global_rank = int(getattr(self.world_group, "rank", 0)) sp_world_size = int(self.training_config.distributed.sp_size or 1) diff --git a/fastvideo/train/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py index e30a4bbc6..cb251a867 100644 --- a/fastvideo/train/models/wangame/wangame.py +++ b/fastvideo/train/models/wangame/wangame.py @@ -189,7 +189,7 @@ def on_train_start(self) -> None: seed = tc.data.seed if seed is None: raise ValueError("training.data.seed must be set " - "for distillation") + "for training") global_rank = int(getattr(self.world_group, "rank", 0)) sp_world_size = int(tc.distributed.sp_size or 1) diff --git a/fastvideo/train/trainer.py b/fastvideo/train/trainer.py index 46a638cf4..1bba98fa1 100644 --- a/fastvideo/train/trainer.py +++ b/fastvideo/train/trainer.py @@ -11,6 +11,7 @@ from tqdm.auto import tqdm from fastvideo.distributed import get_sp_group, get_world_group +from fastvideo.train.methods.base import TrainingMethod from fastvideo.train.utils.tracking import build_tracker if TYPE_CHECKING: @@ -80,7 +81,7 @@ def _get_current_vsa_sparsity(self, step: int) -> float: def run( self, - method: torch.nn.Module, + method: TrainingMethod, *, dataloader: Any, max_steps: int, @@ -93,11 +94,8 @@ def run( int(tc.loop.gradient_accumulation_steps or 1), ) - if hasattr(method, "set_tracker"): - method.set_tracker(self.tracker) # type: ignore[attr-defined] - - if hasattr(method, "on_train_start"): - method.on_train_start() # type: ignore[attr-defined] + method.set_tracker(self.tracker) + method.on_train_start() resume_from_checkpoint = (tc.checkpoint.resume_from_checkpoint or "") if checkpoint_manager is not None: @@ -105,11 +103,8 @@ def run( if resumed_step is not None: start_step = int(resumed_step) - if hasattr(method, "log_validation"): - method.log_validation(start_step) # type: ignore[attr-defined] - - if hasattr(method, "optimizers_zero_grad"): - method.optimizers_zero_grad(start_step) # type: ignore[attr-defined] + method.log_validation(start_step) + method.optimizers_zero_grad(start_step) data_stream = self._iter_dataloader(dataloader) progress = tqdm( @@ -126,25 +121,17 @@ def run( metric_sums: dict[str, float] = {} for accum_iter in range(grad_accum): batch = next(data_stream) - if hasattr(method, "single_train_step"): - loss_map, outputs, step_metrics = method.single_train_step( # type: ignore[attr-defined] - batch, - step, - current_vsa_sparsity=(current_vsa_sparsity), - ) - else: - raise AttributeError("method must implement " - "single_train_step()") - - if hasattr(method, "backward"): - method.backward( # type: ignore[attr-defined] - loss_map, - outputs, - grad_accum_rounds=grad_accum, - ) - else: - total_loss = (loss_map["total_loss"] / grad_accum) - total_loss.backward() + loss_map, outputs, step_metrics = method.single_train_step( + batch, + step, + current_vsa_sparsity=(current_vsa_sparsity), + ) + + method.backward( + loss_map, + outputs, + grad_accum_rounds=grad_accum, + ) for k, v in loss_map.items(): if isinstance(v, torch.Tensor): @@ -161,10 +148,8 @@ def run( f".metrics[{k!r}]"), ) - if hasattr(method, "optimizers_schedulers_step"): - method.optimizers_schedulers_step(step) # type: ignore[attr-defined] - if hasattr(method, "optimizers_zero_grad"): - method.optimizers_zero_grad(step) # type: ignore[attr-defined] + method.optimizers_schedulers_step(step) + method.optimizers_zero_grad(step) metrics = {k: v / grad_accum for k, v in loss_sums.items()} metrics.update({k: v / grad_accum for k, v in metric_sums.items()}) @@ -176,8 +161,7 @@ def run( if checkpoint_manager is not None: checkpoint_manager.maybe_save(step) - if hasattr(method, "log_validation"): - method.log_validation(step) # type: ignore[attr-defined] + method.log_validation(step) if checkpoint_manager is not None: checkpoint_manager.save_final(max_steps) diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index 2802a7838..2ccc5c774 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -403,7 +403,7 @@ class CheckpointConfig: class CheckpointManager: - """Role-based checkpoint manager for distillation runtime. + """Role-based checkpoint manager for training runtime. - Checkpoint policy lives in YAML (via TrainingArgs fields). - Resume path is typically provided via CLI (``--resume-from-checkpoint``). diff --git a/fastvideo/train/utils/training_config.py b/fastvideo/train/utils/training_config.py index a4fc15f4a..f70e054bc 100644 --- a/fastvideo/train/utils/training_config.py +++ b/fastvideo/train/utils/training_config.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Typed distillation training config — replaces TrainingArgs.""" +"""Typed training config — replaces TrainingArgs.""" from __future__ import annotations diff --git a/fastvideo/train/validators/wan.py b/fastvideo/train/validators/wan.py index 9f736ef33..6e459d6c7 100644 --- a/fastvideo/train/validators/wan.py +++ b/fastvideo/train/validators/wan.py @@ -59,7 +59,7 @@ class _ValidationStepResult: class WanValidator: - """Phase 2 standalone validator for Wan distillation.""" + """Phase 2 standalone validator for Wan training.""" def __init__( self, diff --git a/fastvideo/train/validators/wangame.py b/fastvideo/train/validators/wangame.py index 3dee07759..519089349 100644 --- a/fastvideo/train/validators/wangame.py +++ b/fastvideo/train/validators/wangame.py @@ -59,7 +59,7 @@ class _ValidationStepResult: class WanGameValidator: - """Standalone validator for WanGame distillation.""" + """Standalone validator for WanGame training.""" def __init__( self, diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index e2f5b8dc7..6d32e09ba 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -12,14 +12,14 @@ logger = init_logger(__name__) -def run_distillation_from_config( +def run_training_from_config( config_path: str, *, dry_run: bool = False, resume_from_checkpoint: str | None = None, override_output_dir: str | None = None, ) -> None: - """YAML-only distillation entrypoint (schema v2).""" + """YAML-only training entrypoint (schema v2).""" from fastvideo.distributed import ( maybe_init_distributed_environment_and_model_parallel, ) @@ -67,21 +67,14 @@ def run_distillation_from_config( keep_last=int(tc.checkpoint.checkpoints_total_limit or 0), ) - get_rng_generators = getattr(method, "get_rng_generators", None) - if not callable(get_rng_generators): - model = getattr(method, "model", None) - get_rng_generators = getattr(model, "get_rng_generators", None) - if not callable(get_rng_generators): - get_rng_generators = None - checkpoint_manager = CheckpointManager( - role_models=(getattr(method, '_role_models', None) or {}), - optimizers=(getattr(method, '_optimizer_dict', None) or {}), - lr_schedulers=(getattr(method, '_lr_scheduler_dict', None) or {}), + role_models=method._role_models, + optimizers=method._optimizer_dict, + lr_schedulers=method._lr_scheduler_dict, dataloader=dataloader, output_dir=tc.checkpoint.output_dir, config=ckpt_config, - get_rng_generators=get_rng_generators, + get_rng_generators=method.get_rng_generators, ) trainer.run( @@ -99,16 +92,16 @@ def main(args: Any) -> None: resume_from_checkpoint = getattr(args, "resume_from_checkpoint", None) override_output_dir = getattr(args, "override_output_dir", None) logger.info( - "Starting distillation from config=%s", + "Starting training from config=%s", config_path, ) - run_distillation_from_config( + run_training_from_config( config_path, dry_run=dry_run, resume_from_checkpoint=resume_from_checkpoint, override_output_dir=override_output_dir, ) - logger.info("Distillation completed") + logger.info("Training completed") if __name__ == "__main__": @@ -118,7 +111,7 @@ def main(args: Any) -> None: "--config", type=str, required=True, - help=("Path to distillation YAML config " + help=("Path to training YAML config " "(schema v2)."), ) parser.add_argument( From 7fe51fc93835f09f7d0f6ddad7bc1acef6bdc450 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Wed, 4 Mar 2026 23:53:24 +0000 Subject: [PATCH 185/214] remove getting trainable using getattr --- fastvideo/train/methods/distribution_matching/dmd2.py | 6 +++--- fastvideo/train/methods/fine_tuning/dfsft.py | 2 +- fastvideo/train/methods/fine_tuning/finetune.py | 2 +- fastvideo/train/models/base.py | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fastvideo/train/methods/distribution_matching/dmd2.py b/fastvideo/train/methods/distribution_matching/dmd2.py index e443b0256..21cf4b3a6 100644 --- a/fastvideo/train/methods/distribution_matching/dmd2.py +++ b/fastvideo/train/methods/distribution_matching/dmd2.py @@ -65,16 +65,16 @@ def __init__( self.teacher = role_models["teacher"] self.critic = role_models["critic"] - if not getattr(self.student, "_trainable", True): + if not self.student._trainable: raise ValueError( "DMD2Method requires student to be trainable" ) - if getattr(self.teacher, "_trainable", True): + if self.teacher._trainable: raise ValueError( "DMD2Method requires teacher to be " "non-trainable" ) - if not getattr(self.critic, "_trainable", True): + if not self.critic._trainable: raise ValueError( "DMD2Method requires critic to be trainable" ) diff --git a/fastvideo/train/methods/fine_tuning/dfsft.py b/fastvideo/train/methods/fine_tuning/dfsft.py index d59ad9488..e16e6f062 100644 --- a/fastvideo/train/methods/fine_tuning/dfsft.py +++ b/fastvideo/train/methods/fine_tuning/dfsft.py @@ -31,7 +31,7 @@ def __init__( if "student" not in role_models: raise ValueError("DFSFT requires role 'student'") - if not getattr(self.student, "_trainable", True): + if not self.student._trainable: raise ValueError( "DFSFT requires student to be trainable" ) diff --git a/fastvideo/train/methods/fine_tuning/finetune.py b/fastvideo/train/methods/fine_tuning/finetune.py index 2c10339da..09bc1005e 100644 --- a/fastvideo/train/methods/fine_tuning/finetune.py +++ b/fastvideo/train/methods/fine_tuning/finetune.py @@ -31,7 +31,7 @@ def __init__( raise ValueError( "FineTuneMethod requires role 'student'" ) - if not getattr(self.student, "_trainable", True): + if not self.student._trainable: raise ValueError( "FineTuneMethod requires student to be " "trainable" diff --git a/fastvideo/train/models/base.py b/fastvideo/train/models/base.py index dff3c26cc..d74406278 100644 --- a/fastvideo/train/models/base.py +++ b/fastvideo/train/models/base.py @@ -25,6 +25,7 @@ class ModelBase(ABC): transformer: torch.nn.Module noise_scheduler: Any + _trainable: bool # ------------------------------------------------------------------ # Lifecycle From c8f6d085cb9a62010c53767335870f470ade3c6a Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 5 Mar 2026 00:04:44 +0000 Subject: [PATCH 186/214] timestep fix for dfsft --- fastvideo/train/methods/fine_tuning/dfsft.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fastvideo/train/methods/fine_tuning/dfsft.py b/fastvideo/train/methods/fine_tuning/dfsft.py index e16e6f062..8a0d36c31 100644 --- a/fastvideo/train/methods/fine_tuning/dfsft.py +++ b/fastvideo/train/methods/fine_tuning/dfsft.py @@ -149,6 +149,11 @@ def single_train_step( ) t_inhom = schedule_timesteps[timestep_indices] + # Override the homogeneous timesteps from prepare_batch + # so that set_forward_context (in predict_noise and + # backward) receives the correct per-chunk timesteps. + training_batch.timesteps = t_inhom + noise = getattr(training_batch, "noise", None) if noise is None: noise = torch.randn_like(clean_latents) From f94a72589935c227f96837ef1f0873b93f232be9 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 5 Mar 2026 00:09:49 +0000 Subject: [PATCH 187/214] finetune vsa and wangame yaml --- ...2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 72 +++++++++++++++++ .../finetune_wangame2.1_i2v_1.3B.yaml | 78 +++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml create mode 100644 examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml diff --git a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml new file mode 100644 index 000000000..b81fc33a8 --- /dev/null +++ b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml @@ -0,0 +1,72 @@ +# V3 config: Wan 2.1 T2V 1.3B finetune with VSA (phase 3.4, 0.9 sparsity). +# +# Uses _target_-based instantiation — each model role is an independent +# class instance; the method class is resolved directly from the YAML. + +models: + student: + _target_: fastvideo.train.models.wan.WanModel + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + +method: + _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod + attn_kind: vsa + +training: + distributed: + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + data: + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + train_batch_size: 1 + training_cfg_rate: 0.1 + seed: 1000 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + optimizer: + learning_rate: 1.0e-6 + betas: [0.9, 0.999] + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + loop: + max_train_steps: 4000 + gradient_accumulation_steps: 1 + + checkpoint: + output_dir: outputs/phase3.4_wan2.1_finetune_vsa_0.9_v3 + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + checkpoints_total_limit: 3 + + tracker: + project_name: distillation_phase3_r + run_name: phase3.4_wan_finetune_vsa_0.9_v3 + + model: + enable_gradient_checkpointing_type: full + VSA_sparsity: 0.9 + VSA_decay_rate: 0.03 + VSA_decay_interval_steps: 1 + +validation: + enabled: true + dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + every_steps: 50 + sampling_steps: [50] + guidance_scale: 5.0 + +pipeline: + flow_shift: 3 + sampler_kind: ode diff --git a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml new file mode 100644 index 000000000..9963db43d --- /dev/null +++ b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml @@ -0,0 +1,78 @@ +# V3 config: WanGame 2.1 I2V 1.3B finetune (dense attention). +# +# Uses _target_-based instantiation — each model role is an independent +# class instance; the method class is resolved directly from the YAML. + +models: + student: + _target_: fastvideo.train.models.wangame.WanGameModel + init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps + trainable: true + +method: + _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod + attn_kind: dense + +training: + distributed: + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 8 + hsdp_shard_dim: 1 + + data: + data_path: >- + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, + /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 + dataloader_num_workers: 4 + train_batch_size: 1 + training_cfg_rate: 0.0 + seed: 1000 + num_latent_t: 20 + num_height: 352 + num_width: 640 + num_frames: 77 + + optimizer: + learning_rate: 1.0e-5 + betas: [0.9, 0.999] + weight_decay: 1.0e-4 + lr_scheduler: constant + lr_warmup_steps: 0 + max_grad_norm: 1.0 + + loop: + max_train_steps: 20000 + gradient_accumulation_steps: 1 + + checkpoint: + output_dir: outputs/wangame_finetune_v3 + training_state_checkpointing_steps: 1000 + weight_only_checkpointing_steps: 1000 + checkpoints_total_limit: 2 + + tracker: + project_name: distillation_wangame_r + run_name: wangame_finetune_v3 + + model: + enable_gradient_checkpointing_type: full + +validation: + enabled: true + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [40] + sampler_kind: ode + guidance_scale: 1.0 + +pipeline: + flow_shift: 3 + sampler_kind: ode From f3f36296099616ad2d6bd81c792688908a7f5275 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 5 Mar 2026 02:50:51 +0000 Subject: [PATCH 188/214] validator vsa sparsity --- ...etune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 10 ++++++---- examples/distillation/refactor/run.sh | 3 +++ fastvideo/train/utils/moduleloader.py | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml index b81fc33a8..8a22e3d07 100644 --- a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml +++ b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml @@ -51,14 +51,16 @@ training: checkpoints_total_limit: 3 tracker: - project_name: distillation_phase3_r + project_name: distillation_wangame_r run_name: phase3.4_wan_finetune_vsa_0.9_v3 model: enable_gradient_checkpointing_type: full - VSA_sparsity: 0.9 - VSA_decay_rate: 0.03 - VSA_decay_interval_steps: 1 + + vsa: + sparsity: 0.9 + decay_rate: 0.03 + decay_interval_steps: 1 validation: enabled: true diff --git a/examples/distillation/refactor/run.sh b/examples/distillation/refactor/run.sh index 7c77256f7..492a1188d 100755 --- a/examples/distillation/refactor/run.sh +++ b/examples/distillation/refactor/run.sh @@ -36,6 +36,9 @@ LOG_DIR="${LOG_DIR:-examples/distillation/refactor}" mkdir -p "${LOG_DIR}" LOG_FILE="${LOG_DIR}/${CONFIG_NAME}_${TIMESTAMP}.log" +source ~/conda/miniconda/bin/activate +conda activate alexfv + echo "=== Distillation Training ===" echo "Config: ${CONFIG}" echo "Num GPUs: ${NUM_GPUS}" diff --git a/fastvideo/train/utils/moduleloader.py b/fastvideo/train/utils/moduleloader.py index 845adc0cd..648b9ecc4 100644 --- a/fastvideo/train/utils/moduleloader.py +++ b/fastvideo/train/utils/moduleloader.py @@ -62,6 +62,7 @@ def make_inference_args( args.inference_mode = True args.mode = ExecutionMode.INFERENCE args.dit_cpu_offload = True + args.VSA_sparsity = tc.vsa.sparsity return args From 06788467d756f8dd8efd03c8edc09412f2878906 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 5 Mar 2026 22:43:23 +0000 Subject: [PATCH 189/214] validation callback --- .../refactor/dfsft_wangame_causal_v3.yaml | 22 +- ...2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 14 +- .../finetune_wangame2.1_i2v_1.3B.yaml | 16 +- .../self_forcing_wangame_causal_v3.yaml | 21 +- fastvideo/train/callbacks/__init__.py | 15 + fastvideo/train/callbacks/callback.py | 160 ++++ fastvideo/train/callbacks/validation.py | 804 ++++++++++++++++++ fastvideo/train/methods/base.py | 75 -- .../methods/distribution_matching/dmd2.py | 70 -- fastvideo/train/models/wan/wan.py | 10 - fastvideo/train/models/wangame/wangame.py | 10 - fastvideo/train/trainer.py | 22 +- fastvideo/train/utils/config.py | 10 + fastvideo/train/validators/__init__.py | 13 +- fastvideo/train/validators/base.py | 48 +- fastvideo/train/validators/wan.py | 369 +------- fastvideo/train/validators/wangame.py | 631 +------------- fastvideo/training/distillation.py | 6 +- 18 files changed, 1109 insertions(+), 1207 deletions(-) create mode 100644 fastvideo/train/callbacks/__init__.py create mode 100644 fastvideo/train/callbacks/callback.py create mode 100644 fastvideo/train/callbacks/validation.py diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index 8360750e8..2bc84b883 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -67,16 +67,18 @@ training: model: enable_gradient_checkpointing_type: full -validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [40] - rollout_mode: streaming - sampler_kind: ode - ode_solver: euler - guidance_scale: 1.0 - num_frames: 69 +callbacks: + validation: + _target_: fastvideo.train.callbacks.validation.ValidationCallback + pipeline_target: fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline.WanGameCausalDMDPipeline + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [40] + rollout_mode: streaming + sampler_kind: ode + ode_solver: euler + guidance_scale: 1.0 + num_frames: 69 pipeline: flow_shift: 3 diff --git a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml index 8a22e3d07..583a36a65 100644 --- a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml +++ b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml @@ -62,12 +62,14 @@ training: decay_rate: 0.03 decay_interval_steps: 1 -validation: - enabled: true - dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - every_steps: 50 - sampling_steps: [50] - guidance_scale: 5.0 +callbacks: + validation: + _target_: fastvideo.train.callbacks.validation.ValidationCallback + pipeline_target: fastvideo.pipelines.basic.wan.wan_pipeline.WanPipeline + dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + every_steps: 50 + sampling_steps: [50] + guidance_scale: 5.0 pipeline: flow_shift: 3 diff --git a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml index 9963db43d..29aeac1d3 100644 --- a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml +++ b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml @@ -65,13 +65,15 @@ training: model: enable_gradient_checkpointing_type: full -validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [40] - sampler_kind: ode - guidance_scale: 1.0 +callbacks: + validation: + _target_: fastvideo.train.callbacks.validation.ValidationCallback + pipeline_target: fastvideo.pipelines.basic.wan.wangame_i2v_pipeline.WanGameActionImageToVideoPipeline + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [40] + sampler_kind: ode + guidance_scale: 1.0 pipeline: flow_shift: 3 diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index 2acdf842e..f1725649c 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -100,15 +100,18 @@ training: model: enable_gradient_checkpointing_type: null -validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [4] - sampler_kind: sde - rollout_mode: streaming - guidance_scale: 1.0 - num_frames: 69 +callbacks: + validation: + _target_: fastvideo.train.callbacks.validation.ValidationCallback + pipeline_target: fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline.WanGameCausalDMDPipeline + dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json + every_steps: 100 + sampling_steps: [4] + sampler_kind: sde + rollout_mode: streaming + guidance_scale: 1.0 + num_frames: 69 + dmd_denoising_steps: [1000, 750, 500, 250] pipeline: flow_shift: 3 diff --git a/fastvideo/train/callbacks/__init__.py b/fastvideo/train/callbacks/__init__.py new file mode 100644 index 000000000..8649ee40f --- /dev/null +++ b/fastvideo/train/callbacks/__init__.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 + +from fastvideo.train.callbacks.callback import ( + Callback, + CallbackDict, +) +from fastvideo.train.callbacks.validation import ( + ValidationCallback, +) + +__all__ = [ + "Callback", + "CallbackDict", + "ValidationCallback", +] diff --git a/fastvideo/train/callbacks/callback.py b/fastvideo/train/callbacks/callback.py new file mode 100644 index 000000000..cbc0bd808 --- /dev/null +++ b/fastvideo/train/callbacks/callback.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Callback base class and CallbackDict manager. + +Adapted from FastGen's callback pattern to FastVideo's types. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, TYPE_CHECKING + +from fastvideo.logger import init_logger +from fastvideo.train.utils.instantiate import instantiate + +if TYPE_CHECKING: + from fastvideo.train.methods.base import TrainingMethod + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) + +logger = init_logger(__name__) + + +class Callback: + """Base callback with no-op hooks. + + Subclasses override whichever hooks they need. The + ``training_config`` and ``method`` attributes are set by + ``CallbackDict`` after instantiation. + """ + + training_config: TrainingConfig + method: TrainingMethod + + def on_train_start( + self, + method: TrainingMethod, + iteration: int = 0, + ) -> None: + pass + + def on_training_step_end( + self, + method: TrainingMethod, + loss_dict: dict[str, Any], + iteration: int = 0, + ) -> None: + pass + + def on_validation_begin( + self, + method: TrainingMethod, + iteration: int = 0, + ) -> None: + pass + + def on_validation_end( + self, + method: TrainingMethod, + iteration: int = 0, + ) -> None: + pass + + def on_train_end( + self, + method: TrainingMethod, + iteration: int = 0, + ) -> None: + pass + + def state_dict(self) -> dict[str, Any]: + return {} + + def load_state_dict( + self, state_dict: dict[str, Any], + ) -> None: + pass + + +class CallbackDict: + """Manages a collection of named callbacks. + + Instantiates each callback from its ``_target_`` config and + dispatches hook calls to all registered callbacks. + """ + + def __init__( + self, + callback_configs: dict[str, dict[str, Any]], + training_config: TrainingConfig, + ) -> None: + self._callbacks: dict[str, Callback] = {} + if not callback_configs: + return + for name, cb_cfg in callback_configs.items(): + if "_target_" not in cb_cfg: + logger.warning( + "Callback %r is missing '_target_', " + "skipping: %s", + name, + cb_cfg, + ) + continue + logger.info( + "Instantiating callback %r: %s", + name, + cb_cfg, + ) + cb = instantiate(cb_cfg) + if not isinstance(cb, Callback): + raise TypeError( + f"Callback {name!r} resolved to " + f"{type(cb).__name__}, expected a " + f"Callback subclass." + ) + cb.training_config = training_config + self._callbacks[name] = cb + + def __getattr__( + self, method_name: str, + ) -> Callable[..., Any]: + if method_name.startswith("_"): + raise AttributeError(method_name) + + if method_name == "state_dict": + + def _state_dict() -> dict[str, Any]: + return { + n: cb.state_dict() + for n, cb in self._callbacks.items() + } + + return _state_dict + + if method_name == "load_state_dict": + + def _load_state_dict( + state_dict: dict[str, Any], + ) -> None: + for n, cb in self._callbacks.items(): + if n in state_dict: + cb.load_state_dict(state_dict[n]) + else: + logger.warning( + "Callback %r not found in " + "checkpoint.", + n, + ) + + return _load_state_dict + + def _dispatch(*args: Any, **kwargs: Any) -> None: + for cb in self._callbacks.values(): + fn = getattr(cb, method_name, None) + if fn is None: + continue + if not callable(fn): + continue + fn(*args, **kwargs) + + return _dispatch diff --git a/fastvideo/train/callbacks/validation.py b/fastvideo/train/callbacks/validation.py new file mode 100644 index 000000000..7ff770cf9 --- /dev/null +++ b/fastvideo/train/callbacks/validation.py @@ -0,0 +1,804 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Validation callback (unified replacement for WanValidator +and WanGameValidator). + +All configuration is read from the YAML ``callbacks.validation`` +section. The pipeline class is resolved from +``pipeline_target``. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, TYPE_CHECKING + +import imageio +import numpy as np +import torch +import torchvision +from einops import rearrange +from torch.utils.data import DataLoader + +from fastvideo.configs.sample import SamplingParam +from fastvideo.dataset.validation_dataset import ( + ValidationDataset, ) +from fastvideo.distributed import ( + get_sp_group, + get_world_group, +) +from fastvideo.logger import init_logger +from fastvideo.pipelines import ForwardBatch +from fastvideo.train.callbacks.callback import Callback +from fastvideo.train.utils.instantiate import resolve_target +from fastvideo.train.utils.moduleloader import ( + make_inference_args, ) +from fastvideo.training.trackers import DummyTracker +from fastvideo.utils import shallow_asdict + +if TYPE_CHECKING: + from fastvideo.train.methods.base import TrainingMethod + from fastvideo.train.utils.training_config import ( + TrainingConfig, ) + +logger = init_logger(__name__) + + +@dataclass(slots=True) +class _ValidationStepResult: + videos: list[list[np.ndarray]] + captions: list[str] + + +class ValidationCallback(Callback): + """Generic validation callback driven entirely by YAML + config. + + Works with any pipeline that follows the + ``PipelineCls.from_pretrained(...)`` + ``pipeline.forward()`` + contract (Wan, WanGame parallel, WanGame causal/DMD, etc.). + """ + + def __init__( + self, + *, + pipeline_target: str, + dataset_file: str, + every_steps: int = 100, + sampling_steps: list[int] | None = None, + sampler_kind: str = "ode", + ode_solver: str | None = None, + guidance_scale: float | None = None, + num_frames: int | None = None, + output_dir: str | None = None, + sampling_timesteps: list[int] | None = None, + rollout_mode: str = "parallel", + **pipeline_kwargs: Any, + ) -> None: + self.pipeline_target = str(pipeline_target) + self.dataset_file = str(dataset_file) + self.every_steps = int(every_steps) + self.sampling_steps = ( + [int(s) for s in sampling_steps] + if sampling_steps + else [40] + ) + self.sampler_kind = str(sampler_kind) + self.ode_solver = ( + str(ode_solver) if ode_solver is not None + else None + ) + self.guidance_scale = ( + float(guidance_scale) + if guidance_scale is not None + else None + ) + self.num_frames = ( + int(num_frames) if num_frames is not None + else None + ) + self.output_dir = ( + str(output_dir) if output_dir is not None + else None + ) + self.sampling_timesteps = ( + [int(s) for s in sampling_timesteps] + if sampling_timesteps is not None + else None + ) + self.rollout_mode = str(rollout_mode) + self.pipeline_kwargs = dict(pipeline_kwargs) + + # Set after on_train_start. + self._pipeline: Any | None = None + self._pipeline_key: tuple[Any, ...] | None = None + self._sampling_param: SamplingParam | None = None + self.tracker: Any = DummyTracker() + self.validation_random_generator: ( + torch.Generator | None + ) = None + self.seed: int = 0 + + # ---------------------------------------------------------- + # Callback hooks + # ---------------------------------------------------------- + + def on_train_start( + self, + method: TrainingMethod, + iteration: int = 0, + ) -> None: + self.method = method + tc = self.training_config + + self.world_group = get_world_group() + self.sp_group = get_sp_group() + self.global_rank = self.world_group.rank + self.rank_in_sp_group = ( + self.sp_group.rank_in_group + ) + self.sp_world_size = self.sp_group.world_size + + seed = tc.data.seed + if seed is None: + raise ValueError( + "training.data.seed must be set " + "for validation" + ) + self.seed = int(seed) + self.validation_random_generator = ( + torch.Generator(device="cpu").manual_seed( + self.seed + ) + ) + + tracker = getattr(method, "tracker", None) + if tracker is not None: + self.tracker = tracker + + def on_validation_begin( + self, + method: TrainingMethod, + iteration: int = 0, + ) -> None: + if self.every_steps <= 0: + return + if iteration % self.every_steps != 0: + return + + self._run_validation(method, iteration) + + # ---------------------------------------------------------- + # Core validation logic + # ---------------------------------------------------------- + + def _run_validation( + self, + method: TrainingMethod, + step: int, + ) -> None: + tc = self.training_config + transformer = method.student.transformer + was_training = bool( + getattr(transformer, "training", False) + ) + + output_dir = ( + self.output_dir + or tc.checkpoint.output_dir + ) + + # For streaming SDE pipelines we may need to + # temporarily set dmd_denoising_steps on + # pipeline_config. + old_dmd_denoising_steps = getattr( + tc.pipeline_config, + "dmd_denoising_steps", + None, + ) + try: + transformer.eval() + num_sp_groups = ( + self.world_group.world_size + // self.sp_group.world_size + ) + + for num_inference_steps in self.sampling_steps: + self._maybe_set_dmd_denoising_steps( + tc, + num_inference_steps, + ) + + result = self._run_validation_for_steps( + num_inference_steps, + transformer=transformer, + ) + + if self.rank_in_sp_group != 0: + continue + + if self.global_rank == 0: + all_videos = list(result.videos) + all_captions = list(result.captions) + for sp_idx in range( + 1, num_sp_groups + ): + src = ( + sp_idx * self.sp_world_size + ) + recv_v = ( + self.world_group.recv_object( + src=src + ) + ) + recv_c = ( + self.world_group.recv_object( + src=src + ) + ) + all_videos.extend(recv_v) + all_captions.extend(recv_c) + + os.makedirs( + output_dir, exist_ok=True, + ) + video_filenames: list[str] = [] + sp = self._get_sampling_param() + for i, video in enumerate(all_videos): + fname = os.path.join( + output_dir, + f"validation_step_{step}" + f"_inference_steps_" + f"{num_inference_steps}" + f"_video_{i}.mp4", + ) + imageio.mimsave( + fname, + video, + fps=sp.fps, + ) + video_filenames.append(fname) + + video_logs = [] + for fname, cap in zip( + video_filenames, + all_captions, + strict=True, + ): + art = self.tracker.video( + fname, caption=cap, + ) + if art is not None: + video_logs.append(art) + if video_logs: + logs = { + f"validation_videos_" + f"{num_inference_steps}" + f"_steps": video_logs + } + self.tracker.log_artifacts( + logs, step, + ) + else: + self.world_group.send_object( + result.videos, dst=0, + ) + self.world_group.send_object( + result.captions, dst=0, + ) + finally: + if hasattr(tc.pipeline_config, "dmd_denoising_steps"): + tc.pipeline_config.dmd_denoising_steps = ( + old_dmd_denoising_steps + ) + if was_training: + transformer.train() + + def _maybe_set_dmd_denoising_steps( + self, + tc: TrainingConfig, + num_inference_steps: int, + ) -> None: + """Set dmd_denoising_steps on pipeline_config for + streaming SDE validation.""" + if self.rollout_mode != "streaming": + return + if self.sampler_kind != "sde": + return + if self.sampling_timesteps is not None: + tc.pipeline_config.dmd_denoising_steps = ( # type: ignore[union-attr] + list(self.sampling_timesteps) + ) + else: + timesteps = np.linspace( + 1000, 0, int(num_inference_steps), + ) + tc.pipeline_config.dmd_denoising_steps = [ # type: ignore[union-attr] + int(max(0, min(1000, round(t)))) + for t in timesteps + ] + + # Also set any pipeline-specific kwargs from + # YAML (e.g. dmd_denoising_steps override). + pk = self.pipeline_kwargs + if "dmd_denoising_steps" in pk: + tc.pipeline_config.dmd_denoising_steps = [ # type: ignore[union-attr] + int(s) + for s in pk["dmd_denoising_steps"] + ] + + # ---------------------------------------------------------- + # Pipeline management + # ---------------------------------------------------------- + + def _get_sampling_param(self) -> SamplingParam: + if self._sampling_param is None: + self._sampling_param = ( + SamplingParam.from_pretrained( + self.training_config.model_path + ) + ) + return self._sampling_param + + def _get_pipeline( + self, + *, + transformer: torch.nn.Module, + ) -> Any: + key = ( + id(transformer), + self.rollout_mode, + self.sampler_kind, + str(self.ode_solver), + ) + if ( + self._pipeline is not None + and self._pipeline_key == key + ): + return self._pipeline + + tc = self.training_config + PipelineCls = resolve_target(self.pipeline_target) + flow_shift = getattr( + tc.pipeline_config, "flow_shift", None, + ) + + kwargs: dict[str, Any] = { + "inference_mode": True, + "sampler_kind": self.sampler_kind, + "loaded_modules": { + "transformer": transformer, + }, + "tp_size": tc.distributed.tp_size, + "sp_size": tc.distributed.sp_size, + "num_gpus": tc.distributed.num_gpus, + "pin_cpu_memory": ( + tc.distributed.pin_cpu_memory + ), + "dit_cpu_offload": True, + } + if flow_shift is not None: + kwargs["flow_shift"] = float(flow_shift) + if self.ode_solver is not None: + kwargs["ode_solver"] = str(self.ode_solver) + + # For streaming pipelines, build and inject a + # scheduler if needed. + if self.rollout_mode == "streaming": + scheduler = self._build_streaming_scheduler( + flow_shift, + ) + if scheduler is not None: + kwargs["loaded_modules"]["scheduler"] = ( + scheduler + ) + + self._pipeline = PipelineCls.from_pretrained( + tc.model_path, **kwargs, + ) + self._pipeline_key = key + return self._pipeline + + def _build_streaming_scheduler( + self, flow_shift: float | None, + ) -> Any | None: + """Build scheduler for streaming validation.""" + if flow_shift is None: + return None + + if self.sampler_kind == "sde": + from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, ) + + return FlowMatchEulerDiscreteScheduler( + shift=float(flow_shift), + ) + + # ODE mode — choose based on ode_solver. + ode_solver_norm = ( + str(self.ode_solver).strip().lower() + if self.ode_solver is not None + else "unipc" + ) + if ode_solver_norm in { + "unipc", + "unipc_multistep", + "multistep", + }: + from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( + FlowUniPCMultistepScheduler, ) + + return FlowUniPCMultistepScheduler( + shift=float(flow_shift), + ) + if ode_solver_norm in { + "euler", + "flowmatch", + "flowmatch_euler", + }: + from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, ) + + return FlowMatchEulerDiscreteScheduler( + shift=float(flow_shift), + ) + + raise ValueError( + f"Unknown ode_solver for streaming " + f"validation: {self.ode_solver!r}" + ) + + # ---------------------------------------------------------- + # Batch preparation + # ---------------------------------------------------------- + + def _prepare_validation_batch( + self, + sampling_param: SamplingParam, + validation_batch: dict[str, Any], + num_inference_steps: int, + ) -> ForwardBatch: + tc = self.training_config + + sampling_param.prompt = validation_batch["prompt"] + sampling_param.height = tc.data.num_height + sampling_param.width = tc.data.num_width + sampling_param.num_inference_steps = int( + num_inference_steps + ) + sampling_param.data_type = "video" + if self.guidance_scale is not None: + sampling_param.guidance_scale = float( + self.guidance_scale + ) + sampling_param.seed = self.seed + + # image_path for I2V pipelines. + img_path = ( + validation_batch.get("image_path") + or validation_batch.get("video_path") + ) + if img_path is not None: + sampling_param.image_path = img_path + + temporal_compression_factor = int( + tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio # type: ignore[union-attr] + ) + default_num_frames = ( + (tc.data.num_latent_t - 1) + * temporal_compression_factor + + 1 + ) + if self.num_frames is not None: + sampling_param.num_frames = int( + self.num_frames + ) + else: + sampling_param.num_frames = int( + default_num_frames + ) + + latents_size = [ + (sampling_param.num_frames - 1) // 4 + 1, + sampling_param.height // 8, + sampling_param.width // 8, + ] + n_tokens = ( + latents_size[0] + * latents_size[1] + * latents_size[2] + ) + + sampling_timesteps_tensor = ( + torch.tensor( + [int(s) for s in self.sampling_timesteps], + dtype=torch.long, + ) + if self.sampling_timesteps is not None + else None + ) + + inference_args = make_inference_args( + tc, model_path=tc.model_path, + ) + + batch = ForwardBatch( + **shallow_asdict(sampling_param), + latents=None, + generator=self.validation_random_generator, + n_tokens=n_tokens, + eta=0.0, + VSA_sparsity=tc.vsa.sparsity, + timesteps=sampling_timesteps_tensor, + sampling_timesteps=sampling_timesteps_tensor, + ) + batch._inference_args = inference_args # type: ignore[attr-defined] + + # Conditionally set I2V / WanGame fields. + if ( + "image" in validation_batch + and validation_batch["image"] is not None + ): + batch.pil_image = validation_batch["image"] + + self._maybe_set_action_conds( + batch, validation_batch, sampling_param, + ) + return batch + + def _maybe_set_action_conds( + self, + batch: ForwardBatch, + validation_batch: dict[str, Any], + sampling_param: SamplingParam, + ) -> None: + """Set keyboard_cond / mouse_cond on the batch if + present in the dataset.""" + target_len = int(sampling_param.num_frames) + + if ( + "keyboard_cond" in validation_batch + and validation_batch["keyboard_cond"] + is not None + ): + kb = torch.as_tensor( + validation_batch["keyboard_cond"] + ).to(dtype=torch.bfloat16) + if kb.ndim == 3 and kb.shape[0] == 1: + kb = kb.squeeze(0) + if kb.ndim != 2: + raise ValueError( + "validation keyboard_cond must have" + " shape (T, K), got " + f"{tuple(kb.shape)}" + ) + if kb.shape[0] > target_len: + kb = kb[:target_len] + elif kb.shape[0] < target_len: + pad = torch.zeros( + ( + target_len - kb.shape[0], + kb.shape[1], + ), + dtype=kb.dtype, + device=kb.device, + ) + kb = torch.cat([kb, pad], dim=0) + batch.keyboard_cond = kb.unsqueeze(0) + + if ( + "mouse_cond" in validation_batch + and validation_batch["mouse_cond"] + is not None + ): + mc = torch.as_tensor( + validation_batch["mouse_cond"] + ).to(dtype=torch.bfloat16) + if mc.ndim == 3 and mc.shape[0] == 1: + mc = mc.squeeze(0) + if mc.ndim != 2: + raise ValueError( + "validation mouse_cond must have " + "shape (T, 2), got " + f"{tuple(mc.shape)}" + ) + if mc.shape[0] > target_len: + mc = mc[:target_len] + elif mc.shape[0] < target_len: + pad = torch.zeros( + ( + target_len - mc.shape[0], + mc.shape[1], + ), + dtype=mc.dtype, + device=mc.device, + ) + mc = torch.cat([mc, pad], dim=0) + batch.mouse_cond = mc.unsqueeze(0) + + # ---------------------------------------------------------- + # Post-processing + # ---------------------------------------------------------- + + def _post_process_validation_frames( + self, + frames: list[np.ndarray], + batch: ForwardBatch, + ) -> list[np.ndarray]: + """Overlay action indicators if conditions present.""" + keyboard_cond = getattr(batch, "keyboard_cond", None) + mouse_cond = getattr(batch, "mouse_cond", None) + if keyboard_cond is None and mouse_cond is None: + return frames + + try: + from fastvideo.models.dits.matrixgame.utils import ( + draw_keys_on_frame, + draw_mouse_on_frame, + ) + except Exception as e: + logger.warning( + "Action overlay unavailable: %s", e, + ) + return frames + + if ( + keyboard_cond is not None + and torch.is_tensor(keyboard_cond) + ): + keyboard_np = ( + keyboard_cond.squeeze(0) + .detach() + .cpu() + .float() + .numpy() + ) + else: + keyboard_np = None + + if ( + mouse_cond is not None + and torch.is_tensor(mouse_cond) + ): + mouse_np = ( + mouse_cond.squeeze(0) + .detach() + .cpu() + .float() + .numpy() + ) + else: + mouse_np = None + + key_names = ["W", "S", "A", "D", "left", "right"] + processed: list[np.ndarray] = [] + for fi, frame in enumerate(frames): + frame = np.ascontiguousarray(frame.copy()) + if ( + keyboard_np is not None + and fi < len(keyboard_np) + ): + keys = { + key_names[i]: bool( + keyboard_np[fi, i] + ) + for i in range( + min( + len(key_names), + int(keyboard_np.shape[1]), + ) + ) + } + draw_keys_on_frame( + frame, keys, mode="universal", + ) + if ( + mouse_np is not None + and fi < len(mouse_np) + ): + pitch = float(mouse_np[fi, 0]) + yaw = float(mouse_np[fi, 1]) + draw_mouse_on_frame(frame, pitch, yaw) + processed.append(frame) + return processed + + # ---------------------------------------------------------- + # Validation loop + # ---------------------------------------------------------- + + def _run_validation_for_steps( + self, + num_inference_steps: int, + *, + transformer: torch.nn.Module, + ) -> _ValidationStepResult: + tc = self.training_config + pipeline = self._get_pipeline( + transformer=transformer, + ) + sampling_param = self._get_sampling_param() + + dataset = ValidationDataset(self.dataset_file) + dataloader = DataLoader( + dataset, batch_size=None, num_workers=0, + ) + + inference_args = make_inference_args( + tc, model_path=tc.model_path, + ) + + videos: list[list[np.ndarray]] = [] + captions: list[str] = [] + + for validation_batch in dataloader: + batch = self._prepare_validation_batch( + sampling_param, + validation_batch, + num_inference_steps, + ) + + assert ( + batch.prompt is not None + and isinstance(batch.prompt, str) + ) + captions.append(batch.prompt) + + with torch.no_grad(): + output_batch = pipeline.forward( + batch, inference_args, + ) + + samples = output_batch.output.cpu() + if self.rank_in_sp_group != 0: + continue + + video = rearrange( + samples, "b c t h w -> t b c h w", + ) + frames: list[np.ndarray] = [] + for x in video: + x = torchvision.utils.make_grid( + x, nrow=6, + ) + x = ( + x.transpose(0, 1) + .transpose(1, 2) + .squeeze(-1) + ) + frames.append( + (x * 255).numpy().astype(np.uint8) + ) + frames = ( + self._post_process_validation_frames( + frames, batch, + ) + ) + videos.append(frames) + + return _ValidationStepResult( + videos=videos, captions=captions, + ) + + # ---------------------------------------------------------- + # State management + # ---------------------------------------------------------- + + def state_dict(self) -> dict[str, Any]: + state: dict[str, Any] = {} + if self.validation_random_generator is not None: + state["validation_rng"] = ( + self.validation_random_generator.get_state() + ) + return state + + def load_state_dict( + self, state_dict: dict[str, Any], + ) -> None: + rng_state = state_dict.get("validation_rng") + if ( + rng_state is not None + and self.validation_random_generator is not None + ): + self.validation_random_generator.set_state( + rng_state + ) diff --git a/fastvideo/train/methods/base.py b/fastvideo/train/methods/base.py index 30e02a862..8e01c0525 100644 --- a/fastvideo/train/methods/base.py +++ b/fastvideo/train/methods/base.py @@ -9,19 +9,6 @@ import torch from fastvideo.train.models.base import ModelBase -from fastvideo.train.utils.validation import ( - is_validation_enabled, - parse_validation_dataset_file, - parse_validation_every_steps, - parse_validation_guidance_scale, - parse_validation_num_frames, - parse_validation_ode_solver, - parse_validation_output_dir, - parse_validation_rollout_mode, - parse_validation_sampler_kind, - parse_validation_sampling_steps, -) -from fastvideo.train.validators.base import ValidationRequest LogScalar = float | int | torch.Tensor @@ -67,16 +54,6 @@ def __init__( def set_tracker(self, tracker: Any) -> None: self.tracker = tracker - student = self._role_models.get("student") - if student is None: - return - validator = getattr(student, "validator", None) - if validator is None: - return - if hasattr(validator, "set_tracker"): - validator.set_tracker(tracker) - elif hasattr(validator, "tracker"): - validator.tracker = tracker # type: ignore[attr-defined] @abstractmethod def single_train_step( @@ -155,60 +132,8 @@ def get_rng_generators( student_gens = self.student.get_rng_generators() generators.update(student_gens) - if is_validation_enabled(self.validation_config): - validation_gen = ( - self.student.validator.validation_random_generator - ) - if isinstance(validation_gen, torch.Generator): - generators["validation_cpu"] = validation_gen - return generators - def log_validation(self, iteration: int) -> None: - if not is_validation_enabled(self.validation_config): - return - - every_steps = parse_validation_every_steps( - self.validation_config, - ) - if every_steps <= 0: - return - if iteration % every_steps != 0: - return - - request = self._build_validation_request() - self.student.validator.log_validation( - iteration, request=request, - ) - - def _build_validation_request(self) -> ValidationRequest: - """Build the ``ValidationRequest`` for validation. - - Override in subclasses that need custom parameters (e.g. - ``sampling_timesteps`` for DMD2). - """ - vc = self.validation_config - sampling_steps = parse_validation_sampling_steps(vc) - guidance_scale = parse_validation_guidance_scale(vc) - sampler_kind = parse_validation_sampler_kind( - vc, default="ode", - ) - ode_solver = parse_validation_ode_solver( - vc, sampler_kind=sampler_kind, - ) - return ValidationRequest( - sample_handle=self.student, - dataset_file=parse_validation_dataset_file(vc), - sampling_steps=sampling_steps, - sampler_kind=sampler_kind, - rollout_mode=parse_validation_rollout_mode(vc), - ode_solver=ode_solver, - sampling_timesteps=None, - guidance_scale=guidance_scale, - num_frames=parse_validation_num_frames(vc), - output_dir=parse_validation_output_dir(vc), - ) - @staticmethod def _parse_attn_kind( raw: Any, diff --git a/fastvideo/train/methods/distribution_matching/dmd2.py b/fastvideo/train/methods/distribution_matching/dmd2.py index 21cf4b3a6..d2f36ca4d 100644 --- a/fastvideo/train/methods/distribution_matching/dmd2.py +++ b/fastvideo/train/methods/distribution_matching/dmd2.py @@ -14,17 +14,6 @@ build_optimizer_and_scheduler, clip_grad_norm_if_needed, ) -from fastvideo.train.utils.validation import ( - parse_validation_dataset_file, - parse_validation_guidance_scale, - parse_validation_num_frames, - parse_validation_ode_solver, - parse_validation_output_dir, - parse_validation_rollout_mode, - parse_validation_sampler_kind, - parse_validation_sampling_steps, -) -from fastvideo.train.validators.base import ValidationRequest from fastvideo.train.utils.config import ( get_optional_float, get_optional_int, @@ -253,65 +242,6 @@ def optimizers_schedulers_step( ) super().optimizers_schedulers_step(iteration) - # Override: DMD2-specific validation request with - # sampling_timesteps and default="sde". - def _build_validation_request( - self, - ) -> ValidationRequest: - vc = self.validation_config - sampling_steps = parse_validation_sampling_steps(vc) - - sampling_timesteps: list[int] | None = None - raw_timesteps = vc.get( - "sampling_timesteps", None - ) - if raw_timesteps is None: - raw_timesteps = self.method_config.get( - "dmd_denoising_steps", None - ) - if isinstance(raw_timesteps, list) and raw_timesteps: - sampling_timesteps = [ - int(s) for s in raw_timesteps - ] - - if not sampling_steps: - if sampling_timesteps is None: - return ValidationRequest( - sample_handle=self.student - ) - sampling_steps = [int(len(sampling_timesteps))] - - sampler_kind = parse_validation_sampler_kind( - vc, default="sde" - ) - ode_solver = parse_validation_ode_solver( - vc, sampler_kind=sampler_kind - ) - if ( - sampling_timesteps is not None - and sampler_kind != "sde" - ): - raise ValueError( - "method_config.validation." - "sampling_timesteps is " - "only valid when sampler_kind='sde'" - ) - - return ValidationRequest( - sample_handle=self.student, - dataset_file=parse_validation_dataset_file(vc), - sampling_steps=sampling_steps, - sampler_kind=sampler_kind, - rollout_mode=parse_validation_rollout_mode(vc), - ode_solver=ode_solver, - sampling_timesteps=sampling_timesteps, - guidance_scale=parse_validation_guidance_scale( - vc - ), - num_frames=parse_validation_num_frames(vc), - output_dir=parse_validation_output_dir(vc), - ) - def _parse_rollout_mode( self, ) -> Literal["simulate", "data_latent"]: diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index 7ef411146..557bf7512 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -133,16 +133,6 @@ def init_preprocessors(self, training_config: TrainingConfig) -> None: self._init_timestep_mechanics() - # Optional validator. - validation_cfg = training_config.validation - if validation_cfg: - validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) - if validation_enabled: - from fastvideo.train.validators.wan import ( - WanValidator, ) - - self.validator = WanValidator(training_config=training_config) - from fastvideo.dataset.dataloader.schema import ( pyarrow_schema_t2v, ) from fastvideo.train.utils.dataloader import ( diff --git a/fastvideo/train/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py index cb251a867..47159c6de 100644 --- a/fastvideo/train/models/wangame/wangame.py +++ b/fastvideo/train/models/wangame/wangame.py @@ -142,16 +142,6 @@ def init_preprocessors(self, training_config: TrainingConfig) -> None: self._init_timestep_mechanics() - # Optional validator. - validation_cfg = training_config.validation - if validation_cfg: - validation_enabled = bool(validation_cfg.get("enabled", bool(validation_cfg))) - if validation_enabled: - from fastvideo.train.validators.wangame import ( - WanGameValidator, ) - - self.validator = WanGameValidator(training_config=training_config) - from fastvideo.dataset.dataloader.schema import ( pyarrow_schema_wangame, ) from fastvideo.train.utils.dataloader import ( diff --git a/fastvideo/train/trainer.py b/fastvideo/train/trainer.py index 1bba98fa1..3900177d3 100644 --- a/fastvideo/train/trainer.py +++ b/fastvideo/train/trainer.py @@ -11,6 +11,7 @@ from tqdm.auto import tqdm from fastvideo.distributed import get_sp_group, get_world_group +from fastvideo.train.callbacks.callback import CallbackDict from fastvideo.train.methods.base import TrainingMethod from fastvideo.train.utils.tracking import build_tracker @@ -45,6 +46,8 @@ def __init__( training_config: TrainingConfig, *, config: dict[str, Any] | None = None, + callback_configs: dict[str, dict[str, Any]] + | None = None, ) -> None: self.training_config = training_config self.world_group = get_world_group() @@ -56,6 +59,10 @@ def __init__( training_config.checkpoint, config=config, ) + self.callbacks = CallbackDict( + callback_configs or {}, + training_config, + ) def _iter_dataloader(self, dataloader: Any) -> Iterator[dict[str, Any]]: data_iter = iter(dataloader) @@ -103,7 +110,12 @@ def run( if resumed_step is not None: start_step = int(resumed_step) - method.log_validation(start_step) + self.callbacks.on_train_start( + method, iteration=start_step, + ) + self.callbacks.on_validation_begin( + method, iteration=start_step, + ) method.optimizers_zero_grad(start_step) data_stream = self._iter_dataloader(dataloader) @@ -161,7 +173,13 @@ def run( if checkpoint_manager is not None: checkpoint_manager.maybe_save(step) - method.log_validation(step) + self.callbacks.on_validation_begin( + method, iteration=step, + ) + + self.callbacks.on_train_end( + method, iteration=max_steps, + ) if checkpoint_manager is not None: checkpoint_manager.save_final(max_steps) diff --git a/fastvideo/train/utils/config.py b/fastvideo/train/utils/config.py index a576ede26..390b11478 100644 --- a/fastvideo/train/utils/config.py +++ b/fastvideo/train/utils/config.py @@ -31,6 +31,7 @@ class RunConfig: method: dict[str, Any] training: TrainingConfig validation: dict[str, Any] + callbacks: dict[str, dict[str, Any]] raw: dict[str, Any] @@ -440,6 +441,14 @@ def load_run_config(path: str) -> RunConfig: validation = _require_mapping( validation_raw, where="validation") + # --- callbacks --- + callbacks_raw = cfg.get("callbacks", None) + if callbacks_raw is None: + callbacks: dict[str, dict[str, Any]] = {} + else: + callbacks = _require_mapping( + callbacks_raw, where="callbacks") + # --- pipeline config --- pipeline_config = _parse_pipeline_config( cfg, models=models) @@ -458,5 +467,6 @@ def load_run_config(path: str) -> RunConfig: method=method, training=training, validation=validation, + callbacks=callbacks, raw=cfg, ) diff --git a/fastvideo/train/validators/__init__.py b/fastvideo/train/validators/__init__.py index b50574b29..b439ff081 100644 --- a/fastvideo/train/validators/__init__.py +++ b/fastvideo/train/validators/__init__.py @@ -1,10 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 +"""Deprecated — use fastvideo.train.callbacks instead.""" -from fastvideo.train.validators.base import Validator, ValidationRequest -from fastvideo.train.validators.wan import WanValidator +from fastvideo.train.callbacks.callback import Callback +from fastvideo.train.callbacks.validation import ( + ValidationCallback, +) + +# Backwards-compatible aliases. +Validator = Callback +WanValidator = ValidationCallback __all__ = [ "Validator", - "ValidationRequest", "WanValidator", + "ValidationCallback", ] diff --git a/fastvideo/train/validators/base.py b/fastvideo/train/validators/base.py index dd89b9de1..76a0892e4 100644 --- a/fastvideo/train/validators/base.py +++ b/fastvideo/train/validators/base.py @@ -1,39 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 +"""Deprecated — use fastvideo.train.callbacks instead. -from __future__ import annotations - -from dataclasses import dataclass -from abc import ABC, abstractmethod -from typing import Literal - -from fastvideo.train.models.base import ModelBase +Kept as import shims for backwards compatibility. +""" +from __future__ import annotations -@dataclass(slots=True) -class ValidationRequest: - """Method-provided validation configuration overrides. - - Validators are model-plugin-specific (e.g. Wan sampling), but should remain - method-agnostic. A method may override key sampling parameters by passing a - request object here. - """ - - sample_handle: ModelBase | None = None - dataset_file: str | None = None - sampling_steps: list[int] | None = None - sampler_kind: Literal["ode", "sde"] | None = None - rollout_mode: Literal["parallel", "streaming"] | None = None - ode_solver: str | None = None - sampling_timesteps: list[int] | None = None - guidance_scale: float | None = None - # Optional override for validation video length. When set, validators may - # truncate/pad auxiliary sequences (e.g. WanGame keyboard/mouse) to match. - num_frames: int | None = None - output_dir: str | None = None +import warnings +from fastvideo.train.callbacks.callback import Callback -class Validator(ABC): +warnings.warn( + "fastvideo.train.validators.base is deprecated. " + "Use fastvideo.train.callbacks instead.", + DeprecationWarning, + stacklevel=2, +) - @abstractmethod - def log_validation(self, step: int, *, request: ValidationRequest | None = None) -> None: - raise NotImplementedError +# Provide the old names as aliases so that existing +# imports do not break immediately. +Validator = Callback +ValidationRequest = None # type: ignore[assignment] diff --git a/fastvideo/train/validators/wan.py b/fastvideo/train/validators/wan.py index 6e459d6c7..3526dc8da 100644 --- a/fastvideo/train/validators/wan.py +++ b/fastvideo/train/validators/wan.py @@ -1,364 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 -"""Wan validator (model-family validation backend). +"""Deprecated — use fastvideo.train.callbacks.ValidationCallback. -Config keys used: -- `training` (TrainingConfig): - - `data.seed`, `model_path` - - `data.num_height`, `data.num_width`, `data.num_latent_t` - - `distributed.tp_size`, `distributed.sp_size`, - `distributed.num_gpus`, `distributed.pin_cpu_memory` - - `pipeline_config.flow_shift` - - `pipeline_config.vae_config.arch_config - .temporal_compression_ratio` - - `vsa.sparsity` -- `training.validation.*` (typically parsed by a method into - `ValidationRequest`): - - `dataset_file`, `sampling_steps`, `guidance_scale` - - `sampler_kind` (`ode`/`sde`), `ode_solver` (`euler`/`unipc`), - `rollout_mode` +Kept as an import shim for backwards compatibility. """ from __future__ import annotations -import os -from dataclasses import dataclass -from typing import Any, TYPE_CHECKING +import warnings -import imageio -import numpy as np -import torch -import torchvision -from einops import rearrange -from torch.utils.data import DataLoader +from fastvideo.train.callbacks.validation import ( + ValidationCallback, +) -from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.validation_dataset import ( - ValidationDataset, ) -from fastvideo.distributed import get_sp_group, get_world_group -from fastvideo.logger import init_logger -from fastvideo.pipelines import ForwardBatch -from fastvideo.train.utils.moduleloader import ( - make_inference_args, ) -from fastvideo.train.validators.base import ( - ValidationRequest, ) -from fastvideo.pipelines.basic.wan.wan_pipeline import WanPipeline -from fastvideo.training.trackers import DummyTracker -from fastvideo.utils import shallow_asdict +warnings.warn( + "fastvideo.train.validators.wan is deprecated. " + "Use fastvideo.train.callbacks.ValidationCallback " + "instead.", + DeprecationWarning, + stacklevel=2, +) -if TYPE_CHECKING: - from fastvideo.train.utils.training_config import ( - TrainingConfig, ) - -logger = init_logger(__name__) - - -@dataclass(slots=True) -class _ValidationStepResult: - videos: list[list[np.ndarray]] - captions: list[str] - - -class WanValidator: - """Phase 2 standalone validator for Wan training.""" - - def __init__( - self, - *, - training_config: TrainingConfig, - tracker: Any | None = None, - ) -> None: - self.training_config = training_config - self.tracker = tracker or DummyTracker() - - self.world_group = get_world_group() - self.sp_group = get_sp_group() - self.global_rank = self.world_group.rank - self.rank_in_sp_group = self.sp_group.rank_in_group - self.sp_world_size = self.sp_group.world_size - - seed = training_config.data.seed - if seed is None: - raise ValueError("training.data.seed must be set for validation") - self.seed = int(seed) - self.validation_random_generator = (torch.Generator(device="cpu").manual_seed(self.seed)) - - self._pipeline: WanPipeline | None = None - self._pipeline_key: (tuple[int, str, str] | None) = None - self._sampling_param: SamplingParam | None = None - - def set_tracker(self, tracker: Any) -> None: - self.tracker = tracker - - def _get_sampling_param(self) -> SamplingParam: - if self._sampling_param is None: - self._sampling_param = (SamplingParam.from_pretrained(self.training_config.model_path)) - return self._sampling_param - - def _get_pipeline( - self, - *, - transformer: torch.nn.Module, - sampler_kind: str, - ode_solver: str | None, - ) -> WanPipeline: - key = ( - id(transformer), - str(sampler_kind), - str(ode_solver), - ) - if (self._pipeline is not None and self._pipeline_key == key): - return self._pipeline - - tc = self.training_config - flow_shift = getattr(tc.pipeline_config, "flow_shift", None) - - kwargs: dict[str, Any] = { - "inference_mode": True, - "sampler_kind": str(sampler_kind), - "loaded_modules": { - "transformer": transformer - }, - "tp_size": tc.distributed.tp_size, - "sp_size": tc.distributed.sp_size, - "num_gpus": tc.distributed.num_gpus, - "pin_cpu_memory": tc.distributed.pin_cpu_memory, - "dit_cpu_offload": True, - } - if flow_shift is not None: - kwargs["flow_shift"] = float(flow_shift) - if ode_solver is not None: - kwargs["ode_solver"] = str(ode_solver) - - self._pipeline = WanPipeline.from_pretrained(tc.model_path, **kwargs) - self._pipeline_key = key - return self._pipeline - - def _prepare_validation_batch( - self, - sampling_param: SamplingParam, - validation_batch: dict[str, Any], - num_inference_steps: int, - *, - sampling_timesteps: list[int] | None = None, - guidance_scale: float | None = None, - ) -> ForwardBatch: - tc = self.training_config - - sampling_param.prompt = validation_batch["prompt"] - sampling_param.height = tc.data.num_height - sampling_param.width = tc.data.num_width - sampling_param.num_inference_steps = (num_inference_steps) - sampling_param.data_type = "video" - if guidance_scale is not None: - sampling_param.guidance_scale = float(guidance_scale) - sampling_param.seed = self.seed - - latents_size = [ - (sampling_param.num_frames - 1) // 4 + 1, - sampling_param.height // 8, - sampling_param.width // 8, - ] - n_tokens = (latents_size[0] * latents_size[1] * latents_size[2]) - - temporal_compression_factor = int(tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio) - num_frames = ((tc.data.num_latent_t - 1) * temporal_compression_factor + 1) - sampling_param.num_frames = int(num_frames) - - sampling_timesteps_tensor = (torch.tensor( - [int(s) for s in sampling_timesteps], - dtype=torch.long, - ) if sampling_timesteps is not None else None) - - # Build TrainingArgs for inference to pass - # to pipeline.forward(). - inference_args = make_inference_args(tc, model_path=tc.model_path) - - batch = ForwardBatch( - **shallow_asdict(sampling_param), - latents=None, - generator=self.validation_random_generator, - n_tokens=n_tokens, - eta=0.0, - VSA_sparsity=tc.vsa.sparsity, - timesteps=sampling_timesteps_tensor, - sampling_timesteps=sampling_timesteps_tensor, - ) - # Store inference_args on batch for pipeline access. - batch._inference_args = inference_args # type: ignore[attr-defined] - return batch - - def _run_validation_for_steps( - self, - num_inference_steps: int, - *, - dataset_file: str, - transformer: torch.nn.Module, - sampler_kind: str, - ode_solver: str | None, - sampling_timesteps: list[int] | None = None, - guidance_scale: float | None = None, - ) -> _ValidationStepResult: - tc = self.training_config - pipeline = self._get_pipeline( - transformer=transformer, - sampler_kind=sampler_kind, - ode_solver=ode_solver, - ) - sampling_param = self._get_sampling_param() - - dataset = ValidationDataset(dataset_file) - dataloader = DataLoader(dataset, batch_size=None, num_workers=0) - - # Build inference args once for this validation run. - inference_args = make_inference_args(tc, model_path=tc.model_path) - - videos: list[list[np.ndarray]] = [] - captions: list[str] = [] - - for validation_batch in dataloader: - batch = self._prepare_validation_batch( - sampling_param, - validation_batch, - num_inference_steps, - sampling_timesteps=sampling_timesteps, - guidance_scale=guidance_scale, - ) - - assert (batch.prompt is not None and isinstance(batch.prompt, str)) - captions.append(batch.prompt) - - with torch.no_grad(): - output_batch = pipeline.forward(batch, inference_args) - - samples = output_batch.output.cpu() - if self.rank_in_sp_group != 0: - continue - - video = rearrange(samples, "b c t h w -> t b c h w") - frames: list[np.ndarray] = [] - for x in video: - x = torchvision.utils.make_grid(x, nrow=6) - x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) - frames.append((x * 255).numpy().astype(np.uint8)) - videos.append(frames) - - return _ValidationStepResult(videos=videos, captions=captions) - - def log_validation( - self, - step: int, - *, - request: ValidationRequest | None = None, - ) -> None: - if request is None: - raise ValueError("WanValidator.log_validation requires a " - "ValidationRequest") - - dataset_file = getattr(request, "dataset_file", None) - if not dataset_file: - raise ValueError("ValidationRequest.dataset_file must be " - "provided by the method") - - guidance_scale = getattr(request, "guidance_scale", None) - validation_steps = getattr(request, "sampling_steps", None) - if not validation_steps: - raise ValueError("ValidationRequest.sampling_steps must be " - "provided by the method") - sampler_kind = (getattr(request, "sampler_kind", None) or "ode") - rollout_mode = getattr(request, "rollout_mode", None) - if rollout_mode not in {None, "parallel"}: - raise ValueError("WanValidator only supports " - "rollout_mode='parallel'. " - f"Got rollout_mode={rollout_mode!r}.") - ode_solver = getattr(request, "ode_solver", None) - sampling_timesteps = getattr(request, "sampling_timesteps", None) - if sampling_timesteps is not None: - expected = int(len(sampling_timesteps)) - for steps in validation_steps: - if int(steps) != expected: - raise ValueError("validation_sampling_steps must " - "match " - "len(request.sampling_timesteps)=" - f"{expected} when " - "sampling_timesteps is provided, " - f"got {validation_steps!r}.") - - sample_handle = getattr(request, "sample_handle", None) - if sample_handle is None: - raise ValueError("ValidationRequest.sample_handle must be " - "provided by the method") - transformer = sample_handle.transformer - was_training = bool(getattr(transformer, "training", False)) - - tc = self.training_config - output_dir = (getattr(request, "output_dir", None) or tc.checkpoint.output_dir) - - try: - transformer.eval() - - num_sp_groups = (self.world_group.world_size // self.sp_group.world_size) - - for num_inference_steps in validation_steps: - result = self._run_validation_for_steps( - num_inference_steps, - dataset_file=str(dataset_file), - transformer=transformer, - sampler_kind=str(sampler_kind), - ode_solver=(str(ode_solver) if ode_solver is not None else None), - sampling_timesteps=sampling_timesteps, - guidance_scale=guidance_scale, - ) - - if self.rank_in_sp_group != 0: - continue - - if self.global_rank == 0: - all_videos = list(result.videos) - all_captions = list(result.captions) - for sp_group_idx in range(1, num_sp_groups): - src_rank = (sp_group_idx * self.sp_world_size) - recv_videos = (self.world_group.recv_object(src=src_rank)) - recv_captions = (self.world_group.recv_object(src=src_rank)) - all_videos.extend(recv_videos) - all_captions.extend(recv_captions) - - os.makedirs(output_dir, exist_ok=True) - video_filenames: list[str] = [] - sampling_param = (self._get_sampling_param()) - for i, video in enumerate(all_videos): - filename = os.path.join( - output_dir, - f"validation_step_{step}" - f"_inference_steps_" - f"{num_inference_steps}" - f"_video_{i}.mp4", - ) - imageio.mimsave( - filename, - video, - fps=sampling_param.fps, - ) - video_filenames.append(filename) - - video_logs = [] - for filename, caption in zip( - video_filenames, - all_captions, - strict=True, - ): - video_artifact = self.tracker.video(filename, caption=caption) - if video_artifact is not None: - video_logs.append(video_artifact) - if video_logs: - logs = { - f"validation_videos_" - f"{num_inference_steps}" - f"_steps": video_logs - } - self.tracker.log_artifacts(logs, step) - else: - self.world_group.send_object(result.videos, dst=0) - self.world_group.send_object(result.captions, dst=0) - finally: - if was_training: - transformer.train() +WanValidator = ValidationCallback diff --git a/fastvideo/train/validators/wangame.py b/fastvideo/train/validators/wangame.py index 519089349..68e9aad87 100644 --- a/fastvideo/train/validators/wangame.py +++ b/fastvideo/train/validators/wangame.py @@ -1,626 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 -"""WanGame validator (model-family validation backend). +"""Deprecated — use fastvideo.train.callbacks.ValidationCallback. -Config keys used: -- `training` (TrainingConfig): - - `data.seed`, `model_path` - - `data.num_height`, `data.num_width`, `data.num_latent_t` - - `distributed.tp_size`, `distributed.sp_size`, - `distributed.num_gpus`, `distributed.pin_cpu_memory` - - `pipeline_config.flow_shift` - - `pipeline_config.vae_config.arch_config - .temporal_compression_ratio` - - `vsa.sparsity` -- `training.validation.*` (typically parsed by a method into - `ValidationRequest`): - - `dataset_file`, `sampling_steps`, `guidance_scale` - - `sampler_kind` (`ode`/`sde`), `ode_solver` - (`euler`/`unipc`) - - `rollout_mode` (`parallel`/`streaming`), `num_frames` +Kept as an import shim for backwards compatibility. """ from __future__ import annotations -import os -from dataclasses import dataclass -from typing import Any, TYPE_CHECKING +import warnings -import imageio -import numpy as np -import torch -import torchvision -from einops import rearrange -from torch.utils.data import DataLoader +from fastvideo.train.callbacks.validation import ( + ValidationCallback, +) -from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.validation_dataset import ( - ValidationDataset, ) -from fastvideo.distributed import get_sp_group, get_world_group -from fastvideo.logger import init_logger -from fastvideo.pipelines import ForwardBatch -from fastvideo.train.utils.moduleloader import ( - make_inference_args, ) -from fastvideo.train.validators.base import ( - ValidationRequest, ) -from fastvideo.training.trackers import DummyTracker -from fastvideo.utils import shallow_asdict +warnings.warn( + "fastvideo.train.validators.wangame is deprecated. " + "Use fastvideo.train.callbacks.ValidationCallback " + "instead.", + DeprecationWarning, + stacklevel=2, +) -if TYPE_CHECKING: - from fastvideo.train.utils.training_config import ( - TrainingConfig, ) - -logger = init_logger(__name__) - - -@dataclass(slots=True) -class _ValidationStepResult: - videos: list[list[np.ndarray]] - captions: list[str] - - -class WanGameValidator: - """Standalone validator for WanGame training.""" - - def __init__( - self, - *, - training_config: TrainingConfig, - tracker: Any | None = None, - ) -> None: - self.training_config = training_config - self.tracker = tracker or DummyTracker() - - self.world_group = get_world_group() - self.sp_group = get_sp_group() - self.global_rank = self.world_group.rank - self.rank_in_sp_group = self.sp_group.rank_in_group - self.sp_world_size = self.sp_group.world_size - - seed = training_config.data.seed - if seed is None: - raise ValueError("training.data.seed must be set for " - "validation") - self.seed = int(seed) - self.validation_random_generator = (torch.Generator(device="cpu").manual_seed(self.seed)) - - self._pipeline: Any | None = None - self._pipeline_key: (tuple[int, str, str, str] | None) = None - self._sampling_param: SamplingParam | None = None - - def set_tracker(self, tracker: Any) -> None: - self.tracker = tracker - - def _post_process_validation_frames( - self, - frames: list[np.ndarray], - batch: ForwardBatch, - ) -> list[np.ndarray]: - """Optionally overlay action indicators.""" - - keyboard_cond = getattr(batch, "keyboard_cond", None) - mouse_cond = getattr(batch, "mouse_cond", None) - if keyboard_cond is None and mouse_cond is None: - return frames - - try: - from fastvideo.models.dits.matrixgame.utils import ( - draw_keys_on_frame, - draw_mouse_on_frame, - ) - except Exception as e: - logger.warning( - "WanGame action overlay is unavailable: %s", - e, - ) - return frames - - if (keyboard_cond is not None and torch.is_tensor(keyboard_cond)): - keyboard_np = (keyboard_cond.squeeze(0).detach().cpu().float().numpy()) - else: - keyboard_np = None - - if (mouse_cond is not None and torch.is_tensor(mouse_cond)): - mouse_np = (mouse_cond.squeeze(0).detach().cpu().float().numpy()) - else: - mouse_np = None - - key_names = ["W", "S", "A", "D", "left", "right"] - - processed_frames: list[np.ndarray] = [] - for frame_idx, frame in enumerate(frames): - frame = np.ascontiguousarray(frame.copy()) - - if (keyboard_np is not None and frame_idx < len(keyboard_np)): - keys = { - key_names[i]: bool(keyboard_np[frame_idx, i]) - for i in range(min( - len(key_names), - int(keyboard_np.shape[1]), - )) - } - draw_keys_on_frame(frame, keys, mode="universal") - - if (mouse_np is not None and frame_idx < len(mouse_np)): - pitch = float(mouse_np[frame_idx, 0]) - yaw = float(mouse_np[frame_idx, 1]) - draw_mouse_on_frame(frame, pitch, yaw) - - processed_frames.append(frame) - - return processed_frames - - def _get_sampling_param(self) -> SamplingParam: - if self._sampling_param is None: - self._sampling_param = (SamplingParam.from_pretrained(self.training_config.model_path)) - return self._sampling_param - - def _get_pipeline( - self, - *, - transformer: torch.nn.Module, - rollout_mode: str, - sampler_kind: str, - ode_solver: str | None, - ) -> Any: - rollout_mode = str(rollout_mode).strip().lower() - sampler_kind = str(sampler_kind).strip().lower() - key = ( - id(transformer), - rollout_mode, - sampler_kind, - str(ode_solver), - ) - if (self._pipeline is not None and self._pipeline_key == key): - return self._pipeline - - tc = self.training_config - - if rollout_mode == "parallel": - if sampler_kind not in {"ode", "sde"}: - raise ValueError("Unknown sampler_kind for WanGame " - f"validation: {sampler_kind!r}") - - flow_shift = getattr(tc.pipeline_config, "flow_shift", None) - - from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import ( - WanGameActionImageToVideoPipeline, ) - - kwargs: dict[str, Any] = { - "inference_mode": True, - "sampler_kind": sampler_kind, - "loaded_modules": { - "transformer": transformer, - }, - "tp_size": tc.distributed.tp_size, - "sp_size": tc.distributed.sp_size, - "num_gpus": tc.distributed.num_gpus, - "pin_cpu_memory": (tc.distributed.pin_cpu_memory), - "dit_cpu_offload": True, - } - if flow_shift is not None: - kwargs["flow_shift"] = float(flow_shift) - if ode_solver is not None: - kwargs["ode_solver"] = str(ode_solver) - - self._pipeline = (WanGameActionImageToVideoPipeline.from_pretrained( - tc.model_path, - **kwargs, - )) - elif rollout_mode == "streaming": - if sampler_kind not in {"ode", "sde"}: - raise ValueError("Unknown sampler_kind for WanGame " - "streaming validation: " - f"{sampler_kind!r}") - - flow_shift = getattr(tc.pipeline_config, "flow_shift", None) - if flow_shift is None: - raise ValueError("pipeline_config.flow_shift must be set " - "for WanGame validation") - - if sampler_kind == "sde": - if ode_solver is not None: - raise ValueError("ode_solver is only valid when " - "sampler_kind='ode', got " - f"ode_solver={ode_solver!r}.") - from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, ) - - scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) - else: - from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, ) - from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( - FlowUniPCMultistepScheduler, ) - - ode_solver_norm = (str(ode_solver).strip().lower() if ode_solver is not None else "unipc") - if ode_solver_norm in { - "unipc", - "unipc_multistep", - "multistep", - }: - scheduler = (FlowUniPCMultistepScheduler(shift=float(flow_shift))) - elif ode_solver_norm in { - "euler", - "flowmatch", - "flowmatch_euler", - }: - scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) - else: - raise ValueError("Unknown ode_solver for WanGame " - "streaming validation: " - f"{ode_solver!r} (expected 'unipc'" - " or 'euler').") - - from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( - WanGameCausalDMDPipeline, ) - - kwargs = { - "inference_mode": True, - "flow_shift": (float(flow_shift) if flow_shift is not None else None), - "sampler_kind": sampler_kind, - "loaded_modules": { - "transformer": transformer, - "scheduler": scheduler, - }, - "tp_size": tc.distributed.tp_size, - "sp_size": tc.distributed.sp_size, - "num_gpus": tc.distributed.num_gpus, - "pin_cpu_memory": (tc.distributed.pin_cpu_memory), - "dit_cpu_offload": True, - } - if kwargs["flow_shift"] is None: - kwargs.pop("flow_shift") - if ode_solver is not None: - kwargs["ode_solver"] = str(ode_solver) - - self._pipeline = (WanGameCausalDMDPipeline.from_pretrained( - tc.model_path, - **kwargs, - )) - else: - raise ValueError("Unknown rollout_mode for WanGame " - f"validation: {rollout_mode!r}. Expected " - "'parallel' or 'streaming'.") - - self._pipeline_key = key - return self._pipeline - - def _prepare_validation_batch( - self, - sampling_param: SamplingParam, - validation_batch: dict[str, Any], - num_inference_steps: int, - *, - sampling_timesteps: list[int] | None = None, - guidance_scale: float | None = None, - num_frames: int | None = None, - ) -> ForwardBatch: - tc = self.training_config - - sampling_param.prompt = validation_batch["prompt"] - sampling_param.height = tc.data.num_height - sampling_param.width = tc.data.num_width - sampling_param.image_path = (validation_batch.get("image_path") or validation_batch.get("video_path")) - sampling_param.num_inference_steps = int(num_inference_steps) - sampling_param.data_type = "video" - if guidance_scale is not None: - sampling_param.guidance_scale = float(guidance_scale) - sampling_param.seed = self.seed - - temporal_compression_factor = int(tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio) - default_num_frames = ((tc.data.num_latent_t - 1) * temporal_compression_factor + 1) - if num_frames is not None: - sampling_param.num_frames = int(num_frames) - else: - sampling_param.num_frames = int(default_num_frames) - - latents_size = [ - (sampling_param.num_frames - 1) // 4 + 1, - sampling_param.height // 8, - sampling_param.width // 8, - ] - n_tokens = (latents_size[0] * latents_size[1] * latents_size[2]) - - sampling_timesteps_tensor = (torch.tensor( - [int(s) for s in sampling_timesteps], - dtype=torch.long, - ) if sampling_timesteps is not None else None) - - batch = ForwardBatch( - **shallow_asdict(sampling_param), - latents=None, - generator=self.validation_random_generator, - n_tokens=n_tokens, - eta=0.0, - VSA_sparsity=tc.vsa.sparsity, - timesteps=sampling_timesteps_tensor, - sampling_timesteps=sampling_timesteps_tensor, - ) - if ("image" in validation_batch and validation_batch["image"] is not None): - batch.pil_image = validation_batch["image"] - - if ("keyboard_cond" in validation_batch and validation_batch["keyboard_cond"] is not None): - keyboard_cond = torch.as_tensor(validation_batch["keyboard_cond"]).to(dtype=torch.bfloat16) - if (keyboard_cond.ndim == 3 and keyboard_cond.shape[0] == 1): - keyboard_cond = keyboard_cond.squeeze(0) - if keyboard_cond.ndim != 2: - raise ValueError("validation keyboard_cond must have " - "shape (T, K) (or (1, T, K)), " - f"got {tuple(keyboard_cond.shape)}") - target_len = int(sampling_param.num_frames) - if keyboard_cond.shape[0] > target_len: - keyboard_cond = keyboard_cond[:target_len] - elif keyboard_cond.shape[0] < target_len: - pad = torch.zeros( - ( - target_len - keyboard_cond.shape[0], - keyboard_cond.shape[1], - ), - dtype=keyboard_cond.dtype, - device=keyboard_cond.device, - ) - keyboard_cond = torch.cat([keyboard_cond, pad], dim=0) - batch.keyboard_cond = keyboard_cond.unsqueeze(0) - - if ("mouse_cond" in validation_batch and validation_batch["mouse_cond"] is not None): - mouse_cond = torch.as_tensor(validation_batch["mouse_cond"]).to(dtype=torch.bfloat16) - if (mouse_cond.ndim == 3 and mouse_cond.shape[0] == 1): - mouse_cond = mouse_cond.squeeze(0) - if mouse_cond.ndim != 2: - raise ValueError("validation mouse_cond must have shape" - " (T, 2) (or (1, T, 2)), " - f"got {tuple(mouse_cond.shape)}") - target_len = int(sampling_param.num_frames) - if mouse_cond.shape[0] > target_len: - mouse_cond = mouse_cond[:target_len] - elif mouse_cond.shape[0] < target_len: - pad = torch.zeros( - ( - target_len - mouse_cond.shape[0], - mouse_cond.shape[1], - ), - dtype=mouse_cond.dtype, - device=mouse_cond.device, - ) - mouse_cond = torch.cat([mouse_cond, pad], dim=0) - batch.mouse_cond = mouse_cond.unsqueeze(0) - - return batch - - def _run_validation_for_steps( - self, - num_inference_steps: int, - *, - dataset_file: str, - transformer: torch.nn.Module, - rollout_mode: str, - sampler_kind: str, - ode_solver: str | None, - sampling_timesteps: list[int] | None = None, - guidance_scale: float | None = None, - num_frames: int | None = None, - ) -> _ValidationStepResult: - tc = self.training_config - pipeline = self._get_pipeline( - transformer=transformer, - rollout_mode=rollout_mode, - sampler_kind=sampler_kind, - ode_solver=ode_solver, - ) - sampling_param = self._get_sampling_param() - - dataset = ValidationDataset(dataset_file) - dataloader = DataLoader(dataset, batch_size=None, num_workers=0) - - # Build inference args once for this validation run. - inference_args = make_inference_args(tc, model_path=tc.model_path) - - videos: list[list[np.ndarray]] = [] - captions: list[str] = [] - - for validation_batch in dataloader: - batch = self._prepare_validation_batch( - sampling_param, - validation_batch, - num_inference_steps, - sampling_timesteps=sampling_timesteps, - guidance_scale=guidance_scale, - num_frames=num_frames, - ) - - assert (batch.prompt is not None and isinstance(batch.prompt, str)) - captions.append(batch.prompt) - - with torch.no_grad(): - output_batch = pipeline.forward(batch, inference_args) - - samples = output_batch.output.cpu() - if self.rank_in_sp_group != 0: - continue - - video = rearrange(samples, "b c t h w -> t b c h w") - frames: list[np.ndarray] = [] - for x in video: - x = torchvision.utils.make_grid(x, nrow=6) - x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) - frames.append((x * 255).numpy().astype(np.uint8)) - frames = (self._post_process_validation_frames(frames, batch)) - videos.append(frames) - - return _ValidationStepResult(videos=videos, captions=captions) - - def log_validation( - self, - step: int, - *, - request: ValidationRequest | None = None, - ) -> None: - if request is None: - raise ValueError("WanGameValidator.log_validation requires " - "a ValidationRequest") - - dataset_file = getattr(request, "dataset_file", None) - if not dataset_file: - raise ValueError("ValidationRequest.dataset_file must be " - "provided by the method") - - guidance_scale = getattr(request, "guidance_scale", None) - validation_steps = getattr(request, "sampling_steps", None) - if not validation_steps: - raise ValueError("ValidationRequest.sampling_steps must be " - "provided by the method") - sampler_kind = (getattr(request, "sampler_kind", None) or "ode") - rollout_mode_raw = (getattr(request, "rollout_mode", None) or "parallel") - if not isinstance(rollout_mode_raw, str): - raise ValueError("ValidationRequest.rollout_mode must be a " - "string when set, got " - f"{type(rollout_mode_raw).__name__}") - rollout_mode = rollout_mode_raw.strip().lower() - if rollout_mode not in {"parallel", "streaming"}: - raise ValueError("ValidationRequest.rollout_mode must be " - "one of {parallel, streaming}, got " - f"{rollout_mode_raw!r}") - ode_solver = getattr(request, "ode_solver", None) - num_frames_raw = getattr(request, "num_frames", None) - if num_frames_raw is None: - num_frames = None - elif isinstance(num_frames_raw, bool): - raise ValueError("ValidationRequest.num_frames must be an " - "int when set") - elif isinstance(num_frames_raw, int): - num_frames = int(num_frames_raw) - else: - raise ValueError("ValidationRequest.num_frames must be an " - "int when set, got " - f"{type(num_frames_raw).__name__}") - if num_frames is not None and num_frames <= 0: - raise ValueError("ValidationRequest.num_frames must be > 0 " - "when set") - if rollout_mode == "streaming": - sampler_kind_norm = (str(sampler_kind).strip().lower()) - if sampler_kind_norm not in {"ode", "sde"}: - raise ValueError("WanGame validation " - "rollout_mode='streaming' requires " - "sampler_kind to be one of " - "{'ode', 'sde'}, got " - f"{sampler_kind!r}.") - if (sampler_kind_norm == "sde" and ode_solver is not None): - raise ValueError("WanGame validation " - "rollout_mode='streaming' only " - "supports ode_solver when " - "sampler_kind='ode', got " - f"ode_solver={ode_solver!r}.") - sampling_timesteps = getattr(request, "sampling_timesteps", None) - if sampling_timesteps is not None: - expected = int(len(sampling_timesteps)) - for steps in validation_steps: - if int(steps) != expected: - raise ValueError("validation_sampling_steps must " - "match " - "len(request.sampling_timesteps)=" - f"{expected} when " - "sampling_timesteps is provided, " - f"got {validation_steps!r}.") - - sample_handle = getattr(request, "sample_handle", None) - if sample_handle is None: - raise ValueError("ValidationRequest.sample_handle must be " - "provided by the method") - transformer = sample_handle.transformer - was_training = bool(getattr(transformer, "training", False)) - - tc = self.training_config - output_dir = (getattr(request, "output_dir", None) or tc.checkpoint.output_dir) - - # For streaming SDE, we need to set - # dmd_denoising_steps on pipeline_config. - old_dmd_denoising_steps = getattr( - tc.pipeline_config, - "dmd_denoising_steps", - None, - ) - try: - transformer.eval() - - num_sp_groups = (self.world_group.world_size // self.sp_group.world_size) - - for num_inference_steps in validation_steps: - if (rollout_mode == "streaming" and str(sampler_kind).strip().lower() == "sde"): - if sampling_timesteps is not None: - tc.pipeline_config.dmd_denoising_steps = list(sampling_timesteps) - else: - import numpy as np - - timesteps = np.linspace( - 1000, - 0, - int(num_inference_steps), - ) - tc.pipeline_config.dmd_denoising_steps = [int(max(0, min(1000, round(t)))) for t in timesteps] - - result = self._run_validation_for_steps( - num_inference_steps, - dataset_file=str(dataset_file), - transformer=transformer, - rollout_mode=rollout_mode, - sampler_kind=str(sampler_kind), - ode_solver=(str(ode_solver) if ode_solver is not None else None), - sampling_timesteps=sampling_timesteps, - guidance_scale=guidance_scale, - num_frames=num_frames, - ) - - if self.rank_in_sp_group != 0: - continue - - if self.global_rank == 0: - all_videos = list(result.videos) - all_captions = list(result.captions) - for sp_group_idx in range(1, num_sp_groups): - src_rank = (sp_group_idx * self.sp_world_size) - recv_videos = (self.world_group.recv_object(src=src_rank)) - recv_captions = (self.world_group.recv_object(src=src_rank)) - all_videos.extend(recv_videos) - all_captions.extend(recv_captions) - - os.makedirs(output_dir, exist_ok=True) - video_filenames: list[str] = [] - sampling_param = (self._get_sampling_param()) - for i, video in enumerate(all_videos): - filename = os.path.join( - output_dir, - f"validation_step_{step}" - f"_inference_steps_" - f"{num_inference_steps}" - f"_video_{i}.mp4", - ) - imageio.mimsave( - filename, - video, - fps=sampling_param.fps, - ) - video_filenames.append(filename) - - video_logs = [] - for filename, caption in zip( - video_filenames, - all_captions, - strict=True, - ): - video_artifact = self.tracker.video(filename, caption=caption) - if video_artifact is not None: - video_logs.append(video_artifact) - if video_logs: - logs = { - f"validation_videos_" - f"{num_inference_steps}" - f"_steps": video_logs - } - self.tracker.log_artifacts(logs, step) - else: - self.world_group.send_object(result.videos, dst=0) - self.world_group.send_object(result.captions, dst=0) - finally: - tc.pipeline_config.dmd_denoising_steps = (old_dmd_denoising_steps) - if was_training: - transformer.train() +WanGameValidator = ValidationCallback diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 6d32e09ba..6bea477b3 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -53,7 +53,11 @@ def run_training_from_config( "build_from_config succeeded.") return - trainer = Trainer(tc, config=cfg.raw) + trainer = Trainer( + tc, + config=cfg.raw, + callback_configs=cfg.callbacks, + ) # Attach the exact YAML used for this run to the # tracker (e.g., W&B Files). From 3b45f231f1d72df0bf18bc876e7765c003a745a4 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 5 Mar 2026 23:02:41 +0000 Subject: [PATCH 190/214] grad clipping callback --- .../refactor/dfsft_wangame_causal_v3.yaml | 2 + ...2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 2 + .../finetune_wangame2.1_i2v_1.3B.yaml | 2 + .../self_forcing_wangame_causal_v3.yaml | 2 + fastvideo/train/callbacks/__init__.py | 4 + fastvideo/train/callbacks/callback.py | 7 ++ fastvideo/train/callbacks/grad_clip.py | 74 +++++++++++++++++++ fastvideo/train/methods/base.py | 11 +++ .../methods/distribution_matching/dmd2.py | 21 ++---- fastvideo/train/methods/fine_tuning/dfsft.py | 11 --- .../train/methods/fine_tuning/finetune.py | 11 --- fastvideo/train/trainer.py | 10 +++ fastvideo/train/utils/checkpoint.py | 21 ++++++ fastvideo/training/distillation.py | 1 + 14 files changed, 144 insertions(+), 35 deletions(-) create mode 100644 fastvideo/train/callbacks/grad_clip.py diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index 2bc84b883..62da444c1 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -68,6 +68,8 @@ training: enable_gradient_checkpointing_type: full callbacks: + grad_clip: + _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback validation: _target_: fastvideo.train.callbacks.validation.ValidationCallback pipeline_target: fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline.WanGameCausalDMDPipeline diff --git a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml index 583a36a65..4c98c67a5 100644 --- a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml +++ b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml @@ -63,6 +63,8 @@ training: decay_interval_steps: 1 callbacks: + grad_clip: + _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback validation: _target_: fastvideo.train.callbacks.validation.ValidationCallback pipeline_target: fastvideo.pipelines.basic.wan.wan_pipeline.WanPipeline diff --git a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml index 29aeac1d3..68879d153 100644 --- a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml +++ b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml @@ -66,6 +66,8 @@ training: enable_gradient_checkpointing_type: full callbacks: + grad_clip: + _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback validation: _target_: fastvideo.train.callbacks.validation.ValidationCallback pipeline_target: fastvideo.pipelines.basic.wan.wangame_i2v_pipeline.WanGameActionImageToVideoPipeline diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index f1725649c..d56e0f06a 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -101,6 +101,8 @@ training: enable_gradient_checkpointing_type: null callbacks: + grad_clip: + _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback validation: _target_: fastvideo.train.callbacks.validation.ValidationCallback pipeline_target: fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline.WanGameCausalDMDPipeline diff --git a/fastvideo/train/callbacks/__init__.py b/fastvideo/train/callbacks/__init__.py index 8649ee40f..71027e00f 100644 --- a/fastvideo/train/callbacks/__init__.py +++ b/fastvideo/train/callbacks/__init__.py @@ -4,6 +4,9 @@ Callback, CallbackDict, ) +from fastvideo.train.callbacks.grad_clip import ( + GradNormClipCallback, +) from fastvideo.train.callbacks.validation import ( ValidationCallback, ) @@ -11,5 +14,6 @@ __all__ = [ "Callback", "CallbackDict", + "GradNormClipCallback", "ValidationCallback", ] diff --git a/fastvideo/train/callbacks/callback.py b/fastvideo/train/callbacks/callback.py index cbc0bd808..465f0b9df 100644 --- a/fastvideo/train/callbacks/callback.py +++ b/fastvideo/train/callbacks/callback.py @@ -46,6 +46,13 @@ def on_training_step_end( ) -> None: pass + def on_before_optimizer_step( + self, + method: TrainingMethod, + iteration: int = 0, + ) -> None: + pass + def on_validation_begin( self, method: TrainingMethod, diff --git a/fastvideo/train/callbacks/grad_clip.py b/fastvideo/train/callbacks/grad_clip.py new file mode 100644 index 000000000..74168b987 --- /dev/null +++ b/fastvideo/train/callbacks/grad_clip.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Gradient norm clipping callback. + +Clips gradients on modules returned by +``method.get_grad_clip_targets()`` before the optimizer step. +Optionally logs per-module grad norms to the tracker. +""" + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +from fastvideo.logger import init_logger +from fastvideo.train.callbacks.callback import Callback +from fastvideo.train.utils.optimizer import ( + clip_grad_norm_if_needed, +) + +if TYPE_CHECKING: + from fastvideo.train.methods.base import TrainingMethod + +logger = init_logger(__name__) + + +class GradNormClipCallback(Callback): + """Clip gradient norms before the optimizer step. + + ``max_grad_norm`` defaults to + ``training_config.optimizer.max_grad_norm`` when not + provided explicitly in the YAML config. + """ + + def __init__( + self, + *, + max_grad_norm: float | None = None, + log_grad_norms: bool = False, + ) -> None: + self._max_grad_norm_override = max_grad_norm + self._log_grad_norms = bool(log_grad_norms) + + @property + def _max_grad_norm(self) -> float: + if self._max_grad_norm_override is not None: + return float(self._max_grad_norm_override) + return float( + self.training_config.optimizer.max_grad_norm + ) + + def on_before_optimizer_step( + self, + method: TrainingMethod, + iteration: int = 0, + ) -> None: + max_norm = self._max_grad_norm + if max_norm <= 0.0: + return + + targets = method.get_grad_clip_targets(iteration) + tracker = getattr(method, "tracker", None) + + for name, module in targets.items(): + grad_norm = clip_grad_norm_if_needed( + module, max_norm, + ) + if ( + self._log_grad_norms + and tracker is not None + and grad_norm > 0.0 + ): + tracker.log( + {f"grad_norm/{name}": grad_norm}, + iteration, + ) diff --git a/fastvideo/train/methods/base.py b/fastvideo/train/methods/base.py index 8e01c0525..1ad06c3b1 100644 --- a/fastvideo/train/methods/base.py +++ b/fastvideo/train/methods/base.py @@ -121,6 +121,17 @@ def optimizers_zero_grad( # -- Shared hooks (override in subclasses as needed) -- + def get_grad_clip_targets( + self, iteration: int, + ) -> dict[str, torch.nn.Module]: + """Return modules whose gradients should be clipped. + + Override in subclasses to add/conditionally include + modules (e.g. critic, conditionally student). + Default: student transformer. + """ + return {"student": self.student.transformer} + def on_train_start(self) -> None: self.student.on_train_start() diff --git a/fastvideo/train/methods/distribution_matching/dmd2.py b/fastvideo/train/methods/distribution_matching/dmd2.py index d2f36ca4d..ac0c31eff 100644 --- a/fastvideo/train/methods/distribution_matching/dmd2.py +++ b/fastvideo/train/methods/distribution_matching/dmd2.py @@ -12,7 +12,6 @@ from fastvideo.train.models.base import ModelBase from fastvideo.train.utils.optimizer import ( build_optimizer_and_scheduler, - clip_grad_norm_if_needed, ) from fastvideo.train.utils.config import ( get_optional_float, @@ -226,21 +225,17 @@ def get_lr_schedulers( schedulers.append(self._student_lr_scheduler) return schedulers - # TrainingMethod override: optimizers_schedulers_step - def optimizers_schedulers_step( + # TrainingMethod override: get_grad_clip_targets + def get_grad_clip_targets( self, iteration: int, - ) -> None: - max_grad_norm = ( - self.training_config.optimizer.max_grad_norm - ) + ) -> dict[str, torch.nn.Module]: + targets: dict[str, torch.nn.Module] = {} if self._should_update_student(iteration): - clip_grad_norm_if_needed( - self.student.transformer, max_grad_norm + targets["student"] = ( + self.student.transformer ) - clip_grad_norm_if_needed( - self.critic.transformer, max_grad_norm - ) - super().optimizers_schedulers_step(iteration) + targets["critic"] = self.critic.transformer + return targets def _parse_rollout_mode( self, diff --git a/fastvideo/train/methods/fine_tuning/dfsft.py b/fastvideo/train/methods/fine_tuning/dfsft.py index 8a0d36c31..4f91110a2 100644 --- a/fastvideo/train/methods/fine_tuning/dfsft.py +++ b/fastvideo/train/methods/fine_tuning/dfsft.py @@ -12,7 +12,6 @@ from fastvideo.train.models.base import ModelBase from fastvideo.train.utils.optimizer import ( build_optimizer_and_scheduler, - clip_grad_norm_if_needed, ) @@ -250,16 +249,6 @@ def get_lr_schedulers( del iteration return [self._student_lr_scheduler] - # TrainingMethod override: optimizers_schedulers_step - def optimizers_schedulers_step( - self, iteration: int, - ) -> None: - clip_grad_norm_if_needed( - self.student.transformer, - self.training_config.optimizer.max_grad_norm, - ) - super().optimizers_schedulers_step(iteration) - def _parse_chunk_size(self, raw: Any) -> int: if raw in (None, ""): return 3 diff --git a/fastvideo/train/methods/fine_tuning/finetune.py b/fastvideo/train/methods/fine_tuning/finetune.py index 09bc1005e..cf7dc3139 100644 --- a/fastvideo/train/methods/fine_tuning/finetune.py +++ b/fastvideo/train/methods/fine_tuning/finetune.py @@ -12,7 +12,6 @@ from fastvideo.train.models.base import ModelBase from fastvideo.train.utils.optimizer import ( build_optimizer_and_scheduler, - clip_grad_norm_if_needed, ) @@ -188,16 +187,6 @@ def get_lr_schedulers( del iteration return [self._student_lr_scheduler] - # TrainingMethod override: optimizers_schedulers_step - def optimizers_schedulers_step( - self, iteration: int, - ) -> None: - clip_grad_norm_if_needed( - self.student.transformer, - self.training_config.optimizer.max_grad_norm, - ) - super().optimizers_schedulers_step(iteration) - def _init_optimizers_and_schedulers(self) -> None: tc = self.training_config diff --git a/fastvideo/train/trainer.py b/fastvideo/train/trainer.py index 3900177d3..b44fe57a5 100644 --- a/fastvideo/train/trainer.py +++ b/fastvideo/train/trainer.py @@ -160,6 +160,9 @@ def run( f".metrics[{k!r}]"), ) + self.callbacks.on_before_optimizer_step( + method, iteration=step, + ) method.optimizers_schedulers_step(step) method.optimizers_zero_grad(step) @@ -170,12 +173,19 @@ def run( if self.global_rank == 0 and metrics: self.tracker.log(metrics, step) + self.callbacks.on_training_step_end( + method, metrics, iteration=step, + ) + if checkpoint_manager is not None: checkpoint_manager.maybe_save(step) self.callbacks.on_validation_begin( method, iteration=step, ) + self.callbacks.on_validation_end( + method, iteration=step, + ) self.callbacks.on_train_end( method, iteration=max_steps, diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index 2ccc5c774..cbc568c2d 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -396,6 +396,21 @@ def __init__(self, modules: dict[str, torch.nn.Module]) -> None: self.add_module(name, module) +class _CallbackStateWrapper: + """Wraps a CallbackDict for DCP save/load.""" + + def __init__(self, callbacks: Any) -> None: + self._callbacks = callbacks + + def state_dict(self) -> dict[str, Any]: + return self._callbacks.state_dict() + + def load_state_dict( + self, state_dict: dict[str, Any], + ) -> None: + self._callbacks.load_state_dict(state_dict) + + @dataclass(slots=True) class CheckpointConfig: save_steps: int @@ -423,6 +438,7 @@ def __init__( output_dir: str, config: CheckpointConfig, get_rng_generators: Callable[[], dict[str, torch.Generator]] | None = None, + callbacks: Any | None = None, ) -> None: self.bundle = bundle self.role_models = role_models or {} @@ -432,6 +448,7 @@ def __init__( self.output_dir = str(output_dir) self.config = config self._get_rng_generators = get_rng_generators + self._callbacks = callbacks self._last_saved_step: int | None = None def _build_role_states_from_model( @@ -514,6 +531,10 @@ def _build_states(self) -> dict[str, Any]: continue states[f"random_state.{name}"] = RandomStateWrapper(gen) + # Callback state (e.g. EMA shadow weights, validation RNG). + if self._callbacks is not None and _is_stateful(self._callbacks): + states["callbacks"] = _CallbackStateWrapper(self._callbacks) + return states def _checkpoint_dir(self, step: int) -> Path: diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 6bea477b3..02e405cfe 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -79,6 +79,7 @@ def run_training_from_config( output_dir=tc.checkpoint.output_dir, config=ckpt_config, get_rng_generators=method.get_rng_generators, + callbacks=trainer.callbacks, ) trainer.run( From 6c30e257802b9eaf77ce5b9e018d9a87dbe82557 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 5 Mar 2026 23:32:10 +0000 Subject: [PATCH 191/214] ema callback implementation --- fastvideo/train/callbacks/__init__.py | 4 + fastvideo/train/callbacks/ema.py | 199 ++++++++++++++++++ fastvideo/train/models/base.py | 59 ++++++ fastvideo/train/models/wan/wan.py | 7 + fastvideo/train/models/wangame/wangame.py | 7 + .../train/models/wangame/wangame_causal.py | 2 + fastvideo/train/utils/checkpoint.py | 8 + 7 files changed, 286 insertions(+) create mode 100644 fastvideo/train/callbacks/ema.py diff --git a/fastvideo/train/callbacks/__init__.py b/fastvideo/train/callbacks/__init__.py index 71027e00f..23334280a 100644 --- a/fastvideo/train/callbacks/__init__.py +++ b/fastvideo/train/callbacks/__init__.py @@ -4,6 +4,9 @@ Callback, CallbackDict, ) +from fastvideo.train.callbacks.ema import ( + EMACallback, +) from fastvideo.train.callbacks.grad_clip import ( GradNormClipCallback, ) @@ -14,6 +17,7 @@ __all__ = [ "Callback", "CallbackDict", + "EMACallback", "GradNormClipCallback", "ValidationCallback", ] diff --git a/fastvideo/train/callbacks/ema.py b/fastvideo/train/callbacks/ema.py new file mode 100644 index 000000000..d03d912a2 --- /dev/null +++ b/fastvideo/train/callbacks/ema.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: Apache-2.0 +"""EMA (Exponential Moving Average) callback. + +Updates EMA shadow weights after each training step. The model owns +the EMA network (created by ``ModelBase._setup_ema``); this callback +only performs the ``lerp_`` update. +""" + +from __future__ import annotations + +from typing import Any, TYPE_CHECKING + +import torch + +from fastvideo.logger import init_logger +from fastvideo.train.callbacks.callback import Callback + +if TYPE_CHECKING: + from fastvideo.train.methods.base import TrainingMethod + +logger = init_logger(__name__) + + +class EMACallback(Callback): + """Update EMA parameters after each optimizer step. + + The EMA network lives on the model (``method.student.``). + If the model was created with ``use_ema=false``, the callback + detects this at train start and disables itself gracefully. + + Supports three beta strategies: + - ``constant``: fixed ``beta`` every step. + - ``power``: ``(1 - 1/t)^(gamma+1)``. + - ``halflife``: half-life in k-images with optional ramp-up. + """ + + def __init__( + self, + *, + type: str = "constant", + beta: float = 0.9999, + gamma: float = 16.97, + ema_halflife_kimg: float = 500.0, + ema_rampup_ratio: float | None = 0.05, + start_iter: int = 0, + ema_name: str = "ema", + batch_size: int = 1, + ) -> None: + self._type = str(type) + self._beta = float(beta) + self._gamma = float(gamma) + self._ema_halflife_kimg = float(ema_halflife_kimg) + self._ema_rampup_ratio = ( + float(ema_rampup_ratio) + if ema_rampup_ratio is not None + else None + ) + self._start_iter = int(start_iter) + self._ema_name = str(ema_name) + self._batch_size = int(batch_size) + self._enabled = True + + # ---------------------------------------------------------- + # Hooks + # ---------------------------------------------------------- + + def on_train_start( + self, + method: TrainingMethod, + iteration: int = 0, + ) -> None: + student = method.student + ema = getattr(student, self._ema_name, None) + if ema is None: + self._enabled = False + logger.info( + "EMA %r not found on student model; " + "EMA callback disabled.", + self._ema_name, + ) + return + + assert not ema.training, ( + f"EMA {self._ema_name} should be in eval mode" + ) + for name, p in ema.named_parameters(): + assert not p.requires_grad, ( + f"EMA parameter {name} should not " + f"require gradients" + ) + + def on_training_step_end( + self, + method: TrainingMethod, + loss_dict: dict[str, Any], + iteration: int = 0, + ) -> None: + if not self._enabled: + return + + if iteration < self._start_iter: + return + if iteration == self._start_iter: + logger.info( + "Starting EMA %r updates at iteration %d.", + self._ema_name, + iteration, + ) + + beta = self._compute_beta(iteration) + student = method.student + ema = getattr(student, self._ema_name) + ema_state = ema.state_dict() + + with torch.no_grad(): + for name, p_net in ( + student.transformer.named_parameters() + ): + full = self._gather_full(p_net) + ema_key = name.replace( + "_checkpoint_wrapped_module.", "", + ) + if ema_key not in ema_state: + if iteration == self._start_iter: + logger.warning( + "EMA param %r not found, " + "skipping.", + ema_key, + ) + continue + ema_p = ema_state[ema_key] + val = full.to( + device=ema_p.device, + dtype=ema_p.dtype, + ) + if iteration == self._start_iter: + ema_p.copy_(val) + else: + ema_p.lerp_(val, 1.0 - beta) + + for name, buf in ( + student.transformer.named_buffers() + ): + if name in ema_state: + ema_state[name].copy_( + buf.to( + device=ema_state[name].device, + dtype=ema_state[name].dtype, + ) + ) + + tracker = getattr(method, "tracker", None) + if tracker is not None: + tracker.log( + {f"ema/{self._ema_name}_beta": beta}, + iteration, + ) + + # ---------------------------------------------------------- + # Beta strategies + # ---------------------------------------------------------- + + def _compute_beta(self, iteration: int) -> float: + if self._type == "constant": + return self._beta + if self._type == "power": + it = max(iteration, 1) + return (1.0 - 1.0 / it) ** (self._gamma + 1) + if self._type == "halflife": + return self._halflife_beta(iteration) + raise ValueError( + f"Invalid EMA type: {self._type!r}" + ) + + def _halflife_beta(self, iteration: int) -> float: + hl_nimg = self._ema_halflife_kimg * 1000.0 + cur_nimg = iteration * self._batch_size + if self._ema_rampup_ratio is not None: + hl_nimg = min( + hl_nimg, + cur_nimg * self._ema_rampup_ratio, + ) + return 0.5 ** ( + self._batch_size / max(hl_nimg, 1e-8) + ) + + # ---------------------------------------------------------- + # FSDP helper + # ---------------------------------------------------------- + + @staticmethod + def _gather_full( + param: torch.Tensor, + ) -> torch.Tensor: + if hasattr(param, "full_tensor"): + if param.device.type == "cpu": + return param.to("cuda").full_tensor() + return param.full_tensor() + return param diff --git a/fastvideo/train/models/base.py b/fastvideo/train/models/base.py index d74406278..0c62bb401 100644 --- a/fastvideo/train/models/base.py +++ b/fastvideo/train/models/base.py @@ -2,16 +2,21 @@ from __future__ import annotations +import copy from abc import ABC, abstractmethod from typing import Any, Literal, TYPE_CHECKING import torch +from fastvideo.logger import init_logger + if TYPE_CHECKING: from fastvideo.train.utils.training_config import ( TrainingConfig, ) from fastvideo.pipelines import TrainingBatch +logger = init_logger(__name__) + class ModelBase(ABC): """Per-role model instance. @@ -26,6 +31,7 @@ class ModelBase(ABC): transformer: torch.nn.Module noise_scheduler: Any _trainable: bool + _use_ema: list[str] # ------------------------------------------------------------------ # Lifecycle @@ -45,6 +51,59 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: """Return RNG generators for checkpoint resume.""" return {} + # ------------------------------------------------------------------ + # EMA + # ------------------------------------------------------------------ + + def _setup_ema(self) -> None: + """Create EMA copies of the transformer. + + Call after ``self.transformer`` is set and before FSDP wrapping. + Each name in ``self._use_ema`` becomes an attribute holding a + deep copy of ``self.transformer`` in eval mode with no gradients. + """ + if not getattr(self, "_use_ema", None): + return + for name in self._use_ema: + if hasattr(self, name): + logger.warning( + "EMA network %r already exists, " + "skipping initialization.", + name, + ) + continue + logger.info( + "Initializing EMA network %r from " + "transformer", + name, + ) + ema = copy.deepcopy(self.transformer) + ema.eval().requires_grad_(False) + setattr(self, name, ema) + + @property + def ema_dict(self) -> dict[str, torch.nn.Module]: + """Return dict of EMA networks (empty if EMA disabled).""" + if not getattr(self, "_use_ema", None): + return {} + return { + name: getattr(self, name) + for name in self._use_ema + if hasattr(self, name) + } + + @property + def transformer_inference(self) -> torch.nn.Module: + """Return the transformer for inference. + + Returns the first EMA network when available, + otherwise the training transformer. + """ + ema = self.ema_dict + if ema: + return next(iter(ema.values())) + return self.transformer + # ------------------------------------------------------------------ # Timestep helpers # ------------------------------------------------------------------ diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index 557bf7512..1fdd62167 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -72,10 +72,16 @@ def __init__( flow_shift: float = 3.0, enable_gradient_checkpointing_type: str | None = None, + use_ema: list[str] | bool = False, ) -> None: self._init_from = str(init_from) self._trainable = bool(trainable) + if isinstance(use_ema, bool): + self._use_ema = ["ema"] if use_ema else [] + else: + self._use_ema = list(use_ema) + transformer = load_module_from_path( model_path=self._init_from, module_type="transformer", @@ -90,6 +96,7 @@ def __init__( checkpointing_type=(enable_gradient_checkpointing_type), ) self.transformer = transformer + self._setup_ema() self.noise_scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) diff --git a/fastvideo/train/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py index 47159c6de..60d6202fe 100644 --- a/fastvideo/train/models/wangame/wangame.py +++ b/fastvideo/train/models/wangame/wangame.py @@ -67,10 +67,16 @@ def __init__( disable_custom_init_weights: bool = False, flow_shift: float = 3.0, enable_gradient_checkpointing_type: str | None = None, + use_ema: list[str] | bool = False, ) -> None: self._init_from = str(init_from) self._trainable = bool(trainable) + if isinstance(use_ema, bool): + self._use_ema = ["ema"] if use_ema else [] + else: + self._use_ema = list(use_ema) + self.transformer = self._load_transformer( init_from=self._init_from, trainable=self._trainable, @@ -78,6 +84,7 @@ def __init__( enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), training_config=training_config, ) + self._setup_ema() self.noise_scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) diff --git a/fastvideo/train/models/wangame/wangame_causal.py b/fastvideo/train/models/wangame/wangame_causal.py index 902ed7824..21c0f40e7 100644 --- a/fastvideo/train/models/wangame/wangame_causal.py +++ b/fastvideo/train/models/wangame/wangame_causal.py @@ -45,6 +45,7 @@ def __init__( disable_custom_init_weights: bool = False, flow_shift: float = 3.0, enable_gradient_checkpointing_type: str | None = None, + use_ema: list[str] | bool = False, ) -> None: super().__init__( init_from=init_from, @@ -53,6 +54,7 @@ def __init__( flow_shift=flow_shift, enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), training_config=training_config, + use_ema=use_ema, ) self._streaming_caches: dict[tuple[int, str], _StreamingCaches] = {} diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index cbc568c2d..1c1ef4b93 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -240,6 +240,8 @@ def maybe_warmstart_role_modules( if model is not None: if model.transformer is not None: modules["transformer"] = model.transformer + for ema_name, ema_module in model.ema_dict.items(): + modules[ema_name] = ema_module elif bundle is not None: handle = bundle.role(str(role)) modules = dict(handle.modules) @@ -333,6 +335,8 @@ def _copy_or_link(src: str, dest: str) -> None: if model is not None: if model.transformer is not None: modules["transformer"] = model.transformer + for ema_name, ema_module in model.ema_dict.items(): + modules[ema_name] = ema_module elif bundle is not None: handle = bundle.role(str(role)) modules = dict(handle.modules) @@ -464,6 +468,10 @@ def _build_role_states_from_model( if model.transformer is not None: modules["transformer"] = model.transformer + # Include EMA networks in checkpoint. + for ema_name, ema_module in model.ema_dict.items(): + modules[ema_name] = ema_module + container = _RoleModuleContainer(modules) for module_name, module in modules.items(): From 6a0032a008737e368672541135489b2f208c20af Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Thu, 5 Mar 2026 23:51:13 +0000 Subject: [PATCH 192/214] ema and corresponding validation --- .../refactor/dfsft_wangame_causal_v3.yaml | 4 ++ ...2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 4 ++ .../finetune_wangame2.1_i2v_1.3B.yaml | 4 ++ .../self_forcing_wangame_causal_v3.yaml | 4 ++ fastvideo/train/callbacks/ema.py | 25 ++++---- fastvideo/train/callbacks/validation.py | 3 +- fastvideo/train/methods/base.py | 45 ++++++++++++++ fastvideo/train/models/base.py | 59 ------------------- fastvideo/train/models/wan/wan.py | 7 --- fastvideo/train/models/wangame/wangame.py | 7 --- .../train/models/wangame/wangame_causal.py | 2 - fastvideo/train/utils/checkpoint.py | 12 ++-- fastvideo/training/distillation.py | 9 +++ 13 files changed, 87 insertions(+), 98 deletions(-) diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index 62da444c1..b10e5b8e4 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -12,6 +12,7 @@ models: method: _target_: fastvideo.train.methods.fine_tuning.dfsft.DiffusionForcingSFTMethod attn_kind: dense + # use_ema: true chunk_size: 3 min_timestep_ratio: 0.02 max_timestep_ratio: 0.98 @@ -70,6 +71,9 @@ training: callbacks: grad_clip: _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + # ema: + # _target_: fastvideo.train.callbacks.ema.EMACallback + # beta: 0.9999 validation: _target_: fastvideo.train.callbacks.validation.ValidationCallback pipeline_target: fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline.WanGameCausalDMDPipeline diff --git a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml index 4c98c67a5..92a06051f 100644 --- a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml +++ b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml @@ -12,6 +12,7 @@ models: method: _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod attn_kind: vsa + # use_ema: true training: distributed: @@ -65,6 +66,9 @@ training: callbacks: grad_clip: _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + # ema: + # _target_: fastvideo.train.callbacks.ema.EMACallback + # beta: 0.9999 validation: _target_: fastvideo.train.callbacks.validation.ValidationCallback pipeline_target: fastvideo.pipelines.basic.wan.wan_pipeline.WanPipeline diff --git a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml index 68879d153..f1d2cbbd7 100644 --- a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml +++ b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml @@ -12,6 +12,7 @@ models: method: _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod attn_kind: dense + # use_ema: true training: distributed: @@ -68,6 +69,9 @@ training: callbacks: grad_clip: _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + # ema: + # _target_: fastvideo.train.callbacks.ema.EMACallback + # beta: 0.9999 validation: _target_: fastvideo.train.callbacks.validation.ValidationCallback pipeline_target: fastvideo.pipelines.basic.wan.wangame_i2v_pipeline.WanGameActionImageToVideoPipeline diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index d56e0f06a..a1e49a8ff 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -29,6 +29,7 @@ models: method: _target_: fastvideo.train.methods.distribution_matching.self_forcing.SelfForcingMethod + # use_ema: true rollout_mode: simulate generator_update_interval: 5 real_score_guidance_scale: 3.5 @@ -103,6 +104,9 @@ training: callbacks: grad_clip: _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + # ema: + # _target_: fastvideo.train.callbacks.ema.EMACallback + # beta: 0.9999 validation: _target_: fastvideo.train.callbacks.validation.ValidationCallback pipeline_target: fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline.WanGameCausalDMDPipeline diff --git a/fastvideo/train/callbacks/ema.py b/fastvideo/train/callbacks/ema.py index d03d912a2..328e39a76 100644 --- a/fastvideo/train/callbacks/ema.py +++ b/fastvideo/train/callbacks/ema.py @@ -24,8 +24,8 @@ class EMACallback(Callback): """Update EMA parameters after each optimizer step. - The EMA network lives on the model (``method.student.``). - If the model was created with ``use_ema=false``, the callback + The EMA network lives on the method (``method.ema``). + If the method was created with ``use_ema: false``, the callback detects this at train start and disables itself gracefully. Supports three beta strategies: @@ -43,7 +43,6 @@ def __init__( ema_halflife_kimg: float = 500.0, ema_rampup_ratio: float | None = 0.05, start_iter: int = 0, - ema_name: str = "ema", batch_size: int = 1, ) -> None: self._type = str(type) @@ -56,7 +55,6 @@ def __init__( else None ) self._start_iter = int(start_iter) - self._ema_name = str(ema_name) self._batch_size = int(batch_size) self._enabled = True @@ -69,19 +67,17 @@ def on_train_start( method: TrainingMethod, iteration: int = 0, ) -> None: - student = method.student - ema = getattr(student, self._ema_name, None) + ema = getattr(method, "ema", None) if ema is None: self._enabled = False logger.info( - "EMA %r not found on student model; " + "EMA not found on method; " "EMA callback disabled.", - self._ema_name, ) return assert not ema.training, ( - f"EMA {self._ema_name} should be in eval mode" + "EMA should be in eval mode" ) for name, p in ema.named_parameters(): assert not p.requires_grad, ( @@ -103,18 +99,17 @@ def on_training_step_end( if iteration == self._start_iter: logger.info( "Starting EMA %r updates at iteration %d.", - self._ema_name, + "ema", iteration, ) beta = self._compute_beta(iteration) - student = method.student - ema = getattr(student, self._ema_name) + ema = method.ema ema_state = ema.state_dict() with torch.no_grad(): for name, p_net in ( - student.transformer.named_parameters() + method.student.transformer.named_parameters() ): full = self._gather_full(p_net) ema_key = name.replace( @@ -139,7 +134,7 @@ def on_training_step_end( ema_p.lerp_(val, 1.0 - beta) for name, buf in ( - student.transformer.named_buffers() + method.student.transformer.named_buffers() ): if name in ema_state: ema_state[name].copy_( @@ -152,7 +147,7 @@ def on_training_step_end( tracker = getattr(method, "tracker", None) if tracker is not None: tracker.log( - {f"ema/{self._ema_name}_beta": beta}, + {"ema/beta": beta}, iteration, ) diff --git a/fastvideo/train/callbacks/validation.py b/fastvideo/train/callbacks/validation.py index 7ff770cf9..edd49189b 100644 --- a/fastvideo/train/callbacks/validation.py +++ b/fastvideo/train/callbacks/validation.py @@ -178,7 +178,8 @@ def _run_validation( step: int, ) -> None: tc = self.training_config - transformer = method.student.transformer + # Use EMA transformer for validation when available. + transformer = method.transformer_inference was_training = bool( getattr(transformer, "training", False) ) diff --git a/fastvideo/train/methods/base.py b/fastvideo/train/methods/base.py index 1ad06c3b1..f56de8027 100644 --- a/fastvideo/train/methods/base.py +++ b/fastvideo/train/methods/base.py @@ -2,14 +2,18 @@ from __future__ import annotations +import copy from abc import ABC, abstractmethod from collections.abc import Sequence from typing import Any, Literal, cast import torch +from fastvideo.logger import init_logger from fastvideo.train.models.base import ModelBase +logger = init_logger(__name__) + LogScalar = float | int | torch.Tensor @@ -41,6 +45,9 @@ def __init__( self.validation_config: dict[str, Any] = dict( getattr(cfg, "validation", {}) or {} ) + self._use_ema: bool = bool( + self.method_config.get("use_ema", False) + ) # Build nn.ModuleDict for FSDP / checkpoint visibility. self.role_modules = torch.nn.ModuleDict() @@ -52,6 +59,44 @@ def __init__( if mods: self.role_modules[role] = torch.nn.ModuleDict(mods) + self._setup_ema() + + # ------------------------------------------------------------------ + # EMA + # ------------------------------------------------------------------ + + def _setup_ema(self) -> None: + """Create EMA copy of student transformer. + + Called at the end of ``__init__``, before FSDP wrapping. + Only acts when ``use_ema: true`` is set in method config. + """ + if not self._use_ema: + return + logger.info( + "Initializing EMA from student transformer", + ) + ema = copy.deepcopy(self.student.transformer) + ema.eval().requires_grad_(False) + self.ema = ema + # Register in role_modules for FSDP / checkpoint. + if "student" not in self.role_modules: + self.role_modules["student"] = ( + torch.nn.ModuleDict() + ) + self.role_modules["student"]["ema"] = ema # type: ignore[index] + + @property + def transformer_inference(self) -> torch.nn.Module: + """Return EMA transformer for inference if available.""" + if self._use_ema: + ema = getattr(self, "ema", None) + if ema is not None: + return ema + return self.student.transformer + + # ------------------------------------------------------------------ + def set_tracker(self, tracker: Any) -> None: self.tracker = tracker diff --git a/fastvideo/train/models/base.py b/fastvideo/train/models/base.py index 0c62bb401..d74406278 100644 --- a/fastvideo/train/models/base.py +++ b/fastvideo/train/models/base.py @@ -2,21 +2,16 @@ from __future__ import annotations -import copy from abc import ABC, abstractmethod from typing import Any, Literal, TYPE_CHECKING import torch -from fastvideo.logger import init_logger - if TYPE_CHECKING: from fastvideo.train.utils.training_config import ( TrainingConfig, ) from fastvideo.pipelines import TrainingBatch -logger = init_logger(__name__) - class ModelBase(ABC): """Per-role model instance. @@ -31,7 +26,6 @@ class ModelBase(ABC): transformer: torch.nn.Module noise_scheduler: Any _trainable: bool - _use_ema: list[str] # ------------------------------------------------------------------ # Lifecycle @@ -51,59 +45,6 @@ def get_rng_generators(self) -> dict[str, torch.Generator]: """Return RNG generators for checkpoint resume.""" return {} - # ------------------------------------------------------------------ - # EMA - # ------------------------------------------------------------------ - - def _setup_ema(self) -> None: - """Create EMA copies of the transformer. - - Call after ``self.transformer`` is set and before FSDP wrapping. - Each name in ``self._use_ema`` becomes an attribute holding a - deep copy of ``self.transformer`` in eval mode with no gradients. - """ - if not getattr(self, "_use_ema", None): - return - for name in self._use_ema: - if hasattr(self, name): - logger.warning( - "EMA network %r already exists, " - "skipping initialization.", - name, - ) - continue - logger.info( - "Initializing EMA network %r from " - "transformer", - name, - ) - ema = copy.deepcopy(self.transformer) - ema.eval().requires_grad_(False) - setattr(self, name, ema) - - @property - def ema_dict(self) -> dict[str, torch.nn.Module]: - """Return dict of EMA networks (empty if EMA disabled).""" - if not getattr(self, "_use_ema", None): - return {} - return { - name: getattr(self, name) - for name in self._use_ema - if hasattr(self, name) - } - - @property - def transformer_inference(self) -> torch.nn.Module: - """Return the transformer for inference. - - Returns the first EMA network when available, - otherwise the training transformer. - """ - ema = self.ema_dict - if ema: - return next(iter(ema.values())) - return self.transformer - # ------------------------------------------------------------------ # Timestep helpers # ------------------------------------------------------------------ diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index 1fdd62167..557bf7512 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -72,16 +72,10 @@ def __init__( flow_shift: float = 3.0, enable_gradient_checkpointing_type: str | None = None, - use_ema: list[str] | bool = False, ) -> None: self._init_from = str(init_from) self._trainable = bool(trainable) - if isinstance(use_ema, bool): - self._use_ema = ["ema"] if use_ema else [] - else: - self._use_ema = list(use_ema) - transformer = load_module_from_path( model_path=self._init_from, module_type="transformer", @@ -96,7 +90,6 @@ def __init__( checkpointing_type=(enable_gradient_checkpointing_type), ) self.transformer = transformer - self._setup_ema() self.noise_scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) diff --git a/fastvideo/train/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py index 60d6202fe..47159c6de 100644 --- a/fastvideo/train/models/wangame/wangame.py +++ b/fastvideo/train/models/wangame/wangame.py @@ -67,16 +67,10 @@ def __init__( disable_custom_init_weights: bool = False, flow_shift: float = 3.0, enable_gradient_checkpointing_type: str | None = None, - use_ema: list[str] | bool = False, ) -> None: self._init_from = str(init_from) self._trainable = bool(trainable) - if isinstance(use_ema, bool): - self._use_ema = ["ema"] if use_ema else [] - else: - self._use_ema = list(use_ema) - self.transformer = self._load_transformer( init_from=self._init_from, trainable=self._trainable, @@ -84,7 +78,6 @@ def __init__( enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), training_config=training_config, ) - self._setup_ema() self.noise_scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) diff --git a/fastvideo/train/models/wangame/wangame_causal.py b/fastvideo/train/models/wangame/wangame_causal.py index 21c0f40e7..902ed7824 100644 --- a/fastvideo/train/models/wangame/wangame_causal.py +++ b/fastvideo/train/models/wangame/wangame_causal.py @@ -45,7 +45,6 @@ def __init__( disable_custom_init_weights: bool = False, flow_shift: float = 3.0, enable_gradient_checkpointing_type: str | None = None, - use_ema: list[str] | bool = False, ) -> None: super().__init__( init_from=init_from, @@ -54,7 +53,6 @@ def __init__( flow_shift=flow_shift, enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), training_config=training_config, - use_ema=use_ema, ) self._streaming_caches: dict[tuple[int, str], _StreamingCaches] = {} diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index 1c1ef4b93..0c916cfe4 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -240,8 +240,6 @@ def maybe_warmstart_role_modules( if model is not None: if model.transformer is not None: modules["transformer"] = model.transformer - for ema_name, ema_module in model.ema_dict.items(): - modules[ema_name] = ema_module elif bundle is not None: handle = bundle.role(str(role)) modules = dict(handle.modules) @@ -335,8 +333,6 @@ def _copy_or_link(src: str, dest: str) -> None: if model is not None: if model.transformer is not None: modules["transformer"] = model.transformer - for ema_name, ema_module in model.ema_dict.items(): - modules[ema_name] = ema_module elif bundle is not None: handle = bundle.role(str(role)) modules = dict(handle.modules) @@ -436,6 +432,7 @@ def __init__( *, bundle: Any = None, role_models: dict[str, ModelBase] | None = None, + extra_role_modules: dict[str, dict[str, torch.nn.Module]] | None = None, optimizers: dict[str, torch.optim.Optimizer] | None = None, lr_schedulers: dict[str, Any] | None = None, dataloader: Any, @@ -446,6 +443,7 @@ def __init__( ) -> None: self.bundle = bundle self.role_models = role_models or {} + self.extra_role_modules = extra_role_modules or {} self.optimizers = optimizers or {} self.lr_schedulers = lr_schedulers or {} self.dataloader = dataloader @@ -468,9 +466,9 @@ def _build_role_states_from_model( if model.transformer is not None: modules["transformer"] = model.transformer - # Include EMA networks in checkpoint. - for ema_name, ema_module in model.ema_dict.items(): - modules[ema_name] = ema_module + # Include extra modules (e.g. EMA) owned by the method. + for name, mod in self.extra_role_modules.get(role, {}).items(): + modules[name] = mod container = _RoleModuleContainer(modules) diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 02e405cfe..682e3e4f3 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -7,6 +7,8 @@ import sys from typing import Any +import torch + from fastvideo.logger import init_logger logger = init_logger(__name__) @@ -71,8 +73,15 @@ def run_training_from_config( keep_last=int(tc.checkpoint.checkpoints_total_limit or 0), ) + # Build extra role modules (e.g. EMA) for checkpointing. + extra_role_modules: dict[str, dict[str, torch.nn.Module]] = {} + ema = getattr(method, "ema", None) + if ema is not None: + extra_role_modules["student"] = {"ema": ema} + checkpoint_manager = CheckpointManager( role_models=method._role_models, + extra_role_modules=extra_role_modules, optimizers=method._optimizer_dict, lr_schedulers=method._lr_scheduler_dict, dataloader=dataloader, From 793d1842e89bf2b63ffb7a2f0f4b5cf134f79fa0 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 00:13:58 +0000 Subject: [PATCH 193/214] remove legacy bundle design --- fastvideo/train/dispatch.py | 1 - fastvideo/train/doc/utils/checkpoint.md | 22 +++---- fastvideo/train/utils/checkpoint.py | 79 ++++--------------------- 3 files changed, 22 insertions(+), 80 deletions(-) diff --git a/fastvideo/train/dispatch.py b/fastvideo/train/dispatch.py index df16c9614..e70d09f32 100644 --- a/fastvideo/train/dispatch.py +++ b/fastvideo/train/dispatch.py @@ -54,7 +54,6 @@ def build_from_config(cfg: RunConfig, ) -> tuple[TrainingConfig, TrainingMethod, # Warmstart uses the model's transformer directly. model = role_models[role] maybe_warmstart_role_modules( - bundle=None, role=str(role), init_from_checkpoint=str(init_from_checkpoint), checkpoint_role=(str(checkpoint_role) if checkpoint_role else None), diff --git a/fastvideo/train/doc/utils/checkpoint.md b/fastvideo/train/doc/utils/checkpoint.md index a9e88f7d8..8243b8325 100644 --- a/fastvideo/train/doc/utils/checkpoint.md +++ b/fastvideo/train/doc/utils/checkpoint.md @@ -1,27 +1,29 @@ -# `fastvideo/distillation/utils/checkpoint.py` +# `fastvideo/train/utils/checkpoint.py` **目的** -- Phase 2 的 role-based checkpoint/save-resume 管理： - - 按 role 保存/恢复 modules、optimizers、schedulers +- Role-based checkpoint/save-resume 管理： + - 按 role 保存/恢复 modules、optimizers、schedulers（仅 trainable roles） - 可选保存 dataloader 状态（如果 dataloader 是 stateful） - - 保存 RNG（全局 RNG + method 暴露的额外 generators，例如 model/validator 的 RNG） + - 保存 RNG（全局 RNG + method 暴露的额外 generators） + - 保存 callback 状态（如 validation RNG） + - 支持 extra_role_modules（如 EMA shadow weights） **关键类型** -- `DistillCheckpointConfig` +- `CheckpointConfig` - `save_steps` / `keep_last` -- `DistillCheckpointManager` +- `CheckpointManager` - `maybe_resume(resume_from_checkpoint=...) -> step | None` - `maybe_save(step)` - `save_final(step)` **关键机制** -- 只对 `handle.trainable == True` 的 role 保存 optimizer/scheduler 状态。 +- 只对 `model._trainable == True` 的 role 保存 optimizer/scheduler 状态。 - 使用 `torch.distributed.checkpoint (dcp)` 做分布式 checkpoint。 - `resume_from_checkpoint` 支持： - `checkpoint-` 目录 - `checkpoint-/dcp` - `output_dir`（自动选择最新 checkpoint） -**与 Method 的关系** -- 该文件假设：训练开始前 `RoleHandle.optimizers/lr_schedulers` 已经就绪。 - Phase 2.9 开始，它们通常由 method（例如 `DMD2Method`）在构造时创建并写回 handle。 +**独立函数** +- `maybe_warmstart_role_modules`: 从 DCP checkpoint best-effort 加载模型权重（不恢复 optimizer/step）。 +- `save_role_pretrained`: 导出 role modules 为 diffusers-style 模型目录。 diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index 0c916cfe4..ce998ba13 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -202,11 +202,10 @@ def load_state_dict(self, state_dict: dict[str, Any]) -> None: def maybe_warmstart_role_modules( *, - bundle: Any = None, role: str, init_from_checkpoint: str | None, checkpoint_role: str | None = None, - model: ModelBase | None = None, + model: ModelBase, ) -> None: """Warmstart model modules for `role` from a Phase 2/3 DCP checkpoint. @@ -215,10 +214,6 @@ def maybe_warmstart_role_modules( - does not advance `start_step` The checkpoint directory is expected to be ``checkpoint-/dcp/*.distcp``. - - In the new ``_target_``-based flow, pass ``model`` (a ``ModelBase`` instance) - instead of ``bundle``. The legacy ``bundle`` parameter is accepted but - ignored when ``model`` is provided. """ if not init_from_checkpoint: @@ -235,17 +230,9 @@ def maybe_warmstart_role_modules( if not dcp_dir.is_dir(): raise FileNotFoundError(f"Missing dcp dir under checkpoint: {dcp_dir}") - # Build modules dict: new path uses ModelBase, legacy uses RoleManager. modules: dict[str, torch.nn.Module] = {} - if model is not None: - if model.transformer is not None: - modules["transformer"] = model.transformer - elif bundle is not None: - handle = bundle.role(str(role)) - modules = dict(handle.modules) - else: - raise ValueError("maybe_warmstart_role_modules requires either " - "'model' or 'bundle'") + if model.transformer is not None: + modules["transformer"] = model.transformer available_modules = _get_dcp_role_module_names(dcp_dir, role=str(checkpoint_role)) @@ -284,13 +271,12 @@ def maybe_warmstart_role_modules( def save_role_pretrained( *, - bundle: Any = None, role: str, base_model_path: str, output_dir: str, module_names: list[str] | None = None, overwrite: bool = False, - model: ModelBase | None = None, + model: ModelBase, ) -> str: """Export a role's modules into a diffusers-style model directory. @@ -298,9 +284,6 @@ def save_role_pretrained( ``PipelineComponentLoader`` (i.e., has ``model_index.json``, ``transformer/``, ``vae/``, and other pipeline components copied from ``base_model_path``). - - In the new ``_target_``-based flow, pass ``model`` (a ``ModelBase`` - instance) instead of ``bundle``. """ # Resolve HF IDs to local directories (same behavior as module loader). @@ -328,17 +311,9 @@ def _copy_or_link(src: str, dest: str) -> None: _barrier() - # Build modules dict from model or bundle. modules: dict[str, torch.nn.Module] = {} - if model is not None: - if model.transformer is not None: - modules["transformer"] = model.transformer - elif bundle is not None: - handle = bundle.role(str(role)) - modules = dict(handle.modules) - else: - raise ValueError("save_role_pretrained requires either " - "'model' or 'bundle'") + if model.transformer is not None: + modules["transformer"] = model.transformer if module_names is None: module_names = sorted(modules.keys()) @@ -422,16 +397,12 @@ class CheckpointManager: - Checkpoint policy lives in YAML (via TrainingArgs fields). - Resume path is typically provided via CLI (``--resume-from-checkpoint``). - - Accepts either a ``role_models`` dict (new ``_target_``-based flow) - or a legacy ``bundle`` (``RoleManager``). """ def __init__( self, *, - bundle: Any = None, - role_models: dict[str, ModelBase] | None = None, + role_models: dict[str, ModelBase], extra_role_modules: dict[str, dict[str, torch.nn.Module]] | None = None, optimizers: dict[str, torch.optim.Optimizer] | None = None, lr_schedulers: dict[str, Any] | None = None, @@ -441,8 +412,7 @@ def __init__( get_rng_generators: Callable[[], dict[str, torch.Generator]] | None = None, callbacks: Any | None = None, ) -> None: - self.bundle = bundle - self.role_models = role_models or {} + self.role_models = role_models self.extra_role_modules = extra_role_modules or {} self.optimizers = optimizers or {} self.lr_schedulers = lr_schedulers or {} @@ -489,41 +459,12 @@ def _build_role_states_from_model( return states - def _build_role_states_from_handle( - self, - role: str, - handle: Any, - ) -> dict[str, Any]: - if not handle.trainable: - return {} - - states: dict[str, Any] = {} - container = _RoleModuleContainer(handle.modules) - - for module_name, module in handle.modules.items(): - states[f"roles.{role}.{module_name}"] = ModelWrapper(module) - - for name, optimizer in handle.optimizers.items(): - states[f"optimizers.{role}.{name}"] = OptimizerWrapper( - container, - optimizer, - ) - - for name, scheduler in handle.lr_schedulers.items(): - states[f"schedulers.{role}.{name}"] = SchedulerWrapper(scheduler) - - return states - def _build_states(self) -> dict[str, Any]: states: dict[str, Any] = {} # Models/opts/schedulers are role-scoped. - if self.role_models: - for role, model in self.role_models.items(): - states.update(self._build_role_states_from_model(role, model)) - elif self.bundle is not None: - for role, handle in self.bundle.roles.items(): - states.update(self._build_role_states_from_handle(role, handle)) + for role, model in self.role_models.items(): + states.update(self._build_role_states_from_model(role, model)) # Dataloader (optional but recommended for exact resume). if _is_stateful(self.dataloader): From 56a8ee8a17934a1b3ec68124766a9bb865c3a869 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 00:18:04 +0000 Subject: [PATCH 194/214] better checkpointing --- fastvideo/train/methods/base.py | 58 +++++++++++++++++++++ fastvideo/train/utils/checkpoint.py | 80 ++++------------------------- fastvideo/training/distillation.py | 14 +---- 3 files changed, 69 insertions(+), 83 deletions(-) diff --git a/fastvideo/train/methods/base.py b/fastvideo/train/methods/base.py index f56de8027..06b63a3d7 100644 --- a/fastvideo/train/methods/base.py +++ b/fastvideo/train/methods/base.py @@ -11,6 +11,13 @@ from fastvideo.logger import init_logger from fastvideo.train.models.base import ModelBase +from fastvideo.train.utils.checkpoint import _RoleModuleContainer +from fastvideo.training.checkpointing_utils import ( + ModelWrapper, + OptimizerWrapper, + RandomStateWrapper, + SchedulerWrapper, +) logger = init_logger(__name__) @@ -136,6 +143,57 @@ def _optimizer_dict(self) -> dict[str, Any]: def _lr_scheduler_dict(self) -> dict[str, Any]: ... + def checkpoint_state(self) -> dict[str, Any]: + """Return DCP-ready checkpoint state for all trainable roles. + + Keys follow the convention: + ``roles..``, ``optimizers.``, + ``schedulers.``, ``random_state.*``. + """ + states: dict[str, Any] = {} + + for role, model in self._role_models.items(): + if not getattr(model, "_trainable", False): + continue + + modules: dict[str, torch.nn.Module] = {} + if model.transformer is not None: + modules["transformer"] = model.transformer + ema = getattr(self, "ema", None) + if role == "student" and ema is not None: + modules["ema"] = ema + + container = _RoleModuleContainer(modules) + + for module_name, module in modules.items(): + states[ + f"roles.{role}.{module_name}" + ] = ModelWrapper(module) + + opt = self._optimizer_dict.get(role) + if opt is not None: + states[ + f"optimizers.{role}" + ] = OptimizerWrapper(container, opt) + + sched = self._lr_scheduler_dict.get(role) + if sched is not None: + states[ + f"schedulers.{role}" + ] = SchedulerWrapper(sched) + + # RNG states. + states["random_state"] = RandomStateWrapper(None) + for name, gen in ( + self.get_rng_generators() or {} + ).items(): + if gen is not None: + states[ + f"random_state.{name}" + ] = RandomStateWrapper(gen) + + return states + def backward( self, loss_map: dict[str, torch.Tensor], diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index ce998ba13..a56206d0a 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -5,7 +5,6 @@ import os import re import shutil -from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from typing import Any @@ -21,12 +20,6 @@ from fastvideo.train.models.base import ModelBase from fastvideo.logger import init_logger -from fastvideo.training.checkpointing_utils import ( - ModelWrapper, - OptimizerWrapper, - RandomStateWrapper, - SchedulerWrapper, -) logger = init_logger(__name__) @@ -359,10 +352,11 @@ def _copy_or_link(src: str, dest: str) -> None: class _RoleModuleContainer(torch.nn.Module): - """Ephemeral container to expose multiple role modules as a single module. + """Ephemeral container to expose multiple role modules as a single + ``nn.Module``. - Needed because `OptimizerWrapper` expects a single root module covering all - parameters owned by the optimizer. + Used by ``OptimizerWrapper`` which expects a single root module + covering all parameters owned by the optimizer. """ def __init__(self, modules: dict[str, torch.nn.Module]) -> None: @@ -402,85 +396,31 @@ class CheckpointManager: def __init__( self, *, - role_models: dict[str, ModelBase], - extra_role_modules: dict[str, dict[str, torch.nn.Module]] | None = None, - optimizers: dict[str, torch.optim.Optimizer] | None = None, - lr_schedulers: dict[str, Any] | None = None, + method: Any, dataloader: Any, output_dir: str, config: CheckpointConfig, - get_rng_generators: Callable[[], dict[str, torch.Generator]] | None = None, callbacks: Any | None = None, ) -> None: - self.role_models = role_models - self.extra_role_modules = extra_role_modules or {} - self.optimizers = optimizers or {} - self.lr_schedulers = lr_schedulers or {} + self.method = method self.dataloader = dataloader self.output_dir = str(output_dir) self.config = config - self._get_rng_generators = get_rng_generators self._callbacks = callbacks self._last_saved_step: int | None = None - def _build_role_states_from_model( - self, - role: str, - model: ModelBase, - ) -> dict[str, Any]: - if not getattr(model, "_trainable", False): - return {} - - states: dict[str, Any] = {} - modules: dict[str, torch.nn.Module] = {} - if model.transformer is not None: - modules["transformer"] = model.transformer - - # Include extra modules (e.g. EMA) owned by the method. - for name, mod in self.extra_role_modules.get(role, {}).items(): - modules[name] = mod - - container = _RoleModuleContainer(modules) - - for module_name, module in modules.items(): - states[f"roles.{role}.{module_name}"] = ModelWrapper(module) - - # Optimizers/schedulers are keyed by role name in the flat dicts. - for name, optimizer in self.optimizers.items(): - if name.startswith(f"{role}.") or name == role: - states[f"optimizers.{name}"] = OptimizerWrapper( - container, - optimizer, - ) - - for name, scheduler in self.lr_schedulers.items(): - if name.startswith(f"{role}.") or name == role: - states[f"schedulers.{name}"] = SchedulerWrapper(scheduler) - - return states - def _build_states(self) -> dict[str, Any]: - states: dict[str, Any] = {} - - # Models/opts/schedulers are role-scoped. - for role, model in self.role_models.items(): - states.update(self._build_role_states_from_model(role, model)) + states: dict[str, Any] = self.method.checkpoint_state() # Dataloader (optional but recommended for exact resume). if _is_stateful(self.dataloader): states["dataloader"] = self.dataloader - # RNG states: always save global RNG; also save adapter-provided generators. - states["random_state"] = RandomStateWrapper(None) - if self._get_rng_generators is not None: - for name, gen in (self._get_rng_generators() or {}).items(): - if gen is None: - continue - states[f"random_state.{name}"] = RandomStateWrapper(gen) - # Callback state (e.g. EMA shadow weights, validation RNG). if self._callbacks is not None and _is_stateful(self._callbacks): - states["callbacks"] = _CallbackStateWrapper(self._callbacks) + states["callbacks"] = _CallbackStateWrapper( + self._callbacks, + ) return states diff --git a/fastvideo/training/distillation.py b/fastvideo/training/distillation.py index 682e3e4f3..1008ce5a1 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/training/distillation.py @@ -7,8 +7,6 @@ import sys from typing import Any -import torch - from fastvideo.logger import init_logger logger = init_logger(__name__) @@ -73,21 +71,11 @@ def run_training_from_config( keep_last=int(tc.checkpoint.checkpoints_total_limit or 0), ) - # Build extra role modules (e.g. EMA) for checkpointing. - extra_role_modules: dict[str, dict[str, torch.nn.Module]] = {} - ema = getattr(method, "ema", None) - if ema is not None: - extra_role_modules["student"] = {"ema": ema} - checkpoint_manager = CheckpointManager( - role_models=method._role_models, - extra_role_modules=extra_role_modules, - optimizers=method._optimizer_dict, - lr_schedulers=method._lr_scheduler_dict, + method=method, dataloader=dataloader, output_dir=tc.checkpoint.output_dir, config=ckpt_config, - get_rng_generators=method.get_rng_generators, callbacks=trainer.callbacks, ) From 4bbb27e573d55c6194801e8056b5df6e9c4cb7f3 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 00:33:38 +0000 Subject: [PATCH 195/214] entry point, dcp to diffuers conversion. --- fastvideo/train/entrypoint/__init__.py | 1 + .../train/entrypoint/dcp_to_diffusers.py | 299 ++++++++++++++++++ .../entrypoint/train.py} | 79 +++-- fastvideo/train/utils/checkpoint.py | 31 +- fastvideo/train/validators/__init__.py | 17 - fastvideo/train/validators/base.py | 23 -- fastvideo/train/validators/wan.py | 23 -- fastvideo/train/validators/wangame.py | 23 -- 8 files changed, 385 insertions(+), 111 deletions(-) create mode 100644 fastvideo/train/entrypoint/__init__.py create mode 100644 fastvideo/train/entrypoint/dcp_to_diffusers.py rename fastvideo/{training/distillation.py => train/entrypoint/train.py} (61%) delete mode 100644 fastvideo/train/validators/__init__.py delete mode 100644 fastvideo/train/validators/base.py delete mode 100644 fastvideo/train/validators/wan.py delete mode 100644 fastvideo/train/validators/wangame.py diff --git a/fastvideo/train/entrypoint/__init__.py b/fastvideo/train/entrypoint/__init__.py new file mode 100644 index 000000000..988131360 --- /dev/null +++ b/fastvideo/train/entrypoint/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/fastvideo/train/entrypoint/dcp_to_diffusers.py b/fastvideo/train/entrypoint/dcp_to_diffusers.py new file mode 100644 index 000000000..700e62b8e --- /dev/null +++ b/fastvideo/train/entrypoint/dcp_to_diffusers.py @@ -0,0 +1,299 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Convert a DCP training checkpoint to a diffusers-style model directory. + +Works on a single GPU regardless of how many GPUs were used for training +(DCP handles resharding automatically). + +Usage (no torchrun needed):: + + python -m fastvideo.train.entrypoint.dcp_to_diffusers \ + --checkpoint /path/to/checkpoint-1000 \ + --output-dir /path/to/diffusers_output + +Or with torchrun (also fine):: + + torchrun --nproc_per_node=1 \ + -m fastvideo.train.entrypoint.dcp_to_diffusers \ + --checkpoint ... --output-dir ... + +The checkpoint must contain ``metadata.json`` (written by +``CheckpointManager``). If the checkpoint predates metadata +support, pass ``--config`` explicitly to provide the training +YAML. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from typing import Any + +from fastvideo.logger import init_logger + +logger = init_logger(__name__) + + +def _ensure_distributed() -> None: + """Set up a single-process distributed env if needed. + + When running under ``torchrun`` the env vars are already set. + For plain ``python`` we fill in the minimum required vars so + that ``init_process_group`` succeeds with world_size=1. + """ + for key, default in [ + ("RANK", "0"), + ("LOCAL_RANK", "0"), + ("WORLD_SIZE", "1"), + ("MASTER_ADDR", "127.0.0.1"), + ("MASTER_PORT", "29500"), + ]: + os.environ.setdefault(key, default) + + +def convert( + *, + checkpoint_dir: str, + output_dir: str, + config_path: str | None = None, + role: str = "student", + overwrite: bool = False, +) -> str: + """Load a DCP checkpoint and export as a diffusers model. + + Returns the path to the exported model directory. + """ + _ensure_distributed() + + from fastvideo.distributed import ( + maybe_init_distributed_environment_and_model_parallel, + ) + from fastvideo.train.dispatch import build_from_config + from fastvideo.train.utils.checkpoint import ( + CheckpointManager, + _resolve_resume_checkpoint, + save_role_pretrained, + ) + from fastvideo.train.utils.config import ( + RunConfig, + load_run_config, + ) + + import torch.distributed.checkpoint as dcp + + # -- Resolve checkpoint directory -- + resolved = _resolve_resume_checkpoint( + checkpoint_dir, output_dir=checkpoint_dir, + ) + dcp_dir = resolved / "dcp" + if not dcp_dir.is_dir(): + raise FileNotFoundError( + f"Missing dcp/ under {resolved}" + ) + + # -- Obtain config -- + cfg: RunConfig + if config_path is not None: + cfg = load_run_config(config_path) + else: + metadata = CheckpointManager.load_metadata( + resolved + ) + raw_config = metadata.get("config") + if raw_config is None: + raise ValueError( + "Checkpoint metadata.json does not " + "contain 'config'. Pass --config " + "explicitly." + ) + cfg = _run_config_from_raw(raw_config) + + tc = cfg.training + + # -- Init distributed (1 GPU is enough; DCP reshards) -- + maybe_init_distributed_environment_and_model_parallel( + tp_size=1, sp_size=1, + ) + + # Override distributed config so model loading uses 1 GPU. + tc.distributed.tp_size = 1 + tc.distributed.sp_size = 1 + tc.distributed.num_gpus = 1 + tc.distributed.hsdp_replicate_dim = 1 + tc.distributed.hsdp_shard_dim = 1 + + # -- Build model (loads pretrained weights + FSDP) -- + _, method, _, _ = build_from_config(cfg) + + # -- Load DCP weights into the model -- + states = method.checkpoint_state() + logger.info( + "Loading DCP checkpoint from %s", resolved, + ) + dcp.load(states, checkpoint_id=str(dcp_dir)) + + # -- Export to diffusers format -- + model = method._role_models[role] + base_model_path = str(tc.model_path) + if not base_model_path: + raise ValueError( + "Cannot determine base_model_path from " + "config. Ensure models.student.init_from " + "is set." + ) + + logger.info( + "Exporting role=%s to %s (base=%s)", + role, + output_dir, + base_model_path, + ) + result = save_role_pretrained( + role=role, + base_model_path=base_model_path, + output_dir=output_dir, + overwrite=overwrite, + model=model, + ) + logger.info("Export complete: %s", result) + return result + + +def _run_config_from_raw( + raw: dict[str, Any], +) -> Any: + """Reconstruct a RunConfig from a raw config dict. + + This mirrors ``load_run_config`` but operates on an + already-parsed dict (from metadata.json) instead of + reading from a YAML file. + """ + from fastvideo.train.utils.config import ( + RunConfig, + _build_training_config, + _parse_pipeline_config, + _require_mapping, + _require_str, + ) + + models_raw = _require_mapping( + raw.get("models"), where="models", + ) + models: dict[str, dict[str, Any]] = {} + for role_key, model_cfg_raw in models_raw.items(): + role_str = _require_str( + role_key, where="models.", + ) + model_cfg = _require_mapping( + model_cfg_raw, + where=f"models.{role_str}", + ) + models[role_str] = dict(model_cfg) + + method_raw = _require_mapping( + raw.get("method"), where="method", + ) + method = dict(method_raw) + + validation_raw = raw.get("validation", None) + validation: dict[str, Any] = ( + _require_mapping( + validation_raw, where="validation", + ) + if validation_raw is not None + else {} + ) + + callbacks_raw = raw.get("callbacks", None) + callbacks: dict[str, dict[str, Any]] = ( + _require_mapping( + callbacks_raw, where="callbacks", + ) + if callbacks_raw is not None + else {} + ) + + pipeline_config = _parse_pipeline_config( + raw, models=models, + ) + + training_raw = _require_mapping( + raw.get("training"), where="training", + ) + t = dict(training_raw) + t.pop("validation", None) + training = _build_training_config( + t, + models=models, + pipeline_config=pipeline_config, + validation=validation, + ) + + return RunConfig( + models=models, + method=method, + training=training, + validation=validation, + callbacks=callbacks, + raw=raw, + ) + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + "Convert a DCP training checkpoint to a " + "diffusers-style model directory. " + "Only 1 GPU needed (DCP reshards " + "automatically)." + ), + ) + parser.add_argument( + "--checkpoint", + type=str, + required=True, + help=( + "Path to checkpoint- dir, its dcp/ " + "subdir, or an output_dir (auto-picks " + "latest)." + ), + ) + parser.add_argument( + "--output-dir", + type=str, + required=True, + help="Destination for the diffusers model.", + ) + parser.add_argument( + "--config", + type=str, + default=None, + help=( + "Training YAML config. If omitted, read " + "from checkpoint metadata.json." + ), + ) + parser.add_argument( + "--role", + type=str, + default="student", + help="Role to export (default: student).", + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Overwrite output-dir if it exists.", + ) + args = parser.parse_args(sys.argv[1:]) + + convert( + checkpoint_dir=args.checkpoint, + output_dir=args.output_dir, + config_path=args.config, + role=args.role, + overwrite=args.overwrite, + ) + + +if __name__ == "__main__": + main() diff --git a/fastvideo/training/distillation.py b/fastvideo/train/entrypoint/train.py similarity index 61% rename from fastvideo/training/distillation.py rename to fastvideo/train/entrypoint/train.py index 1008ce5a1..ebb074a37 100644 --- a/fastvideo/training/distillation.py +++ b/fastvideo/train/entrypoint/train.py @@ -1,4 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 +"""YAML-only training entrypoint. + +Usage:: + + torchrun --nproc_per_node= -m fastvideo.train.entrypoint.train \ + --config path/to/run.yaml +""" from __future__ import annotations @@ -22,22 +29,23 @@ def run_training_from_config( """YAML-only training entrypoint (schema v2).""" from fastvideo.distributed import ( - maybe_init_distributed_environment_and_model_parallel, ) + maybe_init_distributed_environment_and_model_parallel, + ) from fastvideo.train import Trainer from fastvideo.train.utils.checkpoint import ( CheckpointConfig, CheckpointManager, ) - from fastvideo.train.dispatch import ( - build_from_config, ) - from fastvideo.train.utils.config import ( - load_run_config, ) + from fastvideo.train.dispatch import build_from_config + from fastvideo.train.utils.config import load_run_config cfg = load_run_config(config_path) tc = cfg.training if resume_from_checkpoint is not None: - tc.checkpoint.resume_from_checkpoint = str(resume_from_checkpoint) + tc.checkpoint.resume_from_checkpoint = str( + resume_from_checkpoint + ) if override_output_dir is not None: tc.checkpoint.output_dir = str(override_output_dir) @@ -46,11 +54,15 @@ def run_training_from_config( tc.distributed.sp_size, ) - _, method, dataloader, start_step = build_from_config(cfg) + _, method, dataloader, start_step = build_from_config( + cfg + ) if dry_run: - logger.info("Dry-run: config parsed and " - "build_from_config succeeded.") + logger.info( + "Dry-run: config parsed and " + "build_from_config succeeded." + ) return trainer = Trainer( @@ -67,8 +79,13 @@ def run_training_from_config( ) ckpt_config = CheckpointConfig( - save_steps=int(tc.checkpoint.training_state_checkpointing_steps or 0), - keep_last=int(tc.checkpoint.checkpoints_total_limit or 0), + save_steps=int( + tc.checkpoint.training_state_checkpointing_steps + or 0 + ), + keep_last=int( + tc.checkpoint.checkpoints_total_limit or 0 + ), ) checkpoint_manager = CheckpointManager( @@ -77,6 +94,7 @@ def run_training_from_config( output_dir=tc.checkpoint.output_dir, config=ckpt_config, callbacks=trainer.callbacks, + raw_config=cfg.raw, ) trainer.run( @@ -91,8 +109,12 @@ def run_training_from_config( def main(args: Any) -> None: config_path = str(args.config) dry_run = bool(args.dry_run) - resume_from_checkpoint = getattr(args, "resume_from_checkpoint", None) - override_output_dir = getattr(args, "override_output_dir", None) + resume_from_checkpoint = getattr( + args, "resume_from_checkpoint", None + ) + override_output_dir = getattr( + args, "override_output_dir", None + ) logger.info( "Starting training from config=%s", config_path, @@ -108,35 +130,44 @@ def main(args: Any) -> None: if __name__ == "__main__": argv = sys.argv - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + description="YAML-only training entrypoint.", + ) parser.add_argument( "--config", type=str, required=True, - help=("Path to training YAML config " - "(schema v2)."), + help=( + "Path to training YAML config (schema v2)." + ), ) parser.add_argument( "--dry-run", action="store_true", - help=("Parse config and build runtime, but do not " - "start training."), + help=( + "Parse config and build runtime, " + "but do not start training." + ), ) parser.add_argument( "--resume-from-checkpoint", type=str, default=None, - help=("Path to a checkpoint directory " - "(checkpoint-), its 'dcp/' subdir, " - "or an output_dir containing checkpoints " - "(auto-picks latest)."), + help=( + "Path to a checkpoint directory " + "(checkpoint-), its 'dcp/' subdir, " + "or an output_dir containing checkpoints " + "(auto-picks latest)." + ), ) parser.add_argument( "--override-output-dir", type=str, default=None, - help=("Override training.output_dir from YAML " - "(useful for repeated runs)."), + help=( + "Override training.output_dir from YAML " + "(useful for repeated runs)." + ), ) args = parser.parse_args(argv[1:]) main(args) diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index a56206d0a..1b6de78b0 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import re import shutil @@ -401,12 +402,14 @@ def __init__( output_dir: str, config: CheckpointConfig, callbacks: Any | None = None, + raw_config: dict[str, Any] | None = None, ) -> None: self.method = method self.dataloader = dataloader self.output_dir = str(output_dir) self.config = config self._callbacks = callbacks + self._raw_config = raw_config self._last_saved_step: int | None = None def _build_states(self) -> dict[str, Any]: @@ -453,13 +456,39 @@ def save(self, step: int) -> None: states = self._build_states() if _rank() == 0: - logger.info("Saving Phase 2 checkpoint to %s", checkpoint_dir) + logger.info( + "Saving checkpoint to %s", checkpoint_dir, + ) + self._write_metadata(checkpoint_dir, step) dcp.save(states, checkpoint_id=str(dcp_dir)) _barrier() self._last_saved_step = step self._cleanup_old_checkpoints() + def _write_metadata( + self, checkpoint_dir: Path, step: int, + ) -> None: + metadata: dict[str, Any] = {"step": step} + if self._raw_config is not None: + metadata["config"] = self._raw_config + meta_path = checkpoint_dir / "metadata.json" + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(metadata, f, indent=2) + + @staticmethod + def load_metadata( + checkpoint_dir: str | Path, + ) -> dict[str, Any]: + """Read ``metadata.json`` from a checkpoint dir.""" + meta_path = Path(checkpoint_dir) / "metadata.json" + if not meta_path.is_file(): + raise FileNotFoundError( + f"No metadata.json in {checkpoint_dir}" + ) + with open(meta_path, encoding="utf-8") as f: + return json.load(f) # type: ignore[no-any-return] + def maybe_resume(self, *, resume_from_checkpoint: str | None) -> int | None: if not resume_from_checkpoint: return None diff --git a/fastvideo/train/validators/__init__.py b/fastvideo/train/validators/__init__.py deleted file mode 100644 index b439ff081..000000000 --- a/fastvideo/train/validators/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Deprecated — use fastvideo.train.callbacks instead.""" - -from fastvideo.train.callbacks.callback import Callback -from fastvideo.train.callbacks.validation import ( - ValidationCallback, -) - -# Backwards-compatible aliases. -Validator = Callback -WanValidator = ValidationCallback - -__all__ = [ - "Validator", - "WanValidator", - "ValidationCallback", -] diff --git a/fastvideo/train/validators/base.py b/fastvideo/train/validators/base.py deleted file mode 100644 index 76a0892e4..000000000 --- a/fastvideo/train/validators/base.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Deprecated — use fastvideo.train.callbacks instead. - -Kept as import shims for backwards compatibility. -""" - -from __future__ import annotations - -import warnings - -from fastvideo.train.callbacks.callback import Callback - -warnings.warn( - "fastvideo.train.validators.base is deprecated. " - "Use fastvideo.train.callbacks instead.", - DeprecationWarning, - stacklevel=2, -) - -# Provide the old names as aliases so that existing -# imports do not break immediately. -Validator = Callback -ValidationRequest = None # type: ignore[assignment] diff --git a/fastvideo/train/validators/wan.py b/fastvideo/train/validators/wan.py deleted file mode 100644 index 3526dc8da..000000000 --- a/fastvideo/train/validators/wan.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Deprecated — use fastvideo.train.callbacks.ValidationCallback. - -Kept as an import shim for backwards compatibility. -""" - -from __future__ import annotations - -import warnings - -from fastvideo.train.callbacks.validation import ( - ValidationCallback, -) - -warnings.warn( - "fastvideo.train.validators.wan is deprecated. " - "Use fastvideo.train.callbacks.ValidationCallback " - "instead.", - DeprecationWarning, - stacklevel=2, -) - -WanValidator = ValidationCallback diff --git a/fastvideo/train/validators/wangame.py b/fastvideo/train/validators/wangame.py deleted file mode 100644 index 68e9aad87..000000000 --- a/fastvideo/train/validators/wangame.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Deprecated — use fastvideo.train.callbacks.ValidationCallback. - -Kept as an import shim for backwards compatibility. -""" - -from __future__ import annotations - -import warnings - -from fastvideo.train.callbacks.validation import ( - ValidationCallback, -) - -warnings.warn( - "fastvideo.train.validators.wangame is deprecated. " - "Use fastvideo.train.callbacks.ValidationCallback " - "instead.", - DeprecationWarning, - stacklevel=2, -) - -WanGameValidator = ValidationCallback From 093e130bb41b2df0406029ff569fc7e4eac0c93a Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 01:42:29 +0000 Subject: [PATCH 196/214] validation allow no video. --- examples/distillation/refactor/run.sh | 2 +- fastvideo/train/callbacks/callback.py | 27 ++++++++++++++++++------- fastvideo/train/callbacks/validation.py | 5 ++++- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/examples/distillation/refactor/run.sh b/examples/distillation/refactor/run.sh index 492a1188d..da809abb9 100755 --- a/examples/distillation/refactor/run.sh +++ b/examples/distillation/refactor/run.sh @@ -55,7 +55,7 @@ torchrun \ --nproc_per_node "${NUM_GPUS}" \ --master_addr "${MASTER_ADDR}" \ --master_port "${MASTER_PORT}" \ - fastvideo/training/distillation.py \ + fastvideo/train/entrypoint/train.py \ --config "${CONFIG}" \ "$@" \ 2>&1 | tee "${LOG_FILE}" diff --git a/fastvideo/train/callbacks/callback.py b/fastvideo/train/callbacks/callback.py index 465f0b9df..b44c7dc07 100644 --- a/fastvideo/train/callbacks/callback.py +++ b/fastvideo/train/callbacks/callback.py @@ -19,6 +19,13 @@ logger = init_logger(__name__) +# Well-known callback names that don't need ``_target_`` in YAML. +_BUILTIN_CALLBACKS: dict[str, str] = { + "grad_clip": "fastvideo.train.callbacks.grad_clip.GradNormClipCallback", + "validation": "fastvideo.train.callbacks.validation.ValidationCallback", + "ema": "fastvideo.train.callbacks.ema.EMACallback", +} + class Callback: """Base callback with no-op hooks. @@ -99,14 +106,20 @@ def __init__( if not callback_configs: return for name, cb_cfg in callback_configs.items(): + cb_cfg = dict(cb_cfg) if "_target_" not in cb_cfg: - logger.warning( - "Callback %r is missing '_target_', " - "skipping: %s", - name, - cb_cfg, - ) - continue + if name in _BUILTIN_CALLBACKS: + cb_cfg["_target_"] = ( + _BUILTIN_CALLBACKS[name] + ) + else: + logger.warning( + "Callback %r is missing " + "'_target_', skipping: %s", + name, + cb_cfg, + ) + continue logger.info( "Instantiating callback %r: %s", name, diff --git a/fastvideo/train/callbacks/validation.py b/fastvideo/train/callbacks/validation.py index edd49189b..2fd693837 100644 --- a/fastvideo/train/callbacks/validation.py +++ b/fastvideo/train/callbacks/validation.py @@ -479,7 +479,10 @@ def _prepare_validation_batch( validation_batch.get("image_path") or validation_batch.get("video_path") ) - if img_path is not None: + if img_path is not None and ( + img_path.startswith("http") + or os.path.isfile(img_path) + ): sampling_param.image_path = img_path temporal_compression_factor = int( From 175dbb9cab93128484be83f5e0ed10ece1a6e6dd Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 02:37:07 +0000 Subject: [PATCH 197/214] fix gradient ckpting and dit precision not identified problem --- .gitignore | 1 + fastvideo/train/models/wan/wan.py | 14 ++++++++++++-- fastvideo/train/utils/moduleloader.py | 8 +++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d2217678b..a464f92a4 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ env *.npy weights/ slurm_outputs/ +fastvideo/distillation # SSIM test outputs fastvideo/tests/ssim/generated_videos/ diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index 557bf7512..240ba28ba 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -84,10 +84,20 @@ def __init__( override_transformer_cls_name=("WanTransformer3DModel"), ) transformer = apply_trainable(transformer, trainable=self._trainable) - if (self._trainable and enable_gradient_checkpointing_type): + # Fall back to training_config.model if not set on the + # model YAML section directly. + ckpt_type = ( + enable_gradient_checkpointing_type + or getattr( + getattr(training_config, "model", None), + "enable_gradient_checkpointing_type", + None, + ) + ) + if self._trainable and ckpt_type: transformer = apply_activation_checkpointing( transformer, - checkpointing_type=(enable_gradient_checkpointing_type), + checkpointing_type=ckpt_type, ) self.transformer = transformer diff --git a/fastvideo/train/utils/moduleloader.py b/fastvideo/train/utils/moduleloader.py index 648b9ecc4..7d18db197 100644 --- a/fastvideo/train/utils/moduleloader.py +++ b/fastvideo/train/utils/moduleloader.py @@ -31,11 +31,17 @@ def _make_training_args( model_path: str, ) -> TrainingArgs: """Build a TrainingArgs for PipelineComponentLoader.""" + pipeline_config = tc.pipeline_config or PipelineConfig() + # Propagate dit_precision from TrainingConfig to PipelineConfig + # so that TransformerLoader.load() picks up the correct + # default_dtype (e.g. fp32 master weights for training). + if tc.dit_precision and tc.dit_precision != pipeline_config.dit_precision: + pipeline_config.dit_precision = tc.dit_precision return TrainingArgs( model_path=model_path, mode=ExecutionMode.DISTILLATION, inference_mode=False, - pipeline_config=(tc.pipeline_config or PipelineConfig()), + pipeline_config=pipeline_config, num_gpus=tc.distributed.num_gpus, tp_size=tc.distributed.tp_size, sp_size=tc.distributed.sp_size, From ffe7ed8e544a333b09894aa41239ef06c2702fa0 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 02:51:39 +0000 Subject: [PATCH 198/214] minor fixing --- .../self_forcing_wangame_causal_v3.yaml | 12 +- fastvideo/train/dispatch.py | 33 +--- fastvideo/train/doc/utils/checkpoint.md | 1 - fastvideo/train/doc/utils/config.md | 2 +- fastvideo/train/entrypoint/train.py | 2 +- fastvideo/train/utils/checkpoint.py | 156 ------------------ fastvideo/train/utils/config.py | 29 ++++ 7 files changed, 41 insertions(+), 194 deletions(-) diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index a1e49a8ff..b8e07925d 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -3,29 +3,27 @@ # Uses _target_-based instantiation — each model role is an independent # class instance; the method class is resolved directly from the YAML. # -# - Teacher weights come from a DFSFT causal checkpoint (DCP format). -# - Student/Critic are warmstarted from the same checkpoint. +# To warmstart from a DCP checkpoint, first convert it to diffusers format +# using `dcp_to_diffusers`, then point `init_from` at the converted directory. models: student: _target_: fastvideo.train.models.wangame.WanGameCausalModel + # TODO: update to converted diffusers path init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true - init_from_checkpoint: outputs/wangame_dfsft_causal_v3/persistent/checkpoint-22000 teacher: _target_: fastvideo.train.models.wangame.WanGameCausalModel + # TODO: update to converted diffusers path init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: false disable_custom_init_weights: true - init_from_checkpoint: outputs/wangame_dfsft_causal_v3/persistent/checkpoint-22000 - init_from_checkpoint_role: student critic: _target_: fastvideo.train.models.wangame.WanGameCausalModel + # TODO: update to converted diffusers path init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps trainable: true disable_custom_init_weights: true - init_from_checkpoint: outputs/wangame_dfsft_causal_v3/persistent/checkpoint-22000 - init_from_checkpoint_role: student method: _target_: fastvideo.train.methods.distribution_matching.self_forcing.SelfForcingMethod diff --git a/fastvideo/train/dispatch.py b/fastvideo/train/dispatch.py index e70d09f32..d6d8d9976 100644 --- a/fastvideo/train/dispatch.py +++ b/fastvideo/train/dispatch.py @@ -21,10 +21,9 @@ def build_from_config(cfg: RunConfig, ) -> tuple[TrainingConfig, TrainingMethod, """Build method + dataloader from a v3 run config. 1. Instantiate each model in ``cfg.models`` via ``_target_``. - 2. Resolve the method class from ``cfg.method["_target_"]``. - 3. Construct the method with ``(cfg=cfg, role_models=..., - validator=...)``. - 4. Return ``(training_args, method, dataloader, start_step)``. + 2. Resolve the method class from ``cfg.method["_target_"]`` + and construct it with ``(cfg=cfg, role_models=...)``. + 3. Return ``(training_args, method, dataloader, start_step)``. """ from fastvideo.train.models.base import ModelBase @@ -38,29 +37,7 @@ def build_from_config(cfg: RunConfig, ) -> tuple[TrainingConfig, TrainingMethod, f"ModelBase subclass, got {type(model).__name__}") role_models[role] = model - # --- 2. Warm-start from checkpoint if needed --- - from fastvideo.train.utils.checkpoint import ( - maybe_warmstart_role_modules, ) - - for role, model_cfg in cfg.models.items(): - init_from_checkpoint = model_cfg.get("init_from_checkpoint", None) - if not init_from_checkpoint: - continue - checkpoint_role = model_cfg.get("init_from_checkpoint_role", None) - if (checkpoint_role is not None and not isinstance(checkpoint_role, str)): - raise ValueError(f"models.{role}.init_from_checkpoint_role " - "must be a string when set, got " - f"{type(checkpoint_role).__name__}") - # Warmstart uses the model's transformer directly. - model = role_models[role] - maybe_warmstart_role_modules( - role=str(role), - init_from_checkpoint=str(init_from_checkpoint), - checkpoint_role=(str(checkpoint_role) if checkpoint_role else None), - model=model, - ) - - # --- 3. Build method --- + # --- 2. Build method --- method_cfg = dict(cfg.method) method_target = str(method_cfg.pop("_target_")) method_cls = resolve_target(method_target) @@ -73,7 +50,7 @@ def build_from_config(cfg: RunConfig, ) -> tuple[TrainingConfig, TrainingMethod, role_models=role_models, ) - # --- 4. Gather dataloader and start_step --- + # --- 3. Gather dataloader and start_step --- dataloader = (getattr(student, "dataloader", None) if student is not None else None) start_step = int(getattr(student, "start_step", 0) if student is not None else 0) diff --git a/fastvideo/train/doc/utils/checkpoint.md b/fastvideo/train/doc/utils/checkpoint.md index 8243b8325..79c2b4a41 100644 --- a/fastvideo/train/doc/utils/checkpoint.md +++ b/fastvideo/train/doc/utils/checkpoint.md @@ -25,5 +25,4 @@ - `output_dir`（自动选择最新 checkpoint） **独立函数** -- `maybe_warmstart_role_modules`: 从 DCP checkpoint best-effort 加载模型权重（不恢复 optimizer/step）。 - `save_role_pretrained`: 导出 role modules 为 diffusers-style 模型目录。 diff --git a/fastvideo/train/doc/utils/config.md b/fastvideo/train/doc/utils/config.md index 24cac22d9..d1d62d541 100644 --- a/fastvideo/train/doc/utils/config.md +++ b/fastvideo/train/doc/utils/config.md @@ -59,7 +59,7 @@ - `trainable`: 是否训练该 role（只影响 `requires_grad`/模式；具体 optimizer 由 method 决定） - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” - `extra: dict`：保留 `roles.` 下除上述字段外的所有 key/value， - 交给 model plugin / method 解释（例如 `roles.student.init_from_checkpoint: ...`） + 交给 model plugin / method 解释 ## 3) Builder 装配相关（build-time / run-time 边界） - model plugin（`@register_model`）直接构建并返回一个 `ModelBase` 实例： diff --git a/fastvideo/train/entrypoint/train.py b/fastvideo/train/entrypoint/train.py index ebb074a37..7b02509b4 100644 --- a/fastvideo/train/entrypoint/train.py +++ b/fastvideo/train/entrypoint/train.py @@ -67,7 +67,7 @@ def run_training_from_config( trainer = Trainer( tc, - config=cfg.raw, + config=cfg.resolved_config(), callback_configs=cfg.callbacks, ) diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index 1b6de78b0..663d618d5 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -16,7 +16,6 @@ from torch.distributed.checkpoint.state_dict import ( StateDictOptions, get_model_state_dict, - set_model_state_dict, ) from fastvideo.train.models.base import ModelBase @@ -108,161 +107,6 @@ def _resolve_resume_checkpoint(resume_from_checkpoint: str, *, output_dir: str) f"containing such checkpoints. Got: {path} (output_dir={out}).") -def _get_dcp_role_module_names(dcp_dir: Path, role: str) -> set[str]: - """Inspect a DCP checkpoint and return module names present for `roles..*`.""" - - if _rank() == 0: - reader = dcp.FileSystemReader(str(dcp_dir)) - metadata = reader.read_metadata() - module_names: set[str] = set() - prefix = f"roles.{role}." - for key in metadata.state_dict_metadata: - if not key.startswith(prefix): - continue - parts = key.split(".") - if len(parts) >= 3 and parts[2]: - module_names.add(parts[2]) - packed: list[str] = sorted(module_names) - else: - packed = [] - - if dist.is_available() and dist.is_initialized(): - obj_list: list[Any] = [packed] - dist.broadcast_object_list(obj_list, src=0) - packed = obj_list[0] - - return set(str(name) for name in packed) - - -def _get_dcp_role_module_param_keys(dcp_dir: Path, *, role: str, module_name: str) -> set[str]: - """Return inner param keys present under `roles...*` in a DCP checkpoint. - - Example checkpoint key: - roles.student.transformer.blocks.0.to_q.weight - Returned inner key: - blocks.0.to_q.weight - - This is used for warmstart to avoid DCP planner failures when the target - module expects parameters that are not present in the checkpoint (e.g., - older checkpoints produced under different wrapping/filtering). - """ - - if _rank() == 0: - reader = dcp.FileSystemReader(str(dcp_dir)) - metadata = reader.read_metadata() - prefix = f"roles.{role}.{module_name}." - keys: set[str] = set() - for key in metadata.state_dict_metadata: - if not key.startswith(prefix): - continue - inner = key[len(prefix):] - if inner: - keys.add(inner) - packed: list[str] = sorted(keys) - else: - packed = [] - - if dist.is_available() and dist.is_initialized(): - obj_list: list[Any] = [packed] - dist.broadcast_object_list(obj_list, src=0) - packed = obj_list[0] - - return set(str(name) for name in packed) - - -class _WarmstartModelWrapper(torch.distributed.checkpoint.stateful.Stateful): - """A StateDict wrapper used for warmstart (best-effort load). - - Unlike training checkpoints, warmstart is not required to be an exact - resume. We therefore restrict the *expected* parameter keys to those - actually present in the checkpoint to prevent DCP load planning failures. - """ - - def __init__(self, model: torch.nn.Module, *, allowed_keys: set[str]) -> None: - self.model = model - self.allowed_keys = allowed_keys - - def state_dict(self) -> dict[str, Any]: - state_dict = get_model_state_dict(self.model) # type: ignore[no-any-return] - return {k: v for k, v in state_dict.items() if k in self.allowed_keys} - - def load_state_dict(self, state_dict: dict[str, Any]) -> None: - set_model_state_dict( - self.model, - model_state_dict=state_dict, - options=StateDictOptions(strict=False), - ) - - -def maybe_warmstart_role_modules( - *, - role: str, - init_from_checkpoint: str | None, - checkpoint_role: str | None = None, - model: ModelBase, -) -> None: - """Warmstart model modules for `role` from a Phase 2/3 DCP checkpoint. - - This is **not** a training resume: - - only loads role modules (no optimizer/scheduler/dataloader/RNG state) - - does not advance `start_step` - - The checkpoint directory is expected to be ``checkpoint-/dcp/*.distcp``. - """ - - if not init_from_checkpoint: - return - - if checkpoint_role is None: - checkpoint_role = role - - resolved = _resolve_resume_checkpoint( - str(init_from_checkpoint), - output_dir=str(init_from_checkpoint), - ) - dcp_dir = resolved / "dcp" - if not dcp_dir.is_dir(): - raise FileNotFoundError(f"Missing dcp dir under checkpoint: {dcp_dir}") - - modules: dict[str, torch.nn.Module] = {} - if model.transformer is not None: - modules["transformer"] = model.transformer - - available_modules = _get_dcp_role_module_names(dcp_dir, role=str(checkpoint_role)) - - states: dict[str, Any] = {} - for module_name, module in modules.items(): - if module_name not in available_modules: - continue - allowed_keys = _get_dcp_role_module_param_keys( - dcp_dir, - role=str(checkpoint_role), - module_name=str(module_name), - ) - if not allowed_keys: - continue - states[f"roles.{checkpoint_role}.{module_name}"] = _WarmstartModelWrapper( - module, - allowed_keys=allowed_keys, - ) - - if not states: - raise ValueError(f"init_from_checkpoint={resolved} does not contain any saved modules for " - f"checkpoint_role={checkpoint_role!r}. Available modules in checkpoint: " - f"{sorted(available_modules)}") - - if _rank() == 0: - logger.info( - "Warmstarting role=%s from checkpoint=%s (checkpoint_role=%s, modules=%s)", - role, - resolved, - checkpoint_role, - sorted(states.keys()), - ) - dcp.load(states, checkpoint_id=str(dcp_dir)) - _barrier() - - def save_role_pretrained( *, role: str, diff --git a/fastvideo/train/utils/config.py b/fastvideo/train/utils/config.py index 390b11478..dac534b10 100644 --- a/fastvideo/train/utils/config.py +++ b/fastvideo/train/utils/config.py @@ -34,6 +34,35 @@ class RunConfig: callbacks: dict[str, dict[str, Any]] raw: dict[str, Any] + def resolved_config(self) -> dict[str, Any]: + """Return a fully-resolved config dict with defaults. + + Suitable for logging to W&B so that every parameter + (including defaults) is visible. + """ + import dataclasses + + def _safe_asdict(obj: Any) -> Any: + if dataclasses.is_dataclass(obj) and not isinstance(obj, type): + return { + f.name: _safe_asdict(getattr(obj, f.name)) + for f in dataclasses.fields(obj) + if not callable(getattr(obj, f.name)) + } + if isinstance(obj, dict): + return {k: _safe_asdict(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return type(obj)(_safe_asdict(v) for v in obj) + return obj + + resolved: dict[str, Any] = {} + resolved["models"] = dict(self.models) + resolved["method"] = dict(self.method) + resolved["training"] = _safe_asdict(self.training) + resolved["validation"] = dict(self.validation) + resolved["callbacks"] = dict(self.callbacks) + return resolved + # ---- parsing helpers (kept for use by methods) ---- From 5291ffd9e509bd6f39fb61f50f8d34a745667e34 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 02:54:01 +0000 Subject: [PATCH 199/214] dispatch->builder --- fastvideo/train/entrypoint/dcp_to_diffusers.py | 2 +- fastvideo/train/entrypoint/train.py | 2 +- fastvideo/train/{dispatch.py => utils/builder.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename fastvideo/train/{dispatch.py => utils/builder.py} (100%) diff --git a/fastvideo/train/entrypoint/dcp_to_diffusers.py b/fastvideo/train/entrypoint/dcp_to_diffusers.py index 700e62b8e..eae651e95 100644 --- a/fastvideo/train/entrypoint/dcp_to_diffusers.py +++ b/fastvideo/train/entrypoint/dcp_to_diffusers.py @@ -68,7 +68,7 @@ def convert( from fastvideo.distributed import ( maybe_init_distributed_environment_and_model_parallel, ) - from fastvideo.train.dispatch import build_from_config + from fastvideo.train.utils.builder import build_from_config from fastvideo.train.utils.checkpoint import ( CheckpointManager, _resolve_resume_checkpoint, diff --git a/fastvideo/train/entrypoint/train.py b/fastvideo/train/entrypoint/train.py index 7b02509b4..1c253004e 100644 --- a/fastvideo/train/entrypoint/train.py +++ b/fastvideo/train/entrypoint/train.py @@ -36,7 +36,7 @@ def run_training_from_config( CheckpointConfig, CheckpointManager, ) - from fastvideo.train.dispatch import build_from_config + from fastvideo.train.utils.builder import build_from_config from fastvideo.train.utils.config import load_run_config cfg = load_run_config(config_path) diff --git a/fastvideo/train/dispatch.py b/fastvideo/train/utils/builder.py similarity index 100% rename from fastvideo/train/dispatch.py rename to fastvideo/train/utils/builder.py From b2b7de39ea36f126c9626f0a9a47b88f16a3b4d0 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 03:01:21 +0000 Subject: [PATCH 200/214] grad norm must be set in callback system --- .../refactor/dfsft_wangame_causal_v3.yaml | 2 +- ...2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 2 +- .../finetune_wangame2.1_i2v_1.3B.yaml | 2 +- .../self_forcing_wangame_causal_v3.yaml | 2 +- fastvideo/train/callbacks/grad_clip.py | 17 +++--------- .../methods/distribution_matching/dmd2.py | 27 ++++++++++++------- fastvideo/train/utils/config.py | 2 -- fastvideo/train/utils/training_config.py | 1 - 8 files changed, 26 insertions(+), 29 deletions(-) diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index b10e5b8e4..fb3b0e941 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -50,7 +50,6 @@ training: weight_decay: 1.0e-4 lr_scheduler: constant lr_warmup_steps: 0 - max_grad_norm: 1.0 loop: max_train_steps: 20000 @@ -71,6 +70,7 @@ training: callbacks: grad_clip: _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + max_grad_norm: 1.0 # ema: # _target_: fastvideo.train.callbacks.ema.EMACallback # beta: 0.9999 diff --git a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml index 92a06051f..10f6ea028 100644 --- a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml +++ b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml @@ -39,7 +39,6 @@ training: weight_decay: 0.01 lr_scheduler: constant lr_warmup_steps: 0 - max_grad_norm: 1.0 loop: max_train_steps: 4000 @@ -66,6 +65,7 @@ training: callbacks: grad_clip: _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + max_grad_norm: 1.0 # ema: # _target_: fastvideo.train.callbacks.ema.EMACallback # beta: 0.9999 diff --git a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml index f1d2cbbd7..4edc3f10b 100644 --- a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml +++ b/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml @@ -47,7 +47,6 @@ training: weight_decay: 1.0e-4 lr_scheduler: constant lr_warmup_steps: 0 - max_grad_norm: 1.0 loop: max_train_steps: 20000 @@ -69,6 +68,7 @@ training: callbacks: grad_clip: _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + max_grad_norm: 1.0 # ema: # _target_: fastvideo.train.callbacks.ema.EMACallback # beta: 0.9999 diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml index b8e07925d..c72a4dc8c 100644 --- a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml @@ -81,7 +81,6 @@ training: weight_decay: 0.01 lr_scheduler: constant lr_warmup_steps: 0 - max_grad_norm: 1.0 loop: max_train_steps: 4000 @@ -102,6 +101,7 @@ training: callbacks: grad_clip: _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback + max_grad_norm: 1.0 # ema: # _target_: fastvideo.train.callbacks.ema.EMACallback # beta: 0.9999 diff --git a/fastvideo/train/callbacks/grad_clip.py b/fastvideo/train/callbacks/grad_clip.py index 74168b987..f8d445422 100644 --- a/fastvideo/train/callbacks/grad_clip.py +++ b/fastvideo/train/callbacks/grad_clip.py @@ -25,28 +25,19 @@ class GradNormClipCallback(Callback): """Clip gradient norms before the optimizer step. - ``max_grad_norm`` defaults to - ``training_config.optimizer.max_grad_norm`` when not - provided explicitly in the YAML config. + ``max_grad_norm`` must be set explicitly in the callback + config (``callbacks.grad_clip.max_grad_norm``). """ def __init__( self, *, - max_grad_norm: float | None = None, + max_grad_norm: float = 0.0, log_grad_norms: bool = False, ) -> None: - self._max_grad_norm_override = max_grad_norm + self._max_grad_norm = float(max_grad_norm) self._log_grad_norms = bool(log_grad_norms) - @property - def _max_grad_norm(self) -> float: - if self._max_grad_norm_override is not None: - return float(self._max_grad_norm_override) - return float( - self.training_config.optimizer.max_grad_norm - ) - def on_before_optimizer_step( self, method: TrainingMethod, diff --git a/fastvideo/train/methods/distribution_matching/dmd2.py b/fastvideo/train/methods/distribution_matching/dmd2.py index ac0c31eff..12b59824b 100644 --- a/fastvideo/train/methods/distribution_matching/dmd2.py +++ b/fastvideo/train/methods/distribution_matching/dmd2.py @@ -348,22 +348,28 @@ def _init_optimizers_and_schedulers(self) -> None: scheduler_name=student_sched, ) - # Critic optimizer/scheduler — read from method - # config. + # Critic optimizer/scheduler — must be set in + # method config. critic_lr_raw = get_optional_float( self.method_config, "fake_score_learning_rate", where="method.fake_score_learning_rate", ) - critic_lr = float(critic_lr_raw or 0.0) - if critic_lr == 0.0: - critic_lr = student_lr + if critic_lr_raw is None or critic_lr_raw == 0.0: + raise ValueError( + "method.fake_score_learning_rate must " + "be set to a positive value" + ) + critic_lr = float(critic_lr_raw) critic_betas_raw = self.method_config.get( "fake_score_betas", None ) if critic_betas_raw is None: - critic_betas_raw = tc.optimizer.betas + raise ValueError( + "method.fake_score_betas must be set " + "(e.g. [0.0, 0.999])" + ) critic_betas = parse_betas( critic_betas_raw, where="method.fake_score_betas", @@ -372,9 +378,12 @@ def _init_optimizers_and_schedulers(self) -> None: critic_sched_raw = self.method_config.get( "fake_score_lr_scheduler", None ) - critic_sched = str( - critic_sched_raw or student_sched - ) + if critic_sched_raw is None: + raise ValueError( + "method.fake_score_lr_scheduler must " + "be set (e.g. 'constant')" + ) + critic_sched = str(critic_sched_raw) critic_params = [ p for p in self.critic.transformer.parameters() diff --git a/fastvideo/train/utils/config.py b/fastvideo/train/utils/config.py index dac534b10..e00c9b07e 100644 --- a/fastvideo/train/utils/config.py +++ b/fastvideo/train/utils/config.py @@ -360,8 +360,6 @@ def _build_training_config( o.get("lr_power", 0.0) or 0.0), min_lr_ratio=float( o.get("min_lr_ratio", 0.5) or 0.5), - max_grad_norm=float( - o.get("max_grad_norm", 0.0) or 0.0), ), loop=TrainingLoopConfig( max_train_steps=int( diff --git a/fastvideo/train/utils/training_config.py b/fastvideo/train/utils/training_config.py index f70e054bc..b47d5a8cd 100644 --- a/fastvideo/train/utils/training_config.py +++ b/fastvideo/train/utils/training_config.py @@ -43,7 +43,6 @@ class OptimizerConfig: lr_num_cycles: int = 0 lr_power: float = 0.0 min_lr_ratio: float = 0.5 - max_grad_norm: float = 0.0 @dataclass(slots=True) From 4e3bbec7943aebe72998d2cd47da990cc97eecf9 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 03:09:45 +0000 Subject: [PATCH 201/214] minor --- fastvideo/train/doc/utils/checkpoint.md | 4 +- .../train/entrypoint/dcp_to_diffusers.py | 133 +++++++++++++++++- fastvideo/train/utils/checkpoint.py | 95 ------------- 3 files changed, 133 insertions(+), 99 deletions(-) diff --git a/fastvideo/train/doc/utils/checkpoint.md b/fastvideo/train/doc/utils/checkpoint.md index 79c2b4a41..669f7ca87 100644 --- a/fastvideo/train/doc/utils/checkpoint.md +++ b/fastvideo/train/doc/utils/checkpoint.md @@ -24,5 +24,5 @@ - `checkpoint-/dcp` - `output_dir`（自动选择最新 checkpoint） -**独立函数** -- `save_role_pretrained`: 导出 role modules 为 diffusers-style 模型目录。 +**相关** +- `dcp_to_diffusers` entrypoint 包含 `_save_role_pretrained`：导出 role modules 为 diffusers-style 模型目录。 diff --git a/fastvideo/train/entrypoint/dcp_to_diffusers.py b/fastvideo/train/entrypoint/dcp_to_diffusers.py index eae651e95..b28cff67e 100644 --- a/fastvideo/train/entrypoint/dcp_to_diffusers.py +++ b/fastvideo/train/entrypoint/dcp_to_diffusers.py @@ -51,6 +51,136 @@ def _ensure_distributed() -> None: os.environ.setdefault(key, default) +def _save_role_pretrained( + *, + role: str, + base_model_path: str, + output_dir: str, + module_names: list[str] | None = None, + overwrite: bool = False, + model: Any, +) -> str: + """Export a role's modules into a diffusers-style model dir. + + Produces a ``model_path`` loadable by + ``PipelineComponentLoader`` (``model_index.json``, + ``transformer/``, ``vae/``, etc. copied from + ``base_model_path``). + """ + import shutil + from pathlib import Path + + import torch + import torch.distributed as dist + from torch.distributed.checkpoint.state_dict import ( + StateDictOptions, + get_model_state_dict, + ) + + from fastvideo.utils import maybe_download_model + + def _rank() -> int: + if dist.is_available() and dist.is_initialized(): + return int(dist.get_rank()) + return 0 + + def _barrier() -> None: + if dist.is_available() and dist.is_initialized(): + dist.barrier() + + local_base = Path( + maybe_download_model(str(base_model_path)) + ).resolve() + dst = Path( + os.path.expanduser(str(output_dir)) + ).resolve() + + if _rank() == 0: + if dst.exists(): + if overwrite: + shutil.rmtree(dst, ignore_errors=True) + else: + raise FileExistsError( + f"Refusing to overwrite existing " + f"directory: {dst}. " + "Pass --overwrite to replace it." + ) + + def _copy_or_link(src: str, dest: str) -> None: + try: + os.link(src, dest) + except OSError: + shutil.copy2(src, dest) + + logger.info( + "Creating pretrained export dir at %s " + "(base=%s)", dst, local_base, + ) + shutil.copytree( + local_base, dst, symlinks=True, + copy_function=_copy_or_link, + ) + + _barrier() + + modules: dict[str, torch.nn.Module] = {} + if model.transformer is not None: + modules["transformer"] = model.transformer + + if module_names is None: + module_names = sorted(modules.keys()) + + for module_name in module_names: + if module_name not in modules: + raise KeyError( + f"Role {role!r} does not have module " + f"{module_name!r}. " + f"Available: {sorted(modules.keys())}" + ) + + module_dir = dst / module_name + if not module_dir.is_dir(): + raise FileNotFoundError( + f"Export directory missing component " + f"dir {module_name!r}: {module_dir}" + ) + + options = StateDictOptions( + full_state_dict=True, cpu_offload=True, + ) + state_dict = get_model_state_dict( + modules[module_name], options=options, + ) + + if _rank() == 0: + for path in module_dir.glob("*.safetensors"): + path.unlink(missing_ok=True) + + tensor_state: dict[str, torch.Tensor] = {} + for key, value in state_dict.items(): + if not isinstance(value, torch.Tensor): + raise TypeError( + f"Expected tensor in state_dict " + f"for {module_name}.{key}, " + f"got {type(value).__name__}" + ) + tensor_state[key] = value.detach().cpu() + + from safetensors.torch import save_file + + out_path = module_dir / "model.safetensors" + logger.info( + "Saving %s weights to %s (%s tensors)", + module_name, out_path, + len(tensor_state), + ) + save_file(tensor_state, str(out_path)) + + _barrier() + + return str(dst) + + def convert( *, checkpoint_dir: str, @@ -72,7 +202,6 @@ def convert( from fastvideo.train.utils.checkpoint import ( CheckpointManager, _resolve_resume_checkpoint, - save_role_pretrained, ) from fastvideo.train.utils.config import ( RunConfig, @@ -148,7 +277,7 @@ def convert( output_dir, base_model_path, ) - result = save_role_pretrained( + result = _save_role_pretrained( role=role, base_model_path=base_model_path, output_dir=output_dir, diff --git a/fastvideo/train/utils/checkpoint.py b/fastvideo/train/utils/checkpoint.py index 663d618d5..62166dde7 100644 --- a/fastvideo/train/utils/checkpoint.py +++ b/fastvideo/train/utils/checkpoint.py @@ -13,12 +13,6 @@ import torch import torch.distributed as dist import torch.distributed.checkpoint as dcp -from torch.distributed.checkpoint.state_dict import ( - StateDictOptions, - get_model_state_dict, -) - -from fastvideo.train.models.base import ModelBase from fastvideo.logger import init_logger logger = init_logger(__name__) @@ -107,95 +101,6 @@ def _resolve_resume_checkpoint(resume_from_checkpoint: str, *, output_dir: str) f"containing such checkpoints. Got: {path} (output_dir={out}).") -def save_role_pretrained( - *, - role: str, - base_model_path: str, - output_dir: str, - module_names: list[str] | None = None, - overwrite: bool = False, - model: ModelBase, -) -> str: - """Export a role's modules into a diffusers-style model directory. - - This is intended to produce a ``model_path`` that can be loaded by - ``PipelineComponentLoader`` (i.e., has ``model_index.json``, - ``transformer/``, ``vae/``, and other pipeline components copied - from ``base_model_path``). - """ - - # Resolve HF IDs to local directories (same behavior as module loader). - from fastvideo.utils import maybe_download_model - - local_base = Path(maybe_download_model(str(base_model_path))).resolve() - dst = Path(os.path.expanduser(str(output_dir))).resolve() - - if _rank() == 0: - if dst.exists(): - if overwrite: - shutil.rmtree(dst, ignore_errors=True) - else: - raise FileExistsError(f"Refusing to overwrite existing directory: {dst}. " - "Pass overwrite=True to replace it.") - - def _copy_or_link(src: str, dest: str) -> None: - try: - os.link(src, dest) - except OSError: - shutil.copy2(src, dest) - - logger.info("Creating pretrained export dir at %s (base=%s)", dst, local_base) - shutil.copytree(local_base, dst, symlinks=True, copy_function=_copy_or_link) - - _barrier() - - modules: dict[str, torch.nn.Module] = {} - if model.transformer is not None: - modules["transformer"] = model.transformer - - if module_names is None: - module_names = sorted(modules.keys()) - - for module_name in module_names: - if module_name not in modules: - raise KeyError(f"Role {role!r} does not have module {module_name!r}. " - f"Available: {sorted(modules.keys())}") - - module_dir = dst / module_name - if not module_dir.is_dir(): - raise FileNotFoundError(f"Export directory missing component dir {module_name!r}: {module_dir}") - - options = StateDictOptions(full_state_dict=True, cpu_offload=True) - state_dict = get_model_state_dict(modules[module_name], options=options) - - if _rank() == 0: - # Remove existing *.safetensors to avoid loading duplicate weights. - for path in module_dir.glob("*.safetensors"): - path.unlink(missing_ok=True) - - tensor_state: dict[str, torch.Tensor] = {} - for key, value in state_dict.items(): - if not isinstance(value, torch.Tensor): - raise TypeError(f"Expected tensor in state_dict for {module_name}.{key}, " - f"got {type(value).__name__}") - tensor_state[key] = value.detach().cpu() - - from safetensors.torch import save_file - - out_path = module_dir / "model.safetensors" - logger.info( - "Saving %s weights to %s (%s tensors)", - module_name, - out_path, - len(tensor_state), - ) - save_file(tensor_state, str(out_path)) - - _barrier() - - return str(dst) - - class _RoleModuleContainer(torch.nn.Module): """Ephemeral container to expose multiple role modules as a single ``nn.Module``. From 8d83d47b3fd4a97d71cac57fb89b9800356ea24e Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 03:10:19 +0000 Subject: [PATCH 202/214] remove doc --- fastvideo/train/doc/README.md | 53 ---- fastvideo/train/doc/RFC-v1-en.md | 82 ----- fastvideo/train/doc/RFC-v1.md | 293 ------------------ fastvideo/train/doc/__init__.md | 13 - fastvideo/train/doc/dispatch.md | 26 -- fastvideo/train/doc/methods/__init__.md | 17 - fastvideo/train/doc/methods/base.md | 26 -- .../doc/methods/consistency_model/__init__.md | 8 - .../methods/distribution_matching/__init__.md | 13 - .../doc/methods/distribution_matching/dmd2.md | 62 ---- .../train/doc/methods/fine_tuning/__init__.md | 19 -- .../train/doc/methods/fine_tuning/dfsft.md | 33 -- .../train/doc/methods/fine_tuning/finetune.md | 45 --- .../knowledge_distillation/__init__.md | 10 - fastvideo/train/doc/models/__init__.md | 15 - fastvideo/train/doc/models/base.md | 27 -- fastvideo/train/doc/models/wan.md | 42 --- fastvideo/train/doc/models/wangame.md | 35 --- fastvideo/train/doc/roles.md | 21 -- fastvideo/train/doc/trainer.md | 30 -- fastvideo/train/doc/utils/__init__.md | 12 - fastvideo/train/doc/utils/checkpoint.md | 28 -- fastvideo/train/doc/utils/config.md | 76 ----- fastvideo/train/doc/utils/dataloader.md | 18 -- fastvideo/train/doc/utils/module_state.md | 13 - fastvideo/train/doc/utils/moduleloader.md | 19 -- fastvideo/train/doc/utils/tracking.md | 14 - fastvideo/train/doc/validators/__init__.md | 9 - fastvideo/train/doc/validators/base.md | 17 - fastvideo/train/doc/validators/wan.md | 29 -- fastvideo/train/doc/validators/wangame.md | 16 - 31 files changed, 1121 deletions(-) delete mode 100644 fastvideo/train/doc/README.md delete mode 100644 fastvideo/train/doc/RFC-v1-en.md delete mode 100644 fastvideo/train/doc/RFC-v1.md delete mode 100644 fastvideo/train/doc/__init__.md delete mode 100644 fastvideo/train/doc/dispatch.md delete mode 100644 fastvideo/train/doc/methods/__init__.md delete mode 100644 fastvideo/train/doc/methods/base.md delete mode 100644 fastvideo/train/doc/methods/consistency_model/__init__.md delete mode 100644 fastvideo/train/doc/methods/distribution_matching/__init__.md delete mode 100644 fastvideo/train/doc/methods/distribution_matching/dmd2.md delete mode 100644 fastvideo/train/doc/methods/fine_tuning/__init__.md delete mode 100644 fastvideo/train/doc/methods/fine_tuning/dfsft.md delete mode 100644 fastvideo/train/doc/methods/fine_tuning/finetune.md delete mode 100644 fastvideo/train/doc/methods/knowledge_distillation/__init__.md delete mode 100644 fastvideo/train/doc/models/__init__.md delete mode 100644 fastvideo/train/doc/models/base.md delete mode 100644 fastvideo/train/doc/models/wan.md delete mode 100644 fastvideo/train/doc/models/wangame.md delete mode 100644 fastvideo/train/doc/roles.md delete mode 100644 fastvideo/train/doc/trainer.md delete mode 100644 fastvideo/train/doc/utils/__init__.md delete mode 100644 fastvideo/train/doc/utils/checkpoint.md delete mode 100644 fastvideo/train/doc/utils/config.md delete mode 100644 fastvideo/train/doc/utils/dataloader.md delete mode 100644 fastvideo/train/doc/utils/module_state.md delete mode 100644 fastvideo/train/doc/utils/moduleloader.md delete mode 100644 fastvideo/train/doc/utils/tracking.md delete mode 100644 fastvideo/train/doc/validators/__init__.md delete mode 100644 fastvideo/train/doc/validators/base.md delete mode 100644 fastvideo/train/doc/validators/wan.md delete mode 100644 fastvideo/train/doc/validators/wangame.md diff --git a/fastvideo/train/doc/README.md b/fastvideo/train/doc/README.md deleted file mode 100644 index a7f857147..000000000 --- a/fastvideo/train/doc/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Distillation docs (file-by-file) - -本目录用于帮助 reviewer/贡献者快速理解 `fastvideo/distillation/` 的 Phase 2/2.9 架构。 - -设计原则（对应 Phase 2.9）： -- **Trainer** 只做 infra（loop/accum/日志/ckpt/validate 调用），不包含算法策略。 -- **Method** 只做算法（loss + update policy + 需要哪些 roles）。 -- **Model plugin** 负责装配 + primitives（build-time：加载 modules、构建 bundle/dataloader/validator； - step-time：实现 `ModelBase` 的 primitives，例如 `prepare_batch/predict_*/backward/...`）。 - API 以 operation 为中心，不以 role 为中心（避免 role 爆炸）。 - -快速入口（从运行到训练）： -`fastvideo/training/distillation.py` → `utils.config.load_distill_run_config()` → -`dispatch.build_from_config()` → -`ModelBase(model) + DistillMethod` → `DistillTrainer.run()` - ---- - -## Index - -### Core -- `__init__.md` -- `utils/__init__.md` -- `utils/config.md` -- `utils/dataloader.md` -- `utils/moduleloader.md` -- `utils/module_state.md` -- `utils/tracking.md` -- `utils/checkpoint.md` -- `dispatch.md` -- `roles.md` -- `trainer.md` - -### models/ -- `models/__init__.md` -- `models/base.md` -- `models/wan.md` -- `models/wangame.md` - -### methods/ -- `methods/__init__.md` -- `methods/base.md` -- `methods/distribution_matching/__init__.md` -- `methods/distribution_matching/dmd2.md` -- `methods/consistency_model/__init__.md` -- `methods/knowledge_distillation/__init__.md` -- `methods/fine_tuning/__init__.md` -- `methods/fine_tuning/finetune.md` - -### validators/ -- `validators/__init__.md` -- `validators/base.md` -- `validators/wan.md` diff --git a/fastvideo/train/doc/RFC-v1-en.md b/fastvideo/train/doc/RFC-v1-en.md deleted file mode 100644 index 4bb892152..000000000 --- a/fastvideo/train/doc/RFC-v1-en.md +++ /dev/null @@ -1,82 +0,0 @@ -## 1) File Structure - -```text -fastvideo/distillation/ - trainer.py # Builds the training loop; calls method-provided train_one_step - dispatch.py # Auto-dispatch via @register_method/@register_model; builds (training_args, method, dataloader, start_step) from config - roles.py # Wraps model resources in RoleHandle to tag roles (teacher/student/critic/...) - models/ - components.py # intermediate variable during dispatch-time model construction; records all components - wan.py # loads Wan. model-loading logic differs across families - ... - adapters/ - base.py # turns loaded components into runtime primitives: predict_x0/add_noise/backward/... - wan.py # wan-specific adapter. - ... - methods/ - base.py # DistillMethod base; methods must provide e.g. train-one-step for trainer to call - distribution_matching/ - dmd2.py # DMD2 distillation (student/teacher/critic) - fine_tuning/ - finetune.py # SFT finetuning (student only) - knowledge_distillation/ - consistency_model/ - validators/ - base.py # inference differs by model; validators are model-specific. - wan.py # validator backed by WanPipeline. - utils/ - config.py # yaml parser - dataloader.py - moduleloader.py - module_state.py # apply_trainable(...): standardizes requires_grad + train/eval - tracking.py # wandb tracker (owned by trainer) - checkpoint.py # save/resume - -``` - -``` yaml -recipe: - family: wan - method: dmd2 - -roles: - student: - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - critic: - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - -training: - seed: 0 - output_dir: /path/to/out - data_path: /path/to/parquet - max_train_steps: 4000 - train_batch_size: 1 - dataloader_num_workers: 4 - num_gpus: 8 - - log_validation: true - validation_steps: 50 - validation_dataset_file: /path/to/validation.json - validation_sampling_steps: "8" # legacy-style string；method 可选择覆写 - validation_guidance_scale: "5.0" # legacy-style string；method 可选择覆写 - - trackers: ["wandb"] - tracker_project_name: my_project - wandb_run_name: my_run - -pipeline_config: - flow_shift: 3 - sampler_kind: ode - -method_config: - rollout_mode: simulate - dmd_denoising_steps: [999, 750, 500, 250, 0] - generator_update_interval: 1 - real_score_guidance_scale: 1.0 - attn_kind: dense -``` diff --git a/fastvideo/train/doc/RFC-v1.md b/fastvideo/train/doc/RFC-v1.md deleted file mode 100644 index d341c9696..000000000 --- a/fastvideo/train/doc/RFC-v1.md +++ /dev/null @@ -1,293 +0,0 @@ - -## 1) 文件结构（带注释） - -```text -fastvideo/distillation/ - trainer.py # 构建training loop 调用method提供的train_one_step接口 - dispatch.py # 根据@register_method和@register_model自动识别类型，根据config构建 (training_args, method, dataloader, start_step) - roles.py # RoleHandle模型外面包一层role的字段，用于区分teacher/student/critic。 - - models/ - components.py # ModelComponent dispatch构建模型过程中的中间查无 记录了模型的各个组成部分 - wan.py # 加载Wan模型。不同模型的加载逻辑有所不同（例如ltx就是double stream）。 - ... - - adapters/ - base.py # Adapter本质是一个把已经load好的模型转变为运行时可用api的框架 把Model转变为predict_x0/add_noise/backward/... - wan.py # 针对Wan的Adapter。 - ... - - methods/ - base.py # DistillMethod基类，需要Method提供必要的例如train one step的接口 - distribution_matching/ - dmd2.py # DMD2Method：DMD2 distillation（student/teacher/critic） - fine_tuning/ - finetune.py # FineTuneMethod：SFT/finetuning（只有 student） - knowledge_distillation/ - consistency_model/ - - validators/ - base.py # 不同模型推理方式不同，需要根据模型类型写不同的validator。 - wan.py # 调用WanPipeline的validator - - utils/ - config.py # yaml parser - dataloader.py - moduleloader.py - module_state.py # apply_trainable(...)：统一 requires_grad + train/eval - tracking.py # wandb tracker，由trainer管理 - checkpoint.py# save/resume -``` - -统一入口（YAML-only）： -- `fastvideo/training/distillation.py` - ---- - -## 2) 关键接口（contracts，注释版） - -### 2.1 `roles.py`：RoleManager / RoleHandle - -```py -# RoleHandle：一个 role 的“资源包”（method 只通过 RoleHandle 操作 modules/optimizers） -RoleHandle: - modules: dict[str, nn.Module] # e.g. {"transformer": ..., "transformer_2": ...} - optimizers: dict[str, Optimizer] # method 创建并写回 - lr_schedulers: dict[str, Any] # method 创建并写回 - trainable: bool # 来自 roles..trainable（只影响 module 状态） - -# RoleManager：roles 的容器（role key 不限于 student/teacher/critic） -RoleManager: - roles: dict[str, RoleHandle] - require_roles([...]) # method 用它声明依赖（早失败、错误信息清晰） -``` - -### 2.2 `dispatch.py`：registry + build_from_config() - -```py -# 目标：新增一个 model plugin 或 method 的成本是 O(1)，而不是写 N×M 组合函数 - -@register_model("wan") -def build_wan_components(cfg) -> ModelComponents: ... - -@register_method("dmd2") -class DMD2Method(DistillMethod): ... - -build_from_config(cfg): - components = model_builder(cfg) # -> ModelComponents - method = method_cls.build( # -> DistillMethod instance - cfg=cfg, - bundle=components.bundle, - adapter=components.adapter, - validator=components.validator, - ) - return (training_args, method, dataloader, start_step) -``` - -### 2.3 `trainer.py`：DistillTrainer（infra only） - -```py -# Trainer 只看见 method（算法对象），不看见 roles 的语义，也不看见模型细节。 - -DistillTrainer(training_args, config=raw_yaml): - tracker = build_tracker(training_args, config=raw_yaml) - -run(method, dataloader, max_steps, start_step, checkpoint_manager?): - method.set_tracker(tracker) # 给 method/validator 注入 tracker（artifact logging） - method.on_train_start()? # 可选：让 method/adapter 做一次性初始化 - for step in steps: - loss_map, outputs, metrics = method.single_train_step(...) - method.backward(loss_map, outputs)? # 可选覆写（forward_context / ctx-aware backward） - method.optimizers_schedulers_step(step)? - method.optimizers_zero_grad(step)? - method.log_validation(step)? # 可选：method-managed validation - checkpoint_manager.maybe_save(step)? - tracker.log(...) -``` - -### 2.4 `adapters/`：Adapter 应提供哪些运行时 primitive？ - -> 说明：`DistillAdapter` 基类只约束最小接口；具体 method 通过自己的 `Protocol` -> 显式声明需要哪些 primitive（duck typing）。这样避免把 DMD2 的需求硬塞进所有 adapter。 - -当前方法族常用的 primitives（operation-centric）： - -```py -# batch/conditioning -prepare_batch(raw_batch, current_vsa_sparsity=..., latents_source={"data"|"zeros"}) -> TrainingBatch -on_train_start()? # seed/RNG/negative conditioning/cache（可选） -get_rng_generators()? # ckpt 时保存 RNG（可选） - -# timestep/noise -num_train_timesteps -> int -shift_and_clamp_timestep(t) -> t -add_noise(clean_latents, noise, t) -> noisy_latents - -# forward primitives（不区分 role，只吃 handle） -predict_x0(handle, noisy_latents, t, batch, conditional, attn_kind=...) -> x0 -predict_noise(handle, noisy_latents, t, batch, conditional, attn_kind=...) -> noise_like - -# backward（为了适配 forward_context / activation ckpt 等约束） -backward(loss, ctx, grad_accum_rounds=...) -> None -``` - -### 2.5 `methods/base.py`：一个 Method 应有哪些接口？ - -```py -class DistillMethod(nn.Module): - # dispatch 用 build(...) 统一装配实例（避免每个 method 写 build_*_method boilerplate） - @classmethod - def build(cfg, bundle, adapter, validator) -> DistillMethod - - # 必需：训练一步 - def single_train_step(batch, iteration, current_vsa_sparsity=...) -> (loss_map, outputs, metrics) - # - loss_map 必须包含 total_loss（或 method 覆写 backward 自己处理） - # - metrics 用于额外标量日志（trainer 会统一 log） - - # 必需：update policy（算法语义） - def get_optimizers(iteration) -> Sequence[Optimizer] - def get_lr_schedulers(iteration) -> Sequence[Any] - - # 可选：更复杂的 backward / 多 ctx / forward_context 约束 - def backward(loss_map, outputs, grad_accum_rounds=...) -> None - - # 可选：validation policy（method-managed） - def log_validation(step) -> None -``` - -### 2.6 `validators/`：Validator（family-specific，method-controlled） - -```py -ValidationRequest: - sample_handle: RoleHandle # method 指定要采样哪个 role（通常 student） - sampler_kind: {"ode"|"sde"}? # method 指定采样 loop 类型 - sampling_steps: list[int]? # 展示用：要跑多少步（可能有多个） - sampling_timesteps: list[int]? # few-step schedule（与 sampling_steps 一致时更可控） - guidance_scale: float? - output_dir: str? - -DistillValidator: - log_validation(step, request=ValidationRequest?) -> None -``` - ---- - -## 3) 当前接受的 YAML config 格式（schema v2） - -> 入口：`fastvideo/training/distillation.py --config /abs/path/to/run.yaml` -> -> 特性： -> - **YAML-only**：不与 legacy CLI configs merge -> - `training:` 大部分字段直接映射 `TrainingArgs.from_kwargs(...)` -> - `method_config:` 保持 dict（研究迭代快；由 method 自己解释/校验） - -```yaml -# ---- 必需：选择 model plugin + method ---- -recipe: - family: wan # dispatch key：models/wan.py - method: dmd2 # dispatch key：methods/**/dmd2.py - -# ---- 必需：定义 roles（role key 是任意字符串）---- -roles: - student: - # family 可省略：默认继承 recipe.family - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - critic: - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - -# ---- 必需：训练参数（大部分字段直接走 TrainingArgs）---- -training: - seed: 0 - output_dir: /path/to/out - data_path: /path/to/parquet - max_train_steps: 4000 - train_batch_size: 1 - dataloader_num_workers: 4 - num_gpus: 8 - - # validation（由 method 调用 validator；validator 具体做采样与记录） - log_validation: true - validation_steps: 50 - validation_dataset_file: /path/to/validation.json - validation_sampling_steps: "8" # legacy-style string；method 可选择覆写 - validation_guidance_scale: "5.0" # legacy-style string；method 可选择覆写 - - # tracker（由 trainer 构建并持有；不建议在 model plugin 里构建） - trackers: ["wandb"] - tracker_project_name: my_project - wandb_run_name: my_run - -# ---- 可选：pipeline config（传入 TrainingArgs.pipeline_config）---- -# 只能提供一个：pipeline_config 或 pipeline_config_path -pipeline_config: - flow_shift: 3 - sampler_kind: ode -# pipeline_config_path: /abs/path/to/pipeline_config.yaml - -# ---- 可选：method 私有参数（dict，method 自己解析/校验）---- -method_config: - # DMD2 示例 - rollout_mode: simulate - dmd_denoising_steps: [999, 750, 500, 250, 0] - generator_update_interval: 1 - real_score_guidance_scale: 1.0 - attn_kind: dense -``` - ---- - -## 4) 为什么这样设计（取舍逻辑） - -### 4.1 “很多东西都能写进 config”——为什么还要 model plugin？ - -可以把“模块名/类名/参数”写进 YAML，但最终仍需要一段代码来： -- 解释这些配置（动态 import / 默认值 / 校验 / 失败时给出清晰错误信息）； -- 做 build-time 的工程装配（模块是否存在、可选模块、shared components 复用、role 组合约束等）； -- 把 FastVideo 现实代码的差异（schema、parallel/offload、module packing 约定）收敛起来。 - -因此我们把这层解释器命名为 **model plugin（`models/.py`）**： -- config 负责“选择 + 超参” -- model plugin 负责“把选择落地成可运行组件（ModelComponents）” - -### 4.2 为什么 adapter 基类很薄（`DistillAdapter` 只有 prepare_batch）？ - -如果把 `predict_x0/add_noise/backward/...` 全塞进一个巨大的 adapter 基类： -- 你会把某个算法（例如 DMD2）的需求固化成“全体 adapter 必须实现”的硬约束； -- 未来新增方法会被迫实现一堆不需要的接口（耦合上升、可维护性变差）。 - -当前策略： -- adapter 的稳定边界保持最小 -- 每个 method 用 `Protocol` 显式声明自己需要哪些 primitives（代码可读、依赖清晰） - -### 4.3 为什么 optimizer/scheduler 由 method 创建？ - -optimizer cadence / 多优化器更新比例 / critic 的超参等都属于算法语义。 -如果放在 model plugin，会导致： -- model plugin 需要理解 DMD2/finetune/... 的算法细节（污染 build-time 层） -- 同一个 family 随着方法增多出现“把算法 if/else 塞进 models/”的风险 - -### 4.4 为什么 tracker 由 trainer 构建并持有？ - -tracker 是 infra 资源（日志/文件/媒体记录），生命周期属于训练 loop： -- trainer 负责创建 tracker，并统一 `tracker.log(...)` -- method 只产出 metrics；若 method-managed validation 需要 log 视频，则通过 `method.set_tracker(...)` - 把 tracker 注入到 validator（而不是让 model plugin 构建并传递 tracker） - -### 4.5 为什么 `method_config` 是 dict？ - -研究/工程迭代中，方法参数变化频繁；强类型 schema 会带来迁移成本。 -我们把稳定边界结构化（`recipe/roles/training`），把快速变化的部分留给 dict： -- method 自己解析/校验（并给出明确错误提示） -- 解析 helper（int/float/betas）放在 `utils/config.py` 复用，减少重复代码 - -### 4.6 为什么需要 `dispatch.py` 的 registry？ - -目标是避免“模型×方法”的组合爆炸： -- 新增一个 family → 加一个 model plugin 并注册 -- 新增一个 method → 加一个 method 文件并注册 -- 不需要写 25 个 build 函数或 if/else 分支 diff --git a/fastvideo/train/doc/__init__.md b/fastvideo/train/doc/__init__.md deleted file mode 100644 index 506c128a8..000000000 --- a/fastvideo/train/doc/__init__.md +++ /dev/null @@ -1,13 +0,0 @@ -# `fastvideo/distillation/__init__.py` - -**目的** -- 提供 distillation 子系统的最小“公共入口”，避免上层到处 import 内部实现细节。 - -**当前导出** -- `DistillTrainer`：训练 loop（infra only） -- `RoleManager` / `RoleHandle`：multi-role 模型与训练状态容器 - -**设计意图** -- 让上层（例如 `fastvideo/training/distillation.py`）只依赖稳定 API： - - `DistillTrainer.run(method, dataloader, ...)` - - `bundle.role("student")` 等 role 访问模式 diff --git a/fastvideo/train/doc/dispatch.md b/fastvideo/train/doc/dispatch.md deleted file mode 100644 index dbf7210ae..000000000 --- a/fastvideo/train/doc/dispatch.md +++ /dev/null @@ -1,26 +0,0 @@ -# `fastvideo/distillation/dispatch.py` - -**目的** -- 把 “可扩展注册（models/methods）” 与 “从 YAML 装配可运行 runtime” 收敛到一个入口文件： - - 新增 model 的成本 ≈ N - - 新增 method 的成本 ≈ M - - 不需要写 N×M 的 if/else 组合逻辑 - -**关键概念** -- `ModelBuilder(cfg) -> ModelBase`（一个可运行的 model plugin 实例） -- `DistillMethod` class（算法实现） - - `@register_method("...")` 直接注册到 class - - class 需要实现 `DistillMethod.build(cfg, bundle, model, validator)` - -**关键 API** -- `register_model(name)` / `register_method(name)`：装饰器注册 -- `get_model(name)` / `get_method(name)`：查询（会触发内置注册） -- `available_models()` / `available_methods()` -- `build_from_config(cfg) -> (training_args, method, dataloader, start_step)` - - 选择 model plugin：`get_model(cfg.recipe.family)` - - 选择 method：`get_method(cfg.recipe.method)` - - 返回值是一个 tuple，避免额外引入 `Runtime` 概念。 - -**边界** -- ✅ 这里只做“装配 + dispatch”，不包含训练 loop / loss / rollout / optimizer policy。 -- ✅ method 层保持算法高内聚；model plugin 层保持集成高内聚。 diff --git a/fastvideo/train/doc/methods/__init__.md b/fastvideo/train/doc/methods/__init__.md deleted file mode 100644 index 38764da0c..000000000 --- a/fastvideo/train/doc/methods/__init__.md +++ /dev/null @@ -1,17 +0,0 @@ -# `fastvideo/distillation/methods/__init__.py` - -**目的** -- 提供 method 层（算法层）的统一入口与可发现性。 - -**当前导出** -- `DistillMethod`：算法基类（抽象） -- `DMD2Method`：distribution matching 目录下的一个具体方法实现 -- `FineTuneMethod`：fine tuning 目录下的一个具体方法实现（only student） - -**设计意图** -- method 层应当是 **模型无关** 的（不 import 具体 pipeline/模型实现）； - 任何 model plugin 细节都通过 `ModelBase` primitives（protocol/抽象接口）注入。 - -**实现细节** -- 该模块对 `DMD2Method` 使用 lazy import（`__getattr__`），避免 dispatch 在 - import 时触发循环依赖（circular import）；dispatch 侧会在第一次查询时触发内置注册。 diff --git a/fastvideo/train/doc/methods/base.md b/fastvideo/train/doc/methods/base.md deleted file mode 100644 index 0fc795246..000000000 --- a/fastvideo/train/doc/methods/base.md +++ /dev/null @@ -1,26 +0,0 @@ -# `fastvideo/distillation/methods/base.py` - -**定位** -- `DistillMethod` 是算法层抽象： - - 负责实现 `single_train_step()`（loss 构造） - - 负责定义 update policy（哪些 optimizer/scheduler 在何时 step） - - 不负责训练 loop（由 `DistillTrainer` 承担） - -**关键点** -- `DistillMethod` 持有 `bundle: RoleManager`，并把所有 role 的 modules 放进 - `self.role_modules: ModuleDict`，便于 DDP/FSDP/ckpt 系统统一发现参数。 - -**需要子类实现的抽象方法** -- `build(cfg, bundle, model, validator)`（classmethod） -- `single_train_step(batch, iteration, current_vsa_sparsity=...)` - - 返回：`(loss_map, outputs, metrics)` - - `loss_map: dict[str, Tensor]`：必须包含 `total_loss`（用于 backward） - - `metrics: dict[str, scalar]`：额外要 log 的标量（float/int/0-dim Tensor） -- `get_optimizers(iteration)` -- `get_lr_schedulers(iteration)` - -**默认实现** -- `set_tracker(tracker)`：由 trainer 注入 tracker，并尽力转发到 `self.validator` -- `backward()`：对 `loss_map["total_loss"]` 做 backward（子类可覆写以处理多 ctx） -- `optimizers_schedulers_step()`：按 `get_optimizers/get_lr_schedulers` 的结果 step -- `optimizers_zero_grad()`：对当前 iteration 的 optimizers 清梯度 diff --git a/fastvideo/train/doc/methods/consistency_model/__init__.md b/fastvideo/train/doc/methods/consistency_model/__init__.md deleted file mode 100644 index fe5838270..000000000 --- a/fastvideo/train/doc/methods/consistency_model/__init__.md +++ /dev/null @@ -1,8 +0,0 @@ -# `fastvideo/distillation/methods/consistency_model/__init__.py` - -**状态** -- 当前是占位目录（`__all__ = []`），用于未来加入 Consistency Model（CM）相关方法。 - -**期望的演进方向** -- 通过 `@register_method("cm")`（示例）注册具体实现。 -- method 只包含算法与 update policy；model plugin（`ModelBase`）提供运行时 primitives。 diff --git a/fastvideo/train/doc/methods/distribution_matching/__init__.md b/fastvideo/train/doc/methods/distribution_matching/__init__.md deleted file mode 100644 index 8109b2587..000000000 --- a/fastvideo/train/doc/methods/distribution_matching/__init__.md +++ /dev/null @@ -1,13 +0,0 @@ -# `fastvideo/distillation/methods/distribution_matching/__init__.py` - -**定位** -- distribution matching 类方法的集合目录。 - -**当前实现** -- `DMD2Method`（见 `dmd2.md`） - -**扩展** -- 新增方法时建议保持： - - 算法逻辑在 method - - model plugin 细节通过 `ModelBase` primitives 注入 - - 注册通过 `@register_method("")` diff --git a/fastvideo/train/doc/methods/distribution_matching/dmd2.md b/fastvideo/train/doc/methods/distribution_matching/dmd2.md deleted file mode 100644 index c2dd32a77..000000000 --- a/fastvideo/train/doc/methods/distribution_matching/dmd2.md +++ /dev/null @@ -1,62 +0,0 @@ -# `fastvideo/distillation/methods/distribution_matching/dmd2.py` - -**定位** -- `DMD2Method`：DMD2 distillation 的算法实现（method layer）。 -- 该文件可以出现 DMD2/critic/fake_score 等算法术语；这些语义不应泄漏到 model plugin/validator。 - -**依赖与边界** -- ✅ 不 import 任何具体模型/管线实现（Wan/SDXL/...）。 -- ✅ 只依赖： - - `RoleManager`/`RoleHandle`（获取 student/teacher/critic） - - model plugin 提供的 primitives（通过 `_DMD2Model` Protocol / `ModelBase`） - -**需要的 roles** -- `student`：可训练（trainable=true） -- `teacher`：冻结（trainable=false） -- `critic`：可训练（trainable=true） - -**算法结构（高层）** -1) `prepare_batch()`：交给 model plugin -2) student 更新（按 `generator_update_interval` 控制频率） - - 先做 student rollout 得到 `generator_pred_x0` - - 再计算 DMD loss（teacher CFG 引导 + critic 输出构造梯度） -3) critic 更新（每 step） - - 使用 student rollout（no-grad）构造 flow matching loss -4) backward - - 由于 Wan 的 forward_context 约束，需要 `model.backward(loss, ctx)` - -**few-step rollout policy（Phase 2.9）** -- rollout 的 step list / simulate 逻辑由 method 管理： - - `method_config.dmd_denoising_steps` → `_get_denoising_step_list()` - - `method_config.rollout_mode={simulate|data_latent}` - - `simulate`：batch 不要求 `vae_latent`（model 用零 latents 构造形状） - - `data_latent`：batch 必须带 `vae_latent` - - 可选 `warp_denoising_step`（通过 model.noise_scheduler.timesteps duck-typing） -- model plugin 只提供单步 primitives： - - `predict_x0()` / `predict_noise()` / `add_noise()` - -**optimizer/scheduler 的归属（Phase 2.9）** -- `DMD2Method` 在初始化时创建并写回： - - student 的 optimizer/scheduler：使用 `training.learning_rate/betas/lr_scheduler` - - critic 的 optimizer/scheduler：优先使用 `training.fake_score_*` 覆盖（否则回退到 student） -- 这样 Wan model plugin 完全不需要理解 DMD2 的 critic 超参，从 build-time 层面解耦。 - -**validation 的归属（Phase 2.9）** -- `DMD2Method` 持有 model-plugin-specific `validator`（build-time 注入），并在 `log_validation()` 中调用。 -- method 通过 `ValidationRequest` 明确传入 sampling steps/guidance 等参数； - 同时通过 `ValidationRequest.sample_handle` 指定要采样的模型（通常是 student）， - validator 负责执行采样与记录，保持 method-agnostic。 - -**配置语义的 TODO（Phase 3）** -- Phase 3.1 已引入 `method_config`，并将 rollout 的关键 knob 迁移到 method 层： - - `rollout_mode` - - `dmd_denoising_steps` - - `generator_update_interval` - - `real_score_guidance_scale` -- Phase 3.2 已完成：sampling loop/timesteps 由 method 在 `ValidationRequest` 中显式指定 - （`sampler_kind` + `sampling_timesteps`），validator 使用 `WanPipeline` 执行采样； - distillation config 不再需要 `pipeline_config.dmd_denoising_steps` 这种重复字段。 - -**注册方式（Phase 3.4）** -- `DMD2Method` 通过 `@register_method("dmd2")` 直接注册到 class。 -- 由 `DMD2Method.build(...)` 负责把 `cfg.method_config` 等注入到实例。 diff --git a/fastvideo/train/doc/methods/fine_tuning/__init__.md b/fastvideo/train/doc/methods/fine_tuning/__init__.md deleted file mode 100644 index 474e8fdbb..000000000 --- a/fastvideo/train/doc/methods/fine_tuning/__init__.md +++ /dev/null @@ -1,19 +0,0 @@ -# `fastvideo/distillation/methods/fine_tuning/` - -**目的** -- 以 “method” 的形式实现 finetuning / SFT：把它看作一种特殊的 distillation recipe（只有 `student` + dataset）。 - -**当前实现** -- `finetune.py`：`FineTuneMethod` - - 只要求 `roles.student` - - loss/policy 在 method 层 - - 复用同一套 trainer/roles/model plugin/validator/checkpoint 基础设施 -- `dfsft.py`：`DiffusionForcingSFTMethod`（`recipe.method: dfsft`） - - 只要求 `roles.student` - - 训练目标仍是 SFT/flow-matching loss，但使用 **chunk-wise inhomogeneous timesteps** - （`t_inhom`）作为 diffusion forcing baseline - -**设计要点** -- model plugin 通过 `ModelBase` 提供 operation-centric primitives（`prepare_batch / predict_* / backward`）， - 不内置 finetune 的 loss 语义（由 method 管理）。 -- model plugin 负责 build-time：加载 student modules、shared components（VAE/scheduler）、dataloader、validator。 diff --git a/fastvideo/train/doc/methods/fine_tuning/dfsft.md b/fastvideo/train/doc/methods/fine_tuning/dfsft.md deleted file mode 100644 index 382ea5cb2..000000000 --- a/fastvideo/train/doc/methods/fine_tuning/dfsft.md +++ /dev/null @@ -1,33 +0,0 @@ -# `fastvideo/distillation/methods/fine_tuning/dfsft.py` - -**定位** -- `@register_method("dfsft")`：Diffusion-Forcing SFT（DFSFT）baseline。 -- 只训练 `roles.student`，不依赖 teacher/critic。 -- 目标：在 SFT/flow-matching loss 的基础上，引入 **chunk-wise inhomogeneous timesteps** - （`t_inhom`）来覆盖“历史上下文不总是干净”的噪声分布（为 causal/streaming 训练做铺垫）。 - -**核心训练逻辑（单步）** -1) `model.prepare_batch(...)` 产出 `TrainingBatch`（包含 `x0` video latents + conditioning）。 -2) 采样 `t_inhom`： - - 先采样每个 chunk 的 timestep index（`method_config.chunk_size` 控制 chunk 划分） - - 再 repeat 到每帧（`[B, T_lat]`） -3) 采样 `noise ~ N(0, I)`，得到 `x_t = model.add_noise(x0, noise, t_inhom_flat)` -4) 学生预测 `pred = model.predict_noise(student, x_t, t_inhom, batch, ...)` -5) loss： - - 默认 flow-matching：`MSE(pred, noise - x0)` - - 若 `training.precondition_outputs=true`：precondition 到 `x0` 再回归 `x0` - -**关键 config** -- `method_config`（DFSFT 专属） - - `chunk_size: int`（默认 3）：chunk-wise timestep 的 block size - - `min_timestep_ratio / max_timestep_ratio: float`：采样 index 范围（映射到 scheduler 的 train steps） - - `attn_kind: dense|vsa`：选择 model 的 dense/VSA attention metadata 路径 - -**约束** -- 如果 student transformer 暴露 `num_frame_per_block`，DFSFT 会要求 - `method_config.chunk_size == transformer.num_frame_per_block`，否则直接报错（避免配错造成语义不一致）。 - -**Validation** -- DFSFT 依赖 `training.validation`（由 method 驱动 `validator.log_validation(...)`）。 -- 当前 validator 仍是 “full-video pipeline” 语义；真正的 streaming/causal rollout - 仍需要在后续阶段实现（避免把 rollout policy 藏进 validator/model plugin）。 diff --git a/fastvideo/train/doc/methods/fine_tuning/finetune.md b/fastvideo/train/doc/methods/fine_tuning/finetune.md deleted file mode 100644 index 57c199fd8..000000000 --- a/fastvideo/train/doc/methods/fine_tuning/finetune.md +++ /dev/null @@ -1,45 +0,0 @@ -# `fastvideo/distillation/methods/fine_tuning/finetune.py` - -## 目的 -- 将 “finetuning / SFT” 以 `DistillMethod` 的方式接入 Phase 2+ 架构： - - 复用 `DistillTrainer`（infra loop / accum / step / ckpt / validate 调用） - - 复用 `RoleManager`（角色容器，finetune 只需要 `student`） - - 复用 model plugin（加载与 primitives） - -finetune 可以被视为一种特殊的 distillation recipe：**只有 student + dataset**。 - -## 角色依赖 -- 必需：`student` -- 不需要：`teacher / critic / reward / ...` - -方法会强制： -- `roles.student.trainable=true` - -## 核心训练逻辑 -`FineTuneMethod.single_train_step()`： -1. `model.prepare_batch(..., latents_source="data")` -2. 用 student 做 `model.predict_noise(student, noisy_latents, timesteps, batch, conditional=True)` -3. 计算 loss（与 legacy `training_pipeline.py` 对齐）： - - 默认（`training.precondition_outputs=false`）： - - target = `noise - x0` - - loss = `mse(pred, target)` - - 若 `training.precondition_outputs=true`： - - 先 precondition 到 `x0`：`pred_x0 = x_t - sigma * pred` - - loss = `mse(pred_x0, x0)` -4. backward 通过 `model.backward(loss, ctx, ...)` 执行（确保 forward-context/activation ckpt 兼容） - -## Optimizer / Scheduler -- 由 method 创建（而非 model plugin）： - - 使用 `training.learning_rate / training.betas / training.lr_scheduler / ...` - - 只为 `student` role 创建 `optimizer + lr_scheduler` - -## Validation -- `FineTuneMethod.log_validation()` 构造 `ValidationRequest(sample_handle=student, ...)` -- 具体 pipeline 与采样 loop 由 validator + `pipeline_config.sampler_kind` 决定（默认 `ode`） - -## 配置示例 -- `examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` - -## 注册方式（Phase 3.4） -- `FineTuneMethod` 通过 `@register_method("finetune")` 直接注册到 class。 -- 由 `FineTuneMethod.build(...)` 负责把 `cfg.method_config` 等注入到实例。 diff --git a/fastvideo/train/doc/methods/knowledge_distillation/__init__.md b/fastvideo/train/doc/methods/knowledge_distillation/__init__.md deleted file mode 100644 index 2c224f718..000000000 --- a/fastvideo/train/doc/methods/knowledge_distillation/__init__.md +++ /dev/null @@ -1,10 +0,0 @@ -# `fastvideo/distillation/methods/knowledge_distillation/__init__.py` - -**状态** -- 当前是占位目录（`__all__ = []`），用于未来加入经典 KD 类方法（logit/feature matching 等）。 - -**期望的扩展方式** -- 新增 KD method 时： - - method 定义需要哪些 roles（student/teacher/aux_teacher/...） - - model plugin 只负责加载这些 roles 的 modules 并构建 `RoleManager`/dataloader/validator（可选） - - optimizer/scheduler 由 method 创建并写回 handle diff --git a/fastvideo/train/doc/models/__init__.md b/fastvideo/train/doc/models/__init__.md deleted file mode 100644 index 6de0a486f..000000000 --- a/fastvideo/train/doc/models/__init__.md +++ /dev/null @@ -1,15 +0,0 @@ -# `fastvideo/distillation/models/__init__.py` - -**目的** -- models 是 “model plugins”： - - 从 YAML config 读取 role spec - - 加载模型模块（transformer/vae/...） - - 构建 `RoleManager` - - 构建 dataloader / validator（可选） - - **同时实现 `ModelBase` 的运行时 primitives** - -**为什么需要 model plugins？** -- 把 “装配/加载/数据/分布式细节” 与 “算法/rollout/loss/update policy” 分离： - - model plugin 专注 build-time 高内聚 - - method 专注算法高内聚 - - entrypoint/dispatch 不需要 N×M 组合逻辑 diff --git a/fastvideo/train/doc/models/base.md b/fastvideo/train/doc/models/base.md deleted file mode 100644 index b6b765729..000000000 --- a/fastvideo/train/doc/models/base.md +++ /dev/null @@ -1,27 +0,0 @@ -# `fastvideo/distillation/models/base.py` - -**定位** -- 定义 `ModelBase` 抽象接口：由 model plugin 提供的 *运行时 primitives*（operation-centric）。 -- Method 层不应该 import/依赖任何具体 pipeline/model，只依赖这些 primitives（duck typing / 抽象基类）。 - -**核心思想** -- `ModelBase` **不关心 roles 语义**（student/teacher/critic/... 由 method 定义）。 -- `ModelBase` 提供 “对某个 role handle 执行某个操作” 的 API： - - `predict_noise(handle, ...)` - - `predict_x0(handle, ...)` - - `add_noise(...)` - - `prepare_batch(...)` -- 这样可以避免 “每个 role 一个函数” 的 role 爆炸。 - -**接口概览（必需）** -- `prepare_batch(raw_batch, current_vsa_sparsity, latents_source) -> TrainingBatch` -- `add_noise(clean_latents, noise, timestep) -> Tensor` -- `predict_noise(handle, noisy_latents, timestep, batch, conditional, cfg_uncond?, attn_kind) -> Tensor` -- `predict_x0(handle, noisy_latents, timestep, batch, conditional, cfg_uncond?, attn_kind) -> Tensor` -- `num_train_timesteps` / `shift_and_clamp_timestep(timestep)` -- `on_train_start()` -- `backward(loss, ctx, grad_accum_rounds)` - -**接口概览（可选）** -- `get_rng_generators() -> dict[str, torch.Generator]` - - Trainer/ckpt manager 用于保存 RNG state，实现 “exact resume”。 diff --git a/fastvideo/train/doc/models/wan.md b/fastvideo/train/doc/models/wan.md deleted file mode 100644 index c6222ebb4..000000000 --- a/fastvideo/train/doc/models/wan.md +++ /dev/null @@ -1,42 +0,0 @@ -# `fastvideo/distillation/models/wan.py` - -**定位** -- `@register_model("wan")` 的 model plugin（实现：`WanModel(cfg)`）： - - 负责把 YAML config 装配成一个可运行的 `WanModel`（同时实现 `ModelBase` primitives） - - 包含 Wan 特有的模块加载、shared components、dataloader schema 等逻辑 - -**产物** -- `WanModel` 实例（关键字段）： - - `training_args`, `bundle`, `dataloader`, `validator`, `start_step` - - 以及 `ModelBase` 的 primitives（`prepare_batch/add_noise/predict_*/backward/...`） - -**主要职责** -1) **加载 shared components** - - `vae`：从 student 的 base `model_path` 加载 - - `noise_scheduler`：`FlowMatchEulerDiscreteScheduler(shift=flow_shift)` -2) **按 roles 加载 transformer 模块** - - 对每个 role：加载 `transformer`（teacher 可选 `transformer_2`） - - 根据 `RoleSpec.trainable` 设置 `requires_grad` + `train()/eval()` - - 可选开启 activation checkpoint（仅对 trainable role） -3) **构建 bundle / dataloader** - - `bundle = RoleManager(roles=role_handles)` - - dataloader：parquet + `pyarrow_schema_t2v` - - runtime primitives 由 `WanModel` 直接实现（不再额外分一层 `*Adapter` 类/文件） -4) **tracker / validator（可选）** - - validator：`WanValidator`（当 `training.validation.enabled=true`，或 `training.validation` 非空） - - model plugin 只负责构建并返回 `validator` - - tracker 由 trainer 构建并注入到 method/validator（`method.set_tracker(...)`） - - validator 本身不应 hardcode `bundle.role("student")` 等角色语义； - method 通过 `ValidationRequest.sample_handle` 指定要采样的模型 - - 是否调用、用什么采样配置由 method 决定（method-managed validation） - -**Phase 2.9 的关键变化** -- ✅ model plugin 不再创建 optimizers/schedulers。 - - 这类 update policy（哪些 role 训练、各自超参）属于 method/算法语义。 - - 当前由 `DMD2Method` 在初始化时创建并写回 `RoleHandle.optimizers/lr_schedulers`。 - - ✅ model plugin 不再构建/持有 tracker。 - - tracker 属于 infra：由 `DistillTrainer` 构建并持有。 - -**注意 / TODO** -- YAML 中目前仍使用 `training.fake_score_*` 这类字段作为 DMD2 的 critic 超参来源； - Phase 3 计划把它们迁移到 `method_config`，进一步减少 “training_args 承载算法语义”。 diff --git a/fastvideo/train/doc/models/wangame.md b/fastvideo/train/doc/models/wangame.md deleted file mode 100644 index fccf60380..000000000 --- a/fastvideo/train/doc/models/wangame.md +++ /dev/null @@ -1,35 +0,0 @@ -# `fastvideo/distillation/models/wangame.py` - -**定位** -- `@register_model("wangame")` 的 model plugin（实现：`WanGameModel(cfg)`）： - - 负责把 YAML config 装配成一个可运行的 `WanGameModel`（同时实现 `ModelBase` primitives） - - 包含 WanGame 特有的模块加载、dataset schema、validator 选择等逻辑 - -**产物** -- `WanGameModel` 实例（关键字段）： - - `training_args`, `bundle`, `dataloader`, `validator`, `start_step` - - 以及 `ModelBase` 的 primitives（`prepare_batch/add_noise/predict_*/backward/...`） - -**主要职责** -1) **加载 shared components** - - `vae`：从 `training.model_path`（默认 student.path）加载 - - `noise_scheduler`：`FlowMatchEulerDiscreteScheduler(shift=flow_shift)` -2) **按 roles 加载 transformer 模块** - - 对每个 role：加载 `transformer`（可选 `transformer_2`） - - role-level transformer 类型由 `roles..family` 决定（而不是 `variant`）： - - `roles..family: wangame` → `WanGameActionTransformer3DModel` - - `roles..family: wangame_causal` → `CausalWanGameActionTransformer3DModel` - - 根据 `RoleSpec.trainable` 设置 `requires_grad` - - 可选开启 activation checkpoint（仅对 trainable role） -3) **构建 bundle / dataloader / validator** - - `bundle = RoleManager(roles=role_handles)` - - dataloader：parquet + `pyarrow_schema_wangame` - - validator（可选）：`WanGameValidator`（当 `training.validation.enabled=true`，或 `training.validation` 非空） - - runtime primitives 由 `WanGameModel` 直接实现（不再额外分一层 `*Adapter` 类/文件） - -**关于 roles.family** -- `recipe.family: wangame`（bidirectional）： - - 只支持 `roles..family="wangame"`，否则直接报错。 -- `recipe.family: wangame_causal`（causal-capable）： - - 支持 `roles..family in {"wangame","wangame_causal"}`，用于 self-forcing - 等场景下的 “student causal + teacher/critic bidirectional” 组合。 diff --git a/fastvideo/train/doc/roles.md b/fastvideo/train/doc/roles.md deleted file mode 100644 index c9e377d9d..000000000 --- a/fastvideo/train/doc/roles.md +++ /dev/null @@ -1,21 +0,0 @@ -# `fastvideo/distillation/roles.py` - -**目的** -- 用统一的数据结构表达 “多角色（roles）参与的训练/蒸馏”： - - roles 是字符串 key（`"student"`, `"teacher"`, `"critic"`, `"reward"`, ...） - - 每个 role 对应一个 `RoleHandle` - -**关键类型** -- `RoleHandle` - - `modules: dict[str, nn.Module]`：该 role 持有的模块（例如 `transformer`） - - `optimizers: dict[str, Optimizer]` / `lr_schedulers: dict[str, Any]` - - `trainable: bool` - - `require_module(name)`：强制获取模块（缺失则报错） -- `RoleManager` - - `roles: dict[str, RoleHandle]` - - `require_roles([...])`：method 在构造时校验依赖的 role 是否齐全 - - `role(name)`：获取 handle - -**Phase 2.9 约定** -- model plugin 负责 **load modules + 设置 trainable** 并创建 `RoleManager` -- method 负责 **(按算法) 创建 optimizers/schedulers** 并写回对应的 `RoleHandle` diff --git a/fastvideo/train/doc/trainer.md b/fastvideo/train/doc/trainer.md deleted file mode 100644 index 26a5a7d0d..000000000 --- a/fastvideo/train/doc/trainer.md +++ /dev/null @@ -1,30 +0,0 @@ -# `fastvideo/distillation/trainer.py` - -**目的** -- 提供与算法无关的训练 loop（infra only），把 “怎么训练” 固定下来，把 - “训练什么（loss/rollout/update policy）” 留给 method。 - -**关键类型** -- `DistillTrainer` - - `run(method, dataloader, max_steps, ...)` - - 支持： - - grad accumulation - - tracker logging（rank0；tracker 由 trainer 构建并持有） - - validation hook（`method.log_validation(step)`） - - checkpoint hook（通过 `checkpoint_manager` 注入） - -**与 Method 的契约** -`run()` 通过 duck-typing 调用（存在则调用）： -- `method.set_tracker(tracker)`（注入 tracker；用于 method-managed validation artifacts） -- `method.on_train_start()` -- `method.single_train_step(batch, step, current_vsa_sparsity=...)` - - 返回：`(loss_map, outputs, metrics)` - - `loss_map`：用于 backward 与默认标量 logging（`total_loss` 必需） - - `metrics`：额外的标量指标（非 loss，但希望记录到 tracker） -- `method.backward(loss_map, outputs, grad_accum_rounds=...)` -- `method.optimizers_schedulers_step(step)` -- `method.optimizers_zero_grad(step)` - -**重要边界** -- trainer 不应知道 roles（student/teacher/critic/...）也不应知道具体算法； - optimizer cadence、multi-optimizer 更新策略都应由 method 决定并暴露为 hook。 diff --git a/fastvideo/train/doc/utils/__init__.md b/fastvideo/train/doc/utils/__init__.md deleted file mode 100644 index 6a6ccb8a9..000000000 --- a/fastvideo/train/doc/utils/__init__.md +++ /dev/null @@ -1,12 +0,0 @@ -# `fastvideo/distillation/utils/` - -**目的** -- 放置 distillation 子系统的中性工具代码（不属于某个 model plugin / method）。 - -当前包含： -- `config.py`：YAML loader + schema/types（`DistillRunConfig`）。 -- `dataloader.py`：通用 dataloader 构建（按 dataset kind/schema 复用 FastVideo 现有实现）。 -- `moduleloader.py`：通用组件加载（`PipelineComponentLoader` 的薄封装）。 -- `module_state.py`：module 的 trainable 状态设置（`requires_grad` + train/eval）。 -- `tracking.py`：tracker 初始化（wandb / tensorboard 等；由 trainer 持有）。 -- `checkpoint.py`：role-based checkpoint/save-resume（Phase 2 runtime）。 diff --git a/fastvideo/train/doc/utils/checkpoint.md b/fastvideo/train/doc/utils/checkpoint.md deleted file mode 100644 index 669f7ca87..000000000 --- a/fastvideo/train/doc/utils/checkpoint.md +++ /dev/null @@ -1,28 +0,0 @@ -# `fastvideo/train/utils/checkpoint.py` - -**目的** -- Role-based checkpoint/save-resume 管理： - - 按 role 保存/恢复 modules、optimizers、schedulers（仅 trainable roles） - - 可选保存 dataloader 状态（如果 dataloader 是 stateful） - - 保存 RNG（全局 RNG + method 暴露的额外 generators） - - 保存 callback 状态（如 validation RNG） - - 支持 extra_role_modules（如 EMA shadow weights） - -**关键类型** -- `CheckpointConfig` - - `save_steps` / `keep_last` -- `CheckpointManager` - - `maybe_resume(resume_from_checkpoint=...) -> step | None` - - `maybe_save(step)` - - `save_final(step)` - -**关键机制** -- 只对 `model._trainable == True` 的 role 保存 optimizer/scheduler 状态。 -- 使用 `torch.distributed.checkpoint (dcp)` 做分布式 checkpoint。 -- `resume_from_checkpoint` 支持： - - `checkpoint-` 目录 - - `checkpoint-/dcp` - - `output_dir`（自动选择最新 checkpoint） - -**相关** -- `dcp_to_diffusers` entrypoint 包含 `_save_role_pretrained`：导出 role modules 为 diffusers-style 模型目录。 diff --git a/fastvideo/train/doc/utils/config.md b/fastvideo/train/doc/utils/config.md deleted file mode 100644 index d1d62d541..000000000 --- a/fastvideo/train/doc/utils/config.md +++ /dev/null @@ -1,76 +0,0 @@ -# `fastvideo/distillation/utils/config.py` - -**目的** -- 把 distillation 的 **YAML loader + schema/types + runtime 装配类型** 集中在一个更直觉的位置， - 减少文件级概念数量。 - -备注： -- `dispatch.build_from_config()` 负责把 YAML 装配成 `(training_args, method, dataloader, start_step)`。 -- tracker 由 `DistillTrainer` 构建并持有（避免 model plugin 变成 infra owner）。 - -这里包含： - -## 1) YAML loader（schema v2；YAML-only） - -**核心 API** -- `load_distill_run_config(path) -> DistillRunConfig` - -**核心产物** -- `DistillRunConfig` - - `recipe: RecipeSpec`（选择 family + method） - - `roles: dict[str, RoleSpec]`（来自 YAML 的 `roles:`） - - `training_args: TrainingArgs`（来自 YAML 的 `training:`，并注入 entrypoint invariants） - - `validation: dict`（来自 `training.validation:`，由 method 解释并驱动验证） - - `method_config: dict`（来自 YAML 的 `method_config:`，传给 method 解释） - - `raw: dict`（原始 YAML，便于 tracker 记录/复现） - -**YAML 结构（schema v2）** -- `recipe: {family: ..., method: ...}` -- `roles: {: {family?, path, trainable?}, ...}` -- `training: {...}`（大部分字段复用 `TrainingArgs.from_kwargs()`） -- `training.validation: {...}`（validation 配置；method 也会读取） -- `default_pipeline_config` 或 `default_pipeline_config_path`（默认 pipeline config） -- `method_config: {...}`（算法/recipe 专属超参） - -**Validation 参数建议归属** -- 统一放在 `training.validation:`（框架层固定字段 + method 按需字段）。 -- trainer 每步调用 `method.log_validation(step)`；是否真正执行由 method 基于 - `training.validation.every_steps` 决定。 - -**实现要点** -- `_resolve_existing_file()`：要求传入真实存在的路径（不做 overlay/fallback） -- 默认分布式 size： - - `num_gpus` 默认 1 - - `tp_size` 默认 1 - - `sp_size` 默认 `num_gpus`（保持与现有 pipeline 的期望一致） -- 训练模式 invariants（由入口强制注入）： - - `mode = DISTILLATION` - - `inference_mode = False` - - `dit_precision` 默认 `fp32`（master weights） - - `dit_cpu_offload = False` - -## 2) Schema / run config 相关（轻量选择项） -- `RecipeSpec` - - `family`: family 名称（例如 `"wan"`） - - `method`: method 名称（例如 `"dmd2"`） -- `RoleSpec` - - `family`: 该 role 的 family（默认可继承 `recipe.family`） - - `path`: 模型权重路径（HF repo 或本地目录） - - `trainable`: 是否训练该 role（只影响 `requires_grad`/模式；具体 optimizer 由 method 决定） - - `disable_custom_init_weights`: 是否禁用 family 的“加载时自定义 init weights 逻辑” - - `extra: dict`：保留 `roles.` 下除上述字段外的所有 key/value， - 交给 model plugin / method 解释 - -## 3) Builder 装配相关（build-time / run-time 边界） -- model plugin（`@register_model`）直接构建并返回一个 `ModelBase` 实例： - - `training_args` - - `bundle` - - `dataloader` - - `validator`（可选；model-specific） - - `start_step`（用于 resume / warm-start） -- `dispatch.build_from_config()` 选择 model/method 并返回 `(training_args, method, dataloader, start_step)`。 - -## 4) 通用解析 helpers（method_config / optimizer 等） -- `get_optional_int(mapping, key, where=...)` -- `get_optional_float(mapping, key, where=...)` -- `parse_betas(raw, where=...)` diff --git a/fastvideo/train/doc/utils/dataloader.md b/fastvideo/train/doc/utils/dataloader.md deleted file mode 100644 index ec9d21f35..000000000 --- a/fastvideo/train/doc/utils/dataloader.md +++ /dev/null @@ -1,18 +0,0 @@ -# `fastvideo/distillation/utils/dataloader.py` - -**目的** -- 把 “dataloader 构建” 从 model plugin（原 families/，现 `models/`）中抽离出来， - 让插件更聚焦在加载模块与组装 `RoleManager` + `ModelBase` primitives。 - -**当前包含** -- `build_parquet_t2v_train_dataloader(training_args, parquet_schema=...)` - - 复用 FastVideo 现有的 `build_parquet_map_style_dataloader(...)` - - 仅做最小封装：从 `training_args` 读取必要参数（data_path/batch/workers/seed/cfg_rate/text_len 等） -- `build_parquet_wangame_train_dataloader(training_args, parquet_schema=...)` - - 面向 wangame 的 parquet schema（I2V + action） - - 不依赖 `pipeline_config.text_encoder_configs`（schema 不含 text embedding） - -**边界** -- 这里不包含 model/pipeline 语义（例如 Wan 的 forward/backward 细节）。 -- 若未来要支持更多 dataset kind（webdataset / precomputed / i2v / ode-init ...）， - 推荐在本目录新增更通用的 builder（或引入 `DataSpec` 再做统一 dispatch）。 diff --git a/fastvideo/train/doc/utils/module_state.md b/fastvideo/train/doc/utils/module_state.md deleted file mode 100644 index 51d2bc685..000000000 --- a/fastvideo/train/doc/utils/module_state.md +++ /dev/null @@ -1,13 +0,0 @@ -# `fastvideo/distillation/utils/module_state.py` - -**目的** -- 提供最小且通用的 module 训练状态设置，避免 model plugin 里到处散落： - - `requires_grad_(...)` - - `train()` / `eval()` - -**当前包含** -- `apply_trainable(module, trainable: bool)` - -**边界** -- ✅ 不涉及 optimizer/scheduler（由 method 管理）。 -- ✅ 不涉及激活检查点策略（由 model plugin 在加载后按需启用）。 diff --git a/fastvideo/train/doc/utils/moduleloader.md b/fastvideo/train/doc/utils/moduleloader.md deleted file mode 100644 index ea298836b..000000000 --- a/fastvideo/train/doc/utils/moduleloader.md +++ /dev/null @@ -1,19 +0,0 @@ -# `fastvideo/distillation/utils/moduleloader.py` - -**目的** -- 把 “从 FastVideo 模型路径加载某个子模块（transformer/vae/…）” 的通用逻辑收敛成一个 util， - 便于多个 model plugin 复用，避免每个 plugin 都复制一份 loader 细节。 - -**当前包含** -- `load_module_from_path(model_path, module_type, training_args, disable_custom_init_weights=False, override_transformer_cls_name=None)` - - 解析/下载 `model_path` - - 读取 FastVideo 的 per-module config entry - - 调用 `PipelineComponentLoader.load_module(...)` - - 可选跳过自定义 init weights（legacy flag：`_loading_teacher_critic_model`） - - 可选临时覆盖 `training_args.override_transformer_cls_name` - - 用于 model plugin 按 role 选择 transformer 类（例如根据 `roles..family` - 选择 `WanGameActionTransformer3DModel` / `CausalWanGameActionTransformer3DModel`） - -**边界** -- ✅ 这里只做 “单模块加载”，不做 role 语义、也不做 optimizer/scheduler。 -- ✅ “哪些模块需要加载/共享/复用” 仍由 model plugin 决定。 diff --git a/fastvideo/train/doc/utils/tracking.md b/fastvideo/train/doc/utils/tracking.md deleted file mode 100644 index 76f6aa437..000000000 --- a/fastvideo/train/doc/utils/tracking.md +++ /dev/null @@ -1,14 +0,0 @@ -# `fastvideo/distillation/utils/tracking.py` - -**目的** -- 把 tracker 初始化从 model plugin 中抽离出来，避免 “模型集成层” 持有 infra 细节。 - -**当前包含** -- `build_tracker(training_args, config=...)` - - 读取 `training_args.trackers / training_args.tracker_project_name / output_dir / wandb_run_name` - - 只在 global rank0 选择真实 tracker；其余 rank 返回 no-op tracker - - tracker log dir 默认在 `output_dir/tracker/` - -**设计意图** -- tracker 属于 infra：由 `DistillTrainer` 构建并持有；method 只负责产出要 log 的 - metrics/媒体（video/image/file 等，tracker API 里常叫 artifacts）。 diff --git a/fastvideo/train/doc/validators/__init__.md b/fastvideo/train/doc/validators/__init__.md deleted file mode 100644 index e14c687f3..000000000 --- a/fastvideo/train/doc/validators/__init__.md +++ /dev/null @@ -1,9 +0,0 @@ -# `fastvideo/distillation/validators/__init__.py` - -**目的** -- 统一导出 validator 接口与 Wan validator 实现。 - -**当前导出** -- `DistillValidator`：最小 validation 接口 -- `ValidationRequest`：method 提供的 validation overrides -- `WanValidator`：Wan model plugin 的 validation 采样与记录实现 diff --git a/fastvideo/train/doc/validators/base.md b/fastvideo/train/doc/validators/base.md deleted file mode 100644 index 50ae760e1..000000000 --- a/fastvideo/train/doc/validators/base.md +++ /dev/null @@ -1,17 +0,0 @@ -# `fastvideo/distillation/validators/base.py` - -**目的** -- 定义 distillation validator 的最小抽象接口。 - -**接口** -- `log_validation(step: int, request: ValidationRequest | None = None) -> None` - -`ValidationRequest` 用于 method 覆盖关键采样配置（dataset/steps/guidance/output_dir 等），让 validator -保持 model-plugin-specific、但 method-agnostic。 - -`ValidationRequest.sample_handle` 用于由 method 明确指定“本次 validation 要采样哪个模型/权重” -（例如 student / student_ema / refiner / ...）。validator 不应自行 hardcode 角色语义。 - -**设计意图** -- trainer 只调用 `method.log_validation(step)`。 -- method 决定是否做 validation，并把 `ValidationRequest` 传给 model-plugin-specific validator。 diff --git a/fastvideo/train/doc/validators/wan.md b/fastvideo/train/doc/validators/wan.md deleted file mode 100644 index 2700e52c1..000000000 --- a/fastvideo/train/doc/validators/wan.md +++ /dev/null @@ -1,29 +0,0 @@ -# `fastvideo/distillation/validators/wan.py` - -**定位** -- `WanValidator`：Wan distillation 的 validation/sampling 实现（Phase 2 standalone）。 -- 负责： - - 读取 validation dataset（json） - - 调用 Wan pipeline 生成视频样本 - - 保存 mp4 到 `output_dir` - - 通过 tracker（例如 wandb）记录媒体（video/image/file 等；tracker API 里常叫 artifacts） - -**关键点** -- validator 运行在分布式环境下： - - 以 SP group 为单位做采样，最终由 global rank0 聚合写文件与 log -- 通过 `ValidationRequest.sample_handle` 获取本次要采样的 transformer， - 并以 `loaded_modules={"transformer": transformer}` 复用训练中的权重。 -- method 通过 `ValidationRequest` 覆盖采样配置（例如 sampling steps / guidance / output_dir）。 - -**依赖** -- 使用统一的 `WanPipeline` 做采样推理： - - `ValidationRequest.sampler_kind={ode|sde}` 选择 denoising loop - - `ValidationRequest.sampling_timesteps` 提供 few-step schedule（写入 `ForwardBatch.sampling_timesteps`） -- 这样 validator 不再依赖 `Pipeline`（例如 `WanDMDPipeline`），保持 method-agnostic。 - -**输入约定** -- dataset 由 method 通过 `ValidationRequest.dataset_file` 提供（validator 不再从 `TrainingArgs` 读取）。 - -**Tracker 注入** -- `WanValidator` 不再要求 build-time 传入 tracker； - trainer 会通过 `method.set_tracker(...)` 把 tracker 注入到 validator。 diff --git a/fastvideo/train/doc/validators/wangame.md b/fastvideo/train/doc/validators/wangame.md deleted file mode 100644 index 2ec6b1378..000000000 --- a/fastvideo/train/doc/validators/wangame.md +++ /dev/null @@ -1,16 +0,0 @@ -# `fastvideo/distillation/validators/wangame.py` - -**定位** -- `WanGameValidator` 是 WanGame 的 standalone validator： - - 由 model plugin 构建并返回（当 `training.validation.enabled=true`，或 `training.validation` 非空） - - 由 method 决定何时调用，并通过 `ValidationRequest` 指定采样细节（包含 dataset 与采样策略） - -**pipeline 选择** -- 统一使用 `WanGameActionImageToVideoPipeline`，并通过 `sampler_kind={ode|sde}` 切换采样 loop 语义。 - -**调用方式（method-managed validation）** -- method 通过 `ValidationRequest(sample_handle=..., dataset_file=..., sampler_kind=..., ...)` 指定： - - 用哪个 role 的 transformer 做采样（通常是 student） - - validation dataset（`dataset_file`） - - 采样步数（`sampling_steps`） - - 若是 SDE 采样：`sampling_timesteps`（few-step rollout 的 explicit steps） From 5fe161ef3fad613ff403ab2d99dfc1c823056b5c Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 03:28:24 +0000 Subject: [PATCH 203/214] example yaml --- .gitignore | 10 +- .../distill_wan2.1_t2v_1.3B_dmd2.yaml | 89 ++++++++ examples/distillation/refactor/example.yaml | 214 ++++++++++++++++++ 3 files changed, 304 insertions(+), 9 deletions(-) create mode 100644 examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml create mode 100644 examples/distillation/refactor/example.yaml diff --git a/.gitignore b/.gitignore index a464f92a4..28f4b3792 100644 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,6 @@ env *.npy weights/ slurm_outputs/ -fastvideo/distillation # SSIM test outputs fastvideo/tests/ssim/generated_videos/ @@ -85,11 +84,4 @@ docs/distillation/examples/ !assets/videos/**/*.mp4 dmd_t2v_output/ -preprocess_output_text/ -wangame_1.3b_overfit -wangame_1.3b_overfit_output -wangame_lingbot_test - -ode_train_output -sf_train_output -log +preprocess_output_text/ \ No newline at end of file diff --git a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml new file mode 100644 index 000000000..32efa8f5a --- /dev/null +++ b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml @@ -0,0 +1,89 @@ +# DMD2 distillation: Wan 2.1 T2V 1.3B (teacher 50-step -> student 4-step). +# +# - Teacher: frozen pretrained Wan 2.1 T2V 1.3B +# - Student: trainable, initialized from the same pretrained weights +# - Critic: trainable, initialized from the same pretrained weights +# - Validation: 4-step SDE sampling + +models: + student: + _target_: fastvideo.train.models.wan.WanModel + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + _target_: fastvideo.train.models.wan.WanModel + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: false + disable_custom_init_weights: true + critic: + _target_: fastvideo.train.models.wan.WanModel + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + disable_custom_init_weights: true + +method: + _target_: fastvideo.train.methods.distribution_matching.dmd2.DMD2Method + rollout_mode: simulate + generator_update_interval: 5 + dmd_denoising_steps: [1000, 750, 500, 250] + + # Critic optimizer (required — no fallback to training.optimizer) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: [0.0, 0.999] + fake_score_lr_scheduler: constant + +training: + distributed: + num_gpus: 8 + sp_size: 1 + tp_size: 1 + hsdp_replicate_dim: 1 + hsdp_shard_dim: 8 + + data: + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + train_batch_size: 1 + training_cfg_rate: 0.1 + seed: 1000 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + optimizer: + learning_rate: 2.0e-6 + betas: [0.9, 0.999] + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + + loop: + max_train_steps: 10000 + gradient_accumulation_steps: 1 + + checkpoint: + output_dir: outputs/wan2.1_dmd2_4steps + training_state_checkpointing_steps: 1000 + checkpoints_total_limit: 3 + + tracker: + project_name: distillation_wan + run_name: wan2.1_dmd2_4steps + + model: + enable_gradient_checkpointing_type: full + +callbacks: + grad_clip: + max_grad_norm: 1.0 + validation: + pipeline_target: fastvideo.pipelines.basic.wan.wan_pipeline.WanPipeline + dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + every_steps: 100 + sampling_steps: [4] + sampler_kind: sde + guidance_scale: 5.0 + +pipeline: + flow_shift: 3 diff --git a/examples/distillation/refactor/example.yaml b/examples/distillation/refactor/example.yaml new file mode 100644 index 000000000..fbd0d2972 --- /dev/null +++ b/examples/distillation/refactor/example.yaml @@ -0,0 +1,214 @@ +# ============================================================================== +# Full configuration reference for fastvideo.train +# +# Legend: +# [TYPED] — parsed into a typed dataclass; fields are validated with +# defaults. Unknown keys are silently ignored. +# [FREE] — free-form dict passed as-is to the target class / method. +# Keys depend on the _target_ class constructor / method_config. +# [RESOLVED] — parsed by PipelineConfig.from_kwargs(); auto-populated from +# the model's config files. Only scalar overrides are useful. +# ============================================================================== + +# ------------------------------------------------------------------------------ +# models: [FREE] +# +# Each role is instantiated via _target_(*, training_config=..., **kwargs). +# Keys here are constructor kwargs of the _target_ class (e.g. WanModel). +# You can define any role name (student, teacher, critic, etc.). +# ------------------------------------------------------------------------------ +models: + student: + _target_: fastvideo.train.models.wan.WanModel # required + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers # required: HF repo or local path + trainable: true # default: true + disable_custom_init_weights: false # default: false + flow_shift: 3.0 # default: 3.0 + enable_gradient_checkpointing_type: null # default: null (falls back to training.model) + + teacher: + _target_: fastvideo.train.models.wan.WanModel + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: false + disable_custom_init_weights: true + + critic: + _target_: fastvideo.train.models.wan.WanModel + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + disable_custom_init_weights: true + +# ------------------------------------------------------------------------------ +# method: [FREE] +# +# Instantiated via _target_(*, cfg=RunConfig, role_models=...). +# All keys besides _target_ are available in self.method_config (a plain dict). +# Keys depend entirely on the method class. +# ------------------------------------------------------------------------------ +method: + _target_: fastvideo.train.methods.distribution_matching.dmd2.DMD2Method # required + + # --- DMD2-specific keys (read from self.method_config) --- + rollout_mode: simulate # required: "simulate" or "data_latent" + generator_update_interval: 5 # default: 1 + dmd_denoising_steps: [1000, 750, 500, 250] # SDE timestep schedule + + # Critic optimizer (all required — no fallback) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: [0.0, 0.999] + fake_score_lr_scheduler: constant + + # CFG conditioning policy (optional) + # cfg_uncond: + # on_missing: error # "error" or "ignore" + # text: keep # "keep", "zero", "drop", "negative_prompt" + # image: keep # "keep", "zero", "drop" + # action: keep # "keep", "zero", "drop" + + # --- FineTuneMethod keys (if using finetune instead) --- + # _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod + # attn_kind: vsa # "dense" or "vsa" + # use_ema: false + +# ------------------------------------------------------------------------------ +# training: [TYPED] -> TrainingConfig +# +# Every field below has a typed default. Unknown keys are ignored. +# ------------------------------------------------------------------------------ +training: + + # --- training.distributed [TYPED] -> DistributedConfig --- + distributed: + num_gpus: 8 # default: 1 + tp_size: 1 # default: 1 + sp_size: 1 # default: 1 (defaults to num_gpus in loader) + hsdp_replicate_dim: 1 # default: 1 + hsdp_shard_dim: 8 # default: -1 (defaults to num_gpus in loader) + pin_cpu_memory: false # default: false + + # --- training.data [TYPED] -> DataConfig --- + data: + data_path: data/my_dataset # default: "" + train_batch_size: 1 # default: 1 + dataloader_num_workers: 4 # default: 0 + training_cfg_rate: 0.1 # default: 0.0 + seed: 1000 # default: 0 + num_height: 448 # default: 0 + num_width: 832 # default: 0 + num_latent_t: 20 # default: 0 + num_frames: 77 # default: 0 + + # --- training.optimizer [TYPED] -> OptimizerConfig --- + # Note: only for the student optimizer. Critic optimizer is in method config. + optimizer: + learning_rate: 2.0e-6 # default: 0.0 + betas: [0.9, 0.999] # default: [0.9, 0.999] + weight_decay: 0.01 # default: 0.0 + lr_scheduler: constant # default: "constant" + lr_warmup_steps: 0 # default: 0 + lr_num_cycles: 0 # default: 0 + lr_power: 0.0 # default: 0.0 + min_lr_ratio: 0.5 # default: 0.5 + + # --- training.loop [TYPED] -> TrainingLoopConfig --- + loop: + max_train_steps: 10000 # default: 0 + gradient_accumulation_steps: 1 # default: 1 + + # --- training.checkpoint [TYPED] -> CheckpointConfig --- + checkpoint: + output_dir: outputs/my_run # default: "" + resume_from_checkpoint: "" # default: "" (or use --resume-from-checkpoint CLI) + training_state_checkpointing_steps: 1000 # default: 0 (disabled) + checkpoints_total_limit: 3 # default: 0 (keep all) + + # --- training.tracker [TYPED] -> TrackerConfig --- + tracker: + trackers: [] # default: [] (auto-adds "wandb" if project_name is set) + project_name: my_project # default: "fastvideo" + run_name: my_run # default: "" + + # --- training.vsa [TYPED] -> VSAConfig --- + vsa: + sparsity: 0.0 # default: 0.0 (0.0 = disabled) + decay_rate: 0.0 # default: 0.0 + decay_interval_steps: 0 # default: 0 + + # --- training.model [TYPED] -> ModelTrainingConfig --- + model: + weighting_scheme: uniform # default: "uniform" + logit_mean: 0.0 # default: 0.0 + logit_std: 1.0 # default: 1.0 + mode_scale: 1.0 # default: 1.0 + precondition_outputs: false # default: false + moba_config: {} # default: {} + enable_gradient_checkpointing_type: full # default: null ("full" or null) + + # --- training top-level [TYPED] --- + dit_precision: fp32 # default: "fp32" (master weight precision) + # model_path: ... # default: "" (auto-derived from models.student.init_from) + +# ------------------------------------------------------------------------------ +# callbacks: [FREE] +# +# Each callback is instantiated via _target_(*, **kwargs). +# The callback name (e.g. "grad_clip") is arbitrary — only _target_ matters. +# training_config is injected automatically (not from YAML). +# ------------------------------------------------------------------------------ +callbacks: + + # --- GradNormClipCallback --- + grad_clip: + _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback # optional if using default registry + max_grad_norm: 1.0 # default: 0.0 (0.0 = disabled) + log_grad_norms: false # default: false + + # --- EMACallback --- + # ema: + # _target_: fastvideo.train.callbacks.ema.EMACallback + # type: constant # default: "constant" ("constant", "power", "halflife") + # beta: 0.9999 # default: 0.9999 (for constant type) + # gamma: 16.97 # default: 16.97 (for power type) + # ema_halflife_kimg: 500.0 # default: 500.0 (for halflife type) + # ema_rampup_ratio: 0.05 # default: 0.05 (for halflife type) + # start_iter: 0 # default: 0 + # batch_size: 1 # default: 1 + + # --- ValidationCallback --- + validation: + _target_: fastvideo.train.callbacks.validation.ValidationCallback # optional if using default registry + pipeline_target: fastvideo.pipelines.basic.wan.wan_pipeline.WanPipeline # required + dataset_file: path/to/validation.json # required + every_steps: 100 # default: 100 + sampling_steps: [4] # default: [40] + sampler_kind: sde # default: "ode" (use "sde" for few-step distilled models) + ode_solver: null # default: null ("unipc", "euler"; only for ode) + guidance_scale: 5.0 # default: null (uses model default) + num_frames: null # default: null (derived from training.data) + output_dir: null # default: null (falls back to training.checkpoint.output_dir) + sampling_timesteps: null # default: null (explicit timestep list for SDE) + rollout_mode: parallel # default: "parallel" ("parallel" or "streaming") + +# ------------------------------------------------------------------------------ +# pipeline: [RESOLVED] -> PipelineConfig +# +# Parsed by PipelineConfig.from_kwargs(). Most fields are auto-populated from +# the model's config files (vae_config, dit_config, text_encoder_configs, etc.). +# Only scalar overrides are typically needed here. +# ------------------------------------------------------------------------------ +pipeline: + flow_shift: 3 # default: null (model-specific) + # flow_shift_sr: null # default: null (super-resolution shift) + # embedded_cfg_scale: 6.0 # default: 6.0 + # is_causal: false # default: false + # vae_tiling: true # default: true + # vae_sp: true # default: true + # disable_autocast: false # default: false + +# ------------------------------------------------------------------------------ +# validation: [FREE] (legacy — prefer callbacks.validation instead) +# +# Stored as training.validation dict. Some methods may still read it. +# Prefer setting validation config in the callbacks section above. +# ------------------------------------------------------------------------------ +# validation: {} From a4b96911a62fd3bac35bf3d668dd4f56d702bf23 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 03:39:56 +0000 Subject: [PATCH 204/214] solver -> target. remove legact valiadition key --- .../refactor/dfsft_wangame_causal_v3.yaml | 2 +- .../distill_wan2.1_t2v_1.3B_dmd2.yaml | 1 + examples/distillation/refactor/example.yaml | 12 +-- fastvideo/train/callbacks/validation.py | 76 +++++-------------- .../train/entrypoint/dcp_to_diffusers.py | 12 --- fastvideo/train/utils/config.py | 20 +---- fastvideo/train/utils/training_config.py | 1 - 7 files changed, 26 insertions(+), 98 deletions(-) diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml index fb3b0e941..e2e77db67 100644 --- a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml +++ b/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml @@ -82,7 +82,7 @@ callbacks: sampling_steps: [40] rollout_mode: streaming sampler_kind: ode - ode_solver: euler + scheduler_target: fastvideo.models.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler guidance_scale: 1.0 num_frames: 69 diff --git a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml index 32efa8f5a..763350b31 100644 --- a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml +++ b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml @@ -83,6 +83,7 @@ callbacks: every_steps: 100 sampling_steps: [4] sampler_kind: sde + sampling_timesteps: [1000, 750, 500, 250] guidance_scale: 5.0 pipeline: diff --git a/examples/distillation/refactor/example.yaml b/examples/distillation/refactor/example.yaml index fbd0d2972..f025d47b1 100644 --- a/examples/distillation/refactor/example.yaml +++ b/examples/distillation/refactor/example.yaml @@ -182,7 +182,9 @@ callbacks: every_steps: 100 # default: 100 sampling_steps: [4] # default: [40] sampler_kind: sde # default: "ode" (use "sde" for few-step distilled models) - ode_solver: null # default: null ("unipc", "euler"; only for ode) + scheduler_target: null # default: null (_target_ for scheduler class, e.g. + # fastvideo.models.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler + # fastvideo.models.schedulers.scheduling_flow_unipc_multistep.FlowUniPCMultistepScheduler) guidance_scale: 5.0 # default: null (uses model default) num_frames: null # default: null (derived from training.data) output_dir: null # default: null (falls back to training.checkpoint.output_dir) @@ -204,11 +206,3 @@ pipeline: # vae_tiling: true # default: true # vae_sp: true # default: true # disable_autocast: false # default: false - -# ------------------------------------------------------------------------------ -# validation: [FREE] (legacy — prefer callbacks.validation instead) -# -# Stored as training.validation dict. Some methods may still read it. -# Prefer setting validation config in the callbacks section above. -# ------------------------------------------------------------------------------ -# validation: {} diff --git a/fastvideo/train/callbacks/validation.py b/fastvideo/train/callbacks/validation.py index 2fd693837..b0b152591 100644 --- a/fastvideo/train/callbacks/validation.py +++ b/fastvideo/train/callbacks/validation.py @@ -67,7 +67,7 @@ def __init__( every_steps: int = 100, sampling_steps: list[int] | None = None, sampler_kind: str = "ode", - ode_solver: str | None = None, + scheduler_target: str | None = None, guidance_scale: float | None = None, num_frames: int | None = None, output_dir: str | None = None, @@ -84,8 +84,9 @@ def __init__( else [40] ) self.sampler_kind = str(sampler_kind) - self.ode_solver = ( - str(ode_solver) if ode_solver is not None + self.scheduler_target = ( + str(scheduler_target) + if scheduler_target is not None else None ) self.guidance_scale = ( @@ -350,7 +351,7 @@ def _get_pipeline( id(transformer), self.rollout_mode, self.sampler_kind, - str(self.ode_solver), + self.scheduler_target, ) if ( self._pipeline is not None @@ -380,19 +381,13 @@ def _get_pipeline( } if flow_shift is not None: kwargs["flow_shift"] = float(flow_shift) - if self.ode_solver is not None: - kwargs["ode_solver"] = str(self.ode_solver) - - # For streaming pipelines, build and inject a - # scheduler if needed. - if self.rollout_mode == "streaming": - scheduler = self._build_streaming_scheduler( - flow_shift, + + # Build and inject a scheduler if target is set. + scheduler = self._build_scheduler(flow_shift) + if scheduler is not None: + kwargs["loaded_modules"]["scheduler"] = ( + scheduler ) - if scheduler is not None: - kwargs["loaded_modules"]["scheduler"] = ( - scheduler - ) self._pipeline = PipelineCls.from_pretrained( tc.model_path, **kwargs, @@ -400,54 +395,19 @@ def _get_pipeline( self._pipeline_key = key return self._pipeline - def _build_streaming_scheduler( + def _build_scheduler( self, flow_shift: float | None, ) -> Any | None: - """Build scheduler for streaming validation.""" + """Build scheduler from ``scheduler_target``.""" + if self.scheduler_target is None: + return None if flow_shift is None: return None - if self.sampler_kind == "sde": - from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, ) - - return FlowMatchEulerDiscreteScheduler( - shift=float(flow_shift), - ) - - # ODE mode — choose based on ode_solver. - ode_solver_norm = ( - str(self.ode_solver).strip().lower() - if self.ode_solver is not None - else "unipc" - ) - if ode_solver_norm in { - "unipc", - "unipc_multistep", - "multistep", - }: - from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( - FlowUniPCMultistepScheduler, ) - - return FlowUniPCMultistepScheduler( - shift=float(flow_shift), - ) - if ode_solver_norm in { - "euler", - "flowmatch", - "flowmatch_euler", - }: - from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, ) - - return FlowMatchEulerDiscreteScheduler( - shift=float(flow_shift), - ) - - raise ValueError( - f"Unknown ode_solver for streaming " - f"validation: {self.ode_solver!r}" + SchedulerCls = resolve_target( + self.scheduler_target ) + return SchedulerCls(shift=float(flow_shift)) # ---------------------------------------------------------- # Batch preparation diff --git a/fastvideo/train/entrypoint/dcp_to_diffusers.py b/fastvideo/train/entrypoint/dcp_to_diffusers.py index b28cff67e..a62dde639 100644 --- a/fastvideo/train/entrypoint/dcp_to_diffusers.py +++ b/fastvideo/train/entrypoint/dcp_to_diffusers.py @@ -324,15 +324,6 @@ def _run_config_from_raw( ) method = dict(method_raw) - validation_raw = raw.get("validation", None) - validation: dict[str, Any] = ( - _require_mapping( - validation_raw, where="validation", - ) - if validation_raw is not None - else {} - ) - callbacks_raw = raw.get("callbacks", None) callbacks: dict[str, dict[str, Any]] = ( _require_mapping( @@ -350,19 +341,16 @@ def _run_config_from_raw( raw.get("training"), where="training", ) t = dict(training_raw) - t.pop("validation", None) training = _build_training_config( t, models=models, pipeline_config=pipeline_config, - validation=validation, ) return RunConfig( models=models, method=method, training=training, - validation=validation, callbacks=callbacks, raw=raw, ) diff --git a/fastvideo/train/utils/config.py b/fastvideo/train/utils/config.py index e00c9b07e..704362b98 100644 --- a/fastvideo/train/utils/config.py +++ b/fastvideo/train/utils/config.py @@ -30,7 +30,6 @@ class RunConfig: models: dict[str, dict[str, Any]] method: dict[str, Any] training: TrainingConfig - validation: dict[str, Any] callbacks: dict[str, dict[str, Any]] raw: dict[str, Any] @@ -59,7 +58,6 @@ def _safe_asdict(obj: Any) -> Any: resolved["models"] = dict(self.models) resolved["method"] = dict(self.method) resolved["training"] = _safe_asdict(self.training) - resolved["validation"] = dict(self.validation) resolved["callbacks"] = dict(self.callbacks) return resolved @@ -286,7 +284,6 @@ def _build_training_config( *, models: dict[str, dict[str, Any]], pipeline_config: Any, - validation: dict[str, Any], ) -> TrainingConfig: """Build TrainingConfig from nested training: YAML.""" d = dict(t.get("distributed", {}) or {}) @@ -417,7 +414,6 @@ def _build_training_config( "enable_gradient_checkpointing_type" )), ), - validation=validation, pipeline_config=pipeline_config, model_path=model_path, dit_precision=str( @@ -429,7 +425,7 @@ def load_run_config(path: str) -> RunConfig: """Load a run config from YAML. Expected top-level keys: ``models``, ``method``, - ``training`` (nested), and optionally ``validation`` + ``training`` (nested), and optionally ``callbacks`` and ``pipeline``. """ path = _resolve_existing_file(path) @@ -460,14 +456,6 @@ def load_run_config(path: str) -> RunConfig: "method must have a '_target_' key") method = dict(method_raw) - # --- validation --- - validation_raw = cfg.get("validation", None) - if validation_raw is None: - validation: dict[str, Any] = {} - else: - validation = _require_mapping( - validation_raw, where="validation") - # --- callbacks --- callbacks_raw = cfg.get("callbacks", None) if callbacks_raw is None: @@ -484,16 +472,14 @@ def load_run_config(path: str) -> RunConfig: training_raw = _require_mapping( cfg.get("training"), where="training") t = dict(training_raw) - t.pop("validation", None) training = _build_training_config( - t, models=models, pipeline_config=pipeline_config, - validation=validation) + t, models=models, + pipeline_config=pipeline_config) return RunConfig( models=models, method=method, training=training, - validation=validation, callbacks=callbacks, raw=cfg, ) diff --git a/fastvideo/train/utils/training_config.py b/fastvideo/train/utils/training_config.py index b47d5a8cd..0167751db 100644 --- a/fastvideo/train/utils/training_config.py +++ b/fastvideo/train/utils/training_config.py @@ -94,7 +94,6 @@ class TrainingConfig: tracker: TrackerConfig = field(default_factory=TrackerConfig) vsa: VSAConfig = field(default_factory=VSAConfig) model: ModelTrainingConfig = field(default_factory=ModelTrainingConfig) - validation: dict = field(default_factory=dict) pipeline_config: PipelineConfig | None = None model_path: str = "" dit_precision: str = "fp32" From 7c2648ac5606709562855ab3df4a5f0887181146 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 03:42:37 +0000 Subject: [PATCH 205/214] move device from build pre to init --- fastvideo/train/models/wan/wan.py | 3 +-- fastvideo/train/models/wangame/wangame.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/fastvideo/train/models/wan/wan.py b/fastvideo/train/models/wan/wan.py index 240ba28ba..3267e18fb 100644 --- a/fastvideo/train/models/wan/wan.py +++ b/fastvideo/train/models/wan/wan.py @@ -112,7 +112,7 @@ def __init__( self.world_group: Any = None self.sp_group: Any = None - self.device: Any = None + self.device: Any = get_local_torch_device() self.noise_random_generator: (torch.Generator | None) = None self.noise_gen_cuda: torch.Generator | None = None @@ -139,7 +139,6 @@ def init_preprocessors(self, training_config: TrainingConfig) -> None: self.world_group = get_world_group() self.sp_group = get_sp_group() - self.device = get_local_torch_device() self._init_timestep_mechanics() diff --git a/fastvideo/train/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py index 47159c6de..1d7a25855 100644 --- a/fastvideo/train/models/wangame/wangame.py +++ b/fastvideo/train/models/wangame/wangame.py @@ -90,7 +90,7 @@ def __init__( self.world_group: Any = None self.sp_group: Any = None - self.device: Any = None + self.device: Any = get_local_torch_device() self.noise_random_generator: (torch.Generator | None) = None self.noise_gen_cuda: torch.Generator | None = None @@ -138,7 +138,6 @@ def init_preprocessors(self, training_config: TrainingConfig) -> None: self.world_group = get_world_group() self.sp_group = get_sp_group() - self.device = get_local_torch_device() self._init_timestep_mechanics() From 1d86df3ff3dcf40ce5aea204015efe36243df175 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 21:08:50 +0000 Subject: [PATCH 206/214] minor config --- .../refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml | 1 + ...finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml index 763350b31..6e2a1451b 100644 --- a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml +++ b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml @@ -25,6 +25,7 @@ method: _target_: fastvideo.train.methods.distribution_matching.dmd2.DMD2Method rollout_mode: simulate generator_update_interval: 5 + real_score_guidance_scale: 3.5 dmd_denoising_steps: [1000, 750, 500, 250] # Critic optimizer (required — no fallback to training.optimizer) diff --git a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml index 10f6ea028..888ad2491 100644 --- a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml +++ b/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml @@ -12,7 +12,7 @@ models: method: _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod attn_kind: vsa - # use_ema: true + use_ema: true training: distributed: @@ -66,9 +66,9 @@ callbacks: grad_clip: _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback max_grad_norm: 1.0 - # ema: - # _target_: fastvideo.train.callbacks.ema.EMACallback - # beta: 0.9999 + ema: + _target_: fastvideo.train.callbacks.ema.EMACallback + beta: 0.9999 validation: _target_: fastvideo.train.callbacks.validation.ValidationCallback pipeline_target: fastvideo.pipelines.basic.wan.wan_pipeline.WanPipeline From c9bfc657d0bf410fb078a0039774c22e36de6fbc Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 21:51:33 +0000 Subject: [PATCH 207/214] fix dmd2 and selfforcing cfg --- fastvideo/train/methods/distribution_matching/dmd2.py | 4 ++-- fastvideo/train/methods/distribution_matching/self_forcing.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fastvideo/train/methods/distribution_matching/dmd2.py b/fastvideo/train/methods/distribution_matching/dmd2.py index 12b59824b..9c2e07ef9 100644 --- a/fastvideo/train/methods/distribution_matching/dmd2.py +++ b/fastvideo/train/methods/distribution_matching/dmd2.py @@ -726,9 +726,9 @@ def _dmd_loss( cfg_uncond=self._cfg_uncond, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + ( + real_cfg_x0 = real_uncond_x0 + ( real_cond_x0 - real_uncond_x0 - ) * (guidance_scale - 1) + ) * guidance_scale denom = torch.abs( generator_pred_x0 - real_cfg_x0 diff --git a/fastvideo/train/methods/distribution_matching/self_forcing.py b/fastvideo/train/methods/distribution_matching/self_forcing.py index 3a8c8c083..ae2547bd5 100644 --- a/fastvideo/train/methods/distribution_matching/self_forcing.py +++ b/fastvideo/train/methods/distribution_matching/self_forcing.py @@ -561,7 +561,7 @@ def _dmd_loss( conditional=False, attn_kind="dense", ) - real_cfg_x0 = real_cond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale + real_cfg_x0 = real_uncond_x0 + (real_cond_x0 - real_uncond_x0) * guidance_scale denom = torch.abs(generator_pred_x0 - real_cfg_x0).mean() grad = (faker_x0 - real_cfg_x0) / denom From 9b69bbefeb553aaa2b524d46a9a81906e7742dd5 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 22:35:36 +0000 Subject: [PATCH 208/214] rfc and dmd minor --- .../distill_wan2.1_t2v_1.3B_dmd2.yaml | 26 ++-- examples/distillation/refactor/rfc.md | 142 ++++++++++++++++++ 2 files changed, 155 insertions(+), 13 deletions(-) create mode 100644 examples/distillation/refactor/rfc.md diff --git a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml index 6e2a1451b..63aa765b4 100644 --- a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml +++ b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml @@ -1,9 +1,9 @@ -# DMD2 distillation: Wan 2.1 T2V 1.3B (teacher 50-step -> student 4-step). +# DMD2 distillation: Wan 2.1 T2V 1.3B (teacher 50-step -> student 8-step). # # - Teacher: frozen pretrained Wan 2.1 T2V 1.3B # - Student: trainable, initialized from the same pretrained weights # - Critic: trainable, initialized from the same pretrained weights -# - Validation: 4-step SDE sampling +# - Validation: 8-step SDE sampling models: student: @@ -26,7 +26,7 @@ method: rollout_mode: simulate generator_update_interval: 5 real_score_guidance_scale: 3.5 - dmd_denoising_steps: [1000, 750, 500, 250] + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] # Critic optimizer (required — no fallback to training.optimizer) fake_score_learning_rate: 8.0e-6 @@ -45,7 +45,7 @@ training: data_path: data/Wan-Syn_77x448x832_600k dataloader_num_workers: 4 train_batch_size: 1 - training_cfg_rate: 0.1 + training_cfg_rate: 0.0 seed: 1000 num_latent_t: 20 num_height: 448 @@ -54,23 +54,23 @@ training: optimizer: learning_rate: 2.0e-6 - betas: [0.9, 0.999] + betas: [0.0, 0.999] weight_decay: 0.01 lr_scheduler: constant lr_warmup_steps: 0 loop: - max_train_steps: 10000 + max_train_steps: 4000 gradient_accumulation_steps: 1 checkpoint: - output_dir: outputs/wan2.1_dmd2_4steps + output_dir: outputs/wan2.1_dmd2_8steps training_state_checkpointing_steps: 1000 checkpoints_total_limit: 3 tracker: project_name: distillation_wan - run_name: wan2.1_dmd2_4steps + run_name: wan2.1_dmd2_8steps model: enable_gradient_checkpointing_type: full @@ -81,11 +81,11 @@ callbacks: validation: pipeline_target: fastvideo.pipelines.basic.wan.wan_pipeline.WanPipeline dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - every_steps: 100 - sampling_steps: [4] + every_steps: 50 + sampling_steps: [8] sampler_kind: sde - sampling_timesteps: [1000, 750, 500, 250] - guidance_scale: 5.0 + sampling_timesteps: [1000, 850, 700, 550, 350, 275, 200, 125] + guidance_scale: 6.0 pipeline: - flow_shift: 3 + flow_shift: 8 diff --git a/examples/distillation/refactor/rfc.md b/examples/distillation/refactor/rfc.md new file mode 100644 index 000000000..8ac81430d --- /dev/null +++ b/examples/distillation/refactor/rfc.md @@ -0,0 +1,142 @@ + + +## 1) File Structure + +fastvideo/train/ + trainer.py # Training loop; calls method.train_one_step() + models/ + base.py # BaseModel ABC: predict_x0, add_noise, backward, ... + wan/ + wan.py # Wan model loader + wangame/ + wangame.py # WanGame model loader + wangame_causal.py + methods/ + base.py # DistillMethod base; methods provide train_one_step + distribution_matching/ + dmd2.py # DMD2 distillation (student/teacher/critic) + self_forcing.py # Self-forcing distillation + fine_tuning/ + finetune.py # SFT finetuning (student only) + dfsft.py # Distribution-free SFT + knowledge_distillation/ + consistency_model/ + callbacks/ + callback.py # CallbackDict registry + grad_clip.py # Gradient clipping + optional per-module norm logging + validation.py # Periodic validation via inference pipeline + ema.py # EMA weight averaging + entrypoint/ + train.py # YAML-only CLI entrypoint (torchrun -m fastvideo.train.entrypoint.train) + dcp_to_diffusers.py # Checkpoint conversion + utils/ + config.py # YAML parser -> RunConfig + builder.py # build_from_config: instantiate models, method, dataloader + instantiate.py # _target_ based instantiation + training_config.py # TrainingConfig dataclass (all training settings with defaults) + dataloader.py # Dataset / dataloader construction + moduleloader.py # Dynamic module import + module_state.py # apply_trainable(): requires_grad + train/eval + optimizer.py # Optimizer construction + tracking.py # W&B tracker (owned by trainer) + checkpoint.py # Save/resume with DCP + validation.py # Validation helpers + +By this design, we only need a YAML config to train different models using different methods. +Models declare `_target_` to select the model class; methods declare `_target_` to select the method class. +Current code: https://github.com/FoundationResearch/FastVideo/tree/distill1/fastvideo/train + +DMD2 Distillation, Self-Forcing, SFT, and DFSFT are tested on Wan / WanGame. + +Current supported models: Wan, WanGame. +Current supported methods: DMD2, Self-Forcing, SFT, DFSFT. + +Feedbacks are highly welcome! + + +## 2) Example YAML (DMD2 8-step) + +```yaml +models: + student: + _target_: fastvideo.train.models.wan.WanModel + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + teacher: + _target_: fastvideo.train.models.wan.WanModel + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: false + disable_custom_init_weights: true + critic: + _target_: fastvideo.train.models.wan.WanModel + init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers + trainable: true + disable_custom_init_weights: true + +method: + _target_: fastvideo.train.methods.distribution_matching.dmd2.DMD2Method + rollout_mode: simulate + generator_update_interval: 5 + real_score_guidance_scale: 3.5 + dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] + + # Critic optimizer (required) + fake_score_learning_rate: 8.0e-6 + fake_score_betas: [0.0, 0.999] + fake_score_lr_scheduler: constant + +training: + distributed: + num_gpus: 8 + sp_size: 1 + tp_size: 1 + + data: + data_path: data/Wan-Syn_77x448x832_600k + dataloader_num_workers: 4 + train_batch_size: 1 + training_cfg_rate: 0.0 + seed: 1000 + num_latent_t: 20 + num_height: 448 + num_width: 832 + num_frames: 77 + + optimizer: + learning_rate: 2.0e-6 + betas: [0.0, 0.999] + weight_decay: 0.01 + lr_scheduler: constant + lr_warmup_steps: 0 + + loop: + max_train_steps: 4000 + gradient_accumulation_steps: 1 + + checkpoint: + output_dir: outputs/wan2.1_dmd2_8steps + training_state_checkpointing_steps: 1000 + checkpoints_total_limit: 3 + + tracker: + project_name: distillation_wan + run_name: wan2.1_dmd2_8steps + + model: + enable_gradient_checkpointing_type: full + +callbacks: + grad_clip: + max_grad_norm: 1.0 + validation: + pipeline_target: fastvideo.pipelines.basic.wan.wan_pipeline.WanPipeline + dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json + every_steps: 100 + sampling_steps: [8] + sampler_kind: sde + sampling_timesteps: [1000, 850, 700, 550, 350, 275, 200, 125] + guidance_scale: 6.0 + +pipeline: + flow_shift: 8 +``` From 68c08d189a203911ea4cfc4522464c29938ee2af Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 22:52:24 +0000 Subject: [PATCH 209/214] remove npys --- .../distill_wan2.1_t2v_1.3B_dmd2.yaml | 4 +- .../action/000000_action.npy | Bin 2994 -> 0 bytes .../action/000001_action.npy | Bin 2994 -> 0 bytes .../action/000002_action.npy | Bin 2994 -> 0 bytes .../action/000003_action.npy | Bin 2994 -> 0 bytes .../action/000004_action.npy | Bin 2994 -> 0 bytes .../action/000005_action.npy | Bin 2994 -> 0 bytes .../action/000006_action.npy | Bin 2994 -> 0 bytes .../action/000007_action.npy | Bin 2994 -> 0 bytes .../action/000008_action.npy | Bin 2994 -> 0 bytes .../action/000009_action.npy | Bin 2994 -> 0 bytes .../action/000010_action.npy | Bin 2994 -> 0 bytes .../action/000011_action.npy | Bin 2994 -> 0 bytes .../action/000012_action.npy | Bin 2994 -> 0 bytes .../action/000013_action.npy | Bin 2994 -> 0 bytes .../action/000014_action.npy | Bin 2994 -> 0 bytes .../action/000015_action.npy | Bin 2994 -> 0 bytes .../action/000016_action.npy | Bin 2994 -> 0 bytes .../action/000017_action.npy | Bin 2994 -> 0 bytes .../action/000018_action.npy | Bin 2994 -> 0 bytes .../action/000019_action.npy | Bin 2994 -> 0 bytes .../action/000020_action.npy | Bin 2994 -> 0 bytes .../action/000021_action.npy | Bin 2994 -> 0 bytes .../action/000022_action.npy | Bin 2994 -> 0 bytes .../action/000023_action.npy | Bin 2994 -> 0 bytes .../action/000024_action.npy | Bin 2994 -> 0 bytes .../action/000025_action.npy | Bin 2994 -> 0 bytes .../action/000026_action.npy | Bin 2994 -> 0 bytes .../action/000027_action.npy | Bin 2994 -> 0 bytes .../action/000028_action.npy | Bin 2994 -> 0 bytes .../action/000029_action.npy | Bin 2994 -> 0 bytes .../action/000030_action.npy | Bin 2994 -> 0 bytes .../action/000031_action.npy | Bin 2994 -> 0 bytes .../action/000032_action.npy | Bin 2994 -> 0 bytes .../action/000033_action.npy | Bin 2994 -> 0 bytes .../action/000034_action.npy | Bin 2994 -> 0 bytes .../action/000035_action.npy | Bin 2994 -> 0 bytes .../action/000036_action.npy | Bin 2994 -> 0 bytes .../action/000037_action.npy | Bin 2994 -> 0 bytes .../action/000038_action.npy | Bin 2994 -> 0 bytes .../action/000039_action.npy | Bin 2994 -> 0 bytes .../action/000040_action.npy | Bin 2994 -> 0 bytes .../action/000041_action.npy | Bin 2994 -> 0 bytes .../action/000042_action.npy | Bin 2994 -> 0 bytes .../action/000043_action.npy | Bin 2994 -> 0 bytes .../action/000044_action.npy | Bin 2994 -> 0 bytes .../action/000045_action.npy | Bin 2994 -> 0 bytes .../action/000046_action.npy | Bin 2994 -> 0 bytes .../action/000047_action.npy | Bin 2994 -> 0 bytes .../action/000048_action.npy | Bin 2994 -> 0 bytes .../action/000049_action.npy | Bin 2994 -> 0 bytes .../action/000050_action.npy | Bin 2994 -> 0 bytes .../action/000051_action.npy | Bin 2994 -> 0 bytes .../action/000052_action.npy | Bin 2994 -> 0 bytes .../action/000053_action.npy | Bin 2994 -> 0 bytes .../action/000054_action.npy | Bin 2994 -> 0 bytes .../action/000055_action.npy | Bin 2994 -> 0 bytes .../action/000056_action.npy | Bin 2994 -> 0 bytes .../action/000057_action.npy | Bin 2994 -> 0 bytes .../action/000058_action.npy | Bin 2994 -> 0 bytes .../action/000059_action.npy | Bin 2994 -> 0 bytes .../action/000060_action.npy | Bin 2994 -> 0 bytes .../action/000061_action.npy | Bin 2994 -> 0 bytes .../action/000062_action.npy | Bin 2994 -> 0 bytes .../action/000063_action.npy | Bin 2994 -> 0 bytes .../causal_wangame_ode_init/action/README.md | 66 ------------------ 66 files changed, 2 insertions(+), 68 deletions(-) delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000000_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000001_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000002_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000003_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000004_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000005_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000006_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000007_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000008_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000009_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000010_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000011_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000012_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000013_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000014_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000015_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000016_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000017_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000018_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000019_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000020_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000021_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000022_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000023_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000024_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000025_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000026_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000027_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000028_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000029_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000030_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000031_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000032_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000033_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000034_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000035_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000036_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000037_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000038_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000039_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000040_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000041_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000042_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000043_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000044_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000045_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000046_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000047_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000048_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000049_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000050_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000051_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000052_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000053_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000054_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000055_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000056_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000057_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000058_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000059_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000060_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000061_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000062_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/000063_action.npy delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/action/README.md diff --git a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml index 63aa765b4..7f3a0daf9 100644 --- a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml +++ b/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml @@ -25,7 +25,7 @@ method: _target_: fastvideo.train.methods.distribution_matching.dmd2.DMD2Method rollout_mode: simulate generator_update_interval: 5 - real_score_guidance_scale: 3.5 + real_score_guidance_scale: 4.5 dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] # Critic optimizer (required — no fallback to training.optimizer) @@ -70,7 +70,7 @@ training: tracker: project_name: distillation_wan - run_name: wan2.1_dmd2_8steps + run_name: wan2.1_dmd2_8steps_cfg4.5 model: enable_gradient_checkpointing_type: full diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000000_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000000_action.npy deleted file mode 100644 index 8a6ab2ab14b855f0691ea93076134c495f72926b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeHIF-yZh6i!lO#X37UWlIVrlY@v*sNi5QRl&hQ!X>#9vB`zYMF_UgrP{!4d4m5| zbBRMJZVE22TBwCFv;Vzu)Tm#Y{&VWJt9g`0sO$DI3P*JXdW=A zfa^^q@O~1uB^5Zz8qQT8-ryE2M;%5zu7|8hv8dOkCs1zMqaIfm;JWVRXB`bVV+MZt z086m=-V-i-f@lbl7U_aC>VH!@H_q-WL<6qPj=OAUTWpG8n|$;|Q>$^?+cwI4tBBM= z>d<`~(ht%P(hpq)q#wF!{LTLW^@QenO!5r>IOer_%paALjn=(duB8J1r+~0*j%u*K E06JO3F8}}l diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000001_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000001_action.npy deleted file mode 100644 index e649b6c7273d76092c69f063d53cc78ae717dfbf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeHIze~eF6uzX!igk8y%62J~Ob#MKp@M_KR0RhI376zb#3mOmi4bg|i`u|#d4m6~ z=7&NQw+=4%hIjAYJ-)|%e0Q(3Z!Rux74(Rnuul_@<6|8!rg)^|8RID@k;Ax8>+~6! zC$!2Z0f}j~H-;m9sc&YxtF1RYK#ht0C=27QuJD_nzNDZ`*AG(XK_TGy-7BPiDGQBNofuxIzKR`91g%+|bp?{8fwHosW#bm8^+sarn@P7={lFdlv F%uh&;#4i8< diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000002_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000002_action.npy deleted file mode 100644 index 3ee9a9c38c0fc4b02f535415193cac4201c448a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeH|ze~eF6vr>AF=Cw^oU&aCC6j}QP^jQwFjc|9LBb`ua$=K|OCkhY=%O}oTb|&5 ztN9@`h?|1Tz2R~1-Qx@Q<-@(wxw*K!RnQ}P!U0Wuj*ktznBbv-r!h}CiQJe6v~Hh~ zSwgFR!pNLfwK*6XOXGL3yV`ogebj{5kFs!{Tdp5-YK7S>y(gTL9K@c>sUJs4$}>NO z5`@-Fg3zO&9Vx*^&v2@E@Cw&p+RA|SgchXvoYwQ|$KVv{SEL#8no^cAl=zfIxNe`J zS&BtJC1{St+8zwerTM$qU2VNteWeM$pXBkpaQu)m?8NyjyGM+n0{EWKa7dFhV|kcC z1zc~Cz>OqoAr-jFGn}dcyuvlut~ww+u18rh$2DG?96`Bmw|ZP%faiJVA0nFMjxq?! zJ6M9vcOGyNP{bl28qsS~Vti4$SMK%;kUrO@hh4UlZ){4 DWcb7{ diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000004_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000004_action.npy deleted file mode 100644 index 9ce5230372d7f3a39ac9eb871b638708e37e2d55..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeH|ze~eF6vva)7_rU{PT7(|$>bm+6e>6vOjU4jkZ?(^L~MR=xd_1)x>Q@ZEl=>j z)%@_lh?|3h_lA4C?>)ZoK0J7(b8~TdD@%{klM&#|C&saE%qPZBH>L@p1jTMb0$j^y zXqMqh&q9>qsx}8heWCv@c9&bPMqg@z?Z$#5$#x!a;U$O!5GtWsn}*t_rF~^@KSS7O%JlH7+sQXKsdgK`Y4uRcQ9B#YZ6$QU zffMNNhuD)Y4^D}5;aTXMK$n-`)8)Y_aV|UyofGKt`X79ty3kCGl03sdo_VD@^9P&p TTIoU(l_2m}1nQN|QT13Kr_=zg diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000005_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000005_action.npy deleted file mode 100644 index 02cd7ab2e0691d85f98d3d6140435496260cf2dd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGacRJ{HmZH3|;+@f9~Tc6;6 ztNmf2h}#V=d62xkm&X_K@g=Xc?yj!yCG?D5utyV@;ZqGSXLzjPdBjpiLOWs}tVY?BdMvyJiN5UA%fgjk6x>1;*7ni3sz+mwlxM_0=Cp^TynL1AoZn`EeiZ>=zoU(Wakjdn<(2lXi4*=S99{ zLhvAX=-x!_g&u?+gdX|~2tD-K*#3WjszY-nh_ZzK+2-Y{%^wxhjod$0tOW!A$3QLF Ij8v2H4YmmYtpET3 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000007_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000007_action.npy deleted file mode 100644 index b0f8a4d3e79be73f379dc65f8aca9f50c5a87b48..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeH|u}i~16vi*9F=Cw^oU$c_lF30tC{%DTn5y95Amy4|iP$FLk_f>Tx>Q@ZEl=>j z)ui!(5jO`B?hW_defRj_ejL2Qy}7u&RmdZGGJTeK+&nhS*}&`?<~ZUhr{OT-KCA3A zI!RdRCjpIFdA2%TV{ZJeb{AW(W}9qE?1$Mj&h4QWab{1mNqSE?r#XndA!l9`CMnOn z6bca9rW1r7`gKbI4q3vfGJ+Sl2Fp=8K}%?Un#Zi_R|iM1Ue#AEq0GQ_-ShXdHjC3s zVYI%3Iap%nf#su!avuUU(5lct|BQ65ob6`_+Cm*4eoZ_1mPV?y{+nLEijYOG-l0k# z2{gG#K207-3Avz39|<&hNj^;;NC~;1N*@U{c}YG^9!Lqfp#Fa!$_XvBX_O`G$1$&# YWBy>>U#a+1sgl6o5~#jxM$5(i0K`53tpET3 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000008_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000008_action.npy deleted file mode 100644 index 8a754f020b29a79843f069690896426397c4eb96..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarJu diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000010_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000010_action.npy deleted file mode 100644 index e97638cfc59b49de2c798709bf1dc5e988c22fd0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGarc diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000015_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000015_action.npy deleted file mode 100644 index a8c7fba01479245739ac912e26c76249940eec91..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGadYGO8=YfuXjGsP-=`FMtPVNrsKtwm|?!i9xA&#(cH_R z0;V>Vz_iG3ODeFWS2&klc!ygsEO`_RnCfRmLU_G0JArc3ehrws1lzVRziMloHOas& zA7BLr+j}B~8)NE2pad!h4YVIf>&DuBgJ8&%`EeKRYzs}|Xwy%PH?^L$y={AuZxsQd_sMpnHt7 GXMO{}?!+$u diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000017_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000017_action.npy deleted file mode 100644 index 29db3beb5a0180f62dc11684c506f12e0f4d6823..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV=$$M z!Q>WD7>&ATG_MTSyfSps=M-QK-NTxjUs{}swVG!FR`XJwP903hK|RVI4S|6R0Z@`H IO#~8p0K)3TF8}}l diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000018_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000018_action.npy deleted file mode 100644 index 19c0eba669a3dd77bd29b276e8edc30fc4accbae..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeHIze~eF6uzX!h;?>w%62J~Ob#MKp@M_KR0RhI376zb#O4Q=LHQM-HZGY9~<)~Y&&!R8* zS-Sxt;fKxiL;WhD&1PyNbP>9=4E&Y{s5UfLf+$Pqk8578*8D*+UddfpaTW~x9|N^z IGg2kv6WmzDF8}}l diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000019_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000019_action.npy deleted file mode 100644 index d93f0a1d14b4a16ea50a17b888fde5e361295bf7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaD0l59Oa|z(GVE85CA3F(nKJk F2LRtx#4i8< diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000020_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000020_action.npy deleted file mode 100644 index 7ae951a8fe075b3a6fd4df8416b8ce72f5845f52..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gW3Z(M zklO~En?~I;nr8-co*CTfa|*DA?qSW%FD*{RTFo;7t9hwTrw%6MARc9phQPpu04T|p ICISgP0L~-CF8}}l diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000023_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000023_action.npy deleted file mode 100644 index 6b483a2bec1c0413132a0c85bf31608b3502db32..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gW3Z(M znA--MyGGqJnr~of6GV@u2V&F16krYA!s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaJBn^=j4xl7kng}HH E0GKp!5dZ)H diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000031_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000031_action.npy deleted file mode 100644 index 1194a0be934c4e5c1227135515c854c21ba7ec3b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa58g`>JBn^=j4xl7kng}HH E0Pa0;kpKVy diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000032_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000032_action.npy deleted file mode 100644 index e7e2d0927f525b43f0349ba6b67f2274b971ee76..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa#@QB diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000033_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000033_action.npy deleted file mode 100644 index 3fa255b1c2bc1622532016d512ed5663c470b398..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtV1ihC_RN{RAl4{78V&=M N4nRq^G!aPX0RZ{t!Ts?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtV1ihC_RJYu5Nnhk4TphB M2cRTdng}HH0GV2ufB*mh diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000036_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000036_action.npy deleted file mode 100644 index 7c8fcfc3e36467407f735033f3046053a3a46cd8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^DytQ@6B!(kxP M0Vv6qCISgP0Q|(6fB*mh diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000037_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000037_action.npy deleted file mode 100644 index b5931a32db1828ad94d4331a87407ef9b3a30d55..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaCtc)$aDZo KvZaYYLJt572$_HY diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000038_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000038_action.npy deleted file mode 100644 index 9ee6cd72d22f8d652297a3233aae843455b24416..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmeH|F-yZh6vr>AF=Cw^oU$c_lF30tC{%DTn5y95Amx%=iP$FPk_f>Tx>OsutxxdF z_zmi3(p=+#5jO`1?+y2O|9kx5{dn+}?$z1(wSw-^gBg&-r{z=-s zv3{IR3SMyoQ?S_99m%~I(*Qy>)VOJ=e_A>h&gLV8O{Na^zq%cLb3@#%`=axqW}Rv_ zk*iATl7k>n-w(MbbsmC};3Bfr1%Wy*$*0akP!e23mbxHN=k-7MKzX5=Hj1-^{CMWo d^33lQ-Gz$K+iLa75y>n${N*0-mCbPZ*dI`T{%-&P diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000039_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000039_action.npy deleted file mode 100644 index 0fcbcf9672085c68454284c531e01fb3e1d58ab4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*+NS5bqtON5f$t N(*Y>SmL>uTJpiBfmr(!! diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000042_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000042_action.npy deleted file mode 100644 index 8c5c83c9621edd2cebc15b626566703dbc90d9e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaF9sn^lZqWb$ diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000043_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000043_action.npy deleted file mode 100644 index 2adbe6dc83d774b4be358ba663492ab2f1c0522d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV>CUC zrian=fE)s&=>a)3C@&wT0Bh(T*4+Hk;#92FJQJ{*m+ExtU}9i6d*+PoC>{-ifldRU KBwLyYB=i73YHra0 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000044_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000044_action.npy deleted file mode 100644 index 9adb526a17cc55cf37cc428cfd9179f000fe64dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpgqv>HZ zJs^j`XnH^njnVRf8tGvQu!ino&CM?@PQ_ZyGXbl4sZOU3CI*JHXU^;$#iL;`&}jga KWJ?o)gdPB}#=p@3 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000045_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000045_action.npy deleted file mode 100644 index 71a0e0328d53d2e67d08525c56f3a522b7e6b025..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaboS6e>6vnyTR7AmNf+iP$FPk_f>Tx~L7@mM8e3 z>Sxki;(-x22M6yB@7?|H@rV2I;1%}O@cdeq?xhFa$FWEBLrtHL^q!`VBa#plIuY?P zlTXnk#-$zyXokzX(d}sq?RT-g+<4JDQWbPJOsBKVbi9aQbDB<)TSN%TfbKX1dr=rC zB=r&~fT~p|(01t8Ed^N8Bb>-CJi{dzmfQ_mRP~c=hFQNdI)HptKeebl2ivyK-pi8zJIDpad#24cecU)`hkC1VM)?s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaF9sub@t$+Xk diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000053_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000053_action.npy deleted file mode 100644 index 453f150131c01ef3ec735f0a5f57b0182bd615d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaF9soaGt^fc4 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000056_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000056_action.npy deleted file mode 100644 index be2aa4c1be1b7d2740993ce4a526f41f2fcd4fff..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gV=$$M z!Q>WD7>&ATG_MTSyfSps=M-QK-NTxjUs{}swVG!FR`XJwP901P3}?@ru?2Z-lpYO- OflLRWBwLyYB=i7i`k8s?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaHpggEKvV zTs0augEMYM<7G5nAR$0(I7|W7&^@fV`K86FSgUy^U^Oq*>D0l*z;O1=8C#ICqx5Jv P3}iY0CE3zMAfX2Uf@qn5 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000059_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000059_action.npy deleted file mode 100644 index 40745ce3833ddf43899d8251217dda4ad6d65eb2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGaD0l*z;O1=8C#Iiqx5Jv3}iY0 LCE3zMAfX2Ug=Cq4 diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000060_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000060_action.npy deleted file mode 100644 index 8c808a87ef94fe7b6846fc69915e658deb33c4fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGas?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gW3Z(M zklO~En?~I;nr8-co*CTfa|*DA?qSW%FD*{RTFo;7t9hwTrw%3thO=kR*n$FjlpYO- OflLRWBwLyYB=i7uGns$@ diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/action/000063_action.npy b/examples/training/consistency_finetune/causal_wangame_ode_init/action/000063_action.npy deleted file mode 100644 index 4cd0c40eab41a204a9d9943087d8de28bc893553..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2994 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1Js?sp{MeV9+C$SDIT;sTZG|UzDnsTbfgnnOIbmSUIJKC%z~( zIX|zsq^LBxWXj|zJ*;4rQ+n9*QXonuPw{4Go#M=xG^Kq?&=d`CM$HW79@dnS%7Rpo zdM1C1DV-emrP0OjGa|rcadQ=<|9;4wf8V<-gW3Z(M znA--MyGGqJnr~of6GV@u2V&F16krYA![S] + Static -17. Switch [S]->[W] + Static -18. Switch [A]->[D] + Static -19. Switch [D]->[A] + Static -20. Switch [W]->[A] + Static -21. Switch [W]->[D] + Static -22. Switch [S]->[A] + Static -23. Switch [S]->[D] + Static -24. No Key + Switch [left]->[right] -25. No Key + Switch [right]->[left] -26. No Key + Switch [up]->[down] -27. No Key + Switch [down]->[up] -28. No Key + Switch [up_left]->[up_right] -29. No Key + Switch [up_right]->[up_left] -30. No Key + Switch [left]->[up] -31. No Key + Switch [right]->[down] -32. Hold [W] + Hold [left] -33. Hold [S] + Hold [left] -34. Hold [W] + Hold [right] -35. Hold [S] + Hold [right] -36. Hold [A] + Hold [up] -37. Hold [D] + Hold [up] -38. Hold [WA] + Hold [down] -39. Hold [WD] + Hold [down] -40. Hold [W] + Hold [up_left] -41. Hold [S] + Hold [up_left] -42. Hold [W] + Hold [up_right] -43. Hold [S] + Hold [up_right] -44. Hold [A] + Hold [down_left] -45. Hold [D] + Hold [down_right] -46. Hold [WA] + Hold [right] -47. Hold [WD] + Hold [left] -48. Hold [W] + Switch [left]->[right] -49. Hold [W] + Switch [right]->[left] -50. Hold [W] + Switch [up]->[down] -51. Hold [W] + Switch [down]->[up] -52. Hold [W] + Switch [left]->[up] -53. Hold [W] + Switch [right]->[up] -54. Hold [W] + Switch [left]->[down] -55. Hold [W] + Switch [right]->[down] -56. Switch [W]->[S] + Hold [up] -57. Switch [S]->[W] + Hold [up] -58. Switch [A]->[D] + Hold [up] -59. Switch [D]->[A] + Hold [up] -60. Switch [W]->[A] + Hold [up] -61. Switch [W]->[D] + Hold [up] -62. Switch [S]->[A] + Hold [up] -63. Switch [S]->[D] + Hold [up] From 449c009e8f4ae7f5309b30e9b38968c83d6af863 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 23:01:55 +0000 Subject: [PATCH 210/214] remove dev doc, remove other phases doc --- dev/config.md | 166 ----- dev/design.md | 690 ------------------ dev/distill_structure.md | 262 ------- dev/fastgen.md | 468 ------------ dev/phase_add_causal_wangame_dfsft.md | 326 --------- dev/phase_add_wangame.md | 365 --------- dev/phase_causal.md | 172 ----- dev/phase_causal2.md | 171 ----- dev/phase_mode.md | 238 ------ dev/phase_refactor.md | 676 ----------------- dev/phases/phase_0.md | 227 ------ dev/phases/phase_1.md | 110 --- dev/phases/phase_2.md | 226 ------ dev/phases/phase_2_9.md | 309 -------- dev/phases/phase_3.md | 263 ------- dev/refactor.md | 216 ------ .../distill_wan2.1_t2v_1.3B_dmd2_8steps.sh | 112 --- examples/distillation/phase0/temp.sh | 134 ---- .../distill_wan2.1_t2v_1.3B_dmd2_8steps.sh | 114 --- examples/distillation/phase1/run.md | 3 - examples/distillation/phase1/temp.sh | 136 ---- examples/distillation/phase2/README.md | 27 - .../distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml | 84 --- .../phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh | 29 - examples/distillation/phase2/temp.sh | 49 -- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml | 83 --- examples/distillation/phase2_9/temp.sh | 42 -- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml | 82 --- examples/distillation/phase3_1/temp.sh | 43 -- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml | 84 --- examples/distillation/phase3_2/temp.sh | 42 -- .../finetune_wan2.1_t2v_1.3B_phase3.3.yaml | 64 -- ...finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml | 76 -- examples/distillation/phase3_3/temp-vsa.sh | 43 -- examples/distillation/phase3_3/temp.sh | 42 -- .../distillation/phase3_4/distill-temp.sh | 42 -- ..._wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml | 83 --- .../phase3_4/finetune-vsa-temp.sh | 44 -- .../finetune_wan2.1_t2v_1.3B_phase3.4.yaml | 69 -- ...finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml | 76 -- ...2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml | 76 -- ...2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 76 -- .../dfsft_wangame2.1_i2v_1.3B_causal.yaml | 88 --- ...fsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm | 64 -- ...dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml | 88 --- ...ame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml | 115 --- ...4steps_causal_teacher_ckpt22000_nocfg.yaml | 115 --- ..._causal_teacher_ckpt22000_nocfg_4n8g.slurm | 64 -- ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 130 ---- ...wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml | 115 --- ..._causal_teacher_ckpt22000_nocfg_4n8g.slurm | 63 -- ...s_causal_teacher_ckpt22000_nocfg_4n8g.yaml | 139 ---- .../wangame/finetune_wangame2.1_i2v_1.3B.yaml | 82 --- ...ngame2.1_i2v_1.3B_bidirectional_4n8g.slurm | 64 -- ...angame2.1_i2v_1.3B_bidirectional_4n8g.yaml | 88 --- examples/distillation/wangame/run.sh | 42 -- .../dfsft_wangame_causal_v3.yaml | 0 .../distill_wan2.1_t2v_1.3B_dmd2.yaml | 0 .../refactor => train}/example.yaml | 0 ...2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 0 .../finetune_wangame2.1_i2v_1.3B.yaml | 0 .../{distillation/refactor => train}/rfc.md | 0 .../{distillation/refactor => train}/run.sh | 0 .../self_forcing_wangame_causal_v3.yaml | 0 64 files changed, 7897 deletions(-) delete mode 100644 dev/config.md delete mode 100644 dev/design.md delete mode 100644 dev/distill_structure.md delete mode 100644 dev/fastgen.md delete mode 100644 dev/phase_add_causal_wangame_dfsft.md delete mode 100644 dev/phase_add_wangame.md delete mode 100644 dev/phase_causal.md delete mode 100644 dev/phase_causal2.md delete mode 100644 dev/phase_mode.md delete mode 100644 dev/phase_refactor.md delete mode 100644 dev/phases/phase_0.md delete mode 100644 dev/phases/phase_1.md delete mode 100644 dev/phases/phase_2.md delete mode 100644 dev/phases/phase_2_9.md delete mode 100644 dev/phases/phase_3.md delete mode 100644 dev/refactor.md delete mode 100644 examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh delete mode 100644 examples/distillation/phase0/temp.sh delete mode 100644 examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh delete mode 100644 examples/distillation/phase1/run.md delete mode 100644 examples/distillation/phase1/temp.sh delete mode 100644 examples/distillation/phase2/README.md delete mode 100644 examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml delete mode 100644 examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh delete mode 100644 examples/distillation/phase2/temp.sh delete mode 100644 examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml delete mode 100644 examples/distillation/phase2_9/temp.sh delete mode 100644 examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml delete mode 100644 examples/distillation/phase3_1/temp.sh delete mode 100644 examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml delete mode 100644 examples/distillation/phase3_2/temp.sh delete mode 100644 examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml delete mode 100644 examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml delete mode 100644 examples/distillation/phase3_3/temp-vsa.sh delete mode 100644 examples/distillation/phase3_3/temp.sh delete mode 100644 examples/distillation/phase3_4/distill-temp.sh delete mode 100644 examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml delete mode 100644 examples/distillation/phase3_4/finetune-vsa-temp.sh delete mode 100644 examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml delete mode 100644 examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml delete mode 100644 examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml delete mode 100644 examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml delete mode 100644 examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml delete mode 100644 examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm delete mode 100644 examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml delete mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml delete mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml delete mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm delete mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml delete mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml delete mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm delete mode 100644 examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml delete mode 100644 examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml delete mode 100644 examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.slurm delete mode 100644 examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml delete mode 100644 examples/distillation/wangame/run.sh rename examples/{distillation/refactor => train}/dfsft_wangame_causal_v3.yaml (100%) rename examples/{distillation/refactor => train}/distill_wan2.1_t2v_1.3B_dmd2.yaml (100%) rename examples/{distillation/refactor => train}/example.yaml (100%) rename examples/{distillation/refactor => train}/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml (100%) rename examples/{distillation/refactor => train}/finetune_wangame2.1_i2v_1.3B.yaml (100%) rename examples/{distillation/refactor => train}/rfc.md (100%) rename examples/{distillation/refactor => train}/run.sh (100%) rename examples/{distillation/refactor => train}/self_forcing_wangame_causal_v3.yaml (100%) diff --git a/dev/config.md b/dev/config.md deleted file mode 100644 index d1e61d0cf..000000000 --- a/dev/config.md +++ /dev/null @@ -1,166 +0,0 @@ -# Distillation YAML config (schema v2) - -本文档描述当前 distillation 入口所使用的 **YAML schema v2**、字段含义与设计取舍。 - -相关实现： -- YAML loader：`fastvideo/distillation/utils/config.py::load_distill_run_config` -- Entrypoint：`fastvideo/training/distillation.py` -- Schema/类型定义：`fastvideo/distillation/utils/config.py` -- 示例 YAML（examples）：`examples/distillation/` - -## 1) 入口与约束（非常重要） - -distillation 入口 **只接受一个真实存在的 YAML 文件路径**（不 merge legacy CLI/configs， -也不做路径补全/overlay）。YAML 是 single source of truth。 - -运行方式（示意）： - -```bash -python fastvideo/training/distillation.py \ - --config /abs/path/to/examples/distillation//xxx.yaml -``` - -CLI 仅保留少量 **runtime override**（不属于“实验定义”的内容）： -- `--resume-from-checkpoint`：从 checkpoint 恢复 -- `--override-output-dir`：临时覆盖输出目录（方便重复跑实验） -- `--dry-run`：只 parse + build runtime，不启动训练 - -## 2) YAML 顶层结构（schema v2） - -```yaml -recipe: # 选择 family + method（只负责“选什么”） -roles: # role -> role spec（谁参与） -training: # infra 参数（直接映射到 TrainingArgs） -default_pipeline_config: # 默认 pipeline config（可 inline） -method_config: # method/algorithm 超参（方法侧 single source of truth） -# 或者 default_pipeline_config_path: /abs/path/to/pipeline_config.json|yaml -``` - -loader 规则： -- `pipeline_config` 与 `pipeline_config_path` **二选一**，不能同时提供。 -- `training` 会被传入 `TrainingArgs.from_kwargs(**training_kwargs)`；我们不重造一套训练参数体系。 -- 缺少 `recipe:` 会直接报错（schema v1 的 `distill:` 不再支持）。 - -## 3) `recipe`: 选择 family 与 method - -```yaml -recipe: - family: wan - method: dmd2 -``` - -用途： -- registry dispatch：选择 `models/.py` + `methods/.py` 的组合（N+M，而非 N×M）。 -- 语义更通用：未来把 finetuning 也纳入时不会出现 `distill.method=finetune` 的别扭表达。 - -## 4) `roles`: role-based 参与者 - -```yaml -roles: - student: - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true -``` - -字段含义（见 `fastvideo/distillation/utils/config.py`）： -- `family`：可选；默认继承 `recipe.family` -- `path`：模型路径 / hub 名称（由 family 负责加载） -- `trainable`：该 role 的参数是否参与训练（影响 `requires_grad`/train/eval） -- `disable_custom_init_weights`：可选；禁用 family 的 “加载时自定义 init weights 逻辑” - -设计原因： -- role 只是 key；framework 不强行规定 “canonical roles”。method 决定它需要哪些 roles。 -- `trainable` 表示训练意图；method 仍可施加算法不变量（例如 DMD2 强制 teacher frozen）。 - -## 5) `training`: 直接映射到 `TrainingArgs` - -`training:` 下的 key 基本上就是 `TrainingArgs` 字段（`fastvideo/fastvideo_args.py`），例如： -- 分布式：`num_gpus`, `sp_size`, `tp_size` -- 数据：`data_path`, `dataloader_num_workers`, shape/batch 相关字段 -- 输出：`output_dir`, `max_train_steps`, `seed`, `checkpoints_total_limit` -- 优化器默认值：`learning_rate`, `betas`, `lr_scheduler`, ... -- tracking/validation：`log_validation`, `validation_*`, `tracker_project_name`, ... - -loader 会注入/补全的 invariants（见 `fastvideo/distillation/utils/config.py`）： -- `mode = ExecutionMode.DISTILLATION` -- `inference_mode = False` -- `dit_precision` 默认 `fp32`（master weights） -- `dit_cpu_offload = False` -- 分布式尺寸默认值（`num_gpus/tp_size/sp_size/hsdp_*`） -- `training.model_path` 若缺失，默认使用 `roles.student.path`（供 pipeline_config registry 使用） - -关于 validation 参数的归属（当前约定）： -- `training.validation`：用于描述 validation（method 也会读取这一段） - - 固定字段（框架层会用到）： - - `enabled`（bool，可省略；有 section 默认启用） - - `dataset_file`（str） - - `every_steps`（int） - - 采样字段（method 按需读取并转成 `ValidationRequest`）： - - `sampling_steps`（list[int] / int / str） - - `guidance_scale`（float，可选） - - `sampler_kind`（ode|sde，可选） - - `sampling_timesteps`（list[int]，可选；DMD2/SDE few-step 才需要） - -备注： -- `DistillTrainer` 不再读取 `training.log_validation/validation_steps/...` 做调度； - trainer 每步调用 `method.log_validation(step)`，method 决定是否执行 validation。 - -## 6) `default_pipeline_config` / `default_pipeline_config_path` - -两种写法（二选一）： - -1) inline（适合少量 override）： - -```yaml -default_pipeline_config: - flow_shift: 8 -``` - -1) path（适合复用大型 config 文件）： - -```yaml -default_pipeline_config_path: /abs/path/to/wan_1.3B_t2v_pipeline.json -``` - -常见字段（非穷举）： -- `flow_shift`：Wan 的 flow-matching shift（影响 noise schedule）。 -- `sampler_kind`：`ode|sde`，选择 sampling loop 语义（`WanPipeline` 内部切换）。 - -备注（重要）： -- `default_pipeline_config` 是 “模型/pipeline 的默认 config”（例如 `flow_shift`、`vae_config`）。 - method/validator 的采样语义不应再依赖它；采样策略应由 method 通过 `ValidationRequest` - 显式传入。 - -## 7) `method_config`: method/algorithm 专属超参 - -`method_config` 由 method 自己解释。以 DMD2 为例： - -```yaml -method_config: - rollout_mode: simulate # {simulate|data_latent} - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] -``` - -其中： -- `rollout_mode` 替代 legacy 的 `training.simulate_generator_forward`： - - `simulate`：adapter 用零 latents 构造 batch（不依赖 `vae_latent`） - - `data_latent`：dataset batch 必须提供 `vae_latent` -- `dmd_denoising_steps` 是 method 的 few-step schedule single source of truth。 - -## 8) 最小可运行示例（Wan few-step DMD2） - -参考 `examples/distillation/` 下的可运行 YAML： -- `examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` -- `examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml` -- `examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` -- `examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml` diff --git a/dev/design.md b/dev/design.md deleted file mode 100644 index 111729900..000000000 --- a/dev/design.md +++ /dev/null @@ -1,690 +0,0 @@ -# Distill 重构设计（吸收 FastGen 架构）：`roles={...}` + Method/Trainer/Adapter 解耦 - -本文是基于： - -- FastVideo 当前 distill 实现：`dev/distill_structure.md` -- FastGen distillation pipeline 的优秀结构：`dev/fastgen_structure.md` - -做出的“面向落地”的重构设计草案。重点是把 **算法**（DMD2/Self-forcing/…） -与 **模型/管线**（Wan/其它架构）彻底解耦，并让训练循环（Trainer）保持长期稳定。 - ---- - -## 0. TL;DR（推荐最终形态） - -把 FastGen 的四层结构迁移到 FastVideo，并显式引入 `roles={...}`： - -- `DistillTrainer`：只做训练基础设施（循环、分布式、grad accum、logging、ckpt、validate） -- `DistillMethod`：一个“可训练对象”，封装 distill 算法 + 多角色模型 + 多优化器/交替更新 -- `DistillAdapter`：把具体 pipeline/network 适配成统一的 noise/forward/CFG/cache 接口 -- `RoleManager`：`roles={student, teacher, critic, ...}` 的统一容器（含 optim/ema/fsdp 策略） -- `ConditioningProvider`（或 dataset 常量注入）：显式提供 `neg_condition` 等 conditioning 常量 - -关键原则：**Trainer 不认识 teacher/critic，也不写 DMD/SF 的 if/else。** - ---- - -## 1. 现状与痛点（FastVideo） - -（细节见 `dev/distill_structure.md`，这里仅列架构层面的痛点） - -- **Wan 耦合**：normalize/layout/transformer override 等散落在训练代码里，换模型不可能仅靠配置 -- **算法分叉**：DMD2 与 Self-forcing 各自维护训练 loop，扩展新算法/新模型成本高且容易 drift -- **conditioning 隐式依赖**：`neg_condition/uncond` 可能依赖 validation 初始化等副作用 -- **多网络调度不稳**：交替更新时，scheduler step 与 optimizer step 不严格对齐（会引入 lr 偏差） -- **专家/双 transformer 逻辑分散**：MoE/expert 选择与“哪个 timestep 更新哪个 expert”缺乏单点抽象 - ---- - -## 2. FastGen 架构要点（我们要吸收什么） - -### 2.1 FastGen 的四层分离（核心） - -FastGen 把 distillation 的复杂度拆成： - -1. `Trainer`：训练循环与分布式/accum/validate/ckpt/callbacks -2. `Method(FastGenModel)`：算法 + 多网络容器 + 多优化器调度（交替更新在这里） -3. `Network`：统一 forward contract（pred_type/features/cache），并提供少量 hook -4. `Dataset/Preprocess`：提供 `real/condition/neg_condition`，并支持常量注入 - -这种结构的长期价值是：**训练循环不随算法增长而膨胀**，算法复用与组合能力强。 - -### 2.2 FastGen → FastVideo 对照表（建议直接照搬） - -| FastGen 概念 | FastVideo 目标概念 | 说明 | -|---|---|---| -| `Trainer.run(model)` | `DistillTrainer.run(method)` | Trainer 只依赖统一接口 | -| `FastGenModel.single_train_step()` | `DistillMethod.single_train_step()` | 返回 `loss_map["total_loss"]` | -| `get_optimizers()/get_lr_schedulers()` | 同名接口 | 交替更新/多 optim 的唯一入口 | -| `model_dict/optimizer_dict/scheduler_dict` | 同名映射 | checkpoint 以 role 命名空间泛化 | -| `DDPWrapper.single_train_step()` | `DDPTrainStepWrapper`（可选） | 让训练 step 吃到 DDP hooks | -| dataset `files_map/presets_map` | dataset 常量注入（推荐） | `neg_condition` 不再隐式依赖 | -| callbacks + checkpointer + autoresume | 回调 + checkpoint + resume | 基础设施通用化，算法不介入 | -| meta-init + rank0 broadcast（FSDP2） | 可选的大模型加载策略 | teacher/critic ≥10B 时显著收益 | - -### 2.3 我们希望复制的“具体模式”（不止抽象名词） - -- **统一训练入口**：Trainer 每个（micro）step 都只做： - - `loss_map, outputs = method_ddp.single_train_step(batch, iter)` - - backward 只对 `loss_map["total_loss"]` - - 在 accum 最后一步调用： - - `method.optimizers_schedulers_step(iter)` - - `method.optimizers_zero_grad(iter)` -- **交替更新收敛到 method**：更新 student/critic 的比例完全由 - `get_optimizers(iter)` 决定，Trainer 永不写 role-aware 的分支。 -- **conditioning 显式化**：`neg_condition` 最好是 dataset 常量（或启动时一次性缓存/广播）， - 绝不依赖 validation 副作用。 -- **role 命名空间 checkpoint**：把保存/加载做成 “按 role key 的映射”，未来加模型不会改协议。 - ---- - -## 3. 总体架构（FastVideo 版本） - -### 3.1 一次训练的总数据流（推荐） - -```text -CLI/YAML config - -> build RoleManager(roles={student, teacher, critic?, ...}) - -> build DistillAdapter.from_pipelines(bundle) # pipeline/network 适配 - -> build DistillMethod(adapter, bundle, method_cfg) - -> DistillTrainer(trainer_cfg, callbacks, checkpointer).run(method) -``` - -### 3.2 分层职责（把边界画清楚） - -1. **Data/Conditioning 层** - - dataloader 输出：`real`、`condition`、`neg_condition`（可选）以及 I2V/V2V 的额外条件 - - `ConditioningProvider`：若 dataloader 不提供 `neg_condition`，则构建并缓存/广播 - -2. **Adapter/Network 层（模型相关）** - - `DistillAdapter`：layout/normalize/noise schedule/CFG/forward/(可选)cache - - 每个架构一个 adapter：`WanAdapter`、`HunyuanAdapter`、`LTX2Adapter`… - -3. **Method 层（算法相关 + 多网络训练）** - - `DistillMethod` 基类（FastGenModel analog） - - `DMD2Method` / `SelfForcingMethod` /（未来）`TeacherOnlyMethod` 等 - -4. **Trainer/Engine 层（基础设施）** - - `DistillTrainer.run(method)`：DDP/FSDP、grad accum、日志、验证、断点、回调 - - Trainer 永不写 DMD/SF 专有逻辑 - ---- - -## 4. 核心对象与接口（建议 API） - -### 4.1 `RoleManager`：角色显式化（外部输入） - -目标：让入口层显式传入 `roles={student, teacher, critic, ...}`，并把所有 -“训练态（optim/ema/fsdp 策略）”结构化地挂在 role 下。 - -```text -RoleManager - roles: dict[str, RoleHandle] # key == "student"/"teacher"/"critic"/... - -RoleHandle - modules: dict[str, nn.Module] # e.g. {"transformer": ..., "transformer_2": ...} - trainable: bool - precision: optional # bf16/fp16/fp32 - fsdp_policy: optional # shard strategy / ignored modules - ema: optional - optimizers/schedulers: optional -``` - -约定： - -- `role` 只是一个字符串 key；Trainer/Checkpoint 对所有 role **一视同仁**（不做“主次”区分）。 -- 为了可读性，推荐使用一些常见命名（非强制）： - `student`, `teacher`, `critic`, `discriminator`, `reward`, `refiner`, `aux_teacher`, ... -- 每个 `DistillMethod` 应显式声明并在初始化时校验自己需要的 roles - （例如 DMD2 需要 `student+teacher+critic`，teacher-only 只需要 `student+teacher`）。 - -### 4.2 `DistillAdapter`：把 pipeline/network 适配成算法可消费接口 - -adapter 的职责是“怎么调用模型”，而不是“什么时候更新谁”。建议接口包含： - -- noise & target： - - `add_noise(x0, noise, t) -> x_t` - - `pred_to_x0(pred, x_t, t)`（或统一为 `pred_to_target`） -- forward： - - `forward(role, x_t, t, cond, *, fwd_pred_type=..., neg_cond=None, cfg=None, caches=None)` -- layout & normalize（按模型需要）： - - `to_model_layout(x)` / `from_model_layout(x)` - - `normalize_latents` / `denormalize_latents` -- conditioning： - - `encode_condition(raw_cond) -> cond` - - `encode_neg_condition(raw_neg_cond) -> neg_cond`（或由 dataset 提供 embedding） -- cache（可选，Self-forcing 用）： - - `supports_kv_cache` - - `clear_caches(role=...)` - -此外建议 adapter 暴露 capabilities，避免 method 靠 if/else 猜： - -```text -adapter.capabilities = { - "supports_cfg": True/False, - "supports_kv_cache": True/False, - "supported_pred_types": {...}, - "supports_features": True/False, - "supports_expert_routing": ..., -} -``` - -### 4.3 `DistillMethod`：算法 + 多网络 + 多优化器调度（核心） - -这是 FastGen 最值得抄的点：把 distill 的关键复杂度集中在 method。 - -**最小接口（强制）** - -- `single_train_step(batch, iteration) -> (loss_map, outputs)` - - `loss_map` 必须包含 `total_loss` - - `outputs` 仅用于日志/验证/可视化 -- `get_optimizers(iteration)` / `get_lr_schedulers(iteration)` - - 返回本次 iteration 应该 step 的 optimizer/scheduler 列表（交替更新就在这里实现） -- `optimizers_schedulers_step(iteration)` / `optimizers_zero_grad(iteration)` - - Trainer 只调用它们，不关心内部有哪些 optimizer -- `model_dict/optimizer_dict/scheduler_dict` - - 给 CheckpointManager 使用（key == role 或 role 内模块） - -**建议能力（可选但推荐）** - -- `autocast()` / `grad_scaler`：统一 AMP 管理，Trainer 不关心精度细节 -- `sample_for_logging(...)`：返回可调用的采样函数或采样结果，Trainer 不写采样逻辑 -- `set_trainable(role, enabled)`：method 内部统一 `requires_grad_` 切换（Self-forcing/critic alternation） - -> 直接收益：scheduler step 的粒度天然与 optimizer step 对齐，避免 update ratio 引入 lr 偏差。 - -### 4.4 `DistillTrainer`：完全算法无关的训练循环 - -Trainer 只依赖 method 的统一接口，推荐对齐 FastGen 的关键形态： - -- grad accumulation：Trainer 计算 `sync_grads`，并在 DDP/FSDP 下用 context 禁止同步 -- forward/backward：只围绕 `loss_map["total_loss"]` -- step/zero_grad：只在 accum 最后一次调用 method 接口 -- validate：可复用 `single_train_step`（no_grad + autocast），并允许 method 扩展额外 eval -- callbacks：把 EMA / grad clip / logger / profiler 等都做成回调（可保存状态） - -**DDP 的一个关键实现点（强烈建议照 FastGen）** - -如果 `single_train_step` 不是 `forward()`，DDP 的隐式 hooks 可能不生效。 -FastGen 用 `DDPWrapper` 临时把 `module.forward` 指到 `single_train_step`， -然后通过 `ddp_model(*args)` 触发 hooks。 - -在 FastVideo 落地时建议二选一： - -1) `DistillMethod.forward = single_train_step`（简单，但 forward 被占用） -2) 实现一个 `DDPTrainStepWrapper.single_train_step()`（推荐，行为更明确） - -### 4.5 `CheckpointManager`：围绕 role 命名空间泛化 - -建议统一协议： - -- 输入：`model_dict/optimizer_dict/scheduler_dict/grad_scaler/callback_state/iteration` -- 输出：按 role key 保存（尤其是 FSDP sharded state） - -并显式支持： - -- **只导出 student** 的 inference 权重（teacher/critic 不随推理包分发） -- **兼容旧 distill ckpt**：至少提供一次性迁移脚本或兼容 loader - -### 4.6 `Callback` 系统：把“训练周边复杂度”解耦出去 - -把这些都做成 callbacks（并进入 checkpoint），Trainer/Method 都不硬编码： - -- EMA 更新 -- grad clipping -- logging（wandb/tensorboard/本地） -- profiler/step timer -- param count / debug dumps -- autoresume（从 ckpt 恢复 callback 状态） - ---- - -## 5. 关键设计决策（每条含原因） - -### 设计 1：引入 `DistillMethod`（Method 中心，而非 Algorithm/Trainer 中心） - -**设计** - -- DMD2/Self-forcing/未来算法都实现为 `DistillMethod` 子类 -- Method 负责多角色模型、交替更新、optim/sched/EMA、缓存生命周期等 - -**原因** - -- distill 的“本质复杂度”就是多网络 + 多优化器调度；放在 Method 最自然 -- Trainer 只需要稳定地做基础设施，长期维护成本最低 - -### 设计 2：`roles={...}` 显式输入 + `RoleManager` 结构化承载训练态 - -**设计** - -- 配置/CLI 显式给出 `student/teacher/critic?` -- `RoleManager` 统一挂载冻结策略、precision、FSDP 策略、EMA、optim/sched - -**原因** - -- 角色显式化是解耦的前提，且天然支持未来扩展更多角色 -- checkpoint 也可以自然以 role 命名空间组织 - -### 设计 3：Trainer 固化成“只认一个接口”的稳定循环 - -**设计** - -- Trainer 仅调用： - - `loss_map, outputs = method_ddp.single_train_step(...)` - - backward `total_loss` - - `method.optimizers_schedulers_step()` / `optimizers_zero_grad()` -- validate 也尽可能复用 `single_train_step` - -**原因** - -- 彻底杜绝算法越写越多导致 trainer 分叉、难以测试和维护 -- validate 复用训练逻辑能减少“训练/验证 drift” - -### 设计 4：交替更新/多 optimizer 调度统一走 `get_optimizers()`（FastGen 关键模式） - -**设计** - -- DMD2： - - `iter % student_update_freq == 0`：更新 student - - 否则更新 critic（+ 可选 discriminator） -- Self-forcing： - - 复用相同调度，只替换 student rollout - -**原因** - -- update ratio 的复杂度从 trainer 中消失，且扩展更多 role 不改 trainer -- scheduler step 与 optimizer step 自动对齐（减少 lr 偏差） - -### 设计 5：adapter 明确 capability，而不是靠 if/else 猜 - -**设计** - -- adapter 暴露 `capabilities`，method 启动时检查依赖 -- 自适应训练：method 根据 capability 选择路径或报错/降级 - -**原因** - -- 新增模型架构时，差异集中在 adapter；method 保持稳定 -- 失败要早失败（init-time），避免跑到中途才出形状/feature 错 - -### 设计 6：`neg_condition` 变成“数据常量”或“启动一次性缓存/广播” - -**设计** - -两条路径（二选一或同时支持）： - -1) dataset 常量注入：dataloader 直接输出 `neg_condition` embedding -2) provider 缓存：训练开始时用 adapter 编码 negative prompt，并缓存/广播 - -**原因** - -- 消除 “依赖 validation 初始化 uncond embedding” 这类隐式耦合 -- negative prompt embedding 通常是常量，适合缓存（性能更稳） - -### 设计 7：用 role 命名空间做 checkpoint 协议（对齐 `model_dict`） - -**设计** - -- `model_dict/optimizer_dict/scheduler_dict` 的 key 直接是 role -- FSDP 情况下按 role key 分 shard 保存；非 FSDP rank0 写单文件 - -**原因** - -- 多网络 distill 的保存/加载本质就是 “多个命名空间的 state” -- 未来新增/删减 role 不改变 checkpoint 顶层协议 - -### 设计 8：DDP 训练 step 触发 hooks（借鉴 FastGen 的 wrapper 技巧） - -**设计** - -- 为 DDP 场景提供 `DDPTrainStepWrapper.single_train_step()`： - 临时重定向 `forward` 到 `single_train_step`，再调用 `ddp_model(...)` - -**原因** - -- 让 “训练 step != forward” 的结构仍能享受 DDP 的正确行为与 hooks -- 避免强行把算法逻辑写进 `forward` 导致语义混乱 - -### 设计 9（可选）：meta-init + rank0 broadcast 的大模型加载 - -**设计** - -- teacher/critic ≥10B 时，非 rank0 以 `torch.device("meta")` 构建空权重 -- rank0 加载权重，FSDP wrap 后 broadcast - -**原因** - -- 显著减少多机启动 I/O contention 与峰值内存 - ---- - -## 6. 配置与 CLI 形态（渐进式） - -> Phase 2 开始，新的 distillation 入口 **不再兼容旧式 CLI 传参**。 -> 我们只接受新的结构化配置（YAML），让一次运行可读、可复现、可审查。 - -### 6.1 最小可用（建议先落地） - -**Phase 2+ 目标**：一个 YAML 配置文件描述一次运行（distill/finetune/…），入口只需要： - -- `fastvideo/training/distillation.py --config path/to/run.yaml` - -除此之外的训练参数/模型选择/方法选择，都写入 YAML。 - -### 6.2 复杂配置（建议支持） - -- `--roles_json path/to/roles.json` - - per-role precision/offload/trainable/fsdp_policy/ckpt_path 等 - -### 6.3 YAML schema v2（Phase 3）：`recipe` + `method_config` - -说明： -- Phase 2 的 YAML schema v1 使用 `distill:` 顶层（历史原因） -- Phase 3.1 已升级为 schema v2：用 `recipe:` 顶层，并引入 `method_config:`（语义更通用） - - 入口只接受 schema v2（不再兼容 `distill:`） - -schema v2 的 “单次运行” 配置示意（字段可迭代）： - -```yaml -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true - -training: - output_dir: outputs/... - max_train_steps: 4000 - seed: 1000 - # ... (TrainingArgs/FastVideoArgs 的字段) - -pipeline_config: - # 支持直接内联覆盖，也支持只给 pipeline_config_path - # pipeline_config_path: fastvideo/configs/wan_1.3B_t2v_pipeline.json - flow_shift: 8 - -method_config: - # method-specific 超参（不进入 TrainingArgs；由 method 自行解析） - rollout_mode: simulate - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] - generator_update_interval: 5 - real_score_guidance_scale: 3.5 -``` - -**解析策略（最优雅且低风险）** - -- 新入口的 parser 只保留 `--config run.yaml`（以及少量 meta flags，如 `--dry-run`）。 -- 训练相关的所有参数（TrainingArgs/FastVideoArgs/pipeline_config/method/roles）都来自 YAML。 -- 解析流程： - 1) `yaml.safe_load` 得到 dict - 2) 规范化/校验 schema（recipe/roles/training/pipeline_config/method_config/...） - 3) 将 `training:` 与 `pipeline_config:` 合成 kwargs，调用 `TrainingArgs.from_kwargs(**kwargs)` - （由现有 PipelineConfig/PreprocessConfig 负责子配置实例化与校验） - -这样不需要推翻现有 TrainingArgs/FastVideoArgs 体系，但从入口层面彻底摒弃旧式 CLI 传参方式。 - -### 6.4 YAML configs in `examples/`（Phase 3.4+） - -我们不再使用 `outside/` overlay。为了避免修改主仓库的 `fastvideo/configs/` 树，同时让配置更直觉可运行， -distillation 的 runnable YAML 统一放在： - -- `examples/distillation//*.yaml` - -并约定： -- distillation 入口 **不做任何自动补全/overlay 重写**： - - 用户传入的 `--config` 必须是一个真实存在的文件路径（通常位于 `examples/distillation/` 下） - - config 内引用的其它路径（如 `pipeline_config_path`）也必须是 **真实路径** - -### 6.5 配置系统演进（可选吸收 FastGen 的优点） - -FastGen 的 python config + instantiate + override 很优秀，但 FastVideo 现阶段可以先： - -- 保留现有 YAML/argparse -- 内部把配置整理成结构化 dataclass： - - `TrainerConfig / DataConfig / MethodConfig / ModelsConfig` -- 后续再逐步引入“延迟实例化/override”能力（不阻塞 distill 重构） - ---- - -## 7. 目录落地建议（FastVideo 内） - -Phase 0 的实践表明：先把新框架以 **additive** 方式落地到一个独立目录最稳妥。 -目前已选择并落地在 `fastvideo/distillation/`（建议继续沿用该路径，避免再迁一次目录）。 - -建议结构（已部分实现）： - -- `fastvideo/distillation/roles.py`：`RoleManager/RoleHandle` -- `fastvideo/distillation/adapters/`：`WanAdapter`（Phase 1 已落地；后续新增更多 adapter） -- `fastvideo/distillation/methods/`：`base.py`、`distribution_matching/dmd2.py`、（目标）`self_forcing.py` -- `fastvideo/distillation/trainer.py`：`DistillTrainer` -- `fastvideo/distillation/dispatch.py`：把 “config -> model components -> method -> runtime” 的胶水集中起来 -- `fastvideo/training/distillation.py`：通用入口（YAML-only：`--config path/to/run.yaml`） -- `fastvideo/distillation/utils/checkpoint.py`：role-based `CheckpointManager` -- （后续）`fastvideo/distillation/callbacks/`：EMA/clip/log/profiler 等 - -旧入口（如 `fastvideo/training/*distillation_pipeline.py`）先保留， -通过 flag 切新旧框架做 A/B 对齐。 - ---- - -## 8. 迁移计划（低风险） - -### Phase 0（已完成）：框架落地 + Wan(DMD2) 跑通（过渡实现） - -Phase 0 的定位在实践中更明确了：它是“**把旧 Wan distill pipeline 包一层新框架壳**”， -先把训练循环/多 optimizer 调度/validation hook 等基础设施固定下来，再逐步解耦。 - -- ✅ 新增 `DistillTrainer/DistillMethod/RoleManager` 的骨架，并跑通 WAN distill -- ✅ 用单测锁定关键语义：scheduler step 与 optimizer step 对齐 - - `generator_update_interval > 1` 时不会“空 step scheduler” -- ✅ 为后续解耦铺路：把 “roles={student,teacher,critic}” 显式化到 bundle - -Phase 0 明确没有做（刻意延期）： - -- ❌ v2 path 的 checkpoint/save/resume（role-based） -- ❌ `DMD2Method` 的真正算法解耦（目前仍调用旧 pipeline 内部函数） -- ❌ Self-forcing v2 迁移 - -### Phase 1（已完成）：算法与模型真正解耦（先把 DMD2 “抠出来”） - -Phase 1 的核心目标：把 Phase 0 的“脚手架耦合”逐步替换为 **Method(算法) + Adapter(模型)** -的稳定边界，让其它模型/其它方法可以复用 Trainer。 - -Phase 1 的“辉煌”（落地与收益）： - -- ✅ 通用算法 method：`fastvideo/distillation/methods/distribution_matching/dmd2.py::DMD2Method` - - 算法层不再调用 legacy pipeline 私有算法函数 - - 依赖面缩到 adapter primitives（通过 `Protocol` 约束 surface） -- ✅ 真正的 WAN 适配层：`fastvideo/distillation/adapters/wan.py::WanAdapter` - - `forward_context` 与 backward 重算约束收敛到 adapter（method 只实现算法） - - `ensure_negative_conditioning()` 显式化（不再依赖 validation 的隐式副作用） -- ✅ Dispatch 入口：`fastvideo/distillation/dispatch.py` - - 把 “recipe/roles -> model components -> method -> runtime” 的胶水集中在一处，便于扩展新 method/new model -- ✅ 通用入口：`fastvideo/training/distillation.py` - - Phase 1 仍是 CLI 选择：`--distill-model` + `--distill-method` - - Phase 2 起将切换为 **YAML-only**（见第 6 节），并逐步废弃这套 CLI -- ✅ 训练效果对齐：Phase 1 跑出来的 WAN DMD2 与 Phase 0/baseline 行为一致（已实测） - -### Phase 2（已完成）：彻底脱离 legacy distill pipeline（让新框架可独立存在） - -你提的建议我同意：Phase 2 应该把 Phase 1 仍然残留的 legacy 依赖清干净，让新的 distill -代码路径可以 **不依赖** `fastvideo/training/*distillation_pipeline.py` 和 -`WanDistillationPipeline` 仍可运行训练与验证。 - -为了降低风险，建议 Phase 2 按 “先 validation、再 builder/runtime、最后清理入口” 的顺序推进。 - -#### Phase 2.1：Validation 独立化（优先级最高，收益最大） - -- 目标：`WanAdapter.log_validation()` 不再调用 legacy `pipeline._log_validation(...)` -- 建议实现： - - 新增 `fastvideo/distillation/validation/`（或 `fastvideo/distillation/validators/`） - - 由 adapter 提供 `build_validator(...)` 或直接实现 `adapter.sample(...)` - - 复用模块化 inference pipeline（例如 `fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py`） - 来生成视频并交给 tracker 记录 -- 收益：彻底消除 “validation 初始化副作用/属性缺失” 这类隐式耦合与脆弱点 - -#### Phase 2.2：Builder/Runtime 脱离 pipeline（roles/spec -> instantiate） - -- 目标：`fastvideo/training/distillation.py` 不再先 instantiate `WanDistillationPipeline` -- 建议实现： - - 定义结构化 spec：`RoleSpec/ModelSpec`（role -> {family, path, precision, trainable,...}） - - 配置形态落地（Phase 2 必做）： - - `--config path/to/run.yaml`（YAML 为 single source of truth；CLI 仅指定配置路径） - - runnable YAML：放在 `examples/distillation//*.yaml`，入口只接受真实路径（不做 overlay 寻路） - - （可选）保留 `--models_json` 作为“程序生成配置”的接口 - - builder 根据 spec： - - 加载 modules（student/teacher/critic） - - 构建 role-based optimizers/schedulers - - 组装 `RoleManager + Adapter + Method` - - 构建 dataloader（直接复用 dataset 代码，不经由 legacy pipeline class） - - 不新增入口文件：直接增强 `fastvideo/training/distillation.py`，并把它定义为 **YAML-only distill entrypoint** - - 仅支持 `--config run.yaml`（以及少量 meta flags），不再兼容旧式 CLI configs - - legacy distill 继续通过原有 `fastvideo/training/*distillation_pipeline.py` 入口运行（两套路径并存） -- 收益：distill 路径具备真正的“模型/算法 catalog + instantiate”，开始能支持更多模型家族 - -#### Phase 2.3：role-based checkpoint/save/resume（新框架自洽） - -- 目标：新框架训练可 save/resume，且协议围绕 role 命名空间（不再绑死 WAN pipeline） -- 建议实现： - - `fastvideo/distillation/utils/checkpoint.py`：保存/加载 modules + optimizers + schedulers + RNG states - - 明确兼容策略：兼容旧格式（若必要）或提供一次性转换脚本 - -#### Phase 2.4（Deferred）：收敛与清理（暂不做；完全解耦后手动处理） - -本轮 Phase 2 采用 **非侵入式** 策略：只新增新路径所需的代码，不做 legacy 代码搬家/清理。 -当 Phase 2.1/2.2/2.3 全部完成、并且新框架可以独立运行后，再由你手动清理旧入口/旧实现。 - -在 Phase 1 的稳定边界之上，Phase 2 再做“功能扩展 + 旧实现收敛”： - -- Self-forcing v2：`SelfForcingMethod(DMD2Method)`（只覆写 student rollout / cache 生命周期） - - 并把 ODE-init（若需要）归类为 **student 初始化策略**（builder/config 层），而不是 Trainer 特例 -- role-based checkpoint/save/resume（v2 path） -- 新增更多 adapter（Hunyuan/LTX2/LongCat…） -- 新增更多 method（teacher-only、多 teacher、KD 轨迹蒸馏等） -- 逐步冻结或移除旧 distill pipeline（保留兼容入口亦可） - -### Phase 2.9（已完成）：A+B+Families 语义收敛（为 Phase 3 铺路） - -动机：Phase 2 已经完成“独立可跑”，但仍存在几个会阻碍长期扩展的结构性问题： - -- adapter API 仍有一定 **role-centric / method-specific** 倾向（例如 `sample_dmd_timestep()`、`teacher/critic` 分叉函数） -- entrypoint/builder 的组合 dispatch 未来如果扩展，会走向 N×M（写大量组合函数或 if/else） - -因此引入一个 **Phase 2.9**，先把语义边界收敛好，再开始 Phase 3（配置 schema 升级 + finetune）。 - -Phase 2.9 目标（简称 A+B+Families）： - -- A) 把 adapter API 收敛为 **operation-centric**（role 只是 key，不暴露 teacher/critic/student 专用函数） -- B) 把 “timestep sampling policy” 等 **算法策略** 挪回 method（adapter 只保留 scheduler 语义转换） -- Families) 引入 family registry + method registry，完成 **优雅 dispatch（N+M）** - - 这一步做完后，Phase 3 不再需要改 entrypoint/builder 的 dispatch 逻辑 - -详细执行清单见：`dev/phases/phase_2_9.md` - -补充：Phase 2.9 实践过程中暴露了一个之前设计阶段没充分考虑的点—— -**distillation/sampling method 可能不仅仅改变 timesteps/scheduler，而是会改变 denoising loop 本身** -（例如 DMD2 的 SDE 风格 `pred_x0 -> add_noise(next_t, eps)` 与常规 ODE/solver 风格 `scheduler.step(...)`）。 -这会直接导致 validation sampling 的 drift：即使训练一致，选不同 loop 的 pipeline 也可能出不同视频。 - -Phase 2.9 为了端到端与 legacy apples-to-apples，对 Wan DMD2 的 validation 暂时使用 -`WanDMDPipeline`（SDE rollout）以避免漂移；Phase 3.2 已将其升级为可插拔的 ODE/SDE sampler： -- `WanPipeline` 通过 `pipeline_config.sampler_kind={ode|sde}` 选择 sampling loop -- distillation validation 由 method 通过 `ValidationRequest` 显式指定 sampler + timesteps -- `WanDMDPipeline` 退化为兼容 wrapper（新框架不再依赖） - -### Phase 3：3.1 Config schema v2 + 3.2 ODE/SDE sampler + 3.3 Finetuning - -Phase 3 的定位：在 Phase 2.9 已经完成“优雅 dispatch + adapter/method 语义收敛”的基础上： - -#### Phase 3.1：配置语义升级（`distill` -> `recipe`，引入 `method_config`） - -动机： -- `distill.method=finetune` 语义别扭，因为 finetune 是一种训练 recipe，不一定是“蒸馏”。 -- method-specific 参数长期塞进 `training:`（TrainingArgs）/`pipeline_config:` 会让配置语义越来越混杂。 -- Phase 2.9 暴露过 “method knob 泄漏到 adapter” 的问题（例如 `simulate_generator_forward`），因此需要引入 - `method_config` 来做干净的边界收敛（Phase 3.1 已解决该耦合）。 - -Phase 3.1 计划把 YAML schema 升级为： - -```yaml -recipe: {family: wan, method: dmd2} # 只负责 “选什么” -roles: {student: ..., teacher: ...} # 参与者 -training: {...} # infra 参数（映射到 TrainingArgs） -pipeline_config: {...} # backbone/pipeline config（模型侧） -method_config: {...} # algorithm/method 超参（方法侧） -``` - -迁移策略（建议）： -- DMD2：把 `generator_update_interval`, `real_score_guidance_scale`, - `dmd_denoising_steps`, `rollout_mode`（替代 `simulate_generator_forward`）迁移到 `method_config`。 -- `training:` 保持纯 infra（分布式、优化器默认值、ckpt、logging、数据路径等）。 - -#### Phase 3.2：统一 sampling 语义（ODE/SDE sampler 可插拔） - -背景：DMD2/Consistency 等方法可能需要不同的 denoising loop（不仅是 timesteps/scheduler），如果靠 -“每个 method 一个 pipeline 变体（例如 `WanDMDPipeline`）”会重新回到 N×M 的组合爆炸。 - -状态：**已完成**（`WanPipeline` 支持 `pipeline_config.sampler_kind={ode|sde}`，distillation validation -通过 `ValidationRequest` 显式传入 sampler + timesteps，`WanDMDPipeline` 仅作为兼容 wrapper 保留）。 - -目标： -- `WanPipeline`（以及未来其它 family）通过参数选择 sampler/integrator（`ode|sde`）。 -- validation 由 method/method_config 显式指定 sampler + steps/timesteps；validator 只做 dataset+logging。 -- 新框架不再依赖 `WanDMDPipeline`（可保留 legacy 兼容）。 - -#### Phase 3.3：把 finetuning 作为一种 method 接入框架 - -目标：把 finetune 作为一种 method（only `student` + dataset）接入同一套 -`RoleManager + Adapter + Method + Trainer + (Validator/Checkpoint)` 基础设施，并让其配置语义与 -Phase 3.1 的 `recipe/method_config` 对齐。 - -状态：**已完成**（`FineTuneMethod` + Phase 3.3 示例 YAML + one-shot 脚本）。 - -建议落地形态（Phase 3 落地到代码时）： -- 新增 method：`fastvideo/distillation/methods/fine_tuning/finetune.py::FineTuneMethod` - - `bundle.require_roles(["student"])` - - update policy 永远只 step student 的 optimizer/scheduler -- 为 finetune 定义 adapter contract（类似 `_DMD2Adapter` 的做法）： - - `_FineTuneAdapter(Protocol)`：`prepare_batch()` + `predict_*()` + `backward()` - - **不建议把 `compute_loss()` 放进 adapter**：loss/policy 属于 method（参考 FastGen 的 `SFTModel.single_train_step()`） - - Wan 侧由 `WanAdapter` 实现该 contract；若未来出现 family-specific 分支，优先在 adapter 内部消化而不是膨胀 method - -Finetune 的 config（示意，schema v2）： - -```yaml -recipe: {family: wan, method: finetune} -roles: - student: - family: wan - path: ... - trainable: true -training: {...} -pipeline_config: {...} -method_config: - pred_type: x0 - loss: flow_matching -``` - ---- - -## 9. Guardrails / 测试建议（避免重构“跑得通但不可维护”） - -- **scheduler step 对齐测试**：交替更新下，未 step 的 optimizer 对应 scheduler 不应 step -- **batch_size > 1**：消除所有隐式 `B==1` 的 reshape/unflatten 假设 -- **role 可选性**：critic 可选时应有清晰报错/降级路径（teacher-only） -- **conditioning 显式性**：训练开始前必须具备 `neg_condition`（来自数据或 provider） -- **checkpoint roundtrip**：save → load → loss 不发散（最小 smoke test） diff --git a/dev/distill_structure.md b/dev/distill_structure.md deleted file mode 100644 index d5195fbb9..000000000 --- a/dev/distill_structure.md +++ /dev/null @@ -1,262 +0,0 @@ -# FastVideo Distill 训练结构梳理（现状） - -本文是对当前仓库 distill 训练代码的“结构/逻辑”阅读笔记，目标是把： - -- 入口在哪里 -- 训练时有哪些模型（student/teacher/critic） -- 每一步在算什么 loss、更新谁 -- Self-forcing 相对 DMD 的差异 -- 目前实现的耦合点/限制 - -写清楚，方便后续重构和对齐实现细节。 - -## 代码入口与文件分布 - -- 训练主逻辑 - - `fastvideo/training/distillation_pipeline.py`：`DistillationPipeline` - （DMD/DMD2 风格 distillation） - - `fastvideo/training/self_forcing_distillation_pipeline.py`： - `SelfForcingDistillationPipeline`（Self-forcing distillation） -- Wan 封装/可运行入口（torchrun 直接跑这些文件） - - `fastvideo/training/wan_distillation_pipeline.py`： - `WanDistillationPipeline` - - `fastvideo/training/wan_self_forcing_distillation_pipeline.py`： - `WanSelfForcingDistillationPipeline` - - `fastvideo/training/wan_i2v_distillation_pipeline.py`： - `WanI2VDistillationPipeline` -- 推理/验证用 pipeline（用于 `_log_validation`） - - `fastvideo/pipelines/basic/wan/wan_dmd_pipeline.py`：`WanDMDPipeline` - - `fastvideo/pipelines/basic/wan/wan_causal_dmd_pipeline.py`： - `WanCausalDMDPipeline` - - I2V：`fastvideo/pipelines/basic/wan/wan_i2v_dmd_pipeline.py` -- 断点续训/保存 - - `fastvideo/training/training_utils.py`： - `save_distillation_checkpoint` / `load_distillation_checkpoint` -- 测试 - - `fastvideo/tests/training/distill/test_distill_dmd.py` - - `fastvideo/tests/training/self-forcing/test_self_forcing.py` - -## 先说结论：当前 distill 覆盖范围 - -1. **模型侧目前明显偏 Wan** - - `TrainingPipeline._normalize_dit_input` 固定走 - `normalize_dit_input('wan', ...)` - - teacher/critic 加载时强制设置 - `training_args.override_transformer_cls_name = "WanTransformer3DModel"` - - 因此“想 distill 其它架构”目前不是简单换配置就能跑通的。 - -2. **算法侧两条主线** - - **DMD/DMD2**：`DistillationPipeline` - （student + teacher(real score) + critic(fake score)） - - **Self-forcing**：`SelfForcingDistillationPipeline` - （在 DMD2 框架里，把 student 前向改成 causal/self-forcing 的 - blockwise generation + KV cache） - -3. **默认/隐含假设** - - DMD 路径里多处 `unflatten(0, (1, T))`，等价于默认 `batch_size == 1` - （脚本里确实几乎都设 `--train_batch_size 1`）。 - - DMD loss 需要 teacher 的 cond/uncond 两次前向做 CFG；uncond embedding - 目前依赖 validation 阶段用 negative prompt 编码得到 - （见 “unconditional conditioning 的来源” 一节）。 - -## DistillationPipeline（DMD/DMD2）训练逻辑 - -### 1) 参与训练/冻结的模块（roles） - -`DistillationPipeline.load_modules()` + `initialize_training_pipeline()` 搭出 -“三模型”结构： - -- **student / generator**：`self.transformer` - - 来自 `--pretrained_model_name_or_path` - - 这是要训练的主模型 - - VAE：`self.vae` 同样来自 student pipeline，但在 distill 中 **冻结** -- **teacher / real score**：`self.real_score_transformer` - - 来自 `--real_score_model_path` - - `requires_grad_(False)` + `eval()` -- **critic / fake score**：`self.fake_score_transformer` - - 来自 `--fake_score_model_path` - - **要训练**，有自己的一套 optimizer/scheduler -- （可选）MoE/双 transformer 支持（teacher/critic 优先） - - teacher/critic 会尝试加载 `transformer_2` - - 用 `boundary_ratio`（→ `boundary_timestep`）决定高噪/低噪 expert 选择： - - `_get_real_score_transformer(t)` - - `_get_fake_score_transformer(t)`（并设置 `train_fake_score_transformer_2`） - -> 备注：student 侧也可能有 `transformer_2`（例如 Wan2.2），但 DMD 路径的 -> optimizer step 目前主要写死在 `self.transformer` 上；self-forcing 路径则会 -> 在 `optimizer` / `optimizer_2` 间二选一。重构时建议统一“哪个 timestep -> 更新哪个 expert”的决策来源与实现方式。 - -### 2) 关键超参/调度 - -- `generator_update_interval` - - `step % generator_update_interval == 0` 才更新 student（generator） - - critic 每 step 都更新 -- `pipeline_config.dmd_denoising_steps` → `self.denoising_step_list` - - 这是 “student 训练/模拟推理” 用到的一组离散 timestep - - 若 `--warp_denoising_step`，会根据 scheduler 的 time shift 做一次映射 -- timestep sampling 范围（用于 DMD loss / critic loss 的随机 timestep） - - `min_timestep = min_timestep_ratio * num_train_timesteps` - - `max_timestep = max_timestep_ratio * num_train_timesteps` - - 最终都会 clamp 到 `[min_timestep, max_timestep]` -- teacher CFG - - `real_score_guidance_scale` - -### 3) batch / tensor 形状约定（T2V） - -从 parquet dataloader 读到的典型字段： - -- `vae_latent`: `[B, C, T_lat, H_lat, W_lat]` -- `text_embedding`, `text_attention_mask` -- `info_list`（日志用） - -进入 distill 训练后，关键变换： - -- `_prepare_dit_inputs` 里把 `latents` permute 成 `[B, T_lat, C, H, W]` - 并把 `self.video_latent_shape` 记录为这个形状 -- `_build_distill_input_kwargs` 会再 permute 回 `[B, C, T_lat, H, W]` - 作为 transformer 的 `hidden_states` - -### 4) 一个 step 内发生什么（train_one_step） - -`train_one_step` 的大体结构： - -1. 收集 `gradient_accumulation_steps` 个 batch - - 每个 batch 会构建 attention metadata，并复制出一份 `attn_metadata_vsa` - - `attn_metadata_vsa`：保留当前 VSA sparsity（稀疏注意力） - - `attn_metadata`：强行把 `VSA_sparsity = 0`（等价 dense） - - 代码用法上：student forward 常用 `attn_metadata_vsa`， - teacher/critic forward 用 `attn_metadata`（dense） - -2. **可选更新 student（generator）** - - 条件：`current_step % generator_update_interval == 0` - - 对每个 accumulated batch： - - 先算 `generator_pred_video` - - `--simulate_generator_forward`：`_generator_multi_step_simulation_forward` - （从纯噪声开始按 `denoising_step_list` 模拟推理，最后一步保留梯度） - - 否则：`_generator_forward` - （对数据里的 `vae_latent` 加噪声做一次 denoise；这里把 batch 维度 - `unflatten` 写死成 `(1, T)`，隐含 `B=1`） - - 再算 `dmd_loss = _dmd_forward(generator_pred_video, batch)` 并反传 - - clip grad → `self.optimizer.step()` → EMA update（若开启） - -3. **更新 critic（fake score）**（每个 step 都做） - - 对每个 accumulated batch： - - 先 `generator_pred_video`（no_grad） - - 再随机采样 `fake_score_timestep`，把 `generator_pred_video` 加噪声得到 - `noisy_generator_pred_video` - - critic 预测 `fake_score_pred_noise` - - 目标是 `target = noise - generator_pred_video` - - `flow_matching_loss = mean((pred - target)^2)` - - clip grad → critic optimizer step → critic scheduler step - - 注意：代码里 `self.lr_scheduler.step()`（student 的 lr scheduler）也会在 - critic 更新后每 step 都跑一次，即使这个 step 没有更新 student。 - 如果 `generator_update_interval > 1`，这会让 student 的 lr schedule - “按 step 走”而不是“按 optimizer step 走”。 - -### 5) DMD loss 细节（_dmd_forward） - -简化写成： - -- 输入：`x = generator_pred_video`（student 产出的 clean latent） -- 随机采样 timestep `t` -- 加噪声：`x_t = add_noise(x, eps, t)` -- critic（fake score）得到 `x_hat_fake` -- teacher（real score）分别做 cond/uncond 得到 `x_hat_real_cond`, - `x_hat_real_uncond`，再做 CFG： - - `x_hat_real = x_hat_real_cond + (x_hat_real_cond - x_hat_real_uncond) * s` - -- 构造一个“梯度方向”（代码变量名为 `grad`）： - - `g = (x_hat_fake - x_hat_real) / mean(abs(x - x_hat_real))` - -- DMD2 风格的 stop-grad target： - - `L = 0.5 * mse(x, (x - g).detach())` - -并把中间 latent 存到 `training_batch.dmd_latent_vis_dict` 里用于可视化。 - -### 6) unconditional conditioning 的来源 - -DMD loss 里 teacher 需要 uncond forward： - -- `training_batch.unconditional_dict` 来自 `_prepare_dit_inputs`，但它只在 - `self.negative_prompt_embeds` 已经存在时才会被创建 -- `self.negative_prompt_embeds` 当前是在 `_log_validation` 里，通过 - `validation_pipeline.prompt_encoding_stage` 对 `SamplingParam.negative_prompt` - 编码得到（会同时设置 attention mask） -- 因此目前的“默认用法”是：训练脚本基本都会开 `--log_validation`，保证在 - train 开始前 `_log_validation` 跑一次，从而初始化 negative prompt embedding - -> 重构时建议显式化这条依赖：uncond embedding 不应依赖 -> “是否开启 validation logging”。 - -## SelfForcingDistillationPipeline（Self-forcing）训练逻辑 - -Self-forcing pipeline 继承自 `DistillationPipeline`，但有三个核心变化： - -### 1) scheduler 换成 SelfForcingFlowMatchScheduler - -`initialize_training_pipeline` 里把 `self.noise_scheduler` 替换为 -`SelfForcingFlowMatchScheduler(..., training=True, extra_one_step=True)`， -并引入 self-forcing 的一批超参： - -- `dfake_gen_update_ratio`：generator 每 N step 更新一次（其余 step 只更 critic） -- `num_frame_per_block`, `independent_first_frame`, `same_step_across_blocks`, - `last_step_only`, `context_noise` 等 - -### 2) generator forward 变成 causal/blockwise + KV cache - -`_generator_multi_step_simulation_forward` 在 self-forcing 里被彻底重写： - -- 按 block 生成视频（每 block `num_frame_per_block` 帧） -- 每个 block 内做 denoising loop（遍历 `denoising_step_list`），并用随机 - `exit_flag` 决定在哪个 timestep 停止并保留梯度 -- KV cache / cross-attn cache - - `_initialize_simulation_caches` 预分配 cache 张量 - - 每生成完一段会用 `context_noise`（通常 0）再跑一次 timestep=0 的 forward - 来更新 cache（no_grad） -- gradient masking - - 当生成帧数 > 最小帧数时，会把一部分早期帧 detach，确保只在后面某些帧上回传梯度 - - 额外还有 “last 21 frames” 的处理逻辑：把更早的帧 decode→encode 变成 - image latent，再拼回去 - -总体上，这是把“推理时的 autoregressive/cached 过程”搬进训练。 - -### 3) train_one_step 变成“交替训练 generator / critic” - -- `train_generator = (step % dfake_gen_update_ratio == 0)` -- generator 更新时 - - `generator_loss` 默认仍然走 `_dmd_forward` - （所以仍依赖 teacher + critic） - - optimizer step 会在 `transformer` 和 `transformer_2` 之间二选一 - （基于 `self.train_transformer_2`） -- critic 更新时 - - `critic_loss` 仍然是 `faker_score_forward`（flow matching） - -## Checkpoint / Resume（distill 专用） - -`save_distillation_checkpoint` / `load_distillation_checkpoint` 的要点： - -- 保存 generator + critic 的 distributed checkpoint（可用于 resume） -- 同时保存 “consolidated generator weights”（用于推理/部署） -- 支持（可选）MoE 的 `transformer_2` / `critic_2` / `real_score_2` -- 保存 random state（保证 resume 后 timestep/noise 采样一致） -- EMA 以单独文件的方式保存（避免不同 rank 形状不一致） - -## 目前实现中最重要的耦合点/限制（重构 checklist） - -- **Wan-only 逻辑散落多处** - - normalize 走 `'wan'` - - teacher/critic 强制 `WanTransformer3DModel` -- **batch_size 隐式假设为 1** - - `_generator_forward`, `_dmd_forward`, `faker_score_forward` 里对 - `add_noise(...).unflatten(0, (1, T))` 的写法 -- **uncond embedding 的初始化路径不显式** - - 依赖 `_log_validation` 运行过才能构造 `unconditional_dict` -- **student lr scheduler 的 step 粒度** - - 当前按 “训练 step” 走，而不是按 “generator optimizer step” 走 -- **MoE/dual-transformer 的选择与更新逻辑不统一** - - teacher/critic 有 boundary 选择逻辑 - - student 的 optimizer 选择逻辑在 DMD 与 self-forcing 两条路径不一致 diff --git a/dev/fastgen.md b/dev/fastgen.md deleted file mode 100644 index e5b25b951..000000000 --- a/dev/fastgen.md +++ /dev/null @@ -1,468 +0,0 @@ -# FastGen Framework Summary - -Reference codebase: `~/alex/pkgs/FastGen` - -## Naming Convention (Important!) - -FastGen's naming maps to concepts differently than FastVideo: - -| FastGen term | FastVideo term | What it actually is | -|---|---|---| -| `FastGenNetwork` | `ModelBase` | Neural network + noise schedule | -| `FastGenModel` (e.g. `DMD2Model`) | `DistillMethod` (e.g. `DMD2Method`) | Training algorithm / method | -| `Trainer` | `DistillTrainer` | Training loop orchestrator | - -In FastGen: **"model" = method**, **"network" = model**. - ---- - -## Architecture Overview - -``` -train.py - -> load python config (attrs + OmegaConf + Hydra overrides) - -> model = instantiate(config.model_class) # a FastGenModel subclass - -> Trainer(config).run(model) - -> DDP/FSDP wrap - -> model.init_optimizers() - -> checkpointer.load(...) - -> for iter in range(max_iter): - for accum_iter: - data = preprocess_data(batch) # VAE/text encode - loss_map, outputs = model.single_train_step(data, iter) - backward(loss_map["total_loss"]) - model.optimizers_schedulers_step(iter) - model.optimizers_zero_grad(iter) - maybe_validate(model) - maybe_save_checkpoint(model) -``` - -**Key invariant**: Trainer only ever calls `single_train_step()` and reads -`loss_map["total_loss"]`. All algorithm complexity lives inside the model -(method) class. - ---- - -## Layer 1: FastGenNetwork (the neural network) - -**File**: `fastgen/networks/network.py` - -Abstract base that wraps a transformer/UNet with its noise schedule: - -```python -class FastGenNetwork(ABC, torch.nn.Module): - def __init__(self, net_pred_type="x0", schedule_type="edm", **kwargs): - self.net_pred_type = net_pred_type # "x0", "eps", "v", "flow" - self.noise_scheduler = get_noise_schedule(schedule_type, **kwargs) - - @abstractmethod - def forward(self, x_t, t, condition=None, *, - fwd_pred_type=None, # override pred type - return_features_early=False, # for discriminator - feature_indices=None, - return_logvar=False, - **fwd_kwargs) -> Tensor: ... - -class CausalFastGenNetwork(FastGenNetwork): - """Adds chunk_size, total_num_frames, clear_caches().""" - @abstractmethod - def clear_caches(self): ... -``` - -The network owns the noise schedule, which provides: -- `forward_process(x0, eps, t)` — add noise: `alpha(t)*x0 + sigma(t)*eps` -- `sample_t(n, time_dist_type, ...)` — sample training timesteps -- `sample_t_inhom(n, seq_len, chunk_size, ...)` — per-chunk timestep sampling (causal) -- `x0_to_eps()`, `eps_to_x0()`, `flow_to_x0()`, `convert_model_output()` — conversions -- `latents(noise, t_init)` — scale noise by sigma for initial latent - -### Noise Schedule Hierarchy - -``` -BaseNoiseSchedule (abstract) -├── EDMNoiseSchedule # sigma(t)=t, alpha(t)=1, t∈[0.002, 80] -├── AlphasNoiseSchedule # from alphas_cumprod -│ ├── SDNoiseSchedule -│ ├── SDXLNoiseSchedule -│ └── CogVideoXNoiseSchedule -├── RFNoiseSchedule # rectified flow: alpha(t)=1-t, sigma(t)=t -└── TrigNoiseSchedule # alpha(t)=cos(t), sigma(t)=sin(t) -``` - -Time sampling distributions: `uniform`, `lognormal`, `logitnormal`, `polynomial`, -`shifted`, `log_t`. - ---- - -## Layer 2: FastGenModel (the training method) - -**File**: `fastgen/methods/model.py` (~717 lines) - -Template-method base class that all training algorithms inherit from: - -```python -class FastGenModel(torch.nn.Module): - # --- Construction --- - def build_model(self): - self.net = instantiate(config.net) # student network - self.build_teacher() # optional frozen teacher - self._setup_ema() # optional EMA copies - - def build_teacher(self): - self.teacher = instantiate(config.teacher or config.net) - self.teacher.eval().requires_grad_(False) - - # --- Polymorphic dictionaries (key for checkpoint/FSDP) --- - @property - def model_dict(self) -> dict: # {"net": ..., "teacher": ..., "ema": ...} - @property - def fsdp_dict(self) -> dict: # subset to be FSDP-sharded - @property - def ema_dict(self) -> dict: # all EMA networks - @property - def optimizer_dict(self) -> dict: - @property - def scheduler_dict(self) -> dict: - - # --- Training interface (overridden by subclasses) --- - @abstractmethod - def single_train_step(self, data, iteration) -> (loss_map, outputs): ... - - @abstractmethod - def _get_outputs(self, gen_data, ...) -> dict: ... - - # --- Optimizer management --- - def init_optimizers(self): ... - def get_optimizers(self, iteration) -> list[Optimizer]: ... - def get_lr_schedulers(self, iteration) -> list[Scheduler]: ... - def optimizers_schedulers_step(self, iteration): ... - def optimizers_zero_grad(self, iteration): ... - - # --- Inference --- - @staticmethod - def generator_fn(net, noise, condition, t_list, ...): ... - def _student_sample_loop(self, net, noise, condition, ...): ... - - # --- Precision --- - def set_precision(self, precision, precision_amp, ...): ... -``` - -### Method Inheritance Tree - -``` -FastGenModel -├── SFTModel # supervised fine-tuning -│ └── CausalSFTModel -├── KDModel # knowledge distillation (pre-paired data) -│ └── CausalKDModel -├── CMModel # consistency model -│ ├── TCMModel # two-stage consistency -│ ├── SCMModel # simplified consistency (TrigFlow) -│ └── MeanFlowModel # mean flow matching -├── DMD2Model # distribution matching distillation v2 -│ ├── FdistillModel # f-divergence weighted DMD -│ ├── LADDModel # latent adversarial diffusion distillation -│ └── CausVidModel # causal video DMD -│ └── SelfForcingModel # self-forcing (causal rollout) -``` - ---- - -## Layer 3: Trainer (training loop) - -**File**: `fastgen/trainer.py` (~544 lines) - -Completely algorithm-agnostic. Handles: -- DDP/FSDP wrapping -- Gradient accumulation with sync control -- Data preprocessing (VAE encode, text encode) -- Validation (reuses `single_train_step` with no_grad) -- Checkpoint save/load -- Callback system (EMA update, grad clip, logging, profiling) - -The trainer never knows about roles, multiple networks, or alternating updates. -All of that is encapsulated in the model (method) class. - ---- - -## Method Details - -### SFTModel (Supervised Fine-Tuning) - -**File**: `fastgen/methods/fine_tuning/sft.py` - -Simple diffusion training with optional CFG dropout: - -``` -t = sample_t(batch_size) -eps = randn_like(real_data) -x_t = forward_process(real_data, eps, t) -condition = mix_with_neg_condition(condition, cond_dropout_prob) -pred = net(x_t, t, condition) -loss = denoising_score_matching_loss(pred, x0=real_data, eps=eps, t=t) -``` - -Condition dropout: randomly replaces condition with `neg_condition` per sample, -with configurable `cond_dropout_prob` and `keys_no_dropout` for selective dropout. - -### KDModel (Knowledge Distillation) - -**File**: `fastgen/methods/knowledge_distillation/KD.py` - -Learns from pre-constructed teacher ODE trajectories: - -- **Single-step**: student maps pure noise → x0, loss = MSE(pred, target) -- **Multi-step**: student maps intermediate path point → x0 - - `path` tensor: `[B, num_inf_steps, C, F, H, W]` (pre-computed teacher trajectory) - - Samples random t from `t_list`, gathers corresponding path point - - Supports 2-step and 4-step distillation - -**CausalKDModel**: Uses `sample_t_inhom()` for per-chunk timestep sampling. - -### DMD2Model (Distribution Matching Distillation v2) - -**File**: `fastgen/methods/distribution_matching/dmd2.py` (~532 lines) - -Three networks: student (`net`), teacher (frozen), fake_score (critic). -Optional: discriminator (GAN loss). - -**Alternating updates** controlled by `student_update_freq`: - -``` -if iter % student_update_freq == 0: - # Student update - gen_data = net(input, t_student) # generate x0 - x_t = forward_process(gen_data, eps, t) # re-noise - teacher_x0 = teacher(x_t, t, cfg=True) # teacher prediction (CFG) - fake_score_x0 = fake_score(x_t, t).detach() # critic prediction - - vsd_loss = variational_score_distillation_loss(gen_data, teacher_x0, fake_score_x0) - gan_loss = gan_loss_generator(discriminator(features)) # optional - total_loss = vsd_loss + gan_weight * gan_loss - - optimizers = [net_optimizer] -else: - # Critic + discriminator update - fake_score_loss = denoising_score_matching_loss(fake_score(x_t, t), gen_data, eps, t) - gan_loss = gan_loss_discriminator(real_logits, fake_logits) # optional - - optimizers = [fake_score_optimizer, discriminator_optimizer] -``` - -**VSD Loss** (the core DMD2 gradient): - -```python -def variational_score_distillation_loss(gen_data, teacher_x0, fake_score_x0, - additional_scale=None): - w = 1 / (|gen_data - teacher_x0|.mean() + eps) # adaptive weight - if additional_scale: w *= additional_scale # f-div weighting - vsd_grad = (fake_score_x0 - teacher_x0) * w - pseudo_target = gen_data - vsd_grad - return 0.5 * MSE(gen_data, pseudo_target) -``` - -### FdistillModel (F-Divergence Distillation) - -**File**: `fastgen/methods/distribution_matching/f_distill.py` - -Extends DMD2 with importance weighting via learned discriminator logits: - -```python -f_div_weighting = { - "rkl": lambda r: 1, # reverse KL - "kl": lambda r: r, # forward KL - "js": lambda r: 1-1/(1+r), # Jensen-Shannon - "sf": lambda r: 1/(1+r), # squared Hellinger - "sh": lambda r: r**0.5, # Hellinger - ... -} -``` - -Density ratio estimated from discriminator logits → used as `additional_scale` -in VSD loss. Optional EMA histogram normalization across timestep bins. - -### CausVidModel / SelfForcingModel (Causal Video Distillation) - -**Files**: `fastgen/methods/distribution_matching/causvid.py`, `self_forcing.py` - -**CausVidModel** extends DMD2 for causal (autoregressive) video generation: -- Per-chunk timestep sampling via `sample_t_inhom()` -- Autoregressive rollout with KV cache management -- Context noise injection for cache warmup - -**SelfForcingModel** extends CausVid, only overrides `gen_data_from_net()`: -- Blockwise causal rollout with random exit timesteps -- Only exit steps retain gradients (memory efficient) -- Exit steps broadcast-synced across ranks - -Inheritance chain: `SelfForcingModel → CausVidModel → DMD2Model → FastGenModel` - -### CMModel (Consistency Model) - -**File**: `fastgen/methods/consistency_model/CM.py` - -Learns consistency trajectory: - -``` -r = t_to_r_sigmoid(t, ratio) # map t → smaller r -y_t = forward_process(x0, eps, t) -y_r = ode_solver(teacher, y_t, t, r) # CD mode (or forward_process for CT) -D_yt = net(y_t, t) -D_yr = net(y_r, r).detach() -loss = ||D_yt - D_yr|| / (t - r) # weighted consistency loss -``` - -Weighting options: `default` (1/(t-r)), `c_out`, `sigma_sq`. - -### TCMModel (Two-Stage Consistency) - -**File**: `fastgen/methods/consistency_model/TCM.py` - -Wraps network in `TCMPrecond` (boundary conditions for consistency). -Two training stages with different loss formulations. - -### SCMModel (Simplified Consistency / TrigFlow) - -**File**: `fastgen/methods/consistency_model/sCM.py` - -Uses `TrigFlowPrecond` with trigonometric noise schedule. -Supports pseudo-Huber loss and logvar-based adaptive weighting. - -### MeanFlowModel - -**File**: `fastgen/methods/consistency_model/mean_flow.py` (~503 lines) - -Velocity-parameterized consistency training with mean flow matching. - ---- - -## Loss Functions (Centralized) - -**File**: `fastgen/methods/common_loss.py` - -```python -def denoising_score_matching_loss(pred_type, net_pred, x0, eps, t): - """Unified DSM loss for all prediction types.""" - if pred_type == "x0": target = x0 - elif pred_type == "eps": target = eps - elif pred_type == "v": target = alpha(t)*eps - sigma(t)*x0 - elif pred_type == "flow": target = eps - x0 - return MSE(net_pred, target) - -def variational_score_distillation_loss(gen_data, teacher_x0, fake_score_x0, - additional_scale=None): - """VSD loss with adaptive weighting (DMD2 core).""" - w = 1 / (|gen_data - teacher_x0|.mean() + eps) - if additional_scale: w *= additional_scale - pseudo_target = gen_data - (fake_score_x0 - teacher_x0) * w - return 0.5 * MSE(gen_data, pseudo_target) - -def gan_loss_generator(fake_logits): - return softplus(-fake_logits).mean() - -def gan_loss_discriminator(real_logits, fake_logits): - return softplus(fake_logits).mean() + softplus(-real_logits).mean() -``` - ---- - -## Configuration System - -**Structure**: attrs dataclasses + OmegaConf + LazyCall + Hydra overrides - -### Three-level config hierarchy - -1. **Base config** (`fastgen/configs/config.py`): - - ```python - @attrs.define - class BaseModelConfig: - net: dict # network config (LazyCall) - teacher: Optional[dict] = None - guidance_scale: Optional[float] - net_optimizer: dict - net_scheduler: dict - sample_t_cfg: SampleTConfig # timestep distribution params - input_shape: list[int] - pretrained_model_path: str - use_ema: Any = False - student_sample_steps: int = 1 - fsdp_meta_init: bool = False - - @attrs.define - class SampleTConfig: - time_dist_type: str = "uniform" - train_p_mean: float = -1.1 - train_p_std: float = 2.0 - min_t: float = 0.002 - max_t: float = 80.0 - t_list: Optional[list[float]] = None - ``` - -2. **Method configs** (`fastgen/configs/methods/config_*.py`): - - `config_dmd2.py`: adds `fake_score_optimizer`, `discriminator`, `student_update_freq` - - `config_f_distill.py`: adds `FdistillConfig` (f_div type, ratio bounds) - - `config_cm.py`: adds `CMLossConfig` (weighting, use_cd, ratio) - -3. **Experiment configs** (`fastgen/configs/experiments//config_.py`): - - Concrete settings: model paths, input_shape, lr, t_list, etc. - -### LazyCall / instantiate - -Config entries use `LazyCall` to record `_target_` + kwargs. Objects are created -via `instantiate(cfg)` at runtime, enabling full pluggability. - ---- - -## Distributed Training - -### DDP Trick - -`DDPWrapper` temporarily redirects `module.forward` → `single_train_step` so -that DDP's forward/backward hooks fire correctly even though the training logic -isn't a standard forward pass. - -### FSDP2 with Meta-Init - -For large models (10B+): -1. Non-rank0 processes build model on `torch.device("meta")` (zero memory) -2. Rank0 loads weights -3. FSDP wrap with `sync_module_states` broadcasts weights - -### Checkpoint - -- **Non-FSDP**: rank0 saves single `.pth` (model + optim + scheduler + iteration) -- **FSDP**: `torch.distributed.checkpoint` per `model_dict` key (sharded), - plus scalar state in `.pth` - -Checkpoint keys derived directly from `model_dict` / `optimizer_dict` / `scheduler_dict` -properties — naturally supports multi-network methods. - ---- - -## Data Pipeline - -**File**: `fastgen/datasets/wds_dataloaders.py` - -WebDataset-based with two key features: - -- `files_map`: load constants from files (e.g., pre-computed neg_condition embedding) -- `presets_map`: inject literal constants into every batch - -`Trainer.preprocess_data()` handles optional online encoding: -- VAE: `data["real"] = net.vae.encode(data["real"])` -- Text: `data["condition"] = net.text_encoder.encode(data["condition"])` -- I2V/V2W: first-frame conditioning, image embeddings - ---- - -## Key Design Principles - -1. **Trainer is algorithm-agnostic**: only knows `single_train_step()` + `total_loss` -2. **Method = multi-network container + algorithm**: owns nets, optimizers, update logic -3. **Alternating updates via `get_optimizers(iter)`**: no trainer changes needed -4. **Network owns noise schedule**: method layer doesn't reimplement diffusion math -5. **Centralized loss functions**: all methods share `common_loss.py` -6. **Inheritance for algorithm variants**: SelfForcing only overrides `gen_data_from_net()` -7. **Polymorphic dicts for checkpoint**: `model_dict`/`optimizer_dict` scale to any number of roles -8. **Config-driven pluggability**: LazyCall + instantiate for all components diff --git a/dev/phase_add_causal_wangame_dfsft.md b/dev/phase_add_causal_wangame_dfsft.md deleted file mode 100644 index 11c379fb7..000000000 --- a/dev/phase_add_causal_wangame_dfsft.md +++ /dev/null @@ -1,326 +0,0 @@ -# Phase: Add Causal WanGame + Diffusion-Forcing SFT (DF-SFT) - -> 目标：在现有 distillation framework 上，新增一种 **causal Wangame** 的 -> supervised finetuning 方法（DFSFT），用于把 Wangame 从「双向 / bidirectional」 -> 的训练范式，迁移到「因果 / causal」范式。 -> -> 参考实现：FastGen 的 `CausalSFTModel`（diffusion forcing for SFT）。 - ---- - -## 0. 背景与动机 - -我们已经把 Wangame 的 `finetune` 与 `dmd2` 跑通。 -下一步要做 **bidirectional -> causal**。 - -我建议先落地一个 **Diffusion-Forcing SFT**（DFSFT）baseline： - -- 仅训练 `student`（SFT/DSM loss，和 FastGen 对齐）； -- 使用 **block-wise inhomogeneous timesteps**（`t_inhom`，按 chunk 采样）， - 让 causal student 在训练时就面对“历史上下文不一定是干净的”的分布； -- 不引入 teacher/critic 依赖，降低第一版风险。 - -> 这不是“few-step distill”。它是“训练一个 causal 的基础模型”。 -> 如果后面要把 causal Wangame distill 成 4/8 steps，再做 CausVid/DMD2 -> diffusion-forcing distillation 更合适。 - ---- - -## 1. 本阶段产物（Deliverables） - -- [ ] **Model 侧**：Wangame 支持 `causal` 变体（通过 role 的 `extra` 参数触发） -- [ ] **Method 侧**：新增 `dfsft`（diffusion-forcing SFT）方法 -- [ ] **Examples**：新增一份 DFSFT 的 YAML + temp.sh（端到端可跑） -- [ ] **Validation（关键变更）**：支持“**真正的 causal rollout**”验证，并支持 - **context noise**： - - causal rollout 指 **streaming / chunk-wise** 生成（KV-cache + block processing）， - 而不是“用 causal transformer 但仍一次性 full-video forward”； - - **目标是 ODE-style 的 causal streaming**（用 `scheduler.step(...)` 做数值推进）， - 而不是 DMD/SDE-style 的 `pred_x0 -> add_noise(next_t, eps)` rollout； - - 若要同时保留 SDE-style（用于对照/legacy 对齐），应当由 config 显式选择， - 而不是在 validator 里隐式“偷换语义”。 - ---- - -## 2. 配置语义（Config） - -### 2.1 Causal variant（Role extra） - -不新增新的 family（避免 `wangame_causal` 这种对外语义膨胀）。 -仍然是： - -```yaml -roles: - student: - family: wangame - path: ... - trainable: true - # extra fields (RoleSpec.extra) - variant: causal - # （可选）更细粒度的 causal invariant：用于表达‘是哪一种因果约束/训练范式’ - # 例如：strict / block / sliding_window / bidirectional_train_causal_eval ... - causal_invariant: block -``` - -- `variant: causal` 由 `models/wangame` 插件解释。 -- 未来如果需要更细粒度，可扩展为： - - `variant: causal|bidirectional` - - `causal: true|false` - - `num_frames_per_block` / `sliding_window_num_frames`（可选） - -### 2.2 DFSFT method config（与 FastGen 对齐） - -推荐：把 DFSFT 的关键 knob 放到 `method_config`： - -```yaml -recipe: - family: wangame - method: dfsft - -method_config: - # diffusion-forcing (SFT) 核心：按 chunk 采样 inhomogeneous t - chunk_size: 3 - - # t sampling（可以复用我们已有的 min/max ratio 语义；最终落到 [0,1000]） - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - - # 可选：更接近“history noisy cache”的效果（第一版可以先不做） - context_noise: 0 -``` - -说明： -- `chunk_size`：决定 `t_inhom` 的 block 划分（FastGen 也用 chunk_size）。 - - 对 Wang/Wangame，建议默认 3（与 `num_frames_per_block` 一致）。 -- `context_noise`：**context noise timestep**（整型）。用于 causal rollout 时 - “更新 KV cache 的上下文噪声水平”（见第 4 节 Validation 设计）。 - -### 2.3 Validation config（真正的 causal rollout + context noise） - -我们需要区分两种 validation 语义： - -1) **full-video rollout**（非 streaming）：`WanGameActionImageToVideoPipeline.forward(...)` -2) **streaming causal rollout**（推荐用于 causal student 的验证）：`WanGameCausalDMDPipeline.streaming_*` - -建议把 validation 的配置拆成两类： - -- **run/trainer 级（什么时候验证、用什么 validation dataset）**：放在 `training:` 下； -- **method 级（怎么采样、走什么 rollout 语义）**：放在 `training.validation:` 下（method 读取）。 - -这样 validator 不需要 “从 training_args 猜 method 语义”，而 method 也不会越权去管理 dataloader。 - -建议把选择权交给 method（或 method_config），让 validator 只负责执行： - -```yaml -training: - validation: - # full | streaming_causal - rollout_mode: streaming_causal - - # 对 full rollout：ode/sde 选择 sampling loop - # 对 streaming_causal：同样允许 ode/sde，但这是一个“明确的语义选择” - sampler_kind: ode - - # 采样步数（对 ODE pipeline 即 num_inference_steps） - sampling_steps: [40] - - # SDE/DMD: 需要显式 step list（few-step schedule） - # sampling_timesteps: [1000,750,500,250] - # warp_denoising_step: true - - # causal cache 的 context noise（timestep） - context_noise: 0 -``` - -> 备注： -> - 现阶段的 `WanGameCausalDMDPipeline / MatrixGameCausalDenoisingStage` 是 DMD/SDE-style； -> 要实现 **ODE-style streaming**，需要新增一个 causal ODE denoising stage（见第 4 节）。 -> - 我们仍可以保留 SDE-style streaming（用于对照/legacy 对齐），但必须由 config 显式选择， -> 避免 “training/validation 语义混用导致 reviewer 困惑”。 - -### 2.4 default_pipeline_config：ODE solver 选择（可选） - -我们目前的 `pipeline_config.sampler_kind=ode` 默认会选择 `FlowUniPCMultistepScheduler` -（见 `fastvideo/pipelines/samplers/wan.py`）。为了做对照实验/调试，建议增加一个可选字段： - -```yaml -default_pipeline_config: - sampler_kind: ode - ode_solver: unipc # unipc | euler -``` - -约束（推荐强校验）： -- `sampler_kind=sde` 时不允许 `ode_solver=unipc`（因为 SDE/DMD-style rollout 需要 - `add_noise(next_t, eps)`；UniPC 的 `add_noise` 对 “任意 timestep 值” 不鲁棒）。 -- `ode_solver=euler` 时应强制 deterministic（`stochastic_sampling=false`），否则就变成 - “SDE-like Euler”。 - ---- - -## 3. 训练逻辑（DFSFT 的算法定义） - -目标：对齐 FastGen `CausalSFTModel`（`fastgen/methods/fine_tuning/sft.py`）。 - -核心步骤（单 step）： - -1) 取真实数据 `x0`（video latents）。 -2) 采样 `eps_inhom ~ N(0, I)`。 -3) 采样 `t_inhom`：形状 **[B, T_lat]**，按 chunk/block-wise 采样，chunk 内 - timestep 相同。 -4) 前向扩散：`x_t = add_noise(x0, eps_inhom, t_inhom)` -5) 学生预测：`pred = student(x_t, t_inhom, cond)`（预测 noise/x0/v，取决于 - adapter/model 的 pred_type） -6) DSM loss：对齐噪声调度器语义（最简单是 MSE(pred_eps, eps_inhom)）。 - -关键点： -- DFSFT 不需要 teacher。 -- “diffusion forcing”体现在 `t_inhom`（按 chunk 的独立噪声水平），而不是 - 直接对 KV tensor 加噪。 - ---- - -## 4. 代码改动清单（按文件） - -### 4.1 models（Wangame causal variant） - -- [ ] `fastvideo/distillation/models/wangame.py` - - 读取 `role_spec.extra.get("variant")`（或 `causal: true`） - - 当 `variant == "causal"`：加载 transformer 时覆盖 cls 为 - `CausalWanGameTransformer3DModel`（FastVideo 已存在该类： - `fastvideo/models/dits/wangame/causal_model.py`） - - 目标：**同一份 ckpt 既可作为 bidirectional student，也可作为 causal - student 初始化**（如果 state_dict 不兼容，需要记录为风险点并加 fallback）。 - -> 备注：如果实现细节需要拆文件，可以内部新增 -> `fastvideo/distillation/models/wangame/causal.py`，但对外 family -> 仍然是 `wangame`。 - -### 4.2 methods（新增 dfsft） - -- [ ] `fastvideo/distillation/methods/fine_tuning/dfsft.py`（新增） - - `@register_method("dfsft")` - - 仅依赖 `roles.student` - - `single_train_step()`：实现第 3 节 DFSFT - - 复用现有 finetune 的 optimizer/lr scheduler wiring - -- [ ] `fastvideo/distillation/methods/__init__.py` - - 暴露/导入新方法（取决于我们当前 registry/dispatch 的约定） - -- [ ] （可能需要）`fastvideo/distillation/adapters/wangame.py` - - 确认 `predict_noise/add_noise` 支持 `timestep` 为 **[B, T_lat]** - - 如果当前只支持 [B]，需要扩展并加形状检查。 - -### 4.3 examples（端到端验证） - -- [ ] `examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_dfsft_causal.yaml` - - `roles.student.variant: causal` - - `recipe.method: dfsft` - - `training.validation.dataset_file / training.validation.every_steps` - - `training.validation.sampling_steps: [40]` - -- [ ] `examples/distillation/wangame/dfsft-temp.sh`（新增） - - 跟现在 `run.sh` 一样只负责 export CONFIG + torchrun - -### 4.4 validation（真正的 causal rollout） - -> 这是本阶段新增的关键设计点：**不要**默认复用“full-video validator”来验证 causal 模型。 - -- [ ] `fastvideo/distillation/validators/wangame.py`（或新增 `wangame_causal.py`） - - 支持 `rollout_mode: streaming_causal`： - - pipeline：**需要 ODE-style 的 causal streaming pipeline** - - 方案 A（推荐）：扩展/新增 `WanGameCausalPipeline`（同一条 pipeline），内部按 - `sampler_kind={ode|sde}` 选择不同 denoising stage - - 方案 B（过渡）：保留 `WanGameCausalDMDPipeline`（仅 SDE），但这不满足本阶段目标 - - ODE-style streaming 的调用方式（与现有 streaming API 对齐）： - 1) `pipeline.streaming_reset(batch, fastvideo_args)` - 2) 循环 `pipeline.streaming_step(...)` 直到生成完成 - 3) 聚合每个 chunk 的 frames，拼成完整 video 再落盘/上报 tracker - - 支持 `context_noise`（对齐 legacy self-forcing 语义）： - - cache update 前对 context latent 做一次显式 `scheduler.add_noise(x0, eps, t_context)` - 再 forward 更新 KV cache（避免 “只改 timestep embedding，效果像没开”） - -- [ ] `fastvideo/pipelines/stages/`（新增 causal ODE denoising stage） - - `MatrixGameCausalOdeDenoisingStage`（或同等命名） - - block/chunk 框架与 `MatrixGameCausalDenoisingStage` 一致（KV-cache + action） - - block 内 loop 使用 `scheduler.step(...)`（ODE loop） - - **每个 block 必须 reset solver state**（见风险点：UniPC 多步历史不能跨 block 泄漏） - -- [ ] `fastvideo/distillation/validators/base.py` - - 若需要：扩展 `ValidationRequest` 以携带 - - `rollout_mode` - - `context_noise` - - `sampling_timesteps`（已有） - - 目标：把“验证用什么 pipeline/rollout”的决策交给 method。 - ---- - -## 5. 验收标准（Definition of Done） - -- [ ] DFSFT 端到端可跑（不需要 teacher/critic） -- [ ] step0 validation 能出视频，不 crash -- [ ] 训练若干步后，validation 质量有可见提升 -- [ ] 同一份 wangame checkpoint：bidirectional finetune 和 causal dfsft - 都能启动（若 causal 需要不同 ckpt，要明确写在配置/README） -- [ ] 支持用 streaming causal rollout 做验证（并能开启/关闭 context noise） - ---- - -## 6. 风险点 / 需要提前确认的问题 - -1) **权重兼容**：`CausalWanGameTransformer3DModel` 是否能直接 load - bidirectional wangame 的 transformer 权重。 - - 如果不能：需要一个 conversion 逻辑（或要求 user 提供 causal init ckpt）。 - -2) **t_inhom 的 shape 语义**： - - Wangame transformer 是否真正支持 [B, T_lat] timesteps； - - scheduler.add_noise 是否支持 per-frame timesteps（不支持就需要 reshape - 或 per-frame add_noise）。 - -3) **chunk_size 与模型结构对齐**： - - DFSFT 的 chunk_size 是否必须等于模型的 `num_frames_per_block`； - - 如果用户配错，建议直接 error。 - -4) **“40-step causal” 的含义**： - - DFSFT 训练的是基础模型；推理时可以设 `num_inference_steps=40`。 - - 但“few-step（4/8）”仍需要 distillation（DMD2/CM/CausVid）。 - -5) **“真正 causal rollout” vs “full-video rollout”**（重要决策点）： - - 如果我们只用 `WanGameActionImageToVideoPipeline.forward(...)` 做验证， - 这并不能覆盖 deployment 的 streaming/KV-cache 语义； - - 但 streaming rollout 目前依赖 `WanGameCausalDMDPipeline`（DMD/SDE-style step list）， - 因此需要明确： - - DFSFT baseline 先用 streaming + step list 验证（更贴近 causal 部署）； - - 或者额外实现一个 “streaming ODE” sampler（更大工程，建议后置）。 - -6) **step list / context noise 的配置入口**（重要决策点）： - - 现有 streaming pipeline 读取 `pipeline_config.dmd_denoising_steps / warp_denoising_step / context_noise`； - - 我们的新框架更希望把 few-step schedule/rollout knobs 放到 `method_config` 或 `training.validation`； - - 因此需要一个明确策略： - - 方案 A（最小改动）：validator 在构建 validation pipeline 时，把 - `training.validation.*` 写入 `args_copy.pipeline_config.*`（validation-only）； - - 方案 B（更干净）：把 streaming pipeline 改为优先读 `ForwardBatch.sampling_timesteps` - - `ValidationRequest.context_noise`，彻底摆脱 pipeline_config 依赖（工程量更大）。 - -7) **context noise 的“语义一致性”**（潜在坑）： - - legacy self-forcing 里 context noise 是： - `x0 -> add_noise(x0, eps, context_timestep)` 再更新 cache； - - 现有 `MatrixGameCausalDenoisingStage._update_context_cache` 只把 timestep 传给 transformer， - 是否也应显式 add_noise（以匹配语义）需要确认，否则“开了 context noise 但效果像没开”。 - -8) **ODE solver 的 state reset（新增风险点，必须明确）**： - - `FlowUniPCMultistepScheduler` 是 multistep solver，内部有历史状态； - - 在 streaming causal rollout 中，**每个 block/chunk** 都应当像独立的 ODE 求解过程： - - 要么每个 block 调用一次 `scheduler.set_timesteps(...)`（会清空 `_step_index` 和历史） - - 要么为每个 block 构建新的 scheduler 实例 - - 否则会出现 solver 历史跨 block 泄漏，导致质量漂移且很难定位。 - ---- - -## 7. FastGen 对照（便于后续实现 CausVid） - -- DFSFT (SFT + diffusion forcing)： - - `fastgen/methods/fine_tuning/sft.py::CausalSFTModel` - - `fastgen/networks/noise_schedule.py::sample_t_inhom_sft` - -- Diffusion-forcing distillation（未来）： - - `fastgen/methods/distribution_matching/causvid.py::CausVidModel` diff --git a/dev/phase_add_wangame.md b/dev/phase_add_wangame.md deleted file mode 100644 index 39912ce06..000000000 --- a/dev/phase_add_wangame.md +++ /dev/null @@ -1,365 +0,0 @@ -# Phase：Add WanGame（把 Wangame 接入新 distillation 框架） - -> 目标：在 **不回到 legacy training pipeline** 的前提下，让 `fastvideo/training/distillation.py` -> 可以通过 YAML（schema v2）跑起 **WanGame** 的 finetune / distill（优先 finetune）。 -> -> 本文件最初用于“代码层面规划”，现在也用来记录已落地的实现与遗留 TODO。 - ---- - -## 当前进展（已落地） - -> 下面是“已经写进代码库并通过静态检查（`compileall` + ruff）”的部分。 -> GPU 端到端训练/验证需要你在有 GPU 的机器上跑（我们这边环境可能没有 driver）。 - -### ✅ 已实现（最小可用：finetune） - -- `recipe.family: wangame` + `recipe.method: finetune` -- 新增 model plugin / adapter / validator： - - `fastvideo/distillation/models/wangame.py` - - `fastvideo/distillation/adapters/wangame.py` - - `fastvideo/distillation/validators/wangame.py` -- builtin dispatch 注册： - - `fastvideo/distillation/dispatch.py:ensure_builtin_registrations()` -- dataloader helper（复用 parquet loader，支持 `path:N` + 多路径）： - - `fastvideo/distillation/utils/dataloader.py:build_parquet_wangame_train_dataloader()` -- examples（可直接跑）： - - `examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml` - - 已把 legacy `finetune_wangame.slurm` 的 `DATA_DIR` 语义搬进来 - - 用 YAML folded string（`>-`）让超长 `data_path` 更可读 - - `examples/distillation/wangame/finetune-temp.sh` - -### ✅ 已实现（为 future DMD2 做好 primitives） - -- `WanGameAdapter` 已提供 DMD2 所需的 operation primitives： - - `num_train_timesteps / shift_and_clamp_timestep / add_noise` - - `predict_noise / predict_x0 / backward` - - attention metadata（dense/vsa）+ forward_context -- `WanGameValidator` 支持根据 `ValidationRequest.sampler_kind` 选 pipeline： - - `ode` -> `WanGameActionImageToVideoPipeline` - - `sde` -> `WanGameCausalDMDPipeline` - -### ✅ 关键对齐说明（legacy vs 新框架） - -- **训练 noising scheduler**：新框架 wangame 训练使用 - `FlowMatchEulerDiscreteScheduler`，与 legacy training loop（`TrainingPipeline`） - 一致（训练阶段并不使用 UniPC 做 noising）。 -- **validation sampler**：validator 走 pipeline（ODE/SDE）时，仍由 pipeline - 自己持有对应 scheduler（例如 ODE pipeline 使用 `FlowUniPCMultistepScheduler`）。 - ---- - -## 0) 我们现在在 FastVideo 里哪里能看到 WanGame？ - -WanGame 不是“想象中的模型”，FastVideo 里已经有一整套（legacy）实现： - -- **模型（DiT）**：`fastvideo/models/dits/wangame/*` - - 关键类：`WanGameActionTransformer3DModel` -- **pipeline config**：`fastvideo/configs/models/dits/wangamevideo.py` + - `fastvideo/configs/pipelines/wan.py:WanGameI2V480PConfig` -- **推理 pipeline（ODE）**：`fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py` - - `WanGameActionImageToVideoPipeline` -- **推理 pipeline（SDE/DMD）**：`fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py` - - `WanGameCausalDMDPipeline` -- **训练 pipeline（legacy）**：`fastvideo/training/wangame_training_pipeline.py` - - 重点参考：`_prepare_dit_inputs()` + `_build_input_kwargs()` -- **数据 schema**：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_wangame` - -结论：要把 WanGame 接到新框架，核心工作是把 legacy pipeline 里 -“raw batch -> model forward primitives”的逻辑迁入 **adapter**。 - ---- - -## 1) 现在我们的新框架需要什么（接口映射） - -新框架（Phase 3.x）的一条 run path 是： - -- `fastvideo/training/distillation.py` - -> parse YAML (`fastvideo/distillation/utils/config.py`) - -> `fastvideo/distillation/dispatch.py:build_runtime_from_config()` - -> `fastvideo/distillation/trainer.py:DistillTrainer.run(method, dataloader, ...)` - -其中 **model plugin** 必须返回： - -- `RoleManager`（roles -> RoleHandle -> modules/optimizers/schedulers） -- `Adapter`（operation-centric primitives） -- `DataLoader` -- `Validator`（可选，method 决定怎么调用） - -因此“加 Wangame”就是补齐一个新的 model plugin + adapter + validator。 - ---- - -## 2) 设计目标（Add WanGame 的 Definition of Done） - -### ✅ 最小可用（建议先交付这一档） - -- 支持 `recipe.family: wangame` -- 支持 `recipe.method: finetune`（优先） -- `log_validation: true` 时能正确走 `WanGameActionImageToVideoPipeline`（ODE） -- 训练 input 与 legacy `fastvideo/training/wangame_training_pipeline.py` 一致： - - I2V concat：`noisy_video_latents + mask + image_latents` - - action conditioning：`viewmats / Ks / action` 来自 `process_custom_actions(...)` - -### ✅ 进阶（第二档） - -- 支持 `recipe.method: dmd2`（distill） -- validation 根据 `ValidationRequest.sampler_kind` 切换： - - `ode` -> `WanGameActionImageToVideoPipeline` - - `sde` -> `WanGameCausalDMDPipeline` - -> 注意：WanGame 的训练通常不依赖 text CFG；DMD2 中的 uncond/cond -> 在 wangame 上可能等价（见风险点）。 - ---- - -## 3) 文件改动规划（建议的最小集合） - -### 3.1 新增：model plugin - -- [x] `fastvideo/distillation/models/wangame.py` - - `@register_model("wangame")` - - 主要职责： - - 设置 `training_args.override_transformer_cls_name = "WanGameActionTransformer3DModel"` - （必要时增加可配置项，支持 causal transformer） - - 加载 shared：`vae`（用于 `normalize_dit_input("wan", ...)`）+ `noise_scheduler` - - 为每个 role 加载 `transformer`（以及可选 `transformer_2`） - - `apply_trainable(...)` + activation ckpt（沿用 wan plugin 逻辑） - - 构建 `RoleManager` - - 构建 `WanGameAdapter` - - dataloader：使用 `pyarrow_schema_wangame` - - `log_validation` 时构建 `WanGameValidator` - -### 3.2 新增：adapter - -- [x] `fastvideo/distillation/adapters/wangame.py` - - 复用 `WanAdapter` 的通用 mechanics（timestep/noise/attn_metadata/backward 的模式） - - 重点实现（对齐 legacy `wangame_training_pipeline.py`）： - - `prepare_batch(raw_batch, current_vsa_sparsity, latents_source=...)` - - 从 parquet batch 取： - - `vae_latent`（video x0） - - `clip_feature`（image_embeds） - - `first_frame_latent`（image_latents） - - `keyboard_cond` / `mouse_cond` - - `pil_image`（给 validation 或 debug） - - 计算 timesteps/noise/sigmas，得到 noisy_video_latents - - 构造 I2V 输入： - - 生成 `mask_lat_size`（与 legacy 一致） - - `noisy_model_input = cat([noisy_video_latents, mask, image_latents], dim=1)` - - `process_custom_actions(...)` -> `viewmats / Ks / action` - - 组装 transformer forward 所需的 `input_kwargs` - - `predict_noise(handle, noisy_latents, t, batch, conditional, attn_kind)` - - `predict_x0(handle, noisy_latents, t, batch, conditional, attn_kind)` - - **关键约束**：`noisy_latents` 参数应代表“noisy video latents”； - adapter 内部用 batch 里的 `image_latents/mask/action` 拼出完整 hidden_states。 - - `add_noise(clean_latents, noise, t)`：只对 video latent 做 add_noise - - `backward(loss, ctx, ...)`：延续现有 forward_context 机制 - -> 备注：`conditional` 在 wangame 上可能不区分（先忽略即可），但接口必须收敛一致， -> 以兼容 `DMD2Method` 的调用方式。 - -### 3.3 新增：validator - -- [x] `fastvideo/distillation/validators/wangame.py` - - API 对齐 `WanValidator`：`log_validation(step, request=ValidationRequest)` - - pipeline 选择： - - `request.sampler_kind == "ode"`：`WanGameActionImageToVideoPipeline` - - `request.sampler_kind == "sde"`：`WanGameCausalDMDPipeline` - - batch 构造参考 legacy： - - `fastvideo/training/wangame_training_pipeline.py:_prepare_validation_batch` - - 只要 method 提供 `sample_handle`（通常 student），validator 就能跑。 - -### 3.4 改动：dispatch builtin registrations - -- [x] `fastvideo/distillation/dispatch.py:ensure_builtin_registrations()` - - 显式 import 新增的 `fastvideo.distillation.models.wangame` - -### 3.5 可选：dataloader util - -当前 `fastvideo/distillation/utils/dataloader.py` 只有 T2V helper。 -wangame 需要 I2V+action schema，因此建议： - -- [x] `fastvideo/distillation/utils/dataloader.py` - - 新增 `build_parquet_wangame_train_dataloader(training_args, parquet_schema=pyarrow_schema_wangame)` - - 内部仍调用 `fastvideo.dataset.build_parquet_map_style_dataloader(...)` - -TODO（更通用的方向，暂不做）： -- [ ] 扩展到更多 dataset kind（webdataset / precomputed / ode-init ...）， - 并用更统一的 config/dispatch 管理（例如 `DataSpec`）。 - ---- - -## 4) 配置（YAML）规划（schema v2） - -建议新增一个最小 finetune 配置（示意）： - -```yaml -recipe: - family: wangame - method: finetune - -roles: - student: - family: wangame - path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers - trainable: true - -training: - data_path: /abs/path/to/wangame/preprocessed/combined_parquet_dataset - # shape / dist / lr / validation 同 wan 的 schema 写法即可 - -pipeline_config: - flow_shift: 5 - sampler_kind: ode - -method_config: - attn_kind: dense -``` - -### 4.1 roles 的“自由扩展字段”策略（保留核心字段强校验） - -建议 roles 仍强制解析这三个核心字段（保证错误信息清晰）： -- `family / path / trainable` - -但允许 roles 下出现其它任意 key，并把它们原样保留（例如 `RoleSpec.extra`）， -由 `models/wangame.py` 自行解释（例如 action-only train patterns / cls_name / init 行为）。 - -如果要支持 DMD2（第二档），需要： -- roles 增加 teacher/critic（如果暂时没有更大 teacher，可先 teacher==student 跑通 e2e） -- `pipeline_config.sampler_kind: sde`（validation 走 `WanGameCausalDMDPipeline`） -- `method_config` 增加 DMD2 必需字段（`rollout_mode`, `dmd_denoising_steps`, ...） - ---- - -## 5) 关于目录结构：要不要 `models/wan/wan.py + models/wan/wangame.py`？ - -**不需要强制做**。 - -当前 dispatch 只关心注册 key（`@register_model("...")`），文件放哪都行。 -因此建议先保持扁平： - -- `fastvideo/distillation/models/wan.py` -- `fastvideo/distillation/models/wangame.py` - -等到 wan 系变体更多时（wan_t2v / wan_i2v / wangame / lingbot / ...）， -再做结构化重排成 `models/wan/*`，并把共同的 loader 逻辑抽成内部 helper。 - ---- - -## 6) 风险点 / 决策点（需要提前讲清楚） - -1) **DMD2 的 CFG（cond/uncond）语义在 wangame 上可能不存在** - 先说明：在我们新框架里，`DMD2Method` 的 teacher CFG 语义来自 “文本 CFG”： - - `conditional=True`：用 prompt（正向）作为 conditioning； - - `conditional=False`：用 negative / uncond prompt 作为 conditioning； - - 最终 teacher 的 “real score” 用类似： - `real = uncond + guidance_scale * (cond - uncond)`。 - - 这在 Wan(T2V) 上成立，因为： - - legacy / 我们的 `WanAdapter` 都显式构造了 negative prompt embeds（`ensure_negative_conditioning()`）， - 并把它塞进 `training_batch.unconditional_dict`，从而 `conditional` flag 有真实语义差异。 - - 但在 **WanGame** 上，legacy 训练/推理几乎完全不依赖 text： - - legacy train：`fastvideo/training/wangame_training_pipeline.py:_build_input_kwargs()` - 明确设置： - - `"encoder_hidden_states": training_batch.encoder_hidden_states # None (no text conditioning)` - - `"encoder_hidden_states_image": image_embeds`（I2V conditioning） - - `"viewmats" / "Ks" / "action"`（action conditioning） - - legacy inference pipeline：`fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py` - 没有 `TextEncodingStage`；因此 `ForwardBatch.prompt_embeds` 为空。 - `LatentPreparationStage` 会把空 prompt embeds 变成一个 dummy（形状 `[B, 0, hidden_size]`），并且： - - `batch.do_classifier_free_guidance = False` - （见 `fastvideo/pipelines/stages/latent_preparation.py`） - - legacy scripts/示例也在暗示“没 CFG”： - - `examples/inference/basic/basic_wangame.py` 里 `guidance_scale=1.0` - - `examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm` - 里 `validation_guidance_scale "1.0"` - - 因此：如果我们直接把 DMD2 的 `conditional/unconditional` 套到 wangame 上， - 在 **不重新定义 uncond 的前提下**，它们很可能等价（cond == uncond），导致： - - `real_cond_x0 - real_uncond_x0 ≈ 0` - - guidance_scale 形式上存在，但语义上失效 - - 训练可能仍能跑通，但已经不是“文本 CFG 意义下的 DMD2” - - **建议的落地策略（按风险从低到高）：** - - A（最小可用，推荐先做）：在 `wangame` 的 adapter 里把 `conditional` 当作 no-op - （cond/uncond 同路），并在文档里明确 “wangame 暂不支持文本 CFG”。 - - B（定义 wangame 的 uncond 语义）：由 `method_config` 显式声明 uncond 的定义，例如： - - `uncond_mode: none|zero_action|zero_image|zero_both` - - `zero_action`：把 action/viewmats/Ks 置零或置 None（需要确认 transformer 对 None 的容忍度） - - `zero_image`：把 `encoder_hidden_states_image` 置零（保持 shape） - 这样 `conditional` 才有可解释的差异。 - 归属建议： - - 放在 `method_config`：因为它只在 “需要 CFG 的算法” 中有意义；finetune 等方法不应被迫理解 uncond 语义。 - - 但执行必然落在 adapter（如何 zero_action/zero_image 是模型相关的），因此 adapter 需要提供一个 operation - 来承接该语义（例如“构造 uncond conditioning variant”）。 - - **新增 TODO（需要实现）** - - [ ] 为 wangame + DMD2 引入 `method_config.uncond_mode` - - DMD2Method：读取该字段，并在 teacher CFG 时把 `conditional=False` - 映射到对应的 “uncond variant” - - WanGameAdapter：提供可解释的 uncond 变体构造（避免硬编码 DMD2 名词），例如： - - `conditional=False` 时按 `uncond_mode` 将 action/image conditioning 置零 - - 或提供一个更显式的 operation（如 `build_conditioning_variant(...)`） - -2) **train_action_only / action_warmup_steps（细粒度 trainable）** - legacy `wangame_training_pipeline.py` 支持更细粒度的训练策略： - - `train_action_only`：冻结 base DiT，只训练 action 相关参数（pattern-based） - - `action_warmup_steps`：前 N 步把 action 参数 `requires_grad=False`，之后再打开 - - 重要补充：这两个 knob 在 FastVideo 的 `TrainingArgs` 里已经存在（`fastvideo/fastvideo_args.py`）， - 也就是说 **YAML 的 `training:` 段本身就能表达**： - - `training.train_action_only: true` - - `training.action_train_target: both|action_mlp|prope` - - `training.action_warmup_steps: 1000` - - 我们新框架目前的问题不是 “config 表达不了”，而是： - - roles 只有 `trainable: bool`（整模块开关），无法表达“同一个 transformer 内只训练某些 param” - - warmup 属于 step-time policy，应该由 method（或 method_config）驱动，而不是 loader 一次性决定 - - **关于你提的方案：roles 允许自由 dict（而非完全结构化）** - - 优点：wangame 这类 family 很容易出现 role-specific knobs（比如 per-role train patterns / cls_name / init 行为）， - 不需要每加一个字段就改全局 schema。 - - 缺点：弱化静态校验，typo 会变成 runtime 才爆；文档/IDE 提示也会更差。 - - **折中建议（更可控）：** - - roles 仍解析核心字段（`family/path/trainable`），但把未知字段原样保留到 `RoleSpec.extra`（或 `role_cfg_raw`）， - 由 `models/wangame.py` 自己读取解释。 - - 这样既能满足“roles 自由 dict 扩展”，也不牺牲核心字段的错误提示质量。 - -3) **validation pipeline 的 ODE/SDE 差异** - 现状：FastVideo 已经把 wangame 分成了两个 inference pipeline： - - ODE：`WanGameActionImageToVideoPipeline`（`fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py`） - - SDE/DMD：`WanGameCausalDMDPipeline`（`fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py`） - - 对新框架而言，最小可用的做法是： - - `validators/wangame.py` 根据 `ValidationRequest.sampler_kind` 选择 pipeline class - - 你提的“一个 pipeline 支持多 sampler（ode/sde）”是好方向（我们对 Wan 已经这么做过）： - - 优点：减少 pipeline 分叉/重复逻辑，validation 不容易“走错范式” - - 代价：需要对 wangame pipeline 做更侵入式的重构（引入 `pipelines/samplers/`，并把 denoising loop 抽出来） - - **建议顺序：** - - 接入阶段先做 validator 选 pipeline（改动小、风险低） - - 稳定后再把 wangame pipeline 也升级为 `sampler_kind` 可切换（更优雅，但属于额外工程） - ---- - -## 7) 验收（我们如何确认接入是对的） - -- finetune： - - e2e 跑通（2~10 steps）+ 能输出 validation 视频 - - 与 legacy finetune 的 step0/stepN 视觉趋势大体一致（不追求完全 bitwise） -- distill（若做第二档）： - - e2e 跑通（few-step） - - validation 选择 `sde` 时，视觉应接近 legacy DMD pipeline 的 sampling 形态 - ---- - -## 8) 目前遗留 / 下一步（WanGame 接入方向） - -- [ ] DMD2 on wangame 的 `uncond` 语义（`method_config.uncond_mode`） -- [ ] action-only / warmup（把 legacy 的 `train_action_only / action_warmup_steps` - 接到新框架：归属在 method/role policy 而非 model plugin） -- [ ] 若需要减少 ODE/SDE pipeline 分叉：将 wangame inference pipeline 也升级为 - `sampler_kind` 可切换（侵入式更强，建议放更后面的 phase） diff --git a/dev/phase_causal.md b/dev/phase_causal.md deleted file mode 100644 index 41c26436a..000000000 --- a/dev/phase_causal.md +++ /dev/null @@ -1,172 +0,0 @@ -# Phase: Causal（对齐 FastGen 的 causal / cache 设计） - -> 目标：让 **causal/streaming** 能力像 FastGen 一样成为“可选增强”，只落在 causal 变体上； -> 同时把 **预处理器（preprocessors：VAE / text encoder / image encoder / …）** 的管理收敛到 -> **student**，避免 teacher/critic 重复加载与显存浪费，并减少跨 role 的耦合。 - -## 0. 当前进度（Snapshot） - -截至目前（实现/验证以 WangGame causal 为主）： - -- ✅ `ModelBase` / `CausalModelBase`：cache 语义不进入 `ModelBase`，只存在于 causal 扩展接口中。 -- ✅ `models/` 层级结构：按 family + variant 拆分（`wan/`、`wangame/`、`wangame_causal.py`）。 -- ✅ preprocessors 收敛：VAE 等 shared components 只在 model plugin 内加载一次； - teacher/critic 仅加载各自 transformer，不重复加载 preprocessors。 -- ✅ WangGame causal streaming primitives：cache 由 causal model 内部持有，method 不传 KV 张量。 -- ✅ Self-Forcing 要求 student 为 causal（不允许 cache-free rollout）。 -- ✅ 方案 A（Activation Checkpointing × Streaming KV-cache 的一致性修复）： - - grad-enabled streaming forward 会对 `kv_cache[*].{global_end_index, local_end_index}` 做 snapshot， - 避免 backward recompute 读到被后续 `store_kv=True` 更新后的 index 导致 `CheckpointError`； - - checkpoint-safe 模式会把 KV-cache capacity 扩到 `training.num_frames * frame_seq_length`， - 避免 forward→backward 生命周期内发生 rolling/eviction 破坏重算一致性。 - -## 1. 背景与动机 - -在 FastVideo 的 distillation 框架里，我们会遇到典型组合： - -- teacher：bidirectional（质量高、但不适合 streaming rollout） -- student：causal（需要 KV-cache / streaming rollout） - -如果把 KV-cache 相关的接口（`kv_cache` / `crossattn_cache` / `current_start` / …）直接塞进 -通用的 `predict_noise/predict_x0`，会导致： - -- **所有模型**都被迫兼容 causal 的 cache 语义（污染接口）； -- method 代码不得不理解具体 cache 结构（污染算法层）； -- 加一个新模型/新任务时，很容易“接口爆炸”。 - -FastGen 的经验是： - -- **只有 causal network** 需要并实现 cache（`store_kv=True` + internal caches）； -- **预处理器只在 student 侧管理/按需初始化**（VAE/text encoder 主要用于数据编码与可视化 decode）， - teacher forward 通常直接吃 latent，不需要再加载一份 VAE。 - -本 phase 将这两条经验收敛到我们的 distillation 框架中。 - -## 2. 关键原则 - -### 2.1 只有 student 管 preprocessors - -定义： - -- preprocessors = `vae` / `text_encoder` / `image_encoder` / （未来可能的）action encoder 等。 - -策略： - -- **只在 student 对应的 model plugin 中初始化/持有** preprocessors（必要时 lazy init）。 -- teacher/critic role **不再重复加载** preprocessors。 -- 训练/蒸馏计算以 latent 为主，validation/wandb 可视化 decode 时再用 student 的 VAE。 - -收益： - -- 显存更友好（尤其是多 role / 大模型 teacher）。 -- 语义更清晰：teacher 的职责是提供 score/target，而不是承担数据编解码。 - -### 2.2 causal 能力只落在 *_causal 变体 - -将“streaming rollout / KV-cache / blockwise causal mask / context noise”等能力视为 -**causal 变体的 runtime primitive**，而不是所有模型的通用能力。 - -对应的 method（例如 self-forcing/DFSFT/causal validator）如果需要 cache， -应当只依赖 causal 变体提供的可选接口（或不透明 `cache_state`），而非强制所有模型支持。 - -#### CausalModelBase：不污染 ModelBase 的接口 - -为了避免把 `kv_cache/crossattn_cache/...` 这类实现细节塞进通用的 -`ModelBase.predict_noise/predict_x0`，我们引入一个 causal 专属基类： - -- `ModelBase`：保持通用 primitives（`predict_noise/predict_x0/add_noise/...`），**不出现 cache**。 -- `CausalModelBase(ModelBase)`：仅定义 causal/streaming 所需的最小契约，形态参考 FastGen。 - -FastGen 的经验是：cache 是 **causal network 内部可变状态**，method 不传递 cache 张量本体。 -method 只通过 “清空缓存 + 运行 forward（可选 store_kv）” 来驱动 streaming rollout。 - -因此在我们的 `CausalModelBase` 中更优雅的 API 组合是： - -- `clear_caches(handle, cache_tag=...)`：开始新 rollout / 新 validation request 前清空。 -- `predict_noise_streaming(..., cache_tag=..., store_kv=..., cur_start_frame=...)`： - 在 causal 路径中通过 `store_kv` 控制是否写入历史 cache。 - -而不是设计成： - -- `predict_noise_kvcache(kv_cache=...)`：这会迫使 method 管理 cache 生命周期与结构（污染算法层）， - 且最终仍需要额外的 reset/init API。 - -## 3. 目录层级拆分（models） - -将 `fastvideo/distillation/models/*` 拆成更清晰的层级结构，按 “family + variant” 组织： - -```text -fastvideo/distillation/models/ - base.py # ModelBase：方法无关的 operation-centric primitives - wan/ - __init__.py - wan.py # Wan（T2V）bidirectional primitives - wangame/ - __init__.py - wangame.py # WanGame bidirectional primitives（不含 cache） - wangame_causal.py # WanGame causal primitives（包含 streaming/cache） -``` - -注： - -- `wangame_causal.py` 的职责是提供 “causal 专属的 predict/rollout primitive”，例如： - - `clear_caches()`（或 `clear_*`）等 cache 生命周期管理； - - `store_kv` / cache reset / cache update 的封装（cache **由 causal 变体内部持有**）； - - streaming rollout 所需的 block/chunk 约束。 -- `wangame.py` 保持纯 bidirectional（不引入 cache 语义）。 - -## 4. 运行时组织（role → 独立网络实例） - -对齐 FastGen：**每个 role 持有自己的 transformer 实例**（student/teacher/critic 可以是不同类/不同权重）， -但 preprocessors 只由 student 管理。 - -落地方式（保持我们现有 RoleManager/RoleHandle 架构）： - -- `RoleHandle.modules["transformer"]`：每个 role 独立的 denoiser/transformer -- `ModelBase`（或 family-specific model plugin）： - - 统一持有 student preprocessors（VAE/text encoder…） - - 统一实现 batch→forward primitives 的规范化 - - 根据 role 的 variant 选择对应的 bidi/causal 实现路径（必要时由 `roles..extra.variant` 决定） - -## 5. 风险点与验证 - -### 5.1 latent 语义一致性 - -该设计隐含前提是：teacher/student/critic 共享同一 latent 语义（通常意味着同一 VAE 语义）。 -如果未来要支持跨 family（例如 teacher=SDXL, student=Wan），需要额外的对齐层（暂不在本 phase 处理）。 - -### 5.2 cache 生命周期与 no-grad 语义 - -causal cache 的更新通常应在 `torch.no_grad()` 下进行，以避免历史 cache 引入梯度/爆显存。 -需要在实现阶段明确： - -- cache 初始化/重置的时机（每个 rollout / 每个 validation request） -- cache 的 dtype/device 与 FSDP/activation checkpoint 的交互 - -补充（已落地）： - -- 在 self-forcing 的 streaming rollout 中，`store_kv=True` 会在 forward 过程中更新 cache； - 但当 `training.enable_gradient_checkpointing_type: full` 开启时，torch 会在 backward 对 - checkpointed blocks 进行 recompute，这要求 forward→backward 生命周期内“cache 可观测状态” - 必须稳定（至少对同一 forward 调用而言）。 -- 我们采用“方案 A”在 `models/wangame/wangame_causal.py` 内部做了两点保证： - - **index snapshot**：对每次 grad-enabled 的 streaming forward，快照 cache 的 end-index 张量； - - **capacity 扩容**：checkpoint-safe 时扩大 cache，保证 cache append-only（不触发 rolling/eviction）。 - -### 5.3 验证策略 - -最小验证集合： - -- bidirectional Wan/WanGame：原有 DMD2/finetune 训练与 validator 不回归； -- causal WanGame：streaming rollout 能跑通（block mask 正确、cache 正确更新）； -- 多 role：teacher=bidi + student=causal 能正确构建并 forward。 - -## 6. TODO（实施时的文件清单） - -- [x] `fastvideo/distillation/models/` 目录结构调整（新增子目录、移动文件、更新 imports） -- [x] preprocessors 收敛到 student：移除 teacher/critic 侧的 preprocessor 初始化与依赖 -- [x] `wangame_causal.py`：封装 cache/streaming primitives（仅 causal 需要） -- [x] 更新 `dispatch.py` / `register_model` 的 import 路径（保持注册行为不变） -- [x] 方案 A：支持 activation checkpointing 下的 streaming rollout（index snapshot + capacity 扩容） -- [ ] 进一步工程化：把“checkpoint-safe cache”做成可显式开关（必要时打印内存提示/风险） -- [ ] 更新必要的 doc（只更新本 phase 相关文档） diff --git a/dev/phase_causal2.md b/dev/phase_causal2.md deleted file mode 100644 index e7abb6a9b..000000000 --- a/dev/phase_causal2.md +++ /dev/null @@ -1,171 +0,0 @@ -# Phase: Causal2（更贴近 FastGen 的“student 管 shared components”语义） - -> 目标：进一步把 “shared / expensive components（VAE / text encoder / image encoder / …）” -> 的生命周期与加载语义收敛到 **student**，并让一个 run 内部自然支持 -> “student/teacher/critic 多模型（transformers）并存”，而不引入额外的中间概念。 - -## 0. 当前进度（Snapshot） - -截至目前： - -- ✅ 修复 DCP checkpoint 在开启 activation checkpointing（`enable_gradient_checkpointing_type: full/ops/...`） - 时漏存大量 transformer 参数的问题（根因是 wrapper 导致 `named_parameters()` 与 - `get_model_state_dict()` key 不一致，见 5. 风险与边界）。 -- ✅ warmstart（`roles.*.init_from_checkpoint`）改为 best-effort：仅加载 checkpoint 里实际存在的 keys， - 避免 DCP planner 因 “Missing key” 直接失败（对于历史不完整 ckpt 也能继续跑，但缺失的权重当然无法凭空恢复）。 -- ✅ 新增 `recipe.family: wangame_causal`（更 FastGen 风格）：causal by default。 -- ✅ 落地 `roles.shared_component_role`（默认 student），用于显式指定 shared components owner - （并把 `training.model_path` 的默认来源切换到该 role）。 -- ✅ 文档化：补齐 causal model/method 文件头的 config keys 声明（降低阅读成本）。 - -## 1. 背景：FastGen 的语义是什么？ - -FastGen 在配置层面并不使用 `variant: causal` 这种字段，而是通过“选择不同的 net 配置类” -来表达“我要跑 causal 还是 bidirectional”： - -- `config.model.net = CausalWan_1_3B_Config` → student 是 causal -- `config.model.teacher = Wan_1_3B_Config` → teacher 仍然可以是 bidirectional - -它的核心思路可以概括成两点： - -1) **student/net 是主模型（main network）**：很多“数据侧/推理侧的 shared 组件” - 会天然围绕它组织（例如 preprocessors、一些全局 shape 约束等）。 -2) **teacher 只是额外的网络实例**：提供 target/score，不应该因为 teacher 存在就把 - preprocessors 再加载一份、再维护一份生命周期。 - -## 2. 我们想对齐的“语义”是什么？ - -### 2.1 只让 student 负责 shared components（preprocessors） - -定义 shared components（示例）： - -- VAE（latent encode/decode + latent norm） -- text encoder / tokenizer（如果训练/验证需要 text conditioning） -- image encoder（I2V 可能需要） -- 其它可能跨 role 复用、且体积/显存开销大的组件 - -原则： - -- **一个 run 只加载一份 shared components**，默认由 `roles.student` 持有/提供。 -- teacher/critic role **只加载它们各自的 transformer**（以及必要的轻量配置），不再重复加载 VAE 等。 - -收益： - -- 显存与初始化成本显著降低（尤其 teacher 大模型时）。 -- 更清晰的职责划分：teacher 是“提供 target 的网络”，不是“数据编解码提供者”。 - -### 2.2 一个 run 直接持有多个模型（多 role transformers） - -对齐 FastGen：“一个训练 job 同时持有 student + teacher（+ critic/…）”。 - -在我们的框架里，最自然的落地是： - -- `RoleManager` 仍然是“多 role modules”的容器（student/teacher/critic/...）。 -- model plugin 负责： - - 构建每个 role 的 transformer（以及 trainable 开关/activation checkpointing 等）； - - 加载 shared components（只加载一次）； - - 把 dataloader batch 规范化为 methods 可用的 forward primitives； - - 提供 operation-centric primitives（`add_noise / predict_noise / predict_x0 / backward / ...`）。 - -也就是说：**“多模型并存”是运行时事实，不需要额外引入 ModelComponents 之类中间层**。 - -## 3. 配置（YAML）层面的约定 - -### 3.1 shared components 的来源：`roles.shared_component_role`（默认 student） - -为了把 “谁负责 shared components” 变成**显式语义**（而不是硬编码只认 student）， -引入一个模型层面的配置段： - -```yaml -roles: - # 哪个 role 负责加载/持有 shared components（VAE / text encoder / image encoder / …） - # 默认：student - shared_component_role: student -``` - -约束（从简）： - -- 该字段必须是一个 role name（string），且 `roles.` 必须存在；否则直接报错。 -- shared components 的加载路径 = `roles[shared_component_role].path` -- `training.model_path` 的默认来源也应当改为这个 role（用于 pipeline registry / 组件加载等）。 - -动机： - -- 对齐 FastGen 的 “main network” 语义：它是一个概念，不一定永远叫 `student`。 -- 未来如果出现 “只有 teacher 但没有 student 的 recipe”（例如纯评估/蒸馏变体），也能工作。 - -从简规则（不做 fallback / 不做复杂合并）： - -- 如果不写 `roles.shared_component_role`，默认等价于 `student`。 - -未来如果需要支持 “teacher/student family 不同”，再引入更复杂的机制（例如多套 shared components）。 - -### 3.2 causal / bidirectional 的选择（FastGen 风格） - -我们可以同时支持两种表达方式（但优先推荐 FastGen 风格）： - -- FastGen 风格（推荐）：通过 `recipe.family` 选择模型变体 - - `recipe.family: wangame` / `recipe.family: wangame_causal` - - `wangame_causal` 默认所有 role 走 causal transformer（除非 role 显式声明 bidi） -- 兼容表达（可选）：`roles..variant: causal|bidirectional` - - 用于 “student causal + teacher bidirectional” 等混合场景 - -注：即使使用 `recipe.family: wangame_causal`，仍建议保留 per-role override 的能力， -以覆盖 FastGen 常见组合（student causal + teacher bidi）。 - -## 4. 代码落地点（文件与职责） - -以 wangame 为例： - -```text -fastvideo/distillation/models/wangame/ - common.py # role transformer 构建的共享逻辑（不涉及 preprocessors） - wangame.py # bidi 版本：加载 shared components + bidi primitives - wangame_causal.py # causal 版本：在 wangame.py 基础上增加 cache/streaming primitives - __init__.py # register_model("wangame") (and maybe "wangame_causal") -``` - -关键点： - -- `common.py` 只负责 “按 role 构建 transformer handle”，不加载 VAE/text encoder 等。 -- `wangame.py / wangame_causal.py` 只在 **一个地方**加载 shared components（来自 student path），并复用给所有 role。 -- methods 永远通过 `model.predict_* (handle=RoleHandle, ...)` 这类 operation-centric API 调用网络； - methods 不直接“知道/管理”VAE 等加载细节。 - -## 5. 风险与边界（明确不做） - -- **跨 family shared components**：例如 teacher=SDXL, student=Wan。 - - 这会带来 latent 语义不一致、conditioning schema 不一致等问题。 - - 本 phase 不解决；遇到则应当在构建期直接 error，避免 silent mismatch。 -- **让 config 变成无结构 dict 并把语义搬进 utils**： - - “路径可配置”不等于“语义可抽象”；加载/调度的语义仍然高度 family 相关。 - - 仍坚持 model plugin 层吸收 family 差异，methods 保持算法纯净。 - -### 5.1 DCP checkpoint × activation checkpointing 的 key 对齐（已修复） - -问题现象： - -- 开启 activation checkpointing 时，部分模块会被 `checkpoint_wrapper(...)` 包裹； -- 此时 `named_parameters()` 看到的 key 形如： - - `blocks.0._checkpoint_wrapped_module.weight` -- 但 `torch.distributed.checkpoint.state_dict.get_model_state_dict()` 返回的 key 仍是： - - `blocks.0.weight` -- 如果用 “named_parameters 的 key 集合” 去过滤 `get_model_state_dict()`，就会把 blocks.* 全部过滤掉， - 导致 DCP checkpoint **漏存大部分 transformer 权重**（训练能跑，但 ckpt 不可用 / warmstart 会 Missing key）。 - -修复策略： - -- 在 `ModelWrapper.state_dict()` 里对 `._checkpoint_wrapped_module.` 做 canonicalize： - - 既保留原始 key，也加入 unwrapped key（把 `._checkpoint_wrapped_module.` 替换成 `.`） - - 从而保证 `get_model_state_dict()` 的 key 能被正确匹配并保存。 - -对历史 checkpoint 的影响： - -- 旧 checkpoint 里缺失的权重无法恢复（除非另有完整的 pretrained/export）；但 warmstart 现在会 best-effort 加载已有 keys， - 不再因为 DCP planner strict missing key 直接 crash。 - -## 6. TODO（实施清单） - -- [x] 新增 `recipe.family: wangame_causal`（更 FastGen 风格），默认所有 role 为 causal -- [x] 落地 `roles.shared_component_role`（默认 student）用于 shared components 的 owner（含：`training.model_path` 默认来源切换） -- [x] 文档化：每个 model/method 文件开头声明本文件会读取的 config keys（降低阅读成本） diff --git a/dev/phase_mode.md b/dev/phase_mode.md deleted file mode 100644 index 142b33c1d..000000000 --- a/dev/phase_mode.md +++ /dev/null @@ -1,238 +0,0 @@ -# Phase：Model Plugin “去 Adapter 化”（命名收敛 + 去掉 ModelComponents 中间层） - -> 本阶段只做 **结构与命名** 的收敛，目标是让代码读起来更直观、更少概念。 -> 训练/验证行为应保持不变（同 YAML、同 loss、同 pipeline 选择）。 -> -> 你提出的方向：`ModelBase / WanModel / WanGameModel`，模型构建由 `__init__`（或 `from_config`）自行管理， -> 从而不再需要 `ModelComponents` 这种“中间件”——我认为是合理且优雅的。 - ---- - -## 0) 背景：为什么现在 “Adapter” 这个词让人困惑 - -当前 `fastvideo/distillation/models/{wan,wangame}.py` 里所谓的 `*Adapter`，实际上已经承担了： - -- family 相关的 runtime primitives（`prepare_batch/add_noise/predict_noise/predict_x0/backward/...`） -- shared components 的持有（`vae/noise_scheduler`）与生命周期管理 -- 训练期状态（RNG、negative conditioning cache 等） - -这更像“模型运行时封装（model plugin / runtime model）”，而不是一个“薄 adapter”。 - -同时 `ModelComponents` 只是把这些东西再打包一次，本质信息冗余。 - ---- - -## 1) 目标 / 非目标 - -### ✅ 目标（本阶段要做到） - -1. **命名收敛**：彻底去掉 `Adapter` 这个词。 - - `ModelAdapter` → `ModelBase` - - `WanAdapter` → `WanModel` - - `WanGameAdapter` → `WanGameModel` -2. **去掉 build-time 中间件**：删除 `ModelComponents`（以及对应的 `models/components.py`）。 - - “构建产物”不再是 dataclass，而是 **模型对象本身**（它天然持有 bundle/dataloader/validator）。 -3. **dispatch 更直觉**： - - `@register_model("wan")` 直接注册 `WanModel` 类（或其 factory），而不是注册 `build_wan_components()` 这种函数。 - - `dispatch.build_runtime_from_config()` 变成：`model = WanModel(cfg)` → `method = ...build(..., model=model)`。 -4. **保持 method 仍是 role-centric**： - - method 仍通过 role（student/teacher/critic/…）决定算法逻辑； - - model 只提供 operation-centric primitives，不因为 role 增多而出现“role 爆炸式 API”。 - -### ❌ 非目标（本阶段不做） - -- 不改 YAML schema（保持现有 schema v2）。 -- 不改算法行为（DMD2/finetune/dfsft 的 loss、rollout、optimizer cadence 不变）。 -- 不改 pipeline/validator 选择逻辑（仍由 method/validator 按 config 决定）。 -- 不引入 multi-family roles（先保持 `recipe.family` 主导一个 family）。 - ---- - -## 2) 新的核心抽象：ModelBase / WanModel / WanGameModel - -### 2.1 `ModelBase`（替代 `ModelAdapter` + `ModelComponents`） - -建议文件：`fastvideo/distillation/models/base.py` - -`ModelBase` 是一个 **“模型插件对象”**，它既是构建产物，也是 runtime boundary。 - -它必须持有（构建期产物）： - -- `training_args` -- `bundle: RoleManager` -- `dataloader` -- `validator`（可选） -- `start_step`（用于 resume） - -同时提供（runtime primitives）： - -- `num_train_timesteps` -- `shift_and_clamp_timestep(t)` -- `on_train_start()` / `get_rng_generators()` -- `prepare_batch(...)` -- `add_noise(...)` -- `predict_noise(handle, ...)` -- `predict_x0(handle, ...)` -- `backward(loss, ctx, grad_accum_rounds)` - -> 说明：这基本就是现有 `ModelAdapter` 的抽象 + `ModelComponents` 的字段合并。 - ---- - -## 3) 代码改动设计（按文件列 TODO） - -### 3.1 `fastvideo/distillation/models/adapter.py` → `.../models/base.py` - -- [ ] 文件改名：`adapter.py` → `base.py` -- [ ] 类改名：`ModelAdapter` → `ModelBase` -- [ ] 文档/注释同步：强调这是 “model plugin object”，而非薄 adapter - -### 3.2 `fastvideo/distillation/models/components.py` 删除 - -- [ ] 删除 `ModelComponents` dataclass -- [ ] 删除其在 dispatch / models 插件中的引用 - -### 3.3 `fastvideo/distillation/models/wan.py` - -将结构从： - -- `class WanAdapter(ModelAdapter): ...` -- `@register_model("wan") def build_wan_components(...) -> ModelComponents` - -改成： - -- `@register_model("wan") class WanModel(ModelBase): ...` - -#### 建议实现形态 - -```py -@register_model("wan") -class WanModel(ModelBase): - def __init__(self, *, cfg: DistillRunConfig) -> None: - # 1) parse + validate cfg - # 2) build shared components (vae/noise_scheduler) - # 3) build roles -> RoleManager - # 4) build validator (optional) - # 5) build dataloader - # 6) init runtime caches (rng / negative conditioning state) -``` - -把原本 `WanAdapter` 的方法体原封不动迁到 `WanModel` 上即可（第一版只做搬迁/改名）。 - -> 注意：`ensure_negative_conditioning()` 目前依赖 `prompt_handle`（student transformer + prompt encoding pipeline）。 -> `WanModel` 仍可用 `self.student_handle = self.bundle.role("student")` 解决。 - -### 3.4 `fastvideo/distillation/models/wangame.py` - -同 `wan.py`： - -- `WanGameAdapter` → `WanGameModel` -- `build_wangame_components(...) -> ModelComponents` → `WanGameModel(cfg)` - -需要保持： - -- streaming validation 的 `num_frames` 约束（`1 + 4k` 且 latent 可被 `num_frame_per_block` 整除） -- validator pipeline 选择逻辑不变（parallel vs streaming / ode vs sde / ode_solver） - -### 3.5 `fastvideo/distillation/dispatch.py` - -当前： - -- `_MODELS: dict[str, ModelBuilder]`（builder 返回 `ModelComponents`） - -改为： - -- `_MODELS: dict[str, type[ModelBase]]`（或 `Callable[[DistillRunConfig], ModelBase]`） - -并把 `build_runtime_from_config()` 改为： - -```py -model_cls = get_model(cfg.recipe.family) -model = model_cls(cfg=cfg) - -method_cls = get_method(cfg.recipe.method) -method = method_cls.build(cfg=cfg, bundle=model.bundle, model=model, validator=model.validator) - -return DistillRuntime( - training_args=model.training_args, - method=method, - dataloader=model.dataloader, - start_step=model.start_step, -) -``` - -> 这里建议把传参从 `adapter=` 改名为 `model=`，让含义更直观。 - -### 3.6 `fastvideo/distillation/methods/base.py`（以及所有 methods） - -目标：把 method 里对 `self.adapter` 的依赖改成对 `self.model` 的依赖。 - -- [ ] `DistillMethod.build(...)` 签名建议改为： - - `build(cfg, bundle, model, validator)`（或更简化：`build(cfg, model)`） -- [ ] methods 内部字段： - - `self.model` 替代 `self.adapter` -- [ ] `on_train_start()` 里调用 `self.model.on_train_start()` -- [ ] `get_rng_generators()` 读取 `self.model.get_rng_generators()` - -> 这一改动对行为应是 0 diff（只是字段名变化）。 - -### 3.7 文档与 “Config keys used” 头部更新 - -- [ ] `models/*.py` 的 “Config keys used” 头部同步改成 “consumed by WanModel/WanGameModel” -- [ ] 方法文件、validator 文件不强制改（但建议把 “adapter” 的文字替换为 “model”） - ---- - -## 4) 兼容性与风险点（需要提前明确） - -### 4.1 导入/注册顺序（避免循环 import） - -要求： - -- `dispatch.ensure_builtin_registrations()` 仍然显式 import `fastvideo.distillation.models.wan` 等模块， - 让 `@register_model` 在 import 时完成注册。 -- `models/*.py` 只 import `register_model`（不要反向 import dispatch 里的 heavy objects）。 - -### 4.2 Checkpoint/RNG 断点续训 - -目前 checkpoint manager 通过： - -- `runtime.method.get_rng_generators()`（优先） -- fallback 到 `runtime.method.adapter.get_rng_generators()` - -重构后建议： - -- `runtime.method.get_rng_generators()` 永远存在并返回 `self.model.get_rng_generators()`， - 不再做 adapter fallback。 - -### 4.3 行为不变（Definition of Done） - -必须满足： - -- `fastvideo/training/distillation.py --config ` 能跑通： - - wan: DMD2 / finetune - - wangame: finetune / dmd2 / dfsft（尤其 streaming validation） -- 静态检查至少通过： - - `python -m py_compile`（相关文件） - - `ruff check`（相关文件） - ---- - -## 5) 实施顺序（推荐最小风险落地） - -1) **纯改名 + 0 行为改动** -- 先在 models 内部把 `WanAdapter` 改名 `WanModel`（类名、注释、引用） -- `ModelAdapter` 改名 `ModelBase` - -1) **去掉 ModelComponents** -- model plugin 不再返回 dataclass，而是返回模型对象本身 -- dispatch 改为实例化模型对象 - -1) **methods 参数名收敛** -- `adapter` 全部改为 `model` - -做到这一步就能得到一个更直觉的结构： - -- `models/` 里就是“模型插件对象” -- `methods/` 只看 `model`（operation-centric）+ `bundle`（role-centric） -- dispatch 就是 “cfg -> model -> method -> trainer” diff --git a/dev/phase_refactor.md b/dev/phase_refactor.md deleted file mode 100644 index 802728b62..000000000 --- a/dev/phase_refactor.md +++ /dev/null @@ -1,676 +0,0 @@ -# Phase: Refactor — `_target_` instantiate-first, drop the string registry - -## 0) FastGen hierarchy (the reference) - -``` -FastGenNetwork (ABC, nn.Module) ← per-role backbone; owns transformer + noise_scheduler - CausalFastGenNetwork (FastGenNetwork) ← extends with chunked processing + KV caches - -FastGenModel (nn.Module) ← base method; owns self.net, self.teacher, optimizers - DMD2Model ← adds self.fake_score, self.discriminator - SelfForcingModel ← causal rollout - SFTModel -``` - -**Key lessons:** - -1. **`CausalFastGenNetwork` extends `FastGenNetwork`** — it inherits the shared contract - (noise_scheduler, init_preprocessors, forward) and adds streaming/chunked methods. - -2. **Every role instance owns its own `noise_scheduler`** — student and teacher each get - an independent instance of the same schedule. No sharing needed. This is because - `FastGenNetwork.__init__` always calls `self.set_noise_schedule()`. - The teacher's `forward(x_t, t, fwd_pred_type="x0")` uses its own scheduler internally - for `pred_noise → x0` conversion. No "who owns the scheduler" problem. - -3. **No mixin class.** FastGen doesn't need one because student and teacher are the - **same class** (`WanNetwork`). The only construction-time difference is that - `FastGenModel.build_model()` calls `self.net.init_preprocessors()` on the student, - but NOT on the teacher. The class itself is neutral — it has `init_preprocessors()` - as a method, but whether it's called is the caller's decision. - For us: same principle. Every `ModelBase` has `init_preprocessors()`. The method's - `build()` calls it only on the student. No mixin. No `is_student` flag. - -4. **Method owns all optimizers** (`self.net_optimizer`, `self.fake_score_optimizer`, …) - via `init_optimizers()`. Subclasses extend with `super().init_optimizers()`. - -5. **`model_dict` / `optimizer_dict` / `scheduler_dict`** are properties on the method, - used by the trainer for checkpointing. - -6. **Dataloader is external** — passed into trainer separately, not owned by the model. - ---- - -## 1) Our hierarchy - -``` -fastvideo/distillation/ -├── models/ -│ ├── base.py ModelBase; CausalModelBase(ModelBase) adds streaming -│ ├── wan/ -│ │ └── model.py WanModel(ModelBase) -│ └── wangame/ -│ ├── model.py WanGameModel(ModelBase) -│ └── model_causal.py WanGameCausalModel(CausalModelBase) -├── methods/ -│ ├── base.py DistillMethod (analogous to FastGenModel) -│ └── distribution_matching/ -│ ├── dmd2.py DMD2Method -│ └── self_forcing.py SelfForcingMethod -└── utils/ - └── instantiate.py resolve_target, instantiate -``` - -**Every model instance has its own `noise_scheduler`** (constructed in `__init__`). -VAE and `prepare_batch` are initialized via `init_preprocessors()`, called only on -the student by `DistillMethod.__init__()`. No mixin. No flag. - -**DistillMethod** owns role model objects (`self.student`, `self.teacher`, …) and -all optimizers as attributes. `RoleManager` is retired. - ---- - -## 2) Class interfaces - -### 2.1 ModelBase and CausalModelBase - -`CausalModelBase` **extends** `ModelBase`, mirroring FastGen's design where -`CausalFastGenNetwork(FastGenNetwork)` inherits the shared contract and adds -streaming methods. This avoids duplicating the entire bidirectional interface. -Methods that require a causal student type-check with `isinstance(student, CausalModelBase)`. - -```python -# fastvideo/distillation/models/base.py - -class ModelBase(ABC): - """Per-role model. Every instance owns its own noise_scheduler. - init_preprocessors() is only called on the student by DistillMethod.__init__(). - """ - transformer: nn.Module - noise_scheduler: Any # always set in __init__ - - def init_preprocessors(self, training_args: Any) -> None: - """Load VAE, seed RNGs. Analogous to FastGenNetwork.init_preprocessors(). - Only called on the student. Default: no-op (teacher/critic skip this). - """ - pass - - def on_train_start(self) -> None: - pass - - def get_rng_generators(self) -> dict[str, torch.Generator]: - return {} - - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: str = "data", - ) -> TrainingBatch: - raise NotImplementedError( - f"{type(self).__name__}.prepare_batch() requires init_preprocessors() " - "(student only)." - ) - - def add_noise(self, clean: Tensor, noise: Tensor, timestep: Tensor) -> Tensor: - raise NotImplementedError - - @abstractmethod - def predict_noise( - self, - noisy_latents: Tensor, - timestep: Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict | None = None, - attn_kind: str = "dense", - ) -> Tensor: ... - - @abstractmethod - def predict_x0(self, ...) -> Tensor: ... - - def backward(self, loss: Tensor, ctx: Any, *, grad_accum_rounds: int = 1) -> None: - (loss / max(1, grad_accum_rounds)).backward() - - -class CausalModelBase(ModelBase): - """Extends ModelBase with streaming/causal methods. - Mirrors FastGen: CausalFastGenNetwork(FastGenNetwork) adds chunked processing + KV caches. - Inherits all bidirectional methods; adds streaming variants. - """ - - @abstractmethod - def predict_noise_streaming( - self, - noisy_latents: Tensor, - timestep: Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cache_tag: str = "pos", - store_kv: bool = False, - cur_start_frame: int = 0, - cfg_uncond: dict | None = None, - attn_kind: str = "dense", - ) -> Tensor | None: ... - - @abstractmethod - def predict_x0_streaming(self, ...) -> Tensor | None: ... - - @abstractmethod - def clear_caches(self, *, cache_tag: str = "pos") -> None: ... -``` - -### 2.2 Dynamic config — `__init__` signature as schema - -We abandon structured config dataclasses (no `DistillRunConfig`, no `RecipeSpec`, -no `attrs` configs). Instead, each `_target_` class declares its own schema via its -`__init__` type annotations. The `instantiate()` utility: - -1. Pops `_target_` and resolves the class. -2. Inspects the constructor signature to find accepted parameters. -3. Warns on any unexpected keys (catches typos like `trainabel` instead of `trainable`). -4. Passes only recognized keys to the constructor. - -This is the "whole config dict flows through" model: callers don't need to know which -keys a constructor cares about, and adding a field to a class means adding it to -`__init__` — no schema file to maintain in parallel. But unlike a blind `**kwargs` -pass-through, we catch silent misconfiguration at startup. - -```python -# fastvideo/distillation/utils/instantiate.py - -import importlib -import inspect -import logging -from typing import Any - -logger = logging.getLogger(__name__) - - -def resolve_target(target: str) -> type: - module_path, cls_name = target.rsplit(".", 1) - return getattr(importlib.import_module(module_path), cls_name) - - -def instantiate(cfg: dict[str, Any], /, **extra: Any) -> Any: - """_target_ instantiation with signature-based validation. - - 1. Pops _target_, resolves the class. - 2. Inspects __init__ signature — if it accepts **kwargs, all keys pass through. - Otherwise, warns on unrecognized keys and filters them out. - 3. Catches typos at startup instead of silently applying defaults. - """ - cfg = dict(cfg) - target_str = cfg.pop("_target_") - cls = resolve_target(target_str) - merged = {**cfg, **extra} - - sig = inspect.signature(cls.__init__) - params = sig.parameters - has_var_keyword = any( - p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values() - ) - - if not has_var_keyword: - # Strict mode: only pass recognized params, warn on extras. - accepted = { - name for name, p in params.items() - if p.kind in ( - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.KEYWORD_ONLY, - ) and name != "self" - } - unexpected = set(merged) - accepted - if unexpected: - logger.warning( - "%s.__init__ does not accept: %s (accepted: %s). " - "Check for typos in YAML.", - cls.__name__, sorted(unexpected), sorted(accepted), - ) - merged = {k: v for k, v in merged.items() if k in accepted} - - return cls(**merged) -``` - -**Each class is self-documenting via its explicit params.** Constructors that want -to be strict omit `**kwargs`; those that legitimately need pass-through keep it. - -```python -class WanGameModel(ModelBase): - def __init__( - self, - *, - init_from: str, # required — TypeError at startup if missing from YAML - trainable: bool = True, # optional with default - # No **kwargs — instantiate() will warn on unrecognized keys - ) -> None: ... -``` - -### 2.4 Concrete model classes (WanGame example) - -```python -# fastvideo/distillation/models/wangame/model.py - -class WanGameModel(ModelBase): - """Bidirectional WanGame model. Can be student or teacher/critic. - Always builds noise_scheduler in __init__. - VAE and prepare_batch are activated by init_preprocessors() — only for student. - """ - - def __init__(self, *, init_from: str, trainable: bool = True) -> None: - self.transformer = load_transformer(init_from, cls="WanGameActionTransformer3DModel") - apply_trainable(self.transformer, trainable=trainable) - # noise_scheduler always built — same as FastGenNetwork.__init__ calling set_noise_schedule() - self.noise_scheduler = FlowMatchEulerDiscreteScheduler(...) - # preprocessor state — inactive until init_preprocessors() is called - self.vae: Any | None = None - self._noise_gen_cpu: torch.Generator | None = None - self._noise_gen_cuda: torch.Generator | None = None - self._init_from = init_from - - def init_preprocessors(self, training_args: Any) -> None: - """Activate as student. Analogous to FastGenNetwork.init_preprocessors().""" - self.training_args = training_args - self.vae = load_module_from_path(self._init_from, module_type="vae", ...) - self.device = get_local_torch_device() - self._init_timestep_mechanics() - - def on_train_start(self) -> None: - seed = self.training_args.seed - self._noise_gen_cpu = torch.Generator(device="cpu").manual_seed(seed) - self._noise_gen_cuda = torch.Generator(device=self.device).manual_seed(seed) - - def prepare_batch(self, raw_batch, *, ...) -> TrainingBatch: - if self.vae is None: - raise RuntimeError("prepare_batch requires init_preprocessors() (student only)") - # ... full WanGame batch preparation - return training_batch - - def add_noise(self, clean, noise, timestep) -> Tensor: - return self.noise_scheduler.add_noise(...) - - def predict_noise(self, noisy_latents, timestep, batch, *, conditional, ...) -> Tensor: - # Uses self.noise_scheduler (always available) and self.transformer - with autocast(...), set_forward_context(...): - kwargs = self._build_input_kwargs(noisy_latents, timestep, batch, conditional=conditional, ...) - return self.transformer(**kwargs).permute(0, 2, 1, 3, 4) - - def predict_x0(self, noisy_latents, timestep, batch, *, conditional, ...) -> Tensor: - pred_noise = self.predict_noise(noisy_latents, timestep, batch, conditional=conditional, ...) - # Uses self.noise_scheduler — always available, no dependency on student - return pred_noise_to_pred_video(pred_noise, noisy_latents, timestep, self.noise_scheduler) - - - -# fastvideo/distillation/models/wangame/model_causal.py - -class WanGameCausalModel(CausalModelBase): - """Causal WanGame. Extends CausalModelBase with streaming ops. - Same init_preprocessors() pattern — called only on student. - """ - def __init__(self, *, init_from: str, trainable: bool = True) -> None: - self.transformer = load_transformer(init_from, cls="CausalWanGameTransformer") - apply_trainable(self.transformer, trainable=trainable) - self.noise_scheduler = FlowMatchEulerDiscreteScheduler(...) - self.vae = None - ... - - def init_preprocessors(self, training_args: Any) -> None: - # Same pattern as WanGameModel.init_preprocessors() - ... - - def predict_noise_streaming(self, ...) -> Tensor | None: ... - def predict_x0_streaming(self, ...) -> Tensor | None: ... - def clear_caches(self, *, cache_tag: str = "pos") -> None: ... -``` - -### 2.5 DistillMethod (analogous to FastGenModel) - -Like FastGen's `FastGenModel.__init__` which calls `build_model()` → `init_preprocessors()` -internally, our `DistillMethod.__init__` is the single entry point. Each subclass's -`__init__` validates roles, calls `init_preprocessors()` on the student, and calls -`init_optimizers()`. No separate `build()` classmethod — one construction path, no -risk of someone calling `__init__` directly and skipping validation. - -```python -class DistillMethod(nn.Module, ABC): - - def __init__( - self, - *, - cfg: RunConfig, - role_models: dict[str, ModelBase], - validator: Any | None, - ) -> None: - """Subclasses override __init__, call super().__init__(), then: - 1. Validate and assign role models (self.student, self.teacher, ...). - 2. Call self.student.init_preprocessors(cfg.training_args). - 3. Call self.init_optimizers(). - Dataloader is NOT passed — trainer owns it. - """ - super().__init__() - - @abstractmethod - def init_optimizers(self) -> None: ... - - @abstractmethod - def single_train_step(self, raw_batch, iteration, **kwargs) -> ...: ... - - @abstractmethod - def get_optimizers(self, iteration: int) -> list[Optimizer]: ... - - @abstractmethod - def get_lr_schedulers(self, iteration: int) -> list[Any]: ... - - @property - @abstractmethod - def model_dict(self) -> dict[str, nn.Module]: ... - - @property - @abstractmethod - def optimizer_dict(self) -> dict[str, Optimizer]: ... - - @property - @abstractmethod - def scheduler_dict(self) -> dict[str, Any]: ... -``` - -### 2.6 SelfForcingMethod (example) - -```python -class SelfForcingMethod(DistillMethod): - - def __init__(self, *, cfg, role_models, validator): - super().__init__(cfg=cfg, role_models=role_models, validator=validator) - - # 1. Validate and assign roles - student = role_models["student"] - teacher = role_models["teacher"] - critic = role_models["critic"] - - if not isinstance(student, CausalModelBase): - raise TypeError( - f"SelfForcingMethod requires CausalModelBase student, " - f"got {type(student).__name__}" - ) - - self.student = student - self.teacher = teacher - self.critic = critic - self.validator = validator - self.cfg = cfg.method - - # 2. Init preprocessors on student only — same as FastGen's build_model() - self.student.init_preprocessors(cfg.training_args) - - # 3. Init optimizers (after preprocessors, before FSDP wrapping) - self.init_optimizers() - - def init_optimizers(self) -> None: - self.student_optimizer = build_optimizer(self.student.transformer, ...) - self.student_lr_scheduler = build_scheduler(self.student_optimizer, ...) - self.critic_optimizer = build_optimizer(self.critic.transformer, ...) - self.critic_lr_scheduler = build_scheduler(self.critic_optimizer, ...) - - def single_train_step(self, raw_batch, iteration, **kwargs): - batch = self.student.prepare_batch(raw_batch, ...) # student owns this - noisy = self.student.add_noise(batch.latents, batch.noise, batch.timesteps) - - student_x0 = self.student.predict_x0(noisy, batch.timesteps, batch, ...) - teacher_x0 = self.teacher.predict_x0(noisy, batch.timesteps, batch, ...) - # teacher uses its OWN noise_scheduler for pred_noise→x0 — no sharing needed - ... - - @property - def model_dict(self): - return {"student": self.student.transformer, "critic": self.critic.transformer} - - @property - def optimizer_dict(self): - return {"student": self.student_optimizer, "critic": self.critic_optimizer} - - @property - def scheduler_dict(self): - return {"student": self.student_lr_scheduler, "critic": self.critic_lr_scheduler} -``` - ---- - -## 3) Final YAML - -### WanGame Self-Forcing - -```yaml -log: - project: fastvideo - group: wangame - name: self_forcing_causal_student_bidi_teacher - wandb_mode: online - -trainer: - output_dir: outputs/wangame_self_forcing_refactor - max_train_steps: 100000 - seed: 1000 - mixed_precision: bf16 - grad_accum_rounds: 1 - -data: - _target_: fastvideo.distillation.utils.dataloader.ParquetDataConfig - data_path: /path/to/wangame/parquet - dataloader_num_workers: 4 - -validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json - every_steps: 100 - sampling_steps: [4] - -models: - student: - _target_: fastvideo.distillation.models.wangame.WanGameCausalModel - init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - trainable: true - teacher: - _target_: fastvideo.distillation.models.wangame.WanGameModel - init_from: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers - trainable: false - critic: - _target_: fastvideo.distillation.models.wangame.WanGameModel - init_from: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - trainable: true - -method: - _target_: fastvideo.distillation.methods.distribution_matching.self_forcing.SelfForcingMethod - rollout_mode: simulate - chunk_size: 3 - student_sample_type: sde - context_noise: 0.0 -``` - -No `is_student` flag in YAML. The method's `build()` decides which role gets -`init_preprocessors()`. - -### Wan DMD2 - -```yaml -models: - student: - _target_: fastvideo.distillation.models.wan.WanModel - init_from: /path/to/wan_14b - trainable: true - teacher: - _target_: fastvideo.distillation.models.wan.WanModel - init_from: /path/to/wan_14b - trainable: false - fake_score: - _target_: fastvideo.distillation.models.wan.WanModel - init_from: /path/to/wan_14b - trainable: true - -method: - _target_: fastvideo.distillation.methods.distribution_matching.dmd2.DMD2Method - guidance_scale: 5.0 - student_update_freq: 5 -``` - ---- - -## 4) Assembly flow - -```python -cfg = load_run_config(path) - -# 1. Build per-role models. No init_preprocessors() here — that is the method's job. -# Analogous to FastGenModel instantiating self.net and self.teacher as raw networks. -role_models = { - role: instantiate(model_cfg) # just transformer + noise_scheduler - for role, model_cfg in cfg.models.items() -} - -# 2. Dataloader is owned by the trainer. Build it here for trainer use only. -dataloader = instantiate(cfg.data, training_args=training_args) -validator = build_validator(cfg.validation) if cfg.validation.enabled else None - -# 3. Method __init__ validates roles, calls init_preprocessors() on student, -# then calls init_optimizers(). Single construction path — no separate build(). -# Analogous to FastGenModel.__init__ → build_model() → init_preprocessors(). -method_cls = resolve_target(cfg.method["_target_"]) -method = method_cls( - cfg=cfg, - role_models=role_models, - validator=validator, -) - -# 4. FSDP wrapping — trainer wraps models after construction, before training. -# See §4.1 for details. -trainer = DistillTrainer(training_args) -trainer.setup_fsdp(method) - -# 5. Trainer runs the loop. Trainer is the sole owner of the dataloader. -trainer.run(method, dataloader=dataloader, max_steps=cfg.trainer.max_train_steps, ...) -``` - -### 4.1 FSDP integration - -FSDP wrapping happens **after method construction, before training**. The trainer -is responsible for wrapping — the method and models don't know about FSDP. - -```python -# In DistillTrainer.setup_fsdp(method): - -# 1. Wrap trainable models in FSDP using method.model_dict. -for name, module in method.model_dict.items(): - wrapped = FSDP(module, ...) - # Replace the transformer on the owning model object. - # method.student.transformer = wrapped, etc. - -# 2. Non-trainable models (teacher) are cast to precision and moved to device, -# but NOT wrapped in FSDP (no gradients needed). - -# 3. Preprocessors (VAE, text encoders) on the student are moved to device/dtype. -# They live outside FSDP — they are frozen inference-only modules. - -# 4. After FSDP wrapping, optimizers reference the wrapped parameters. -# This means init_optimizers() must run BEFORE FSDP wrapping, and the -# optimizer param groups automatically track the FSDP-wrapped params -# (PyTorch FSDP handles this transparently for params created pre-wrap). -``` - -**Key FSDP decisions:** - -| Concern | Approach | -|---|---| -| Which models get wrapped? | `method.model_dict` — only trainable models | -| Who owns wrapping? | Trainer, not method or model | -| Teacher FSDP? | Configurable: `cfg.trainer.add_teacher_to_fsdp` (default false for frozen teacher) | -| Meta-device init? | `cfg.trainer.fsdp_meta_init` — load weights on meta device, broadcast after wrap | -| Precision? | `cfg.trainer.fsdp_precision` — FSDP MixedPrecision policy applied at wrap time | -| Optimizer creation order? | `init_optimizers()` in `__init__` → FSDP wrap → optimizer params auto-track | - ---- - -## 5) What is retired / changed - -| Current | New | -|---|---| -| `dispatch.py` string registry | Deleted; `_target_` + `instantiate()` | -| `ModelBase` — dispatches via `RoleHandle` arg | Retired; per-role instance owns one transformer | -| `CausalModelBase(ModelBase)` subclass (broken contract) | `CausalModelBase(ModelBase)` clean inheritance — mirrors FastGen | -| `RoleHandle` / `RoleManager` | Retired; method owns `self.student`, `self.teacher`, etc. | -| Optimizers on `RoleHandle.optimizers` | On method: `self.student_optimizer`, etc. | -| `DistillMethod.build(cfg, bundle, model, validator)` | `DistillMethod.__init__(cfg, role_models, validator)` — no dataloader, no build() | -| `recipe.family` / `recipe.method` YAML | Deleted; `models.._target_` + `method._target_` | -| `roles.*` YAML section | `models.*` with `_target_`, `init_from`, `trainable` | -| `is_student` / mixin / SharedContext | None of these. `build()` calls `init_preprocessors()` on student. | - ---- - -## 6) Naming cleanup (separate PR) - -- `DistillMethod` → `Method` -- `DistillRunConfig` → `RunConfig` -- Entrypoint `distillation.py` → `training.py` (optional) - ---- - -## 7) TODO (wangame first, then wan) - -**Infrastructure** - -- [ ] `fastvideo/distillation/utils/instantiate.py` - - `resolve_target(target: str) -> type` - - `instantiate(cfg: dict, **extra) -> Any` - - Uses `inspect.signature()` for kwargs validation and warning on unrecognized keys (see §2.2) - -- [ ] `fastvideo/distillation/utils/config.py` - - New `RunConfig` dataclass - - New `load_run_config(path)` parser - - Keep `load_distill_run_config` as deprecated shim - -**Base classes** - -- [ ] `fastvideo/distillation/models/base.py` - - New `ModelBase` ABC with `init_preprocessors` (no-op default), `prepare_batch` (raises), `add_noise` (raises) - - New `CausalModelBase(ModelBase)` — inherits shared contract, adds streaming methods - - Retire old `ModelBase` / `CausalModelBase` - -- [ ] `fastvideo/distillation/roles.py` — retire `RoleHandle` / `RoleManager` - -- [ ] `fastvideo/distillation/methods/base.py` - - New `DistillMethod.__init__(cfg, role_models, validator)` — single construction path, no build() - - Add abstract `init_optimizers`, `model_dict`, `optimizer_dict`, `scheduler_dict` - -**WanGame** - -- [ ] `fastvideo/distillation/models/wangame/model.py` - - `WanGameModel(ModelBase)`: always builds `noise_scheduler`; `init_preprocessors()` loads VAE - - No `**kwargs` — let `instantiate()` catch typos - -- [ ] `fastvideo/distillation/models/wangame/model_causal.py` - - `WanGameCausalModel(CausalModelBase)`: same pattern + streaming ops - -**Wan** - -- [ ] `fastvideo/distillation/models/wan/model.py` - - `WanModel(ModelBase)`: always builds `noise_scheduler`; `init_preprocessors()` loads VAE + negative prompt - -**Methods** - -- [ ] Update all `__init__` signatures: `SelfForcingMethod`, `DMD2Method`, `FinetuneMethod`, `DFSFTMethod` -- [ ] `DMD2Method.__init__()`: calls `init_preprocessors()` on student; `init_optimizers()` adds `fake_score_optimizer` - -**FSDP** - -- [ ] `DistillTrainer.setup_fsdp(method)` — wraps `method.model_dict` modules in FSDP -- [ ] Configurable teacher FSDP: `cfg.trainer.add_teacher_to_fsdp` (default false) -- [ ] Meta-device init support: `cfg.trainer.fsdp_meta_init` -- [ ] FSDP MixedPrecision policy: `cfg.trainer.fsdp_precision` - -**Dispatch & configs** - -- [ ] `fastvideo/distillation/dispatch.py`: replace with new assembly flow (or delete) -- [ ] New YAML for wangame self-forcing; validate end-to-end -- [ ] Migrate existing wangame and wan YAML configs diff --git a/dev/phases/phase_0.md b/dev/phases/phase_0.md deleted file mode 100644 index 33da22793..000000000 --- a/dev/phases/phase_0.md +++ /dev/null @@ -1,227 +0,0 @@ -# Phase 0：新 Distill 框架落地 + Wan(DMD2) 跑通（实践记录/执行清单） - -目标（Phase 0 的“可交付”）： - -1. 在 FastVideo 内落地一套 **Method/Trainer/Adapter/Bundle** 的 distill 框架骨架， - 代码可 import，可扩展，不影响现有 `fastvideo/training/*distillation_pipeline.py`。 -2. 用新框架跑通 **Wan 的 DMD2**（student+teacher+critic），并提供一个独立入口脚本， - 便于 A/B 对齐旧实现。 -3. 在 Phase 0 就消除一个当前硬耦合：**DMD2 所需的 uncond/neg_condition 不再依赖 validation 副作用**。 - -非目标（Phase 0 暂不强求）： - -- 完整复刻旧 pipeline 的所有日志/可视化/validation 产物（可以逐步补） -- role-based 通用 checkpoint 协议（Phase 0 先复用现有 distill ckpt utils，后续再迁移） -- 支持除 Wan 以外的模型（Phase 0 只做 `WanAdapter`） - ---- - -## Phase 0 TODO List（Review Checklist） - -> 目的：把 Phase 0 的“可交付”拆成可审查的 checklist，方便你决定是否开启 Phase 1。 - -### Phase 0 可交付（应全部完成） - -- [x] 新 distill 框架骨架落地（Method/Trainer/Adapter/Bundle），且 **不影响旧 pipeline** - - `fastvideo/distillation/` -- [x] 新框架跑通 Wan DMD2（student + teacher + critic） - - 入口：`fastvideo/training/wan_distillation_v2.py` - - method：`fastvideo/distillation/methods/wan_dmd2.py` - - adapter：`fastvideo/distillation/adapters/wan.py` -- [x] 消除 DMD2 的隐式耦合：uncond/negative conditioning **不再依赖 validation 副作用** - - `fastvideo/distillation/adapters/wan.py`（`ensure_negative_conditioning()`） -- [x] 多优化器更新节奏下，optimizer step 与 lr_scheduler step 对齐（避免 lr schedule 漂移） - - `fastvideo/distillation/methods/base.py` - - `fastvideo/tests/distillation/test_phase0_schedule.py` -- [x] 提供 few-step distill 示例脚本（可直接改路径运行） - - `examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh` - - `examples/distillation/phase0/temp.sh` -- [x] validation 能在新 Trainer 中被统一触发（Phase 0 先复用旧 `_log_validation`） - - hook：`fastvideo/distillation/trainer.py` - - 实现：`fastvideo/distillation/methods/wan_dmd2.py::log_validation` - -### Phase 0 明确不做 / 延后到 Phase 1+ - -- [ ] 把 `WanDMD2Method` 演进为通用 `DMD2Method`（算法与 Wan 解耦） -- [ ] 把 `WanPipelineAdapter` 演进为真正 `WanAdapter`（不再调用旧 pipeline 私有 helper） -- [ ] v2 path 的 checkpoint/save/resume（role-based） -- [ ] Self-forcing v2（method + adapter） -- [ ] Builder 层：用 config 直接构建 `models={...}`（不再依赖 legacy pipeline 负责加载） - -## 0. 关键风险与应对 - -### 风险 A：`negative_prompt_embeds` 目前只在 validation 中被初始化 - -现状：`fastvideo/training/distillation_pipeline.py` 里，`_prepare_dit_inputs()` 只有在 -`self.negative_prompt_embeds` 已经存在时才会构造 `unconditional_dict`； -而 `self.negative_prompt_embeds` 是在 `_log_validation()` 里通过 -`validation_pipeline.prompt_encoding_stage(...)` 赋值的。 - -如果不跑 validation，DMD2 会在 real_score 的 uncond forward 直接报错 -（`text_dict cannot be None`）。 - -Phase 0 方案：引入一个最小的 `ConditioningProvider`（Wan 版本），在训练开始前： - -- 从 `SamplingParam.from_pretrained(model_path).negative_prompt` 取 negative prompt 字符串 -- 用一个轻量的 prompt encoder（优先复用 `WanDMDPipeline` 的 `prompt_encoding_stage`） - 计算 `negative_prompt_embeds / negative_prompt_attention_mask` -- 缓存到 adapter/method，并确保每个 step 都能显式提供 uncond conditioning - -这一步是 Phase 0 必做，不然新 Trainer/Method 没法脱离 validation。 - -### 风险 B：Phase 0 会短期“仍然 Wan 耦合” - -为了尽快跑通 + 降低风险，Phase 0 允许 `WanAdapter` 通过 **wrap 现有 pipeline 的 helper -methods**（normalize/noise/timestep/attention metadata/build_input_kwargs 等）实现。 - -后续 Phase 1/2 再把这些 helper 从 pipeline 迁移/重写进 adapter，彻底摆脱旧实现。 - ---- - -## 1. 代码落地点（具体到文件） - -> 约定：Phase 0 把新框架放到 `fastvideo/distillation/`。 - -### 1.1 新增 distill 框架骨架 - -- `fastvideo/distillation/__init__.py` - - 导出 Phase 0 需要的核心类（Trainer/Method/Bundle） -- `fastvideo/distillation/roles.py` - - `RoleHandle` / `ModelBundle`：`roles: dict[str, RoleHandle]` -- `fastvideo/distillation/trainer.py` - - `DistillTrainer`：通用训练循环（grad accum + step/zero_grad），不认识 roles -- `fastvideo/distillation/methods/base.py` - - `DistillMethod` 抽象：`single_train_step()`、`get_optimizers()` 等 -- `fastvideo/distillation/adapters/base.py` - - `DistillAdapter` 抽象：`prepare_batch()`、`forward_*()`、conditioning provider hook - -### 1.2 Phase 0 的 Wan 实现（pipeline-backed，先跑通） - -- `fastvideo/distillation/adapters/wan.py` - - `WanPipelineAdapter`： - - 复用 `fastvideo/training/distillation_pipeline.py` 的 helper 方法做数据准备/forward - - 提供 `ensure_negative_conditioning()`，不依赖 validation -- `fastvideo/distillation/methods/wan_dmd2.py` - - `WanDMD2Method`： - - 实现 DMD2 的 loss 计算（generator_loss + fake_score_loss） - - 实现 update schedule（`generator_update_interval`）与 optimizer/scheduler step 对齐 - -### 1.3 独立入口（不影响旧脚本） - -- `fastvideo/training/wan_distillation_v2.py` - - 行为与 `wan_distillation_pipeline.py` 类似，但走新框架： - - 构建 `WanDistillationPipeline.from_pretrained(...)`（仅用于复用现有加载/优化器/dataloader） - - 构建 `WanPipelineAdapter` + `WanDMD2Method` - - 用 `DistillTrainer.run(...)` 启动训练 - -### 1.4 最小单测（CPU 可跑） - -- `fastvideo/tests/distillation/test_phase0_schedule.py` - - 只测 method 的 optimizer/scheduler 选择逻辑是否与 update ratio 对齐 - - 不依赖 GPU/模型权重 - ---- - -## 2. Phase 0 训练循环的行为约定（便于 A/B） - -为了尽量可对齐旧实现，Phase 0 的新 Trainer 约定： - -- global `step` 仍然按旧 pipeline 的语义：从 `init_steps+1` 开始，到 `max_train_steps` -- grad accumulation 由 Trainer 处理（每个 microbatch 调一次 `single_train_step`，最后 step） -- generator optimizer/scheduler 只在 `step % generator_update_interval == 0` 时 step - （这是对旧实现的一个显式修正：旧实现会每步 step generator scheduler） -- fake_score optimizer/scheduler 每步 step - -关于 backward 的一个 Phase 0 现实约束： - -- 由于 FastVideo 的 attention/kernel 依赖 `set_forward_context(...)`，并且训练里常开 - activation checkpointing，**backward 可能触发 forward 重算**，重算时也必须处于正确的 - forward_context 里。 -- 旧实现通过在 backward 前重新 `set_forward_context` 来保证这一点（且 generator/critic - 的 context 可能不同）。 -- 因此 Phase 0 的接口在 `DistillMethod` 里增加 `backward(loss_map, outputs, grad_accum_rounds)` - 这个 hook：Trainer 调用它，但不关心里面怎么拆分 loss/怎么设置 context。 - 默认实现仍然是对 `total_loss` 做 backward；Wan(DMD2) method 会覆写为 - “generator_loss 在 vsa context 下 backward + fake_score_loss 在 normal context 下 backward”。 - -> 如果后续发现这个 scheduler 行为变化会影响 A/B 对齐，我们可以在 Phase 0 -> 加一个 “legacy 模式开关”；但默认先按“optimizer step 对齐 scheduler step”的正确语义实现。 - ---- - -## 3. 开始实践（本次提交会先做到什么程度） - -本次实现优先级： - -1. 新框架骨架文件可 import（`fastvideo.distillation.*`） -2. `WanPipelineAdapter.ensure_negative_conditioning()` 可在无 validation 的情况下生成 neg embeds -3. `WanDMD2Method.single_train_step()` 能产出 `loss_map["total_loss"]` -4. `DistillTrainer.run()` 能跑若干 step（最小 smoke）并 step optimizers -5. 加一个 schedule 单测，确保 `get_optimizers/get_lr_schedulers` 与 update ratio 对齐 - -后续增量（Phase 0 内可迭代）： - -- checkpoint/resume 接入（优先复用 `save_distillation_checkpoint/load_distillation_checkpoint`） -- validation 接入：已通过 `DistillTrainer` -> `method.log_validation(step)` hook - 接入旧 pipeline 的 `_log_validation`（见 `WanDMD2Method.log_validation()`） - ---- - -## 4. “大设计硬伤”停工汇报标准 - -如果在 Phase 0 实践过程中出现以下情况，我会暂停继续写代码并直接汇报你： - -- `models={...}` + adapter 的抽象无法覆盖 Wan 的关键差异（例如 conditioning/CFG 方式根本不一致） -- DMD2 的计算图要求导致 Method/Trainer 的边界必须反转（Trainer 不可算法无关） -- 现有 pipeline 的 helper 复用导致强耦合无法逐步迁移（必须一次性大重构才可跑通） - ---- - -## 5. Phase 0 的“耦合债务”与命名说明（非常重要，避免未来遗忘） - -### 5.1 为什么现在会有 `WanDMD2Method` 这种名字？ - -结论：这是 **Phase 0 的过渡实现**，名字里带 `Wan` 是“刻意暴露耦合”，防止误用。 - -原因：当前 `fastvideo/distillation/methods/wan_dmd2.py` 并不是一个纯算法层的 DMD2。 -它直接复用/依赖了旧实现 `fastvideo/training/distillation_pipeline.py` 的 Wan-only 私有逻辑： - -- DMD2 的关键计算来自旧 pipeline 的内部函数：`_dmd_forward(...)`、`faker_score_forward(...)`、 - `_generator_forward(...)` 等（它们隐含了 layout/normalize/CFG/uncond 等具体假设） -- `fastvideo/distillation/adapters/wan.py` 也在复用旧 pipeline 的 helper： - `_normalize_dit_input/_prepare_dit_inputs/_build_attention_metadata` - -因此它在语义上等价于：**“把旧 Wan distill pipeline 包了一层 Method/Trainer 外壳”**， -而不是一个可对接任意 adapter 的“通用 DMD2Method”。 - -### 5.2 FastGen 有没有类似的做法？ - -FastGen 的命名与分层更“干净”： - -- 算法层叫 `DMD2Model`（算法名），不会叫 `WanDMD2Model` -- 网络/架构差异在 `networks/*` + config 里选择（网络与算法解耦） - -所以我们现在的 `WanDMD2Method` 更像是 Phase 0 的迁移脚手架，而不是最终形态。 - -### 5.3 TODO（必须做）：把 `WanDMD2Method` 演进为 **算法名 method + 模型名 adapter** - -为了避免“又一次耦合到 Wan”，必须把 Phase 0 的耦合逐步清掉，目标对齐 FastGen： - -1) **把算法从旧 pipeline 里抠出来** - - 新增：`fastvideo/distillation/methods/dmd2.py`（`DMD2Method`，不依赖任何具体模型） - - `DMD2Method` 只依赖 adapter 提供的 primitives（noise/pred_to_x0/teacher_cfg/critic_loss 等） - -2) **把模型差异收敛到 adapter（WanAdapter）** - - 演进：`WanPipelineAdapter` -> `WanAdapter` - - `WanAdapter` 不再调用 `DistillationPipeline` 的私有 helper 方法， - 而是自己实现 normalize/layout/attention metadata/输入 kwargs 组装等 - -3) **最终命名与入口应变成** - - `DMD2Method + WanAdapter`（method 不带模型名） - - `fastvideo/training/wan_distillation_v2.py` 里只选择 adapter，不再选择“WanDMD2Method” - -4) **迁移后应删除/冻结 Phase 0 的 pipeline-backed 版本** - - 避免未来复制粘贴 `WanDMD2Method` 去做其它模型（那会把耦合扩散） - -> 备注：Phase 0 用 `WanDMD2Method` 的意义是“先把训练循环与多 optimizer 调度结构稳定下来”， -> 但我们必须把它当成临时脚手架，Phase 1/2 逐步替换为真正解耦的 method+adapter。 diff --git a/dev/phases/phase_1.md b/dev/phases/phase_1.md deleted file mode 100644 index ee0c48d51..000000000 --- a/dev/phases/phase_1.md +++ /dev/null @@ -1,110 +0,0 @@ -# Phase 1：算法/模型解耦落地（DMD2Method + WanAdapter）+ 通用入口 - -Phase 1 的定位：把 distillation 从 “pipeline god object” 拆成稳定边界，让 **算法** 与 -**模型家族** 解耦合，并且让训练入口开始具备 “选择 + instantiate” 的结构（对齐 FastGen -的 catalog 思路）。 - -目标边界（Phase 1 确认有效）： - -- `DistillTrainer`：训练基础设施（accum/step/log hooks），不懂具体模型/算法细节 -- `DistillMethod`：算法编排（DMD2 / Self-Forcing / CM / ...），只依赖 adapter primitives -- `DistillAdapter`：模型家族适配（Wan / CogVideoX / ...），负责 forward/backward context 与 - pipeline 细节 -- `ModelBundle`：`roles -> handles`（modules/optimizers/schedulers），method 通过 role 取用 - -现状：Wan DMD2 的 Phase 1 训练结果已与旧 baseline 对齐，因此 Phase 0 的脚手架代码路径已移除 -（见 TODO H）。 - ---- - -## Phase 1 非目标（明确延后） - -- Self-forcing v2（以及 ODE-init） -- role-based checkpoint/save/resume 协议统一 -- 完整的 validation 抽象（Phase 1 先保留 `method.log_validation()` hook） -- 多模型家族混搭（例如 student=Wan、teacher=SDXL） - ---- - -## Phase 1 TODO List（Review Checklist） - -### A. 方法目录结构（catalog：多层级 methods） - -- [x] 建立 methods 分层目录（至少有目录与 `__init__.py`） - - `fastvideo/distillation/methods/distribution_matching/` - - `fastvideo/distillation/methods/consistency_model/` - - `fastvideo/distillation/methods/knowledge_distillation/` - - `fastvideo/distillation/methods/fine_tuning/` -- [x] `DMD2Method` 放到 `fastvideo/distillation/methods/distribution_matching/dmd2.py` - -### B. 通用 `DMD2Method`（算法层） - -- [x] 新增 `DMD2Method`（算法实现），并显式 `bundle.require_roles(["student","teacher","critic"])` -- [x] `DMD2Method` 不持有 legacy pipeline，不调用 legacy pipeline 私有算法函数 -- [x] 保留关键语义：`generator_update_interval`（只在该 step 更新 student） - -### C. Adapter primitives 契约（让算法真正可复用） - -- [x] 在 `DMD2Method` 内通过 `Protocol` 定义 DMD2 所需 adapter surface（`_DMD2Adapter`） -- [x] `DMD2Method` 仅依赖 primitives，而不是 Wan 细节 - -### D. `WanAdapter`（模型层：forward/backward/context 全部在 adapter） - -- [x] 新增 `fastvideo/distillation/adapters/wan.py::WanAdapter` -- [x] Wan 侧实现 DMD2 所需 primitives（batch prepare / teacher cfg / critic loss / backward 封装） -- [x] `forward_context` 由 adapter 托管（method 不直接触碰 `set_forward_context`） -- [x] `ensure_negative_conditioning()` 显式化（不依赖 validation 的副作用） - -### E. Builder + 通用入口（config -> instantiate） - -- [x] （Phase 1 过渡实现；Phase 2 已移除）`fastvideo/distillation/builder.py::build_wan_dmd2_method` - - 说明：Phase 1 用 legacy pipeline 启动，快速对齐 baseline -- [x] （Phase 2 已改为 YAML-only）通用 distill 入口：`fastvideo/training/distillation.py` - - Phase 1：CLI 驱动；Phase 2：`--config /real/path/to/distill.yaml` -- [x] （Phase 1 过渡实现；Phase 2 已移除）Wan wrapper：`fastvideo/training/wan_distillation_v3.py` - -### F. 示例脚本（Phase 1） - -- [x] `examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh` -- [x] `examples/distillation/phase1/temp.sh`（可直接改路径启动训练） - -### G. 最小单测（CPU；锁定调度语义） - -- [x] 保留并重命名 optimizer/scheduler 对齐测试： - - `fastvideo/tests/distillation/test_optimizer_scheduler_alignment.py` -- [ ] （可选）为 builder 增加 1 个最小单测（不需要真实模型） - -### H. 移除 Phase 0 脚手架（去除旧路径依赖） - -- [x] 移除 legacy-backed `WanPipelineAdapter` -- [x] 移除旧的强耦合方法 `WanDMD2Method` -- [x] 移除旧入口 `fastvideo/training/wan_distillation_v2.py` -- [ ] 保留 `examples/distillation/phase0/`（暂存脚本用于对照；最终会统一清理） -- [x] 清理代码里残留的 `Phase 0/phase0` 调用与命名 - ---- - -## 仍然依赖 legacy code 的部分（尚未解耦干净） - -Phase 1 已经把 **算法层** 从 legacy pipeline 私有函数中剥离出来，但当前仍保留 “复用 legacy -pipeline 做加载/数据/日志” 的过渡策略，主要耦合点是： - -1. **加载/数据/优化器仍依赖 legacy pipeline** - - `fastvideo/training/distillation.py` 仍通过 `WanDistillationPipeline.from_pretrained(...)` - 完成：模型加载、dataloader 构建、optimizer/scheduler 初始化、tracker 等。 - - `fastvideo/distillation/builder.py` 目前接受的输入还是 `WanDistillationPipeline`。 - -2. **validation 仍复用 legacy pipeline 的实现** - - `fastvideo/distillation/adapters/wan.py::WanAdapter.log_validation()` 仍调用 - legacy 的 `pipeline._log_validation(...)`（并需要 legacy pipeline 的 validation init/RNG）。 - -这些耦合点将是 Phase 2 的主要清理对象（尤其是 validation + builder 输入从 “pipeline” 变为 -“roles/specs”）。 - ---- - -## Definition of Done（Phase 1） - -- `DMD2Method` 存在且可运行，且不依赖 legacy pipeline 私有算法函数 -- Wan DMD2 训练可走 `fastvideo/training/distillation.py --distill-model wan --distill-method dmd2` -- Phase 0 代码路径已移除，避免后续继续扩散耦合 diff --git a/dev/phases/phase_2.md b/dev/phases/phase_2.md deleted file mode 100644 index fc1362059..000000000 --- a/dev/phases/phase_2.md +++ /dev/null @@ -1,226 +0,0 @@ -# Phase 2：让新 Distill 框架 **独立运行**（摆脱 legacy distill pipeline） - -Phase 2 的定位：在 Phase 1 已经验证 “Wan DMD2 训练行为对齐 baseline” 的前提下， -把当前仍然依赖 legacy pipeline 的部分逐步替换掉，使 **新 distill 代码路径可以独立运行** -（训练 / validation / checkpoint-resume），同时 **旧代码仍然可跑**（通过保留旧入口文件）。 - -> 约束：本 phase 采用 **非侵入式** 策略 —— 优先新增代码路径，不强行重构/迁移 legacy 文件。 -> 等到完全解耦之后，旧代码的清理由你手动完成（不在 Phase 2 做“删除/搬家”）。 - -> 额外约束（你拍板）：**不新增任何入口文件**。 -> 新 distill 入口直接落在 `fastvideo/training/distillation.py`，并且 **仅接受新的 YAML configs** ->（不兼容旧式 CLI configs）。legacy distill 继续通过现有 -> `fastvideo/training/*distillation_pipeline.py` 入口运行（两套路径并存）。 - ---- - -## Phase 2 目标（可交付） - -1. **Validation 独立化**：不再调用 legacy `pipeline._log_validation(...)`，避免隐式依赖与脆弱属性。 -2. **Builder/Runtime 脱离 pipeline**：不再依赖 `WanDistillationPipeline.from_pretrained(...)` 来启动训练； - 改为从 `models={role -> spec}` 直接构建 `ModelBundle + Adapter + Method + DataLoader + Tracker`。 -3. **role-based checkpoint/save/resume**：新框架自洽地保存/恢复： - - per-role modules / optimizers / schedulers - - RNG states（含用于噪声采样的 generator） - - StatefulDataLoader（若使用） -4. **YAML 驱动的训练参数解析**：用 `distill.yaml` 描述一次运行；入口只接受新 config（不做 legacy CLI merge）。 -5. **`outside/` overlay workaround**：不修改主仓库 `fastvideo/configs/`，在 distillation 内提供可覆盖的“外部配置根”。 - ---- - -## Phase 2 非目标（明确不做） - -- 清理/删除 legacy distill code（你会在完全解耦后手动清理） -- 将 `fastvideo/training/*` 的通用函数迁移到更中立目录（设计里叫 Phase 2.4，先不做） -- 迁移 Self-forcing v2 / ODE-init（等 distill runtime 自洽后再做） - ---- - -## 当前与 legacy 的关系（Phase 2 目标） - -Phase 2 的目标是：**新 distill 代码路径在 import 与 runtime 两个层面都不再依赖 legacy -distillation pipeline**（`fastvideo/training/*distillation_pipeline.py`）。 - -当前状态（已达成）： - -- Phase 2 entrypoint：`fastvideo/training/distillation.py --config ` -- runtime：`build_runtime_from_config(...)` -- validation：`fastvideo/distillation/validators/wan.py::WanValidator` - -以上链路不再实例化/调用 legacy `WanDistillationPipeline` / `DistillationPipeline._log_validation(...)`。 - ---- - -## Phase 2 TODO List（Review Checklist） - -### A. Validation 独立化（Phase 2.1） - -- [x] 定义通用接口：`fastvideo/distillation/validators/base.py::DistillValidator` - - 入口：`log_validation(step)` - - 行为：rank0 写 artifacts（其余 rank 走 gather/send） -- [x] 实现 `fastvideo/distillation/validators/wan.py::WanValidator`（Wan + T2V 最小版本） - - 复用 `ValidationDataset` - - 使用模块化 inference pipeline：`WanDMDPipeline` - - 支持 few-step：`validation_sampling_steps` + `validation_guidance_scale` + seed/RNG（不依赖 legacy pipeline） -- [x] `fastvideo/distillation/adapters/wan.py::WanAdapter.log_validation()` 支持注入 validator - - Phase 2 路径：走新 validator - - 未提供 validator 时：不做 validation（不再回退到 legacy pipeline） - -### B. Builder/Runtime 脱离 pipeline（Phase 2.2） - -- [x] 定义结构化 spec（角色驱动）：`DistillSpec / RoleSpec` - - 目标：`models={role -> spec}` 成为唯一真相 - - method 自己声明需要哪些 roles（缺失则报错） -- [x] 新增 YAML 配置解析：`fastvideo/distillation/utils/config.py::load_distill_run_config` - - `yaml.safe_load` + 最小 schema 校验（不做 legacy CLI merge） - - schema：`distill + models + training + (pipeline_config|pipeline_config_path)` -- [x] 修改入口：`fastvideo/training/distillation.py` - - `--config` 为必需参数：Phase 2 路径 **YAML-only** - - legacy distill 仍通过旧入口文件可跑（两套路径并存） -- [x] 支持 `outside/`（你拍板的 Phase 2 workaround） - - 新增目录：`fastvideo/distillation/outside/`（视作外部 repo root） - - 覆盖路径：`fastvideo/distillation/outside/` - - **无自动补全/overlay**：config loader 不做路径重写；运行时传入 outside YAML 的真实路径（无 fallback） -- [x] 实现 standalone runtime builder：`fastvideo/distillation/builder.py::build_runtime_from_config` - - 直接加载 modules（student/teacher/critic）并构建 `ModelBundle` - - 构建 per-role optimizers/schedulers（复用 TrainingArgs 超参） - - 构建 dataloader（`build_parquet_map_style_dataloader`） - - 初始化 tracker（复用 `fastvideo/training/trackers/`） - - 通过 `WanAdapter(validator=...)` 接入独立 validation -- [x] 移除 Phase 1 legacy bridge（不影响 Phase 2） - - `fastvideo/distillation/builder.py::build_wan_dmd2_method` 已移除 - - `fastvideo/distillation/adapters/wan.py` 已移除 legacy pipeline fallback - - `fastvideo/training/wan_distillation_v3.py` 已移除 - -### C. role-based checkpoint/save/resume（Phase 2.3） - -- [x] 新增 `fastvideo/distillation/utils/checkpoint.py::DistillCheckpointManager` - - 保存内容（Phase 2 路径）： - - **trainable roles** 的 `modules/optimizers/lr_schedulers`（teacher 默认 frozen，不保存） - - `StatefulDataLoader`（Wan-Syn parquet loader 是 `torchdata.stateful_dataloader.StatefulDataLoader`） - - RNG states： - - 全局 torch/numpy/python/cuda - - adapter 暴露的 generators（`WanAdapter.get_rng_generators()`：noise_cpu/noise_cuda/validation_cpu） - - 保存位置： - - `${output_dir}/checkpoint-/dcp/`（torch.distributed.checkpoint） - - 旧 checkpoint 清理： - - 由 `training.checkpoints_total_limit` 控制（只保留最近 N 个） -- [x] 将 checkpoint manager 接入 `fastvideo/distillation/trainer.py::DistillTrainer` - - resume 参数来自 CLI：`fastvideo/training/distillation.py --resume-from-checkpoint ` - - 支持传入：`checkpoint-` / `checkpoint-/dcp` / `output_dir`（自动选最新） - - checkpoint 策略来自 YAML（TrainingArgs 字段）： - - `training.training_state_checkpointing_steps`: 训练态 checkpoint 保存间隔（<=0 关闭保存） - - `training.checkpoints_total_limit`: 保留最近 N 个 checkpoint（<=0 不清理） - - 行为： - - `on_train_start()` 之后再 `dcp.load(...)`，确保 adapter generators 已创建，可恢复其状态 - - checkpoint 保存发生在 **每步训练完成**（optim step + zero_grad）之后、validation 之前 - -### D. 示例脚本（Phase 2） - -- [x] 最小 smoke（训练 + few-step validation）：`examples/distillation/phase2/temp.sh` -- [x] Save/Resume 用法说明：`examples/distillation/phase2/README.md` - -### E. 最小单测（可选但建议） - -- [ ] `models_json` schema 解析 + role 校验 -- [ ] checkpoint roundtrip（mock modules + optimizer）不 crash - ---- - -## 关键设计（具体到代码） - -### 2.1 Validation 独立化：`DistillValidator` / `WanValidator` - -**核心原则**：validation 只是一段 “inference + decode + log”，它应当： - -- 不依赖训练 pipeline 的内部属性（例如 `validation_random_generator`） -- 不要求 `pipeline.train()` 被调用才“初始化齐全” - -建议落地： - -- 新增目录：`fastvideo/distillation/validators/` - - `base.py`：`DistillValidator` 抽象 - - `wan.py`：`WanValidator` - -`WanValidator` 的实现思路（最小版本）： - -1. 从 `training_args.validation_dataset_file` 构建 `ValidationDataset` -2. 取若干条 prompt（rank0） -3. 构建/复用一个 inference pipeline： - - `WanDMDPipeline.from_pretrained(student_model_path, loaded_modules=...)` - - 或者直接用 loader 加载 `text_encoder/tokenizer/vae/scheduler`，并注入 student transformer -4. 生成视频并交给 tracker（wandb）记录 - -**风险点（如遇到需要你决策会停下讨论）** - -- validation 要不要严格对齐 legacy `_log_validation` 的所有输出格式（latent vis dict 等）？ - - 建议 Phase 2 先只对齐“视频 artifact + caption”，其余可后续补齐。 - -### 2.2 Builder/Runtime 独立化：roles/spec -> instantiate - -**目标**：从 “先构建 legacy pipeline，再拆 roles” 转成 “roles/spec 直接构建 bundle”。 - -建议新增： - -- `fastvideo/distillation/utils/config.py`（后续实现时收敛到该文件） - - `ModelSpec`：`family/path/revision/precision/...` - - `RoleSpec`：`role/trainable/optimizer/scheduler/...` - - `DistillSpec`：`method + models{role->RoleSpec} + adapter_family(optional)` - -一个最小的 `distill.yaml` 示例（Wan 1.3B 学 14B + critic=1.3B；示意）： - -```yaml -distill: - model: wan - method: dmd2 - -models: - student: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} - teacher: {family: wan, path: Wan-AI/Wan2.1-T2V-14B-Diffusers, trainable: false} - critic: {family: wan, path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers, trainable: true} - -training: - output_dir: outputs/phase2_wan_dmd2 - max_train_steps: 4000 - seed: 1000 -``` - -**加载实现建议（Wan）** - -尽量复用已经存在的、与 distill training pipeline 无关的 loader 工具链： - -- `fastvideo/utils.py::maybe_download_model / verify_model_config_and_directory` -- `fastvideo/models/loader/component_loader.py::PipelineComponentLoader` - -这样我们无需实例化 `WanDistillationPipeline`，也能加载： - -- `transformer`（student/teacher/critic） -- （可选）`transformer_2`（MoE 支持，Phase 2 先保持 optional） -- `vae/text_encoder/tokenizer/scheduler`（用于 adapter + validation） - -**构建 optimizer/scheduler** - -- 在 builder 内按 role 创建 optimizer/scheduler，并放进 `RoleHandle.optimizers/lr_schedulers` -- method 继续用 `get_optimizers/get_lr_schedulers` 做 update policy（Trainer 不关心 role） - -**入口文件** - -- 不新增入口文件；直接增强 `fastvideo/training/distillation.py`： - - 仅支持 `--config distill.yaml`（YAML-only），不再兼容旧式 CLI configs - - legacy pipeline 继续通过现有 `fastvideo/training/*distillation_pipeline.py` 入口运行 - -### 2.3 role-based checkpoint：`DistillCheckpointManager` - -建议新增： - -- `fastvideo/distillation/utils/checkpoint.py` - - `DistillCheckpointManager` - - 内部复用 `fastvideo/training/checkpointing_utils.py` 的 wrappers： - - `ModelWrapper/OptimizerWrapper/SchedulerWrapper/RandomStateWrapper` - -建议 checkpoint 的 “state dict key” 命名空间： - -- `models/{role}/{module_name}` -- `optimizers/{role}/{name}` -- `schedulers/{role}/{name}` -- `random/{name}`（全局 RNG + per-role noise generator） diff --git a/dev/phases/phase_2_9.md b/dev/phases/phase_2_9.md deleted file mode 100644 index 2b4ce9fac..000000000 --- a/dev/phases/phase_2_9.md +++ /dev/null @@ -1,309 +0,0 @@ -# Phase 2.9：A+B+Families 语义收敛（operation-centric adapter + policy 回归 method + 优雅 dispatch） - -Phase 2 已经实现了“新框架可独立运行（不依赖 legacy distill pipeline）”。但从长期扩展的角度， -我们仍有三类结构性问题需要先解决，否则 Phase 3（配置语义升级 + finetune）会被迫在不稳的语义边界上继续堆功能。 - -本 Phase 2.9 的目标是先把语义边界收敛好（A+B+Families），并把 dispatch 做到真正优雅（N+M）， -让 Phase 3 只需要在此基础上加 config schema 与新 method，而不需要再动 entrypoint/builder 的组合逻辑。 - -本文件最初用于 **Phase 2.9 的代码层面设计**；目前 Phase 2.9 已实现，下面 checklist 已同步打勾。 -后续如果在实践过程中发现有小调整，会继续回填；遇到重大风险会停下讨论。 - ---- - -## 0) Phase 2.9 的 “A+B+Families” 是什么？ - -### A) operation-centric adapter API（避免 role 爆炸） - -把 adapter API 从： -- `teacher_predict_x0(...) / critic_predict_x0(...) / backward_student(...) / backward_critic(...)` - -收敛为： -- `predict_x0(handle, ...)` -- `backward(loss, ctx, ...)`（ctx 自带所需信息，不需要 role） - -核心原则： -- adapter **不接触 role 字符串**，只接收 `RoleHandle`（handle）来取 module/optimizer 等资源。 -- method 才持有 “role -> handle” 的语义映射（teacher/critic/student/reward/... 的意义只存在于 method）。 - -为什么要用 handle 而不是 role 字符串？ -- 防止 adapter 内部出现 `if role == "teacher": ...` 这类语义泄漏，长期演化会再次耦合/role 爆炸。 -- 让 adapter 更像 “family runtime primitives”，method 更像 “algorithm orchestration”。 - -### B) policy 回归 method（adapter 只保留 mechanics） - -把 DMD2 的“策略”从 adapter 挪回 method，例如： -- timestep sampling strategy（uniform/课程/分段等） - -adapter 只保留 scheduler 相关的 mechanics： -- `num_train_timesteps` -- `shift/clamp timestep` 的语义转换 - -### Families) build-time 语义收敛 + 优雅 dispatch（N+M） - -引入 `families/` + registry 的目的： -- adapter 专注 runtime/step-time -- family 插件专注 build-time（加载 modules / shared components / dataloader / validator / tracker） -- builder/entrypoint 不写组合 if/else；新增 family 或 method 的成本为 N+M，而不是 N×M - ---- - -## 0.5) 常见疑问（为什么这么拆？） - -### backward 为什么不需要传 handle？ - -因为 backward 本质上只做两件事： -1) 让 autograd 通过 **已有的计算图** 反传（参数由计算图决定，不需要再“选择模块”） -2) 在 FastVideo/Wan 这种依赖全局 forward_context 的实现里，在 backward 期间 **恢复必要的上下文** - -因此 adapter 的 backward 更像： -- `adapter.backward(loss, ctx)`：ctx 里带着 forward_context 所需的 timesteps/attn_metadata 等信息 - -handle 是为 **forward/select module** 服务的：比如选择哪个 transformer（以及 transformer_2/boundary）。而 backward 只需要“把当时 forward 用的上下文恢复回来”，不需要知道 handle。 - -如果未来真的出现 role/handle 相关的 backward 特殊逻辑（例如不同模块需要不同 backward context），也更推荐： -- 把 handle 放进 ctx（由 forward 返回），而不是让 adapter.backward(handle, ...) 成为常态 API。 - -### Families 和 Adapter 的关系是什么？ - -- `Family`（build-time）：负责“装配/构建” - - 从 config 解析 roles -> 加载 modules -> 构建 `ModelBundle` - - 构建 shared components（scheduler/vae/tokenizer/...） - - 构建 dataloader/validator/tracker（可选） - - **实例化 adapter**（把 bundle + shared components 注入进去） - -- `Adapter`（runtime/step-time）：负责“操作/执行” - - prepare_batch / forward_context / attention metadata - - `predict_x0(handle, ...)` 这类 operation-centric primitives - - backward 时恢复 forward_context - -一句话：**Family 负责把一堆零件装成可运行的 runtime；Adapter 负责在训练 step 里把 FastVideo 的模型/管线差异变成可复用的操作。** - -### 每个 family 都一定要有自己的 adapter 吗？能否复用？ - -通常会是 “一个模型家族（Wan/SDXL/Flux/...）至少有一个 adapter”，因为： -- raw batch schema 不同 -- forward_context / attention metadata 的机制不同 -- 预测目标（pred noise / pred x0 / flow-matching 等）的转换不同 - -但并不是强制 1:1： -- 一个 family 可能有多个 adapter（例如不同 pipeline 类型） -- 不同 family 也可能共享一个 adapter 基类或部分 mixin（当 runtime mechanics 很像时） - -### 为什么不把 Family 和 Adapter 合并成一个东西？ - -可以合并，但会带来三个长期问题（这也是我们保留 Families 的核心原因）： -1) **高内聚被破坏**：build-time（下载/加载/优化器/dataloader）和 step-time（forward/backward/context）混在一起，类会膨胀成 God object。 -2) **扩展成本回退到 N×M**：新增 method 时更容易被迫去改“合体的 adapter/builder”，最终又会出现各种 if/else 组合分支。 -3) **语义边界变模糊**：method 想保持纯算法；adapter 想保持纯 runtime；family 想保持纯装配。合并后这些边界会互相污染，导致未来很难做 finetune/新 method 的接入。 - ---- - -## 1) Phase 2.9 交付目标（Definition of Done） - -- `fastvideo/training/distillation.py` 不再硬编码 `wan + dmd2` 分支； - 改为调用通用 `build_runtime_from_config(cfg)`，并通过 registry resolve family/method。 -- `WanAdapter` 对 DMD2 的暴露接口变为 operation-centric： - - 不再暴露 `teacher_*` / `critic_*` / `student_*` 专用函数给 method 使用 - - DMD2Method 通过通用操作（如 `predict_x0(handle=...)`）完成 teacher/critic/student 的调用 -- DMD2 的 timestep sampling policy 从 adapter 迁移到 method（最少把 `sample_dmd_timestep()` 挪走）。 -- few-step rollout 的 step list / simulate 逻辑从 adapter 迁移到 method（未来应进一步移到 `method_config`）。 -- `WanAdapter` 不应包含 method-specific 命名/概念（例如不应依赖 `*DMD*Pipeline` 这类算法命名）。 -- optimizer/scheduler 的创建归属 method（update policy），family 不再出现 DMD2/critic 专属超参（例如 `fake_score_*`）。 -- validation 归属 method：family 构建 `WanValidator`，method 负责决定是否/如何调用（通过 `ValidationRequest` 传参）。 -- Phase 2 的训练行为/结果应尽可能保持一致（同 config 下 loss 形态、validation 产物趋势不应漂移）。 - ---- - -## 2) 非目标（明确不做） - -- 不做 YAML schema v2（`recipe` + `method_config`）升级（留到 Phase 3）。 -- 不新增 finetune method（留到 Phase 3）。 -- 不新增新模型家族（Phase 2.9 只整理 Wan）。 -- 不追求把所有“validation / sample / prompt encode”的实现都完全脱离 pipeline（Phase 2.9 先保证训练路径独立可跑）； - 但 adapter 层需要避免 method-specific policy/rollout 泄漏，并避免 method-specific 命名耦合。 - ---- - -## 3) TODO List（Review Checklist） - -### 3.1 Registry + Families（优雅 dispatch，N+M） - -- [x] 新增 `fastvideo/distillation/registry.py` - - `register_family(name)` / `register_method(name)` 装饰器 - - `get_family(name)` / `get_method(name)`（错误信息包含可用项） - - `ensure_builtin_registrations()`：导入内置 family/method 以完成注册 -- [x] 新增 `fastvideo/distillation/models/` - - `fastvideo/distillation/models/__init__.py` - - `fastvideo/distillation/models/wan.py`：`build_wan_family_artifacts` - - 从 Phase 2 builder 迁移 Wan-specific build-time 逻辑： - - 加载 role modules（transformer/transformer_2/vae 等） - - shared components（scheduler/noise_scheduler） - - dataloader（parquet + schema） - - tracker - - validator（WanValidator） - - adapter 实例化（WanAdapter） -- [x] 改造 `fastvideo/distillation/builder.py` - - 新增/收敛为 `build_runtime_from_config(cfg: DistillRunConfig) -> DistillRuntime` - - `DistillRuntime.method` 类型改为 `DistillMethod`（而不是 `DMD2Method`） - - builder 内部逻辑： - - `family = registry.get_family(cfg.distill.model)`（Phase 2.9 暂用 `distill.model` 作为 family） - - `method = registry.get_method(cfg.distill.method)` - - 调用 family 构建 bundle/adapter/dataloader/tracker - - 调用 method factory 构建 DistillMethod -- [x] 改造入口 `fastvideo/training/distillation.py` - - 删除 `if cfg.distill.model == "wan" and cfg.distill.method == "dmd2": ...` - - 统一走：`runtime = build_runtime_from_config(cfg)` - -### 3.2 Adapter API：从 role-centric 收敛到 operation-centric（A） - -- [x] 更新 `fastvideo/distillation/methods/distribution_matching/dmd2.py` - - `_DMD2Adapter` Protocol 改为 operation-centric： - - `predict_x0(handle, noisy_latents, timestep, batch, *, conditional, attn_kind)` - - `backward(loss, ctx, ...)` - - `select_module(handle, module_name, timestep)`（可选：用于 transformer_2/boundary） - - `timestep_ops`（见 3.3） - - 移除对 `teacher_predict_x0/critic_predict_x0/backward_student/backward_critic` 的直接依赖 -- [x] 更新 `fastvideo/distillation/adapters/wan.py` - - 把 `teacher_predict_x0/critic_predict_x0` 合并为 `predict_x0(handle=...)` - - 把 `backward_student/backward_critic` 合并为 `backward(loss, ctx, ...)` - - 将 `get_teacher_transformer/get_critic_transformer` 改为 `get_transformer(handle, timestep)` - - handle 不包含“语义” -- [x] `fastvideo/distillation/adapters/wan.py` 不再出现 method-specific 命名 - - 移除 `WanDMDPipeline` 依赖，prompt encoding 改用 `WanPipeline` - - adapter 内不再出现 `dmd/DMD` 字眼（避免“命名耦合”） - -### 3.3 Timestep sampling policy 回归 method（B） - -- [x] DMD2Method 内实现 timestep sampling policy： - - `t = torch.randint(0, adapter.num_train_timesteps, ...)`（policy：uniform） - - 然后调用 adapter 的 mechanics： - - `t = adapter.shift_and_clamp_timestep(t)`（mechanics：shift/clamp 语义） -- [x] `WanAdapter` 去掉 `sample_dmd_timestep()`（改为提供 `shift_and_clamp_timestep()`） -- [x] few-step rollout policy 回归 method - - denoising step list / warp mapping / multi-step simulate 全部由 `DMD2Method` 管理 - - adapter 只提供单步 primitives（如 `predict_x0` / `predict_noise` / `add_noise`） - - TODO(Phase 3): 将 `pipeline_config.dmd_denoising_steps` 迁移到 `method_config`，避免“pipeline_config 承载算法语义” - -### 3.4 兼容性与安全落地（降低风险） - -- [x] （选择 direct cut）一次性迁移到 operation-centric API（不保留旧 role-centric API wrapper） -- [x] 明确哪些行为必须保持一致（不引入训练 drift）： - - forward_context 的 ctx 捕获/恢复方式不改变 - - teacher 的 `transformer_2` boundary 逻辑不变 - - validation 路径不回退到 legacy -- [x] Wan family 不再创建 optimizers/schedulers - - `fastvideo/distillation/models/wan.py` 只负责加载 modules + 构建 `ModelBundle` - - `DMD2Method` 在 init 时为 student/critic 创建 optimizers/schedulers（复用 TrainingArgs 字段，未来迁移到 `method_config`） -- [x] Wan validator 归属 method（method 决定是否/如何调用 validation） - - `fastvideo/distillation/validators/wan.py` 当前使用 `WanDMDPipeline` 进行 validation 采样（对齐 DMD2 的 SDE rollout，便于与 legacy apples-to-apples 对比） - - validator 不 hardcode `bundle.role("student")` 等 role 语义；由 method 通过 `ValidationRequest.sample_handle` 指定采样模型 - - validator 不由 adapter 持有/调用；trainer 只调用 `method.log_validation(step)` - - TODO(Phase 3)：通过 “ODE/SDE sampler 可插拔” 的方式淘汰 `WanDMDPipeline`（validator 回到 method-agnostic） - ---- - -## 4) 关键接口草案（更具体一些） - -### 4.1 `predict_x0(...)` 的建议签名 - -```text -predict_x0( - handle: RoleHandle, - noisy_latents: Tensor, - timestep: Tensor, - batch: TrainingBatch, - *, - conditional: bool, - attn_kind: Literal["dense", "vsa"], -) -> Tensor -``` - -解释： -- `handle`：由 method 从 bundle 解析得到（`handle = bundle.role("teacher")` 等），adapter 只使用 handle 取 transformer（以及可选 transformer_2） -- `conditional`：选择 `batch.conditional_dict` 或 `batch.unconditional_dict` -- `attn_kind`：选择 `batch.attn_metadata` 或 `batch.attn_metadata_vsa`（以及对应 ctx） - -### 4.2 `backward(...)` 的建议形态 - -```text -ctx = AdapterBackwardContext(timesteps, attn_metadata) -adapter.backward(loss, ctx, grad_accum_rounds=...) -``` - -ctx 不需要包含 role（role 语义由 method 管理；backward 只需要 forward_context 信息）。 - -### 4.3 timestep mechanics（adapter 提供） - -```text -adapter.num_train_timesteps -> int -adapter.shift_and_clamp_timestep(t: Tensor) -> Tensor -``` - -method 决定如何 sample（policy），adapter 负责把 sample 结果转换为本模型家族的 scheduler 语义（mechanics）。 - ---- - -## 5) 风险点 / 需要你参与决策的地方 - -1) **一次性改动范围**：operation-centric API 迁移是否允许保留旧 API wrapper 一段时间？ - - 我建议允许，以降低重构风险；但最终目标是删除旧 API，避免长期双语义。 - -2) **timestep policy 的抽象边界**： - - Phase 2.9 最小只迁 `sample_dmd_timestep`（uniform + shift/clamp） - - 未来如果要更复杂的采样（课程学习），应该放在 `method_config`（Phase 3） - -3) **registry 的注册方式**：使用显式 `ensure_builtin_registrations()` 还是 import side-effect？ - - 我建议显式 `ensure_builtin_registrations()`，避免 import 顺序导致“没注册”这类隐式 bug。 - ---- - -## 6) 备注：为什么 Phase 2.9 不做 `recipe/method_config`？ - -因为本 phase 的核心风险来自“语义边界调整”： -- adapter/method API 变更 -- dispatch/build 结构变更 - -如果同时改 YAML schema（`distill` -> `recipe`）会叠加变量，出现问题时很难定位。 -因此 Phase 2.9 先保证内部语义正确，再在 Phase 3 做 schema 升级与 finetune 接入。 - ---- - -## 7) End-to-end 验证（建议） - -- 训练脚本：`examples/distillation/phase2_9/temp.sh` - - 默认读取：`fastvideo/distillation/outside/fastvideo/configs/distillation/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` - - 输出建议：`bash examples/distillation/phase2_9/temp.sh |& tee examples/distillation/phase2_9/temp.log` - ---- - -## 8) TODO（Phase 3）：统一 ODE/SDE sampling 语义（淘汰 `WanDMDPipeline`） - -### 8.1 背景：为什么 WanPipeline 和 DMD2 sampling 会不一致？ - -- DMD2 的 few-step rollout（以及 legacy `WanDMDPipeline`）本质是 **SDE 风格**： - 每一步先得到 `pred_x0`，再用 `add_noise(pred_x0, eps, next_t)` **重加噪**进入下一步。 -- `WanPipeline` 的默认采样 loop 是 **ODE/solver 风格**： - 每一步用 `scheduler.step(noise_pred, t, latents)` 直接更新 latents（并包含 CFG 双 forward + mix）。 - -因此即使我们把 timesteps/scheduler “看起来”统一，**只要 loop 更新公式不同，采样结果就可能 drift**。 - -### 8.2 Phase 2.9 的取舍（为了端到端对齐） - -为了确认 Phase 2.9 的新训练链路与 legacy 能 apples-to-apples 对比， -当前 `WanValidator` 直接改为使用 `WanDMDPipeline` 做采样（对齐 SDE rollout）。 - -缺点：validator 仍依赖一个带 method 命名的 pipeline 变体（不够优雅）。 - -### 8.3 Phase 3 的目标：让 sampling 语义可插拔（ODE/SDE），而不是靠 “Pipeline” - -在 Phase 3 我们应把 “denoising loop” 抽象成可注入的 sampler/integrator： -- `OdeSampler`：`scheduler.step(...)`（当前 `DenoisingStage` 的语义） -- `SdeSampler`：`pred_x0 -> add_noise(next_t, eps)`（当前 `DmdDenoisingStage` 的语义） - -然后： -- `WanPipeline` 通过参数选择 `sampler_kind: ode|sde`（默认 ode） -- method（或 `method_config`）在 validation 时显式选择 sampler + step list -- validator 回到 method-agnostic：只做 dataset + logging（不再 import `WanDMDPipeline`） - -> 这个设计会被写入 `dev/phases/phase_3.md` 并在 Phase 3 实施。 diff --git a/dev/phases/phase_3.md b/dev/phases/phase_3.md deleted file mode 100644 index f922cb687..000000000 --- a/dev/phases/phase_3.md +++ /dev/null @@ -1,263 +0,0 @@ -# Phase 3：3.1 Config schema v2 + 3.2 ODE/SDE sampler + 3.3 Finetuning + 3.4 命名/结构整理 - -Phase 2.9 已完成三件关键事情（为 Phase 3 铺路）： -- operation-centric adapter（adapter 不看 role string，只收 `RoleHandle`） -- policy 回归 method（few-step rollout / step list 等在 method 里） -- families + registry + builder（优雅 dispatch：新增 family 或 method 是 N+M，不是 N×M） - -因此 Phase 3 不再聚焦 dispatch；Phase 3 的新增工作按顺序拆成三个子阶段： - -- **Phase 3.1：Config schema v2（`recipe` + `method_config`）** -- **Phase 3.2：ODE/SDE sampler 可插拔（淘汰 `Pipeline`）** -- **Phase 3.3：Finetuning method 接入（only student + dataset）** -- **Phase 3.4：命名/结构整理（降低概念数量 + 更直觉的目录组织）** - -约束（延续前几个 phase）： -- 不新增 entry file：继续使用 `fastvideo/training/distillation.py`。 -- forward/backward context 仍由 adapter 托管；method 只负责算法编排。 - ---- - -## Phase 3.1：Config schema v2（`recipe` + `method_config`） - -### 目标（DoD） -- distillation 入口只接受 **schema v2** 的 YAML（顶层 `recipe:`），并作为 single source of truth。 -- `method_config` 能被解析并传入 method；至少 DMD2 的关键超参从 `method_config` 生效。 -- 将 “method knob” 从 `training_args/pipeline_config` 中剥离出来（逐步迁移），并修复 Phase 2.9 - 残留的语义泄漏： - - `WanAdapter.prepare_batch()` 不再读取 `training_args.simulate_generator_forward`。 - -### Schema v2（示意） - -```yaml -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - -training: {...} # infra（映射到 TrainingArgs） -pipeline_config: {...} # backbone/pipeline（模型侧） -method_config: {...} # algorithm（方法侧） -``` - -### 文件 TODO（实现清单） -- [x] `fastvideo/distillation/utils/config.py` - - 新增 `RecipeSpec(family: str, method: str)` - - `DistillRunConfig` 增加 `recipe` 与 `method_config` -- [x] `fastvideo/distillation/utils/config.py`（包含 YAML loader + schema/dataclass） - - 解析 `recipe:` 与 `method_config:`（默认 `{}`） - - 将 v1 的 `distill:` 视为不再支持（breaking change，直接推进 schema v2） -- [x] `fastvideo/distillation/dispatch.py` - - 从 `cfg.recipe` 取 family/method（不再读 `cfg.distill`） - - build method 时传入 `method_config` -- [x] `fastvideo/distillation/methods/distribution_matching/dmd2.py` - - `DMD2Method(..., method_config=...)` - - 关键参数读取优先级：`method_config` > `training_args`（迁移期平滑） - - `generator_update_interval` - - `real_score_guidance_scale` - - `dmd_denoising_steps`（few-step step list） - - `rollout_mode`（替代 `simulate_generator_forward`） -- [x] `fastvideo/distillation/adapters/wan.py` - - 移除 `training_args.simulate_generator_forward` 的读取（这是 Phase 2.9 的残留耦合） - - 把 batch 形态做成显式 API/参数，让 method 决定： - - 选项 A（推荐）：拆分显式入口 - - `prepare_batch_from_data_latent(raw_batch, ...)`（必须有 `vae_latent`） - - `prepare_batch_from_placeholder_latent(raw_batch, ...)`（不依赖 `vae_latent`） - - 选项 B：保留单入口但显式参数化：`prepare_batch(..., latents_source=...)`（本阶段采用） -- [x] configs / docs - - [x] `examples/distillation/**.yaml` 全部升级到 schema v2 - - [x] 更新 `dev/config.md`（描述 schema v2 与迁移策略） - -### 可运行产物 -- Phase 3.1 YAML：`examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml` -- One-shot 脚本：`examples/distillation/phase3_1/temp.sh` - ---- - -## Phase 3.2：ODE/SDE sampler 可插拔（统一 sampling loop 语义） - -### 背景（为什么需要） -Phase 2.9 已验证：即使统一 timesteps/scheduler，**只要 denoising loop 不同**，sampling 结果仍会 drift： -- ODE/solver 风格：`scheduler.step(noise_pred, t, latents)`（当前 `WanPipeline`） -- SDE 风格：`pred_x0 -> add_noise(next_t, eps)`（DMD2/legacy `WanDMDPipeline`） - -如果继续靠 `Pipeline` 这类 pipeline 变体来选择 loop，会重新走向 N×M 组合爆炸。 - -### 目标（DoD） -- `WanPipeline` 支持通过参数/配置选择 sampler（`ode|sde`），默认 `ode`。 -- distillation 的 validation 由 method/method_config 显式指定 sampler + step list； - validator 回到 method-agnostic，不再 import `WanDMDPipeline`。 -- `WanDMDPipeline` 保留为 legacy 兼容（可选），但新框架不依赖它。 - -### 文件 TODO（实现清单） -- [x] 抽象 sampler（中性命名，不出现 DMD） - - `fastvideo/pipelines/samplers/`：`SamplerKind` + Wan sampler helpers - - `pipeline_config.sampler_kind={ode|sde}`：`WanPipeline` 通过该参数选择 sampling loop -- [x] `fastvideo/pipelines/stages/denoising.py` - - `SdeDenoisingStage`：SDE 风格 rollout（`pred_x0 -> add_noise(next_t, eps)`） - - `SdeDenoisingStage` 接受显式 `batch.sampling_timesteps`（来自 ValidationRequest） - - 继续使用 `batch.generator` 生成每一步注入的 `eps`（可复现） - - 保留 `DmdDenoisingStage = SdeDenoisingStage` alias（legacy pipeline 兼容） -- [x] `fastvideo/pipelines/basic/wan/wan_pipeline.py` - - `WanPipeline` 支持 `sampler_kind={ode|sde}`（单一 pipeline 覆盖两种 loop） -- [x] `fastvideo/distillation/validators/base.py` - - `ValidationRequest` 新增： - - `sampler_kind: Literal["ode", "sde"] | None` - - `sampling_timesteps: list[int] | None` -- [x] `fastvideo/distillation/validators/wan.py` - - 使用 `WanPipeline` + request 的 sampler/timesteps（不再 import `WanDMDPipeline`） -- [x] `fastvideo/distillation/methods/distribution_matching/dmd2.py` - - validation request 指定 `sampler_kind="sde"` + `sampling_timesteps=` - -### 可运行产物 -- Phase 3.2 YAML：`examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml` -- One-shot 脚本：`examples/distillation/phase3_2/temp.sh` - ---- - -## Phase 3.3：Finetuning method（only student） - -### 目标（DoD） -- 新增 `finetune` method，复用同一套 Trainer/Bundle/Adapter/Family/Validator 基础设施。 -- 最小可运行：只需 `roles.student` + dataset 即可训练。 -- finetune 的 method 参数进入 `method_config`（与 Phase 3.1 schema 一致）。 - -### 文件 TODO（实现清单） -- [x] `fastvideo/distillation/methods/fine_tuning/finetune.py` - - `FineTuneMethod(DistillMethod)` + `@register_method("finetune")` - - `bundle.require_roles(["student"])` - - `single_train_step()` 只更新 student -- [ ] （如有必要）为 finetune 定义 adapter contract（类似 `_DMD2Adapter` 的做法） - - 重点：**loss 仍由 method 计算**；adapter 只提供 operation-centric primitives - - `_FineTuneAdapter(Protocol)` 推荐只包含： - - `prepare_batch(...)`（产出 latents/noise/timesteps/sigmas/conditioning） - - `predict_noise(handle, ...)`（以及可选 `predict_x0`） - - `backward(loss, ctx, ...)`（forward-context/activation ckpt 相关） -- [x] configs/examples - - [x] `examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml` - - [x] `examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml` - - [x] `examples/distillation/phase3_3/temp.sh` - - [x] `examples/distillation/phase3_3/temp-vsa.sh` - ---- - -## 备注：关于 `simulate_generator_forward`（Phase 2.9 的残留） - -该耦合已在 Phase 3.1 解决： -- `WanAdapter.prepare_batch()` 不再读取 `training_args.simulate_generator_forward` -- `DMD2Method` 通过 `method_config.rollout_mode` 决定 `latents_source={zeros|data}`， - 并把它作为参数传给 adapter（adapter 只处理 batch 形态，不解释 DMD2 语义） - ---- - -## Phase 3.4：命名/结构整理（降低概念数量 + 更直觉） - -### 背景（为什么要做） - -Phase 3.1~3.3 已经把训练端到端跑通；但目前 `fastvideo/distillation/` 的概念命名偏“框架内部术语”，对新 reviewer 不友好： -- `families/` 读起来像“人类家族”，但它实际承担的是 **model/pipeline contract 的集成/装配层**。 -- `bundle.py` 读起来像“打包”，但它本质是 **roles 管理/索引容器**。 -- `registry.py` / `builder.py` /（以及一些纯 dataclass 文件）分散在多个文件，阅读路径长，容易产生“概念过多”的感受（本阶段已收敛到 `dispatch.py` 与 `utils/config.py`）。 - -我们希望把这些改成更直觉的命名，并把“infra”从“模型集成层”里抽出来。 - -> 备注：此阶段优先做 **低风险、可 review、行为不变（或可控变更）** 的整理。 -> 若某些重排会牵动较大行为差异（例如数据加载完全抽象成独立 registry），可以拆成 3.4.x 逐步落地。 - -**本阶段决策（重要）** -- 不做后向兼容：不保留 re-export shim，不保留旧路径别名。 -- 直接把全 repo 的 import / docs / YAML 示例统一到新语义，允许 breaking change。 - -### 目标（DoD） - -1) **更直觉的目录命名** -- `fastvideo/distillation/families/` → `fastvideo/distillation/models/` - - 语义：这里的 “models” 指 **模型家族/管线 contract 的集成插件**（不是 YAML 的 `roles:`）。 - -1) **roles 容器命名统一** -- `fastvideo/distillation/bundle.py` → `fastvideo/distillation/roles.py` -- `ModelBundle` → `RoleManager` - -1) **把 infra 从 models(原 families) 中解耦合** -- dataloader 构建逻辑从 `models/*` 抽到 `fastvideo/distillation/utils/`（或 `infra/`） -- tracker 初始化从 `models/*` 抽到 `trainer/entrypoint`（更符合“infra 归 infra”） -- checkpointing 相关（`fastvideo/distillation/utils/checkpoint.py`）统一放在 `utils/`（或 `infra/`） - -1) **减少“文件级概念数量”** -- 已将纯 dataclass（原 `specs.py/runtime.py`）合并到 `utils/config.py`，减少“文件级概念数量” -- 已将 YAML loader（原 `yaml_config.py`）合并到 `utils/config.py`（schema+解析逻辑同处） -- `registry.py + builder.py` 可以合并/重命名为更直觉的 `dispatch.py`（保留注册表与 build_runtime 的入口） - -### 具体设计：如何“解耦 dataloader/tracker” - -#### Tracker -现状：tracker 在 `models/wan.py`（原 `families/wan.py`）里由 `_build_tracker()` 创建，并传给 validator。 - -Phase 3.4 目标： -- tracker 由 `fastvideo/training/distillation.py`（entrypoint）或 `DistillTrainer` 创建/持有； -- model plugin 只返回“是否需要 tracker config”（例如 raw config dict），validator 也由 method 触发调用； -- validator 构建可以延迟到 tracker 创建之后（factory/closure），避免 plugin 直接依赖 tracker。 - -#### Dataloader -现状：FastVideo 里 “数据 schema/预处理” 的差异主要来自 **任务/数据形态**， -并不严格等价于 model family（同一 family 内也可能有多种 schema）： - -- parquet 族：`fastvideo/training/training_pipeline.py` 统一走 - `build_parquet_map_style_dataloader(..., parquet_schema=..., text_padding_length=...)`。 - - T2V：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_t2v` - - I2V：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_i2v` - （额外字段如 `clip_feature`/`first_frame_latent`/`pil_image`，见 - `fastvideo/training/wan_i2v_training_pipeline.py`） - - MatrixGame：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_matrixgame` - （额外 action cond，且不使用 text embedding，见 - `fastvideo/training/matrixgame_training_pipeline.py`） - - ODE-init：`fastvideo/dataset/dataloader/schema.py:pyarrow_schema_ode_trajectory_text_only` - （trajectory latents/timesteps，见 `fastvideo/training/ode_causal_pipeline.py`） -- 非 parquet：例如 LTX2 使用 `.precomputed/*.pt` 的数据形态（见 - `fastvideo/dataset/ltx2_precomputed_dataset.py`）。 - -因此 Phase 3.4 的目标应更准确表述为：**model plugin 不负责 data plumbing**； -dataloader 由通用层基于 `DataSpec`/`dataset_kind` 构建，而 family/adapter 只负责把 -batch 转成 forward primitives 所需输入（若需要额外字段，由 `DataSpec` 显式声明）。 - -Phase 3.4 目标： -- model plugin **不直接构建 dataloader**，而是返回一个 `DataSpec`（或 `dataloader_factory`）描述： - - dataset kind（parquet/webdataset/…） - - schema/text padding length/cfg_rate 等必要参数 -- `distillation/utils/data.py`（或 `infra/data.py`）统一执行 “根据 TrainingArgs + DataSpec 构建 dataloader” - -这样做的收益：models(集成层) 文件更短、更聚焦在“加载模块 + 组装 adapter 需要的 shared context”。 - -### 文件 TODO（实现清单） - -命名/结构（行为尽量不变）： -- [x] YAML schema：顶层 `models:` → `roles:`（与 `DistillRunConfig.roles` 对齐） -- [x] YAML loader：`fastvideo/distillation/utils/config.py`（包含 schema + 解析逻辑） -- [x] `fastvideo/distillation/families/` → `fastvideo/distillation/models/`（直接迁移并更新所有 import） -- [x] `fastvideo/distillation/bundle.py` → `fastvideo/distillation/roles.py`（直接迁移并更新所有 import） -- [x] `fastvideo/distillation/specs.py` + `fastvideo/distillation/runtime.py` 合并到 `fastvideo/distillation/utils/config.py` -- [x] `fastvideo/distillation/dispatch.py`（合并 `registry.py` + `builder.py`） - -infra 解耦： -- [x] 新增 `fastvideo/distillation/utils/`（或 `infra/`） - - [x] `utils/tracking.py`：tracker 初始化（rank0 only） - - [x] `utils/data.py`：dataloader 构建（当前先覆盖 parquet T2V） - - [x] `utils/checkpoint.py`：checkpoint manager / config（从 `distillation/checkpoint.py` 迁移） -- [x] `models/*`（原 families）移除 tracker/dataloader/checkpointing 的内联实现（迁移到 `utils/`） -- [x] build-time “装配产物” 统一为 `models/components.py::ModelComponents`（不额外引入 artifacts/factory/spec 抽象） - -docs： -- [x] 更新 `fastvideo/distillation/doc/README.md` 与各文件说明（路径/命名变化） diff --git a/dev/refactor.md b/dev/refactor.md deleted file mode 100644 index d90313163..000000000 --- a/dev/refactor.md +++ /dev/null @@ -1,216 +0,0 @@ -# 重构讨论：把复杂度“移到 config” vs 保持“清晰分层” - -这份文档记录 FastVideo distillation 重构（Phase 2 → 3+）过程中，我们对 -架构抽象与配置复杂度之间取舍的讨论，以及对 Phase 4+ 的可能方向。 - -核心矛盾： - -- **方案 A（配置驱动加载 / 更少概念）** - - 在 YAML 里显式写清模型加载细节（VAE、text encoder 等的路径/开关）。 - - `Method` 直接调用共享的 `utils/` 来完成加载与组装。 - - 希望减少“概念数量”（Family / Adapter / Bundle / Validator / …）。 - -- **方案 B（清晰分层 / 显式边界）** - - `Method` 只关注算法与训练接口（algorithm-only）。 - - 模型/管线差异由 `Adapter`（operation-centric）吸收；构建期组装由 - `Family/Builder` 处理。 - - 配置保持结构化并可校验（schema + validation）。 - -本文档不是最终结论，主要记录： -1) 共识是什么； -2) 分歧是什么； -3) 实际 trade-off； -4) 收敛路径（如何减少概念但不破坏边界）。 - ---- - -## 术语表（当前 FastVideo 语义） - -- **Role**：配置里用于索引参与者的字符串 key，例如 `student`、`teacher`、 - `critic`……（不设“高低贵贱”，只是 key）。 -- **RoleHandle**：每个 role 对应的句柄，包含 `modules/optimizers/schedulers` - 以及类似 `trainable` 的标志。 -- **RoleManager**：持有所有 `RoleHandle` 的容器（role 索引与训练态命名空间）。 -- **Adapter**：训练原语的 operation-centric 接口（如 `prepare_batch`、 - `predict_noise/x0`、`add_noise`、`backward` 等）。它不应包含算法策略。 -- **Model plugin**：构建期工厂：根据 config 组装 roles + adapter + validator。 -- **Validator**：训练期 validation 的采样/记录层，由 method 通过 - `ValidationRequest` 提供关键参数（steps/sampler/guidance/…）。 - ---- - -## 共同目标（共识） - -1) **消除隐式耦合** - - 避免“validation 的 side-effect 初始化训练状态”这种隐藏依赖。 - - 避免算法名（例如 “dmd”）泄漏进 adapter / validator。 - -2) **可扩展** - - 新增 method 不应导致入口文件/组合爆炸。 - - 新增 role 不应导致“每个 role 一个函数”的 API 爆炸。 - -3) **职责清晰** - - 算法决策归 `Method`。 - - 模型/管线的工程差异归 `Adapter` / 构建期组装。 - -4) **配置表达力强，但要安全** - - 配置需要能表达当前 distillation，也要能表达未来训练语义（例如 - finetune = 只有 student 的一种“特殊 distill”）。 - ---- - -## 核心分歧 - -### A) “把抽象复杂度移到 config” - -直觉是： -- 如果 config 能声明 `vae_path`、`text_encoder_path` 等，那么代码可以更 - 简单：少一些抽象层、少一些类。 -- Loading/组装细节可以抽到 `utils/`，由 method 直接调用。 - -### B) “配置驱动加载仍需要边界” - -反驳点： -- 即使 YAML 写清楚了 *路径*，加载/运行的 *语义* 更准确地说是 **pipeline contract 相关** - （也就近似“模型家族相关”）： - - 有哪些子模块（`transformer` / `vae` / `text_encoder` / …）； - - latent 的归一化/布局； - - attention metadata（VSA/VMoBA）如何构建； - - conditioning 的结构与缓存（text dict 的形状、neg/uncond 的初始化与 - broadcast）； - - dtype/offload/并行切分（tp/sp/hsdp）的要求。 - -如果 `Method` 调用 `utils.load_vae(...)`，而 `utils` 内部其实知道 -“Wan 的 VAE 怎么 load、SDXL 的 VAE 怎么 load”，那么 **method 仍然被污染** -（哪怕是间接污染）。我们只是把耦合从“文件 A”搬到了“文件 utils”，并没有 -真正消除耦合。 - ---- - -## 讨论：能否做一个“完全通用”的 config → load → 调度？ - -先澄清“通用”到底指什么： - -- **通用的模块加载机制**：可以统一（FastVideo 大多数 pipeline 的确会走 - `PipelineComponentLoader.load_module()` 这条路径）。 -- **通用的 pipeline contract（需要哪些模块 + 如何初始化 + 如何采样/训练）**： - **很难做到完全通用**，通常只能做到“框架通用 + 插件/实现分家族”。 - -为什么“contract”无法完全靠 config 自动推导？因为差异不仅在 *加载路径*，还在： - -1) **需要加载哪些模块本身就不同** - - Wan：`text_encoder/tokenizer/vae/transformer/scheduler` - - SD3.5：`text_encoder(1/2/3) + tokenizer(1/2/3)`（三套编码器） - - Hunyuan：`text_encoder_2/tokenizer_2`（两套） - - LTX2：额外的 `audio_vae`、`vocoder` - - Cosmos：额外的 `safety_checker` - -2) **即使模块名相同，初始化/替换策略也不同** - - 有些 pipeline 会在 `initialize_pipeline()` 里“重建/替换” scheduler， - 而不是按权重目录加载（例如 Cosmos 使用 `FlowMatchEulerDiscreteScheduler`， - TurboDiffusion 使用 `RCMScheduler`）。 - - Wan 也会基于 `sampler_kind` 构建 scheduler，并改变 stage graph（ODE/SDE）。 - -3) **部分 pipeline 需要特殊的加载/回退逻辑** - - LTX2 的 tokenizer 目录可能不存在，需要从 `text_encoder/gemma` 回退。 - - StepVideo 的 text encoder/clip 不是走 `model_index.json` 加载，而是在 - `initialize_pipeline()` 里手工构建，并加载自定义 `.so`。 - -4) **stage graph 差异是真实存在的** - - refine pipeline（LongCat）会插入 refine init / timestep override 等 stage； - - audio pipeline（LTX2）会有 audio decode stage； - - 不同 pipeline 对 conditioning/timestep/latent 准备的处理逻辑不一致。 - -因此结论不是“完全无法抽象出通用框架”，而是： - -- 我们可以抽象出 **通用框架**（统一入口、统一 YAML schema、统一 Trainer、统一 - Method 接口、统一 Adapter primitives、统一 Validator request/日志框架）。 -- 但要做到“**只靠一份 dict config + 一个通用 utils** 就能自动适配所有模型家族”， - 很容易被上述 contract 差异击穿；最终要么： - - 在 utils 里堆满 if/else（隐式 family），要么 - - 让 method 变成“懂所有 pipeline 语义的上帝对象”（method 污染）。 - -这也是为什么我们目前更倾向于保留“家族层”的存在（哪怕未来换个名字）： -它是“contract 插件层”，负责把 config 映射到 **正确的 pipeline/adapter/runtime 组装**， -把 method 留在算法层。 - ---- - -## “配置驱动加载”擅长什么、不擅长什么 - -配置擅长： -- **选择声明**：用哪个权重、哪些可选模块要加载、从哪里加载。 -- **策略 knob**：例如 `sampler_kind`、`flow_shift`、validation steps、 - update interval 等。 -- **可复现**：把 run 的 YAML 原样上传到 W&B（后续复盘非常方便）。 - -配置不擅长： -- **跨 role 的一致性约束**（例如共享 VAE、latent 空间兼容、scheduler 类型一致）。 -- **防止“看似能跑但悄悄不一致”**（dtype/offload 不一致、negative embedding - 布局不一致、VSA metadata 不一致等）。 - -这些一致性通常需要： -- 单一组装点（builder/family），和/或 -- 一个共享的 context（由 role-bound view 使用）。 - ---- - -## 为什么现在还需要 Family/Builder（务实角度） - -即使只让 student 初始化 preprocessors，我们仍然需要一个地方来： -- 为所有 roles 加载 `RoleHandle.modules`（student/teacher/critic/…）； -- 决定哪些资源共享、哪些 per-role； -- 初始化共享缓存（neg/uncond conditioning 等）； -- 按 run config 构建 validator（采样 pipeline）并保持一致； -- 只在 rank0 创建 tracker / run 元信息。 - -如果移除 model plugin（装配层），工程上往往会以另外一种形式“复活”同样的东西： -- `SharedContext`， -- `RuntimeFactory`， -- 或“method 构造函数里做 assembly”。 - -因此问题变成：我们到底想要 **显式的组装层**，还是 **隐式的组装层**？ -通常显式层更容易 review、更容易写测试、更容易维护 invariant。 - ---- - -## 建议的收敛方案（减少概念，但不把语义推给 method） - -我们可以通过以下方式减少“概念感”，同时保持边界： - -1) **重命名 / 重塑概念** - - 例如 `RoleManager`（替代旧的 `ModelBundle`）（语义更直观）。 - - `Model plugin` → `RuntimeFactory`（如果“Model plugin”这个词仍然难理解）。 - -2) **让 method “看起来像持有多个 adapter”，但底层共享 context** - - 保留一份共享 adapter + context。 - - 增加轻量 per-role view： - - `adapter.bind(handle)` → `RoleAdapterView` - - method 持有 `student_api` / `teacher_api` / …（view），而不是 raw adapter。 - - 既保留共享资源的一致性，又贴近心智模型： - “method 与 role-specific API 交互”。 - -3) **配置保持结构化，但允许受控扩展** - - 保持顶层 schema（例如 `recipe/models/training/pipeline_config/method_config`）。 - - 允许在 `method_config`（或未来的 `family_config`）里放自由 dict， - 但解析与校验集中在一个入口，避免 dict 在全栈到处传。 - -4) **infra ownership 下沉到 trainer/entrypoint** - - tracker 应由 trainer/runtime 持有（或 entrypoint 创建），而不是由 family - 的实现细节决定。 - ---- - -## 下一轮需要明确的问题（可作为下一次讨论议题） - -1) “load preprocessors” 的 policy 应该放在哪里？ - - Family（assembly） vs Adapter（context） vs Method（algorithm）。 - -2) 如果未来支持跨 family distill（Wan teacher → SDXL student），如何保证 - “共享 VAE/latent 空间一致”的 invariant？ - -3) 当前 adapter API 哪些部分仍然过于 role-centric？ - - 目标：尽量 operation-centric primitives。 - -4) 是否需要一个显式的 `SharedContext` 类型？ - - 还是继续把共享状态作为 adapter 的内部实现细节。 diff --git a/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh deleted file mode 100644 index fd8b8166f..000000000 --- a/examples/distillation/phase0/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -set -e -x - -# Phase 0 example: run Wan DMD2 distillation via Method/Trainer entrypoint. -# Validation is supported via `--log_validation` (see `validation_args` below). - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_PORT=${MASTER_PORT:-29503} - -NUM_GPUS=${NUM_GPUS:-1} - -# Models -STUDENT_MODEL_PATH=${STUDENT_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} -# For best distillation, point TEACHER_MODEL_PATH to a stronger teacher (e.g. 14B). -# For a cheaper smoke run, set it to the same 1.3B model. -TEACHER_MODEL_PATH=${TEACHER_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-14B-Diffusers"} -CRITIC_MODEL_PATH=${CRITIC_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} - -# Data (parquet dataset folder) -DATA_DIR=${DATA_DIR:-"your_data_dir"} -VALIDATION_DATASET_FILE=${VALIDATION_DATASET_FILE:-"your_validation_dataset_file"} - -OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase0_wan2.1_t2v_1.3B_dmd2_8steps"} - -training_args=( - --tracker_project_name "phase0_wan_dmd2_8steps" - --output_dir "$OUTPUT_DIR" - --max_train_steps 4000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 21 - --num_height 480 - --num_width 832 - --num_frames 81 - --enable_gradient_checkpointing_type "full" - --simulate_generator_forward -) - -parallel_args=( - --num_gpus "$NUM_GPUS" - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim "$NUM_GPUS" -) - -model_args=( - --model_path "$STUDENT_MODEL_PATH" - --pretrained_model_name_or_path "$STUDENT_MODEL_PATH" - --real_score_model_path "$TEACHER_MODEL_PATH" - --fake_score_model_path "$CRITIC_MODEL_PATH" -) - -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 50 - --validation_sampling_steps "8" - --validation_guidance_scale "6.0" # not used for dmd inference -) - -optimizer_args=( - --learning_rate 1e-5 - --mixed_precision "bf16" - --weight_decay 0.01 - --betas '0.0,0.999' - --max_grad_norm 1.0 - --fake_score_learning_rate 8e-6 - --fake_score_betas '0.0,0.999' -) - -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.0 - --dit_precision "fp32" - --flow_shift 8 - --seed 1000 -) - -dmd_args=( - # 8-step schedule (same as Wan2.2 self-forcing examples) - --dmd_denoising_steps '1000,850,700,550,350,275,200,125' - --min_timestep_ratio 0.02 - --max_timestep_ratio 0.98 - --generator_update_interval 5 - --real_score_guidance_scale 3.5 -) - -torchrun \ ---nnodes 1 \ ---master_port "$MASTER_PORT" \ ---nproc_per_node "$NUM_GPUS" \ - fastvideo/training/wan_distillation_v2.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ - "${dmd_args[@]}" diff --git a/examples/distillation/phase0/temp.sh b/examples/distillation/phase0/temp.sh deleted file mode 100644 index 63743f2c7..000000000 --- a/examples/distillation/phase0/temp.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -set -e -x - -# One-shot launch script for Phase0 (Method/Trainer) Wan DMD2 few-step distillation. -# Uses local Wan-Syn parquet dataset + a small validation json already in this repo. -# -# Notes: -# - By default this runs W&B in offline mode (safer for overnight runs). -# If you want online logging: -# export WANDB_MODE=online -# export WANDB_API_KEY=... -# - Phase0 v2 currently focuses on "can it train + can it validate". -# Checkpoint save/resume is NOT wired yet in v2. - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29503} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -if [[ -z "${NUM_GPUS:-}" ]]; then - if command -v nvidia-smi >/dev/null 2>&1; then - NUM_GPUS=$(nvidia-smi -L | wc -l) - else - NUM_GPUS=1 - fi -fi - -# Models -STUDENT_MODEL_PATH=${STUDENT_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} -TEACHER_MODEL_PATH=${TEACHER_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-14B-Diffusers"} -CRITIC_MODEL_PATH=${CRITIC_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} - -# Data (parquet dataset folder) -DATA_DIR=${DATA_DIR:-"data/Wan-Syn_77x448x832_600k"} -DEFAULT_VALIDATION_DATASET_FILE=\ -"examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json" -VALIDATION_DATASET_FILE=${VALIDATION_DATASET_FILE:-"$DEFAULT_VALIDATION_DATASET_FILE"} - -RUN_ID=${RUN_ID:-"$(date +%Y%m%d_%H%M%S)"} -OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase0_wan2.1_dmd2_8steps_wansyn_${RUN_ID}"} - -training_args=( - --tracker_project_name "phase0_wan_dmd2_8steps_wansyn" - --output_dir "$OUTPUT_DIR" - --max_train_steps 4000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 20 - --num_height 448 - --num_width 832 - --num_frames 77 - --enable_gradient_checkpointing_type "full" - --simulate_generator_forward -) - -parallel_args=( - --num_gpus "$NUM_GPUS" - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim "$NUM_GPUS" -) - -model_args=( - --model_path "$STUDENT_MODEL_PATH" - --pretrained_model_name_or_path "$STUDENT_MODEL_PATH" - --real_score_model_path "$TEACHER_MODEL_PATH" - --fake_score_model_path "$CRITIC_MODEL_PATH" -) - -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 50 - --validation_sampling_steps "8" - --validation_guidance_scale "6.0" -) - -optimizer_args=( - --learning_rate 2e-6 - --mixed_precision "bf16" - --weight_decay 0.01 - --betas '0.0,0.999' - --max_grad_norm 1.0 - --fake_score_learning_rate 8e-6 - --fake_score_betas '0.0,0.999' -) - -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.0 - --dit_precision "fp32" - --flow_shift 8 - --seed 1000 -) - -dmd_args=( - --dmd_denoising_steps '1000,850,700,550,350,275,200,125' - --min_timestep_ratio 0.02 - --max_timestep_ratio 0.98 - --generator_update_interval 5 - --real_score_guidance_scale 3.5 -) - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/wan_distillation_v2.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ - "${dmd_args[@]}" diff --git a/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh deleted file mode 100644 index fe79dc58b..000000000 --- a/examples/distillation/phase1/distill_wan2.1_t2v_1.3B_dmd2_8steps.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -set -e -x - -# Phase 1 example: run Wan DMD2 distillation via DMD2Method + WanAdapter entrypoint. -# Note: validation is currently best-effort; Phase 1 focuses on algorithm/model decoupling. - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_PORT=${MASTER_PORT:-29504} - -NUM_GPUS=${NUM_GPUS:-1} - -# Models -STUDENT_MODEL_PATH=${STUDENT_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} -# For best distillation, point TEACHER_MODEL_PATH to a stronger teacher (e.g. 14B). -# For a cheaper smoke run, set it to the same 1.3B model. -TEACHER_MODEL_PATH=${TEACHER_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-14B-Diffusers"} -CRITIC_MODEL_PATH=${CRITIC_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} - -# Data (parquet dataset folder) -DATA_DIR=${DATA_DIR:-"your_data_dir"} -VALIDATION_DATASET_FILE=${VALIDATION_DATASET_FILE:-"your_validation_dataset_file"} - -OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase1_wan2.1_t2v_1.3B_dmd2_8steps"} - -training_args=( - --tracker_project_name "phase1_wan_dmd2_8steps" - --output_dir "$OUTPUT_DIR" - --max_train_steps 4000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 21 - --num_height 480 - --num_width 832 - --num_frames 81 - --enable_gradient_checkpointing_type "full" - --simulate_generator_forward -) - -parallel_args=( - --num_gpus "$NUM_GPUS" - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim "$NUM_GPUS" -) - -model_args=( - --model_path "$STUDENT_MODEL_PATH" - --pretrained_model_name_or_path "$STUDENT_MODEL_PATH" - --real_score_model_path "$TEACHER_MODEL_PATH" - --fake_score_model_path "$CRITIC_MODEL_PATH" -) - -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 50 - --validation_sampling_steps "8" - --validation_guidance_scale "6.0" # not used for dmd inference -) - -optimizer_args=( - --learning_rate 1e-5 - --mixed_precision "bf16" - --weight_decay 0.01 - --betas '0.0,0.999' - --max_grad_norm 1.0 - --fake_score_learning_rate 8e-6 - --fake_score_betas '0.0,0.999' -) - -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.0 - --dit_precision "fp32" - --flow_shift 8 - --seed 1000 -) - -dmd_args=( - # 8-step schedule (same as Wan2.2 self-forcing examples) - --dmd_denoising_steps '1000,850,700,550,350,275,200,125' - --min_timestep_ratio 0.02 - --max_timestep_ratio 0.98 - --generator_update_interval 5 - --real_score_guidance_scale 3.5 -) - -torchrun \ ---nnodes 1 \ ---master_port "$MASTER_PORT" \ ---nproc_per_node "$NUM_GPUS" \ - fastvideo/training/distillation.py \ - --distill_model "wan" \ - --distill_method "dmd2" \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ - "${dmd_args[@]}" diff --git a/examples/distillation/phase1/run.md b/examples/distillation/phase1/run.md deleted file mode 100644 index 45969dc43..000000000 --- a/examples/distillation/phase1/run.md +++ /dev/null @@ -1,3 +0,0 @@ -# Phase 1 run link (optional) - -Paste your W&B run URL here for quick access. diff --git a/examples/distillation/phase1/temp.sh b/examples/distillation/phase1/temp.sh deleted file mode 100644 index 6559f46c8..000000000 --- a/examples/distillation/phase1/temp.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash -set -e -x - -# One-shot launch script for Phase 1 (DMD2Method + WanAdapter) Wan DMD2 few-step distillation. -# Uses the Wan-Syn parquet dataset + validation json (paths configurable below). -# -# Notes: -# - By default this runs W&B in offline mode (safer for overnight runs). -# If you want online logging: -# export WANDB_MODE=online -# export WANDB_API_KEY=... -# - Phase1 uses the general entrypoint: -# fastvideo/training/distillation.py --distill_model wan --distill_method dmd2 - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29504} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -if [[ -z "${NUM_GPUS:-}" ]]; then - if command -v nvidia-smi >/dev/null 2>&1; then - NUM_GPUS=$(nvidia-smi -L | wc -l) - else - NUM_GPUS=1 - fi -fi - -# Models -STUDENT_MODEL_PATH=${STUDENT_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} -TEACHER_MODEL_PATH=${TEACHER_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-14B-Diffusers"} -CRITIC_MODEL_PATH=${CRITIC_MODEL_PATH:-"Wan-AI/Wan2.1-T2V-1.3B-Diffusers"} - -# Data (parquet dataset folder) -DATA_DIR=${DATA_DIR:-"data/Wan-Syn_77x448x832_600k"} -DEFAULT_VALIDATION_DATASET_FILE=\ -"examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json" -VALIDATION_DATASET_FILE=${VALIDATION_DATASET_FILE:-"$DEFAULT_VALIDATION_DATASET_FILE"} - -RUN_ID=${RUN_ID:-"$(date +%Y%m%d_%H%M%S)"} -OUTPUT_DIR=${OUTPUT_DIR:-"outputs/phase1_wan2.1_dmd2_8steps_wansyn_${RUN_ID}"} - -training_args=( - --tracker_project_name "phase1_wan_dmd2_8steps_wansyn" - --output_dir "$OUTPUT_DIR" - --max_train_steps 4000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 20 - --num_height 448 - --num_width 832 - --num_frames 77 - --enable_gradient_checkpointing_type "full" - --simulate_generator_forward -) - -parallel_args=( - --num_gpus "$NUM_GPUS" - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim "$NUM_GPUS" -) - -model_args=( - --model_path "$STUDENT_MODEL_PATH" - --pretrained_model_name_or_path "$STUDENT_MODEL_PATH" - --real_score_model_path "$TEACHER_MODEL_PATH" - --fake_score_model_path "$CRITIC_MODEL_PATH" -) - -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 50 - --validation_sampling_steps "8" - --validation_guidance_scale "6.0" -) - -optimizer_args=( - --learning_rate 2e-6 - --mixed_precision "bf16" - --weight_decay 0.01 - --betas '0.0,0.999' - --max_grad_norm 1.0 - --fake_score_learning_rate 8e-6 - --fake_score_betas '0.0,0.999' -) - -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.0 - --dit_precision "fp32" - --flow_shift 8 - --seed 1000 -) - -dmd_args=( - --dmd_denoising_steps '1000,850,700,550,350,275,200,125' - --min_timestep_ratio 0.02 - --max_timestep_ratio 0.98 - --generator_update_interval 5 - --real_score_guidance_scale 3.5 -) - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --distill_model "wan" \ - --distill_method "dmd2" \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ - "${dmd_args[@]}" diff --git a/examples/distillation/phase2/README.md b/examples/distillation/phase2/README.md deleted file mode 100644 index f44875065..000000000 --- a/examples/distillation/phase2/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Phase 2 examples - -This folder contains **YAML-only** distillation examples for the Phase 2 -entrypoint: - -- `fastvideo/training/distillation.py --config path/to/distill.yaml` - -Important: - -- Phase 2 does not rewrite config paths automatically. Pass the explicit YAML - path (we keep runnable YAMLs next to the scripts under `examples/distillation/`). - -Start from: - -- `distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml` - -Recommended: - -- Edit the runnable YAML in this folder. -- `temp.sh` (runs the config above; same dataset + validation defaults as Phase0/Phase1). - -Resume: - -- Checkpoints are saved under `${output_dir}/checkpoint-/dcp/` when - `training.training_state_checkpointing_steps > 0` in YAML. -- To resume, pass a checkpoint directory (or an output_dir to auto-pick latest): - - `fastvideo/training/distillation.py --config --resume-from-checkpoint ` diff --git a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml b/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml deleted file mode 100644 index f3d1fb766..000000000 --- a/examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml +++ /dev/null @@ -1,84 +0,0 @@ -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 1 - sp_size: 1 - tp_size: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase2_wan2.1_dmd2_8steps_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (student default) - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Critic overrides (DMD2-specific) - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Method-agnostic knobs - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: phase2_wan_dmd2_8steps_wansyn - wandb_run_name: phase2_wan_dmd2_8steps_wansyn - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "8" - validation_guidance_scale: "6.0" - -pipeline_config: - flow_shift: 8 - # DMD2 validation rollout matches SDE-style generation. - sampler_kind: sde - -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh b/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh deleted file mode 100644 index 1f8012d60..000000000 --- a/examples/distillation/phase2/run_wan2.1_t2v_1.3B_dmd2_8steps.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -set -euo pipefail -set -x - -# NOTE: -# Phase 2 expects an explicit YAML path (YAML-only entrypoint). -# We keep runnable YAML next to this script under: -# examples/distillation/phase2/*.yaml - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_PORT=${MASTER_PORT:-29506} - -CONFIG=${CONFIG:-"examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} -if [[ ! -f "$CONFIG" ]]; then - echo "Missing Phase 2 YAML config at: $CONFIG" >&2 - exit 1 -fi -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --master_port "$MASTER_PORT" \ - --nproc_per_node "$NUM_GPUS" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" diff --git a/examples/distillation/phase2/temp.sh b/examples/distillation/phase2/temp.sh deleted file mode 100644 index 747eba4e1..000000000 --- a/examples/distillation/phase2/temp.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -set -e -x - -# One-shot launch script for Phase 2 (YAML-only, standalone runtime) Wan DMD2 -# few-step distillation. -# -# Uses the same defaults as Phase0/Phase1 temp.sh: -# - parquet dataset folder: data/Wan-Syn_77x448x832_600k -# - validation json: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json -# -# Notes: -# - Phase 2 expects an explicit YAML path (YAML-only entrypoint). -# We keep runnable YAML next to this script under: -# examples/distillation/phase2/*.yaml -# - By default this runs W&B in offline mode (safer for overnight runs). -# If you want online logging: -# export WANDB_MODE=online -# export WANDB_API_KEY=... - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29507} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -CONFIG=${CONFIG:-"examples/distillation/phase2/distill_wan2.1_t2v_1.3B_dmd2_8steps.yaml"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing Phase 2 YAML config at: $CONFIG" >&2 - exit 1 -fi - -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" diff --git a/examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml b/examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml deleted file mode 100644 index 30bc978b4..000000000 --- a/examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml +++ /dev/null @@ -1,83 +0,0 @@ -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 1 - sp_size: 1 - tp_size: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase2.9_wan2.1_dmd2_8steps_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (student default) - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Critic overrides (DMD2-specific) - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Method-agnostic knobs - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: phase2.9_wan_dmd2_8steps_wansyn - wandb_run_name: phase2.9_wan_dmd2_8steps_wansyn - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "8" - validation_guidance_scale: "6.0" - -pipeline_config: - flow_shift: 8 - sampler_kind: sde - -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/examples/distillation/phase2_9/temp.sh b/examples/distillation/phase2_9/temp.sh deleted file mode 100644 index fe5f30fa3..000000000 --- a/examples/distillation/phase2_9/temp.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -euo pipefail -if [[ "${DEBUG:-0}" == "1" ]]; then - set -x -fi - -# One-shot launch script for Phase 2.9 (models + registry dispatch + -# operation-centric adapter + method-managed validation). -# -# Uses the same dataset/validation defaults as phase0/phase1/phase2; the main -# difference is internal wiring/decoupling, not hyperparameters. - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29509} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -CONFIG=${CONFIG:-"examples/distillation/phase2_9/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase2.9.yaml"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" diff --git a/examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml b/examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml deleted file mode 100644 index 5c42dbce7..000000000 --- a/examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml +++ /dev/null @@ -1,82 +0,0 @@ -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.1_wan2.1_dmd2_8steps_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Method-agnostic knobs - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: phase3.1_wan_dmd2_8steps_wansyn - wandb_run_name: phase3.1_wan_dmd2_8steps - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "8" - validation_guidance_scale: "6.0" - -pipeline_config: - flow_shift: 8 - sampler_kind: sde - -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/examples/distillation/phase3_1/temp.sh b/examples/distillation/phase3_1/temp.sh deleted file mode 100644 index 6bff2fc42..000000000 --- a/examples/distillation/phase3_1/temp.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -euo pipefail -if [[ "${DEBUG:-0}" == "1" ]]; then - set -x -fi - -# One-shot launch script for Phase 3.1 (schema v2: `recipe` + `method_config`). -# -# Uses the same dataset/validation defaults as phase0/phase1/phase2/phase2_9; -# the main difference is config semantics (method knobs moved into -# `method_config`) and the removal of `simulate_generator_forward` coupling in -# adapters. - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29510} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -CONFIG=${CONFIG:-"examples/distillation/phase3_1/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.1.yaml"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" diff --git a/examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml b/examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml deleted file mode 100644 index 4e5d16dbf..000000000 --- a/examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml +++ /dev/null @@ -1,84 +0,0 @@ -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.2_wan2.1_dmd2_8steps_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Distillation (method-agnostic knobs) - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: phase3.2_wan_dmd2_8steps_wansyn - wandb_run_name: phase3.2_wan_dmd2_8steps_simulate - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "8" - validation_guidance_scale: "6.0" - -pipeline_config: - flow_shift: 8 - sampler_kind: sde - -method_config: - # Replace legacy `training.simulate_generator_forward`. - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - # Few-step schedule (single source of truth for the method). - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/examples/distillation/phase3_2/temp.sh b/examples/distillation/phase3_2/temp.sh deleted file mode 100644 index 525d742fe..000000000 --- a/examples/distillation/phase3_2/temp.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -euo pipefail -if [[ "${DEBUG:-0}" == "1" ]]; then - set -x -fi - -# One-shot launch script for Phase 3.2 (ODE/SDE sampler selection in `WanPipeline`). -# -# Compared to phase3_1, validation no longer needs `WanDMDPipeline`; the method -# supplies `sampler_kind + sampling_timesteps` via `ValidationRequest`, and -# `WanValidator` runs `WanPipeline` with the requested loop. - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29512} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -CONFIG=${CONFIG:-"examples/distillation/phase3_2/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.2.yaml"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" diff --git a/examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml b/examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml deleted file mode 100644 index 7bc4a1a8b..000000000 --- a/examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml +++ /dev/null @@ -1,64 +0,0 @@ -recipe: - family: wan - method: finetune - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.3_wan2.1_finetune_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Method-agnostic knobs - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: phase3.3_wan_finetune_wansyn - wandb_run_name: phase3.3_wan_finetune - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "50" - validation_guidance_scale: "6.0" - -pipeline_config: - flow_shift: 8 - sampler_kind: ode - -method_config: {} diff --git a/examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml b/examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml deleted file mode 100644 index cdede750f..000000000 --- a/examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml +++ /dev/null @@ -1,76 +0,0 @@ -recipe: - family: wan - method: finetune - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - -training: - # Distributed (mirror legacy Wan VSA finetune scripts) - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.3_wan2.1_finetune_vsa_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (match legacy defaults as closely as possible) - learning_rate: 1.0e-6 - mixed_precision: bf16 - betas: "0.9,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Method-agnostic knobs - training_cfg_rate: 0.1 - enable_gradient_checkpointing_type: full - - # VSA knobs (schedule ramps up sparsity during training) - VSA_sparsity: 0.9 - VSA_decay_rate: 0.03 - VSA_decay_interval_steps: 50 - - # Tracking / validation - tracker_project_name: phase3.3_wan_finetune_vsa_wansyn - wandb_run_name: phase3.3_wan_finetune_vsa - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 200 - validation_sampling_steps: "50" - validation_guidance_scale: "5.0" - -pipeline_config: - # Match legacy Wan VSA finetune scripts. - flow_shift: 1 - sampler_kind: ode - -method_config: - # Use the VSA attention metadata when FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN. - attn_kind: vsa diff --git a/examples/distillation/phase3_3/temp-vsa.sh b/examples/distillation/phase3_3/temp-vsa.sh deleted file mode 100644 index fac3a526a..000000000 --- a/examples/distillation/phase3_3/temp-vsa.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -euo pipefail -if [[ "${DEBUG:-0}" == "1" ]]; then - set -x -fi - -# One-shot launch script for Phase 3.3 finetune (SFT) with VSA backend. -# -# This mirrors legacy finetune scripts: -# - FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN -# - VSA_* knobs live in the YAML (VSA_sparsity / VSA_decay_rate / VSA_decay_interval_steps) - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-VIDEO_SPARSE_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29514} -export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-"/tmp/triton_cache_${USER}_$$"} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -CONFIG=${CONFIG:-"examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_vsa_phase3.3.yaml"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" diff --git a/examples/distillation/phase3_3/temp.sh b/examples/distillation/phase3_3/temp.sh deleted file mode 100644 index 44323a55e..000000000 --- a/examples/distillation/phase3_3/temp.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -euo pipefail -if [[ "${DEBUG:-0}" == "1" ]]; then - set -x -fi - -# One-shot launch script for Phase 3.3 (finetune method on the distillation scaffold). -# -# - Same trainer/bundle/adapter/family infrastructure as distillation. -# - Only `roles.student` is required; the method updates student weights only. -# - Validation is still supported via `ValidationRequest` + `WanValidator`. - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29513} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -CONFIG=${CONFIG:-"examples/distillation/phase3_3/finetune_wan2.1_t2v_1.3B_phase3.3.yaml"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" diff --git a/examples/distillation/phase3_4/distill-temp.sh b/examples/distillation/phase3_4/distill-temp.sh deleted file mode 100644 index 5d92ad5d9..000000000 --- a/examples/distillation/phase3_4/distill-temp.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -euo pipefail -if [[ "${DEBUG:-0}" == "1" ]]; then - set -x -fi - -# Phase 3.4 distillation (DMD2) launcher. -# -# - YAML config lives next to this script. -# - Run is driven by `fastvideo/training/distillation.py --config `. - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29515} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -CONFIG=${CONFIG:-"examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" - diff --git a/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml b/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml deleted file mode 100644 index f7f812c9c..000000000 --- a/examples/distillation/phase3_4/distill_wan2.1_t2v_1.3B_dmd2_8steps_phase3.4.yaml +++ /dev/null @@ -1,83 +0,0 @@ -recipe: - family: wan - method: dmd2 - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - teacher: - family: wan - path: Wan-AI/Wan2.1-T2V-14B-Diffusers - trainable: false - disable_custom_init_weights: true - critic: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.4_wan2.1_dmd2_8steps_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (student default) - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Critic overrides (DMD2-specific) - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Method-agnostic knobs - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: distillation_phase3 - wandb_run_name: phase3.4_wan_dmd2_8steps_simulate_aftermergewangame - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "8" - validation_guidance_scale: "6.0" - -pipeline_config: - flow_shift: 8 - sampler_kind: sde - -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - dmd_denoising_steps: [1000, 850, 700, 550, 350, 275, 200, 125] diff --git a/examples/distillation/phase3_4/finetune-vsa-temp.sh b/examples/distillation/phase3_4/finetune-vsa-temp.sh deleted file mode 100644 index 15a941935..000000000 --- a/examples/distillation/phase3_4/finetune-vsa-temp.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -set -euo pipefail -if [[ "${DEBUG:-0}" == "1" ]]; then - set -x -fi - -# Phase 3.4 finetune (SFT) with VSA backend launcher. -# -# Mirrors legacy finetune scripts: -# - FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN -# - VSA_* knobs live in the YAML (VSA_sparsity / VSA_decay_rate / VSA_decay_interval_steps) - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-VIDEO_SPARSE_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29516} -export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-"/tmp/triton_cache_${USER}_$$"} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -CONFIG=${CONFIG:-"examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" - diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml deleted file mode 100644 index 24ce82021..000000000 --- a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_phase3.4.yaml +++ /dev/null @@ -1,69 +0,0 @@ -recipe: - family: wan - method: finetune - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - -training: - # Distributed (mirror legacy Wan VSA finetune scripts) - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.4_wan2.1_finetune_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 1.0e-6 - mixed_precision: bf16 - betas: "0.9,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Method-agnostic knobs - training_cfg_rate: 0.1 - enable_gradient_checkpointing_type: full - - - # Tracking / validation - tracker_project_name: distillation_phase3 - wandb_run_name: phase3.4_wan_finetune - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "50" - validation_guidance_scale: "5.0" - -pipeline_config: - flow_shift: 3 - sampler_kind: ode - -method_config: {} diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml deleted file mode 100644 index 1c5b5be49..000000000 --- a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4.yaml +++ /dev/null @@ -1,76 +0,0 @@ -recipe: - family: wan - method: finetune - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - -training: - # Distributed (mirror legacy Wan VSA finetune scripts) - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.4_wan2.1_finetune_vsa_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 1.0e-6 - mixed_precision: bf16 - betas: "0.9,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Method-agnostic knobs - training_cfg_rate: 0.1 - enable_gradient_checkpointing_type: full - - # VSA knobs (schedule ramps up sparsity during training) - VSA_sparsity: 0.7 - VSA_decay_rate: 0.03 - VSA_decay_interval_steps: 50 - - # Tracking / validation - tracker_project_name: distillation_phase3 - wandb_run_name: phase3.4_wan_finetune_vsa_0.7_0.03_50 - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "50" - validation_guidance_scale: "5.0" - -pipeline_config: - # Match legacy Wan VSA finetune scripts. - flow_shift: 3 - sampler_kind: ode - -method_config: - # Use the VSA attention metadata when FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN. - attn_kind: vsa diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml deleted file mode 100644 index c8ff8c73a..000000000 --- a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.7sparsity.yaml +++ /dev/null @@ -1,76 +0,0 @@ -recipe: - family: wan - method: finetune - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - -training: - # Distributed (mirror legacy Wan VSA finetune scripts) - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.4_wan2.1_finetune_vsa_0.7_0.03_1_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 1.0e-6 - mixed_precision: bf16 - betas: "0.9,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Method-agnostic knobs - training_cfg_rate: 0.1 - enable_gradient_checkpointing_type: full - - # VSA knobs (schedule ramps up sparsity during training) - VSA_sparsity: 0.7 - VSA_decay_rate: 0.03 - VSA_decay_interval_steps: 1 - - # Tracking / validation - tracker_project_name: distillation_phase3 - wandb_run_name: phase3.4_wan_finetune_vsa_0.7_0.03_1 - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "50" - validation_guidance_scale: "5.0" - -pipeline_config: - # Match legacy Wan VSA finetune scripts. - flow_shift: 3 - sampler_kind: ode - -method_config: - # Use the VSA attention metadata when FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN. - attn_kind: vsa diff --git a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml deleted file mode 100644 index 7e2d92a13..000000000 --- a/examples/distillation/phase3_4/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml +++ /dev/null @@ -1,76 +0,0 @@ -recipe: - family: wan - method: finetune - -roles: - student: - family: wan - path: Wan-AI/Wan2.1-T2V-1.3B-Diffusers - trainable: true - -training: - # Distributed (mirror legacy Wan VSA finetune scripts) - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder) - data_path: data/Wan-Syn_77x448x832_600k - dataloader_num_workers: 4 - - # Batch / shape (matches Wan-Syn 77x448x832) - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 448 - num_width: 832 - num_frames: 77 - - # Output / steps - output_dir: outputs/phase3.4_wan2.1_finetune_vsa_0.7_0.03_1_wansyn - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer - learning_rate: 1.0e-6 - mixed_precision: bf16 - betas: "0.9,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Method-agnostic knobs - training_cfg_rate: 0.1 - enable_gradient_checkpointing_type: full - - # VSA knobs (schedule ramps up sparsity during training) - VSA_sparsity: 0.9 - VSA_decay_rate: 0.03 - VSA_decay_interval_steps: 1 - - # Tracking / validation - tracker_project_name: distillation_phase3 - wandb_run_name: phase3.4_wan_finetune_vsa_0.9_0.03_1 - log_validation: true - validation_dataset_file: examples/training/finetune/Wan2.1-VSA/Wan-Syn-Data/validation_4.json - validation_steps: 50 - validation_sampling_steps: "50" - validation_guidance_scale: "5.0" - -pipeline_config: - # Match legacy Wan VSA finetune scripts. - flow_shift: 3 - sampler_kind: ode - -method_config: - # Use the VSA attention metadata when FASTVIDEO_ATTENTION_BACKEND=VIDEO_SPARSE_ATTN. - attn_kind: vsa diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml deleted file mode 100644 index 75c61ac26..000000000 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal.yaml +++ /dev/null @@ -1,88 +0,0 @@ -recipe: - family: wangame_causal - method: dfsft - -roles: - shared_component_role: student - student: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - -training: - # Distributed (mirror legacy WanGame finetune scripts) - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder) - # - # Supports comma-separated `path` or `path:N` (repeat count) entries. - # N=0 means skip (handy to toggle datasets without deleting lines). - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_dfsft_causal - max_train_steps: 20000 - seed: 1000 - checkpoints_total_limit: 2 - - # Optimizer - learning_rate: 1.0e-5 - mixed_precision: bf16 - betas: "0.9,0.999" - weight_decay: 1.0e-4 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Method-agnostic knobs - training_cfg_rate: 0.0 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: distillation_wangame - wandb_run_name: wangame_dfsft_causal - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [40] - rollout_mode: streaming - sampler_kind: ode - ode_solver: euler - guidance_scale: 1.0 - num_frames: 75 - -default_pipeline_config: - flow_shift: 3 - sampler_kind: ode - -method_config: - attn_kind: dense - chunk_size: 3 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm deleted file mode 100644 index 95b6171e5..000000000 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.slurm +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wangame_dfsft_causal -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=slurm_outputs/wangame_dfsft_causal_4n8g/wangame_dfsft_causal_%j.out -#SBATCH --error=slurm_outputs/wangame_dfsft_causal_4n8g/wangame_dfsft_causal_%j.err -#SBATCH --exclusive - -set -euo pipefail - -# ---- Env (mirror legacy WanGame slurm scripts) ---- -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-INIT,NET} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} - -# Triton cache (avoid cross-job collisions; per-node task is OK in practice) -export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/tmp/triton_cache_${SLURM_JOB_ID}_${SLURM_PROCID}} - -# Rendezvous (torchrun) -export MASTER_PORT=${MASTER_PORT:-29501} -nodes=( $(scontrol show hostnames "$SLURM_JOB_NODELIST") ) -export MASTER_ADDR=${MASTER_ADDR:-${nodes[0]}} - -# W&B (recommended to pass WANDB_API_KEY via environment / secret manager) -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-online} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -source ~/conda/miniconda/bin/activate -conda activate alexfv - -CONFIG=${CONFIG:-"examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml"} -OUTPUT_DIR=${OUTPUT_DIR:-"outputs/wangame_dfsft_causal_4n8g/${SLURM_JOB_ID}"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -# 4 nodes × 8 GPUs -NUM_NODES=${SLURM_NNODES} -GPUS_PER_NODE=8 - -srun torchrun \ - --nnodes "$NUM_NODES" \ - --nproc_per_node "$GPUS_PER_NODE" \ - --rdzv_backend c10d \ - --rdzv_endpoint "$MASTER_ADDR:$MASTER_PORT" \ - --node_rank "$SLURM_PROCID" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" \ - --override-output-dir "$OUTPUT_DIR" - diff --git a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml b/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml deleted file mode 100644 index 73e8b7254..000000000 --- a/examples/distillation/wangame/dfsft_wangame2.1_i2v_1.3B_causal_4n8g.yaml +++ /dev/null @@ -1,88 +0,0 @@ -recipe: - family: wangame_causal - method: dfsft - -roles: - shared_component_role: student - student: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - -training: - # Distributed (4 nodes × 8 GPUs = 32 ranks) - num_gpus: 32 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 1 - hsdp_shard_dim: 32 - - # Data (parquet dataset folder) - # - # Supports comma-separated `path` or `path:N` (repeat count) entries. - # N=0 means skip (handy to toggle datasets without deleting lines). - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_dfsft_causal_4n8g - max_train_steps: 200000 - seed: 1000 - checkpoints_total_limit: 2 - - # Optimizer - learning_rate: 1.0e-5 - mixed_precision: bf16 - betas: "0.9,0.95" - weight_decay: 1.0e-5 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Method-agnostic knobs - training_cfg_rate: 0.0 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: distillation_wangame - wandb_run_name: wangame_dfsft_causal_4n8g - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json - every_steps: 100 - sampling_steps: [40] - rollout_mode: streaming - sampler_kind: ode - ode_solver: euler - guidance_scale: 1.0 - num_frames: 69 - -default_pipeline_config: - flow_shift: 3 - sampler_kind: ode - -method_config: - attn_kind: dense - chunk_size: 3 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml deleted file mode 100644 index 56d405a39..000000000 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_actioncfg.yaml +++ /dev/null @@ -1,115 +0,0 @@ -# WanGame DMD2 distillation (few-step rollout). -# -# - Fill in `roles.teacher.path` with your teacher checkpoint (must be wangame family). -# - This YAML is the single source of truth for the run. - -recipe: - family: wangame - method: dmd2 - -roles: - student: - family: wangame - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - teacher: - family: wangame - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: false - disable_custom_init_weights: true - critic: - family: wangame - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder). - # Supports comma-separated `path` or `path:N` (repeat count) entries. - data_path: >- - /mnt/weka/home/hao.zhang/kaiqin/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_dmd2_8steps_distill - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (student) - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Optimizer (critic / fake-score) - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Distillation (method-agnostic knobs) - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Tracking / validation - tracker_project_name: distillation_wangame - wandb_run_name: wangame_dmd2_4steps_distill_actioncfg - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 50 - sampling_steps: [4] - sampler_kind: sde - guidance_scale: 1.0 - -default_pipeline_config: - flow_shift: 5 - sampler_kind: sde - -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - - # Few-step schedule (single source of truth for the method). - # These are "step indices" and will be warped by the (shifted) scheduler. - warp_denoising_step: true - dmd_denoising_steps: [1000,750,500,250] - - # Define CFG "uncond" semantics (operation-centric). - # - # If all channels are `keep`, then uncond == cond, so CFG becomes a no-op: - # real_cfg = cond + (cond - uncond) * scale == cond - cfg_uncond: - on_missing: error - # For I2V+action, a practical default is "no-action baseline". - action: zero - # Keep the reference image conditioning for I2V. - image: keep - # Text is unused in current wangame training batches, but kept for schema symmetry. - text: keep diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml deleted file mode 100644 index add07c027..000000000 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg.yaml +++ /dev/null @@ -1,115 +0,0 @@ -# WanGame causal DMD2 distillation (40-step teacher -> 4-step student). -# -# - Teacher weights come from a DFSFT causal checkpoint (DCP format). -# - Student/Critic are warmstarted from the same checkpoint (copied from -# checkpoint role `student`). -# - Role `path` still points at a *base* wangame model directory (needed to -# load configs/architectures); `init_from_checkpoint` overwrites weights. - -recipe: - family: wangame_causal - method: dmd2 - -roles: - shared_component_role: student - student: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - teacher: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: false - disable_custom_init_weights: true - init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - init_from_checkpoint_role: student - critic: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - disable_custom_init_weights: true - init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - init_from_checkpoint_role: student - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder). - # Supports comma-separated `path` or `path:N` (repeat count) entries. - data_path: >- - /mnt/weka/home/hao.zhang/kaiqin/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_dmd2_4steps_distill_causal_teacher22000 - max_train_steps: 400000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (student) - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Optimizer (critic / fake-score) - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Distillation (method-agnostic knobs) - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Tracking / validation - tracker_project_name: distillation_wangame - wandb_run_name: wangame_dmd2_4steps_distill_causal_teacher22000 - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 50 - sampling_steps: [4] - sampler_kind: sde - guidance_scale: 1.0 - -default_pipeline_config: - flow_shift: 3 - sampler_kind: sde - -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - - # Few-step schedule (single source of truth for the method). - # These are "step indices" and will be warped by the (shifted) scheduler. - warp_denoising_step: true - dmd_denoising_steps: [1000,750,500,250] - - # Define CFG "uncond" semantics (operation-centric). - cfg_uncond: - on_missing: error - action: keep - image: keep - text: keep diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm deleted file mode 100644 index ebde340d0..000000000 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wangame_dmd2_causal -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=slurm_outputs/wangame_dmd2_causal_teacher22000_4n8g/wangame_dmd2_causal_%j.out -#SBATCH --error=slurm_outputs/wangame_dmd2_causal_teacher22000_4n8g/wangame_dmd2_causal_%j.err -#SBATCH --exclusive - -set -euo pipefail - -# ---- Env (mirror legacy WanGame slurm scripts) ---- -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-INIT,NET} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} - -# Triton cache (avoid cross-job collisions; per-node task is OK in practice) -export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/tmp/triton_cache_${SLURM_JOB_ID}_${SLURM_PROCID}} - -# Rendezvous (torchrun) -export MASTER_PORT=${MASTER_PORT:-29501} -nodes=( $(scontrol show hostnames "$SLURM_JOB_NODELIST") ) -export MASTER_ADDR=${MASTER_ADDR:-${nodes[0]}} - -# W&B (recommended to pass WANDB_API_KEY via environment / secret manager) -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-online} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -source ~/conda/miniconda/bin/activate -conda activate alexfv - -CONFIG=${CONFIG:-"examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml"} -OUTPUT_DIR=${OUTPUT_DIR:-"outputs/wangame_dmd2_causal_teacher22000_4n8g/${SLURM_JOB_ID}"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -# 4 nodes × 8 GPUs -NUM_NODES=${SLURM_NNODES} -GPUS_PER_NODE=8 - -srun torchrun \ - --nnodes "$NUM_NODES" \ - --nproc_per_node "$GPUS_PER_NODE" \ - --rdzv_backend c10d \ - --rdzv_endpoint "$MASTER_ADDR:$MASTER_PORT" \ - --node_rank "$SLURM_PROCID" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" \ - --override-output-dir "$OUTPUT_DIR" - diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml deleted file mode 100644 index 359d34544..000000000 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml +++ /dev/null @@ -1,130 +0,0 @@ -# WanGame causal DMD2 distillation (40-step teacher -> 4-step student). -# -# 4 nodes × 8 GPUs config (32 ranks). -# -# - Teacher weights come from a DFSFT causal checkpoint (DCP format). -# - Student/Critic are warmstarted from the same checkpoint (copied from -# checkpoint role `student`). -# - Role `path` still points at a *base* wangame model directory (needed to -# load configs/architectures); `init_from_checkpoint` overwrites weights. - -recipe: - family: wangame_causal - method: dmd2 - -roles: - shared_component_role: student - student: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - teacher: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: false - disable_custom_init_weights: true - init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - init_from_checkpoint_role: student - critic: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - disable_custom_init_weights: true - init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - init_from_checkpoint_role: student - -training: - # Distributed (4 nodes × 8 GPUs = 32 ranks) - num_gpus: 32 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 1 - hsdp_shard_dim: 32 - - # Data (parquet dataset folder). - # Supports comma-separated `path` or `path:N` (repeat count) entries. - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_dmd2_4steps_distill_causal_teacher22000_4n8g - max_train_steps: 400000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (student) - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Optimizer (critic / fake-score) - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Distillation (method-agnostic knobs) - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Tracking / validation - tracker_project_name: distillation_wangame - wandb_run_name: wangame_dmd2_4steps_distill_causal_teacher22000_4n8g - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 50 - sampling_steps: [4] - sampler_kind: sde - rollout_mode: streaming - guidance_scale: 1.0 - # Streaming causal denoising requires: - # - num_frames divisible by num_frame_per_block (default=3) - # - (num_frames - 1) divisible by 4 (action tensor convention) - num_frames: 69 - -default_pipeline_config: - flow_shift: 3 - sampler_kind: sde - -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - - # Few-step schedule (single source of truth for the method). - # These are "step indices" and will be warped by the (shifted) scheduler. - warp_denoising_step: true - dmd_denoising_steps: [1000,750,500,250] - - # Define CFG "uncond" semantics (operation-centric). - # Here `keep` means uncond == cond (CFG becomes a no-op). - cfg_uncond: - on_missing: error - action: keep - image: keep - text: keep diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml deleted file mode 100644 index 7adcae97b..000000000 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_dmd2_4steps_nocfg.yaml +++ /dev/null @@ -1,115 +0,0 @@ -# WanGame DMD2 distillation (few-step rollout). -# -# - Fill in `roles.teacher.path` with your teacher checkpoint (must be wangame family). -# - This YAML is the single source of truth for the run. - -recipe: - family: wangame - method: dmd2 - -roles: - student: - family: wangame - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - teacher: - family: wangame - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: false - disable_custom_init_weights: true - critic: - family: wangame - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - disable_custom_init_weights: true - -training: - # Distributed - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder). - # Supports comma-separated `path` or `path:N` (repeat count) entries. - data_path: >- - /mnt/weka/home/hao.zhang/kaiqin/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_dmd2_8steps_distill - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (student) - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Optimizer (critic / fake-score) - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Distillation (method-agnostic knobs) - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: full - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Tracking / validation - tracker_project_name: distillation_wangame - wandb_run_name: wangame_dmd2_4steps_distill_nocfg - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 50 - sampling_steps: [4] - sampler_kind: sde - guidance_scale: 1.0 - -default_pipeline_config: - flow_shift: 5 - sampler_kind: sde - -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - - # Few-step schedule (single source of truth for the method). - # These are "step indices" and will be warped by the (shifted) scheduler. - warp_denoising_step: true - dmd_denoising_steps: [1000,750,500,250] - - # Define CFG "uncond" semantics (operation-centric). - # - # If all channels are `keep`, then uncond == cond, so CFG becomes a no-op: - # real_cfg = cond + (cond - uncond) * scale == cond - cfg_uncond: - on_missing: error - # For I2V+action, a practical default is "no-action baseline". - action: keep - # Keep the reference image conditioning for I2V. - image: keep - # Text is unused in current wangame training batches, but kept for schema symmetry. - text: keep diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm deleted file mode 100644 index c7216fb4f..000000000 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.slurm +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wangame_self_forcing_causal -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=slurm_outputs/wangame_self_forcing_causal_teacher22000_4n8g/wangame_self_forcing_causal_%j.out -#SBATCH --error=slurm_outputs/wangame_self_forcing_causal_teacher22000_4n8g/wangame_self_forcing_causal_%j.err -#SBATCH --exclusive - -set -euo pipefail - -# ---- Env (mirror legacy WanGame slurm scripts) ---- -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-INIT,NET} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} - -# Triton cache (avoid cross-job collisions; per-node task is OK in practice) -export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/tmp/triton_cache_${SLURM_JOB_ID}_${SLURM_PROCID}} - -# Rendezvous (torchrun) -export MASTER_PORT=${MASTER_PORT:-29501} -nodes=( $(scontrol show hostnames "$SLURM_JOB_NODELIST") ) -export MASTER_ADDR=${MASTER_ADDR:-${nodes[0]}} - -# W&B (recommended to pass WANDB_API_KEY via environment / secret manager) -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-online} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -source ~/conda/miniconda/bin/activate -conda activate alexfv - -CONFIG=${CONFIG:-"examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml"} -OUTPUT_DIR=${OUTPUT_DIR:-"outputs/wangame_self_forcing_causal_teacher22000_4n8g/${SLURM_JOB_ID}"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -# 4 nodes × 8 GPUs -NUM_NODES=${SLURM_NNODES} -GPUS_PER_NODE=8 - -srun torchrun \ - --nnodes "$NUM_NODES" \ - --nproc_per_node "$GPUS_PER_NODE" \ - --rdzv_backend c10d \ - --rdzv_endpoint "$MASTER_ADDR:$MASTER_PORT" \ - --node_rank "$SLURM_PROCID" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" \ - --override-output-dir "$OUTPUT_DIR" diff --git a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml b/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml deleted file mode 100644 index ece7f0303..000000000 --- a/examples/distillation/wangame/distill_wangame2.1_i2v_1.3B_self_forcing_4steps_causal_teacher_ckpt22000_nocfg_4n8g.yaml +++ /dev/null @@ -1,139 +0,0 @@ -# WanGame causal Self-Forcing distillation (40-step teacher -> 4-step student). -# -# 4 nodes × 8 GPUs config (32 ranks). -# -# - Teacher weights come from a DFSFT causal checkpoint (DCP format). -# - Student/Critic are warmstarted from the same checkpoint (copied from -# checkpoint role `student`). -# - Role `path` still points at a *base* wangame model directory (needed to -# load configs/architectures); `init_from_checkpoint` overwrites weights. - -recipe: - family: wangame_causal - method: self_forcing - -roles: - shared_component_role: student - student: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - teacher: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: false - disable_custom_init_weights: true - init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - init_from_checkpoint_role: student - critic: - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - disable_custom_init_weights: true - init_from_checkpoint: outputs/wangame_dfsft_causal_4n8g/persistent/checkpoint-22000 - init_from_checkpoint_role: student - -training: - # Distributed (4 nodes × 8 GPUs = 32 ranks) - num_gpus: 32 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 1 - hsdp_shard_dim: 32 - - # Data (parquet dataset folder). - # Supports comma-separated `path` or `path:N` (repeat count) entries. - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_self_forcing_4steps_distill_causal_teacher22000_4n8g - max_train_steps: 4000 - seed: 1000 - checkpoints_total_limit: 3 - - # Optimizer (student) - learning_rate: 2.0e-6 - mixed_precision: bf16 - betas: "0.0,0.999" - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Optimizer (critic / fake-score) - fake_score_learning_rate: 8.0e-6 - fake_score_betas: "0.0,0.999" - fake_score_lr_scheduler: constant - - # Distillation (method-agnostic knobs) - training_cfg_rate: 0.0 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - enable_gradient_checkpointing_type: null - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Tracking / validation - tracker_project_name: distillation_wangame - wandb_run_name: wangame_self_forcing_4steps_distill_causal_teacher22000_4n8g - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [4] - sampler_kind: sde - rollout_mode: streaming - guidance_scale: 1.0 - # Streaming causal denoising requires: - # - num_frames divisible by num_frame_per_block (default=3) - # - (num_frames - 1) divisible by 4 (action tensor convention) - num_frames: 69 - -default_pipeline_config: - flow_shift: 3 - sampler_kind: sde - -method_config: - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - - # Few-step schedule (single source of truth for the method). - # These are "step indices" and will be warped by the (shifted) scheduler. - warp_denoising_step: true - dmd_denoising_steps: [1000,750,500,250] - - # Self-Forcing rollout knobs - chunk_size: 3 - student_sample_type: sde - same_step_across_blocks: false - last_step_only: false - context_noise: 0.0 - enable_gradient_in_rollout: true - start_gradient_frame: 0 - - # Define CFG "uncond" semantics (operation-centric). - # Here `keep` means uncond == cond (CFG becomes a no-op). - cfg_uncond: - on_missing: error - action: keep - image: keep - text: keep diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml deleted file mode 100644 index a8c800fe7..000000000 --- a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml +++ /dev/null @@ -1,82 +0,0 @@ -recipe: - family: wangame - method: finetune - -roles: - student: - family: wangame - path: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - -training: - # Distributed (mirror legacy WanGame finetune scripts) - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - # Data (parquet dataset folder) - # - # Supports comma-separated `path` or `path:N` (repeat count) entries. - # N=0 means skip (handy to toggle datasets without deleting lines). - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_finetune - max_train_steps: 20000 - seed: 1000 - checkpoints_total_limit: 2 - - # Optimizer - learning_rate: 1.0e-5 - mixed_precision: bf16 - betas: "0.9,0.999" - weight_decay: 1.0e-4 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Method-agnostic knobs - training_cfg_rate: 0.0 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: distillation_wangame - wandb_run_name: wangame_finetune_from_pretrained - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [40] - sampler_kind: ode - guidance_scale: 1.0 - -default_pipeline_config: - flow_shift: 3 - sampler_kind: ode - -method_config: - attn_kind: dense diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.slurm b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.slurm deleted file mode 100644 index c1219a171..000000000 --- a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.slurm +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wangame_finetune_bidi -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=slurm_outputs/wangame_finetune_bidi_4n8g/wangame_finetune_bidi_%j.out -#SBATCH --error=slurm_outputs/wangame_finetune_bidi_4n8g/wangame_finetune_bidi_%j.err -#SBATCH --exclusive - -set -euo pipefail - -# ---- Env (mirror legacy WanGame slurm scripts) ---- -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-INIT,NET} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} - -# Triton cache (avoid cross-job collisions; per-node task is OK in practice) -export TRITON_CACHE_DIR=${TRITON_CACHE_DIR:-/tmp/triton_cache_${SLURM_JOB_ID}_${SLURM_PROCID}} - -# Rendezvous (torchrun) -export MASTER_PORT=${MASTER_PORT:-29501} -nodes=( $(scontrol show hostnames "$SLURM_JOB_NODELIST") ) -export MASTER_ADDR=${MASTER_ADDR:-${nodes[0]}} - -# W&B (recommended to pass WANDB_API_KEY via environment / secret manager) -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-online} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -source ~/conda/miniconda/bin/activate -conda activate alexfv - -CONFIG=${CONFIG:-"examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml"} -OUTPUT_DIR=${OUTPUT_DIR:-"outputs/wangame_finetune_bidi_4n8g/${SLURM_JOB_ID}"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -# 4 nodes × 8 GPUs -NUM_NODES=${SLURM_NNODES} -GPUS_PER_NODE=8 - -srun torchrun \ - --nnodes "$NUM_NODES" \ - --nproc_per_node "$GPUS_PER_NODE" \ - --rdzv_backend c10d \ - --rdzv_endpoint "$MASTER_ADDR:$MASTER_PORT" \ - --node_rank "$SLURM_PROCID" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" \ - --override-output-dir "$OUTPUT_DIR" - diff --git a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml b/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml deleted file mode 100644 index a6935afb6..000000000 --- a/examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B_bidirectional_4n8g.yaml +++ /dev/null @@ -1,88 +0,0 @@ -# WanGame bidirectional finetune (4 nodes × 8 GPUs). -# -# This is a "from scratch" run in the sense that it does not warmstart from any -# DCP checkpoint (no `init_from_checkpoint`). It starts from the base pretrained -# `roles.student.path` weights. - -recipe: - family: wangame - method: finetune - -roles: - student: - family: wangame - path: weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers - trainable: true - -training: - # Distributed (4 nodes × 8 GPUs = 32 ranks) - num_gpus: 32 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 1 - hsdp_shard_dim: 32 - - # Data (parquet dataset folder) - # - # Supports comma-separated `path` or `path:N` (repeat count) entries. - # N=0 means skip (handy to toggle datasets without deleting lines). - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 - dataloader_num_workers: 4 - - # Batch / shape - train_batch_size: 1 - train_sp_batch_size: 1 - gradient_accumulation_steps: 1 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - # Output / steps - output_dir: outputs/wangame_finetune_bidi_4n8g - max_train_steps: 200000 - seed: 1000 - checkpoints_total_limit: 2 - - # Optimizer - learning_rate: 1.0e-5 - mixed_precision: bf16 - betas: "0.9,0.999" - weight_decay: 1.0e-4 - lr_scheduler: constant - lr_warmup_steps: 0 - max_grad_norm: 1.0 - - # Checkpointing - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - - # Method-agnostic knobs - training_cfg_rate: 0.0 - enable_gradient_checkpointing_type: full - - # Tracking / validation - tracker_project_name: distillation_wangame - wandb_run_name: wangame_finetune_bidirectional_4n8g_from_pretrained - validation: - enabled: true - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json - every_steps: 100 - sampling_steps: [40] - sampler_kind: ode - guidance_scale: 1.0 - -default_pipeline_config: - flow_shift: 3 - sampler_kind: ode - -method_config: - attn_kind: dense diff --git a/examples/distillation/wangame/run.sh b/examples/distillation/wangame/run.sh deleted file mode 100644 index ab786423e..000000000 --- a/examples/distillation/wangame/run.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -euo pipefail -if [[ "${DEBUG:-0}" == "1" ]]; then - set -x -fi - -# WanGame finetune launcher (new distillation framework). -# -# - YAML config lives next to this script. -# - Run is driven by `fastvideo/training/distillation.py --config `. - -export NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} -export TORCH_NCCL_ENABLE_MONITORING=${TORCH_NCCL_ENABLE_MONITORING:-0} -export TOKENIZERS_PARALLELISM=${TOKENIZERS_PARALLELISM:-false} -export FASTVIDEO_ATTENTION_BACKEND=${FASTVIDEO_ATTENTION_BACKEND:-FLASH_ATTN} -export WANDB_BASE_URL=${WANDB_BASE_URL:-"https://api.wandb.ai"} -export WANDB_MODE=${WANDB_MODE:-offline} -export MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} -export MASTER_PORT=${MASTER_PORT:-29515} - -if [[ "$WANDB_MODE" == "online" && -z "${WANDB_API_KEY:-}" ]]; then - echo "WANDB_MODE=online requires WANDB_API_KEY in env." >&2 - exit 1 -fi - -CONFIG=${CONFIG:-"examples/distillation/wangame/finetune_wangame2.1_i2v_1.3B.yaml"} - -if [[ ! -f "$CONFIG" ]]; then - echo "Missing distillation YAML config at: $CONFIG" >&2 - exit 1 -fi - -NUM_GPUS=$(python -c "import yaml; cfg=yaml.safe_load(open('$CONFIG', encoding='utf-8')); print(int(cfg.get('training', {}).get('num_gpus', 1) or 1))") - -torchrun \ - --nnodes 1 \ - --nproc_per_node "$NUM_GPUS" \ - --master_addr "$MASTER_ADDR" \ - --master_port "$MASTER_PORT" \ - fastvideo/training/distillation.py \ - --config "$CONFIG" - diff --git a/examples/distillation/refactor/dfsft_wangame_causal_v3.yaml b/examples/train/dfsft_wangame_causal_v3.yaml similarity index 100% rename from examples/distillation/refactor/dfsft_wangame_causal_v3.yaml rename to examples/train/dfsft_wangame_causal_v3.yaml diff --git a/examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml b/examples/train/distill_wan2.1_t2v_1.3B_dmd2.yaml similarity index 100% rename from examples/distillation/refactor/distill_wan2.1_t2v_1.3B_dmd2.yaml rename to examples/train/distill_wan2.1_t2v_1.3B_dmd2.yaml diff --git a/examples/distillation/refactor/example.yaml b/examples/train/example.yaml similarity index 100% rename from examples/distillation/refactor/example.yaml rename to examples/train/example.yaml diff --git a/examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/train/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml similarity index 100% rename from examples/distillation/refactor/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml rename to examples/train/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml diff --git a/examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml b/examples/train/finetune_wangame2.1_i2v_1.3B.yaml similarity index 100% rename from examples/distillation/refactor/finetune_wangame2.1_i2v_1.3B.yaml rename to examples/train/finetune_wangame2.1_i2v_1.3B.yaml diff --git a/examples/distillation/refactor/rfc.md b/examples/train/rfc.md similarity index 100% rename from examples/distillation/refactor/rfc.md rename to examples/train/rfc.md diff --git a/examples/distillation/refactor/run.sh b/examples/train/run.sh similarity index 100% rename from examples/distillation/refactor/run.sh rename to examples/train/run.sh diff --git a/examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml b/examples/train/self_forcing_wangame_causal_v3.yaml similarity index 100% rename from examples/distillation/refactor/self_forcing_wangame_causal_v3.yaml rename to examples/train/self_forcing_wangame_causal_v3.yaml From 850dfd87c257600d6bf4cdfe1561706a331c5d72 Mon Sep 17 00:00:00 2001 From: alexzms <3036648523@qq.com> Date: Fri, 6 Mar 2026 23:07:25 +0000 Subject: [PATCH 211/214] resolve --- fastvideo/configs/models/dits/__init__.py | 2 +- .../models/dits/wangame/hyworld_action_module.py | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/fastvideo/configs/models/dits/__init__.py b/fastvideo/configs/models/dits/__init__.py index 0af21d5c0..8caddca7e 100644 --- a/fastvideo/configs/models/dits/__init__.py +++ b/fastvideo/configs/models/dits/__init__.py @@ -14,6 +14,6 @@ "HunyuanVideoConfig", "HunyuanVideo15Config", "WanVideoConfig", "StepVideoConfig", "CosmosVideoConfig", "Cosmos25VideoConfig", "LongCatVideoConfig", "LTX2VideoConfig", "HYWorldConfig", - "LingBotWorldVideoConfig", "WanGameVideoConfig", "WanLingBotVideoConfig" + "LingBotWorldVideoConfig", "WanGameVideoConfig", "WanLingBotVideoConfig", "LingBotWorldVideoConfig", "HunyuanGameCraftConfig", "WanVideoConfig" ] diff --git a/fastvideo/models/dits/wangame/hyworld_action_module.py b/fastvideo/models/dits/wangame/hyworld_action_module.py index 0e482c6c2..a0159d12f 100644 --- a/fastvideo/models/dits/wangame/hyworld_action_module.py +++ b/fastvideo/models/dits/wangame/hyworld_action_module.py @@ -240,20 +240,6 @@ def forward(self, # In this case, use LocalAttention (supports different Q/KV lengths) if query_all.shape[1] != key_all.shape[1]: raise ValueError("Q and KV have different sequence lengths") - # KV cache mode: Q has new tokens only, KV has cached + new tokens - # Use LocalAttention which supports different Q/KV lengths - # LocalAttention will use the appropriate backend (SageAttn, FlashAttn, etc.) - if not hasattr(self, '_kv_cache_attn'): - from fastvideo.attention import LocalAttention - self._kv_cache_attn = LocalAttention( - num_heads=self.num_heads, - head_size=self.head_dim, - causal=False, - supported_attention_backends=(AttentionBackendEnum.SAGE_ATTN, - AttentionBackendEnum.FLASH_ATTN, - AttentionBackendEnum.TORCH_SDPA) - ) - hidden_states_all = self._kv_cache_attn(query_all, key_all, value_all) else: # Same sequence length: use DistributedAttention (supports SP) # Create default attention mask if not provided From 3893939688cc2658ea151318d3ab90e3483ed1f8 Mon Sep 17 00:00:00 2001 From: mignonjia Date: Fri, 6 Mar 2026 23:46:59 +0000 Subject: [PATCH 212/214] delete wangame files --- docs/wangame/zero_init_fixes.md | 51 - examples/distill/SFWanGame2.1/distill_dmd.sh | 140 --- .../distill/SFWanGame2.1/distill_dmd.slurm | 161 --- examples/distill/SFWanGame2.1/validation.json | 164 --- examples/distill/WanGame2.1/distill_dmd.slurm | 146 --- examples/distill/WanGame2.1/validation.json | 324 ------ .../inference/basic/basic_causal_wangame.py | 49 - examples/inference/basic/basic_wangame.py | 46 - .../inference/basic/basic_wangame_lingbot.py | 46 - examples/train/dfsft_wangame_causal_v3.yaml | 91 -- .../train/finetune_wangame2.1_i2v_1.3B.yaml | 86 -- .../train/self_forcing_wangame_causal_v3.yaml | 122 --- .../causal_wangame_ode_init/ar_diff.slurm | 126 --- .../finetune_ode_init.sh | 99 -- .../finetune_ode_init.slurm | 131 --- .../launch_preprocess_slurm.sh | 20 - .../ode_finetune_worker.slurm | 61 -- .../preprocess_data.sh | 24 - .../preprocess_worker.slurm | 62 -- .../causal_wangame_ode_init/validation.json | 324 ------ .../validation_same.json | 324 ------ .../WanGame2.1_1.3b_i2v/actions/README.md | 147 --- .../WanGame2.1_1.3b_i2v/actions_81/README.md | 147 --- .../WanGame2.1_1.3b_i2v/finetune_i2v.sh | 93 -- .../WanGame2.1_1.3b_i2v/finetune_i2v.slurm | 120 --- .../finetune_wangame.slurm | 182 ---- .../finetune_wangame_freeze_action.slurm | 191 ---- .../preprocess_wangame_data_i2v.sh | 27 - .../scripts/collect_samples_to_shao.py | 118 --- .../scripts/generate_actions.py | 278 ----- .../scripts/generate_validation.py | 271 ----- .../scripts/generate_validation_static_w.py | 71 -- .../scripts/generate_validation_to_shao.py | 87 -- .../WanGame2.1_1.3b_i2v/validation.json | 404 -------- .../validation_random.json | 324 ------ .../validation_random_8.json | 84 -- .../WanGame2.1_1.3b_i2v/validation_zelda.json | 324 ------ .../action/README.md | 18 - .../finetune_i2v.sh | 102 -- .../finetune_i2v.slurm | 120 --- .../generate_actions.py | 193 ---- .../launch_preprocess_slurm.sh | 20 - .../preprocess_wangame_data_i2v.sh | 27 - .../preprocess_worker.slurm | 61 -- .../validation.json | 14 - .../validation_vizdoom.json | 84 -- fastvideo/configs/models/dits/wangamevideo.py | 122 --- fastvideo/models/dits/wangame/__init__.py | 10 - fastvideo/models/dits/wangame/causal_model.py | 856 ---------------- .../dits/wangame/hyworld_action_module.py | 280 ------ fastvideo/models/dits/wangame/model.py | 422 -------- .../models/dits/wangame_lingbot/__init__.py | 5 - .../models/dits/wangame_lingbot/cam_utils.py | 203 ---- .../models/dits/wangame_lingbot/model.py | 451 --------- .../basic/wan/wangame_causal_dmd_pipeline.py | 89 -- .../basic/wan/wangame_i2v_pipeline.py | 104 -- .../wangame/wangame_preprocess_pipeline.py | 303 ------ ...game_preprocess_pipeline_ode_trajectory.py | 497 --------- fastvideo/train/models/wangame/__init__.py | 7 - fastvideo/train/models/wangame/wangame.py | 816 --------------- .../train/models/wangame/wangame_causal.py | 503 --------- .../training/wangame_ar_diffusion_pipeline.py | 527 ---------- .../training/wangame_distillation_pipeline.py | 517 ---------- .../wangame_lingbot_training_pipeline.py | 418 -------- .../training/wangame_ode_causal_pipeline.py | 659 ------------ ...game_self_forcing_distillation_pipeline.py | 952 ------------------ .../training/wangame_training_pipeline.py | 542 ---------- 67 files changed, 14387 deletions(-) delete mode 100644 docs/wangame/zero_init_fixes.md delete mode 100644 examples/distill/SFWanGame2.1/distill_dmd.sh delete mode 100644 examples/distill/SFWanGame2.1/distill_dmd.slurm delete mode 100644 examples/distill/SFWanGame2.1/validation.json delete mode 100644 examples/distill/WanGame2.1/distill_dmd.slurm delete mode 100644 examples/distill/WanGame2.1/validation.json delete mode 100644 examples/inference/basic/basic_causal_wangame.py delete mode 100644 examples/inference/basic/basic_wangame.py delete mode 100644 examples/inference/basic/basic_wangame_lingbot.py delete mode 100644 examples/train/dfsft_wangame_causal_v3.yaml delete mode 100644 examples/train/finetune_wangame2.1_i2v_1.3B.yaml delete mode 100644 examples/train/self_forcing_wangame_causal_v3.yaml delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/ar_diff.slurm delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/ode_finetune_worker.slurm delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/validation.json delete mode 100644 examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/README.md delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.sh delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.slurm delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/preprocess_wangame_data_i2v.sh delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/collect_samples_to_shao.py delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_to_shao.py delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v/validation_zelda.json delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/README.md delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.sh delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.slurm delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/generate_actions.py delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/launch_preprocess_slurm.sh delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_wangame_data_i2v.sh delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation.json delete mode 100644 examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation_vizdoom.json delete mode 100644 fastvideo/configs/models/dits/wangamevideo.py delete mode 100644 fastvideo/models/dits/wangame/__init__.py delete mode 100644 fastvideo/models/dits/wangame/causal_model.py delete mode 100644 fastvideo/models/dits/wangame/hyworld_action_module.py delete mode 100644 fastvideo/models/dits/wangame/model.py delete mode 100644 fastvideo/models/dits/wangame_lingbot/__init__.py delete mode 100644 fastvideo/models/dits/wangame_lingbot/cam_utils.py delete mode 100644 fastvideo/models/dits/wangame_lingbot/model.py delete mode 100644 fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py delete mode 100644 fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py delete mode 100644 fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline.py delete mode 100644 fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py delete mode 100644 fastvideo/train/models/wangame/__init__.py delete mode 100644 fastvideo/train/models/wangame/wangame.py delete mode 100644 fastvideo/train/models/wangame/wangame_causal.py delete mode 100644 fastvideo/training/wangame_ar_diffusion_pipeline.py delete mode 100644 fastvideo/training/wangame_distillation_pipeline.py delete mode 100644 fastvideo/training/wangame_lingbot_training_pipeline.py delete mode 100644 fastvideo/training/wangame_ode_causal_pipeline.py delete mode 100644 fastvideo/training/wangame_self_forcing_distillation_pipeline.py delete mode 100644 fastvideo/training/wangame_training_pipeline.py diff --git a/docs/wangame/zero_init_fixes.md b/docs/wangame/zero_init_fixes.md deleted file mode 100644 index 3a397657f..000000000 --- a/docs/wangame/zero_init_fixes.md +++ /dev/null @@ -1,51 +0,0 @@ -# Zero Initialization Fixes Summary - -## Problem -New parameters (`action_embedder`, `to_out_prope`) were not learning - weights stayed at zero after training. - -## Root Causes & Fixes - -### 1. FSDP Loader Overwriting Model Initialization - -**File:** `fastvideo/models/loader/fsdp_load.py` - -**Problem:** FSDP loader initialized ALL new parameters (not in checkpoint) with zeros, overwriting the model's `__init__` initialization. - -**Fix:** Added `KAIMING_INIT_PATTERNS` to selectively apply proper initialization: - -```python -ALLOWED_NEW_PARAM_PATTERNS = ["gate_compress", "proj_l", "to_out_prope", "action_embedder"] -KAIMING_INIT_PATTERNS = ["fc_in.weight", "lora_A"] # Input projections need non-zero init - -for new_param_name in unused_keys: - use_kaiming = any(pattern in new_param_name for pattern in KAIMING_INIT_PATTERNS) - if use_kaiming: - nn.init.kaiming_uniform_(tensor, a=math.sqrt(5)) # Non-zero for gradient flow - else: - torch.zeros_like(...) # Zero for output projections (residual behavior) -``` - -**Why:** -- Input projections (`fc_in.weight`) need non-zero weights for gradients to flow -- Output projections (`fc_out.weight`) should be zero-initialized for stable residual learning (ControlNet/adapter pattern) - -### 2. Attention Mask Shape Mismatch - -**File:** `fastvideo/models/dits/wangame/hyworld_action_module.py` - -**Problem:** Attention mask had shape `[B, L]` but query tensor had shape `[2*B, L, ...]` (rope + prope concatenated). The prope batch (second half) had no mask coverage → output was zeros. - -**Fix:** - -```python -# Before (wrong): -attention_mask = torch.ones(batch_size, seq_len, ...) # [B, L] - -# After (correct): -attention_mask = torch.ones(batch_size * 2, seq_len, ...) # [2*B, L] -``` - -## Files Modified - -1. `fastvideo/models/loader/fsdp_load.py` - KAIMING_INIT_PATTERNS -2. `fastvideo/models/dits/wangame/hyworld_action_module.py` - attention mask shape diff --git a/examples/distill/SFWanGame2.1/distill_dmd.sh b/examples/distill/SFWanGame2.1/distill_dmd.sh deleted file mode 100644 index c819ef1aa..000000000 --- a/examples/distill/SFWanGame2.1/distill_dmd.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=t2v -#SBATCH --partition=main -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=dmd_t2v_output/t2v_%j.out -#SBATCH --error=dmd_t2v_output/t2v_%j.err -#SBATCH --exclusive - -# Basic Info -export NCCL_P2P_DISABLE=1 -export TORCH_NCCL_ENABLE_MONITORING=0 -# different cache dir for different processes -# export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} -export MASTER_PORT=29503 -export TOKENIZERS_PARALLELISM=false -export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" -export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=online -export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN - -# Configs -NUM_GPUS=64 - -# Model paths for Self-Forcing DMD distillation: -GENERATOR_MODEL_PATH="../WanGame-2.1" -REAL_SCORE_MODEL_PATH="../WanGame-2.1" # Teacher model -FAKE_SCORE_MODEL_PATH="../WanGame-2.1-" # Critic model - -DATA_DIR="../FastvideoWorldModel-MC/preprocessed" -VALIDATION_DATASET_FILE="examples/distill/SFWanGame2.1/validation.json" - -training_args=( - --tracker_project_name wangame_distill_self_forcing_dmd - --output_dir "checkpoints/wangame_distill_self_forcing_dmd" - --wandb_run_name "0202_1010_steps2000_bs_64" - --max_train_steps 500 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 21 - --num_height 352 - --num_width 640 - --enable_gradient_checkpointing_type "full" - --log_visualization - --simulate_generator_forward - --num_frames 81 - --num_frame_per_block 3 # Frame generation block size for self-forcing - --enable_gradient_masking - --gradient_mask_last_n_frames 21 - # --resume_from_checkpoint "checkpoints/wangame_distill_self_forcing_dmd/checkpoint-100" -) - -parallel_args=( - --num_gpus $NUM_GPUS # 64 - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 # 64 - --hsdp_shard_dim $NUM_GPUS -) - -model_args=( - --model_path $GENERATOR_MODEL_PATH # TODO: check if you can remove this in this script - --pretrained_model_name_or_path $GENERATOR_MODEL_PATH - --real_score_model_path $REAL_SCORE_MODEL_PATH - --fake_score_model_path $FAKE_SCORE_MODEL_PATH -) - -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 50 - --validation_sampling_steps "4" - --validation_guidance_scale "6.0" # not used for dmd inference -) - -optimizer_args=( - --learning_rate 6e-6 - --mixed_precision "bf16" - --training_state_checkpointing_steps 50 - --weight_only_checkpointing_steps 50 - --weight_decay 0.01 - --betas '0.0,0.999' - --max_grad_norm 1.0 -) - -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.0 - --dit_precision "fp32" - --flow_shift 5 - --seed 1000 - --use_ema True - --ema_decay 0.99 - --ema_start_step 100 - --init_weights_from_safetensors "checkpoints/wangame_ode_init_64gpu/checkpoint-2000/transformer" -) - -dmd_args=( - --dmd_denoising_steps '1000,750,500,250' - --min_timestep_ratio 0.02 - --max_timestep_ratio 0.98 - --dfake_gen_update_ratio 5 - --real_score_guidance_scale 3.0 - --fake_score_learning_rate 8e-6 - --fake_score_betas '0.0,0.999' - --warp_denoising_step -) - -self_forcing_args=( - --independent_first_frame False # Whether to treat first frame independently - --same_step_across_blocks True # Whether to use same denoising step across all blocks - --last_step_only False # Whether to only use the last denoising step - --context_noise 0 # Amount of noise to add during context caching (0 = no noise) -) - -torchrun \ ---nnodes 1 \ ---master_port $MASTER_PORT \ ---nproc_per_node $NUM_GPUS \ - fastvideo/training/wangame_self_forcing_distillation_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ - "${dmd_args[@]}" \ - "${self_forcing_args[@]}" diff --git a/examples/distill/SFWanGame2.1/distill_dmd.slurm b/examples/distill/SFWanGame2.1/distill_dmd.slurm deleted file mode 100644 index b3ec595d3..000000000 --- a/examples/distill/SFWanGame2.1/distill_dmd.slurm +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wg-sf -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=log/sf_train_output/ode_%j.out -#SBATCH --error=log/sf_train_output/ode_%j.err -#SBATCH --exclusive - -set -e -x - -# Environment Setup -source ~/conda/miniconda/bin/activate -conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv -export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" - -# Basic Info -export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" -export WANDB_MODE="online" -export NCCL_P2P_DISABLE=1 -export MASTER_PORT=29500 -export NODE_RANK=$SLURM_PROCID -nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) -export MASTER_ADDR=${nodes[0]} -export TOKENIZERS_PARALLELISM=false - -echo "MASTER_ADDR: $MASTER_ADDR" -echo "NODE_RANK: $NODE_RANK" - -RUN_NAME=$(date +"%m%d_%H%M") -echo "RUN_NAME: $RUN_NAME" - -# Model paths for Self-Forcing DMD distillation: -# GENERATOR_MODEL_PATH="../wg_models/WanGame-2.1-Student-VizDoom1k-1000steps-Diffusers" -# GENERATOR_MODEL_PATH="../wg_models/SFWanGame-2.1-0223-9000steps" -GENERATOR_MODEL_PATH="../wg_models/SFWanGame-2.1-0224-4k5steps" -REAL_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Teacher model -FAKE_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Critic model - -# DATA_DIR="../traindata_0222_0030/ode_init_mc_with_mouse/preprocessed_wangame" -DATA_DIR="../traindata_0222_0030/ode_init_mc_Xonly_3k/preprocessed" -VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" - -# Training arguments -training_args=( - --tracker_project_name "wangame_sf" - --output_dir "checkpoints/wangame_sf_${RUN_NAME}" - --wandb_run_name "${RUN_NAME}_bs32" - --max_train_steps 3000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 21 - --num_height 352 - --num_width 640 - --enable_gradient_checkpointing_type "full" - --log_visualization - --simulate_generator_forward - --num_frames 81 - --num_frame_per_block 3 # Frame generation block size for self-forcing - --enable_gradient_masking - --gradient_mask_last_n_frames 21 - # --init_weights_from_safetensors $CKPT_SAFETENSOR -) - -# Parallel arguments -parallel_args=( - --num_gpus 32 - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim 32 -) - -model_args=( - --model_path $GENERATOR_MODEL_PATH # TODO: check if you can remove this in this script - --pretrained_model_name_or_path $GENERATOR_MODEL_PATH - --real_score_model_path $REAL_SCORE_MODEL_PATH - --fake_score_model_path $FAKE_SCORE_MODEL_PATH -) - -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -# Validation arguments -validation_args=( - --log_validation - --log_visualization - --visualization-steps 100 - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 100 - --validation_sampling_steps "4" - --validation_guidance_scale "6.0" -) - -# Optimizer arguments -optimizer_args=( - --learning_rate 6e-6 - --mixed_precision "bf16" - --weight_only_checkpointing_steps 100 - --training_state_checkpointing_steps 100 - --weight_decay 0.01 - --betas '0.0,0.999' - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.0 - --dit_precision "fp32" - --flow_shift 5 - --seed 1000 - --use_ema True - --ema_decay 0.99 - --ema_start_step 200 -) - -dmd_args=( - --dmd_denoising_steps '1000,750,500,250' - --min_timestep_ratio 0.02 - --max_timestep_ratio 0.98 - --dfake_gen_update_ratio 5 - --real_score_guidance_scale 3.0 - --fake_score_learning_rate 8e-6 - --fake_score_betas '0.0,0.999' - --warp_denoising_step -) - -self_forcing_args=( - --independent_first_frame False # Whether to treat first frame independently - --same_step_across_blocks True # Whether to use same denoising step across all blocks - --last_step_only False # Whether to only use the last denoising step - --context_noise 0 # Amount of noise to add during context caching (0 = no noise) -) - -mkdir -p log/sf_train_output - -srun torchrun \ ---nnodes $SLURM_JOB_NUM_NODES \ ---nproc_per_node 8 \ ---node_rank $SLURM_PROCID \ ---rdzv_backend=c10d \ ---rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ - fastvideo/training/wangame_self_forcing_distillation_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ - "${dmd_args[@]}" \ - "${self_forcing_args[@]}" diff --git a/examples/distill/SFWanGame2.1/validation.json b/examples/distill/SFWanGame2.1/validation.json deleted file mode 100644 index d97352dd7..000000000 --- a/examples/distill/SFWanGame2.1/validation.json +++ /dev/null @@ -1,164 +0,0 @@ -{ - "data": [ - { - "caption": "Hold [W] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000000_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [S] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000001_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [A] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000002_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [D] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000003_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [W] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000000_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [S] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000001_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [A] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000002_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [D] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000003_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [W] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000000_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [S] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000001_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [A] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000002_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [D] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000003_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [W] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000000_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [S] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000001_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [A] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000002_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "Hold [D] + Static", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/vizdoom/gen/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/FastVideo_kaiqin/examples/training/finetune/MatrixGame2.0/action/000003_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - } - ] -} \ No newline at end of file diff --git a/examples/distill/WanGame2.1/distill_dmd.slurm b/examples/distill/WanGame2.1/distill_dmd.slurm deleted file mode 100644 index 6e4a800e0..000000000 --- a/examples/distill/WanGame2.1/distill_dmd.slurm +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wg-dmd -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=log/dmd_train_output/dmd_%j.out -#SBATCH --error=log/dmd_train_output/dmd_%j.err -#SBATCH --exclusive - -set -e -x - -# Environment Setup -source ~/conda/miniconda/bin/activate -conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv -export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" - -# Basic Info -export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" -export WANDB_MODE="online" -export NCCL_P2P_DISABLE=1 -export MASTER_PORT=29500 -export NODE_RANK=$SLURM_PROCID -nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) -export MASTER_ADDR=${nodes[0]} -export TOKENIZERS_PARALLELISM=false - -echo "MASTER_ADDR: $MASTER_ADDR" -echo "NODE_RANK: $NODE_RANK" - -RUN_NAME=$(date +"%m%d_%H%M") -echo "RUN_NAME: $RUN_NAME" - -GENERATOR_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" -REAL_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Teacher model -FAKE_SCORE_MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" # Critic model - -DATA_DIR="../traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed" -VALIDATION_DATASET_FILE="examples/distill/WanGame2.1/validation.json" - -# Training arguments -training_args=( - --tracker_project_name "wangame_dmd" - --output_dir "checkpoints/wangame_dmd_${RUN_NAME}" - --wandb_run_name "${RUN_NAME}_dmd" - --max_train_steps 3000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 20 - --num_height 352 - --num_width 640 - --num_frames 77 - --enable_gradient_checkpointing_type "full" - --training_state_checkpointing_steps 500 - --weight_only_checkpointing_steps 500 -) - -# Parallel arguments -parallel_args=( - --num_gpus 32 - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim 32 -) - -model_args=( - --model_path $GENERATOR_MODEL_PATH - --pretrained_model_name_or_path $GENERATOR_MODEL_PATH - --real_score_model_path $REAL_SCORE_MODEL_PATH - --fake_score_model_path $FAKE_SCORE_MODEL_PATH -) - -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -# Validation arguments -validation_args=( - --log_validation - --log_visualization - --visualization-steps 200 - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 200 - --validation_sampling_steps "4" - --validation_guidance_scale "6.0" -) - -# Optimizer arguments -optimizer_args=( - --learning_rate 2e-6 - --mixed_precision "bf16" - --weight_decay 0.01 - --betas '0.0,0.999' - --max_grad_norm 1.0 - --fake_score_learning_rate 8e-6 - --fake_score_betas '0.0,0.999' -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.0 - --dit_precision "fp32" - --flow_shift 5 - --seed 1000 - --use_ema True - --ema_decay 0.99 - --ema_start_step 200 -) - -# DMD-specific arguments -dmd_args=( - --dmd_denoising_steps '1000,750,500,250' - --min_timestep_ratio 0.02 - --max_timestep_ratio 0.98 - --dfake_gen_update_ratio 5 - --real_score_guidance_scale 3.0 - --fake_score_learning_rate 8e-6 - --fake_score_betas '0.0,0.999' - --warp_denoising_step -) - -mkdir -p log/dmd_train_output - -srun torchrun \ ---nnodes $SLURM_JOB_NUM_NODES \ ---nproc_per_node 8 \ ---node_rank $SLURM_PROCID \ ---rdzv_backend=c10d \ ---rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ - fastvideo/training/wangame_distillation_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ - "${dmd_args[@]}" diff --git a/examples/distill/WanGame2.1/validation.json b/examples/distill/WanGame2.1/validation.json deleted file mode 100644 index 2012d50fe..000000000 --- a/examples/distill/WanGame2.1/validation.json +++ /dev/null @@ -1,324 +0,0 @@ -{ - "data": [ - { - "caption": "51", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000051.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000051_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "229", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000229.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000229_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "250", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000250.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000250_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "380", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000380.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000380_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "382", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000382.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000382_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "387", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000387.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000387_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "418", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000418.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000418_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "505", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000505.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000505_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "515", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000515.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000515_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "534", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000534.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000534_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "599", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000599.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000599_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "613", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000613.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000613_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "745", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000745.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000745_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "861", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000861.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000861_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "940", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000940.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000940_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "946", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000946.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000946_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "996", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000996.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000996_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1011", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001011.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001011_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1037", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001037.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001037_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1057", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001057.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001057_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1195", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001195.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001195_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1236", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001236.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001236_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1276", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001276.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001276_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1368", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001368.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001368_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1403", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001403.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001403_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1417", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001417.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001417_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1481", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001481.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001481_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1489", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001489.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001489_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1618", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001618.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001618_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1779", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001779.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001779_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1867", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001867.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001867_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1949", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001949.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001949_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - } - ] -} \ No newline at end of file diff --git a/examples/inference/basic/basic_causal_wangame.py b/examples/inference/basic/basic_causal_wangame.py deleted file mode 100644 index f18056945..000000000 --- a/examples/inference/basic/basic_causal_wangame.py +++ /dev/null @@ -1,49 +0,0 @@ -from fastvideo import VideoGenerator -from fastvideo.configs.pipelines import SelfForcingWanGameI2V480PConfig -from fastvideo.models.dits.matrixgame.utils import create_action_presets -import torch - -BASE_MODEL_PATH = "Wan2.1-Fun-1.3B-InP-Diffusers" -WEIGHTS_PATH = "checkpoints/wangame_ode_init/checkpoint-1200/transformer" - -OUTPUT_PATH = "video_samples_wangame" -IMAGE_PATH = "https://raw.githubusercontent.com/SkyworkAI/Matrix-Game/main/Matrix-Game-2/demo_images/universal/0000.png" - - -def main(): - generator = VideoGenerator.from_pretrained( - BASE_MODEL_PATH, - pipeline_config=SelfForcingWanGameI2V480PConfig(), - num_gpus=1, - use_fsdp_inference=False, - dit_cpu_offload=False, - vae_cpu_offload=False, - text_encoder_cpu_offload=True, - pin_cpu_memory=True, - override_pipeline_cls_name="WanGameCausalDMDPipeline", - override_transformer_cls_name="CausalWanGameActionTransformer3DModel", - init_weights_from_safetensors=WEIGHTS_PATH, - ) - - num_frames = 81 - actions = create_action_presets(num_frames, keyboard_dim=4) - actions["keyboard"] = torch.tensor([[1.0, 0.0, 0.0, 0.0]] * num_frames) - actions["mouse"] = torch.tensor([[0.0, 0.0]] * num_frames) - - generator.generate_video( - prompt="", - image_path=IMAGE_PATH, - mouse_cond=actions["mouse"].unsqueeze(0), - keyboard_cond=actions["keyboard"].unsqueeze(0), - num_frames=num_frames, - height=352, - width=640, - num_inference_steps=40, - guidance_scale=1.0, - output_path=OUTPUT_PATH, - save_video=True, - ) - - -if __name__ == "__main__": - main() diff --git a/examples/inference/basic/basic_wangame.py b/examples/inference/basic/basic_wangame.py deleted file mode 100644 index d5154f824..000000000 --- a/examples/inference/basic/basic_wangame.py +++ /dev/null @@ -1,46 +0,0 @@ -from fastvideo import VideoGenerator -from fastvideo.configs.pipelines import WanGameI2V480PConfig -from fastvideo.models.dits.matrixgame.utils import create_action_presets - -BASE_MODEL_PATH = "Wan2.1-Fun-1.3B-InP-Diffusers" -# WEIGHTS_PATH = "wangame_1.3b_overfit/checkpoint-10000/transformer/diffusion_pytorch_model.safetensors" - -OUTPUT_PATH = "video_samples_wangame" -IMAGE_PATH = "/mnt/fast-disks/hao_lab/kaiqin/traindata_0209_1500/ode_init_mc/images/000000.jpg" - - -def main(): - generator = VideoGenerator.from_pretrained( - BASE_MODEL_PATH, - pipeline_config=WanGameI2V480PConfig(), - num_gpus=1, - use_fsdp_inference=False, - dit_cpu_offload=False, - vae_cpu_offload=False, - text_encoder_cpu_offload=True, - pin_cpu_memory=True, - override_pipeline_cls_name="WanGameActionImageToVideoPipeline", - override_transformer_cls_name="WanGameActionTransformer3DModel", - # init_weights_from_safetensors=WEIGHTS_PATH, - ) - - num_frames = 77 - actions = create_action_presets(num_frames, keyboard_dim=4) - - generator.generate_video( - prompt="", - image_path=IMAGE_PATH, - mouse_cond=actions["mouse"].unsqueeze(0), - keyboard_cond=actions["keyboard"].unsqueeze(0), - num_frames=num_frames, - height=352, - width=640, - num_inference_steps=40, - guidance_scale=1.0, - output_path=OUTPUT_PATH, - save_video=True, - ) - - -if __name__ == "__main__": - main() diff --git a/examples/inference/basic/basic_wangame_lingbot.py b/examples/inference/basic/basic_wangame_lingbot.py deleted file mode 100644 index b30d0f562..000000000 --- a/examples/inference/basic/basic_wangame_lingbot.py +++ /dev/null @@ -1,46 +0,0 @@ -from fastvideo import VideoGenerator -from fastvideo.configs.pipelines import WanLingBotI2V480PConfig -from fastvideo.models.dits.matrixgame.utils import create_action_presets - -BASE_MODEL_PATH = "weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -WEIGHTS_PATH = "wangame_lingbot_test/checkpoint-100/transformer/diffusion_pytorch_model.safetensors" - -OUTPUT_PATH = "video_samples_wangame_lingbot" -IMAGE_PATH = "https://raw.githubusercontent.com/SkyworkAI/Matrix-Game/main/Matrix-Game-2/demo_images/universal/0000.png" - - -def main(): - generator = VideoGenerator.from_pretrained( - BASE_MODEL_PATH, - pipeline_config=WanLingBotI2V480PConfig(), - num_gpus=1, - use_fsdp_inference=False, - dit_cpu_offload=False, - vae_cpu_offload=False, - text_encoder_cpu_offload=True, - pin_cpu_memory=True, - override_pipeline_cls_name="WanLingBotImageToVideoPipeline", - override_transformer_cls_name="WanLingBotTransformer3DModel", - init_weights_from_safetensors=WEIGHTS_PATH, - ) - - num_frames = 77 - actions = create_action_presets(num_frames, keyboard_dim=4) - - generator.generate_video( - prompt="", - image_path=IMAGE_PATH, - mouse_cond=actions["mouse"].unsqueeze(0), - keyboard_cond=actions["keyboard"].unsqueeze(0), - num_frames=num_frames, - height=352, - width=640, - num_inference_steps=40, - guidance_scale=1.0, - output_path=OUTPUT_PATH, - save_video=True, - ) - - -if __name__ == "__main__": - main() diff --git a/examples/train/dfsft_wangame_causal_v3.yaml b/examples/train/dfsft_wangame_causal_v3.yaml deleted file mode 100644 index e2e77db67..000000000 --- a/examples/train/dfsft_wangame_causal_v3.yaml +++ /dev/null @@ -1,91 +0,0 @@ -# V3 config: WanGame causal Diffusion-Forcing SFT (DFSFT). -# -# Uses _target_-based instantiation — each model role is an independent -# class instance; the method class is resolved directly from the YAML. - -models: - student: - _target_: fastvideo.train.models.wangame.WanGameCausalModel - init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - -method: - _target_: fastvideo.train.methods.fine_tuning.dfsft.DiffusionForcingSFTMethod - attn_kind: dense - # use_ema: true - chunk_size: 3 - min_timestep_ratio: 0.02 - max_timestep_ratio: 0.98 - -training: - distributed: - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - data: - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 - dataloader_num_workers: 4 - train_batch_size: 1 - training_cfg_rate: 0.0 - seed: 1000 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - optimizer: - learning_rate: 1.0e-5 - betas: [0.9, 0.999] - weight_decay: 1.0e-4 - lr_scheduler: constant - lr_warmup_steps: 0 - - loop: - max_train_steps: 20000 - gradient_accumulation_steps: 1 - - checkpoint: - output_dir: outputs/wangame_dfsft_causal_v3 - training_state_checkpointing_steps: 1000 - checkpoints_total_limit: 2 - - tracker: - project_name: distillation_wangame_r - run_name: wangame_dfsft_causal_v3 - - model: - enable_gradient_checkpointing_type: full - -callbacks: - grad_clip: - _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback - max_grad_norm: 1.0 - # ema: - # _target_: fastvideo.train.callbacks.ema.EMACallback - # beta: 0.9999 - validation: - _target_: fastvideo.train.callbacks.validation.ValidationCallback - pipeline_target: fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline.WanGameCausalDMDPipeline - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [40] - rollout_mode: streaming - sampler_kind: ode - scheduler_target: fastvideo.models.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler - guidance_scale: 1.0 - num_frames: 69 - -pipeline: - flow_shift: 3 - sampler_kind: ode diff --git a/examples/train/finetune_wangame2.1_i2v_1.3B.yaml b/examples/train/finetune_wangame2.1_i2v_1.3B.yaml deleted file mode 100644 index 4edc3f10b..000000000 --- a/examples/train/finetune_wangame2.1_i2v_1.3B.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# V3 config: WanGame 2.1 I2V 1.3B finetune (dense attention). -# -# Uses _target_-based instantiation — each model role is an independent -# class instance; the method class is resolved directly from the YAML. - -models: - student: - _target_: fastvideo.train.models.wangame.WanGameModel - init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - -method: - _target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod - attn_kind: dense - # use_ema: true - -training: - distributed: - num_gpus: 8 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 8 - hsdp_shard_dim: 1 - - data: - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1 - dataloader_num_workers: 4 - train_batch_size: 1 - training_cfg_rate: 0.0 - seed: 1000 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - optimizer: - learning_rate: 1.0e-5 - betas: [0.9, 0.999] - weight_decay: 1.0e-4 - lr_scheduler: constant - lr_warmup_steps: 0 - - loop: - max_train_steps: 20000 - gradient_accumulation_steps: 1 - - checkpoint: - output_dir: outputs/wangame_finetune_v3 - training_state_checkpointing_steps: 1000 - weight_only_checkpointing_steps: 1000 - checkpoints_total_limit: 2 - - tracker: - project_name: distillation_wangame_r - run_name: wangame_finetune_v3 - - model: - enable_gradient_checkpointing_type: full - -callbacks: - grad_clip: - _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback - max_grad_norm: 1.0 - # ema: - # _target_: fastvideo.train.callbacks.ema.EMACallback - # beta: 0.9999 - validation: - _target_: fastvideo.train.callbacks.validation.ValidationCallback - pipeline_target: fastvideo.pipelines.basic.wan.wangame_i2v_pipeline.WanGameActionImageToVideoPipeline - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [40] - sampler_kind: ode - guidance_scale: 1.0 - -pipeline: - flow_shift: 3 - sampler_kind: ode diff --git a/examples/train/self_forcing_wangame_causal_v3.yaml b/examples/train/self_forcing_wangame_causal_v3.yaml deleted file mode 100644 index c72a4dc8c..000000000 --- a/examples/train/self_forcing_wangame_causal_v3.yaml +++ /dev/null @@ -1,122 +0,0 @@ -# V3 config: WanGame causal Self-Forcing distillation (40-step teacher -> 4-step student). -# -# Uses _target_-based instantiation — each model role is an independent -# class instance; the method class is resolved directly from the YAML. -# -# To warmstart from a DCP checkpoint, first convert it to diffusers format -# using `dcp_to_diffusers`, then point `init_from` at the converted directory. - -models: - student: - _target_: fastvideo.train.models.wangame.WanGameCausalModel - # TODO: update to converted diffusers path - init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - teacher: - _target_: fastvideo.train.models.wangame.WanGameCausalModel - # TODO: update to converted diffusers path - init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: false - disable_custom_init_weights: true - critic: - _target_: fastvideo.train.models.wangame.WanGameCausalModel - # TODO: update to converted diffusers path - init_from: /mnt/weka/home/hao.zhang/kaiqin/wg_models/WanGame-2.1-0223-9000steps - trainable: true - disable_custom_init_weights: true - -method: - _target_: fastvideo.train.methods.distribution_matching.self_forcing.SelfForcingMethod - # use_ema: true - rollout_mode: simulate - generator_update_interval: 5 - real_score_guidance_scale: 3.5 - - # Critic / fake-score optimizer - fake_score_learning_rate: 8.0e-6 - fake_score_betas: [0.0, 0.999] - fake_score_lr_scheduler: constant - - warp_denoising_step: true - dmd_denoising_steps: [1000,750,500,250] - - chunk_size: 3 - student_sample_type: sde - same_step_across_blocks: false - last_step_only: false - context_noise: 0.0 - enable_gradient_in_rollout: true - start_gradient_frame: 0 - - cfg_uncond: - on_missing: error - action: keep - image: keep - text: keep - -training: - distributed: - num_gpus: 32 - sp_size: 1 - tp_size: 1 - hsdp_replicate_dim: 1 - hsdp_shard_dim: 32 - - data: - data_path: >- - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1, - /mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1 - dataloader_num_workers: 4 - train_batch_size: 1 - training_cfg_rate: 0.0 - seed: 1000 - num_latent_t: 20 - num_height: 352 - num_width: 640 - num_frames: 77 - - optimizer: - learning_rate: 2.0e-6 - betas: [0.0, 0.999] - weight_decay: 0.01 - lr_scheduler: constant - lr_warmup_steps: 0 - - loop: - max_train_steps: 4000 - gradient_accumulation_steps: 1 - - checkpoint: - output_dir: outputs/wangame_self_forcing_4steps_v3 - training_state_checkpointing_steps: 1000 - checkpoints_total_limit: 3 - - tracker: - project_name: distillation_wangame_r - run_name: wangame_self_forcing_4steps_v3 - - model: - enable_gradient_checkpointing_type: null - -callbacks: - grad_clip: - _target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback - max_grad_norm: 1.0 - # ema: - # _target_: fastvideo.train.callbacks.ema.EMACallback - # beta: 0.9999 - validation: - _target_: fastvideo.train.callbacks.validation.ValidationCallback - pipeline_target: fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline.WanGameCausalDMDPipeline - dataset_file: examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json - every_steps: 100 - sampling_steps: [4] - sampler_kind: sde - rollout_mode: streaming - guidance_scale: 1.0 - num_frames: 69 - dmd_denoising_steps: [1000, 750, 500, 250] - -pipeline: - flow_shift: 3 - sampler_kind: sde diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/ar_diff.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/ar_diff.slurm deleted file mode 100644 index 1fb83ad48..000000000 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/ar_diff.slurm +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wg-ar-diff -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=log/ar_diff_output/ar_diff_%j.out -#SBATCH --error=log/ar_diff_output/ar_diff_%j.err -#SBATCH --exclusive - -# Environment Setup -source ~/conda/miniconda/bin/activate -conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv -export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" - -export NCCL_P2P_DISABLE=1 -export TORCH_NCCL_ENABLE_MONITORING=0 -export MASTER_PORT=29503 -export TOKENIZERS_PARALLELISM=false -export WANDB_MODE=online -export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN - -# Basic Info -export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" -export WANDB_MODE="online" -export NCCL_P2P_DISABLE=1 -export MASTER_PORT=29500 -export NODE_RANK=$SLURM_PROCID -nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) -export MASTER_ADDR=${nodes[0]} -export TOKENIZERS_PARALLELISM=false - -echo "MASTER_ADDR: $MASTER_ADDR" -echo "NODE_RANK: $NODE_RANK" - -RUN_NAME=$(date +"%m%d_%H%M") -echo "RUN_NAME: $RUN_NAME" - -MODEL_PATH="../wg_models/WanGame-2.1-0223-9000steps" - -DATA_DIR="../traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed" -VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json" - -training_args=( - --tracker_project_name wangame_ar_diffusion - --output_dir "checkpoints/wangame_ar_diffusion_${RUN_NAME}" - --wandb_run_name "${RUN_NAME}_df_bs32" - --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" - --max_train_steps 5000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 15 - --num_height 352 - --num_width 640 - --enable_gradient_checkpointing_type "full" - --num_frames 57 - --num_frame_per_block 3 -) - -parallel_args=( - --num_gpus 32 - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim 32 -) - -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 200 - --validation_sampling_steps "50" - --validation_guidance_scale "1.0" -) - -optimizer_args=( - --learning_rate 2e-5 - --mixed_precision "bf16" - --training_state_checkpointing_steps 500 - --weight_only_checkpointing_steps 500 - --weight_decay 0.01 - --betas '0.9,0.999' - --max_grad_norm 1.0 - --lr_scheduler cosine - --lr_warmup_steps 100 -) - -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.0 - --dit_precision "fp32" - --flow_shift 8 - --seed 42 -) - -mkdir -p log/ar_diff_output - -srun torchrun \ ---nnodes $SLURM_JOB_NUM_NODES \ ---nproc_per_node 8 \ ---node_rank $SLURM_PROCID \ ---rdzv_backend=c10d \ ---rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ - fastvideo/training/wangame_ar_diffusion_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh deleted file mode 100644 index 9a351607b..000000000 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash - -export PYTHONPATH="/mnt/fast-disks/hao_lab/kaiqin/FastVideo_wangame:$PYTHONPATH" -export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" -export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=online -export TOKENIZERS_PARALLELISM=false - -MODEL_PATH="Wan2.1-Fun-1.3B-InP-Diffusers" -DATA_DIR="../traindata_0209_1500/ode_init_mc/preprocessed/combined_parquet_dataset/worker_0" -VALIDATION_DATASET_FILE="$(dirname "$0")/validation.json" -NUM_GPUS=1 -export CUDA_VISIBLE_DEVICES=4,5,6,7 -# IP=[MASTER NODE IP] - -# Training arguments -training_args=( - --tracker_project_name "wangame_ode_init" - --output_dir "checkpoints/wangame_ode_init" - --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" - --wandb_run_name "0213_2100_test" - --max_train_steps 1 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 21 - --num_height 352 - --num_width 640 - --num_frames 81 - --warp_denoising_step - --enable_gradient_checkpointing_type "full" -) - -# Parallel arguments -parallel_args=( - --num_gpus $NUM_GPUS - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim $NUM_GPUS -) - -# Model arguments -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -# Dataset arguments -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 1 -) - -# Validation arguments -validation_args=( - --log_validation - --log-visualization - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 100 - --visualization-steps 100 - --validation_sampling_steps "50" - --validation_guidance_scale "6.0" -) - -# Optimizer arguments -optimizer_args=( - --learning_rate 6e-6 - --mixed_precision "bf16" - --weight_only_checkpointing_steps 200 - --training_state_checkpointing_steps 200 - --weight_decay 1e-4 - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.1 - --multi_phased_distill_schedule "4000-1" - --not_apply_cfg_solver - --dit_precision "fp32" - --num_euler_timesteps 50 - --ema_start_step 0 -) - -# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t -torchrun \ - --nnodes 1 \ - --nproc_per_node $NUM_GPUS \ - fastvideo/training/wangame_ode_causal_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm deleted file mode 100644 index cd02796fe..000000000 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/finetune_ode_init.slurm +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wg-ode -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=ode_train_output/ode_%j.out -#SBATCH --error=ode_train_output/ode_%j.err -#SBATCH --exclusive - -set -e -x - -# Environment Setup -source ~/conda/miniconda/bin/activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv -export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" - -# Basic Info -export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" -export WANDB_MODE=online -export NCCL_P2P_DISABLE=1 -export MASTER_PORT=29500 -export NODE_RANK=$SLURM_PROCID -nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) -export MASTER_ADDR=${nodes[0]} -export TOKENIZERS_PARALLELISM=false - -echo "MASTER_ADDR: $MASTER_ADDR" -echo "NODE_RANK: $NODE_RANK" - -RUN_NAME=$(date +"%m%d_%H%M") -echo "RUN_NAME: $RUN_NAME" - -# Configs -MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" -# DATA_DIR="../traindata_0222_0030/ode_init_mc_Xonly_3k/preprocessed" -DATA_DIR="../traindata_0222_0030/ode_init_mc_random/preprocessed_wangame" -# VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation.json" -VALIDATION_DATASET_FILE="examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json" -CKPT_SAFETENSOR="/mnt/weka/home/hao.zhang/mhuo/FastVideo/wangame_1.3b_1action_rand_from_scratch/checkpoint-9000/transformer/diffusion_pytorch_model.safetensors" - -# Training arguments -training_args=( - --tracker_project_name "wangame_ode_init" - --output_dir "checkpoints/wangame_ode_init_${RUN_NAME}" - --override_transformer_cls_name "CausalWanGameActionTransformer3DModel" - --wandb_run_name "${RUN_NAME}_bs64_random" - --max_train_steps 5000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 2 - --num_latent_t 21 - --num_height 352 - --num_width 640 - --num_frames 81 - --warp_denoising_step - --enable_gradient_checkpointing_type "full" - --init_weights_from_safetensors $CKPT_SAFETENSOR -) - -# Parallel arguments -parallel_args=( - --num_gpus 32 - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim 32 -) - -# Model arguments -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -# Dataset arguments -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 4 -) - -# Validation arguments -validation_args=( - --log_validation - --log-visualization - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 100 - --visualization-steps 100 - --validation_sampling_steps "50" - --validation_guidance_scale "6.0" -) - -# Optimizer arguments -optimizer_args=( - --learning_rate 6e-6 - --mixed_precision "bf16" - --weight_only_checkpointing_steps 500 - --training_state_checkpointing_steps 500 - --weight_decay 1e-4 - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.1 - --multi_phased_distill_schedule "4000-1" - --not_apply_cfg_solver - --dit_precision "fp32" - --num_euler_timesteps 50 -) - -mkdir -p ode_train_output - -srun torchrun \ ---nnodes $SLURM_JOB_NUM_NODES \ ---nproc_per_node 8 \ ---node_rank $SLURM_PROCID \ ---rdzv_backend=c10d \ ---rdzv_endpoint="$MASTER_ADDR:$MASTER_PORT" \ - fastvideo/training/wangame_ode_causal_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh deleted file mode 100644 index a07f03760..000000000 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/launch_preprocess_slurm.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Create output directory if it doesn't exist -mkdir -p preprocess_output - -# Launch 8 jobs, one for each node (Total 64 GPUs) -# Each node processes 8 consecutive files (64 total files / 8 nodes = 8 files per node) -for node_id in {0..3}; do - # Calculate the starting file number for this node - start_file=$((node_id * 8)) - - echo "Launching node $node_id with files merge_${start_file}.txt to merge_$((start_file + 7)).txt" - - sbatch --job-name=wg-pre-${node_id} \ - --output=preprocess_output/wg-node-${node_id}.out \ - --error=preprocess_output/wg-node-${node_id}.err \ - $(pwd)/FastVideo/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm $start_file $node_id -done - -echo "All 4 nodes (32 GPUs) launched successfully!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/ode_finetune_worker.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/ode_finetune_worker.slurm deleted file mode 100644 index 6311b44fa..000000000 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/ode_finetune_worker.slurm +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -#SBATCH --partition=main -#SBATCH --qos=hao -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=8 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=16 -#SBATCH --mem=960G -#SBATCH --exclusive -#SBATCH --time=72:00:00 - -# conda init -source ~/conda/miniconda/bin/activate -conda activate fastvideo_kaiqin - -# Accept parameters from launch script -START_FILE=${1:-0} # Starting file number for this node -NODE_ID=${2:-0} # Node identifier (0-7) - -MODEL_PATH="../Matrix-Game-2.0-Base-Diffusers" -OUTPUT_BASE="../FastvideoWorldModel-MC/preprocessed" - -# Port range calculation -base_port=$((29700 + NODE_ID * 100)) # Using a different port range to avoid collision with other tasks -gpu_ids=(0 1 2 3 4 5 6 7) - -for i in {1..8}; do - port=$((base_port + i)) - gpu=${gpu_ids[((i-1))]} - file_num=$((START_FILE + i - 1)) - - DATA_MERGE_PATH="../FastvideoWorldModel-MC/gen/merge_${file_num}.txt" - OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" - - # CPU binding - start_cpu=$(( (i-1)*2 )) - end_cpu=$(( start_cpu+1 )) - - echo "Starting GPU $gpu processing file merge_${file_num}.txt on port $port" - - CUDA_VISIBLE_DEVICES=$gpu taskset -c ${start_cpu}-${end_cpu} torchrun --nnodes=1 --nproc_per_node=1 --master_port $port \ - FastVideo/fastvideo/pipelines/preprocess/v1_preprocess.py \ - --model_path $MODEL_PATH \ - --data_merge_path $DATA_MERGE_PATH \ - --preprocess_video_batch_size 1 \ - --seed 42 \ - --max_height 352 \ - --max_width 640 \ - --num_frames 81 \ - --flow_shift 5.0 \ - --dataloader_num_workers 0 \ - --output_dir=$OUTPUT_DIR \ - --train_fps 25 \ - --samples_per_file 8 \ - --flush_frequency 8 \ - --video_length_tolerance_range 5 \ - --preprocess_task "matrixgame_ode_trajectory" & -done - -wait -echo "Node $NODE_ID ODE preprocessing blocks completed!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh deleted file mode 100644 index 172de86bf..000000000 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_data.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -export PYTHONPATH="/mnt/fast-disks/hao_lab/kaiqin/FastVideo_wangame:$PYTHONPATH" - -GPU_NUM=1 # 2,4,8 -MODEL_PATH="./Wan2.1-Fun-1.3B-InP-Diffusers" -DATA_MERGE_PATH="../traindata_0209_1500/ode_init_mc/merge.txt" -OUTPUT_DIR="../traindata_0209_1500/ode_init_mc/preprocessed" - -torchrun --nproc_per_node=$GPU_NUM \ - fastvideo/pipelines/preprocess/v1_preprocess.py \ - --model_path $MODEL_PATH \ - --data_merge_path $DATA_MERGE_PATH \ - --preprocess_video_batch_size 1 \ - --seed 42 \ - --max_height 352 \ - --max_width 640 \ - --num_frames 81 \ - --dataloader_num_workers 0 \ - --output_dir=$OUTPUT_DIR \ - --samples_per_file 8 \ - --train_fps 25 \ - --flush_frequency 8 \ - --preprocess_task wangame_ode_trajectory & diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm b/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm deleted file mode 100644 index 79a57304e..000000000 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/preprocess_worker.slurm +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -#SBATCH --partition=main -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=8 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=16 -#SBATCH --mem=960G -#SBATCH --exclusive -#SBATCH --time=72:00:00 - -# conda init -source ~/conda/miniconda/bin/activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv -export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" - -# Accept parameters from launch script -START_FILE=${1:-1} # Starting file number for this node -NODE_ID=${2:-0} # Node identifier (0-7) - -MODEL_PATH="./Wan2.1-Fun-1.3B-InP-Diffusers" -# OUTPUT_BASE="traindata_0222_0030/ode_init_mc_Xonly_3k/preprocessed" -OUTPUT_BASE="traindata_0222_0030/ode_init_mc_same/preprocessed" - -# Port range calculation -base_port=$((29500 + NODE_ID * 100)) -gpu_ids=(0 1 2 3 4 5 6 7) - -for i in {1..8}; do - port=$((base_port + i)) - gpu=${gpu_ids[((i-1))]} - file_num=$((START_FILE + i - 1)) - - # DATA_MERGE_PATH="traindata_0222_0030/ode_init_mc_Xonly_3k/merge_${file_num}.txt" - DATA_MERGE_PATH="traindata_0222_0030/ode_init_mc_same/merge_${file_num}.txt" - OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" - echo "DATA_MERGE_PATH: $DATA_MERGE_PATH" - echo "OUTPUT_DIR: $OUTPUT_DIR" - - # CPU binding (optional, kept from syn.slurm logic) - start_cpu=$(( (i-1)*2 )) - end_cpu=$(( start_cpu+1 )) - - echo "Starting GPU $gpu processing file merge_${file_num}.txt on port $port" - - CUDA_VISIBLE_DEVICES=$gpu taskset -c ${start_cpu}-${end_cpu} torchrun --nnodes=1 --nproc_per_node=1 --master_port $port \ - FastVideo/fastvideo/pipelines/preprocess/v1_preprocess.py \ - --model_path $MODEL_PATH \ - --data_merge_path $DATA_MERGE_PATH \ - --preprocess_video_batch_size 1 \ - --seed 42 \ - --max_height 352 \ - --max_width 640 \ - --num_frames 81 \ - --dataloader_num_workers 0 \ - --output_dir=$OUTPUT_DIR \ - --samples_per_file 8 \ - --train_fps 25 \ - --flush_frequency 8 \ - --preprocess_task wangame_ode_trajectory & -done - -wait -echo "Node $NODE_ID processing blocks completed!" diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json deleted file mode 100644 index bdd9b6e45..000000000 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/validation.json +++ /dev/null @@ -1,324 +0,0 @@ -{ - "data": [ - { - "caption": "51", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000051.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000051_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "229", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000229.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000229_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "250", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000250.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000250_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "380", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000380.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000380_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "382", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000382.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000382_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "387", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000387.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000387_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "418", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000418.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000418_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "505", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000505.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000505_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "515", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000515.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000515_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "534", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000534.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000534_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "599", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000599.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000599_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "613", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000613.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000613_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "745", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000745.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000745_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "861", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000861.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000861_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "940", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000940.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000940_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "946", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000946.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000946_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "996", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000996.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/000996_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1011", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001011.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001011_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1037", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001037.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001037_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1057", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001057.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001057_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1195", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001195.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001195_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1236", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001236.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001236_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1276", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001276.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001276_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1368", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001368.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001368_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1403", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001403.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001403_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1417", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001417.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001417_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1481", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001481.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001481_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1489", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001489.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001489_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1618", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001618.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001618_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1779", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001779.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001779_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1867", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001867.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001867_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1949", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001949.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_Xonly_3k/images/001949_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - } - ] -} \ No newline at end of file diff --git a/examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json b/examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json deleted file mode 100644 index 612000782..000000000 --- a/examples/training/consistency_finetune/causal_wangame_ode_init/validation_same.json +++ /dev/null @@ -1,324 +0,0 @@ -{ - "data": [ - { - "caption": "51", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000051.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000051_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "229", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000229.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000229_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "250", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000250.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000250_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "380", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000380.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000380_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "382", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000382.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000382_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "387", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000387.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000387_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "418", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000418.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000418_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "505", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000505.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000505_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "515", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000515.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000515_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "534", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000534.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000534_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "599", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000599.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000599_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "613", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000613.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000613_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "745", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000745.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000745_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "861", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000861.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000861_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "940", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000940.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000940_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "946", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000946.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000946_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "996", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000996.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/000996_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1011", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001011.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001011_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1037", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001037.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001037_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1057", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001057.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001057_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1195", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001195.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001195_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1236", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001236.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001236_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1276", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001276.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001276_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1368", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001368.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001368_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1403", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001403.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001403_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1417", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001417.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001417_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1481", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001481.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001481_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1489", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001489.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001489_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1618", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001618.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001618_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1779", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001779.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001779_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1867", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001867.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001867_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "1949", - "image_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001949.jpg", - "action_path": "/mnt/weka/home/hao.zhang/kaiqin/traindata_0222_0030/ode_init_mc_random/images/001949_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - } - ] -} \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md deleted file mode 100644 index fa58e4bfe..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/README.md +++ /dev/null @@ -1,147 +0,0 @@ -Total Files: 145 - -00: W -01: S -02: A -03: D -04: WA -05: WD -06: SA -07: SD -08: u -09: d -10: l -11: r -12: ur -13: ul -14: dr -15: dl -16: still -17: W_u -18: W_d -19: W_l -20: W_r -21: W_ur -22: W_ul -23: W_dr -24: W_dl -25: S_u -26: S_d -27: S_l -28: S_r -29: S_ur -30: S_ul -31: S_dr -32: S_dl -33: A_u -34: A_d -35: A_l -36: A_r -37: A_ur -38: A_ul -39: A_dr -40: A_dl -41: D_u -42: D_d -43: D_l -44: D_r -45: D_ur -46: D_ul -47: D_dr -48: D_dl -49: WA_u -50: WA_d -51: WA_l -52: WA_r -53: WA_ur -54: WA_ul -55: WA_dr -56: WA_dl -57: WD_u -58: WD_d -59: WD_l -60: WD_r -61: WD_ur -62: WD_ul -63: WD_dr -64: WD_dl -65: SA_u -66: SA_d -67: SA_l -68: SA_r -69: SA_ur -70: SA_ul -71: SA_dr -72: SA_dl -73: SD_u -74: SD_d -75: SD_l -76: SD_r -77: SD_ur -78: SD_ul -79: SD_dr -80: SD_dl -81: key_2_action_rand_1_f4 -82: key_2_action_rand_2_f4 -83: key_2_action_rand_3_f4 -84: key_2_action_rand_4_f4 -85: key_2_action_rand_1 -86: key_2_action_rand_2 -87: key_2_action_rand_3 -88: key_2_action_rand_4 -89: camera_2_action_rand_1_f4 -90: camera_2_action_rand_2_f4 -91: camera_2_action_rand_3_f4 -92: camera_2_action_rand_4_f4 -93: camera_2_action_rand_1 -94: camera_2_action_rand_2 -95: camera_2_action_rand_3 -96: camera_2_action_rand_4 -97: key_camera_2_action_rand_1_f4 -98: key_camera_2_action_rand_2_f4 -99: key_camera_2_action_rand_3_f4 -100: key_camera_2_action_rand_4_f4 -101: key_camera_2_action_rand_1 -102: key_camera_2_action_rand_2 -103: key_camera_2_action_rand_3 -104: key_camera_2_action_rand_4 -105: key_1_action_rand_1_f4 -106: key_1_action_rand_2_f4 -107: key_1_action_rand_3_f4 -108: key_1_action_rand_4_f4 -109: key_1_action_rand_1 -110: key_1_action_rand_2 -111: key_1_action_rand_3 -112: key_1_action_rand_4 -113: camera_1_action_rand_1_f4 -114: camera_1_action_rand_2_f4 -115: camera_1_action_rand_3_f4 -116: camera_1_action_rand_4_f4 -117: camera_1_action_rand_1 -118: camera_1_action_rand_2 -119: camera_1_action_rand_3 -120: camera_1_action_rand_4 -121: key_camera_1_action_rand_1_f4 -122: key_camera_1_action_rand_2_f4 -123: key_camera_1_action_rand_3_f4 -124: key_camera_1_action_rand_4_f4 -125: key_camera_1_action_rand_1 -126: key_camera_1_action_rand_2 -127: key_camera_1_action_rand_3 -128: key_camera_1_action_rand_4 -129: key_camera_excl_1_action_rand_1_f4 -130: key_camera_excl_1_action_rand_2_f4 -131: key_camera_excl_1_action_rand_3_f4 -132: key_camera_excl_1_action_rand_4_f4 -133: key_camera_excl_1_action_rand_1 -134: key_camera_excl_1_action_rand_2 -135: key_camera_excl_1_action_rand_3 -136: key_camera_excl_1_action_rand_4 -137: key_camera_excl_1_action_rand_1_f4 -138: key_camera_excl_1_action_rand_2_f4 -139: key_camera_excl_1_action_rand_3_f4 -140: key_camera_excl_1_action_rand_4_f4 -141: key_camera_excl_1_action_rand_1 -142: key_camera_excl_1_action_rand_2 -143: key_camera_excl_1_action_rand_3 -144: key_camera_excl_1_action_rand_4 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/README.md b/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/README.md deleted file mode 100644 index de15602b6..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/README.md +++ /dev/null @@ -1,147 +0,0 @@ -Total Files: 145 - -00: W -01: S -02: A -03: D -04: WA -05: WD -06: SA -07: SD -08: u -09: d -10: l -11: r -12: ur -13: ul -14: dr -15: dl -16: still -17: W_u -18: W_d -19: W_l -20: W_r -21: W_ur -22: W_ul -23: W_dr -24: W_dl -25: S_u -26: S_d -27: S_l -28: S_r -29: S_ur -30: S_ul -31: S_dr -32: S_dl -33: A_u -34: A_d -35: A_l -36: A_r -37: A_ur -38: A_ul -39: A_dr -40: A_dl -41: D_u -42: D_d -43: D_l -44: D_r -45: D_ur -46: D_ul -47: D_dr -48: D_dl -49: WA_u -50: WA_d -51: WA_l -52: WA_r -53: WA_ur -54: WA_ul -55: WA_dr -56: WA_dl -57: WD_u -58: WD_d -59: WD_l -60: WD_r -61: WD_ur -62: WD_ul -63: WD_dr -64: WD_dl -65: SA_u -66: SA_d -67: SA_l -68: SA_r -69: SA_ur -70: SA_ul -71: SA_dr -72: SA_dl -73: SD_u -74: SD_d -75: SD_l -76: SD_r -77: SD_ur -78: SD_ul -79: SD_dr -80: SD_dl -81: key_2_action_rand_1_f4 -82: key_2_action_rand_2_f4 -83: key_2_action_rand_3_f4 -84: key_2_action_rand_4_f4 -85: key_2_action_rand_1 -86: key_2_action_rand_2 -87: key_2_action_rand_3 -88: key_2_action_rand_4 -89: camera_2_action_rand_1_f4 -90: camera_2_action_rand_2_f4 -91: camera_2_action_rand_3_f4 -92: camera_2_action_rand_4_f4 -93: camera_2_action_rand_1 -94: camera_2_action_rand_2 -95: camera_2_action_rand_3 -96: camera_2_action_rand_4 -97: key_camera_2_action_rand_1_f4 -98: key_camera_2_action_rand_2_f4 -99: key_camera_2_action_rand_3_f4 -100: key_camera_2_action_rand_4_f4 -101: key_camera_2_action_rand_1 -102: key_camera_2_action_rand_2 -103: key_camera_2_action_rand_3 -104: key_camera_2_action_rand_4 -105: key_1_action_rand_1_f4 -106: key_1_action_rand_2_f4 -107: key_1_action_rand_3_f4 -108: key_1_action_rand_4_f4 -109: key_1_action_rand_1 -110: key_1_action_rand_2 -111: key_1_action_rand_3 -112: key_1_action_rand_4 -113: camera_1_action_rand_1_f4 -114: camera_1_action_rand_2_f4 -115: camera_1_action_rand_3_f4 -116: camera_1_action_rand_4_f4 -117: camera_1_action_rand_1 -118: camera_1_action_rand_2 -119: camera_1_action_rand_3 -120: camera_1_action_rand_4 -121: key_camera_1_action_rand_1_f4 -122: key_camera_1_action_rand_2_f4 -123: key_camera_1_action_rand_3_f4 -124: key_camera_1_action_rand_4_f4 -125: key_camera_1_action_rand_1 -126: key_camera_1_action_rand_2 -127: key_camera_1_action_rand_3 -128: key_camera_1_action_rand_4 -129: key_camera_excl_2_action_rand_1_f4 -130: key_camera_excl_2_action_rand_2_f4 -131: key_camera_excl_2_action_rand_3_f4 -132: key_camera_excl_2_action_rand_4_f4 -133: key_camera_excl_2_action_rand_1 -134: key_camera_excl_2_action_rand_2 -135: key_camera_excl_2_action_rand_3 -136: key_camera_excl_2_action_rand_4 -137: key_camera_excl_1_action_rand_1_f4 -138: key_camera_excl_1_action_rand_2_f4 -139: key_camera_excl_1_action_rand_3_f4 -140: key_camera_excl_1_action_rand_4_f4 -141: key_camera_excl_1_action_rand_1 -142: key_camera_excl_1_action_rand_2 -143: key_camera_excl_1_action_rand_3 -144: key_camera_excl_1_action_rand_4 diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.sh b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.sh deleted file mode 100644 index e9869e1f5..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=offline -export TOKENIZERS_PARALLELISM=false -export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN - -MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" -VALIDATION_DATASET_FILE="mc_wasd_10/validation.json" -NUM_GPUS=4 -# export CUDA_VISIBLE_DEVICES=0,1,2,3 -# IP=[MASTER NODE IP] - -# Training arguments -training_args=( - --tracker_project_name "wangame_1.3b_overfit" - --output_dir "wangame_1.3b_overfit" - --max_train_steps 1500 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 20 - --num_height 352 - --num_width 640 - --num_frames 77 - --enable_gradient_checkpointing_type "full" -) - -# Parallel arguments -parallel_args=( - --num_gpus $NUM_GPUS - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim $NUM_GPUS -) - -# Model arguments -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -# Dataset arguments -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 1 -) - -# Validation arguments -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 100 - --validation_sampling_steps "40" - --validation_guidance_scale "1.0" -) - -# Optimizer arguments -optimizer_args=( - --learning_rate 2e-5 - --mixed_precision "bf16" - --weight_only_checkpointing_steps 1000 - --training_state_checkpointing_steps 1000 - --weight_decay 1e-4 - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.1 - --multi_phased_distill_schedule "4000-1" - --not_apply_cfg_solver - --dit_precision "fp32" - --num_euler_timesteps 50 - --ema_start_step 0 -) - -# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t -torchrun \ - --nnodes 1 \ - --nproc_per_node $NUM_GPUS \ - fastvideo/training/wangame_training_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.slurm deleted file mode 100644 index fc4eb79d2..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_i2v.slurm +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wangame_1.3b_overfit -#SBATCH --partition=main -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=wangame_1.3b_overfit_output/wangame_1.3b_overfit_%j.out -#SBATCH --error=wangame_1.3b_overfit_output/wangame_1.3b_overfit_%j.err -#SBATCH --exclusive - -# Basic Info -export NCCL_P2P_DISABLE=1 -export TORCH_NCCL_ENABLE_MONITORING=0 -export NCCL_DEBUG_SUBSYS=INIT,NET -# different cache dir for different processes -export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} -export MASTER_PORT=29500 -export NODE_RANK=$SLURM_PROCID -nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) -export MASTER_ADDR=${nodes[0]} -export TOKENIZERS_PARALLELISM=false -# export WANDB_API_KEY="8d9f4b39abd68eb4e29f6fc010b7ee71a2207cde" -export WANDB_API_KEY="50632ebd88ffd970521cec9ab4a1a2d7e85bfc45" -# export WANDB_API_KEY='your_wandb_api_key_here' -export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=online -export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN - -source ~/conda/miniconda/bin/activate -conda activate wei-fv-distill -export HOME="/mnt/weka/home/hao.zhang/wei" - -MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" -VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json" -# Configs -NUM_GPUS=8 - -# Training arguments -training_args=( - --tracker_project_name "wangame_1.3b_overfit" - --output_dir "wangame_1.3b_overfit" - --max_train_steps 15000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 20 - --num_height 352 - --num_width 640 - --num_frames 77 - --enable_gradient_checkpointing_type "full" -) - -# Parallel arguments -parallel_args=( - --num_gpus $NUM_GPUS - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim $NUM_GPUS -) - -# Model arguments -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -# Dataset arguments -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 1 -) - -# Validation arguments -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 100 - --validation_sampling_steps "40" - --validation_guidance_scale "1.0" -) - -# Optimizer arguments -optimizer_args=( - --learning_rate 2e-5 - --mixed_precision "bf16" - --weight_only_checkpointing_steps 1000000 - --training_state_checkpointing_steps 10000000 - --weight_decay 1e-4 - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.1 - --multi_phased_distill_schedule "4000-1" - --not_apply_cfg_solver - --dit_precision "fp32" - --num_euler_timesteps 50 - --ema_start_step 0 -) - -# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t -torchrun \ - --nnodes 1 \ - --nproc_per_node $NUM_GPUS \ - fastvideo/training/wangame_training_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm deleted file mode 100644 index a6051f192..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame.slurm +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wangame_1.3b -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=wangame_1.3b_output/wangame_1.3b_%j.out -#SBATCH --error=wangame_1.3b_output/wangame_1.3b_%j.err -#SBATCH --exclusive - -# Basic Info -export NCCL_P2P_DISABLE=1 -export TORCH_NCCL_ENABLE_MONITORING=0 -export NCCL_DEBUG_SUBSYS=INIT,NET -# different cache dir for different processes -export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} -export MASTER_PORT=29501 -export NODE_RANK=$SLURM_PROCID -nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) -export MASTER_ADDR=${nodes[0]} -export TOKENIZERS_PARALLELISM=false -export WANDB_API_KEY="d5b02b05e30d8cb34c7b31c6ae10416fc26dcb66" -export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=online -export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN -export FASTVIDEO_MAP_STYLE_CACHE_DIR="/mnt/weka/home/hao.zhang/mhuo/FastVideo/map_style_cache" - -source ~/conda/miniconda/bin/activate -conda activate mhuo-fv -export HOME="/mnt/weka/home/hao.zhang/mhuo" - -# Configs -NUM_GPUS=8 -NUM_NODES=4 -NUM_TOTAL_GPUS=$((NUM_GPUS * NUM_NODES)) -BS_PER_GPU=1 -GRADIENT_ACCUMULATION_STEPS=1 -WANDB_RUN_NAME="MC_1action_rand_from_scratch" -FREEZE_DIT=False -RUN_DIR="wangame_1.3b_1action_rand_from_scratch" -CHECKPOINTING_STEPS=1000 -ACTION_WARMUP_STEPS=0 -LEARNING_RATE=1e-5 - -MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -# CKPT_SAFETENSOR="wangame_1.3b_with_warmup_lr_1e-5/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" -# -# Data dirs (use one of the following): -# - DATA_DIR_ALL: all datasets below combined (comma-separated) -# - Or a single path / subset; optional ":N" = repeat, ":0" = skip -# -DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0" # Random -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:0" # Doom -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:1" # Static + w only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:1" # w/s/a/d only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:1" # wasd only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:1" # camera l-only and r-only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:1" # camera only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:1" # key_camera_excl_1_action_rand - -VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json" -# -# Single-dir / validation alternatives (comment out DATA_DIR above and uncomment one block): -# MC wasd only: -# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json" -# MC random: -# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" -# Doom: -# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_doom.json" -# Overfit: -# DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_overfit.json" - - -# Training arguments -training_args=( - --tracker_project_name "wangame_1.3b" - --output_dir $RUN_DIR - --wandb_run_name "$WANDB_RUN_NAME" - --max_train_steps 20000 - --train_batch_size $BS_PER_GPU - --train_sp_batch_size $BS_PER_GPU - --gradient_accumulation_steps $GRADIENT_ACCUMULATION_STEPS - --num_latent_t 20 - --num_height 352 - --num_width 640 - --num_frames 77 - --enable_gradient_checkpointing_type "full" - --train_action_only $FREEZE_DIT - --action_warmup_steps $ACTION_WARMUP_STEPS - # --init_weights_from_safetensors $CKPT_SAFETENSOR -) - -# Parallel arguments -parallel_args=( - --num_gpus $NUM_TOTAL_GPUS - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim $NUM_TOTAL_GPUS -) - -# Model arguments -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -# Dataset arguments -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 1 -) - -# Validation arguments -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 100 - --validation_sampling_steps "40" - --validation_guidance_scale "1.0" - --validation_num_samples $NUM_TOTAL_GPUS -) - -# Optimizer arguments -optimizer_args=( - --learning_rate $LEARNING_RATE - --mixed_precision "bf16" - --weight_only_checkpointing_steps 100000 - --training_state_checkpointing_steps $CHECKPOINTING_STEPS - --weight_decay 1e-4 - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 2 - --training_cfg_rate 0.1 - --multi_phased_distill_schedule "4000-1" - --not_apply_cfg_solver - --dit_precision "fp32" - --num_euler_timesteps 50 - --ema_start_step 0 -) - -# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t - -if [ $NUM_NODES -eq 1 ]; then - torchrun \ - --nnodes $NUM_NODES \ - --nproc_per_node $NUM_GPUS \ - fastvideo/training/wangame_training_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" -else - srun torchrun \ - --nnodes $NUM_NODES \ - --nproc_per_node $NUM_GPUS \ - --rdzv_backend c10d \ - --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT \ - --node_rank $SLURM_PROCID \ - fastvideo/training/wangame_training_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" -fi \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm deleted file mode 100644 index fb0d30b24..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/finetune_wangame_freeze_action.slurm +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wangame_1.3b -#SBATCH --partition=main -#SBATCH --nodes=4 -#SBATCH --ntasks=4 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=wangame_1.3b_output/wangame_1.3b_%j.out -#SBATCH --error=wangame_1.3b_output/wangame_1.3b_%j.err -#SBATCH --exclusive - -# Basic Info -export NCCL_P2P_DISABLE=1 -export TORCH_NCCL_ENABLE_MONITORING=0 -export NCCL_DEBUG_SUBSYS=INIT,NET -# different cache dir for different processes -export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} -export MASTER_PORT=29501 -export NODE_RANK=$SLURM_PROCID -nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) -export MASTER_ADDR=${nodes[0]} -export TOKENIZERS_PARALLELISM=false -export WANDB_API_KEY="d5b02b05e30d8cb34c7b31c6ae10416fc26dcb66" -export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=online -export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN -export FASTVIDEO_MAP_STYLE_CACHE_DIR="/mnt/weka/home/hao.zhang/mhuo/FastVideo/map_style_cache" - -source ~/conda/miniconda/bin/activate -conda activate mhuo-fv -export HOME="/mnt/weka/home/hao.zhang/mhuo" - -# Configs -NUM_GPUS=8 -NUM_NODES=4 # TODO: change this to 1 to debug -NUM_TOTAL_GPUS=$((NUM_GPUS * NUM_NODES)) -BS_PER_GPU=1 -GRADIENT_ACCUMULATION_STEPS=1 -WANDB_RUN_NAME="Doom from MC freeze action" -RUN_DIR="wangame_1.3b" -CHECKPOINTING_STEPS=100000 # This means checkpoint every 100000 steps, effectively no ckpt will be saved -ACTION_WARMUP_STEPS=100000 # This means the action modules will be frozen for the first 100000 steps. -# Effectively, during the total 100000 steps, action modules are always frozen. -LEARNING_RATE=1e-5 -# Freeze base DiT, only train action modules -Freeze_DiT=false - -MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -# CKPT_SAFETENSOR="wangame_1.3b_wsad_random_lr_1e-5/checkpoint-2000/transformer/diffusion_pytorch_model.safetensors" -# CKPT_SAFETENSOR="wangame_1.3b_with_warmup_lr_1e-5/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" -CKPT_SAFETENSOR="wangame_1.3b_1action_rand_from_scratch/checkpoint-7000/transformer/diffusion_pytorch_model.safetensors" - -# Data dirs (use one of the following): -# - DATA_DIR_ALL: all datasets below combined (comma-separated) -# - Or a single path / subset; optional ":N" = repeat, ":0" = skip -# -DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed:0" # Random -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed:1" # Doom -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/0_static_plus_w_only/preprocessed:0" # Static + w only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed:0" # w/s/a/d only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed:0" # wasd only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed:0" # camera l-only and r-only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed:0" # camera only -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed:0" # key_camera_excl_1_action_rand -DATA_DIR="$DATA_DIR,/mnt/weka/home/hao.zhang/alex/wm-lab/datas/cache/zelda_overfit:0" # zelda - -VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json" # TODO: double check this, you may remove some MC image and add more doom image -# -# Single-dir / validation alternatives (comment out DATA_DIR above and uncomment one block): -# MC wasd only: -# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_wsad.json" -# MC random: -# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_mc.json" -# Doom: -# DATA_DIR="/mnt/weka/home/hao.zhang/mhuo/traindata_0204_1600/preprocessed" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_doom.json" -# Overfit: -# DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" -# VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v/validation_overfit.json" - - -# Training arguments -training_args=( - --tracker_project_name "wangame_1.3b" - --output_dir $RUN_DIR - --wandb_run_name "$WANDB_RUN_NAME" - --max_train_steps 10000 - --train_batch_size $BS_PER_GPU - --train_sp_batch_size $BS_PER_GPU - --gradient_accumulation_steps $GRADIENT_ACCUMULATION_STEPS - --num_latent_t 20 - --num_height 352 - --num_width 640 - --num_frames 77 - --enable_gradient_checkpointing_type "full" - --action_warmup_steps $ACTION_WARMUP_STEPS - --init_weights_from_safetensors $CKPT_SAFETENSOR - # TODO: check terminal log or log file xxx.err, whether there is a output line saying "Starting training with 1.53 B trainable parameters (total)". If both action modules are frozen, it should be around 1.53B, otherwise it could be 1.6B. -) - -if [ "$FREEZE_DiT" = "true" ]; then - training_args+=(--train_action_only) -fi - -# Parallel arguments -parallel_args=( - --num_gpus $NUM_TOTAL_GPUS - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim $NUM_TOTAL_GPUS -) - -# Model arguments -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -# Dataset arguments -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 1 -) - -# Validation arguments -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 100 - --validation_sampling_steps "40" - --validation_guidance_scale "1.0" - --validation_num_samples $NUM_TOTAL_GPUS -) - -# Optimizer arguments -optimizer_args=( - --learning_rate $LEARNING_RATE - --mixed_precision "bf16" - --weight_only_checkpointing_steps 100000 - --training_state_checkpointing_steps $CHECKPOINTING_STEPS - --weight_decay 1e-4 - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 2 - --training_cfg_rate 0.1 - --multi_phased_distill_schedule "4000-1" - --not_apply_cfg_solver - --dit_precision "fp32" - --num_euler_timesteps 50 - --ema_start_step 0 -) - -# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t - -if [ $NUM_NODES -eq 1 ]; then - torchrun \ - --nnodes $NUM_NODES \ - --nproc_per_node $NUM_GPUS \ - fastvideo/training/wangame_training_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" -else - srun torchrun \ - --nnodes $NUM_NODES \ - --nproc_per_node $NUM_GPUS \ - --rdzv_backend c10d \ - --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT \ - --node_rank $SLURM_PROCID \ - fastvideo/training/wangame_training_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" -fi \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/preprocess_wangame_data_i2v.sh b/examples/training/finetune/WanGame2.1_1.3b_i2v/preprocess_wangame_data_i2v.sh deleted file mode 100644 index 85a4fd0d2..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/preprocess_wangame_data_i2v.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -GPU_NUM=1 # 2,4,8 -MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" -DATA_MERGE_PATH="mc_wasd_10/merge.txt" -OUTPUT_DIR="mc_wasd_10/preprocessed/" - -# export CUDA_VISIBLE_DEVICES=0 -export MASTER_ADDR=localhost -export MASTER_PORT=29500 -export RANK=0 -export WORLD_SIZE=1 - -python fastvideo/pipelines/preprocess/v1_preprocess.py \ - --model_path $MODEL_PATH \ - --data_merge_path $DATA_MERGE_PATH \ - --preprocess_video_batch_size 10 \ - --seed 42 \ - --max_height 352 \ - --max_width 640 \ - --num_frames 77 \ - --dataloader_num_workers 0 \ - --output_dir=$OUTPUT_DIR \ - --samples_per_file 10 \ - --train_fps 25 \ - --flush_frequency 10 \ - --preprocess_task wangame \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/collect_samples_to_shao.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/collect_samples_to_shao.py deleted file mode 100644 index 1b2e886ef..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/collect_samples_to_shao.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -""" -For each data dir (from finetune_wangame.slurm), randomly pick 10 samples (mp4 + action.npy), -copy to to_shao// as 01.mp4, 01_action.npy, ..., 10.mp4, 10_action.npy, -and extract first frame as 01.jpg, ..., 10.jpg. -""" -import os -import random -import shutil - -import cv2 - -# Data dirs from finetune_wangame.slurm (paths with "preprocessed"; we use "video" or "videos" for mp4/npy) -DATA_DIRS = [ - "/mnt/weka/home/hao.zhang/mhuo/traindata_0204_2130/preprocessed", - "/mnt/weka/home/hao.zhang/mhuo/traindata_0205_1330/data/1_wasd_only/preprocessed", - "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/wasdonly_alpha1/preprocessed", - "/mnt/weka/home/hao.zhang/mhuo/traindata_0206_1200/data/camera/preprocessed", - "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/camera4hold_alpha1/preprocessed", - "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/preprocessed", -] - -OUT_ROOT = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao" -NUM_SAMPLES = 10 - - -def get_video_dir(preprocessed_path: str) -> str | None: - """Replace 'preprocessed' with 'video' (or 'videos') to get the dir containing mp4/npy.""" - video_path = preprocessed_path.replace("preprocessed", "video") - if os.path.isdir(video_path): - return video_path - videos_path = preprocessed_path.replace("preprocessed", "videos") - if os.path.isdir(videos_path): - return videos_path - return None - - -# Override short name for specific data dirs (e.g. traindata_0204_2130 -> fully_random) -SHORT_NAME_OVERRIDES: dict[str, str] = { - "traindata_0204_2130": "fully_random", -} - - -def get_short_name(preprocessed_path: str) -> str: - """Short name = parent folder of the preprocessed dir, e.g. 1_wasd_only.""" - name = os.path.basename(os.path.normpath(os.path.dirname(preprocessed_path))) - return SHORT_NAME_OVERRIDES.get(name, name) - - -def find_samples(video_dir: str) -> list[str]: - """Return list of base names (no extension) that have both xxxxxx.mp4 and xxxxxx_action.npy.""" - samples = [] - for f in os.listdir(video_dir): - if f.endswith(".mp4"): - base = f[:-4] - action_path = os.path.join(video_dir, f"{base}_action.npy") - if os.path.isfile(action_path): - samples.append(base) - return samples - - -def extract_first_frame(mp4_path: str, jpg_path: str) -> None: - cap = cv2.VideoCapture(mp4_path) - ret, frame = cap.read() - cap.release() - if ret: - cv2.imwrite(jpg_path, frame) - - -def main() -> None: - random.seed(42) - os.makedirs(OUT_ROOT, exist_ok=True) - total_dir = os.path.join(OUT_ROOT, "total") - os.makedirs(total_dir, exist_ok=True) - total_idx = 0 - - for preprocessed_path in DATA_DIRS: - video_dir = get_video_dir(preprocessed_path) - if video_dir is None: - print(f"Skip (video dir not found): {preprocessed_path}") - continue - - short_name = get_short_name(preprocessed_path) - samples = find_samples(video_dir) - if len(samples) < NUM_SAMPLES: - print(f"Skip {short_name}: only {len(samples)} samples (need {NUM_SAMPLES})") - continue - - chosen = random.sample(samples, NUM_SAMPLES) - out_dir = os.path.join(OUT_ROOT, short_name) - os.makedirs(out_dir, exist_ok=True) - - for i, base in enumerate(chosen, start=1): - num_str = f"{i:02d}" - src_mp4 = os.path.join(video_dir, f"{base}.mp4") - src_npy = os.path.join(video_dir, f"{base}_action.npy") - dst_mp4 = os.path.join(out_dir, f"{num_str}.mp4") - dst_npy = os.path.join(out_dir, f"{num_str}_action.npy") - dst_jpg = os.path.join(out_dir, f"{num_str}.jpg") - - shutil.copy2(src_mp4, dst_mp4) - shutil.copy2(src_npy, dst_npy) - extract_first_frame(dst_mp4, dst_jpg) - - # Copy into total/ with global numbering - total_idx += 1 - t_str = f"{total_idx:02d}" - shutil.copy2(dst_mp4, os.path.join(total_dir, f"{t_str}.mp4")) - shutil.copy2(dst_npy, os.path.join(total_dir, f"{t_str}_action.npy")) - shutil.copy2(dst_jpg, os.path.join(total_dir, f"{t_str}.jpg")) - - print(f"Done: {short_name} -> {out_dir} ({NUM_SAMPLES} samples)") - - print(f"Done: total -> {total_dir} ({total_idx} samples)") - - -if __name__ == "__main__": - main() diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py deleted file mode 100644 index ff8b4e76e..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_actions.py +++ /dev/null @@ -1,278 +0,0 @@ -import os -import numpy as np - -# Configuration -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -BASE_OUTPUT_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "actions_81")) -VIDEO_OUTPUT_DIR = BASE_OUTPUT_DIR - -os.makedirs(VIDEO_OUTPUT_DIR, exist_ok=True) - -CAM_VALUE = 0.1 -FRAME_COUNT = 80 - -# Action Mapping -KEY_TO_INDEX = { - 'W': 0, 'S': 1, 'A': 2, 'D': 3, -} - -VIEW_ACTION_TO_MOUSE = { - "stop": [0.0, 0.0], - "up": [CAM_VALUE, 0.0], - "down": [-CAM_VALUE, 0.0], - "left": [0.0, -CAM_VALUE], - "right": [0.0, CAM_VALUE], - "up_right": [CAM_VALUE, CAM_VALUE], - "up_left": [CAM_VALUE, -CAM_VALUE], - "down_right": [-CAM_VALUE, CAM_VALUE], - "down_left": [-CAM_VALUE, -CAM_VALUE], -} - -def get_multihot_vector(keys_str): - """Convert string like 'WA' to [1, 0, 1, 0, 0, 0]""" - vector = [0.0] * 6 - if not keys_str: - return vector - for char in keys_str.upper(): - if char in KEY_TO_INDEX: - vector[KEY_TO_INDEX[char]] = 1.0 - return vector - -def get_mouse_vector(view_str): - """Convert view string to [x, y]""" - return VIEW_ACTION_TO_MOUSE.get(view_str.lower(), [0.0, 0.0]) - -def generate_sequence(key_seq, mouse_seq): - """ - Generates action arrays based on sequences. - key_seq and mouse_seq must be length FRAME_COUNT. - Duplicates the first frame at the beginning, so output length is FRAME_COUNT + 1. - """ - if len(key_seq) != FRAME_COUNT or len(mouse_seq) != FRAME_COUNT: - raise ValueError("key_seq and mouse_seq must be length FRAME_COUNT") - - keyboard_arr = np.zeros((FRAME_COUNT, 6), dtype=np.float32) - mouse_arr = np.zeros((FRAME_COUNT, 2), dtype=np.float32) - - for i in range(FRAME_COUNT): - keyboard_arr[i] = get_multihot_vector(key_seq[i]) - mouse_arr[i] = get_mouse_vector(mouse_seq[i]) - - keyboard_arr = np.vstack([keyboard_arr[0:1], keyboard_arr]) - mouse_arr = np.vstack([mouse_arr[0:1], mouse_arr]) - - return keyboard_arr, mouse_arr - -def save_action(filename, keyboard_arr, mouse_arr): - if not filename.endswith(".npy"): - filename = f"{filename}.npy" - filepath = os.path.join(VIDEO_OUTPUT_DIR, filename) - - action_dict = { - 'keyboard': keyboard_arr, - 'mouse': mouse_arr - } - np.save(filepath, action_dict) - return filename - - -def build_constant_sequence(value): - return [value] * FRAME_COUNT - - -def build_random_sequence(actions, granularity, rng): - sequence = [] - remaining = FRAME_COUNT - while remaining > 0: - block = granularity if remaining >= granularity else remaining - action = rng.choice(actions) - sequence.extend([action] * block) - remaining -= block - return sequence - - -def build_random_sequence_either_or(key_actions, mouse_actions, granularity, rng): - """Build key_seq and mouse_seq where each block has either key OR mouse, not both.""" - key_seq = [] - mouse_seq = [] - remaining = FRAME_COUNT - while remaining > 0: - block = granularity if remaining >= granularity else remaining - use_key = rng.choice([True, False]) - if use_key: - key_action = rng.choice(key_actions) - mouse_action = "" - else: - key_action = "" - mouse_action = rng.choice(mouse_actions) - key_seq.extend([key_action] * block) - mouse_seq.extend([mouse_action] * block) - remaining -= block - return key_seq, mouse_seq - - -def mouse_short_name(view_str): - mapping = { - "up": "u", - "down": "d", - "left": "l", - "right": "r", - "up_right": "ur", - "up_left": "ul", - "down_right": "dr", - "down_left": "dl", - } - return mapping.get(view_str, "NA") - - -if __name__ == "__main__": - configs = [] - readme_content = [] - rng = np.random.default_rng(42) - - # configs = list of entries - # a entry is a tuple of (key_seq, mouse_seq) - # key_seq is a list of strings, length of FRAME_COUNT, each string is a key in 'W', 'S', 'A', 'D', 'WA', 'WD', 'SA', 'SD' - # mouse_seq is a list of strings, length of FRAME_COUNT, each string is a mouse action in 'up', 'down', 'left', 'right', 'up_right', 'up_left', 'down_right', 'down_left' - - # Naming: 1=WASDudlr (key: W.npy, SA.npy; camera: u.npy; key+camera: W_u.npy, SA_dl.npy). Rand: 1_action=WASD/UDLR+still only, 2_action=full set. 2-6=rand names below. - # Group 1: Constant Keyboard, No Mouse. W.npy, S.npy, WA.npy, SA.npy, ... - keys_basic = ["W", "S", "A", "D", "WA", "WD", "SA", "SD"] - for key in keys_basic: - configs.append( - (key, build_constant_sequence(key), build_constant_sequence("")) - ) - - # Group 2: No Keyboard, Constant Mouse. u.npy, d.npy, ur.npy, ... - mouse_basic = [ - "up", - "down", - "left", - "right", - "up_right", - "up_left", - "down_right", - "down_left", - ] - for mouse in mouse_basic: - name = mouse_short_name(mouse) - configs.append( - (name, build_constant_sequence(""), build_constant_sequence(mouse)) - ) - - # Group 3: Still. still.npy - configs.append(("still", build_constant_sequence(""), build_constant_sequence(""))) - - # Group 4: Constant key + camera. W_u.npy, SA_dl.npy, ... - for key in keys_basic: - for mouse in mouse_basic: - configs.append( - ( - f"{key}_{mouse_short_name(mouse)}", - build_constant_sequence(key), - build_constant_sequence(mouse), - ) - ) - - # Random groups: allow still ("") as an option (WASD+still, UDLR+still, and full sets+still) - keys_basic_still = keys_basic + [""] - mouse_basic_still = mouse_basic + [""] - - # Group 5: key_2_action_rand (full key set). key_2_action_rand_1..4, key_2_action_rand_1_f4..4_f4 - for granularity in (4, 12): - suffix = "_f4" if granularity == 4 else "" - for i in range(1, 5): - key_seq = build_random_sequence(keys_basic_still, granularity, rng) - configs.append( - (f"key_2_action_rand_{i}{suffix}", key_seq, build_constant_sequence("")) - ) - - # Group 6: camera_2_action_rand (full camera set) - for granularity in (4, 12): - suffix = "_f4" if granularity == 4 else "" - for i in range(1, 5): - mouse_seq = build_random_sequence(mouse_basic_still, granularity, rng) - configs.append( - (f"camera_2_action_rand_{i}{suffix}", build_constant_sequence(""), mouse_seq) - ) - - # Group 7: key_camera_2_action_rand (both full sets) - for granularity in (4, 12): - suffix = "_f4" if granularity == 4 else "" - for i in range(1, 5): - key_seq = build_random_sequence(keys_basic_still, granularity, rng) - mouse_seq = build_random_sequence(mouse_basic_still, granularity, rng) - configs.append( - (f"key_camera_2_action_rand_{i}{suffix}", key_seq, mouse_seq) - ) - - # WASD-only (no combined keys) and u/d/l/r-only (no combined directions), with still as option - keys_wasd_only = ["W", "S", "A", "D"] - mouse_udlr_only = ["up", "down", "left", "right"] - keys_wasd_still = keys_wasd_only + [""] - mouse_udlr_still = mouse_udlr_only + [""] - - # Group 8: key_1_action_rand (WASD+still only) - for granularity in (4, 12): - suffix = "_f4" if granularity == 4 else "" - for i in range(1, 5): - key_seq = build_random_sequence(keys_wasd_still, granularity, rng) - configs.append( - (f"key_1_action_rand_{i}{suffix}", key_seq, build_constant_sequence("")) - ) - - # Group 9: camera_1_action_rand (UDLR+still only) - for granularity in (4, 12): - suffix = "_f4" if granularity == 4 else "" - for i in range(1, 5): - mouse_seq = build_random_sequence(mouse_udlr_still, granularity, rng) - configs.append( - (f"camera_1_action_rand_{i}{suffix}", build_constant_sequence(""), mouse_seq) - ) - - # Group 10: key_camera_1_action_rand (WASD+still, UDLR+still) - for granularity in (4, 12): - suffix = "_f4" if granularity == 4 else "" - for i in range(1, 5): - key_seq = build_random_sequence(keys_wasd_still, granularity, rng) - mouse_seq = build_random_sequence(mouse_udlr_still, granularity, rng) - configs.append( - (f"key_camera_1_action_rand_{i}{suffix}", key_seq, mouse_seq) - ) - - # Group 11a: key_camera_excl_2_action_rand (either key OR camera per block, full key + full camera set) - for granularity in (4, 12): - suffix = "_f4" if granularity == 4 else "" - for i in range(1, 5): - key_seq, mouse_seq = build_random_sequence_either_or(keys_basic_still, mouse_basic_still, granularity, rng) - configs.append( - (f"key_camera_excl_2_action_rand_{i}{suffix}", key_seq, mouse_seq) - ) - - # Group 11b: key_camera_excl_1_action_rand (either key OR camera per block, WASD/UDLR+still) - for granularity in (4, 12): - suffix = "_f4" if granularity == 4 else "" - for i in range(1, 5): - key_seq, mouse_seq = build_random_sequence_either_or(keys_wasd_still, mouse_udlr_still, granularity, rng) - configs.append( - (f"key_camera_excl_1_action_rand_{i}{suffix}", key_seq, mouse_seq) - ) - - # Execution - print(f"Preparing to generate {len(configs)} action files...") - - for name, key_seq, mouse_seq in configs: - # Generate Data - kb_arr, ms_arr = generate_sequence(key_seq, mouse_seq) - filename = save_action(name, kb_arr, ms_arr) - readme_content.append(filename.replace(".npy", "")) - - print(f"Generated {filename}") - - readme_path = os.path.join(VIDEO_OUTPUT_DIR, "README.md") - with open(readme_path, "w") as f: - f.write(f"Total Files: {len(readme_content)}\n\n") - for idx, name in enumerate(readme_content): - f.write(f"{idx:02d}: {name}\n") - - print(f"{len(configs)} .npy files generated in {VIDEO_OUTPUT_DIR}") \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py deleted file mode 100644 index 3444ba1eb..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation.py +++ /dev/null @@ -1,271 +0,0 @@ -import json -import os -import shutil - -import cv2 - -train = "zelda" - -if train == "zelda": - height = 480 - width = 832 - num_frames = 81 - action_dir = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81" -elif train == "mc": - height = 352 - width = 640 - num_frames = 77 - action_dir = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions" -else: - raise ValueError(f"Invalid train type: {train}") - -# Output path -output_path = ( - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/" - f"WanGame2.1_1.3b_i2v/validation_{train}.json" -) - -# Fixed fields -fixed_fields = { - "video_path": None, - "num_inference_steps": 40, - "height": height, - "width": width, - "num_frames": num_frames, -} - -# WASDudlr: single key W.npy, single camera u.npy, key+camera w_u.npy -still = os.path.join(action_dir, "still.npy") -key_W = os.path.join(action_dir, "W.npy") -key_S = os.path.join(action_dir, "S.npy") -key_A = os.path.join(action_dir, "A.npy") -key_D = os.path.join(action_dir, "D.npy") -key_wa = os.path.join(action_dir, "WA.npy") -key_s_u = os.path.join(action_dir, "S_u.npy") -camera_u = os.path.join(action_dir, "u.npy") -camera_d = os.path.join(action_dir, "d.npy") -camera_l = os.path.join(action_dir, "l.npy") -camera_r = os.path.join(action_dir, "r.npy") -# key_1_action_rand, camera_1_action_rand (full set); _f4 suffix for granularity 4 -key_1_action_rand_1 = os.path.join(action_dir, "key_1_action_rand_1.npy") -key_1_action_rand_2 = os.path.join(action_dir, "key_1_action_rand_2.npy") -key_1_action_rand_1_f4 = os.path.join(action_dir, "key_1_action_rand_1_f4.npy") -key_1_action_rand_2_f4 = os.path.join(action_dir, "key_1_action_rand_2_f4.npy") -camera_1_action_rand_1 = os.path.join(action_dir, "camera_1_action_rand_1.npy") -camera_1_action_rand_2 = os.path.join(action_dir, "camera_1_action_rand_2.npy") -camera_1_action_rand_1_f4 = os.path.join(action_dir, "camera_1_action_rand_1_f4.npy") -camera_1_action_rand_2_f4 = os.path.join(action_dir, "camera_1_action_rand_2_f4.npy") -key_camera_1_action_rand_1 = os.path.join(action_dir, "key_camera_1_action_rand_1.npy") -key_camera_1_action_rand_2 = os.path.join(action_dir, "key_camera_1_action_rand_2.npy") -key_camera_1_action_rand_1_f4 = os.path.join(action_dir, "key_camera_1_action_rand_1_f4.npy") -key_camera_1_action_rand_2_f4 = os.path.join(action_dir, "key_camera_1_action_rand_2_f4.npy") -key_camera_excl_1_action_rand_1 = os.path.join(action_dir, "key_camera_excl_1_action_rand_1.npy") -key_camera_excl_1_action_rand_2 = os.path.join(action_dir, "key_camera_excl_1_action_rand_2.npy") -key_camera_excl_1_action_rand_1_f4 = os.path.join(action_dir, "key_camera_excl_1_action_rand_1_f4.npy") -key_camera_excl_1_action_rand_2_f4 = os.path.join(action_dir, "key_camera_excl_1_action_rand_2_f4.npy") -# key_2_action_rand, camera_2_action_rand (WASD/UDLR+still) -key_2_action_rand_1 = os.path.join(action_dir, "key_2_action_rand_1.npy") -key_2_action_rand_1_f4 = os.path.join(action_dir, "key_2_action_rand_1_f4.npy") -camera_2_action_rand_1 = os.path.join(action_dir, "camera_2_action_rand_1.npy") -camera_2_action_rand_1_f4 = os.path.join(action_dir, "camera_2_action_rand_1_f4.npy") -key_camera_2_action_rand_1 = os.path.join(action_dir, "key_camera_2_action_rand_1.npy") -key_camera_2_action_rand_1_f4 = os.path.join(action_dir, "key_camera_2_action_rand_1_f4.npy") -key_camera_excl_2_action_rand_1 = os.path.join(action_dir, "key_camera_excl_2_action_rand_1.npy") -key_camera_excl_2_action_rand_1_f4 = os.path.join(action_dir, "key_camera_excl_2_action_rand_1_f4.npy") - - -train_img_zelda_list = [ - # "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/-BxyBxfDKA0_chunk_0292/segment0001.jpg", - # "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/-BxyBxfDKA0_chunk_0292/segment0003.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0006/segment0002.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0067/segment0002.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0484/segment0002.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0019/segment0004.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0140/segment0003.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0300/segment0003.jpg", -] - -val_img_zelda_list = train_img_zelda_list -train_action_zelda_list = [] -for img in train_img_zelda_list: - img_dir = os.path.dirname(img) - basename = os.path.splitext(os.path.basename(img))[0] - action_path = os.path.join( - img_dir, - "postprocess/action/majority_voting/" - "81_frame_no_button", - f"{basename}.npy", - ) - train_action_zelda_list.append(action_path) - - -val_img_mc_list = [ - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg", -] - -# Get train data list -train_mc_data_dir = "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1" -train_mc_idx_list = ["000000", "000500", "001000", "001500", "002000", "002500", "003000", "003500"] -train_mc_img_list = [] -train_mc_action_list = [] - -for idx in train_mc_idx_list: - video_path = os.path.join(train_mc_data_dir, f"videos/{idx}.mp4") - # extract the first frame as image - image_path = os.path.join(train_mc_data_dir, f"first_frame/{idx}.jpg") - os.makedirs(os.path.dirname(image_path), exist_ok=True) - cap = cv2.VideoCapture(video_path) - ret, frame = cap.read() - cap.release() - if ret: - cv2.imwrite(image_path, frame) - train_mc_img_list.append(image_path) - train_mc_action_list.append(os.path.join(train_mc_data_dir, f"videos/{idx}_action.npy")) - - -# Get doom Val data list -val_img_doom_list = [ - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000002.jpg", - "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000003.jpg", -] - -if train == "mc": - val_img_list = val_img_mc_list - train_img_list = train_mc_img_list - train_action_list = train_mc_action_list -elif train == "zelda": - val_img_list = val_img_zelda_list - train_img_list = train_img_zelda_list - train_action_list = train_action_zelda_list -elif train == "doom": - val_img_list = val_img_doom_list -else: - raise ValueError(f"Invalid train type: {train}") - - -holder = 0 # placeholder -# 32 placeholders (idx 0-31). Fill in manually. -a0 = ["00 Val-00: W", val_img_list[0], key_W] -a1 = ["01 Val-01: S", val_img_list[1], key_S] -a2 = ["02 Val-02: A", val_img_list[2], key_A] -a3 = ["03 Val-03: D", val_img_list[3], key_D] -a4 = ["04 Val-04: u", val_img_list[4], camera_u] -a5 = ["05 Val-05: d", val_img_list[5], camera_d] -a6 = ["06 Val-06: l", val_img_list[4], camera_l] -a7 = ["07 Val-07: r", val_img_list[5], camera_r] -a8 = ["08 Val-00: key rand", val_img_list[0], key_1_action_rand_1] -a9 = ["09 Val-01: key rand", val_img_list[1], key_1_action_rand_2] -a10 = ["10 Val-02: camera rand", val_img_list[2], camera_1_action_rand_1] -a11 = ["11 Val-03: camera rand", val_img_list[3], camera_1_action_rand_2] -a12 = ["12 Val-00: key+camera excl rand", val_img_list[0], key_camera_excl_1_action_rand_1] -a13 = ["13 Val-01: key+camera excl rand", val_img_list[1], key_camera_excl_1_action_rand_2] -a14 = ["14 Val-02: key+camera rand", val_img_list[2], key_camera_1_action_rand_1] -a15 = ["15 Val-03: key+camera rand", val_img_list[3], key_camera_1_action_rand_2] -a16 = ["16 Val-04: (simultaneous) key rand", val_img_list[4], key_2_action_rand_1] -a17 = ["17 Val-05: (simultaneous) camera rand", val_img_list[5], camera_2_action_rand_1] -a18 = ["18 Val-06: (simultaneous) key+camera excl rand", val_img_list[5], key_camera_excl_2_action_rand_1] -a19 = ["19 Val-07: (simultaneous) key+camera rand", val_img_list[5], key_camera_2_action_rand_1] -a20 = ["20 Val-08: W+A", val_img_list[0], key_wa] -a21 = ["21 Val-09: S+u", val_img_list[1], key_s_u] -a22 = ["22 Val-08: Still", val_img_list[2], still] -a23 = ["23 Val-09: Still", val_img_list[3], still] -a24 = ["24 Val-06: key+camera excl rand Frame 4", val_img_list[4], key_camera_excl_1_action_rand_1_f4] -a25 = ["25 Val-07: key+camera excl rand Frame 4", val_img_list[5], key_camera_excl_1_action_rand_2_f4] -a26 = ["26 Train-00", train_img_list[0], train_action_list[0]] -a27 = ["27 Train-01", train_img_list[1], train_action_list[1]] -# a28 = ["28 Train-02", train_img_list[2], train_action_list[2]] -# a29 = ["29 Train-03", train_img_list[3], train_action_list[3]] -# a30 = ["30 Train-04", train_img_list[4], train_action_list[4]] -# a31 = ["31 Train-05", train_img_list[5], train_action_list[5]] -a28 = ["28 Doom-00: W", val_img_doom_list[0], key_W] -a29 = ["29 Doom-01: key rand", val_img_doom_list[1], key_1_action_rand_1] -a30 = ["30 Doom-02: camera rand", val_img_doom_list[2], camera_1_action_rand_1] -a31 = ["31 Doom-03: key+camera excl rand", val_img_doom_list[3], key_camera_excl_1_action_rand_1] - -Val_entries = { - 0: a0, - 1: a1, - 2: a2, - 3: a3, - 4: a4, - 5: a5, - 6: a6, - 7: a7, - 8: a8, - 9: a9, - 10: a10, - 11: a11, - 12: a12, - 13: a13, - 14: a14, - 15: a15, - 16: a16, - 17: a17, - 18: a18, - 19: a19, - 20: a20, - 21: a21, - 22: a22, - 23: a23, - 24: a24, - 25: a25, - 26: a26, - 27: a27, - 28: a28, - 29: a29, - 30: a30, - 31: a31, -} - -data = [] -for idx in range(32): - if idx not in Val_entries: - raise ValueError(f"Missing entry for idx {idx}") - caption, image_path, action_path = Val_entries[idx] - data.append( - { - "caption": caption, - "image_path": image_path, - "action_path": action_path, - **fixed_fields, - } - ) - -output = {"data": data} -with open(output_path, "w") as f: - json.dump(output, f, indent=4) - -print(f"Generated {len(data)} entries to {output_path}") - -# Check file all exists - -with open(output_path) as f: - data = json.load(f) - -missing = [] -for i, item in enumerate(data['data']): - for key in ('image_path', 'action_path'): - path = item.get(key) - if path: - import os - if not os.path.isfile(path): - missing.append((i, key, path)) -if missing: - print('Missing paths:') - for idx, key, path in missing: - print(f' [{idx}] {key}: {path}') -else: - print('All paths exist.') - - diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py deleted file mode 100644 index 10e285750..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_static_w.py +++ /dev/null @@ -1,71 +0,0 @@ -import json -import os - -# Paths for two image directories -image_dir_val = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/first_frame" -image_dir_train = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_same_1st_frame_static_plus_w_only/first_frame" - -# Action paths (used for both) -action_still = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/videos/000000_action.npy" -action_w = "/mnt/weka/home/hao.zhang/kaiqin/traindata_0205_1330/data/0_static_plus_w_only/videos/001050_action.npy" - -# Output path -output_path = "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_static_w.json" - -# Fixed fields -fixed_fields = { - "video_path": None, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 -} - -data = [] - -# 16 images from each directory, alternating: val (0,1), train (2,3), val (4,5), train (6,7), ... -for i in range(16): - # Val images: indices 0,1, 4,5, 8,9, ... (pair index 0, 2, 4, ...) - image_path_val = os.path.join(image_dir_val, f"{i:06d}.png") - - # Still action for val - data.append({ - "caption": f"val {i:02d} - Still", - "image_path": image_path_val, - "action_path": action_still, - **fixed_fields - }) - - # W action for val - data.append({ - "caption": f"val {i:02d} - W", - "image_path": image_path_val, - "action_path": action_w, - **fixed_fields - }) - - # Train images: indices 2,3, 6,7, 10,11, ... (pair index 1, 3, 5, ...) - image_path_train = os.path.join(image_dir_train, f"{i:06d}.png") - - # Still action for train - data.append({ - "caption": f"train {i:02d} - Still", - "image_path": image_path_train, - "action_path": action_still, - **fixed_fields - }) - - # W action for train - data.append({ - "caption": f"train {i:02d} - W", - "image_path": image_path_train, - "action_path": action_w, - **fixed_fields - }) - -# Write to file -output = {"data": data} -with open(output_path, "w") as f: - json.dump(output, f, indent=4) - -print(f"Generated {len(data)} entries to {output_path}") diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_to_shao.py b/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_to_shao.py deleted file mode 100644 index 9074c5e25..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/scripts/generate_validation_to_shao.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate a validation JSON where all image_path and action_path entries come from -examples/training/finetune/WanGame2.1_1.3b_i2v/to_shao. - -Scans each subfolder of to_shao for pairs (NN.jpg, NN_action.npy) and emits one -validation entry per pair. Uses the same fixed_fields as generate_validation.py. -""" -import json -import os - -# Paths relative to this script / repo -FINETUNE_DIR = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -) -TO_SHAO_DIR = os.path.join(FINETUNE_DIR, "to_shao") -OUTPUT_PATH = os.path.join(FINETUNE_DIR, "validation_to_shao.json") - -# Same fixed fields as generate_validation.py -FIXED_FIELDS = { - "video_path": None, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77, -} - - -def collect_samples_from_to_shao(): - """Yield (caption, image_path, action_path) for each sample under to_shao.""" - if not os.path.isdir(TO_SHAO_DIR): - raise FileNotFoundError(f"to_shao directory not found: {TO_SHAO_DIR}") - - for subdir_name in sorted(os.listdir(TO_SHAO_DIR)): - subdir = os.path.join(TO_SHAO_DIR, subdir_name) - if not os.path.isdir(subdir): - continue - # Find all NN.jpg (or NN.jpeg) and matching NN_action.npy - for f in sorted(os.listdir(subdir)): - if f.endswith(".jpg") or f.endswith(".jpeg"): - base = f[: -4] if f.endswith(".jpg") else f[:-5] - action_name = f"{base}_action.npy" - action_path = os.path.join(subdir, action_name) - if not os.path.isfile(action_path): - continue - image_path = os.path.join(subdir, f) - caption = f"to_shao/{subdir_name}/{base}" - yield caption, image_path, action_path - - -def main(): - data = [] - for caption, image_path, action_path in collect_samples_from_to_shao(): - data.append( - { - "caption": caption, - "image_path": image_path, - "action_path": action_path, - **FIXED_FIELDS, - } - ) - - output = {"data": data} - with open(OUTPUT_PATH, "w") as f: - json.dump(output, f, indent=4) - - print(f"Generated {len(data)} entries to {OUTPUT_PATH}") - - # Check all paths exist - missing = [] - with open(OUTPUT_PATH) as f: - loaded = json.load(f) - for i, item in enumerate(loaded["data"]): - for key in ("image_path", "action_path"): - path = item.get(key) - if path and not os.path.isfile(path): - missing.append((i, key, path)) - if missing: - print("Missing paths:") - for idx, key, path in missing: - print(f" [{idx}] {key}: {path}") - else: - print("All paths exist.") - - -if __name__ == "__main__": - main() diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json deleted file mode 100644 index 5af4cca74..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation.json +++ /dev/null @@ -1,404 +0,0 @@ -{ - "data": [ - { - "caption": "0", - "image_path": "../../../../mc_wasd_10/validate/000000.jpg", - "action_path": "../../../../mc_wasd_10/videos/000000_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "1", - "image_path": "../../../../mc_wasd_10/validate/000001.jpg", - "action_path": "../../../../mc_wasd_10/videos/000001_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "2", - "image_path": "../../../../mc_wasd_10/validate/000002.jpg", - "action_path": "../../../../mc_wasd_10/videos/000002_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "3", - "image_path": "../../../../mc_wasd_10/validate/000003.jpg", - "action_path": "../../../../mc_wasd_10/videos/000003_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "4", - "image_path": "../../../../mc_wasd_10/validate/000004.jpg", - "action_path": "../../../../mc_wasd_10/videos/000004_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "5", - "image_path": "../../../../mc_wasd_10/validate/000005.jpg", - "action_path": "../../../../mc_wasd_10/videos/000005_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "6", - "image_path": "../../../../mc_wasd_10/validate/000006.jpg", - "action_path": "../../../../mc_wasd_10/videos/000006_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "7", - "image_path": "../../../../mc_wasd_10/validate/000007.jpg", - "action_path": "../../../../mc_wasd_10/videos/000007_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "00. Hold [W] + Static", - "image_path": "../../../../mc_wasd_10/validate/000000.jpg", - "action_path": "action/000000_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "01. Hold [S] + Static", - "image_path": "../../../../mc_wasd_10/validate/000001.jpg", - "action_path": "action/000001_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "02. Hold [A] + Static", - "image_path": "../../../../mc_wasd_10/validate/000002.jpg", - "action_path": "action/000002_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "03. Hold [D] + Static", - "image_path": "../../../../mc_wasd_10/validate/000003.jpg", - "action_path": "action/000003_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "04. Hold [WA] + Static", - "image_path": "../../../../mc_wasd_10/validate/000004.jpg", - "action_path": "action/000004_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "05. Hold [WD] + Static", - "image_path": "../../../../mc_wasd_10/validate/000005.jpg", - "action_path": "action/000005_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "06. Hold [SA] + Static", - "image_path": "../../../../mc_wasd_10/validate/000006.jpg", - "action_path": "action/000006_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "07. Hold [SD] + Static", - "image_path": "../../../../mc_wasd_10/validate/000007.jpg", - "action_path": "action/000007_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "08. No Key + Hold [up]", - "image_path": "../../../../mc_wasd_10/validate/000000.jpg", - "action_path": "action/000008_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "09. No Key + Hold [down]", - "image_path": "../../../../mc_wasd_10/validate/000001.jpg", - "action_path": "action/000009_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "10. No Key + Hold [left]", - "image_path": "../../../../mc_wasd_10/validate/000002.jpg", - "action_path": "action/000010_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "11. No Key + Hold [right]", - "image_path": "../../../../mc_wasd_10/validate/000003.jpg", - "action_path": "action/000011_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "12. No Key + Hold [up_right]", - "image_path": "../../../../mc_wasd_10/validate/000004.jpg", - "action_path": "action/000012_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "13. No Key + Hold [up_left]", - "image_path": "../../../../mc_wasd_10/validate/000005.jpg", - "action_path": "action/000013_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "14. No Key + Hold [down_right]", - "image_path": "../../../../mc_wasd_10/validate/000006.jpg", - "action_path": "action/000014_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "15. No Key + Hold [down_left]", - "image_path": "../../../../mc_wasd_10/validate/000007.jpg", - "action_path": "action/000015_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "00. Hold [W] + Static", - "image_path": "../../../../mc_wasd_10/validate/gen_000000.jpg", - "action_path": "action/000000_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "01. Hold [S] + Static", - "image_path": "../../../../mc_wasd_10/validate/gen_000001.jpg", - "action_path": "action/000001_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "02. Hold [A] + Static", - "image_path": "../../../../mc_wasd_10/validate/gen_000002.jpg", - "action_path": "action/000002_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "03. Hold [D] + Static", - "image_path": "../../../../mc_wasd_10/validate/gen_000003.jpg", - "action_path": "action/000003_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "04. Hold [WA] + Static", - "image_path": "../../../../mc_wasd_10/validate/gen_000004.jpg", - "action_path": "action/000004_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "05. Hold [WD] + Static", - "image_path": "../../../../mc_wasd_10/validate/gen_000005.jpg", - "action_path": "action/000005_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "06. Hold [SA] + Static", - "image_path": "../../../../mc_wasd_10/validate/gen_000006.jpg", - "action_path": "action/000006_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "07. Hold [SD] + Static", - "image_path": "../../../../mc_wasd_10/validate/gen_000007.jpg", - "action_path": "action/000007_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "08. No Key + Hold [up]", - "image_path": "../../../../mc_wasd_10/validate/gen_000000.jpg", - "action_path": "action/000008_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "09. No Key + Hold [down]", - "image_path": "../../../../mc_wasd_10/validate/gen_000001.jpg", - "action_path": "action/000009_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "10. No Key + Hold [left]", - "image_path": "../../../../mc_wasd_10/validate/gen_000002.jpg", - "action_path": "action/000010_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "11. No Key + Hold [right]", - "image_path": "../../../../mc_wasd_10/validate/gen_000003.jpg", - "action_path": "action/000011_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "12. No Key + Hold [up_right]", - "image_path": "../../../../mc_wasd_10/validate/gen_000004.jpg", - "action_path": "action/000012_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "13. No Key + Hold [up_left]", - "image_path": "../../../../mc_wasd_10/validate/gen_000005.jpg", - "action_path": "action/000013_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "14. No Key + Hold [down_right]", - "image_path": "../../../../mc_wasd_10/validate/gen_000006.jpg", - "action_path": "action/000014_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "15. No Key + Hold [down_left]", - "image_path": "../../../../mc_wasd_10/validate/gen_000007.jpg", - "action_path": "action/000015_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - } - ] -} \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json deleted file mode 100644 index 41e51fc79..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random.json +++ /dev/null @@ -1,324 +0,0 @@ -{ - "data": [ - { - "caption": "00 Val-00: W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "01 Val-01: S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "02 Val-02: A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "03 Val-03: D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "04 Val-04: u", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/u.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "05 Val-05: d", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "06 Val-06: l", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/l.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "07 Val-07: r", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/r.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "08 Val-00: key rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "09 Val-01: key rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_2.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "10 Val-02: camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "11 Val-03: camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_2.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "12 Val-00: key+camera excl rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "13 Val-01: key+camera excl rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "14 Val-02: key+camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "15 Val-03: key+camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_1_action_rand_2.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "16 Val-04: (simultaneous) key rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_2_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "17 Val-05: (simultaneous) camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_2_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "18 Val-06: (simultaneous) key+camera excl rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_2_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "19 Val-07: (simultaneous) key+camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_2_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "20 Val-08: W+A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/WA.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "21 Val-09: S+u", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S_u.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "22 Val-08: Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "23 Val-09: Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/humanplay/000013.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "24 Val-06: key+camera excl rand Frame 4", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1_f4.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "25 Val-07: key+camera excl rand Frame 4", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_2_f4.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "26 Train-00", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/first_frame/000500.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/videos/000500_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "27 Train-01", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/first_frame/001000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/traindata_0208_2000/data/wasd4holdrandview_simple_1key1mouse1/videos/001000_action.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "28 Doom-00: W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "29 Doom-01: key rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "30 Doom-02: camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/camera_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "31 Doom-03: key+camera excl rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/key_camera_excl_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - } - ] -} \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json deleted file mode 100644 index 0bde93715..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_random_8.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "data": [ - { - "caption": "00 Val-00: W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/W.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "01 Val-01: S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/S.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "02 Val-02: A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/A.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "03 Val-03: D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000005.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/D.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "04 Val-04: u", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/u.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "05 Val-05: d", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "06 Val-06: l", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000006.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/l.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - }, - { - "caption": "07 Val-07: r", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/mc_wasd_10/validate/000007.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions/r.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 352, - "width": 640, - "num_frames": 77 - } - ] -} \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_zelda.json b/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_zelda.json deleted file mode 100644 index 8ad289afe..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v/validation_zelda.json +++ /dev/null @@ -1,324 +0,0 @@ -{ - "data": [ - { - "caption": "00 Val-00: W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0006/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "01 Val-01: S", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0067/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "02 Val-02: A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0484/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/A.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "03 Val-03: D", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0019/segment0004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/D.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "04 Val-04: u", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0140/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/u.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "05 Val-05: d", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0300/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/d.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "06 Val-06: l", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0140/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/l.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "07 Val-07: r", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0300/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/r.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "08 Val-00: key rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0006/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "09 Val-01: key rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0067/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_2.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "10 Val-02: camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0484/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "11 Val-03: camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0019/segment0004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_2.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "12 Val-00: key+camera excl rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0006/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "13 Val-01: key+camera excl rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0067/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_2.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "14 Val-02: key+camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0484/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "15 Val-03: key+camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0019/segment0004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_1_action_rand_2.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "16 Val-04: (simultaneous) key rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0140/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_2_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "17 Val-05: (simultaneous) camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0300/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_2_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "18 Val-06: (simultaneous) key+camera excl rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0300/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_2_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "19 Val-07: (simultaneous) key+camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0300/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_2_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "20 Val-08: W+A", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0006/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/WA.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "21 Val-09: S+u", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0067/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/S_u.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "22 Val-08: Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0484/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "23 Val-09: Still", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0019/segment0004.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/still.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "24 Val-06: key+camera excl rand Frame 4", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0140/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_1_f4.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "25 Val-07: key+camera excl rand Frame 4", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/N6ObBAt41bg_chunk_0300/segment0003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_2_f4.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "26 Train-00", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0006/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0006/postprocess/action/majority_voting/81_frame_no_button/segment0002.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "27 Train-01", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0067/segment0002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/zelda/5TTrlqAguhQ_chunk_0067/postprocess/action/majority_voting/81_frame_no_button/segment0002.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "28 Doom-00: W", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000000.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/W.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "29 Doom-01: key rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000001.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "30 Doom-02: camera rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000002.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/camera_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - }, - { - "caption": "31 Doom-03: key+camera excl rand", - "image_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/doom/000003.jpg", - "action_path": "/mnt/weka/home/hao.zhang/mhuo/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v/actions_81/key_camera_excl_1_action_rand_1.npy", - "video_path": null, - "num_inference_steps": 40, - "height": 480, - "width": 832, - "num_frames": 81 - } - ] -} \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/README.md b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/README.md deleted file mode 100644 index cdddcfab6..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/action/README.md +++ /dev/null @@ -1,18 +0,0 @@ -Total Files: 16 - -00. Hold [W] + Static -01. Hold [S] + Static -02. Hold [A] + Static -03. Hold [D] + Static -04. Hold [WA] + Static -05. Hold [WD] + Static -06. Hold [SA] + Static -07. Hold [SD] + Static -08. No Key + Hold [up] -09. No Key + Hold [down] -10. No Key + Hold [left] -11. No Key + Hold [right] -12. No Key + Hold [up_right] -13. No Key + Hold [up_left] -14. No Key + Hold [down_right] -15. No Key + Hold [down_left] diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.sh b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.sh deleted file mode 100644 index dc66efdac..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -export WANDB_API_KEY="7ff8b6e8356924f7a6dd51a0342dd1a422ea9352" -export WANDB_BASE_URL="https://api.wandb.ai" -# export WANDB_MODE=online -export WANDB_MODE=offline -export TOKENIZERS_PARALLELISM=false -export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN -export PYTHONPATH=$PYTHONPATH:$(pwd) - -MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -DATA_DIR="../traindata_0205_1330/data/0_static_plus_w_only/preprocessed" -VALIDATION_DATASET_FILE="$(dirname "$0")/validation.json" -NUM_GPUS=1 -# export CUDA_VISIBLE_DEVICES=0,1,2,3 -# IP=[MASTER NODE IP] - -source ~/conda/miniconda/bin/activate -conda activate /mnt/weka/home/hao.zhang/conda/miniconda/envs/mhuo-fv -export PYTHONPATH="/mnt/weka/home/hao.zhang/kaiqin/FastVideo:$PYTHONPATH" - -# Training arguments -training_args=( - --override-pipeline-cls-name "WanLingBotImageToVideoPipeline" - --override-transformer-cls-name "WanLingBotTransformer3DModel" - --tracker_project_name "wangame_lingbot_test" - --output_dir "wangame_lingbot_test" - --max_train_steps 100 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 20 - --num_height 352 - --num_width 640 - --num_frames 77 - --enable_gradient_checkpointing_type "full" -) - -# Parallel arguments -parallel_args=( - --num_gpus $NUM_GPUS - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim $NUM_GPUS -) - -# Model arguments -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -# Dataset arguments -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 1 -) - -# Validation arguments -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 100 - --validation_sampling_steps "40" - --validation_guidance_scale "1.0" -) - -# Optimizer arguments -optimizer_args=( - --learning_rate 2e-5 - --mixed_precision "bf16" - --weight_only_checkpointing_steps 1000 - --training_state_checkpointing_steps 1000 - --weight_decay 1e-4 - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.1 - --multi_phased_distill_schedule "4000-1" - --not_apply_cfg_solver - --dit_precision "fp32" - --num_euler_timesteps 50 - --ema_start_step 0 -) - -# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t -torchrun \ - --nnodes 1 \ - --nproc_per_node $NUM_GPUS \ - fastvideo/training/wangame_lingbot_training_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.slurm deleted file mode 100644 index d40029c7e..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/finetune_i2v.slurm +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=wangame_1.3b_overfit -#SBATCH --partition=main -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=128 -#SBATCH --mem=1440G -#SBATCH --output=wangame_1.3b_overfit_output/wangame_1.3b_overfit_%j.out -#SBATCH --error=wangame_1.3b_overfit_output/wangame_1.3b_overfit_%j.err -#SBATCH --exclusive - -# Basic Info -export NCCL_P2P_DISABLE=1 -export TORCH_NCCL_ENABLE_MONITORING=0 -export NCCL_DEBUG_SUBSYS=INIT,NET -# different cache dir for different processes -export TRITON_CACHE_DIR=/tmp/triton_cache_${SLURM_PROCID} -export MASTER_PORT=29500 -export NODE_RANK=$SLURM_PROCID -nodes=( $(scontrol show hostnames $SLURM_JOB_NODELIST) ) -export MASTER_ADDR=${nodes[0]} -export TOKENIZERS_PARALLELISM=false -# export WANDB_API_KEY="8d9f4b39abd68eb4e29f6fc010b7ee71a2207cde" -export WANDB_API_KEY="50632ebd88ffd970521cec9ab4a1a2d7e85bfc45" -# export WANDB_API_KEY='your_wandb_api_key_here' -export WANDB_BASE_URL="https://api.wandb.ai" -export WANDB_MODE=online -export FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN - -source ~/conda/miniconda/bin/activate -conda activate wei-fv-distill -export HOME="/mnt/weka/home/hao.zhang/wei" - -MODEL_PATH="weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers" -DATA_DIR="mc_wasd_10/preprocessed/combined_parquet_dataset" -VALIDATION_DATASET_FILE="examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation.json" -# Configs -NUM_GPUS=8 - -# Training arguments -training_args=( - --tracker_project_name "wangame_1.3b_overfit" - --output_dir "wangame_1.3b_overfit" - --max_train_steps 15000 - --train_batch_size 1 - --train_sp_batch_size 1 - --gradient_accumulation_steps 1 - --num_latent_t 20 - --num_height 352 - --num_width 640 - --num_frames 77 - --enable_gradient_checkpointing_type "full" -) - -# Parallel arguments -parallel_args=( - --num_gpus $NUM_GPUS - --sp_size 1 - --tp_size 1 - --hsdp_replicate_dim 1 - --hsdp_shard_dim $NUM_GPUS -) - -# Model arguments -model_args=( - --model_path $MODEL_PATH - --pretrained_model_name_or_path $MODEL_PATH -) - -# Dataset arguments -dataset_args=( - --data_path "$DATA_DIR" - --dataloader_num_workers 1 -) - -# Validation arguments -validation_args=( - --log_validation - --validation_dataset_file "$VALIDATION_DATASET_FILE" - --validation_steps 100 - --validation_sampling_steps "40" - --validation_guidance_scale "1.0" -) - -# Optimizer arguments -optimizer_args=( - --learning_rate 2e-5 - --mixed_precision "bf16" - --weight_only_checkpointing_steps 1000000 - --training_state_checkpointing_steps 10000000 - --weight_decay 1e-4 - --max_grad_norm 1.0 -) - -# Miscellaneous arguments -miscellaneous_args=( - --inference_mode False - --checkpoints_total_limit 3 - --training_cfg_rate 0.1 - --multi_phased_distill_schedule "4000-1" - --not_apply_cfg_solver - --dit_precision "fp32" - --num_euler_timesteps 50 - --ema_start_step 0 -) - -# If you do not have 32 GPUs and to fit in memory, you can: 1. increase sp_size. 2. reduce num_latent_t -torchrun \ - --nnodes 1 \ - --nproc_per_node $NUM_GPUS \ - fastvideo/training/wangame_lingbot_training_pipeline.py \ - "${parallel_args[@]}" \ - "${model_args[@]}" \ - "${dataset_args[@]}" \ - "${training_args[@]}" \ - "${optimizer_args[@]}" \ - "${validation_args[@]}" \ - "${miscellaneous_args[@]}" \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/generate_actions.py b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/generate_actions.py deleted file mode 100644 index edc3ad6dd..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/generate_actions.py +++ /dev/null @@ -1,193 +0,0 @@ -import os -import numpy as np - -# Configuration -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -BASE_OUTPUT_DIR = os.path.join(SCRIPT_DIR, 'action') -VIDEO_OUTPUT_DIR = BASE_OUTPUT_DIR -os.makedirs(VIDEO_OUTPUT_DIR, exist_ok=True) - -FRAME_COUNT = 81 -CAM_VALUE = 0.1 - -# Action Mapping -KEY_TO_INDEX = { - 'W': 0, 'S': 1, 'A': 2, 'D': 3, -} - -VIEW_ACTION_TO_MOUSE = { - "stop": [0.0, 0.0], - "up": [CAM_VALUE, 0.0], - "down": [-CAM_VALUE, 0.0], - "left": [0.0, -CAM_VALUE], - "right": [0.0, CAM_VALUE], - "up_right": [CAM_VALUE, CAM_VALUE], - "up_left": [CAM_VALUE, -CAM_VALUE], - "down_right": [-CAM_VALUE, CAM_VALUE], - "down_left": [-CAM_VALUE, -CAM_VALUE], -} - -def get_multihot_vector(keys_str): - """Convert string like 'WA' to [1, 0, 1, 0, 0, 0]""" - vector = [0.0] * 6 - if not keys_str: - return vector - for char in keys_str.upper(): - if char in KEY_TO_INDEX: - vector[KEY_TO_INDEX[char]] = 1.0 - return vector - -def get_mouse_vector(view_str): - """Convert view string to [x, y]""" - return VIEW_ACTION_TO_MOUSE.get(view_str.lower(), [0.0, 0.0]) - -def generate_sequence(key_seq, mouse_seq): - """ - Generates action arrays based on sequences. - """ - keyboard_arr = np.zeros((FRAME_COUNT, 6), dtype=np.float32) - mouse_arr = np.zeros((FRAME_COUNT, 2), dtype=np.float32) - - mid_point = FRAME_COUNT // 2 - - # First Half - k_vec1 = get_multihot_vector(key_seq[0]) - m_vec1 = get_mouse_vector(mouse_seq[0]) - keyboard_arr[:mid_point] = k_vec1 - mouse_arr[:mid_point] = m_vec1 - - # Second Half - k_vec2 = get_multihot_vector(key_seq[1]) - m_vec2 = get_mouse_vector(mouse_seq[1]) - keyboard_arr[mid_point:] = k_vec2 - mouse_arr[mid_point:] = m_vec2 - - return keyboard_arr, mouse_arr - -def save_action(index, keyboard_arr, mouse_arr): - filename = f"{index:06d}_action.npy" - filepath = os.path.join(VIDEO_OUTPUT_DIR, filename) - - action_dict = { - 'keyboard': keyboard_arr, - 'mouse': mouse_arr - } - np.save(filepath, action_dict) - return filename - -def generate_description(key_seq, mouse_seq): - """Generates a human-readable string for the combination.""" - k1, k2 = key_seq - m1, m2 = mouse_seq - - # Format Keyboard Description - if not k1 and not k2: - k_desc = "No Key" - elif k1 == k2: - k_desc = f"Hold [{k1}]" - else: - k_desc = f"Switch [{k1}]->[{k2}]" - - # Format Mouse Description - if m1 == "stop" and m2 == "stop": - m_desc = "Static" - elif m1 == m2: - m_desc = f"Hold [{m1}]" - else: - m_desc = f"Switch [{m1}]->[{m2}]" - - return f"{k_desc} + {m_desc}" - -# ========================================== -# Main Generation Logic -# ========================================== - -configs = [] -readme_content = [] - -# Group 1: Constant Keyboard, No Mouse (0-7) -keys_basic = ['W', 'S', 'A', 'D', 'WA', 'WD', 'SA', 'SD'] -for k in keys_basic: - configs.append(((k, k), ("stop", "stop"))) - -# Group 2: No Keyboard, Constant Mouse (8-15) -mouse_basic = ['up', 'down', 'left', 'right', 'up_right', 'up_left', 'down_right', 'down_left'] -for m in mouse_basic: - configs.append((("", ""), (m, m))) - -# Group 3: Split Keyboard, No Mouse (16-23) -split_keys = [ - ('W', 'S'), ('S', 'W'), - ('A', 'D'), ('D', 'A'), - ('W', 'A'), ('W', 'D'), - ('S', 'A'), ('S', 'D') -] -for k1, k2 in split_keys: - configs.append(((k1, k2), ("stop", "stop"))) - -# Group 4: No Keyboard, Split Mouse (24-31) -split_mouse = [ - ('left', 'right'), ('right', 'left'), - ('up', 'down'), ('down', 'up'), - ('up_left', 'up_right'), ('up_right', 'up_left'), - ('left', 'up'), ('right', 'down') -] -for m1, m2 in split_mouse: - configs.append((("", ""), (m1, m2))) - -# Group 5: Constant Keyboard + Constant Mouse (32-47) -combo_keys = ['W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD', 'W', 'S', 'W', 'S', 'A', 'D', 'WA', 'WD'] -combo_mice = ['left', 'left', 'right', 'right', 'up', 'up', 'down', 'down', 'up_left', 'up_left', 'up_right', 'up_right', 'down_left', 'down_right', 'right', 'left'] -for i in range(16): - configs.append(((combo_keys[i], combo_keys[i]), (combo_mice[i], combo_mice[i]))) - -# Group 6: Constant Keyboard, Split Mouse (48-55) -complex_1_keys = ['W'] * 8 -complex_1_mice = [ - ('left', 'right'), ('right', 'left'), - ('up', 'down'), ('down', 'up'), - ('left', 'up'), ('right', 'up'), - ('left', 'down'), ('right', 'down') -] -for i in range(8): - configs.append(((complex_1_keys[i], complex_1_keys[i]), complex_1_mice[i])) - -# Group 7: Split Keyboard, Constant Mouse (56-63) -complex_2_keys = [ - ('W', 'S'), ('S', 'W'), - ('A', 'D'), ('D', 'A'), - ('W', 'A'), ('W', 'D'), - ('S', 'A'), ('S', 'D') -] -complex_2_mouse = 'up' -for k1, k2 in complex_2_keys: - configs.append(((k1, k2), (complex_2_mouse, complex_2_mouse))) - - -# Execution -print(f"Preparing to generate {len(configs)} action files...") - -for i, (key_seq, mouse_seq) in enumerate(configs): - if i >= 16: break - - # Generate Data - kb_arr, ms_arr = generate_sequence(key_seq, mouse_seq) - filename = save_action(i, kb_arr, ms_arr) - - # Generate Description for README - description = generate_description(key_seq, mouse_seq) - readme_entry = f"{i:02d}. {description}" - readme_content.append(readme_entry) - - print(f"Generated {filename} -> {description}") - -# Write README -readme_path = os.path.join(BASE_OUTPUT_DIR, 'README.md') -with open(readme_path, 'w', encoding='utf-8') as f: - f.write(f"Total Files: {len(readme_content)}\n\n") - for line in readme_content: - f.write(line + '\n') - -print(f"\nProcessing complete.") -print(f"64 .npy files generated in {VIDEO_OUTPUT_DIR}") -print(f"Manifest saved to {readme_path}") \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/launch_preprocess_slurm.sh b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/launch_preprocess_slurm.sh deleted file mode 100644 index c1bc37b5d..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/launch_preprocess_slurm.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Create output directory if it doesn't exist -mkdir -p preprocess_output - -# Launch 8 jobs, one for each node (Total 64 GPUs) -# Each node processes 8 consecutive files (64 total files / 8 nodes = 8 files per node) -for node_id in {0..7}; do - # Calculate the starting file number for this node - start_file=$((node_id * 8)) - - echo "Launching node $node_id with files merge_${start_file}.txt to merge_$((start_file + 7)).txt" - - sbatch --job-name=mg-pre-${node_id} \ - --output=preprocess_output/mg-node-${node_id}.out \ - --error=preprocess_output/mg-node-${node_id}.err \ - $(pwd)/FastVideo/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm $start_file $node_id -done - -echo "All 8 nodes (64 GPUs) launched successfully!" diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_wangame_data_i2v.sh b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_wangame_data_i2v.sh deleted file mode 100644 index 85a4fd0d2..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_wangame_data_i2v.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -GPU_NUM=1 # 2,4,8 -MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" -DATA_MERGE_PATH="mc_wasd_10/merge.txt" -OUTPUT_DIR="mc_wasd_10/preprocessed/" - -# export CUDA_VISIBLE_DEVICES=0 -export MASTER_ADDR=localhost -export MASTER_PORT=29500 -export RANK=0 -export WORLD_SIZE=1 - -python fastvideo/pipelines/preprocess/v1_preprocess.py \ - --model_path $MODEL_PATH \ - --data_merge_path $DATA_MERGE_PATH \ - --preprocess_video_batch_size 10 \ - --seed 42 \ - --max_height 352 \ - --max_width 640 \ - --num_frames 77 \ - --dataloader_num_workers 0 \ - --output_dir=$OUTPUT_DIR \ - --samples_per_file 10 \ - --train_fps 25 \ - --flush_frequency 10 \ - --preprocess_task wangame \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm deleted file mode 100644 index f60659a7b..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/preprocess_worker.slurm +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -#SBATCH --partition=main -#SBATCH --qos=hao -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=8 -#SBATCH --gres=gpu:8 -#SBATCH --cpus-per-task=16 -#SBATCH --mem=960G -#SBATCH --exclusive -#SBATCH --time=72:00:00 - -# conda init -source ~/conda/miniconda/bin/activate -conda activate fastvideo_kaiqin - -# Accept parameters from launch script -START_FILE=${1:-1} # Starting file number for this node -NODE_ID=${2:-0} # Node identifier (0-7) - -MODEL_PATH="weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers" -OUTPUT_BASE="traindata_0204_2130/preprocessed" - -# Port range calculation -base_port=$((29500 + NODE_ID * 100)) -gpu_ids=(0 1 2 3 4 5 6 7) - -for i in {1..8}; do - port=$((base_port + i)) - gpu=${gpu_ids[((i-1))]} - file_num=$((START_FILE + i - 1)) - - DATA_MERGE_PATH="traindata_0204_2130/merge_${file_num}.txt" - OUTPUT_DIR="${OUTPUT_BASE}/gpu_${gpu}_file_${file_num}" - echo "DATA_MERGE_PATH: $DATA_MERGE_PATH" - echo "OUTPUT_DIR: $OUTPUT_DIR" - - # CPU binding (optional, kept from syn.slurm logic) - start_cpu=$(( (i-1)*2 )) - end_cpu=$(( start_cpu+1 )) - - echo "Starting GPU $gpu processing file merge_${file_num}.txt on port $port" - - CUDA_VISIBLE_DEVICES=$gpu taskset -c ${start_cpu}-${end_cpu} torchrun --nnodes=1 --nproc_per_node=1 --master_port $port \ - FastVideo/fastvideo/pipelines/preprocess/v1_preprocess.py \ - --model_path $MODEL_PATH \ - --data_merge_path $DATA_MERGE_PATH \ - --preprocess_video_batch_size 1 \ - --seed 42 \ - --max_height 352 \ - --max_width 640 \ - --num_frames 77 \ - --dataloader_num_workers 0 \ - --output_dir=$OUTPUT_DIR \ - --samples_per_file 8 \ - --train_fps 25 \ - --flush_frequency 8 \ - --preprocess_task wangame & -done - -wait -echo "Node $NODE_ID processing blocks completed!" diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation.json b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation.json deleted file mode 100644 index 3a4a47115..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "data": [ - { - "caption": "00. Hold [W] + Static", - "image_path": "doom/000000.jpg", - "action_path": "action/000000_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 77 - } - ] -} \ No newline at end of file diff --git a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation_vizdoom.json b/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation_vizdoom.json deleted file mode 100644 index 1535db987..000000000 --- a/examples/training/finetune/WanGame2.1_1.3b_i2v_LingBot/validation_vizdoom.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "data": [ - { - "caption": "00. Hold [W] + Static", - "image_path": "doom/000000.jpg", - "action_path": "action/000000_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "01. Hold [S] + Static", - "image_path": "doom/000001.jpg", - "action_path": "action/000001_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "02. Hold [A] + Static", - "image_path": "doom/000002.jpg", - "action_path": "action/000002_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "10. No Key + Hold [left]", - "image_path": "doom/000003.jpg", - "action_path": "action/000010_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "12. No Key + Hold [up_right]", - "image_path": "doom/000004.jpg", - "action_path": "action/000012_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "21. Switch [W]->[D] + Static", - "image_path": "doom/000005.jpg", - "action_path": "action/000021_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "29. No Key + Switch [up_right]->[up_left]", - "image_path": "doom/000006.jpg", - "action_path": "action/000029_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - }, - { - "caption": "50. Hold [W] + Switch [up]->[down]", - "image_path": "doom/000007.jpg", - "action_path": "action/000050_action.npy", - "video_path": null, - "num_inference_steps": 4, - "height": 352, - "width": 640, - "num_frames": 81 - } - ] -} \ No newline at end of file diff --git a/fastvideo/configs/models/dits/wangamevideo.py b/fastvideo/configs/models/dits/wangamevideo.py deleted file mode 100644 index 429d7407c..000000000 --- a/fastvideo/configs/models/dits/wangamevideo.py +++ /dev/null @@ -1,122 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -from dataclasses import dataclass, field - -from fastvideo.configs.models.dits.base import DiTArchConfig, DiTConfig - - -def is_blocks(n: str, m) -> bool: - return "blocks" in n and str.isdigit(n.split(".")[-1]) - - -@dataclass -class WanGameVideoArchConfig(DiTArchConfig): - _fsdp_shard_conditions: list = field(default_factory=lambda: [is_blocks]) - - param_names_mapping: dict = field( - default_factory=lambda: { - r"^patch_embedding\.(.*)$": - r"patch_embedding.proj.\1", - r"^condition_embedder\.text_embedder\.linear_1\.(.*)$": - r"condition_embedder.text_embedder.fc_in.\1", - r"^condition_embedder\.text_embedder\.linear_2\.(.*)$": - r"condition_embedder.text_embedder.fc_out.\1", - r"^condition_embedder\.time_embedder\.linear_1\.(.*)$": - r"condition_embedder.time_embedder.mlp.fc_in.\1", - r"^condition_embedder\.time_embedder\.linear_2\.(.*)$": - r"condition_embedder.time_embedder.mlp.fc_out.\1", - r"^condition_embedder\.time_proj\.(.*)$": - r"condition_embedder.time_modulation.linear.\1", - r"^condition_embedder\.image_embedder\.ff\.net\.0\.proj\.(.*)$": - r"condition_embedder.image_embedder.ff.fc_in.\1", - r"^condition_embedder\.image_embedder\.ff\.net\.2\.(.*)$": - r"condition_embedder.image_embedder.ff.fc_out.\1", - r"^blocks\.(\d+)\.attn1\.to_q\.(.*)$": - r"blocks.\1.to_q.\2", - r"^blocks\.(\d+)\.attn1\.to_k\.(.*)$": - r"blocks.\1.to_k.\2", - r"^blocks\.(\d+)\.attn1\.to_v\.(.*)$": - r"blocks.\1.to_v.\2", - r"^blocks\.(\d+)\.attn1\.to_out\.0\.(.*)$": - r"blocks.\1.to_out.\2", - r"^blocks\.(\d+)\.attn1\.norm_q\.(.*)$": - r"blocks.\1.norm_q.\2", - r"^blocks\.(\d+)\.attn1\.norm_k\.(.*)$": - r"blocks.\1.norm_k.\2", - r"^blocks\.(\d+)\.attn2\.to_out\.0\.(.*)$": - r"blocks.\1.attn2.to_out.\2", - r"^blocks\.(\d+)\.ffn\.net\.0\.proj\.(.*)$": - r"blocks.\1.ffn.fc_in.\2", - r"^blocks\.(\d+)\.ffn\.net\.2\.(.*)$": - r"blocks.\1.ffn.fc_out.\2", - r"^blocks\.(\d+)\.norm2\.(.*)$": - r"blocks.\1.self_attn_residual_norm.norm.\2", - }) - - # Reverse mapping for saving checkpoints: custom -> hf - reverse_param_names_mapping: dict = field(default_factory=lambda: {}) - - # Some LoRA adapters use the original official layer names instead of hf layer names, - # so apply this before the param_names_mapping - lora_param_names_mapping: dict = field( - default_factory=lambda: { - r"^blocks\.(\d+)\.self_attn\.q\.(.*)$": r"blocks.\1.attn1.to_q.\2", - r"^blocks\.(\d+)\.self_attn\.k\.(.*)$": r"blocks.\1.attn1.to_k.\2", - r"^blocks\.(\d+)\.self_attn\.v\.(.*)$": r"blocks.\1.attn1.to_v.\2", - r"^blocks\.(\d+)\.self_attn\.o\.(.*)$": - r"blocks.\1.attn1.to_out.0.\2", - r"^blocks\.(\d+)\.cross_attn\.q\.(.*)$": r"blocks.\1.attn2.to_q.\2", - r"^blocks\.(\d+)\.cross_attn\.k\.(.*)$": r"blocks.\1.attn2.to_k.\2", - r"^blocks\.(\d+)\.cross_attn\.v\.(.*)$": r"blocks.\1.attn2.to_v.\2", - r"^blocks\.(\d+)\.cross_attn\.o\.(.*)$": - r"blocks.\1.attn2.to_out.0.\2", - r"^blocks\.(\d+)\.ffn\.0\.(.*)$": r"blocks.\1.ffn.fc_in.\2", - r"^blocks\.(\d+)\.ffn\.2\.(.*)$": r"blocks.\1.ffn.fc_out.\2", - }) - - patch_size: tuple[int, int, int] = (1, 2, 2) - text_len = 512 - num_attention_heads: int = 40 - attention_head_dim: int = 128 - in_channels: int = 16 - out_channels: int = 16 - text_dim: int = 4096 - freq_dim: int = 256 - ffn_dim: int = 13824 - num_layers: int = 40 - cross_attn_norm: bool = True - qk_norm: str = "rms_norm_across_heads" - eps: float = 1e-6 - image_dim: int | None = None - added_kv_proj_dim: int | None = None - rope_max_seq_len: int = 1024 - pos_embed_seq_len: int | None = None - exclude_lora_layers: list[str] = field(default_factory=lambda: ["embedder"]) - - # Wan MoE - boundary_ratio: float | None = None - - # Causal Wan - local_attn_size: int = -1 # Window size for temporal local attention (-1 indicates global attention) - sink_size: int = 0 # Size of the attention sink, we keep the first `sink_size` frames unchanged when rolling the KV cache - num_frames_per_block: int = 3 - sliding_window_num_frames: int = 21 - - def __post_init__(self): - super().__post_init__() - self.out_channels = self.out_channels or self.in_channels - self.hidden_size = self.num_attention_heads * self.attention_head_dim - self.num_channels_latents = self.out_channels - - -@dataclass -class WanGameVideoConfig(DiTConfig): - arch_config: DiTArchConfig = field(default_factory=WanGameVideoArchConfig) - - prefix: str = "WanGame" - - -@dataclass -class WanLingBotVideoConfig(DiTConfig): - arch_config: DiTArchConfig = field(default_factory=WanGameVideoArchConfig) - - prefix: str = "WanLingBot" diff --git a/fastvideo/models/dits/wangame/__init__.py b/fastvideo/models/dits/wangame/__init__.py deleted file mode 100644 index 1d799e506..000000000 --- a/fastvideo/models/dits/wangame/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .model import WanGameActionTransformer3DModel -from .causal_model import CausalWanGameActionTransformer3DModel -from .hyworld_action_module import WanGameActionTimeImageEmbedding, WanGameActionSelfAttention - -__all__ = [ - "WanGameActionTransformer3DModel", - "CausalWanGameActionTransformer3DModel", - "WanGameActionTimeImageEmbedding", - "WanGameActionSelfAttention", -] diff --git a/fastvideo/models/dits/wangame/causal_model.py b/fastvideo/models/dits/wangame/causal_model.py deleted file mode 100644 index 64bfefafc..000000000 --- a/fastvideo/models/dits/wangame/causal_model.py +++ /dev/null @@ -1,856 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -import math -from typing import Any - -import torch -import torch.nn as nn - -from torch.nn.attention.flex_attention import create_block_mask, flex_attention -from torch.nn.attention.flex_attention import BlockMask -# wan 1.3B model has a weird channel / head configurations and require max-autotune to work with flexattention -# see https://github.com/pytorch/pytorch/issues/133254 -# change to default for other models -flex_attention = torch.compile( - flex_attention, dynamic=False, mode="max-autotune-no-cudagraphs") -import torch.distributed as dist - -from fastvideo.attention import LocalAttention -from fastvideo.configs.models.dits.wangamevideo import WanGameVideoConfig -from fastvideo.distributed.parallel_state import get_sp_world_size -from fastvideo.layers.layernorm import (FP32LayerNorm, LayerNormScaleShift, - RMSNorm, ScaleResidual, - ScaleResidualLayerNormScaleShift) -from fastvideo.layers.linear import ReplicatedLinear -from fastvideo.layers.mlp import MLP -from fastvideo.layers.rotary_embedding import (_apply_rotary_emb, - get_rotary_pos_embed) -from fastvideo.layers.visual_embedding import PatchEmbed -from fastvideo.logger import init_logger -from fastvideo.models.dits.base import BaseDiT -from fastvideo.models.dits.wanvideo import WanI2VCrossAttention -from fastvideo.platforms import AttentionBackendEnum, current_platform - -# Import ActionModule -from fastvideo.models.dits.wangame.hyworld_action_module import ( - WanGameActionTimeImageEmbedding, - WanGameActionSelfAttention -) -from fastvideo.models.dits.hyworld.camera_rope import prope_qkv - -logger = init_logger(__name__) - - -class CausalWanGameCrossAttention(WanI2VCrossAttention): - """Cross-attention for WanGame causal model""" - - def forward(self, x, context, context_lens=None, crossattn_cache=None): - r""" - Args: - x(Tensor): Shape [B, L1, C] - context(Tensor): Shape [B, L2, C] - context_lens(Tensor): Shape [B] - crossattn_cache: Optional cache dict for inference - """ - context_img = context - b, n, d = x.size(0), self.num_heads, self.head_dim - - # compute query, key, value - q = self.norm_q(self.to_q(x)[0]).view(b, -1, n, d) - - if crossattn_cache is not None: - if not crossattn_cache["is_init"]: - crossattn_cache["is_init"] = True - k_img = self.norm_added_k(self.add_k_proj(context_img)[0]).view( - b, -1, n, d) - v_img = self.add_v_proj(context_img)[0].view(b, -1, n, d) - crossattn_cache["k"] = k_img - crossattn_cache["v"] = v_img - else: - k_img = crossattn_cache["k"] - v_img = crossattn_cache["v"] - else: - k_img = self.norm_added_k(self.add_k_proj(context_img)[0]).view( - b, -1, n, d) - v_img = self.add_v_proj(context_img)[0].view(b, -1, n, d) - - img_x = self.attn(q, k_img, v_img) - - # output - x = img_x.flatten(2) - x, _ = self.to_out(x) - return x - - -class CausalWanGameActionSelfAttention(WanGameActionSelfAttention): - - def __init__(self, - dim: int, - num_heads: int, - local_attn_size: int = -1, - sink_size: int = 0, - qk_norm=True, - eps=1e-6) -> None: - super().__init__( - dim=dim, - num_heads=num_heads, - local_attn_size=local_attn_size, - sink_size=sink_size, - qk_norm=qk_norm, - eps=eps, - ) - self.max_attention_size = 32760 if local_attn_size == -1 else local_attn_size * 1560 - - # Local attention for KV-cache inference - self.local_attn = LocalAttention( - num_heads=num_heads, - head_size=self.head_dim, - dropout_rate=0, - softmax_scale=None, - causal=False, - supported_attention_backends=(AttentionBackendEnum.FLASH_ATTN, - AttentionBackendEnum.TORCH_SDPA)) - - @staticmethod - def _masked_flex_attn( - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - block_mask: BlockMask, - ) -> torch.Tensor: - padded_length = math.ceil(query.shape[1] / 128) * 128 - query.shape[1] - if padded_length > 0: - query = torch.cat( - [ - query, - torch.zeros( - [query.shape[0], padded_length, query.shape[2], query.shape[3]], - device=query.device, - dtype=value.dtype, - ), - ], - dim=1, - ) - key = torch.cat( - [ - key, - torch.zeros( - [key.shape[0], padded_length, key.shape[2], key.shape[3]], - device=key.device, - dtype=value.dtype, - ), - ], - dim=1, - ) - value = torch.cat( - [ - value, - torch.zeros( - [value.shape[0], padded_length, value.shape[2], value.shape[3]], - device=value.device, - dtype=value.dtype, - ), - ], - dim=1, - ) - - out = flex_attention( - query=query.transpose(2, 1), - key=key.transpose(2, 1), - value=value.transpose(2, 1), - block_mask=block_mask, - ).transpose(2, 1) - - if padded_length > 0: - out = out[:, :-padded_length] - return out - - def forward(self, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - freqs_cis: tuple[torch.Tensor, torch.Tensor], - block_mask: BlockMask | None = None, - kv_cache: dict | None = None, - current_start: int = 0, - cache_start: int | None = None, - viewmats: torch.Tensor | None = None, - Ks: torch.Tensor | None = None, - is_cache: bool = False): - """ - Forward pass with causal attention. - """ - if cache_start is None: - cache_start = current_start - - if kv_cache is None: - if block_mask is None: - raise ValueError( - "block_mask must be provided for causal training attention") - if viewmats is None or Ks is None: - raise ValueError( - "viewmats and Ks must be provided for WanGame causal attention") - - cos, sin = freqs_cis - query_rope = _apply_rotary_emb( - q, cos, sin, is_neox_style=False).type_as(v) - key_rope = _apply_rotary_emb( - k, cos, sin, is_neox_style=False).type_as(v) - rope_output = self._masked_flex_attn( - query_rope, key_rope, v, block_mask) - - # PRoPE path with the same causal mask. - query_prope, key_prope, value_prope, apply_fn_o = prope_qkv( - q.transpose(1, 2), - k.transpose(1, 2), - v.transpose(1, 2), - viewmats=viewmats, - Ks=Ks, - patches_x=40, - patches_y=22, - ) - query_prope = query_prope.transpose(1, 2) - key_prope = key_prope.transpose(1, 2) - value_prope = value_prope.transpose(1, 2) - prope_output = self._masked_flex_attn( - query_prope, key_prope, value_prope, block_mask) - prope_output = apply_fn_o( - prope_output.transpose(1, 2)).transpose(1, 2) - - return rope_output, prope_output - else: - # Inference mode with KV cache - if viewmats is None or Ks is None: - raise ValueError( - "viewmats and Ks must be provided for WanGame causal attention") - - cos, sin = freqs_cis - roped_query = _apply_rotary_emb(q, cos, sin, is_neox_style=False).type_as(v) - roped_key = _apply_rotary_emb(k, cos, sin, is_neox_style=False).type_as(v) - query_prope, key_prope, value_prope, apply_fn_o = prope_qkv( - q.transpose(1, 2), - k.transpose(1, 2), - v.transpose(1, 2), - viewmats=viewmats, - Ks=Ks, - patches_x=40, - patches_y=22, - ) - query_prope = query_prope.transpose(1, 2).type_as(v) - key_prope = key_prope.transpose(1, 2).type_as(v) - value_prope = value_prope.transpose(1, 2).type_as(v) - - frame_seqlen = q.shape[1] - current_end = current_start + roped_query.shape[1] - sink_tokens = self.sink_size * frame_seqlen - # If we are using local attention and the current KV cache size is larger than the local attention size, we need to truncate the KV cache - kv_cache_size = kv_cache["k"].shape[1] - num_new_tokens = roped_query.shape[1] - - # rope+prope - cache_head_dim = kv_cache["k"].shape[-1] - local_end_index = kv_cache["local_end_index"].item() - - # read cache but never mutate it. - if not is_cache: - if cache_head_dim not in (self.head_dim, self.head_dim * 2): - raise ValueError( - f"Unexpected kv_cache head dim: {cache_head_dim}, " - f"expected {self.head_dim} or {self.head_dim * 2}") - - cache_k_rope = kv_cache["k"][..., :self.head_dim] - cache_v_rope = kv_cache["v"][..., :self.head_dim] - rope_k = torch.cat( - [cache_k_rope[:, :local_end_index], roped_key], dim=1) - rope_v = torch.cat( - [cache_v_rope[:, :local_end_index], v], dim=1) - rope_k = rope_k[:, -self.max_attention_size:] - rope_v = rope_v[:, -self.max_attention_size:] - rope_x = self.local_attn(roped_query, rope_k, rope_v) - - if cache_head_dim == self.head_dim * 2: - cache_k_prope = kv_cache["k"][..., self.head_dim:] - cache_v_prope = kv_cache["v"][..., self.head_dim:] - prope_k = torch.cat( - [cache_k_prope[:, :local_end_index], key_prope], dim=1) - prope_v = torch.cat( - [cache_v_prope[:, :local_end_index], value_prope], dim=1) - prope_k = prope_k[:, -self.max_attention_size:] - prope_v = prope_v[:, -self.max_attention_size:] - prope_x = self.local_attn(query_prope, prope_k, prope_v) - else: - prope_x = self.local_attn( - query_prope, key_prope, value_prope) - - prope_x = apply_fn_o(prope_x.transpose(1, 2)).transpose(1, 2) - return rope_x, prope_x - - # update cache. - if cache_head_dim == self.head_dim: - kv_cache["k"] = torch.cat( - [kv_cache["k"], torch.zeros_like(kv_cache["k"])], dim=-1) - kv_cache["v"] = torch.cat( - [kv_cache["v"], torch.zeros_like(kv_cache["v"])], dim=-1) - elif cache_head_dim != self.head_dim * 2: - raise ValueError( - f"Unexpected kv_cache head dim: {cache_head_dim}, " - f"expected {self.head_dim} or {self.head_dim * 2}") - - cache_k_rope = kv_cache["k"][..., :self.head_dim] - cache_k_prope = kv_cache["k"][..., self.head_dim:] - cache_v_rope = kv_cache["v"][..., :self.head_dim] - cache_v_prope = kv_cache["v"][..., self.head_dim:] - - if self.local_attn_size != -1 and (current_end > kv_cache["global_end_index"].item()) and ( - num_new_tokens + kv_cache["local_end_index"].item() > kv_cache_size): - # Calculate the number of new tokens added in this step - # Shift existing cache content left to discard oldest tokens - # Clone the source slice to avoid overlapping memory error - num_evicted_tokens = num_new_tokens + kv_cache["local_end_index"].item() - kv_cache_size - num_rolled_tokens = kv_cache["local_end_index"].item() - num_evicted_tokens - sink_tokens - cache_k_rope[:, sink_tokens:sink_tokens + num_rolled_tokens] = \ - cache_k_rope[:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() - cache_v_rope[:, sink_tokens:sink_tokens + num_rolled_tokens] = \ - cache_v_rope[:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() - cache_k_prope[:, sink_tokens:sink_tokens + num_rolled_tokens] = \ - cache_k_prope[:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() - cache_v_prope[:, sink_tokens:sink_tokens + num_rolled_tokens] = \ - cache_v_prope[:, sink_tokens + num_evicted_tokens:sink_tokens + num_evicted_tokens + num_rolled_tokens].clone() - # Insert the new keys/values at the end - local_end_index = kv_cache["local_end_index"].item() + current_end - \ - kv_cache["global_end_index"].item() - num_evicted_tokens - local_start_index = local_end_index - num_new_tokens - cache_k_rope[:, local_start_index:local_end_index] = roped_key - cache_v_rope[:, local_start_index:local_end_index] = v - cache_k_prope[:, local_start_index:local_end_index] = key_prope - cache_v_prope[:, local_start_index:local_end_index] = value_prope - else: - # Assign new keys/values directly up to current_end - local_end_index = kv_cache["local_end_index"].item() + current_end - kv_cache["global_end_index"].item() - local_start_index = local_end_index - num_new_tokens - kv_cache["k"] = kv_cache["k"].detach() - kv_cache["v"] = kv_cache["v"].detach() - cache_k_rope = kv_cache["k"][..., :self.head_dim] - cache_k_prope = kv_cache["k"][..., self.head_dim:] - cache_v_rope = kv_cache["v"][..., :self.head_dim] - cache_v_prope = kv_cache["v"][..., self.head_dim:] - # logger.info("kv_cache['k'] is in comp graph: %s", kv_cache["k"].requires_grad or kv_cache["k"].grad_fn is not None) - cache_k_rope[:, local_start_index:local_end_index] = roped_key - cache_v_rope[:, local_start_index:local_end_index] = v - cache_k_prope[:, local_start_index:local_end_index] = key_prope - cache_v_prope[:, local_start_index:local_end_index] = value_prope - - rope_x = self.local_attn( - roped_query, - cache_k_rope[:, max(0, local_end_index - self.max_attention_size):local_end_index], - cache_v_rope[:, max(0, local_end_index - self.max_attention_size):local_end_index] - ) - prope_x = self.local_attn( - query_prope, - cache_k_prope[:, max(0, local_end_index - self.max_attention_size):local_end_index], - cache_v_prope[:, max(0, local_end_index - self.max_attention_size):local_end_index] - ) - prope_x = apply_fn_o(prope_x.transpose(1, 2)).transpose(1, 2) - kv_cache["global_end_index"].fill_(current_end) - kv_cache["local_end_index"].fill_(local_end_index) - - return rope_x, prope_x - - -class CausalWanGameActionTransformerBlock(nn.Module): - - def __init__(self, - dim: int, - ffn_dim: int, - num_heads: int, - local_attn_size: int = -1, - sink_size: int = 0, - qk_norm: str = "rms_norm_across_heads", - cross_attn_norm: bool = False, - eps: float = 1e-6, - added_kv_proj_dim: int | None = None, - supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None, - prefix: str = ""): - super().__init__() - - # 1. Self-attention - self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False) - self.to_q = ReplicatedLinear(dim, dim, bias=True) - self.to_k = ReplicatedLinear(dim, dim, bias=True) - self.to_v = ReplicatedLinear(dim, dim, bias=True) - self.to_out = ReplicatedLinear(dim, dim, bias=True) - - self.attn1 = CausalWanGameActionSelfAttention( - dim, - num_heads, - local_attn_size=local_attn_size, - sink_size=sink_size, - qk_norm=qk_norm, - eps=eps) - - self.hidden_dim = dim - self.num_attention_heads = num_heads - self.local_attn_size = local_attn_size - dim_head = dim // num_heads - - if qk_norm == "rms_norm": - self.norm_q = RMSNorm(dim_head, eps=eps) - self.norm_k = RMSNorm(dim_head, eps=eps) - elif qk_norm == "rms_norm_across_heads": - self.norm_q = RMSNorm(dim, eps=eps) - self.norm_k = RMSNorm(dim, eps=eps) - else: - raise ValueError(f"QK Norm type {qk_norm} not supported") - - assert cross_attn_norm is True - self.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( - dim, - norm_type="layer", - eps=eps, - elementwise_affine=True, - compute_dtype=torch.float32) - - # 2. Cross-attention (I2V only) - self.attn2 = CausalWanGameCrossAttention(dim, - num_heads, - qk_norm=qk_norm, - eps=eps) - # norm3 for FFN input - self.norm3 = LayerNormScaleShift(dim, norm_type="layer", eps=eps, - elementwise_affine=False) - - # 3. Feed-forward - self.ffn = MLP(dim, ffn_dim, act_type="gelu_pytorch_tanh") - self.mlp_residual = ScaleResidual() - - self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) - - # PRoPE output projection (initialized via add_discrete_action_parameters on the model) - self.to_out_prope = ReplicatedLinear(dim, dim, bias=True) - nn.init.zeros_(self.to_out_prope.weight) - if self.to_out_prope.bias is not None: - nn.init.zeros_(self.to_out_prope.bias) - - def forward( - self, - hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor, - temb: torch.Tensor, - freqs_cis: tuple[torch.Tensor, torch.Tensor], - block_mask: BlockMask | None = None, - kv_cache: dict | None = None, - crossattn_cache: dict | None = None, - current_start: int = 0, - cache_start: int | None = None, - viewmats: torch.Tensor | None = None, - Ks: torch.Tensor | None = None, - is_cache: bool = False, - ) -> torch.Tensor: - if hidden_states.dim() == 4: - hidden_states = hidden_states.squeeze(1) - - num_frames = temb.shape[1] - frame_seqlen = hidden_states.shape[1] // num_frames - bs, seq_length, _ = hidden_states.shape - orig_dtype = hidden_states.dtype - - # Cast temb to float32 for scale/shift computation - e = self.scale_shift_table + temb.float() - assert e.shape == (bs, num_frames, 6, self.hidden_dim) - shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(6, dim=2) - - # 1. Self-attention - norm_hidden_states = (self.norm1(hidden_states.float()).unflatten(dim=1, sizes=(num_frames, frame_seqlen)) * - (1 + scale_msa) + shift_msa).to(orig_dtype).flatten(1, 2) - - query, _ = self.to_q(norm_hidden_states) - key, _ = self.to_k(norm_hidden_states) - value, _ = self.to_v(norm_hidden_states) - - if self.norm_q is not None: - query = self.norm_q.forward_native(query) - if self.norm_k is not None: - key = self.norm_k.forward_native(key) - - query = query.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - - # Self-attention with camera PRoPE - attn_output_rope, attn_output_prope = self.attn1( - query, key, value, freqs_cis, - block_mask, kv_cache, current_start, cache_start, - viewmats, Ks, is_cache=is_cache - ) - # Combine rope and prope outputs - attn_output_rope = attn_output_rope.flatten(2) - attn_output_rope, _ = self.to_out(attn_output_rope) - attn_output_prope = attn_output_prope.flatten(2) - attn_output_prope, _ = self.to_out_prope(attn_output_prope) - attn_output = attn_output_rope.squeeze(1) + attn_output_prope.squeeze(1) - - # Self-attention residual + norm in float32 - null_shift = null_scale = torch.zeros(1, device=hidden_states.device, dtype=torch.float32) - norm_hidden_states, hidden_states = self.self_attn_residual_norm( - hidden_states.float(), attn_output.float(), gate_msa, null_shift, null_scale) - hidden_states = hidden_states.type_as(attn_output) - norm_hidden_states = norm_hidden_states.type_as(attn_output) - - # 2. Cross-attention - attn_output = self.attn2(norm_hidden_states.to(orig_dtype), - context=encoder_hidden_states, - context_lens=None, - crossattn_cache=crossattn_cache) - # Cross-attention residual in bfloat16 - hidden_states = hidden_states + attn_output - - # norm3 for FFN input in float32 - norm_hidden_states = self.norm3( - hidden_states.float(), c_shift_msa, c_scale_msa - ).type_as(hidden_states) - - # 3. Feed-forward - ff_output = self.ffn(norm_hidden_states.to(orig_dtype)) - hidden_states = self.mlp_residual(hidden_states.float(), ff_output.float(), c_gate_msa) - hidden_states = hidden_states.to(orig_dtype) - - return hidden_states - - -class CausalWanGameActionTransformer3DModel(BaseDiT): - - _fsdp_shard_conditions = WanGameVideoConfig()._fsdp_shard_conditions - _compile_conditions = WanGameVideoConfig()._compile_conditions - _supported_attention_backends = WanGameVideoConfig()._supported_attention_backends - param_names_mapping = WanGameVideoConfig().param_names_mapping - reverse_param_names_mapping = WanGameVideoConfig().reverse_param_names_mapping - lora_param_names_mapping = WanGameVideoConfig().lora_param_names_mapping - - def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> None: - super().__init__(config=config, hf_config=hf_config) - - inner_dim = config.num_attention_heads * config.attention_head_dim - self.hidden_size = config.hidden_size - self.num_attention_heads = config.num_attention_heads - self.attention_head_dim = config.attention_head_dim - self.in_channels = config.in_channels - self.out_channels = config.out_channels - self.num_channels_latents = config.num_channels_latents - self.patch_size = config.patch_size - self.local_attn_size = config.local_attn_size - self.inner_dim = inner_dim - - # 1. Patch & position embedding - self.patch_embedding = PatchEmbed(in_chans=config.in_channels, - embed_dim=inner_dim, - patch_size=config.patch_size, - flatten=False) - - # 2. Condition embeddings - self.condition_embedder = WanGameActionTimeImageEmbedding( - dim=inner_dim, - time_freq_dim=config.freq_dim, - image_embed_dim=config.image_dim, - ) - - # 3. Transformer blocks - self.blocks = nn.ModuleList([ - CausalWanGameActionTransformerBlock( - inner_dim, - config.ffn_dim, - config.num_attention_heads, - config.local_attn_size, - config.sink_size, - config.qk_norm, - config.cross_attn_norm, - config.eps, - config.added_kv_proj_dim, - supported_attention_backends=self._supported_attention_backends, - prefix=f"{config.prefix}.blocks.{i}") - for i in range(config.num_layers) - ]) - - # 4. Output norm & projection - self.norm_out = LayerNormScaleShift(inner_dim, - norm_type="layer", - eps=config.eps, - elementwise_affine=False, - dtype=torch.float32) - self.proj_out = nn.Linear( - inner_dim, config.out_channels * math.prod(config.patch_size)) - self.scale_shift_table = nn.Parameter(torch.randn(1, 2, inner_dim) / inner_dim**0.5) - - self.gradient_checkpointing = False - - # Causal-specific - self.block_mask = None - self.num_frame_per_block = config.arch_config.num_frames_per_block - assert self.num_frame_per_block <= 3 - - self.__post_init__() - - @staticmethod - def _prepare_blockwise_causal_attn_mask( - device: torch.device | str, num_frames: int = 21, - frame_seqlen: int = 1560, num_frame_per_block=1, local_attn_size=-1 - ) -> BlockMask: - """ - we will divide the token sequence into the following format - [1 latent frame] [1 latent frame] ... [1 latent frame] - We use flexattention to construct the attention mask - """ - total_length = num_frames * frame_seqlen - - # we do right padding to get to a multiple of 128 - padded_length = math.ceil(total_length / 128) * 128 - total_length - - ends = torch.zeros(total_length + padded_length, - device=device, dtype=torch.long) - - # Block-wise causal mask will attend to all elements that are before the end of the current chunk - frame_indices = torch.arange( - start=0, - end=total_length, - step=frame_seqlen * num_frame_per_block, - device=device - ) - - for tmp in frame_indices: - ends[tmp:tmp + frame_seqlen * num_frame_per_block] = tmp + \ - frame_seqlen * num_frame_per_block - - def attention_mask(b, h, q_idx, kv_idx): - if local_attn_size == -1: - return (kv_idx < ends[q_idx]) | (q_idx == kv_idx) - else: - return ((kv_idx < ends[q_idx]) & (kv_idx >= (ends[q_idx] - local_attn_size * frame_seqlen))) | (q_idx == kv_idx) - - block_mask = create_block_mask(attention_mask, B=None, H=None, Q_LEN=total_length + padded_length, - KV_LEN=total_length + padded_length, _compile=False, device=device) - - if not dist.is_initialized() or dist.get_rank() == 0: - print( - f" cache a block wise causal mask with block size of {num_frame_per_block} frames") - print(block_mask) - - return block_mask - - def _forward_inference( - self, - hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor | list[torch.Tensor], - timestep: torch.LongTensor, - encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, - guidance=None, - action: torch.Tensor | None = None, - viewmats: torch.Tensor | None = None, - Ks: torch.Tensor | None = None, - kv_cache: list[dict] | None = None, - crossattn_cache: list[dict] | None = None, - current_start: int = 0, - cache_start: int = 0, - start_frame: int = 0, - is_cache: bool = False, - **kwargs - ) -> torch.Tensor: - r""" - Run the diffusion model with kv caching. - See Algorithm 2 of CausVid paper https://arxiv.org/abs/2412.07772 for details. - This function will be run for num_frame times. - Process the latent frames one by one (1560 tokens each) - """ - orig_dtype = hidden_states.dtype - if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: - encoder_hidden_states_image = encoder_hidden_states_image[0] - - batch_size, num_channels, num_frames, height, width = hidden_states.shape - p_t, p_h, p_w = self.patch_size - post_patch_num_frames = num_frames // p_t - post_patch_height = height // p_h - post_patch_width = width // p_w - - # Get rotary embeddings - d = self.hidden_size // self.num_attention_heads - rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] - freqs_cos, freqs_sin = get_rotary_pos_embed( - (post_patch_num_frames * get_sp_world_size(), post_patch_height, post_patch_width), - self.hidden_size, - self.num_attention_heads, - rope_dim_list, - dtype=torch.float32 if current_platform.is_mps() else torch.float64, - rope_theta=10000, - start_frame=start_frame - ) - freqs_cos = freqs_cos.to(hidden_states.device) - freqs_sin = freqs_sin.to(hidden_states.device) - freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None - - hidden_states = self.patch_embedding(hidden_states) - hidden_states = hidden_states.flatten(2).transpose(1, 2) - - if timestep.dim() == 2: - timestep = timestep.flatten() - - temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( - timestep, action, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) - - # condition_embedder returns: - # - temb: [B*T, dim] where T = post_patch_num_frames - # - timestep_proj: [B*T, 6*dim] - # Reshape to [B, T, 6, dim] for transformer blocks - timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) # [B*T, 6, dim] - timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, self.hidden_size) # [B, T, 6, dim] - - encoder_hidden_states = encoder_hidden_states_image - - # Transformer blocks - for block_idx, block in enumerate(self.blocks): - if torch.is_grad_enabled() and self.gradient_checkpointing: - hidden_states = self._gradient_checkpointing_func( - block, hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, - self.block_mask, - kv_cache[block_idx] if kv_cache else None, - crossattn_cache[block_idx] if crossattn_cache else None, - current_start, cache_start, - viewmats, Ks, is_cache) - else: - hidden_states = block( - hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, - block_mask=self.block_mask, - kv_cache=kv_cache[block_idx] if kv_cache else None, - crossattn_cache=crossattn_cache[block_idx] if crossattn_cache else None, - current_start=current_start, cache_start=cache_start, - viewmats=viewmats, Ks=Ks, is_cache=is_cache) - - # If cache-only mode, return early - if is_cache: - return kv_cache - - # Output norm, projection & unpatchify - # temb is [B*T, dim], reshape to [B, T, 1, dim] - temb = temb.view(batch_size, post_patch_num_frames, -1).unsqueeze(2) # [B, T, 1, dim] - - shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) - hidden_states = self.norm_out(hidden_states, shift, scale) - hidden_states = self.proj_out(hidden_states) - - hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, - post_patch_height, - post_patch_width, p_t, p_h, p_w, - -1) - hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) - output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) - - return output - - def _forward_train( - self, - hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor | list[torch.Tensor], - timestep: torch.LongTensor, - encoder_hidden_states_image: torch.Tensor | list[torch.Tensor] | None = None, - guidance=None, - action: torch.Tensor | None = None, - viewmats: torch.Tensor | None = None, - Ks: torch.Tensor | None = None, - start_frame: int = 0, - **kwargs - ) -> torch.Tensor: - - orig_dtype = hidden_states.dtype - if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: - encoder_hidden_states_image = encoder_hidden_states_image[0] - - batch_size, num_channels, num_frames, height, width = hidden_states.shape - p_t, p_h, p_w = self.patch_size - post_patch_num_frames = num_frames // p_t - post_patch_height = height // p_h - post_patch_width = width // p_w - - # Get rotary embeddings - d = self.hidden_size // self.num_attention_heads - rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] - freqs_cos, freqs_sin = get_rotary_pos_embed( - (post_patch_num_frames * get_sp_world_size(), post_patch_height, post_patch_width), - self.hidden_size, - self.num_attention_heads, - rope_dim_list, - dtype=torch.float32 if current_platform.is_mps() else torch.float64, - rope_theta=10000, - start_frame=start_frame - ) - freqs_cos = freqs_cos.to(hidden_states.device) - freqs_sin = freqs_sin.to(hidden_states.device) - freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None - - # Construct blockwise causal attn mask - if self.block_mask is None: - self.block_mask = self._prepare_blockwise_causal_attn_mask( - device=hidden_states.device, - num_frames=num_frames, - frame_seqlen=post_patch_height * post_patch_width, - num_frame_per_block=self.num_frame_per_block, - local_attn_size=self.local_attn_size - ) - - hidden_states = self.patch_embedding(hidden_states) - hidden_states = hidden_states.flatten(2).transpose(1, 2) - - if timestep.dim() == 2: - timestep = timestep.flatten() - - temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( - timestep, action, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) - - # condition_embedder returns: - # - temb: [B*T, dim] where T = post_patch_num_frames - # - timestep_proj: [B*T, 6*dim] - # Reshape to [B, T, 6, dim] for transformer blocks - timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) # [B*T, 6, dim] - timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, self.hidden_size) # [B, T, 6, dim] - - encoder_hidden_states = encoder_hidden_states_image - - # Transformer blocks - if torch.is_grad_enabled() and self.gradient_checkpointing: - for block in self.blocks: - hidden_states = self._gradient_checkpointing_func( - block, hidden_states, encoder_hidden_states, - timestep_proj, freqs_cis, - self.block_mask, - None, None, # kv_cache, crossattn_cache - 0, None, # current_start, cache_start - viewmats, Ks, False) # viewmats, Ks, is_cache - else: - for block in self.blocks: - hidden_states = block(hidden_states, encoder_hidden_states, - timestep_proj, freqs_cis, - block_mask=self.block_mask, - viewmats=viewmats, Ks=Ks) - - # Output norm, projection & unpatchify - # temb is [B*T, dim], reshape to [B, T, 1, dim] - temb = temb.view(batch_size, post_patch_num_frames, -1).unsqueeze(2) # [B, T, 1, dim] - - shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) - hidden_states = self.norm_out(hidden_states, shift, scale) - hidden_states = self.proj_out(hidden_states) - - hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, - post_patch_height, - post_patch_width, p_t, p_h, p_w, - -1) - hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) - output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) - - return output - - def forward( - self, - *args, - **kwargs - ): - if kwargs.get('kv_cache', None) is not None: - return self._forward_inference(*args, **kwargs) - else: - return self._forward_train(*args, **kwargs) diff --git a/fastvideo/models/dits/wangame/hyworld_action_module.py b/fastvideo/models/dits/wangame/hyworld_action_module.py deleted file mode 100644 index a0159d12f..000000000 --- a/fastvideo/models/dits/wangame/hyworld_action_module.py +++ /dev/null @@ -1,280 +0,0 @@ -import math - -import torch -import torch.nn as nn - -from fastvideo.layers.visual_embedding import TimestepEmbedder, ModulateProjection, timestep_embedding -from fastvideo.platforms import AttentionBackendEnum -from fastvideo.attention import DistributedAttention -from fastvideo.forward_context import set_forward_context -from fastvideo.models.dits.wanvideo import WanImageEmbedding - -from fastvideo.models.dits.hyworld.camera_rope import prope_qkv -from fastvideo.layers.rotary_embedding import _apply_rotary_emb -from fastvideo.layers.mlp import MLP - -class WanGameActionTimeImageEmbedding(nn.Module): - def __init__( - self, - dim: int, - time_freq_dim: int, - image_embed_dim: int | None = None, - ): - super().__init__() - - self.time_freq_dim = time_freq_dim - self.time_embedder = TimestepEmbedder( - dim, frequency_embedding_size=time_freq_dim, act_layer="silu") - self.time_modulation = ModulateProjection(dim, - factor=6, - act_layer="silu") - - self.image_embedder = None - if image_embed_dim is not None: - self.image_embedder = WanImageEmbedding(image_embed_dim, dim) - - self.action_embedder = MLP( - time_freq_dim, - dim, - dim, - bias=True, - act_type="silu" - ) - # Initialize fc_in with kaiming_uniform (same as nn.Linear default) - nn.init.kaiming_uniform_(self.action_embedder.fc_in.weight, a=math.sqrt(5)) - # Initialize fc_out with zeros for residual-like behavior - nn.init.zeros_(self.action_embedder.fc_out.weight) - if self.action_embedder.fc_out.bias is not None: - nn.init.zeros_(self.action_embedder.fc_out.bias) - - def forward( - self, - timestep: torch.Tensor, - action: torch.Tensor, - encoder_hidden_states: torch.Tensor, # Kept for interface compatibility - encoder_hidden_states_image: torch.Tensor | None = None, - timestep_seq_len: int | None = None, - ): - """ - Args: - timestep: [B] diffusion timesteps (one per batch sample) - action: [B, T] action labels (one per frame per batch sample) - - Returns: - temb: [B*T, dim] combined timestep + action embedding - timestep_proj: [B*T, 6*dim] modulation projection - """ - # timestep may be [B] (one per sample) or [B*T] (one per frame, from causal training) - temb = self.time_embedder(timestep, timestep_seq_len) - - # Handle action embedding for batch > 1 - # action shape: [B, T] where B=batch_size, T=num_frames - batch_size = action.shape[0] - num_frames = action.shape[1] - - # Compute action embeddings: [B, T] -> [B*T] -> [B*T, dim] - action_flat = action.flatten() # [B*T] - action_emb = timestep_embedding(action_flat, self.time_freq_dim) - action_embedder_dtype = next(iter(self.action_embedder.parameters())).dtype - if ( - action_emb.dtype != action_embedder_dtype - and action_embedder_dtype != torch.int8 - ): - action_emb = action_emb.to(action_embedder_dtype) - action_emb = self.action_embedder(action_emb).type_as(temb) # [B*T, dim] - - # temb is [B*T, dim] when timestep was already per-frame (causal training), - # or [B, dim] when timestep is per-sample (inference). - # Only expand if temb is per-sample [B, dim]. - if temb.shape[0] == batch_size and num_frames > 1: - # Expand temb: [B, dim] -> [B, T, dim] -> [B*T, dim] - temb_expanded = temb.unsqueeze(1).expand(-1, num_frames, -1) # [B, T, dim] - temb_expanded = temb_expanded.reshape(batch_size * num_frames, -1) # [B*T, dim] - else: - # temb is already [B*T, dim] (per-frame timesteps) - temb_expanded = temb - - # Add action embedding to expanded temb - temb = temb_expanded + action_emb # [B*T, dim] - - timestep_proj = self.time_modulation(temb) # [B*T, 6*dim] - - # MatrixGame does not use text embeddings, so we ignore encoder_hidden_states - - if encoder_hidden_states_image is not None: - assert self.image_embedder is not None - encoder_hidden_states_image = self.image_embedder( - encoder_hidden_states_image) - - encoder_hidden_states = torch.zeros((batch_size, 0, temb.shape[-1]), - device=temb.device, - dtype=temb.dtype) - - return temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image - -class WanGameActionSelfAttention(nn.Module): - """ - Self-attention module with support for: - - Standard RoPE-based attention - - Camera PRoPE-based attention (when viewmats and Ks are provided) - - KV caching for autoregressive generation - """ - - def __init__(self, - dim: int, - num_heads: int, - local_attn_size: int = -1, - sink_size: int = 0, - qk_norm=True, - eps=1e-6) -> None: - assert dim % num_heads == 0 - super().__init__() - self.dim = dim - self.num_heads = num_heads - self.head_dim = dim // num_heads - self.local_attn_size = local_attn_size - self.sink_size = sink_size - self.qk_norm = qk_norm - self.eps = eps - self.max_attention_size = 32760 if local_attn_size == -1 else local_attn_size * 1560 - - # Scaled dot product attention (using DistributedAttention for SP support) - self.attn = DistributedAttention( - num_heads=num_heads, - head_size=self.head_dim, - softmax_scale=None, - causal=False, - supported_attention_backends=(AttentionBackendEnum.FLASH_ATTN, - AttentionBackendEnum.TORCH_SDPA)) - - def forward(self, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - freqs_cis: tuple[torch.Tensor, torch.Tensor], - kv_cache: dict | None = None, - current_start: int = 0, - cache_start: int | None = None, - viewmats: torch.Tensor | None = None, - Ks: torch.Tensor | None = None, - is_cache: bool = False, - attention_mask: torch.Tensor | None = None): - """ - Forward pass with camera PRoPE attention combining standard RoPE and projective positional encoding. - - Args: - q, k, v: Query, key, value tensors [B, L, num_heads, head_dim] - freqs_cis: RoPE frequency cos/sin tensors - kv_cache: KV cache dict (may have None values for training) - current_start: Current position for KV cache - cache_start: Cache start position - viewmats: Camera view matrices for PRoPE [B, cameras, 4, 4] - Ks: Camera intrinsics for PRoPE [B, cameras, 3, 3] - is_cache: Whether to store to KV cache (for inference) - attention_mask: Attention mask [B, L] (1 = attend, 0 = mask) - """ - if cache_start is None: - cache_start = current_start - - # Apply RoPE manually - cos, sin = freqs_cis - query_rope = _apply_rotary_emb(q, cos, sin, is_neox_style=False).type_as(v) - key_rope = _apply_rotary_emb(k, cos, sin, is_neox_style=False).type_as(v) - value_rope = v - - # # DEBUG: Check camera matrices - # if self.training and torch.distributed.get_rank() == 0: - # vm_info = f"viewmats={viewmats.shape if viewmats is not None else None}" - # ks_info = f"Ks={Ks.shape if Ks is not None else None}" - # vm_nonzero = (viewmats != 0).sum().item() if viewmats is not None else 0 - # ks_nonzero = (Ks != 0).sum().item() if Ks is not None else 0 - # print(f"[DEBUG] PRoPE input: {vm_info} nonzero={vm_nonzero}, {ks_info} nonzero={ks_nonzero}", flush=True) - - # Get PRoPE transformed q, k, v - query_prope, key_prope, value_prope, apply_fn_o = prope_qkv( - q.transpose(1, 2), # [B, num_heads, L, head_dim] - k.transpose(1, 2), - v.transpose(1, 2), - viewmats=viewmats, - Ks=Ks, - patches_x=40, # hardcoded for now - patches_y=22, - ) - # PRoPE returns [B, num_heads, L, head_dim], convert to [B, L, num_heads, head_dim] - query_prope = query_prope.transpose(1, 2) - key_prope = key_prope.transpose(1, 2) - value_prope = value_prope.transpose(1, 2) - - # # DEBUG: Check prope_qkv output - # if self.training and torch.distributed.get_rank() == 0: - # q_nz = (query_prope != 0).sum().item() - # k_nz = (key_prope != 0).sum().item() - # v_nz = (value_prope != 0).sum().item() - # print(f"[DEBUG] prope_qkv output: q_nonzero={q_nz}, k_nonzero={k_nz}, v_nonzero={v_nz}", flush=True) - - # KV cache handling - if kv_cache is not None: - cache_key = kv_cache.get("k", None) - cache_value = kv_cache.get("v", None) - - if cache_value is not None and not is_cache: - cache_key_rope, cache_key_prope = cache_key.chunk(2, dim=-1) - cache_value_rope, cache_value_prope = cache_value.chunk(2, dim=-1) - - key_rope = torch.cat([cache_key_rope, key_rope], dim=1) - value_rope = torch.cat([cache_value_rope, value_rope], dim=1) - key_prope = torch.cat([cache_key_prope, key_prope], dim=1) - value_prope = torch.cat([cache_value_prope, value_prope], dim=1) - - if is_cache: - # Store to cache (update input dict directly) - kv_cache["k"] = torch.cat([key_rope, key_prope], dim=-1) - kv_cache["v"] = torch.cat([value_rope, value_prope], dim=-1) - - # Concatenate rope and prope paths (matching original) - query_all = torch.cat([query_rope, query_prope], dim=0) - key_all = torch.cat([key_rope, key_prope], dim=0) - value_all = torch.cat([value_rope, value_prope], dim=0) - - # Check if Q and KV have different sequence lengths (KV cache mode) - # In this case, use LocalAttention (supports different Q/KV lengths) - if query_all.shape[1] != key_all.shape[1]: - raise ValueError("Q and KV have different sequence lengths") - else: - # Same sequence length: use DistributedAttention (supports SP) - # Create default attention mask if not provided - # NOTE: query_all has shape [2*B, L, ...] (rope+prope concatenated), so mask needs 2*B - if attention_mask is None: - batch_size, seq_len = q.shape[0], q.shape[1] - attention_mask = torch.ones(batch_size * 2, seq_len, device=q.device, dtype=q.dtype) - - if q.dtype == torch.float32: - from fastvideo.attention.backends.sdpa import SDPAMetadataBuilder - attn_metadata_builder = SDPAMetadataBuilder - else: - from fastvideo.attention.backends.flash_attn import FlashAttnMetadataBuilder - attn_metadata_builder = FlashAttnMetadataBuilder - attn_metadata = attn_metadata_builder().build( - current_timestep=0, - attn_mask=attention_mask, - ) - with set_forward_context(current_timestep=0, attn_metadata=attn_metadata): - hidden_states_all, _ = self.attn(query_all, key_all, value_all, attention_mask=attention_mask) - - hidden_states_rope, hidden_states_prope = hidden_states_all.chunk(2, dim=0) - - # # DEBUG: Check attention output and apply_fn_o - # if self.training and torch.distributed.get_rank() == 0: - # attn_all_nz = (hidden_states_all != 0).sum().item() - # rope_nz = (hidden_states_rope != 0).sum().item() - # prope_before = (hidden_states_prope != 0).sum().item() - # print(f"[DEBUG] attn output: all_nonzero={attn_all_nz}, rope_nonzero={rope_nz}, prope_before_apply={prope_before}", flush=True) - - hidden_states_prope = apply_fn_o(hidden_states_prope.transpose(1, 2)).transpose(1, 2) - - # # DEBUG: Check after apply_fn_o - # if self.training and torch.distributed.get_rank() == 0: - # prope_after = (hidden_states_prope != 0).sum().item() - # print(f"[DEBUG] prope_after_apply_fn_o={prope_after}", flush=True) - - return hidden_states_rope, hidden_states_prope \ No newline at end of file diff --git a/fastvideo/models/dits/wangame/model.py b/fastvideo/models/dits/wangame/model.py deleted file mode 100644 index 61f539821..000000000 --- a/fastvideo/models/dits/wangame/model.py +++ /dev/null @@ -1,422 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -import math -from typing import Any - -import torch -import torch.nn as nn - -from fastvideo.configs.models.dits.wangamevideo import WanGameVideoConfig -from fastvideo.distributed.parallel_state import get_sp_world_size -from fastvideo.layers.layernorm import (FP32LayerNorm, LayerNormScaleShift, - RMSNorm, ScaleResidual, - ScaleResidualLayerNormScaleShift) -from fastvideo.layers.linear import ReplicatedLinear -from fastvideo.layers.mlp import MLP -from fastvideo.layers.rotary_embedding import (_apply_rotary_emb, - get_rotary_pos_embed) -from fastvideo.layers.visual_embedding import PatchEmbed -from fastvideo.logger import init_logger -from fastvideo.models.dits.base import BaseDiT -from fastvideo.models.dits.wanvideo import WanI2VCrossAttention -from fastvideo.platforms import AttentionBackendEnum, current_platform - -# Import ActionModule -from fastvideo.models.dits.wangame.hyworld_action_module import WanGameActionTimeImageEmbedding, WanGameActionSelfAttention - -logger = init_logger(__name__) - - -class WanGameCrossAttention(WanI2VCrossAttention): - def forward(self, x, context, context_lens=None): - r""" - Args: - x(Tensor): Shape [B, L1, C] - context(Tensor): Shape [B, L2, C] - context_lens(Tensor): Shape [B] - """ - context_img = context - b, n, d = x.size(0), self.num_heads, self.head_dim - - # compute query, key, value - q = self.norm_q(self.to_q(x)[0]).view(b, -1, n, d) - k_img = self.norm_added_k(self.add_k_proj(context_img)[0]).view( - b, -1, n, d) - v_img = self.add_v_proj(context_img)[0].view(b, -1, n, d) - img_x = self.attn(q, k_img, v_img) - - # output - x = img_x.flatten(2) - x, _ = self.to_out(x) - return x - -class WanGameActionTransformerBlock(nn.Module): - """ - Transformer block for WAN Action model with support for: - - Self-attention with RoPE and camera PRoPE - - Cross-attention with text/image context - - Feed-forward network with AdaLN modulation - """ - - def __init__(self, - dim: int, - ffn_dim: int, - num_heads: int, - local_attn_size: int = -1, - sink_size: int = 0, - qk_norm: str = "rms_norm_across_heads", - cross_attn_norm: bool = False, - eps: float = 1e-6, - added_kv_proj_dim: int | None = None, - supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None, - prefix: str = ""): - super().__init__() - - # 1. Self-attention - self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False) - self.to_q = ReplicatedLinear(dim, dim, bias=True) - self.to_k = ReplicatedLinear(dim, dim, bias=True) - self.to_v = ReplicatedLinear(dim, dim, bias=True) - self.to_out = ReplicatedLinear(dim, dim, bias=True) - - self.attn1 = WanGameActionSelfAttention( - dim, - num_heads, - local_attn_size=local_attn_size, - sink_size=sink_size, - qk_norm=qk_norm, - eps=eps) - - self.hidden_dim = dim - self.num_attention_heads = num_heads - self.local_attn_size = local_attn_size - dim_head = dim // num_heads - - if qk_norm == "rms_norm": - self.norm_q = RMSNorm(dim_head, eps=eps) - self.norm_k = RMSNorm(dim_head, eps=eps) - elif qk_norm == "rms_norm_across_heads": - self.norm_q = RMSNorm(dim, eps=eps) - self.norm_k = RMSNorm(dim, eps=eps) - else: - raise ValueError(f"QK Norm type {qk_norm} not supported") - - assert cross_attn_norm is True - self.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( - dim, - norm_type="layer", - eps=eps, - elementwise_affine=True, - compute_dtype=torch.float32) - - # 2. Cross-attention (I2V only for now) - self.attn2 = WanGameCrossAttention(dim, - num_heads, - qk_norm=qk_norm, - eps=eps) - # norm3 for FFN input - self.norm3 = LayerNormScaleShift(dim, norm_type="layer", eps=eps, - elementwise_affine=False) - - # 3. Feed-forward - self.ffn = MLP(dim, ffn_dim, act_type="gelu_pytorch_tanh") - self.mlp_residual = ScaleResidual() - - self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) - - # PRoPE output projection (initialized via add_discrete_action_parameters on the model) - self.to_out_prope = ReplicatedLinear(dim, dim, bias=True) - nn.init.zeros_(self.to_out_prope.weight) - if self.to_out_prope.bias is not None: - nn.init.zeros_(self.to_out_prope.bias) - - def forward( - self, - hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor, - temb: torch.Tensor, - freqs_cis: tuple[torch.Tensor, torch.Tensor], - kv_cache: dict | None = None, - crossattn_cache: dict | None = None, - current_start: int = 0, - cache_start: int | None = None, - viewmats: torch.Tensor | None = None, - Ks: torch.Tensor | None = None, - is_cache: bool = False, - ) -> torch.Tensor: - if hidden_states.dim() == 4: - hidden_states = hidden_states.squeeze(1) - - num_frames = temb.shape[1] - frame_seqlen = hidden_states.shape[1] // num_frames - bs, seq_length, _ = hidden_states.shape - orig_dtype = hidden_states.dtype - - # Cast temb to float32 for scale/shift computation - e = self.scale_shift_table + temb.float() - assert e.shape == (bs, num_frames, 6, self.hidden_dim) - shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(6, dim=2) - - # 1. Self-attention - norm_hidden_states = (self.norm1(hidden_states.float()).unflatten(dim=1, sizes=(num_frames, frame_seqlen)) * - (1 + scale_msa) + shift_msa).to(orig_dtype).flatten(1, 2) - - query, _ = self.to_q(norm_hidden_states) - key, _ = self.to_k(norm_hidden_states) - value, _ = self.to_v(norm_hidden_states) - - if self.norm_q is not None: - query = self.norm_q.forward_native(query) - if self.norm_k is not None: - key = self.norm_k.forward_native(key) - - query = query.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - - # Self-attention with camera PRoPE - attn_output_rope, attn_output_prope = self.attn1( - query, key, value, freqs_cis, - kv_cache, current_start, cache_start, viewmats, Ks, - is_cache=is_cache - ) - # Combine rope and prope outputs - attn_output_rope = attn_output_rope.flatten(2) - attn_output_rope, _ = self.to_out(attn_output_rope) - attn_output_prope = attn_output_prope.flatten(2) - - # # DEBUG: Check if prope input is zero - # if self.training and torch.distributed.get_rank() == 0: - # prope_nonzero = (attn_output_prope != 0).sum().item() - # prope_total = attn_output_prope.numel() - # if prope_nonzero == 0: - # print(f"[DEBUG] to_out_prope INPUT is ALL ZEROS! shape={attn_output_prope.shape}", flush=True) - - attn_output_prope, _ = self.to_out_prope(attn_output_prope) - attn_output = attn_output_rope.squeeze(1) + attn_output_prope.squeeze(1) - - # Self-attention residual + norm in float32 - null_shift = null_scale = torch.zeros(1, device=hidden_states.device, dtype=torch.float32) - norm_hidden_states, hidden_states = self.self_attn_residual_norm( - hidden_states.float(), attn_output.float(), gate_msa, null_shift, null_scale) - hidden_states = hidden_states.type_as(attn_output) - norm_hidden_states = norm_hidden_states.type_as(attn_output) - - # 2. Cross-attention - attn_output = self.attn2(norm_hidden_states.to(orig_dtype), - context=encoder_hidden_states, - context_lens=None) - # Cross-attention residual in bfloat16 - hidden_states = hidden_states + attn_output - - # norm3 for FFN input in float32 - norm_hidden_states = self.norm3( - hidden_states.float(), c_shift_msa, c_scale_msa - ).type_as(hidden_states) - - # 3. Feed-forward - ff_output = self.ffn(norm_hidden_states.to(orig_dtype)) - hidden_states = self.mlp_residual(hidden_states.float(), ff_output.float(), c_gate_msa) - hidden_states = hidden_states.to(orig_dtype) # Cast back to original dtype - - return hidden_states - -class WanGameActionTransformer3DModel(BaseDiT): - """ - WAN Action Transformer 3D Model for video generation with action conditioning. - - Extends the base WAN video model with: - - Action embedding support for controllable generation - - camera PRoPE attention for 3D-aware generation - - KV caching for autoregressive inference - """ - _fsdp_shard_conditions = WanGameVideoConfig()._fsdp_shard_conditions - _compile_conditions = WanGameVideoConfig()._compile_conditions - _supported_attention_backends = WanGameVideoConfig()._supported_attention_backends - param_names_mapping = WanGameVideoConfig().param_names_mapping - reverse_param_names_mapping = WanGameVideoConfig().reverse_param_names_mapping - lora_param_names_mapping = WanGameVideoConfig().lora_param_names_mapping - - def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> None: - super().__init__(config=config, hf_config=hf_config) - - inner_dim = config.num_attention_heads * config.attention_head_dim - self.hidden_size = config.hidden_size - self.num_attention_heads = config.num_attention_heads - self.attention_head_dim = config.attention_head_dim - self.in_channels = config.in_channels - self.out_channels = config.out_channels - self.num_channels_latents = config.num_channels_latents - self.patch_size = config.patch_size - self.local_attn_size = config.local_attn_size - self.inner_dim = inner_dim - - # 1. Patch & position embedding - self.patch_embedding = PatchEmbed(in_chans=config.in_channels, - embed_dim=inner_dim, - patch_size=config.patch_size, - flatten=False) - - # 2. Condition embeddings (with action support) - self.condition_embedder = WanGameActionTimeImageEmbedding( - dim=inner_dim, - time_freq_dim=config.freq_dim, - image_embed_dim=config.image_dim, - ) - - # 3. Transformer blocks - self.blocks = nn.ModuleList([ - WanGameActionTransformerBlock( - inner_dim, - config.ffn_dim, - config.num_attention_heads, - config.local_attn_size, - config.sink_size, - config.qk_norm, - config.cross_attn_norm, - config.eps, - config.added_kv_proj_dim, - supported_attention_backends=self._supported_attention_backends, - prefix=f"{config.prefix}.blocks.{i}") - for i in range(config.num_layers) - ]) - - # 4. Output norm & projection - self.norm_out = LayerNormScaleShift(inner_dim, - norm_type="layer", - eps=config.eps, - elementwise_affine=False, - dtype=torch.float32) - self.proj_out = nn.Linear( - inner_dim, config.out_channels * math.prod(config.patch_size)) - self.scale_shift_table = nn.Parameter(torch.randn(1, 2, inner_dim) / inner_dim**0.5) - - self.gradient_checkpointing = False - - # Causal-specific - self.num_frame_per_block = config.arch_config.num_frames_per_block - assert self.num_frame_per_block <= 3 - - self.__post_init__() - - def forward( - self, - hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor | list[torch.Tensor], - timestep: torch.LongTensor, - encoder_hidden_states_image: torch.Tensor | list[torch.Tensor], - guidance=None, - action: torch.Tensor | None = None, - viewmats: torch.Tensor | None = None, - Ks: torch.Tensor | None = None, - kv_cache: list[dict] | None = None, - crossattn_cache: list[dict] | None = None, - current_start: int = 0, - cache_start: int = 0, - start_frame: int = 0, - is_cache: bool = False, - **kwargs - ) -> torch.Tensor: - """ - Forward pass for both training and inference with KV caching. - - Args: - hidden_states: Video latents [B, C, T, H, W] - encoder_hidden_states: Text embeddings [B, L, D] - timestep: Timestep tensor - encoder_hidden_states_image: Optional image embeddings - action: Action tensor [B, T] for per-frame conditioning - viewmats: Camera view matrices for PRoPE [B, T, 4, 4] - Ks: Camera intrinsics for PRoPE [B, T, 3, 3] - kv_cache: KV cache for autoregressive inference (list of dicts per layer) - crossattn_cache: Cross-attention cache for inference - current_start: Current position for KV cache - cache_start: Cache start position - start_frame: RoPE offset for new frames in autoregressive mode - is_cache: If True, populate KV cache and return early (cache-only mode) - """ - orig_dtype = hidden_states.dtype - # if not isinstance(encoder_hidden_states, torch.Tensor): - # encoder_hidden_states = encoder_hidden_states[0] - if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: - encoder_hidden_states_image = encoder_hidden_states_image[0] - # else: - # encoder_hidden_states_image = None - - batch_size, num_channels, num_frames, height, width = hidden_states.shape - p_t, p_h, p_w = self.patch_size - post_patch_num_frames = num_frames // p_t - post_patch_height = height // p_h - post_patch_width = width // p_w - - # Get rotary embeddings - d = self.hidden_size // self.num_attention_heads - rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] - freqs_cos, freqs_sin = get_rotary_pos_embed( - (post_patch_num_frames * get_sp_world_size(), post_patch_height, post_patch_width), - self.hidden_size, - self.num_attention_heads, - rope_dim_list, - dtype=torch.float32 if current_platform.is_mps() else torch.float64, - rope_theta=10000, - start_frame=start_frame - ) - freqs_cos = freqs_cos.to(hidden_states.device) - freqs_sin = freqs_sin.to(hidden_states.device) - freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None - - hidden_states = self.patch_embedding(hidden_states) - hidden_states = hidden_states.flatten(2).transpose(1, 2) - - if timestep.dim() == 2: - timestep = timestep.flatten() - - temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( - timestep, action, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) - - # condition_embedder returns: - # - temb: [B*T, dim] where T = post_patch_num_frames - # - timestep_proj: [B*T, 6*dim] - # Reshape to [B, T, 6, dim] for transformer blocks - timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) # [B*T, 6, dim] - timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, self.hidden_size) # [B, T, 6, dim] - - encoder_hidden_states = encoder_hidden_states_image - - # Transformer blocks - for block_idx, block in enumerate(self.blocks): - if torch.is_grad_enabled() and self.gradient_checkpointing: - hidden_states = self._gradient_checkpointing_func( - block, hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, - kv_cache[block_idx] if kv_cache else None, - crossattn_cache[block_idx] if crossattn_cache else None, - current_start, cache_start, - viewmats, Ks, is_cache) - else: - hidden_states = block( - hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, - kv_cache[block_idx] if kv_cache else None, - crossattn_cache[block_idx] if crossattn_cache else None, - current_start, cache_start, - viewmats, Ks, is_cache) - - # If cache-only mode, return early - if is_cache: - return kv_cache - - # Output norm, projection & unpatchify - # temb is [B*T, dim], reshape to [B, T, 1, dim] - temb = temb.view(batch_size, post_patch_num_frames, -1).unsqueeze(2) # [B, T, 1, dim] - - shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) - hidden_states = self.norm_out(hidden_states, shift, scale) - hidden_states = self.proj_out(hidden_states) - - hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, - post_patch_height, - post_patch_width, p_t, p_h, p_w, - -1) - hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) - output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) - - return output \ No newline at end of file diff --git a/fastvideo/models/dits/wangame_lingbot/__init__.py b/fastvideo/models/dits/wangame_lingbot/__init__.py deleted file mode 100644 index ad549c889..000000000 --- a/fastvideo/models/dits/wangame_lingbot/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .model import WanLingBotTransformer3DModel - -__all__ = [ - "WanLingBotTransformer3DModel", -] diff --git a/fastvideo/models/dits/wangame_lingbot/cam_utils.py b/fastvideo/models/dits/wangame_lingbot/cam_utils.py deleted file mode 100644 index fb72ec84a..000000000 --- a/fastvideo/models/dits/wangame_lingbot/cam_utils.py +++ /dev/null @@ -1,203 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Adapted from LingBot World: https://github.com/Robbyant/lingbot-world/blob/main/wan/utils/cam_utils.py - -import numpy as np -import os -import torch -from scipy.interpolate import interp1d -from scipy.spatial.transform import Rotation, Slerp - - -# --- Official Code (Leave Unchanged) --- - -def interpolate_camera_poses( - src_indices: np.ndarray, - src_rot_mat: np.ndarray, - src_trans_vec: np.ndarray, - tgt_indices: np.ndarray, -) -> torch.Tensor: - # interpolate translation - interp_func_trans = interp1d( - src_indices, - src_trans_vec, - axis=0, - kind='linear', - bounds_error=False, - fill_value="extrapolate", - ) - interpolated_trans_vec = interp_func_trans(tgt_indices) - - # interpolate rotation - src_quat_vec = Rotation.from_matrix(src_rot_mat) - # ensure there is no sudden change in qw - quats = src_quat_vec.as_quat().copy() # [N, 4] - for i in range(1, len(quats)): - if np.dot(quats[i], quats[i-1]) < 0: - quats[i] = -quats[i] - src_quat_vec = Rotation.from_quat(quats) - slerp_func_rot = Slerp(src_indices, src_quat_vec) - interpolated_rot_quat = slerp_func_rot(tgt_indices) - interpolated_rot_mat = interpolated_rot_quat.as_matrix() - - poses = np.zeros((len(tgt_indices), 4, 4)) - poses[:, :3, :3] = interpolated_rot_mat - poses[:, :3, 3] = interpolated_trans_vec - poses[:, 3, 3] = 1.0 - return torch.from_numpy(poses).float() - - -def SE3_inverse(T: torch.Tensor) -> torch.Tensor: - Rot = T[:, :3, :3] # [B,3,3] - trans = T[:, :3, 3:] # [B,3,1] - R_inv = Rot.transpose(-1, -2) - t_inv = -torch.bmm(R_inv, trans) - T_inv = torch.eye(4, device=T.device, dtype=T.dtype)[None, :, :].repeat(T.shape[0], 1, 1) - T_inv[:, :3, :3] = R_inv - T_inv[:, :3, 3:] = t_inv - return T_inv - - -def compute_relative_poses( - c2ws_mat: torch.Tensor, - framewise: bool = False, - normalize_trans: bool = True, -) -> torch.Tensor: - ref_w2cs = SE3_inverse(c2ws_mat[0:1]) - relative_poses = torch.matmul(ref_w2cs, c2ws_mat) - # ensure identity matrix for 1st frame - relative_poses[0] = torch.eye(4, device=c2ws_mat.device, dtype=c2ws_mat.dtype) - if framewise: - # compute pose between i and i+1 - relative_poses_framewise = torch.bmm(SE3_inverse(relative_poses[:-1]), relative_poses[1:]) - relative_poses[1:] = relative_poses_framewise - if normalize_trans: # note refer to camctrl2: "we scale the coordinate inputs to roughly 1 standard deviation to simplify model learning." - translations = relative_poses[:, :3, 3] # [f, 3] - max_norm = torch.norm(translations, dim=-1).max() - # only normlaize when moving - if max_norm > 0: - relative_poses[:, :3, 3] = translations / max_norm - return relative_poses - - -@torch.no_grad() -def create_meshgrid(n_frames: int, height: int, width: int, bias: float = 0.5, device='cuda', dtype=torch.float32) -> torch.Tensor: - x_range = torch.arange(width, device=device, dtype=dtype) - y_range = torch.arange(height, device=device, dtype=dtype) - grid_y, grid_x = torch.meshgrid(y_range, x_range, indexing='ij') - grid_xy = torch.stack([grid_x, grid_y], dim=-1).view([-1, 2]) + bias # [h*w, 2] - grid_xy = grid_xy[None, ...].repeat(n_frames, 1, 1) # [f, h*w, 2] - return grid_xy - - -def get_plucker_embeddings( - c2ws_mat: torch.Tensor, - Ks: torch.Tensor, - height: int, - width: int, -): - n_frames = c2ws_mat.shape[0] - grid_xy = create_meshgrid(n_frames, height, width, device=c2ws_mat.device, dtype=c2ws_mat.dtype) # [f, h*w, 2] - fx, fy, cx, cy = Ks.chunk(4, dim=-1) # [f, 1] - - i = grid_xy[..., 0] # [f, h*w] - j = grid_xy[..., 1] # [f, h*w] - zs = torch.ones_like(i) # [f, h*w] - xs = (i - cx) / fx * zs - ys = (j - cy) / fy * zs - - directions = torch.stack([xs, ys, zs], dim=-1) # [f, h*w, 3] - directions = directions / directions.norm(dim=-1, keepdim=True) # [f, h*w, 3] - - rays_d = directions @ c2ws_mat[:, :3, :3].transpose(-1, -2) # [f, h*w, 3] - rays_o = c2ws_mat[:, :3, 3] # [f, 3] - rays_o = rays_o[:, None, :].expand_as(rays_d) # [f, h*w, 3] - # rays_dxo = torch.cross(rays_o, rays_d, dim=-1) # [f, h*w, 3] - # note refer to: apt2 - plucker_embeddings = torch.cat([rays_o, rays_d], dim=-1) # [f, h*w, 6] - plucker_embeddings = plucker_embeddings.view([n_frames, height, width, 6]) # [f*h*w, 6] - return plucker_embeddings - - -def get_Ks_transformed( - Ks: torch.Tensor, - height_org: int, - width_org: int, - height_resize: int, - width_resize: int, - height_final: int, - width_final: int, -): - fx, fy, cx, cy = Ks.chunk(4, dim=-1) # [f, 1] - - scale_x = width_resize / width_org - scale_y = height_resize / height_org - - fx_resize = fx * scale_x - fy_resize = fy * scale_y - cx_resize = cx * scale_x - cy_resize = cy * scale_y - - crop_offset_x = (width_resize - width_final) / 2 - crop_offset_y = (height_resize - height_final) / 2 - - cx_final = cx_resize - crop_offset_x - cy_final = cy_resize - crop_offset_y - - Ks_transformed = torch.zeros_like(Ks) - Ks_transformed[:, 0:1] = fx_resize - Ks_transformed[:, 1:2] = fy_resize - Ks_transformed[:, 2:3] = cx_final - Ks_transformed[:, 3:4] = cy_final - - return Ks_transformed - - -# --- Custom --- - -def prepare_camera_embedding( - action_path: str, - num_frames: int, - height: int, - width: int, - spatial_scale: int = 8, -) -> tuple[torch.Tensor, int]: - c2ws = np.load(os.path.join(action_path, "poses.npy")) - len_c2ws = ((len(c2ws) - 1) // 4) * 4 + 1 - num_frames = min(num_frames, len_c2ws) - c2ws = c2ws[:num_frames] - - Ks = torch.from_numpy( - np.load(os.path.join(action_path, "intrinsics.npy")) - ).float() - Ks = get_Ks_transformed( - Ks, - height_org=480, - width_org=832, - height_resize=height, - width_resize=width, - height_final=height, - width_final=width, - ) - Ks = Ks[0] # use first frame - - len_c2ws = len(c2ws) - num_latent_frames = (len_c2ws - 1) // 4 + 1 - c2ws_infer = interpolate_camera_poses( - src_indices=np.linspace(0, len_c2ws - 1, len_c2ws), - src_rot_mat=c2ws[:, :3, :3], - src_trans_vec=c2ws[:, :3, 3], - tgt_indices=np.linspace(0, len_c2ws - 1, num_latent_frames), - ) - c2ws_infer = compute_relative_poses(c2ws_infer, framewise=True) - Ks = Ks.repeat(num_latent_frames, 1) - plucker = get_plucker_embeddings(c2ws_infer, Ks, height, width) # [F, H, W, 6] - - # reshpae - latent_height = height // spatial_scale - latent_width = width // spatial_scale - plucker = plucker.view(num_latent_frames, latent_height, spatial_scale, latent_width, spatial_scale, 6) - plucker = plucker.permute(0, 1, 3, 5, 2, 4).contiguous() - plucker = plucker.view(num_latent_frames, latent_height, latent_width, 6 * spatial_scale * spatial_scale) - c2ws_plucker_emb = plucker.permute(3, 0, 1, 2).contiguous().unsqueeze(0) - - return c2ws_plucker_emb, num_frames \ No newline at end of file diff --git a/fastvideo/models/dits/wangame_lingbot/model.py b/fastvideo/models/dits/wangame_lingbot/model.py deleted file mode 100644 index a19c9b351..000000000 --- a/fastvideo/models/dits/wangame_lingbot/model.py +++ /dev/null @@ -1,451 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -import math -from typing import Any - -import torch -import torch.nn as nn -import torch.nn.functional as F - -from fastvideo.attention import DistributedAttention -from fastvideo.configs.models.dits.wangamevideo import WanGameVideoConfig -from fastvideo.distributed.parallel_state import get_sp_world_size -from fastvideo.layers.layernorm import (FP32LayerNorm, LayerNormScaleShift, - RMSNorm, ScaleResidual, - ScaleResidualLayerNormScaleShift) -from fastvideo.layers.linear import ReplicatedLinear -from fastvideo.layers.mlp import MLP -from fastvideo.layers.rotary_embedding import get_rotary_pos_embed -from fastvideo.layers.visual_embedding import (PatchEmbed, - WanCamControlPatchEmbedding) -from fastvideo.logger import init_logger -from fastvideo.models.dits.base import BaseDiT -from fastvideo.models.dits.wanvideo import (WanI2VCrossAttention, - WanTimeTextImageEmbedding) -from fastvideo.platforms import AttentionBackendEnum, current_platform - - -logger = init_logger(__name__) - - -class LingBotWorldCamConditioner(nn.Module): - - def __init__(self, dim: int) -> None: - super().__init__() - self.cam_injector = MLP(dim, dim, dim, bias=True, act_type="silu") - self.cam_scale_layer = nn.Linear(dim, dim) - self.cam_shift_layer = nn.Linear(dim, dim) - - def forward( - self, - hidden_states: torch.Tensor, - c2ws_plucker_emb: torch.Tensor | None, - ) -> torch.Tensor: - if c2ws_plucker_emb is None: - return hidden_states - assert c2ws_plucker_emb.shape == hidden_states.shape, ( - f"c2ws_plucker_emb shape must match hidden_states shape, got " - f"{tuple(c2ws_plucker_emb.shape)} vs {tuple(hidden_states.shape)}" - ) - c2ws_hidden_states = self.cam_injector(c2ws_plucker_emb) - c2ws_hidden_states = c2ws_hidden_states + c2ws_plucker_emb - cam_scale = self.cam_scale_layer(c2ws_hidden_states) - cam_shift = self.cam_shift_layer(c2ws_hidden_states) - return (1.0 + cam_scale) * hidden_states + cam_shift - - -class WanGameCrossAttention(WanI2VCrossAttention): - def forward(self, x, context, context_lens=None): - r""" - Args: - x(Tensor): Shape [B, L1, C] - context(Tensor): Shape [B, L2, C] - context_lens(Tensor): Shape [B] - """ - context_img = context - b, n, d = x.size(0), self.num_heads, self.head_dim - - # compute query, key, value - q = self.norm_q(self.to_q(x)[0]).view(b, -1, n, d) - k_img = self.norm_added_k(self.add_k_proj(context_img)[0]).view( - b, -1, n, d) - v_img = self.add_v_proj(context_img)[0].view(b, -1, n, d) - img_x = self.attn(q, k_img, v_img) - - # output - x = img_x.flatten(2) - x, _ = self.to_out(x) - return x - - -class WanGameActionTransformerBlock(nn.Module): - """ - Transformer block for WAN Action model with support for: - - Self-attention with RoPE and camera PRoPE - - Cross-attention with text/image context - - Feed-forward network with AdaLN modulation - """ - - def __init__(self, - dim: int, - ffn_dim: int, - num_heads: int, - local_attn_size: int = -1, - sink_size: int = 0, - qk_norm: str = "rms_norm_across_heads", - cross_attn_norm: bool = False, - eps: float = 1e-6, - added_kv_proj_dim: int | None = None, - supported_attention_backends: tuple[AttentionBackendEnum, ...] | None = None, - prefix: str = ""): - super().__init__() - - # 1. Self-attention - self.norm1 = FP32LayerNorm(dim, eps, elementwise_affine=False) - self.to_q = ReplicatedLinear(dim, dim, bias=True) - self.to_k = ReplicatedLinear(dim, dim, bias=True) - self.to_v = ReplicatedLinear(dim, dim, bias=True) - - self.to_out = ReplicatedLinear(dim, dim, bias=True) - self.attn1 = DistributedAttention( - num_heads=num_heads, - head_size=dim // num_heads, - causal=False, - supported_attention_backends=supported_attention_backends, - prefix=f"{prefix}.attn1") - self.hidden_dim = dim - self.num_attention_heads = num_heads - self.local_attn_size = local_attn_size - dim_head = dim // num_heads - if qk_norm == "rms_norm": - self.norm_q = RMSNorm(dim_head, eps=eps) - self.norm_k = RMSNorm(dim_head, eps=eps) - elif qk_norm == "rms_norm_across_heads": - self.norm_q = RMSNorm(dim, eps=eps) - self.norm_k = RMSNorm(dim, eps=eps) - else: - print("QK Norm type not supported") - raise Exception - assert cross_attn_norm is True - self.self_attn_residual_norm = ScaleResidualLayerNormScaleShift( - dim, - norm_type="layer", - eps=eps, - elementwise_affine=True, - dtype=torch.float32, - compute_dtype=torch.float32) - - # 2. Cross-attention (I2V only for now) - self.attn2 = WanGameCrossAttention(dim, - num_heads, - qk_norm=qk_norm, - eps=eps) - # norm3 for FFN input - self.norm3 = LayerNormScaleShift(dim, norm_type="layer", eps=eps, - elementwise_affine=False) - - # 3. Feed-forward - self.ffn = MLP(dim, ffn_dim, act_type="gelu_pytorch_tanh") - self.mlp_residual = ScaleResidual() - - self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) - self.cam_conditioner = LingBotWorldCamConditioner(dim) - - def forward( - self, - hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor, - temb: torch.Tensor, - freqs_cis: tuple[torch.Tensor, torch.Tensor], - kv_cache: dict | None = None, - crossattn_cache: dict | None = None, - current_start: int = 0, - cache_start: int | None = None, - viewmats: torch.Tensor | None = None, - Ks: torch.Tensor | None = None, - c2ws_plucker_emb: torch.Tensor | None = None, - is_cache: bool = False, - ) -> torch.Tensor: - if hidden_states.dim() == 4: - hidden_states = hidden_states.squeeze(1) - - num_frames = temb.shape[1] - frame_seqlen = hidden_states.shape[1] // num_frames - bs, seq_length, _ = hidden_states.shape - orig_dtype = hidden_states.dtype - - # Cast temb to float32 for scale/shift computation - e = self.scale_shift_table + temb.float() - assert e.shape == (bs, num_frames, 6, self.hidden_dim) - shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(6, dim=2) - - # 1. Self-attention - norm_hidden_states = (self.norm1(hidden_states.float()).unflatten( - dim=1, sizes=(num_frames, frame_seqlen)) * - (1 + scale_msa) + shift_msa).to(orig_dtype).flatten(1, 2) - query, _ = self.to_q(norm_hidden_states) - key, _ = self.to_k(norm_hidden_states) - value, _ = self.to_v(norm_hidden_states) - - if self.norm_q is not None: - query = self.norm_q(query) - if self.norm_k is not None: - key = self.norm_k(key) - - query = query.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - key = key.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - value = value.squeeze(1).unflatten(2, (self.num_attention_heads, -1)) - - attn_output, _ = self.attn1(query, key, value, freqs_cis=freqs_cis) - attn_output = attn_output.flatten(2) - attn_output, _ = self.to_out(attn_output) - attn_output = attn_output.squeeze(1) - - # Self-attention residual + norm in float32 - null_shift = null_scale = torch.tensor([0], device=hidden_states.device) - norm_hidden_states, hidden_states = self.self_attn_residual_norm( - hidden_states, attn_output, gate_msa, null_shift, null_scale) - norm_hidden_states, hidden_states = norm_hidden_states.to( - orig_dtype), hidden_states.to(orig_dtype) - # Inject camera condition - # must be applied after the self-attention residual update. - hidden_states = self.cam_conditioner(hidden_states, c2ws_plucker_emb) - norm_hidden_states = self.self_attn_residual_norm.norm(hidden_states) - norm_hidden_states = norm_hidden_states.to(orig_dtype) - - - # 2. Cross-attention - attn_output = self.attn2(norm_hidden_states.to(orig_dtype), - context=encoder_hidden_states, - context_lens=None) - # Cross-attention residual in bfloat16 - hidden_states = hidden_states + attn_output - - # norm3 for FFN input in float32 - norm_hidden_states = self.norm3( - hidden_states.float(), c_shift_msa, c_scale_msa - ).type_as(hidden_states) - - # 3. Feed-forward - ff_output = self.ffn(norm_hidden_states.to(orig_dtype)) - hidden_states = self.mlp_residual(hidden_states.float(), ff_output.float(), c_gate_msa) - hidden_states = hidden_states.to(orig_dtype) # Cast back to original dtype - - return hidden_states - -class WanLingBotTransformer3DModel(BaseDiT): - """ - WAN Action Transformer 3D Model for video generation with action conditioning. - - Extends the base WAN video model with: - - Action embedding support for controllable generation - - camera PRoPE attention for 3D-aware generation - - KV caching for autoregressive inference - """ - _fsdp_shard_conditions = WanGameVideoConfig()._fsdp_shard_conditions - _compile_conditions = WanGameVideoConfig()._compile_conditions - _supported_attention_backends = WanGameVideoConfig()._supported_attention_backends - param_names_mapping = WanGameVideoConfig().param_names_mapping - reverse_param_names_mapping = WanGameVideoConfig().reverse_param_names_mapping - lora_param_names_mapping = WanGameVideoConfig().lora_param_names_mapping - - def __init__(self, config: WanGameVideoConfig, hf_config: dict[str, Any]) -> None: - super().__init__(config=config, hf_config=hf_config) - - inner_dim = config.num_attention_heads * config.attention_head_dim - self.hidden_size = config.hidden_size - self.num_attention_heads = config.num_attention_heads - self.attention_head_dim = config.attention_head_dim - self.in_channels = config.in_channels - self.out_channels = config.out_channels - self.num_channels_latents = config.num_channels_latents - self.patch_size = config.patch_size - self.local_attn_size = config.local_attn_size - self.inner_dim = inner_dim - - # 1. Patch & position embedding - self.patch_embedding = PatchEmbed(in_chans=config.in_channels, - embed_dim=inner_dim, - patch_size=config.patch_size, - flatten=False) - self.patch_embedding_wancamctrl = WanCamControlPatchEmbedding(in_chans=6 * 64, - embed_dim=inner_dim, - patch_size=config.patch_size) - self.c2ws_mlp = MLP(inner_dim, inner_dim, inner_dim, bias=True, act_type="silu") - - # 2. Condition embeddings (image-only) - self.condition_embedder = WanTimeTextImageEmbedding( - dim=inner_dim, - time_freq_dim=config.freq_dim, - text_embed_dim=0, - image_embed_dim=config.image_dim, - ) - - # 3. Transformer blocks - self.blocks = nn.ModuleList([ - WanGameActionTransformerBlock( - inner_dim, - config.ffn_dim, - config.num_attention_heads, - config.local_attn_size, - config.sink_size, - config.qk_norm, - config.cross_attn_norm, - config.eps, - config.added_kv_proj_dim, - supported_attention_backends=self._supported_attention_backends, - prefix=f"{config.prefix}.blocks.{i}") - for i in range(config.num_layers) - ]) - - # 4. Output norm & projection - self.norm_out = LayerNormScaleShift(inner_dim, - norm_type="layer", - eps=config.eps, - elementwise_affine=False, - dtype=torch.float32) - self.proj_out = nn.Linear( - inner_dim, config.out_channels * math.prod(config.patch_size)) - self.scale_shift_table = nn.Parameter(torch.randn(1, 2, inner_dim) / inner_dim**0.5) - - self.gradient_checkpointing = False - - # Causal-specific - self.num_frame_per_block = config.arch_config.num_frames_per_block - assert self.num_frame_per_block <= 3 - - self.__post_init__() - - def forward( - self, - hidden_states: torch.Tensor, - encoder_hidden_states: torch.Tensor | list[torch.Tensor], - timestep: torch.LongTensor, - encoder_hidden_states_image: torch.Tensor | list[torch.Tensor], - guidance=None, - action: torch.Tensor | None = None, - viewmats: torch.Tensor | None = None, - Ks: torch.Tensor | None = None, - c2ws_plucker_emb: torch.Tensor | None = None, - kv_cache: list[dict] | None = None, - crossattn_cache: list[dict] | None = None, - current_start: int = 0, - cache_start: int = 0, - start_frame: int = 0, - is_cache: bool = False, - **kwargs - ) -> torch.Tensor: - """ - Forward pass for both training and inference with KV caching. - - Args: - hidden_states: Video latents [B, C, T, H, W] - encoder_hidden_states: Text embeddings [B, L, D] - timestep: Timestep tensor - encoder_hidden_states_image: Optional image embeddings - action: Action tensor [B, T] for per-frame conditioning - viewmats: Camera view matrices for PRoPE [B, T, 4, 4] - Ks: Camera intrinsics for PRoPE [B, T, 3, 3] - c2ws_plucker_emb: Camera plucker embedding [B, C, T, H, W] - kv_cache: KV cache for autoregressive inference (list of dicts per layer) - crossattn_cache: Cross-attention cache for inference - current_start: Current position for KV cache - cache_start: Cache start position - start_frame: RoPE offset for new frames in autoregressive mode - is_cache: If True, populate KV cache and return early (cache-only mode) - """ - orig_dtype = hidden_states.dtype - if isinstance(encoder_hidden_states, list) and len(encoder_hidden_states) > 0: - encoder_hidden_states = encoder_hidden_states[0] - if isinstance(encoder_hidden_states_image, list) and len(encoder_hidden_states_image) > 0: - encoder_hidden_states_image = encoder_hidden_states_image[0] - # else: - # encoder_hidden_states_image = None - - batch_size, num_channels, num_frames, height, width = hidden_states.shape - p_t, p_h, p_w = self.patch_size - post_patch_num_frames = num_frames // p_t - post_patch_height = height // p_h - post_patch_width = width // p_w - - # Get rotary embeddings - d = self.hidden_size // self.num_attention_heads - rope_dim_list = [d - 4 * (d // 6), 2 * (d // 6), 2 * (d // 6)] - freqs_cos, freqs_sin = get_rotary_pos_embed( - (post_patch_num_frames * get_sp_world_size(), post_patch_height, post_patch_width), - self.hidden_size, - self.num_attention_heads, - rope_dim_list, - dtype=torch.float32 if current_platform.is_mps() else torch.float64, - rope_theta=10000, - start_frame=start_frame - ) - freqs_cos = freqs_cos.to(hidden_states.device) - freqs_sin = freqs_sin.to(hidden_states.device) - freqs_cis = (freqs_cos, freqs_sin) if freqs_cos is not None else None - - hidden_states = self.patch_embedding(hidden_states) - hidden_states = hidden_states.flatten(2).transpose(1, 2) - c2ws_hidden_states = None - if c2ws_plucker_emb is not None: - c2ws_plucker_emb = self.patch_embedding_wancamctrl( - c2ws_plucker_emb.to(device=hidden_states.device, dtype=hidden_states.dtype) - ) - c2ws_hidden_states = self.c2ws_mlp(c2ws_plucker_emb) - c2ws_plucker_emb = c2ws_plucker_emb + c2ws_hidden_states - - if timestep.dim() == 1: - timestep = timestep.unsqueeze(1).expand(-1, post_patch_num_frames) - if timestep.dim() == 2: - timestep = timestep.flatten() - - if encoder_hidden_states is None or ( - isinstance(encoder_hidden_states, torch.Tensor) - and encoder_hidden_states.numel() == 0): - encoder_hidden_states = hidden_states.new_zeros((batch_size, 0, self.hidden_size)) - temb, timestep_proj, encoder_hidden_states, encoder_hidden_states_image = self.condition_embedder( - timestep, encoder_hidden_states, encoder_hidden_states_image=encoder_hidden_states_image) - timestep_proj = timestep_proj.unflatten(1, (6, self.hidden_size)) - timestep_proj = timestep_proj.view(batch_size, post_patch_num_frames, 6, - self.hidden_size) - - encoder_hidden_states = encoder_hidden_states_image - if encoder_hidden_states is None: - encoder_hidden_states = hidden_states.new_zeros((batch_size, 0, self.hidden_size)) - - # Transformer blocks - for block_idx, block in enumerate(self.blocks): - if torch.is_grad_enabled() and self.gradient_checkpointing: - hidden_states = self._gradient_checkpointing_func( - block, hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, - kv_cache[block_idx] if kv_cache else None, - crossattn_cache[block_idx] if crossattn_cache else None, - current_start, cache_start, - viewmats, Ks, c2ws_hidden_states, is_cache) - else: - hidden_states = block( - hidden_states, encoder_hidden_states, timestep_proj, freqs_cis, - kv_cache[block_idx] if kv_cache else None, - crossattn_cache[block_idx] if crossattn_cache else None, - current_start, cache_start, - viewmats, Ks, c2ws_hidden_states, is_cache) - - # If cache-only mode, return early - if is_cache: - return kv_cache - - # Output norm, projection & unpatchify - temb = temb.view(batch_size, post_patch_num_frames, -1).unsqueeze(2) - - shift, scale = (self.scale_shift_table.unsqueeze(1) + temb).chunk(2, dim=2) - hidden_states = self.norm_out(hidden_states, shift, scale) - hidden_states = self.proj_out(hidden_states) - - hidden_states = hidden_states.reshape(batch_size, post_patch_num_frames, - post_patch_height, - post_patch_width, p_t, p_h, p_w, - -1) - hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) - output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) - - return output diff --git a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py deleted file mode 100644 index af6191b4c..000000000 --- a/fastvideo/pipelines/basic/wan/wangame_causal_dmd_pipeline.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""WanGame causal DMD pipeline implementation.""" - -from fastvideo.fastvideo_args import FastVideoArgs -from fastvideo.logger import init_logger -from fastvideo.pipelines import ComposedPipelineBase, LoRAPipeline -from fastvideo.pipelines.samplers.wan import get_wan_sampler_kind - -from fastvideo.pipelines.stages import ( - ConditioningStage, DecodingStage, MatrixGameCausalDenoisingStage, - MatrixGameCausalOdeDenoisingStage, MatrixGameImageEncodingStage, - InputValidationStage, LatentPreparationStage, TextEncodingStage, - TimestepPreparationStage) -from fastvideo.pipelines.stages.image_encoding import ( - MatrixGameImageVAEEncodingStage) - -logger = init_logger(__name__) - - -class WanGameCausalDMDPipeline(LoRAPipeline, ComposedPipelineBase): - _required_config_modules = [ - "vae", "transformer", "scheduler", "image_encoder", "image_processor" - ] - - def create_pipeline_stages(self, fastvideo_args: FastVideoArgs) -> None: - sampler_kind = get_wan_sampler_kind(fastvideo_args) - self.add_stage(stage_name="input_validation_stage", - stage=InputValidationStage()) - - if (self.get_module("text_encoder", None) is not None - and self.get_module("tokenizer", None) is not None): - self.add_stage(stage_name="prompt_encoding_stage", - stage=TextEncodingStage( - text_encoders=[self.get_module("text_encoder")], - tokenizers=[self.get_module("tokenizer")], - )) - - if (self.get_module("image_encoder", None) is not None - and self.get_module("image_processor", None) is not None): - self.add_stage( - stage_name="image_encoding_stage", - stage=MatrixGameImageEncodingStage( - image_encoder=self.get_module("image_encoder"), - image_processor=self.get_module("image_processor"), - )) - - self.add_stage(stage_name="conditioning_stage", - stage=ConditioningStage()) - - if sampler_kind == "ode": - self.add_stage(stage_name="timestep_preparation_stage", - stage=TimestepPreparationStage( - scheduler=self.get_module("scheduler"))) - - self.add_stage(stage_name="latent_preparation_stage", - stage=LatentPreparationStage( - scheduler=self.get_module("scheduler"), - transformer=self.get_module("transformer", None))) - - self.add_stage( - stage_name="image_latent_preparation_stage", - stage=MatrixGameImageVAEEncodingStage(vae=self.get_module("vae"))) - - if sampler_kind == "ode": - denoising_stage = MatrixGameCausalOdeDenoisingStage( - transformer=self.get_module("transformer"), - transformer_2=self.get_module("transformer_2", None), - scheduler=self.get_module("scheduler"), - pipeline=self, - vae=self.get_module("vae"), - ) - else: - denoising_stage = MatrixGameCausalDenoisingStage( - transformer=self.get_module("transformer"), - transformer_2=self.get_module("transformer_2", None), - scheduler=self.get_module("scheduler"), - pipeline=self, - vae=self.get_module("vae"), - ) - - self.add_stage(stage_name="denoising_stage", stage=denoising_stage) - - self.add_stage(stage_name="decoding_stage", - stage=DecodingStage(vae=self.get_module("vae"))) - - logger.info("WanGameCausalDMDPipeline initialized with action support") - - -EntryClass = WanGameCausalDMDPipeline diff --git a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py b/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py deleted file mode 100644 index 307bf48f9..000000000 --- a/fastvideo/pipelines/basic/wan/wangame_i2v_pipeline.py +++ /dev/null @@ -1,104 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""WanGame image-to-video pipeline implementation. - -This module contains an implementation of the WanGame image-to-video pipeline -using the modular pipeline architecture. -""" - -from fastvideo.fastvideo_args import FastVideoArgs -from fastvideo.logger import init_logger -from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase -from fastvideo.pipelines.lora_pipeline import LoRAPipeline -from fastvideo.pipelines.samplers.wan import ( - build_wan_scheduler, - get_wan_sampler_kind, - wan_use_btchw_layout, -) - -# isort: off -from fastvideo.pipelines.stages import ( - ConditioningStage, - DecodingStage, - DenoisingStage, - ImageEncodingStage, - ImageVAEEncodingStage, - InputValidationStage, - LatentPreparationStage, - SdeDenoisingStage, - TimestepPreparationStage, -) - -# isort: on - -logger = init_logger(__name__) - - -class WanGameActionImageToVideoPipeline(LoRAPipeline, ComposedPipelineBase): - - _required_config_modules = [ - "vae", - "transformer", - "scheduler", - "image_encoder", - "image_processor", - ] - - def initialize_pipeline(self, fastvideo_args: FastVideoArgs): - sampler_kind = get_wan_sampler_kind(fastvideo_args) - self.modules["scheduler"] = build_wan_scheduler(fastvideo_args, - sampler_kind) - - def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): - """Set up pipeline stages with proper dependency injection.""" - - sampler_kind = get_wan_sampler_kind(fastvideo_args) - use_btchw_layout = wan_use_btchw_layout(sampler_kind) - - self.add_stage(stage_name="input_validation_stage", - stage=InputValidationStage()) - - self.add_stage( - stage_name="image_encoding_stage", - stage=ImageEncodingStage( - image_encoder=self.get_module("image_encoder"), - image_processor=self.get_module("image_processor"), - ), - ) - - self.add_stage(stage_name="conditioning_stage", - stage=ConditioningStage()) - - if sampler_kind == "ode": - self.add_stage(stage_name="timestep_preparation_stage", - stage=TimestepPreparationStage( - scheduler=self.get_module("scheduler"))) - - self.add_stage(stage_name="latent_preparation_stage", - stage=LatentPreparationStage( - scheduler=self.get_module("scheduler"), - transformer=self.get_module("transformer"), - use_btchw_layout=use_btchw_layout)) - - self.add_stage(stage_name="image_latent_preparation_stage", - stage=ImageVAEEncodingStage(vae=self.get_module("vae"))) - - if sampler_kind == "sde": - self.add_stage(stage_name="denoising_stage", - stage=SdeDenoisingStage( - transformer=self.get_module("transformer"), - scheduler=self.get_module("scheduler"))) - else: - self.add_stage(stage_name="denoising_stage", - stage=DenoisingStage( - transformer=self.get_module("transformer"), - scheduler=self.get_module("scheduler"))) - - self.add_stage(stage_name="decoding_stage", - stage=DecodingStage(vae=self.get_module("vae"))) - - -class WanLingBotImageToVideoPipeline(WanGameActionImageToVideoPipeline): - pass - - -EntryClass = [WanGameActionImageToVideoPipeline, WanLingBotImageToVideoPipeline] diff --git a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline.py b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline.py deleted file mode 100644 index 40c958d76..000000000 --- a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline.py +++ /dev/null @@ -1,303 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -from typing import Any - -import numpy as np -import torch -from PIL import Image - -from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame -from fastvideo.distributed import get_local_torch_device -from fastvideo.fastvideo_args import FastVideoArgs -from fastvideo.forward_context import set_forward_context -from fastvideo.pipelines.preprocess.preprocess_pipeline_base import ( - BasePreprocessPipeline) -from fastvideo.pipelines.stages import ImageEncodingStage - - -class PreprocessPipeline_WanGame(BasePreprocessPipeline): - """I2V preprocessing pipeline implementation.""" - - _required_config_modules = ["vae", "image_encoder", "image_processor"] - - def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): - self.add_stage(stage_name="image_encoding_stage", - stage=ImageEncodingStage( - image_encoder=self.get_module("image_encoder"), - image_processor=self.get_module("image_processor"), - )) - - def get_pyarrow_schema(self): - """Return the PyArrow schema for I2V pipeline.""" - return pyarrow_schema_wangame - - def get_extra_features(self, valid_data: dict[str, Any], - fastvideo_args: FastVideoArgs) -> dict[str, Any]: - - # TODO(will): move these to cpu at some point - self.get_module("image_encoder").to(get_local_torch_device()) - self.get_module("vae").to(get_local_torch_device()) - - features = {} - """Get CLIP features from the first frame of each video.""" - first_frame = valid_data["pixel_values"][:, :, 0, :, :].permute( - 0, 2, 3, 1) # (B, C, T, H, W) -> (B, H, W, C) - _, _, num_frames, height, width = valid_data["pixel_values"].shape - # latent_height = height // self.get_module( - # "vae").spatial_compression_ratio - # latent_width = width // self.get_module("vae").spatial_compression_ratio - - processed_images = [] - # Frame has values between -1 and 1 - for frame in first_frame: - frame = (frame + 1) * 127.5 - frame_pil = Image.fromarray(frame.cpu().numpy().astype(np.uint8)) - processed_img = self.get_module("image_processor")( - images=frame_pil, return_tensors="pt") - processed_images.append(processed_img) - - # Get CLIP features - pixel_values = torch.cat( - [img['pixel_values'] for img in processed_images], - dim=0).to(get_local_torch_device()) - with torch.no_grad(): - image_inputs = {'pixel_values': pixel_values} - with set_forward_context(current_timestep=0, attn_metadata=None): - clip_features = self.get_module("image_encoder")(**image_inputs) - clip_features = clip_features.last_hidden_state - - features["clip_feature"] = clip_features - """Get VAE features from the first frame of each video""" - video_conditions = [] - for frame in first_frame: - processed_img = frame.to(device="cpu", dtype=torch.float32) - processed_img = processed_img.unsqueeze(0).permute(0, 3, 1, - 2).unsqueeze(2) - # (B, H, W, C) -> (B, C, 1, H, W) - video_condition = torch.cat([ - processed_img, - processed_img.new_zeros(processed_img.shape[0], - processed_img.shape[1], num_frames - 1, - height, width) - ], - dim=2) - video_condition = video_condition.to( - device=get_local_torch_device(), dtype=torch.float32) - video_conditions.append(video_condition) - - video_conditions = torch.cat(video_conditions, dim=0) - - with torch.autocast(device_type="cuda", - dtype=torch.float32, - enabled=True): - encoder_outputs = self.get_module("vae").encode(video_conditions) - - latent_condition = encoder_outputs.mean - if (hasattr(self.get_module("vae"), "shift_factor") - and self.get_module("vae").shift_factor is not None): - if isinstance(self.get_module("vae").shift_factor, torch.Tensor): - latent_condition -= self.get_module("vae").shift_factor.to( - latent_condition.device, latent_condition.dtype) - else: - latent_condition -= self.get_module("vae").shift_factor - - if isinstance(self.get_module("vae").scaling_factor, torch.Tensor): - latent_condition = latent_condition * self.get_module( - "vae").scaling_factor.to(latent_condition.device, - latent_condition.dtype) - else: - latent_condition = latent_condition * self.get_module( - "vae").scaling_factor - - # mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, - # latent_width) - # mask_lat_size[:, :, list(range(1, num_frames))] = 0 - # first_frame_mask = mask_lat_size[:, :, 0:1] - # first_frame_mask = torch.repeat_interleave( - # first_frame_mask, - # dim=2, - # repeats=self.get_module("vae").temporal_compression_ratio) - # mask_lat_size = torch.concat( - # [first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2) - # mask_lat_size = mask_lat_size.view( - # batch_size, -1, - # self.get_module("vae").temporal_compression_ratio, latent_height, - # latent_width) - # mask_lat_size = mask_lat_size.transpose(1, 2) - # mask_lat_size = mask_lat_size.to(latent_condition.device) - - # image_latent = torch.concat([mask_lat_size, latent_condition], dim=1) - - features["first_frame_latent"] = latent_condition - - if "action_path" in valid_data and valid_data["action_path"]: - keyboard_cond_list = [] - mouse_cond_list = [] - num_bits = 6 - for action_path in valid_data["action_path"]: - if action_path: - action_data = np.load(action_path, allow_pickle=True) - if isinstance( - action_data, - np.ndarray) and action_data.dtype == np.dtype('O'): - action_dict = action_data.item() - if "keyboard" in action_dict: - keyboard_raw = action_dict["keyboard"] - # Convert 1D bit-flag values to 2D multi-hot encoding - if isinstance(keyboard_raw, np.ndarray): - if keyboard_raw.ndim == 1: - # [T] -> [T, num_bits] - T = len(keyboard_raw) - multi_hot = np.zeros((T, num_bits), - dtype=np.float32) - action_values = keyboard_raw.astype(int) - for bit_idx in range(num_bits): - target_idx = ( - 2 - - (bit_idx % 3)) + 3 * (bit_idx // 3) - if target_idx < num_bits: - multi_hot[:, target_idx] = ( - (action_values >> bit_idx) - & 1).astype(np.float32) - keyboard_cond_list.append(multi_hot) - else: - # If already 2D, pad to num_bits if necessary - k_data = keyboard_raw.astype(np.float32) - if k_data.ndim == 2 and k_data.shape[ - -1] < num_bits: - padding = np.zeros( - (k_data.shape[0], - num_bits - k_data.shape[-1]), - dtype=np.float32) - k_data = np.concatenate( - [k_data, padding], axis=-1) - keyboard_cond_list.append(k_data) - else: - keyboard_cond_list.append(keyboard_raw) - if "mouse" in action_dict: - mouse_cond_list.append(action_dict["mouse"]) - else: - if isinstance(action_data, - np.ndarray) and action_data.ndim == 1: - T = len(action_data) - multi_hot = np.zeros((T, num_bits), - dtype=np.float32) - action_values = action_data.astype(int) - for bit_idx in range(num_bits): - target_idx = ( - 2 - (bit_idx % 3)) + 3 * (bit_idx // 3) - if target_idx < num_bits: - multi_hot[:, target_idx] = ( - (action_values >> bit_idx) & 1).astype( - np.float32) - keyboard_cond_list.append(multi_hot) - else: - # If already 2D, pad to num_bits if necessary - k_data = action_data.astype(np.float32) - if k_data.ndim == 2 and k_data.shape[-1] < num_bits: - padding = np.zeros( - (k_data.shape[0], - num_bits - k_data.shape[-1]), - dtype=np.float32) - k_data = np.concatenate([k_data, padding], - axis=-1) - keyboard_cond_list.append(k_data) - if keyboard_cond_list: - features["keyboard_cond"] = keyboard_cond_list - if mouse_cond_list: - features["mouse_cond"] = mouse_cond_list - - return features - - def create_record( - self, - video_name: str, - vae_latent: np.ndarray, - text_embedding: np.ndarray, - valid_data: dict[str, Any], - idx: int, - extra_features: dict[str, Any] | None = None) -> dict[str, Any]: - """Create a record for the Parquet dataset with CLIP features.""" - record = super().create_record(video_name=video_name, - vae_latent=vae_latent, - text_embedding=text_embedding, - valid_data=valid_data, - idx=idx, - extra_features=extra_features) - - if extra_features and "clip_feature" in extra_features: - clip_feature = extra_features["clip_feature"] - record.update({ - "clip_feature_bytes": clip_feature.tobytes(), - "clip_feature_shape": list(clip_feature.shape), - "clip_feature_dtype": str(clip_feature.dtype), - }) - else: - record.update({ - "clip_feature_bytes": b"", - "clip_feature_shape": [], - "clip_feature_dtype": "", - }) - - if extra_features and "first_frame_latent" in extra_features: - first_frame_latent = extra_features["first_frame_latent"] - record.update({ - "first_frame_latent_bytes": - first_frame_latent.tobytes(), - "first_frame_latent_shape": - list(first_frame_latent.shape), - "first_frame_latent_dtype": - str(first_frame_latent.dtype), - }) - else: - record.update({ - "first_frame_latent_bytes": b"", - "first_frame_latent_shape": [], - "first_frame_latent_dtype": "", - }) - - if extra_features and "pil_image" in extra_features: - pil_image = extra_features["pil_image"] - record.update({ - "pil_image_bytes": pil_image.tobytes(), - "pil_image_shape": list(pil_image.shape), - "pil_image_dtype": str(pil_image.dtype), - }) - else: - record.update({ - "pil_image_bytes": b"", - "pil_image_shape": [], - "pil_image_dtype": "", - }) - - if extra_features and "keyboard_cond" in extra_features: - keyboard_cond = extra_features["keyboard_cond"] - record.update({ - "keyboard_cond_bytes": keyboard_cond.tobytes(), - "keyboard_cond_shape": list(keyboard_cond.shape), - "keyboard_cond_dtype": str(keyboard_cond.dtype), - }) - else: - record.update({ - "keyboard_cond_bytes": b"", - "keyboard_cond_shape": [], - "keyboard_cond_dtype": "", - }) - - if extra_features and "mouse_cond" in extra_features: - mouse_cond = extra_features["mouse_cond"] - record.update({ - "mouse_cond_bytes": mouse_cond.tobytes(), - "mouse_cond_shape": list(mouse_cond.shape), - "mouse_cond_dtype": str(mouse_cond.dtype), - }) - else: - record.update({ - "mouse_cond_bytes": b"", - "mouse_cond_shape": [], - "mouse_cond_dtype": "", - }) - - return record - - -EntryClass = PreprocessPipeline_WanGame diff --git a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py b/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py deleted file mode 100644 index 6335d187e..000000000 --- a/fastvideo/pipelines/preprocess/wangame/wangame_preprocess_pipeline_ode_trajectory.py +++ /dev/null @@ -1,497 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -""" -ODE Trajectory Data Preprocessing pipeline implementation. - -This module contains an implementation of the ODE Trajectory Data Preprocessing pipeline -using the modular pipeline architecture. - -Sec 4.3 of CausVid paper: https://arxiv.org/pdf/2412.07772 -""" - -import os -from collections.abc import Iterator -from typing import Any - -import numpy as np -import pyarrow as pa -import torch -from PIL import Image -from torch.utils.data import DataLoader -from torchdata.stateful_dataloader import StatefulDataLoader -from tqdm import tqdm - -from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset import getdataset -from fastvideo.dataset.dataloader.parquet_io import (ParquetDatasetWriter, - records_to_table) -from fastvideo.dataset.dataloader.record_schema import ( - wangame_ode_record_creator) -from fastvideo.dataset.dataloader.schema import ( - pyarrow_schema_ode_trajectory_wangame) -from fastvideo.distributed import get_local_torch_device -from fastvideo.fastvideo_args import FastVideoArgs -from fastvideo.forward_context import set_forward_context -from fastvideo.logger import init_logger -from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( - SelfForcingFlowMatchScheduler) -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch -from fastvideo.pipelines.preprocess.preprocess_pipeline_base import ( - BasePreprocessPipeline) -from fastvideo.pipelines.stages import (DecodingStage, DenoisingStage, - InputValidationStage, - LatentPreparationStage, - ImageEncodingStage, - TimestepPreparationStage) -from fastvideo.utils import save_decoded_latents_as_video, shallow_asdict - -logger = init_logger(__name__) - - -class PreprocessPipeline_WanGame_ODE_Trajectory(BasePreprocessPipeline): - """ODE Trajectory preprocessing pipeline implementation.""" - - _required_config_modules = [ - "vae", "image_encoder", "image_processor", "transformer", "scheduler" - ] - - preprocess_dataloader: StatefulDataLoader - preprocess_loader_iter: Iterator[dict[str, Any]] - pbar: Any - num_processed_samples: int - - def get_pyarrow_schema(self) -> pa.Schema: - """Return the PyArrow schema for ODE Trajectory pipeline.""" - return pyarrow_schema_ode_trajectory_wangame - - def create_pipeline_stages(self, fastvideo_args: FastVideoArgs): - """Set up pipeline stages with proper dependency injection.""" - assert fastvideo_args.pipeline_config.flow_shift == 5 - self.modules["scheduler"] = SelfForcingFlowMatchScheduler( - shift=fastvideo_args.pipeline_config.flow_shift, - sigma_min=0.0, - extra_one_step=True) - self.modules["scheduler"].set_timesteps(num_inference_steps=48, - denoising_strength=1.0) - - self.add_stage(stage_name="input_validation_stage", - stage=InputValidationStage()) - self.add_stage(stage_name="image_encoding_stage", - stage=ImageEncodingStage( - image_encoder=self.get_module("image_encoder"), - image_processor=self.get_module("image_processor"), - )) - self.add_stage(stage_name="timestep_preparation_stage", - stage=TimestepPreparationStage( - scheduler=self.get_module("scheduler"))) - self.add_stage(stage_name="latent_preparation_stage", - stage=LatentPreparationStage( - scheduler=self.get_module("scheduler"), - transformer=self.get_module("transformer", None))) - self.add_stage(stage_name="denoising_stage", - stage=DenoisingStage( - transformer=self.get_module("transformer"), - scheduler=self.get_module("scheduler"), - pipeline=self, - )) - self.add_stage(stage_name="decoding_stage", - stage=DecodingStage(vae=self.get_module("vae"))) - - def get_extra_features(self, valid_data: dict[str, Any], - fastvideo_args: FastVideoArgs) -> dict[str, Any]: - - # TODO(will): move these to cpu at some point - self.get_module("image_encoder").to(get_local_torch_device()) - self.get_module("vae").to(get_local_torch_device()) - - features = {} - """Get CLIP features from the first frame of each video.""" - first_frame = valid_data["pixel_values"][:, :, 0, :, :].permute( - 0, 2, 3, 1) # (B, C, T, H, W) -> (B, H, W, C) - _, _, num_frames, height, width = valid_data["pixel_values"].shape - # latent_height = height // self.get_module( - # "vae").spatial_compression_ratio - # latent_width = width // self.get_module("vae").spatial_compression_ratio - - processed_images = [] - # Frame has values between -1 and 1 - for frame in first_frame: - frame = (frame + 1) * 127.5 - frame_pil = Image.fromarray(frame.cpu().numpy().astype(np.uint8)) - processed_img = self.get_module("image_processor")( - images=frame_pil, return_tensors="pt") - processed_images.append(processed_img) - - # Get CLIP features - pixel_values = torch.cat( - [img['pixel_values'] for img in processed_images], - dim=0).to(get_local_torch_device()) - with torch.no_grad(): - image_inputs = {'pixel_values': pixel_values} - with set_forward_context(current_timestep=0, attn_metadata=None): - clip_features = self.get_module("image_encoder")(**image_inputs) - clip_features = clip_features.last_hidden_state - - features["clip_feature"] = clip_features - features["pil_image"] = first_frame - """Get VAE features from the first frame of each video""" - video_conditions = [] - for frame in first_frame: - processed_img = frame.to(device="cpu", dtype=torch.float32) - processed_img = processed_img.unsqueeze(0).permute(0, 3, 1, - 2).unsqueeze(2) - # (B, H, W, C) -> (B, C, 1, H, W) - video_condition = torch.cat([ - processed_img, - processed_img.new_zeros(processed_img.shape[0], - processed_img.shape[1], num_frames - 1, - height, width) - ], - dim=2) - video_condition = video_condition.to( - device=get_local_torch_device(), dtype=torch.float32) - video_conditions.append(video_condition) - - video_conditions = torch.cat(video_conditions, dim=0) - - with torch.autocast(device_type="cuda", - dtype=torch.float32, - enabled=True): - encoder_outputs = self.get_module("vae").encode(video_conditions) - - # Use mode() instead of mean - latent_condition = encoder_outputs.mode() - - # Use latents_mean/latents_std normalization to match - vae = self.get_module("vae") - if (hasattr(vae.config, 'latents_mean') - and hasattr(vae.config, 'latents_std')): - latents_mean = torch.tensor(vae.config.latents_mean, - device=latent_condition.device, - dtype=latent_condition.dtype).view( - 1, -1, 1, 1, 1) - latents_std = torch.tensor(vae.config.latents_std, - device=latent_condition.device, - dtype=latent_condition.dtype).view( - 1, -1, 1, 1, 1) - latent_condition = (latent_condition - latents_mean) / latents_std - elif (hasattr(vae, "shift_factor") and vae.shift_factor is not None): - if isinstance(vae.shift_factor, torch.Tensor): - latent_condition -= vae.shift_factor.to(latent_condition.device, - latent_condition.dtype) - else: - latent_condition -= vae.shift_factor - - if isinstance(vae.scaling_factor, torch.Tensor): - latent_condition = latent_condition * vae.scaling_factor.to( - latent_condition.device, latent_condition.dtype) - else: - latent_condition = latent_condition * vae.scaling_factor - - # mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, - # latent_width) - # mask_lat_size[:, :, list(range(1, num_frames))] = 0 - # first_frame_mask = mask_lat_size[:, :, 0:1] - # first_frame_mask = torch.repeat_interleave( - # first_frame_mask, - # dim=2, - # repeats=self.get_module("vae").temporal_compression_ratio) - # mask_lat_size = torch.concat( - # [first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2) - # mask_lat_size = mask_lat_size.view( - # batch_size, -1, - # self.get_module("vae").temporal_compression_ratio, latent_height, - # latent_width) - # mask_lat_size = mask_lat_size.transpose(1, 2) - # mask_lat_size = mask_lat_size.to(latent_condition.device) - - # image_latent = torch.concat([mask_lat_size, latent_condition], dim=1) - - # Create mask_cond: ones for first frame, zeros for rest - # Shape: (B, 16, latent_frames, latent_height, latent_width) - mask_cond = torch.ones_like(latent_condition) - mask_cond[:, :, 1:] = 0 # Set all frames except first to 0 - # Create cond_concat: first 4 channels of mask + all 16 channels of img_cond - # Shape: (B, 20, latent_frames, latent_height, latent_width) - cond_concat = torch.cat([mask_cond[:, :4], latent_condition], dim=1) - features["first_frame_latent"] = cond_concat - - if "action_path" in valid_data and valid_data["action_path"]: - keyboard_cond_list = [None] * len(valid_data["action_path"]) - mouse_cond_list = [None] * len(valid_data["action_path"]) - arch_cfg = self.get_module("transformer").config.arch_config - action_cfg = getattr(arch_cfg, "action_config", {}) or {} - keyboard_dim = action_cfg.get("keyboard_dim_in", None) - for idx, action_path in enumerate(valid_data["action_path"]): - if action_path: - action_data = np.load(action_path, allow_pickle=True) - if isinstance( - action_data, - np.ndarray) and action_data.dtype == np.dtype('O'): - action_dict = action_data.item() - if "keyboard" in action_dict: - keyboard = action_dict["keyboard"].astype( - np.float32) - if keyboard_dim is not None: - if keyboard.ndim >= 2: - keyboard = keyboard[:, :keyboard_dim] - else: - keyboard = keyboard[:keyboard_dim] - keyboard_cond_list[idx] = keyboard - if "mouse" in action_dict: - mouse_cond_list[idx] = action_dict["mouse"].astype( - np.float32) - else: - keyboard = action_data.astype(np.float32) - if keyboard_dim is not None: - if keyboard.ndim >= 2: - keyboard = keyboard[:, :keyboard_dim] - else: - keyboard = keyboard[:keyboard_dim] - keyboard_cond_list[idx] = keyboard - features["keyboard_cond"] = keyboard_cond_list - features["mouse_cond"] = mouse_cond_list - return features - - def preprocess_action_and_trajectory(self, fastvideo_args: FastVideoArgs, - args): - """Preprocess data and generate trajectory information.""" - - for batch_idx, data in enumerate(self.pbar): - if data is None: - continue - - with torch.inference_mode(): - # Filter out invalid samples (those with all zeros) - valid_indices = [] - for i, pixel_values in enumerate(data["pixel_values"]): - if not torch.all( - pixel_values == 0): # Check if all values are zero - valid_indices.append(i) - self.num_processed_samples += len(valid_indices) - - if not valid_indices: - continue - - # Create new batch with only valid samples - valid_data = { - "pixel_values": - torch.stack( - [data["pixel_values"][i] for i in valid_indices]), - "path": [data["path"][i] for i in valid_indices], - } - - if "fps" in data: - valid_data["fps"] = [data["fps"][i] for i in valid_indices] - if "duration" in data: - valid_data["duration"] = [ - data["duration"][i] for i in valid_indices - ] - if "action_path" in data: - valid_data["action_path"] = [ - data["action_path"][i] for i in valid_indices - ] - - pixel_values = valid_data["pixel_values"] - if pixel_values.shape[2] == 1 and args.num_frames is not None: - pixel_values = pixel_values.repeat(1, 1, args.num_frames, 1, - 1) - valid_data["pixel_values"] = pixel_values - - # Get extra features if needed - extra_features = self.get_extra_features( - valid_data, fastvideo_args) - - clip_features = extra_features['clip_feature'] - image_latents = extra_features['first_frame_latent'] - image_latents = image_latents[:, :, :args.num_latent_t] - pil_image = extra_features['pil_image'] - if "keyboard_cond" in extra_features: - keyboard_cond = extra_features['keyboard_cond'] - else: - keyboard_cond = None - if "mouse_cond" in extra_features: - mouse_cond = extra_features['mouse_cond'] - else: - mouse_cond = None - - sampling_params = SamplingParam.from_pretrained(args.model_path) - - trajectory_latents = [] - trajectory_timesteps = [] - trajectory_decoded = [] - - device = get_local_torch_device() - for i in range(len(valid_indices)): - # Collect the trajectory data - batch = ForwardBatch(**shallow_asdict(sampling_params), ) - batch.image_embeds = [clip_features[i].unsqueeze(0)] - batch.image_latent = image_latents[i].unsqueeze(0) - sample_keyboard = keyboard_cond[ - i] if keyboard_cond is not None else None - sample_mouse = mouse_cond[ - i] if mouse_cond is not None else None - if sample_keyboard is not None and sample_mouse is not None: - batch.keyboard_cond = torch.from_numpy( - sample_keyboard).unsqueeze(0).to(device) - batch.mouse_cond = torch.from_numpy( - sample_mouse).unsqueeze(0).to(device) - else: - batch.keyboard_cond = None - batch.mouse_cond = None - batch.num_inference_steps = 48 - batch.return_trajectory_latents = True - # Enabling this will save the decoded trajectory videos. - # Used for debugging. - batch.return_trajectory_decoded = False - batch.height = args.max_height - batch.width = args.max_width - batch.fps = args.train_fps - batch.num_frames = valid_data["pixel_values"].shape[2] - batch.guidance_scale = 6.0 - batch.do_classifier_free_guidance = False - batch.prompt = "" - batch.prompt_embeds = [ - torch.zeros( - (1, 0, self.get_module("transformer").hidden_size), - dtype=torch.bfloat16, - device=device) - ] - - result_batch = self.input_validation_stage( - batch, fastvideo_args) - result_batch = self.timestep_preparation_stage( - result_batch, fastvideo_args) - result_batch.timesteps = result_batch.timesteps.to(device) - result_batch = self.latent_preparation_stage( - result_batch, fastvideo_args) - result_batch = self.denoising_stage(result_batch, - fastvideo_args) - result_batch = self.decoding_stage(result_batch, - fastvideo_args) - - trajectory_latents.append( - result_batch.trajectory_latents.cpu()) - trajectory_timesteps.append( - result_batch.trajectory_timesteps.cpu()) - trajectory_decoded.append(result_batch.trajectory_decoded) - - # Prepare extra features - extra_features = { - "trajectory_latents": trajectory_latents, - "trajectory_timesteps": trajectory_timesteps - } - - if batch.return_trajectory_decoded: - for i, decoded_frames in enumerate(trajectory_decoded): - for j, decoded_frame in enumerate(decoded_frames): - save_decoded_latents_as_video( - decoded_frame, - f"decoded_videos/trajectory_decoded_{i}_{j}.mp4", - args.train_fps) - - # Prepare batch data for Parquet dataset - batch_data: list[dict[str, Any]] = [] - - # Add progress bar for saving outputs - save_pbar = tqdm(enumerate(valid_data["path"]), - desc="Saving outputs", - unit="item", - leave=False) - - for idx, video_path in save_pbar: - video_name = os.path.basename(video_path).split(".")[0] - - clip_feature_np = clip_features[idx].cpu().numpy() - first_frame_latent_np = image_latents[idx].cpu().numpy() - pil_image_np = pil_image[idx].cpu().numpy() - keyboard_cond_np = keyboard_cond[ - idx] if keyboard_cond is not None else None - mouse_cond_np = mouse_cond[ - idx] if mouse_cond is not None else None - - # Get trajectory features for this sample - traj_latents = extra_features["trajectory_latents"][idx] - traj_timesteps = extra_features["trajectory_timesteps"][idx] - if isinstance(traj_latents, torch.Tensor): - traj_latents = traj_latents.cpu().float().numpy() - if isinstance(traj_timesteps, torch.Tensor): - traj_timesteps = traj_timesteps.cpu().float().numpy() - - # Create record for Parquet dataset - record: dict[str, Any] = wangame_ode_record_creator( - video_name=video_name, - clip_feature=clip_feature_np, - first_frame_latent=first_frame_latent_np, - trajectory_latents=traj_latents, - trajectory_timesteps=traj_timesteps, - pil_image=pil_image_np, - keyboard_cond=keyboard_cond_np, - mouse_cond=mouse_cond_np, - caption="") - batch_data.append(record) - - if batch_data: - write_pbar = tqdm(total=1, - desc="Writing to Parquet dataset", - unit="batch") - table = records_to_table(batch_data, - self.get_pyarrow_schema()) - write_pbar.update(1) - write_pbar.close() - - if not hasattr(self, 'dataset_writer'): - self.dataset_writer = ParquetDatasetWriter( - out_dir=self.combined_parquet_dir, - samples_per_file=args.samples_per_file, - ) - self.dataset_writer.append_table(table) - - logger.info("Collected batch with %s samples", len(table)) - - if self.num_processed_samples >= args.flush_frequency: - written = self.dataset_writer.flush() - logger.info("Flushed %s samples to parquet", written) - self.num_processed_samples = 0 - - # Final flush for any remaining samples - if hasattr(self, 'dataset_writer'): - written = self.dataset_writer.flush(write_remainder=True) - if written: - logger.info("Final flush wrote %s samples", written) - - def forward(self, batch: ForwardBatch, fastvideo_args: FastVideoArgs, args): - if not self.post_init_called: - self.post_init() - - self.local_rank = int(os.getenv("RANK", 0)) - os.makedirs(args.output_dir, exist_ok=True) - # Create directory for combined data - self.combined_parquet_dir = os.path.join(args.output_dir, - "combined_parquet_dataset") - os.makedirs(self.combined_parquet_dir, exist_ok=True) - - # Loading dataset - train_dataset = getdataset(args) - - self.preprocess_dataloader = DataLoader( - train_dataset, - batch_size=args.preprocess_video_batch_size, - num_workers=args.dataloader_num_workers, - ) - - self.preprocess_loader_iter = iter(self.preprocess_dataloader) - - self.num_processed_samples = 0 - # Add progress bar for video preprocessing - self.pbar = tqdm(self.preprocess_loader_iter, - desc="Processing videos", - unit="batch", - disable=self.local_rank != 0) - - # Initialize class variables for data sharing - self.video_data: dict[str, Any] = {} # Store video metadata and paths - self.latent_data: dict[str, Any] = {} # Store latent tensors - self.preprocess_action_and_trajectory(fastvideo_args, args) - - -EntryClass = PreprocessPipeline_WanGame_ODE_Trajectory diff --git a/fastvideo/train/models/wangame/__init__.py b/fastvideo/train/models/wangame/__init__.py deleted file mode 100644 index 101e3e7a8..000000000 --- a/fastvideo/train/models/wangame/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""WanGame model plugin package.""" - -from fastvideo.train.models.wangame.wangame import ( - WanGameModel as WanGameModel, ) -from fastvideo.train.models.wangame.wangame_causal import ( - WanGameCausalModel as WanGameCausalModel, ) diff --git a/fastvideo/train/models/wangame/wangame.py b/fastvideo/train/models/wangame/wangame.py deleted file mode 100644 index 1d7a25855..000000000 --- a/fastvideo/train/models/wangame/wangame.py +++ /dev/null @@ -1,816 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""WanGame bidirectional model plugin (per-role instance).""" - -from __future__ import annotations - -import copy -from typing import Any, Literal, TYPE_CHECKING - -import torch - -import fastvideo.envs as envs -from fastvideo.distributed import ( - get_local_torch_device, - get_sp_group, - get_world_group, -) -from fastvideo.forward_context import set_forward_context -from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, ) -from fastvideo.models.utils import pred_noise_to_pred_video -from fastvideo.pipelines import TrainingBatch -from fastvideo.training.activation_checkpoint import ( - apply_activation_checkpointing, ) -from fastvideo.training.training_utils import ( - compute_density_for_timestep_sampling, - get_sigmas, - normalize_dit_input, - shift_timestep, -) -from fastvideo.utils import ( - is_vmoba_available, - is_vsa_available, - set_random_seed, -) - -from fastvideo.train.models.base import ModelBase -from fastvideo.train.utils.module_state import ( - apply_trainable, ) -from fastvideo.train.utils.moduleloader import ( - load_module_from_path, ) - -if TYPE_CHECKING: - from fastvideo.train.utils.training_config import ( - TrainingConfig, ) - -try: - from fastvideo.attention.backends.video_sparse_attn import ( - VideoSparseAttentionMetadataBuilder, ) - from fastvideo.attention.backends.vmoba import ( - VideoMobaAttentionMetadataBuilder, ) -except Exception: - VideoSparseAttentionMetadataBuilder = None # type: ignore[assignment] - VideoMobaAttentionMetadataBuilder = None # type: ignore[assignment] - - -class WanGameModel(ModelBase): - """WanGame per-role model: owns transformer + noise_scheduler.""" - - _transformer_cls_name: str = ("WanGameActionTransformer3DModel") - - def __init__( - self, - *, - init_from: str, - training_config: TrainingConfig, - trainable: bool = True, - disable_custom_init_weights: bool = False, - flow_shift: float = 3.0, - enable_gradient_checkpointing_type: str | None = None, - ) -> None: - self._init_from = str(init_from) - self._trainable = bool(trainable) - - self.transformer = self._load_transformer( - init_from=self._init_from, - trainable=self._trainable, - disable_custom_init_weights=(disable_custom_init_weights), - enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), - training_config=training_config, - ) - - self.noise_scheduler = (FlowMatchEulerDiscreteScheduler(shift=float(flow_shift))) - - # Filled by init_preprocessors (student only). - self.vae: Any = None - self.training_config: TrainingConfig = training_config - self.dataloader: Any = None - self.validator: Any = None - self.start_step: int = 0 - - self.world_group: Any = None - self.sp_group: Any = None - self.device: Any = get_local_torch_device() - - self.noise_random_generator: (torch.Generator | None) = None - self.noise_gen_cuda: torch.Generator | None = None - - self.timestep_shift: float = float(flow_shift) - self.num_train_timestep: int = int(self.noise_scheduler.num_train_timesteps) - self.min_timestep: int = 0 - self.max_timestep: int = self.num_train_timestep - - def _load_transformer( - self, - *, - init_from: str, - trainable: bool, - disable_custom_init_weights: bool, - enable_gradient_checkpointing_type: str | None, - training_config: TrainingConfig, - ) -> torch.nn.Module: - transformer = load_module_from_path( - model_path=init_from, - module_type="transformer", - training_config=training_config, - disable_custom_init_weights=(disable_custom_init_weights), - override_transformer_cls_name=(self._transformer_cls_name), - ) - transformer = apply_trainable(transformer, trainable=trainable) - if (trainable and enable_gradient_checkpointing_type): - transformer = apply_activation_checkpointing( - transformer, - checkpointing_type=(enable_gradient_checkpointing_type), - ) - return transformer - - # ------------------------------------------------------------------ - # Lifecycle - # ------------------------------------------------------------------ - - def init_preprocessors(self, training_config: TrainingConfig) -> None: - """Load VAE, build dataloader, seed RNGs.""" - self.vae = load_module_from_path( - model_path=str(training_config.model_path), - module_type="vae", - training_config=training_config, - ) - - self.world_group = get_world_group() - self.sp_group = get_sp_group() - - self._init_timestep_mechanics() - - from fastvideo.dataset.dataloader.schema import ( - pyarrow_schema_wangame, ) - from fastvideo.train.utils.dataloader import ( - build_parquet_wangame_train_dataloader, ) - - self.dataloader = (build_parquet_wangame_train_dataloader( - training_config.data, - parquet_schema=pyarrow_schema_wangame, - )) - self.start_step = 0 - - # ------------------------------------------------------------------ - # ModelBase overrides: timestep helpers - # ------------------------------------------------------------------ - - @property - def num_train_timesteps(self) -> int: - return int(self.num_train_timestep) - - def shift_and_clamp_timestep(self, timestep: torch.Tensor) -> torch.Tensor: - timestep = shift_timestep( - timestep, - self.timestep_shift, - self.num_train_timestep, - ) - return timestep.clamp(self.min_timestep, self.max_timestep) - - # ------------------------------------------------------------------ - # ModelBase overrides: lifecycle hooks - # ------------------------------------------------------------------ - - def on_train_start(self) -> None: - assert self.training_config is not None - tc = self.training_config - seed = tc.data.seed - if seed is None: - raise ValueError("training.data.seed must be set " - "for training") - - global_rank = int(getattr(self.world_group, "rank", 0)) - sp_world_size = int(tc.distributed.sp_size or 1) - if sp_world_size > 1: - sp_group_seed = int(seed) + (global_rank // sp_world_size) - set_random_seed(sp_group_seed) - else: - set_random_seed(int(seed) + global_rank) - - self.noise_random_generator = torch.Generator(device="cpu").manual_seed(int(seed)) - self.noise_gen_cuda = torch.Generator(device=self.device).manual_seed(int(seed)) - - def get_rng_generators(self, ) -> dict[str, torch.Generator]: - generators: dict[str, torch.Generator] = {} - if self.noise_random_generator is not None: - generators["noise_cpu"] = (self.noise_random_generator) - if self.noise_gen_cuda is not None: - generators["noise_cuda"] = self.noise_gen_cuda - return generators - - # ------------------------------------------------------------------ - # ModelBase overrides: runtime primitives - # ------------------------------------------------------------------ - - def prepare_batch( - self, - raw_batch: dict[str, Any], - *, - current_vsa_sparsity: float = 0.0, - latents_source: Literal["data", "zeros"] = "data", - ) -> TrainingBatch: - assert self.training_config is not None - tc = self.training_config - dtype = self._get_training_dtype() - device = self.device - - training_batch = TrainingBatch(current_vsa_sparsity=current_vsa_sparsity) - infos = raw_batch.get("info_list") - - if latents_source == "zeros": - clip_feature = raw_batch["clip_feature"] - batch_size = int(clip_feature.shape[0]) - vae_config = ( - tc.pipeline_config.vae_config.arch_config # type: ignore[union-attr] - ) - num_channels = int(vae_config.z_dim) - spatial_compression_ratio = int(vae_config.spatial_compression_ratio) - latent_height = (int(tc.data.num_height) // spatial_compression_ratio) - latent_width = (int(tc.data.num_width) // spatial_compression_ratio) - latents = torch.zeros( - batch_size, - num_channels, - int(tc.data.num_latent_t), - latent_height, - latent_width, - device=device, - dtype=dtype, - ) - elif latents_source == "data": - if "vae_latent" not in raw_batch: - raise ValueError("vae_latent not found in batch " - "and latents_source='data'") - latents = raw_batch["vae_latent"] - latents = latents[:, :, :tc.data.num_latent_t] - latents = latents.to(device, dtype=dtype) - else: - raise ValueError(f"Unknown latents_source: " - f"{latents_source!r}") - - if "clip_feature" not in raw_batch: - raise ValueError("clip_feature must be present for WanGame") - image_embeds = raw_batch["clip_feature"].to(device, dtype=dtype) - - if "first_frame_latent" not in raw_batch: - raise ValueError("first_frame_latent must be present " - "for WanGame") - image_latents = raw_batch["first_frame_latent"] - image_latents = image_latents[:, :, :tc.data.num_latent_t] - image_latents = image_latents.to(device, dtype=dtype) - - pil_image = raw_batch.get("pil_image") - if isinstance(pil_image, torch.Tensor): - training_batch.preprocessed_image = (pil_image.to(device=device)) - else: - training_batch.preprocessed_image = pil_image - - keyboard_cond = raw_batch.get("keyboard_cond") - if (isinstance(keyboard_cond, torch.Tensor) and keyboard_cond.numel() > 0): - training_batch.keyboard_cond = (keyboard_cond.to(device, dtype=dtype)) - else: - training_batch.keyboard_cond = None - - mouse_cond = raw_batch.get("mouse_cond") - if (isinstance(mouse_cond, torch.Tensor) and mouse_cond.numel() > 0): - training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) - else: - training_batch.mouse_cond = None - - temporal_compression_ratio = ( - tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio # type: ignore[union-attr] - ) - expected_num_frames = ((tc.data.num_latent_t - 1) * temporal_compression_ratio + 1) - if (training_batch.keyboard_cond is not None - and int(training_batch.keyboard_cond.shape[1]) != int(expected_num_frames)): - raise ValueError("keyboard_cond temporal dim mismatch: " - f"got {int(training_batch.keyboard_cond.shape[1])}, " - f"expected {int(expected_num_frames)}") - if (training_batch.mouse_cond is not None - and int(training_batch.mouse_cond.shape[1]) != int(expected_num_frames)): - raise ValueError("mouse_cond temporal dim mismatch: " - f"got {int(training_batch.mouse_cond.shape[1])}, " - f"expected {int(expected_num_frames)}") - - training_batch.latents = latents - training_batch.encoder_hidden_states = None - training_batch.encoder_attention_mask = None - training_batch.image_embeds = image_embeds - training_batch.image_latents = image_latents - training_batch.infos = infos - - training_batch.latents = normalize_dit_input("wan", training_batch.latents, self.vae) - training_batch = self._prepare_dit_inputs(training_batch) - training_batch = self._build_attention_metadata(training_batch) - - training_batch.attn_metadata_vsa = copy.deepcopy(training_batch.attn_metadata) - if training_batch.attn_metadata is not None: - training_batch.attn_metadata.VSA_sparsity = 0.0 # type: ignore[attr-defined] - - training_batch.mask_lat_size = (self._build_i2v_mask_latents(image_latents)) - viewmats, intrinsics, action_labels = (self._process_actions(training_batch)) - training_batch.viewmats = viewmats - training_batch.Ks = intrinsics - training_batch.action = action_labels - - return training_batch - - def add_noise( - self, - clean_latents: torch.Tensor, - noise: torch.Tensor, - timestep: torch.Tensor, - ) -> torch.Tensor: - b, t = clean_latents.shape[:2] - noisy = self.noise_scheduler.add_noise( - clean_latents.flatten(0, 1), - noise.flatten(0, 1), - timestep, - ).unflatten(0, (b, t)) - return noisy - - def predict_x0( - self, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - cond_inputs = (self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - )) - input_kwargs = (self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs["image_embeds"], - image_latents=cond_inputs["image_latents"], - mask_lat_size=cond_inputs["mask_lat_size"], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs["keyboard_cond"], - )) - transformer = self._get_transformer(timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) - return pred_x0 - - def predict_noise( - self, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor: - device_type = self.device.type - dtype = noisy_latents.dtype - - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - cond_inputs = (self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - )) - input_kwargs = (self._build_distill_input_kwargs( - noisy_latents, - timestep, - image_embeds=cond_inputs["image_embeds"], - image_latents=cond_inputs["image_latents"], - mask_lat_size=cond_inputs["mask_lat_size"], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs["keyboard_cond"], - )) - transformer = self._get_transformer(timestep) - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - return pred_noise - - def backward( - self, - loss: torch.Tensor, - ctx: Any, - *, - grad_accum_rounds: int, - ) -> None: - timesteps, attn_metadata = ctx - with set_forward_context( - current_timestep=timesteps, - attn_metadata=attn_metadata, - ): - (loss / max(1, int(grad_accum_rounds))).backward() - - # ------------------------------------------------------------------ - # Internal helpers - # ------------------------------------------------------------------ - - def _get_training_dtype(self) -> torch.dtype: - return torch.bfloat16 - - def _init_timestep_mechanics(self) -> None: - assert self.training_config is not None - tc = self.training_config - self.timestep_shift = float(tc.pipeline_config.flow_shift # type: ignore[union-attr] - ) - self.num_train_timestep = int(self.noise_scheduler.num_train_timesteps) - self.min_timestep = 0 - self.max_timestep = self.num_train_timestep - - def _sample_timesteps(self, batch_size: int, device: torch.device) -> torch.Tensor: - if self.noise_random_generator is None: - raise RuntimeError("on_train_start() must be called " - "before prepare_batch()") - assert self.training_config is not None - tc = self.training_config - - u = compute_density_for_timestep_sampling( - weighting_scheme=tc.model.weighting_scheme, - batch_size=batch_size, - generator=self.noise_random_generator, - logit_mean=tc.model.logit_mean, - logit_std=tc.model.logit_std, - mode_scale=tc.model.mode_scale, - ) - indices = (u * self.noise_scheduler.config.num_train_timesteps).long() - return self.noise_scheduler.timesteps[indices].to(device=device) - - def _build_attention_metadata(self, training_batch: TrainingBatch) -> TrainingBatch: - assert self.training_config is not None - tc = self.training_config - latents_shape = training_batch.raw_latent_shape - patch_size = ( - tc.pipeline_config.dit_config.patch_size # type: ignore[union-attr] - ) - current_vsa_sparsity = (training_batch.current_vsa_sparsity) - assert latents_shape is not None - assert training_batch.timesteps is not None - - if (envs.FASTVIDEO_ATTENTION_BACKEND == "VIDEO_SPARSE_ATTN"): - if (not is_vsa_available() or VideoSparseAttentionMetadataBuilder is None): - raise ImportError("FASTVIDEO_ATTENTION_BACKEND is " - "VIDEO_SPARSE_ATTN, but " - "fastvideo_kernel is not correctly " - "installed or detected.") - training_batch.attn_metadata = VideoSparseAttentionMetadataBuilder().build( # type: ignore[misc] - raw_latent_shape=latents_shape[2:5], - current_timestep=(training_batch.timesteps), - patch_size=patch_size, - VSA_sparsity=current_vsa_sparsity, - device=self.device, - ) - elif (envs.FASTVIDEO_ATTENTION_BACKEND == "VMOBA_ATTN"): - if (not is_vmoba_available() or VideoMobaAttentionMetadataBuilder is None): - raise ImportError("FASTVIDEO_ATTENTION_BACKEND is " - "VMOBA_ATTN, but fastvideo_kernel " - "(or flash_attn>=2.7.4) is not " - "correctly installed.") - moba_params = tc.model.moba_config.copy() - moba_params.update({ - "current_timestep": (training_batch.timesteps), - "raw_latent_shape": (training_batch.raw_latent_shape[2:5]), - "patch_size": patch_size, - "device": self.device, - }) - training_batch.attn_metadata = VideoMobaAttentionMetadataBuilder().build(** - moba_params) # type: ignore[misc] - else: - training_batch.attn_metadata = None - - return training_batch - - def _prepare_dit_inputs(self, training_batch: TrainingBatch) -> TrainingBatch: - assert self.training_config is not None - tc = self.training_config - latents = training_batch.latents - assert isinstance(latents, torch.Tensor) - batch_size = latents.shape[0] - - if self.noise_gen_cuda is None: - raise RuntimeError("on_train_start() must be called " - "before prepare_batch()") - - noise = torch.randn( - latents.shape, - generator=self.noise_gen_cuda, - device=latents.device, - dtype=latents.dtype, - ) - timesteps = self._sample_timesteps(batch_size, latents.device) - if int(tc.distributed.sp_size or 1) > 1: - self.sp_group.broadcast(timesteps, src=0) - - sigmas = get_sigmas( - self.noise_scheduler, - latents.device, - timesteps, - n_dim=latents.ndim, - dtype=latents.dtype, - ) - noisy_model_input = ((1.0 - sigmas) * latents + sigmas * noise) - - training_batch.noisy_model_input = (noisy_model_input) - training_batch.timesteps = timesteps - training_batch.sigmas = sigmas - training_batch.noise = noise - training_batch.raw_latent_shape = latents.shape - - training_batch.latents = (training_batch.latents.permute(0, 2, 1, 3, 4)) - return training_batch - - def _build_i2v_mask_latents(self, image_latents: torch.Tensor) -> torch.Tensor: - assert self.training_config is not None - tc = self.training_config - temporal_compression_ratio = ( - tc.pipeline_config.vae_config.arch_config.temporal_compression_ratio # type: ignore[union-attr] - ) - num_frames = ((tc.data.num_latent_t - 1) * temporal_compression_ratio + 1) - - ( - batch_size, - _num_channels, - _t, - latent_height, - latent_width, - ) = image_latents.shape - mask_lat_size = torch.ones( - batch_size, - 1, - num_frames, - latent_height, - latent_width, - ) - mask_lat_size[:, :, 1:] = 0 - - first_frame_mask = mask_lat_size[:, :, :1] - first_frame_mask = torch.repeat_interleave( - first_frame_mask, - dim=2, - repeats=temporal_compression_ratio, - ) - mask_lat_size = torch.cat( - [first_frame_mask, mask_lat_size[:, :, 1:]], - dim=2, - ) - mask_lat_size = mask_lat_size.view( - batch_size, - -1, - temporal_compression_ratio, - latent_height, - latent_width, - ) - mask_lat_size = mask_lat_size.transpose(1, 2) - return mask_lat_size.to( - device=image_latents.device, - dtype=image_latents.dtype, - ) - - def _process_actions(self, training_batch: TrainingBatch) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - keyboard_cond = getattr(training_batch, "keyboard_cond", None) - mouse_cond = getattr(training_batch, "mouse_cond", None) - if keyboard_cond is None or mouse_cond is None: - raise ValueError("WanGame batch must provide " - "keyboard_cond and mouse_cond") - - from fastvideo.models.dits.hyworld.pose import ( - process_custom_actions, ) - - batch_size = int(training_batch.noisy_model_input.shape[0] # type: ignore[union-attr] - ) - viewmats_list: list[torch.Tensor] = [] - intrinsics_list: list[torch.Tensor] = [] - action_labels_list: list[torch.Tensor] = [] - for b in range(batch_size): - v, i, a = process_custom_actions(keyboard_cond[b], mouse_cond[b]) - viewmats_list.append(v) - intrinsics_list.append(i) - action_labels_list.append(a) - - viewmats = torch.stack(viewmats_list, dim=0).to(device=self.device, dtype=torch.bfloat16) - intrinsics = torch.stack(intrinsics_list, dim=0).to(device=self.device, dtype=torch.bfloat16) - action_labels = torch.stack(action_labels_list, dim=0).to(device=self.device, dtype=torch.bfloat16) - - num_latent_t = int(training_batch.noisy_model_input.shape[2] # type: ignore[union-attr] - ) - if int(action_labels.shape[1]) != num_latent_t: - raise ValueError("Action conditioning temporal dim " - "mismatch: " - f"action={tuple(action_labels.shape)} " - f"vs latent_t={num_latent_t}") - if int(viewmats.shape[1]) != num_latent_t: - raise ValueError("Viewmats temporal dim mismatch: " - f"viewmats={tuple(viewmats.shape)} " - f"vs latent_t={num_latent_t}") - - return viewmats, intrinsics, action_labels - - def _build_distill_input_kwargs( - self, - noisy_video_latents: torch.Tensor, - timestep: torch.Tensor, - *, - image_embeds: torch.Tensor, - image_latents: torch.Tensor, - mask_lat_size: torch.Tensor, - viewmats: torch.Tensor | None, - Ks: torch.Tensor | None, - action: torch.Tensor | None, - mouse_cond: torch.Tensor | None, - keyboard_cond: torch.Tensor | None, - ) -> dict[str, Any]: - hidden_states = torch.cat( - [ - noisy_video_latents.permute(0, 2, 1, 3, 4), - mask_lat_size, - image_latents, - ], - dim=1, - ) - return { - "hidden_states": hidden_states, - "encoder_hidden_states": None, - "timestep": timestep.to(device=self.device, dtype=torch.bfloat16), - "encoder_hidden_states_image": image_embeds, - "viewmats": viewmats, - "Ks": Ks, - "action": action, - "mouse_cond": mouse_cond, - "keyboard_cond": keyboard_cond, - "return_dict": False, - } - - def _select_cfg_condition_inputs( - self, - batch: TrainingBatch, - *, - conditional: bool, - cfg_uncond: dict[str, Any] | None, - ) -> dict[str, Any]: - image_embeds = batch.image_embeds - image_latents = batch.image_latents - mask_lat_size = batch.mask_lat_size - if image_embeds is None: - raise RuntimeError("WanGameModel requires " - "TrainingBatch.image_embeds") - if image_latents is None: - raise RuntimeError("WanGameModel requires " - "TrainingBatch.image_latents") - if mask_lat_size is None: - raise RuntimeError("WanGameModel requires " - "TrainingBatch.mask_lat_size") - - viewmats = getattr(batch, "viewmats", None) - Ks = getattr(batch, "Ks", None) - action = getattr(batch, "action", None) - mouse_cond = getattr(batch, "mouse_cond", None) - keyboard_cond = getattr(batch, "keyboard_cond", None) - - if conditional or cfg_uncond is None: - return { - "image_embeds": image_embeds, - "image_latents": image_latents, - "mask_lat_size": mask_lat_size, - "viewmats": viewmats, - "Ks": Ks, - "action": action, - "mouse_cond": mouse_cond, - "keyboard_cond": keyboard_cond, - } - - on_missing_raw = cfg_uncond.get("on_missing", "error") - if not isinstance(on_missing_raw, str): - raise ValueError("method_config.cfg_uncond.on_missing " - "must be a string, got " - f"{type(on_missing_raw).__name__}") - on_missing = on_missing_raw.strip().lower() - if on_missing not in {"error", "ignore"}: - raise ValueError("method_config.cfg_uncond.on_missing " - "must be one of {error, ignore}, got " - f"{on_missing_raw!r}") - - supported_channels = {"image", "action"} - for channel, policy_raw in cfg_uncond.items(): - if channel in {"on_missing"}: - continue - if channel in supported_channels: - continue - if policy_raw is None: - continue - if not isinstance(policy_raw, str): - raise ValueError("method_config.cfg_uncond values " - "must be strings, got " - f"{channel}=" - f"{type(policy_raw).__name__}") - policy = policy_raw.strip().lower() - if policy == "keep": - continue - if on_missing == "ignore": - continue - raise ValueError("WanGameModel does not support " - "cfg_uncond channel " - f"{channel!r} (policy={policy!r}). " - "Set cfg_uncond.on_missing=ignore or " - "remove the channel.") - - def _get_policy(channel: str) -> str: - raw = cfg_uncond.get(channel, "keep") - if raw is None: - return "keep" - if not isinstance(raw, str): - raise ValueError("method_config.cfg_uncond values " - "must be strings, got " - f"{channel}={type(raw).__name__}") - policy = raw.strip().lower() - if policy not in {"keep", "zero", "drop"}: - raise ValueError("method_config.cfg_uncond values " - "must be one of " - "{keep, zero, drop}, got " - f"{channel}={raw!r}") - return policy - - image_policy = _get_policy("image") - if image_policy == "zero": - image_embeds = torch.zeros_like(image_embeds) - image_latents = torch.zeros_like(image_latents) - mask_lat_size = torch.zeros_like(mask_lat_size) - elif image_policy == "drop": - raise ValueError("cfg_uncond.image=drop is not supported " - "for WanGame I2V; use " - "cfg_uncond.image=zero or keep.") - - action_policy = _get_policy("action") - if action_policy == "zero": - if (viewmats is None or Ks is None or action is None): - if on_missing == "ignore": - pass - else: - raise ValueError("cfg_uncond.action=zero requires " - "action conditioning tensors, " - "but TrainingBatch is missing " - "{viewmats, Ks, action}.") - else: - viewmats = torch.zeros_like(viewmats) - Ks = torch.zeros_like(Ks) - action = torch.zeros_like(action) - if mouse_cond is not None: - mouse_cond = torch.zeros_like(mouse_cond) - if keyboard_cond is not None: - keyboard_cond = torch.zeros_like(keyboard_cond) - elif action_policy == "drop": - viewmats = None - Ks = None - action = None - mouse_cond = None - keyboard_cond = None - - return { - "image_embeds": image_embeds, - "image_latents": image_latents, - "mask_lat_size": mask_lat_size, - "viewmats": viewmats, - "Ks": Ks, - "action": action, - "mouse_cond": mouse_cond, - "keyboard_cond": keyboard_cond, - } - - def _get_transformer(self, timestep: torch.Tensor) -> torch.nn.Module: - return self.transformer diff --git a/fastvideo/train/models/wangame/wangame_causal.py b/fastvideo/train/models/wangame/wangame_causal.py deleted file mode 100644 index 902ed7824..000000000 --- a/fastvideo/train/models/wangame/wangame_causal.py +++ /dev/null @@ -1,503 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""WanGame causal model plugin (per-role instance, streaming/cache).""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, Literal, TYPE_CHECKING - -import torch - -from fastvideo.forward_context import set_forward_context -from fastvideo.models.utils import pred_noise_to_pred_video - -from fastvideo.train.models.base import CausalModelBase -from fastvideo.train.models.wangame.wangame import WanGameModel - -if TYPE_CHECKING: - from fastvideo.train.utils.training_config import ( - TrainingConfig, ) - - -@dataclass(slots=True) -class _StreamingCaches: - kv_cache: list[dict[str, Any]] - crossattn_cache: list[dict[str, Any]] | None - frame_seq_length: int - local_attn_size: int - sliding_window_num_frames: int - batch_size: int - dtype: torch.dtype - device: torch.device - - -class WanGameCausalModel(WanGameModel, CausalModelBase): - """WanGame per-role model with causal/streaming primitives.""" - - _transformer_cls_name: str = ("CausalWanGameActionTransformer3DModel") - - def __init__( - self, - *, - init_from: str, - training_config: TrainingConfig, - trainable: bool = True, - disable_custom_init_weights: bool = False, - flow_shift: float = 3.0, - enable_gradient_checkpointing_type: str | None = None, - ) -> None: - super().__init__( - init_from=init_from, - trainable=trainable, - disable_custom_init_weights=disable_custom_init_weights, - flow_shift=flow_shift, - enable_gradient_checkpointing_type=(enable_gradient_checkpointing_type), - training_config=training_config, - ) - self._streaming_caches: dict[tuple[int, str], _StreamingCaches] = {} - - # --- CausalModelBase override: clear_caches --- - def clear_caches(self, *, cache_tag: str = "pos") -> None: - self._streaming_caches.pop((id(self), str(cache_tag)), None) - - # --- CausalModelBase override: predict_noise_streaming --- - def predict_noise_streaming( - self, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: Any, - *, - conditional: bool, - cache_tag: str = "pos", - store_kv: bool = False, - cur_start_frame: int = 0, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor | None: - if attn_kind == "dense": - attn_metadata = batch.attn_metadata - elif attn_kind == "vsa": - attn_metadata = batch.attn_metadata_vsa - else: - raise ValueError(f"Unknown attn_kind: {attn_kind!r}") - - cache_tag = str(cache_tag) - cur_start_frame = int(cur_start_frame) - if cur_start_frame < 0: - raise ValueError("cur_start_frame must be >= 0") - - batch_size = int(noisy_latents.shape[0]) - num_frames = int(noisy_latents.shape[1]) - timestep_full = self._ensure_per_frame_timestep( - timestep=timestep, - batch_size=batch_size, - num_frames=num_frames, - device=noisy_latents.device, - ) - - transformer = self._get_transformer(timestep_full) - caches = self._get_or_init_streaming_caches( - cache_tag=cache_tag, - transformer=transformer, - noisy_latents=noisy_latents, - ) - - frame_seq_length = int(caches.frame_seq_length) - kv_cache = caches.kv_cache - crossattn_cache = caches.crossattn_cache - - if (self._should_snapshot_streaming_cache() and torch.is_grad_enabled()): - kv_cache = self._snapshot_kv_cache_indices(kv_cache) - - model_kwargs: dict[str, Any] = { - "kv_cache": kv_cache, - "crossattn_cache": crossattn_cache, - "current_start": cur_start_frame * frame_seq_length, - "start_frame": cur_start_frame, - "is_cache": bool(store_kv), - } - - device_type = self.device.type - dtype = noisy_latents.dtype - with torch.autocast(device_type, dtype=dtype), set_forward_context( - current_timestep=batch.timesteps, - attn_metadata=attn_metadata, - ): - cond_inputs = self._select_cfg_condition_inputs( - batch, - conditional=conditional, - cfg_uncond=cfg_uncond, - ) - cond_inputs = self._slice_cond_inputs_for_streaming( - cond_inputs=cond_inputs, - cur_start_frame=cur_start_frame, - num_frames=num_frames, - ) - input_kwargs = self._build_distill_input_kwargs( - noisy_latents, - timestep_full, - image_embeds=cond_inputs["image_embeds"], - image_latents=cond_inputs["image_latents"], - mask_lat_size=cond_inputs["mask_lat_size"], - viewmats=cond_inputs["viewmats"], - Ks=cond_inputs["Ks"], - action=cond_inputs["action"], - mouse_cond=cond_inputs["mouse_cond"], - keyboard_cond=cond_inputs["keyboard_cond"], - ) - - input_kwargs["timestep"] = timestep_full.to(device=self.device, dtype=torch.long) - input_kwargs.update(model_kwargs) - - if store_kv: - with torch.no_grad(): - _ = transformer(**input_kwargs) - return None - - pred_noise = transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - return pred_noise - - # --- CausalModelBase override: predict_x0_streaming --- - def predict_x0_streaming( - self, - noisy_latents: torch.Tensor, - timestep: torch.Tensor, - batch: Any, - *, - conditional: bool, - cache_tag: str = "pos", - store_kv: bool = False, - cur_start_frame: int = 0, - cfg_uncond: dict[str, Any] | None = None, - attn_kind: Literal["dense", "vsa"] = "dense", - ) -> torch.Tensor | None: - pred_noise = self.predict_noise_streaming( - noisy_latents, - timestep, - batch, - conditional=conditional, - cache_tag=cache_tag, - store_kv=store_kv, - cur_start_frame=cur_start_frame, - cfg_uncond=cfg_uncond, - attn_kind=attn_kind, - ) - if pred_noise is None: - return None - - pred_x0 = pred_noise_to_pred_video( - pred_noise=pred_noise.flatten(0, 1), - noise_input_latent=noisy_latents.flatten(0, 1), - timestep=self.shift_and_clamp_timestep( - self._ensure_per_frame_timestep( - timestep=timestep, - batch_size=int(noisy_latents.shape[0]), - num_frames=int(noisy_latents.shape[1]), - device=noisy_latents.device, - ).flatten()), - scheduler=self.noise_scheduler, - ).unflatten(0, pred_noise.shape[:2]) - return pred_x0 - - # --- internal helpers --- - - def _ensure_per_frame_timestep( - self, - *, - timestep: torch.Tensor, - batch_size: int, - num_frames: int, - device: torch.device, - ) -> torch.Tensor: - if timestep.ndim == 0: - return (timestep.view(1, 1).expand(batch_size, num_frames).to(device=device)) - if timestep.ndim == 1: - if int(timestep.shape[0]) == batch_size: - return (timestep.view(batch_size, 1).expand(batch_size, num_frames).to(device=device)) - raise ValueError("streaming timestep must be scalar, [B], or " - f"[B, T]; got shape={tuple(timestep.shape)}") - if timestep.ndim == 2: - return timestep.to(device=device) - raise ValueError("streaming timestep must be scalar, [B], or [B, T]; " - f"got ndim={int(timestep.ndim)}") - - def _slice_cond_inputs_for_streaming( - self, - *, - cond_inputs: dict[str, Any], - cur_start_frame: int, - num_frames: int, - ) -> dict[str, Any]: - start = int(cur_start_frame) - num_frames = int(num_frames) - if num_frames <= 0: - raise ValueError("num_frames must be positive for streaming") - if start < 0: - raise ValueError("cur_start_frame must be >= 0 for streaming") - end = start + num_frames - - sliced: dict[str, Any] = dict(cond_inputs) - - image_latents = cond_inputs.get("image_latents") - if isinstance(image_latents, torch.Tensor): - sliced["image_latents"] = image_latents[:, :, start:end] - - mask_lat_size = cond_inputs.get("mask_lat_size") - if isinstance(mask_lat_size, torch.Tensor): - sliced["mask_lat_size"] = mask_lat_size[:, :, start:end] - - viewmats = cond_inputs.get("viewmats") - if isinstance(viewmats, torch.Tensor): - sliced["viewmats"] = viewmats[:, start:end] - - Ks = cond_inputs.get("Ks") - if isinstance(Ks, torch.Tensor): - sliced["Ks"] = Ks[:, start:end] - - action = cond_inputs.get("action") - if isinstance(action, torch.Tensor): - sliced["action"] = action[:, start:end] - - temporal_compression_ratio = int( - self.training_config.pipeline_config.vae_config.arch_config.temporal_compression_ratio) - raw_end_frame_idx = (1 + temporal_compression_ratio * max(0, end - 1)) - - mouse_cond = cond_inputs.get("mouse_cond") - if isinstance(mouse_cond, torch.Tensor): - sliced["mouse_cond"] = mouse_cond[:, :raw_end_frame_idx] - - keyboard_cond = cond_inputs.get("keyboard_cond") - if isinstance(keyboard_cond, torch.Tensor): - sliced["keyboard_cond"] = keyboard_cond[:, :raw_end_frame_idx] - - return sliced - - def _get_or_init_streaming_caches( - self, - *, - cache_tag: str, - transformer: torch.nn.Module, - noisy_latents: torch.Tensor, - ) -> _StreamingCaches: - key = (id(self), cache_tag) - cached = self._streaming_caches.get(key) - - batch_size = int(noisy_latents.shape[0]) - dtype = noisy_latents.dtype - device = noisy_latents.device - - frame_seq_length = self._compute_frame_seq_length(transformer, noisy_latents) - local_attn_size = self._get_local_attn_size(transformer) - sliding_window_num_frames = (self._get_sliding_window_num_frames(transformer)) - - meta = ( - frame_seq_length, - local_attn_size, - sliding_window_num_frames, - batch_size, - dtype, - device, - ) - - if cached is not None: - cached_meta = ( - cached.frame_seq_length, - cached.local_attn_size, - cached.sliding_window_num_frames, - cached.batch_size, - cached.dtype, - cached.device, - ) - if cached_meta == meta: - return cached - - kv_cache = self._initialize_kv_cache( - transformer=transformer, - batch_size=batch_size, - dtype=dtype, - device=device, - frame_seq_length=frame_seq_length, - local_attn_size=local_attn_size, - sliding_window_num_frames=sliding_window_num_frames, - checkpoint_safe=(self._should_use_checkpoint_safe_kv_cache()), - ) - crossattn_cache = self._initialize_crossattn_cache(transformer=transformer, device=device) - - caches = _StreamingCaches( - kv_cache=kv_cache, - crossattn_cache=crossattn_cache, - frame_seq_length=frame_seq_length, - local_attn_size=local_attn_size, - sliding_window_num_frames=sliding_window_num_frames, - batch_size=batch_size, - dtype=dtype, - device=device, - ) - self._streaming_caches[key] = caches - return caches - - def _compute_frame_seq_length( - self, - transformer: torch.nn.Module, - noisy_latents: torch.Tensor, - ) -> int: - latent_seq_length = int(noisy_latents.shape[-1]) * int(noisy_latents.shape[-2]) - patch_size = getattr(transformer, "patch_size", None) - if patch_size is None: - patch_size = getattr( - getattr(transformer, "config", None), - "arch_config", - None, - ) - patch_size = getattr(patch_size, "patch_size", None) - if patch_size is None: - raise ValueError("Unable to determine transformer.patch_size " - "for causal streaming") - patch_ratio = int(patch_size[-1]) * int(patch_size[-2]) - if patch_ratio <= 0: - raise ValueError("Invalid patch_size for causal streaming") - return latent_seq_length // patch_ratio - - def _get_sliding_window_num_frames(self, transformer: torch.nn.Module) -> int: - cfg = getattr(transformer, "config", None) - arch_cfg = getattr(cfg, "arch_config", None) - value = (getattr(arch_cfg, "sliding_window_num_frames", None) if arch_cfg is not None else None) - if value is None: - return 15 - return int(value) - - def _get_local_attn_size(self, transformer: torch.nn.Module) -> int: - try: - value = getattr(transformer, "local_attn_size", -1) - except Exception: - value = -1 - if value is None: - return -1 - return int(value) - - def _initialize_kv_cache( - self, - *, - transformer: torch.nn.Module, - batch_size: int, - dtype: torch.dtype, - device: torch.device, - frame_seq_length: int, - local_attn_size: int, - sliding_window_num_frames: int, - checkpoint_safe: bool, - ) -> list[dict[str, Any]]: - num_blocks = len(getattr(transformer, "blocks", [])) - if num_blocks <= 0: - raise ValueError("Unexpected transformer.blocks for causal " - "streaming") - - try: - num_attention_heads = int(transformer.num_attention_heads # type: ignore[attr-defined] - ) - except AttributeError as e: - raise ValueError("Transformer is missing num_attention_heads") from e - - try: - attention_head_dim = int(transformer.attention_head_dim # type: ignore[attr-defined] - ) - except AttributeError: - try: - hidden_size = int(transformer.hidden_size # type: ignore[attr-defined] - ) - except AttributeError as e: - raise ValueError("Transformer is missing attention_head_dim " - "and hidden_size") from e - attention_head_dim = hidden_size // max(1, num_attention_heads) - - if local_attn_size != -1: - kv_cache_size = (int(local_attn_size) * int(frame_seq_length)) - else: - kv_cache_size = int(frame_seq_length) * int(sliding_window_num_frames) - - if checkpoint_safe: - tc = getattr(self, "training_config", None) - total_frames = int(tc.data.num_frames if tc is not None else 0) - if total_frames <= 0: - raise ValueError("training.num_frames must be set to enable " - "checkpoint-safe streaming KV cache; " - f"got {total_frames}") - kv_cache_size = max( - kv_cache_size, - int(frame_seq_length) * total_frames, - ) - - kv_cache: list[dict[str, Any]] = [] - for _ in range(num_blocks): - kv_cache.append({ - "k": - torch.zeros( - [ - batch_size, - kv_cache_size, - num_attention_heads, - attention_head_dim, - ], - dtype=dtype, - device=device, - ), - "v": - torch.zeros( - [ - batch_size, - kv_cache_size, - num_attention_heads, - attention_head_dim, - ], - dtype=dtype, - device=device, - ), - "global_end_index": - torch.zeros((), dtype=torch.long, device=device), - "local_end_index": - torch.zeros((), dtype=torch.long, device=device), - }) - - return kv_cache - - def _should_use_checkpoint_safe_kv_cache(self) -> bool: - tc = getattr(self, "training_config", None) - if tc is not None: - checkpointing_type = tc.model.enable_gradient_checkpointing_type - else: - checkpointing_type = None - return bool(checkpointing_type) and bool(self._trainable) - - def _should_snapshot_streaming_cache(self) -> bool: - return self._should_use_checkpoint_safe_kv_cache() - - def _snapshot_kv_cache_indices(self, kv_cache: list[dict[str, Any]]) -> list[dict[str, Any]]: - snapshot: list[dict[str, Any]] = [] - for block_cache in kv_cache: - global_end_index = block_cache.get("global_end_index") - local_end_index = block_cache.get("local_end_index") - if not isinstance(global_end_index, torch.Tensor) or not isinstance(local_end_index, torch.Tensor): - raise ValueError("Unexpected kv_cache index tensors; expected " - "tensors at kv_cache[*].{global_end_index, " - "local_end_index}") - - copied = dict(block_cache) - copied["global_end_index"] = (global_end_index.detach().clone()) - copied["local_end_index"] = (local_end_index.detach().clone()) - snapshot.append(copied) - return snapshot - - def _initialize_crossattn_cache( - self, - *, - transformer: torch.nn.Module, - device: torch.device, - ) -> list[dict[str, Any]] | None: - num_blocks = len(getattr(transformer, "blocks", [])) - if num_blocks <= 0: - return None - return [{ - "is_init": False, - "k": torch.empty(0, device=device), - "v": torch.empty(0, device=device), - } for _ in range(num_blocks)] diff --git a/fastvideo/training/wangame_ar_diffusion_pipeline.py b/fastvideo/training/wangame_ar_diffusion_pipeline.py deleted file mode 100644 index 7adbbb6c8..000000000 --- a/fastvideo/training/wangame_ar_diffusion_pipeline.py +++ /dev/null @@ -1,527 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -import sys -from copy import deepcopy -from typing import Any, cast - -import numpy as np -import torch -import torch.nn.functional as F - -from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame -from fastvideo.distributed import get_local_torch_device -from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.forward_context import set_forward_context -from fastvideo.logger import init_logger -from fastvideo.models.dits.hyworld.pose import process_custom_actions -from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( - SelfForcingFlowMatchScheduler) -from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( - WanGameCausalDMDPipeline) -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch -from fastvideo.training.training_pipeline import TrainingPipeline -from fastvideo.training.training_utils import ( - clip_grad_norm_while_handling_failing_dtensor_cases) -from fastvideo.utils import shallow_asdict - -logger = init_logger(__name__) - - -class WanGameARDiffusionPipeline(TrainingPipeline): - - _required_config_modules = ["scheduler", "transformer", "vae"] - - def initialize_pipeline(self, fastvideo_args: FastVideoArgs): - self.modules["scheduler"] = SelfForcingFlowMatchScheduler( - shift=fastvideo_args.pipeline_config.flow_shift, - sigma_min=0.0, - extra_one_step=True) - self.modules["scheduler"].set_timesteps(num_inference_steps=1000, - training=True) - - def set_schemas(self): - self.train_dataset_schema = pyarrow_schema_wangame - - def initialize_training_pipeline(self, training_args: TrainingArgs): - super().initialize_training_pipeline(training_args) - - self.vae = self.get_module("vae") - self.vae.requires_grad_(False) - - self.num_frame_per_block = getattr(training_args, 'num_frame_per_block', 3) - self.timestep_shift = training_args.pipeline_config.flow_shift - self.ar_noise_scheduler = SelfForcingFlowMatchScheduler( - shift=self.timestep_shift, sigma_min=0.0, extra_one_step=True) - self.ar_noise_scheduler.set_timesteps(num_inference_steps=1000, - training=True) - - logger.info("AR Diffusion pipeline initialized with " - "num_frame_per_block=%d, timestep_shift=%.1f", - self.num_frame_per_block, self.timestep_shift) - - def initialize_validation_pipeline(self, training_args: TrainingArgs): - logger.info("Initializing validation pipeline...") - args_copy = deepcopy(training_args) - args_copy.inference_mode = True - - validation_scheduler = SelfForcingFlowMatchScheduler( - shift=args_copy.pipeline_config.flow_shift, - sigma_min=0.0, - extra_one_step=True) - validation_scheduler.set_timesteps(num_inference_steps=1000, - training=True) - - num_val_steps = int( - training_args.validation_sampling_steps.split(",")[0]) - step_size = 1000 // num_val_steps - args_copy.pipeline_config.dmd_denoising_steps = list( - range(1000, 0, -step_size)) - args_copy.pipeline_config.warp_denoising_step = True - training_args.pipeline_config.dmd_denoising_steps = ( - args_copy.pipeline_config.dmd_denoising_steps) - training_args.pipeline_config.warp_denoising_step = True - - logger.info("Validation: %d-step causal denoising, " - "dmd_denoising_steps has %d entries", - num_val_steps, - len(args_copy.pipeline_config.dmd_denoising_steps)) - - self.validation_pipeline = WanGameCausalDMDPipeline.from_pretrained( - training_args.model_path, - args=args_copy, - inference_mode=True, - loaded_modules={ - "transformer": self.get_module("transformer"), - "vae": self.get_module("vae"), - "scheduler": validation_scheduler, - }, - tp_size=training_args.tp_size, - sp_size=training_args.sp_size, - num_gpus=training_args.num_gpus, - pin_cpu_memory=training_args.pin_cpu_memory, - dit_cpu_offload=True) - - def _get_timestep( - self, - min_timestep: int, - max_timestep: int, - batch_size: int, - num_frame: int, - num_frame_per_block: int, - uniform_timestep: bool = False, - ) -> torch.Tensor: - """ - Sample per-block timesteps. - """ - device = get_local_torch_device() - if uniform_timestep: - timestep = torch.randint( - min_timestep, max_timestep, [batch_size, 1], - device=device, dtype=torch.long - ).repeat(1, num_frame) - return timestep - else: - timestep = torch.randint( - min_timestep, max_timestep, [batch_size, num_frame], - device=device, dtype=torch.long - ) - # Make the noise level the same within every block - timestep = timestep.reshape( - timestep.shape[0], -1, num_frame_per_block) - timestep[:, :, 1:] = timestep[:, :, 0:1] - timestep = timestep.reshape(timestep.shape[0], -1) - return timestep - - def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: - batch = next(self.train_loader_iter, None) # type: ignore - if batch is None: - self.current_epoch += 1 - logger.info("Starting epoch %s", self.current_epoch) - self.train_dataset.sampler.set_epoch(self.current_epoch) - self.train_loader_iter = iter(self.train_dataloader) - batch = next(self.train_loader_iter) - - latents = batch['vae_latent'] - latents = latents[:, :, :self.training_args.num_latent_t] - clip_features = batch['clip_feature'] - image_latents = batch['first_frame_latent'] - image_latents = image_latents[:, :, :self.training_args.num_latent_t] - pil_image = batch['pil_image'] - infos = batch['info_list'] - - training_batch.latents = latents.to(get_local_torch_device(), - dtype=torch.bfloat16) - training_batch.encoder_hidden_states = None - training_batch.encoder_attention_mask = None - training_batch.preprocessed_image = pil_image.to( - get_local_torch_device()) - training_batch.image_embeds = clip_features.to(get_local_torch_device()) - training_batch.image_latents = image_latents.to( - get_local_torch_device()) - training_batch.infos = infos - - # Action conditioning - if 'mouse_cond' in batch and batch['mouse_cond'].numel() > 0: - training_batch.mouse_cond = batch['mouse_cond'].to( - get_local_torch_device(), dtype=torch.bfloat16) - else: - training_batch.mouse_cond = None - - if 'keyboard_cond' in batch and batch['keyboard_cond'].numel() > 0: - training_batch.keyboard_cond = batch['keyboard_cond'].to( - get_local_torch_device(), dtype=torch.bfloat16) - else: - training_batch.keyboard_cond = None - - # Validate action temporal dimensions match video num_frames - expected_num_frames = (self.training_args.num_latent_t - 1) * 4 + 1 - if training_batch.keyboard_cond is not None: - assert training_batch.keyboard_cond.shape[1] >= expected_num_frames, ( - f"keyboard_cond has {training_batch.keyboard_cond.shape[1]} frames " - f"but need at least {expected_num_frames}") - training_batch.keyboard_cond = training_batch.keyboard_cond[:, :expected_num_frames] - if training_batch.mouse_cond is not None: - assert training_batch.mouse_cond.shape[1] >= expected_num_frames, ( - f"mouse_cond has {training_batch.mouse_cond.shape[1]} frames " - f"but need at least {expected_num_frames}") - training_batch.mouse_cond = training_batch.mouse_cond[:, :expected_num_frames] - - return training_batch - - def _prepare_dit_inputs(self, - training_batch: TrainingBatch) -> TrainingBatch: - """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" - assert self.training_args is not None - latents = training_batch.latents # [B, C, T, H, W] - batch_size = latents.shape[0] - num_latent_t = latents.shape[2] - - # Reshape latents to [B, T, C, H, W] for per-frame operations - latents_btchw = latents.permute(0, 2, 1, 3, 4) # [B, T, C, H, W] - - # Sample per-block independent timestep indices: [B, T] - timestep_indices = self._get_timestep( - min_timestep=0, - max_timestep=self.ar_noise_scheduler.num_train_timesteps, - batch_size=batch_size, - num_frame=num_latent_t, - num_frame_per_block=self.num_frame_per_block, - uniform_timestep=False) - - # Convert indices to actual timestep values: [B, T] - self.ar_noise_scheduler.timesteps = self.ar_noise_scheduler.timesteps.to( - get_local_torch_device()) - timesteps = self.ar_noise_scheduler.timesteps[timestep_indices] - - # Generate noise: [B, T, C, H, W] - noise = torch.randn_like(latents_btchw) - - # Add noise per-frame: noisy = (1-σ) * clean + σ * noise - noisy_latents = self.ar_noise_scheduler.add_noise( - latents_btchw.flatten(0, 1), # [B*T, C, H, W] - noise.flatten(0, 1), # [B*T, C, H, W] - timesteps.flatten(0, 1) # [B*T] - ).unflatten(0, (batch_size, num_latent_t)) # [B, T, C, H, W] - - # Convert back to [B, C, T, H, W] for transformer input - noisy_model_input = noisy_latents.permute(0, 2, 1, 3, 4) - - # I2V concatenation: [mask(1ch), image_latent(16ch)] → 17+16=33 ch total - assert isinstance(training_batch.image_latents, torch.Tensor) - image_latents = training_batch.image_latents.to( - get_local_torch_device(), dtype=torch.bfloat16) - - temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (num_latent_t - 1) * temporal_compression_ratio + 1 - _, num_channels, _, latent_height, latent_width = image_latents.shape - mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, - latent_width) - mask_lat_size[:, :, 1:] = 0 - - first_frame_mask = mask_lat_size[:, :, :1] - first_frame_mask = torch.repeat_interleave( - first_frame_mask, dim=2, repeats=temporal_compression_ratio) - mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], - dim=2) - mask_lat_size = mask_lat_size.view(batch_size, -1, - temporal_compression_ratio, - latent_height, latent_width) - mask_lat_size = mask_lat_size.transpose(1, 2) - mask_lat_size = mask_lat_size.to( - image_latents.device).to(dtype=torch.bfloat16) - - noisy_model_input = torch.cat( - [noisy_model_input, mask_lat_size, image_latents], dim=1) - - # Compute flow-matching training target: target = noise - clean - # Shape: [B, T, C, H, W] - training_target = self.ar_noise_scheduler.training_target( - latents_btchw.flatten(0, 1), - noise.flatten(0, 1), - timesteps.flatten(0, 1) - ).unflatten(0, (batch_size, num_latent_t)) - - # Store everything on training_batch - training_batch.noisy_model_input = noisy_model_input - training_batch.timesteps = timesteps # [B, T] per-frame timesteps - training_batch.noise = noise.permute(0, 2, 1, 3, 4) # [B, C, T, H, W] - training_batch.raw_latent_shape = latents.shape - # Store extra data for the custom loss function - training_batch._ar_training_target = training_target # [B, T, C, H, W] - - return training_batch - - def _build_input_kwargs(self, - training_batch: TrainingBatch) -> TrainingBatch: - """Build transformer kwargs with action conditioning and per-frame timesteps.""" - # Image Embeds for conditioning - image_embeds = training_batch.image_embeds - assert torch.isnan(image_embeds).sum() == 0 - image_embeds = image_embeds.to(get_local_torch_device(), - dtype=torch.bfloat16) - - # Process actions for each batch sample - batch_size = training_batch.noisy_model_input.shape[0] - keyboard_cond = training_batch.keyboard_cond - mouse_cond = training_batch.mouse_cond - - if keyboard_cond is not None and mouse_cond is not None: - viewmats_list, intrinsics_list, action_labels_list = [], [], [] - for b in range(batch_size): - v, i, a = process_custom_actions(keyboard_cond[b], - mouse_cond[b]) - viewmats_list.append(v) - intrinsics_list.append(i) - action_labels_list.append(a) - viewmats = torch.stack(viewmats_list, - dim=0).to(get_local_torch_device(), - dtype=torch.bfloat16) - intrinsics = torch.stack(intrinsics_list, - dim=0).to(get_local_torch_device(), - dtype=torch.bfloat16) - action_labels = torch.stack(action_labels_list, - dim=0).to(get_local_torch_device(), - dtype=torch.bfloat16) - else: - viewmats = None - intrinsics = None - action_labels = None - - # Per-frame timesteps: [B, T] - timesteps = training_batch.timesteps - assert timesteps.ndim == 2, ( - f"Expected per-frame timesteps [B, T], got shape {timesteps.shape}") - - training_batch.input_kwargs = { - "hidden_states": training_batch.noisy_model_input, - "encoder_hidden_states": None, # No text conditioning for WanGame - "timestep": timesteps.to(get_local_torch_device(), - dtype=torch.bfloat16), - "encoder_hidden_states_image": image_embeds, - "viewmats": viewmats, - "Ks": intrinsics, - "action": action_labels, - "num_frame_per_block": self.num_frame_per_block, - "return_dict": False, - } - return training_batch - - def _transformer_forward_and_compute_loss( - self, training_batch: TrainingBatch) -> TrainingBatch: - """ - Run transformer forward pass and compute flow-matching loss. - """ - input_kwargs = training_batch.input_kwargs - - # Forward with causal attention via set_forward_context - with set_forward_context(current_timestep=training_batch.timesteps, - attn_metadata=None, - forward_batch=None): - # model_pred: [B, C, T, H, W] (flow prediction) - model_pred = self.transformer(**input_kwargs) - - # model_pred is [B, C, T, H, W], convert to [B, T, C, H, W] - model_pred_btchw = model_pred.permute(0, 2, 1, 3, 4) - - # Training target: [B, T, C, H, W] - training_target = training_batch._ar_training_target.to( - model_pred_btchw.device, dtype=model_pred_btchw.dtype) - - batch_size, num_frame = model_pred_btchw.shape[:2] - - # Per-frame MSE loss with training weight - # loss shape before weight: [B, T] - loss = F.mse_loss( - model_pred_btchw.float(), - training_target.float(), - reduction='none' - ).mean(dim=(2, 3, 4)) # Average over C, H, W → [B, T] - - # Apply per-timestep training weight from scheduler - timesteps = training_batch.timesteps # [B, T] - weights = self.ar_noise_scheduler.training_weight( - timesteps.flatten(0, 1) - ).unflatten(0, (batch_size, num_frame)) - loss = (loss * weights).mean() - - loss = loss / self.training_args.gradient_accumulation_steps - loss.backward() - - avg_loss = loss.detach().clone() - training_batch.total_loss += avg_loss.item() - - return training_batch - - def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: - """Override to use custom AR diffusion training logic.""" - self.transformer.train() - self.optimizer.zero_grad() - training_batch.total_loss = 0.0 - args = cast(TrainingArgs, self.training_args) - - for _ in range(args.gradient_accumulation_steps): - training_batch = self._get_next_batch(training_batch) - - # Prepare noisy inputs with per-block timesteps + I2V concat - training_batch = self._prepare_dit_inputs(training_batch) - - # Build transformer input kwargs (action conditioning etc.) - training_batch = self._build_input_kwargs(training_batch) - - # Forward + loss - training_batch = self._transformer_forward_and_compute_loss( - training_batch) - - # Clip grad and step - grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( - [p for p in self.transformer.parameters() if p.requires_grad], - args.max_grad_norm if args.max_grad_norm is not None else 0.0) - - self.optimizer.step() - self.lr_scheduler.step() - - if grad_norm is None: - grad_value = 0.0 - else: - try: - if isinstance(grad_norm, torch.Tensor): - grad_value = float(grad_norm.detach().float().item()) - else: - grad_value = float(grad_norm) - except Exception: - grad_value = 0.0 - training_batch.grad_norm = grad_value - training_batch.raw_latent_shape = training_batch.latents.shape - return training_batch - - def _prepare_validation_batch(self, sampling_param: SamplingParam, - training_args: TrainingArgs, - validation_batch: dict[str, Any], - num_inference_steps: int) -> ForwardBatch: - sampling_param.prompt = validation_batch['prompt'] - sampling_param.height = training_args.num_height - sampling_param.width = training_args.num_width - sampling_param.image_path = validation_batch.get( - 'image_path') or validation_batch.get('video_path') - sampling_param.num_inference_steps = num_inference_steps - sampling_param.data_type = "video" - assert self.seed is not None - sampling_param.seed = self.seed - - latents_size = [(sampling_param.num_frames - 1) // 4 + 1, - sampling_param.height // 8, sampling_param.width // 8] - n_tokens = latents_size[0] * latents_size[1] * latents_size[2] - temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (training_args.num_latent_t - - 1) * temporal_compression_factor + 1 - sampling_param.num_frames = num_frames - batch = ForwardBatch( - **shallow_asdict(sampling_param), - latents=None, - generator=torch.Generator(device="cpu").manual_seed(self.seed), - n_tokens=n_tokens, - eta=0.0, - VSA_sparsity=training_args.VSA_sparsity, - ) - if "image" in validation_batch and validation_batch["image"] is not None: - batch.pil_image = validation_batch["image"] - - if "keyboard_cond" in validation_batch and validation_batch[ - "keyboard_cond"] is not None: - keyboard_cond = validation_batch["keyboard_cond"] - keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) - keyboard_cond = keyboard_cond.unsqueeze(0) - batch.keyboard_cond = keyboard_cond - - if "mouse_cond" in validation_batch and validation_batch[ - "mouse_cond"] is not None: - mouse_cond = validation_batch["mouse_cond"] - mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) - mouse_cond = mouse_cond.unsqueeze(0) - batch.mouse_cond = mouse_cond - - return batch - - def _post_process_validation_frames( - self, frames: list[np.ndarray], - batch: ForwardBatch) -> list[np.ndarray]: - """Apply action overlay to validation frames.""" - keyboard_cond = getattr(batch, 'keyboard_cond', None) - mouse_cond = getattr(batch, 'mouse_cond', None) - - if keyboard_cond is None and mouse_cond is None: - return frames - - from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, - draw_mouse_on_frame) - - if keyboard_cond is not None: - keyboard_cond = keyboard_cond.squeeze( - 0).cpu().float().numpy() - if mouse_cond is not None: - mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() - - key_names = ["W", "S", "A", "D", "left", "right"] - - processed_frames = [] - for frame_idx, frame in enumerate(frames): - frame = np.ascontiguousarray(frame.copy()) - - if keyboard_cond is not None and frame_idx < len(keyboard_cond): - keys = { - key_names[i]: bool(keyboard_cond[frame_idx, i]) - for i in range(min(len(key_names), keyboard_cond.shape[1])) - } - draw_keys_on_frame(frame, keys, mode='universal') - - if mouse_cond is not None and frame_idx < len(mouse_cond): - pitch = float(mouse_cond[frame_idx, 0]) - yaw = float(mouse_cond[frame_idx, 1]) - draw_mouse_on_frame(frame, pitch, yaw) - - processed_frames.append(frame) - - return processed_frames - - -def main(args) -> None: - logger.info("Starting WanGame AR diffusion training pipeline...") - - pipeline = WanGameARDiffusionPipeline.from_pretrained( - args.pretrained_model_name_or_path, args=args) - args = pipeline.training_args - pipeline.train() - logger.info("WanGame AR diffusion training pipeline done") - - -if __name__ == "__main__": - argv = sys.argv - from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.utils import FlexibleArgumentParser - parser = FlexibleArgumentParser() - parser = TrainingArgs.add_cli_args(parser) - parser = FastVideoArgs.add_cli_args(parser) - args = parser.parse_args() - args.dit_cpu_offload = False - main(args) diff --git a/fastvideo/training/wangame_distillation_pipeline.py b/fastvideo/training/wangame_distillation_pipeline.py deleted file mode 100644 index c72c60250..000000000 --- a/fastvideo/training/wangame_distillation_pipeline.py +++ /dev/null @@ -1,517 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -import sys -from copy import deepcopy -from typing import Any - -import numpy as np -import torch -import torch.nn.functional as F -from einops import rearrange - -from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame -from fastvideo.distributed import get_local_torch_device -from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.forward_context import set_forward_context -from fastvideo.logger import init_logger -from fastvideo.models.dits.hyworld.pose import process_custom_actions -from fastvideo.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler) -from fastvideo.models.utils import pred_noise_to_pred_video -from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import ( - WanGameActionImageToVideoPipeline) -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch -from fastvideo.training.distillation_pipeline import DistillationPipeline -from fastvideo.training.training_utils import shift_timestep -from fastvideo.utils import is_vsa_available, shallow_asdict - -try: - vsa_available = is_vsa_available() -except Exception: - vsa_available = False - -logger = init_logger(__name__) - - -class WanGameDistillationPipeline(DistillationPipeline): - """ - DMD distillation pipeline for WanGame. - """ - _required_config_modules = ["scheduler", "transformer", "vae"] - - def initialize_pipeline(self, fastvideo_args: FastVideoArgs): - """Initialize WanGame-specific scheduler.""" - self.modules["scheduler"] = FlowMatchEulerDiscreteScheduler( - shift=fastvideo_args.pipeline_config.flow_shift) - - def create_training_stages(self, training_args: TrainingArgs): - """May be used in future refactors.""" - pass - - def set_schemas(self): - self.train_dataset_schema = pyarrow_schema_wangame - - def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: - batch = next(self.train_loader_iter, None) # type: ignore - if batch is None: - self.current_epoch += 1 - logger.info("Starting epoch %s", self.current_epoch) - self.train_loader_iter = iter(self.train_dataloader) - batch = next(self.train_loader_iter) - - device = get_local_torch_device() - dtype = torch.bfloat16 - - clip_feature = batch['clip_feature'] - first_frame_latent = batch['first_frame_latent'] - keyboard_cond = batch.get('keyboard_cond', None) - mouse_cond = batch.get('mouse_cond', None) - infos = batch['info_list'] - - if self.training_args.simulate_generator_forward: - # When simulating, we don't need real VAE latents — just use zeros - batch_size = clip_feature.shape[0] - vae_config = self.training_args.pipeline_config.vae_config.arch_config - num_channels = vae_config.z_dim - spatial_compression_ratio = vae_config.spatial_compression_ratio - - latent_height = self.training_args.num_height // spatial_compression_ratio - latent_width = self.training_args.num_width // spatial_compression_ratio - - latents = torch.zeros( - batch_size, - num_channels, - self.training_args.num_latent_t, - latent_height, - latent_width, - device=device, - dtype=dtype, - ) - else: - if 'vae_latent' not in batch: - raise ValueError( - "vae_latent not found in batch and simulate_generator_forward is False. " - "Either preprocess data with VAE latents or set --simulate_generator_forward." - ) - latents = batch['vae_latent'] - latents = latents[:, :, :self.training_args.num_latent_t] - latents = latents.to(device, dtype=dtype) - - training_batch.latents = latents.to(device, dtype=dtype) - training_batch.encoder_hidden_states = None - training_batch.encoder_attention_mask = None - training_batch.image_embeds = clip_feature.to(device, dtype=dtype) - training_batch.image_latents = first_frame_latent.to(device, dtype=dtype) - - # Action conditioning - if keyboard_cond is not None and keyboard_cond.numel() > 0: - training_batch.keyboard_cond = keyboard_cond.to(device, dtype=dtype) - else: - training_batch.keyboard_cond = None - if mouse_cond is not None and mouse_cond.numel() > 0: - training_batch.mouse_cond = mouse_cond.to(device, dtype=dtype) - else: - training_batch.mouse_cond = None - - training_batch.infos = infos - return training_batch - - def _prepare_dit_inputs(self, - training_batch: TrainingBatch) -> TrainingBatch: - """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" - # First, call parent method to prepare noise, timesteps, etc. for video latents - training_batch = super()._prepare_dit_inputs(training_batch) - - training_batch.conditional_dict = { - "encoder_hidden_states": None, - "encoder_attention_mask": None, - } - training_batch.unconditional_dict = None - - assert isinstance(training_batch.image_latents, torch.Tensor) - image_latents = training_batch.image_latents.to( - get_local_torch_device(), dtype=torch.bfloat16) - - # Build mask + image_latent -> cond_concat (20 channels) - temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (self.training_args.num_latent_t - - 1) * temporal_compression_ratio + 1 - batch_size, num_channels, _, latent_height, latent_width = image_latents.shape - mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, - latent_width) - mask_lat_size[:, :, 1:] = 0 - - first_frame_mask = mask_lat_size[:, :, :1] - first_frame_mask = torch.repeat_interleave( - first_frame_mask, dim=2, repeats=temporal_compression_ratio) - mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], - dim=2) - mask_lat_size = mask_lat_size.view(batch_size, -1, - temporal_compression_ratio, - latent_height, latent_width) - mask_lat_size = mask_lat_size.transpose(1, 2) - mask_lat_size = mask_lat_size.to( - image_latents.device).to(dtype=torch.bfloat16) - - # cond_concat = [mask(4), image_latent(16)] = 20 channels - image_latents = torch.cat([mask_lat_size, image_latents], dim=1) - - if self.sp_world_size > 1: - total_frames = image_latents.shape[2] - # Split cond latents to local SP shard only when tensor is still global. - if total_frames == self.training_args.num_latent_t: - if total_frames % self.sp_world_size != 0: - raise ValueError( - "image_latents temporal dim is not divisible by SP world size: " - f"frames={total_frames}, sp_world_size={self.sp_world_size}" - ) - image_latents = rearrange(image_latents, - "b c (n t) h w -> b c n t h w", - n=self.sp_world_size).contiguous() - image_latents = image_latents[:, :, self.rank_in_sp_group, :, :, - :] - - training_batch.image_latents = image_latents - - return training_batch - - def _build_distill_input_kwargs( - self, noise_input: torch.Tensor, timestep: torch.Tensor, - text_dict: dict[str, torch.Tensor] | None, - training_batch: TrainingBatch) -> TrainingBatch: - """Build model input with WanGame - """ - # Image embeds (CLIP features) for cross-attention conditioning - image_embeds = training_batch.image_embeds - assert torch.isnan(image_embeds).sum() == 0 - image_embeds = image_embeds.to(get_local_torch_device(), - dtype=torch.bfloat16) - - # already prepared in _prepare_dit_inputs - image_latents = training_batch.image_latents - - # Process action conditioning - keyboard_cond = training_batch.keyboard_cond - mouse_cond = training_batch.mouse_cond - - if keyboard_cond is not None and mouse_cond is not None: - viewmats_list = [] - intrinsics_list = [] - action_labels_list = [] - for b in range(noise_input.shape[0]): - viewmats, intrinsics, action_labels = process_custom_actions( - keyboard_cond[b], mouse_cond[b]) - viewmats_list.append(viewmats) - intrinsics_list.append(intrinsics) - action_labels_list.append(action_labels) - - viewmats = torch.stack(viewmats_list, dim=0).to( - device=get_local_torch_device(), dtype=torch.bfloat16) - intrinsics = torch.stack(intrinsics_list, dim=0).to( - device=get_local_torch_device(), dtype=torch.bfloat16) - action_labels = torch.stack(action_labels_list, dim=0).to( - device=get_local_torch_device(), dtype=torch.bfloat16) - else: - viewmats = None - intrinsics = None - action_labels = None - - # I2V concatenation: [noise_input(16ch), image_latents(20ch)] -> 36ch - noisy_model_input = torch.cat( - [noise_input, image_latents.permute(0, 2, 1, 3, 4)], dim=2) - - training_batch.input_kwargs = { - "hidden_states": noisy_model_input.permute(0, 2, 1, 3, 4), - "encoder_hidden_states": None, - "timestep": timestep, - "encoder_hidden_states_image": image_embeds, - "viewmats": viewmats, - "Ks": intrinsics, - "action": action_labels, - "return_dict": False, - } - training_batch.noise_latents = noise_input - - return training_batch - - def _dmd_forward(self, generator_pred_video: torch.Tensor, - training_batch: TrainingBatch) -> torch.Tensor: - """Compute DMD loss for WanGame.""" - original_latent = generator_pred_video - with torch.no_grad(): - timestep = torch.randint(0, - self.num_train_timestep, [1], - device=self.device, - dtype=torch.long) - - timestep = shift_timestep(timestep, self.timestep_shift, - self.num_train_timestep) - - timestep = timestep.clamp(self.min_timestep, self.max_timestep) - - noise = torch.randn(self.video_latent_shape, - device=self.device, - dtype=generator_pred_video.dtype) - - noisy_latent = self.noise_scheduler.add_noise( - generator_pred_video.flatten(0, 1), noise.flatten(0, 1), - timestep).detach().unflatten(0, (1, generator_pred_video.shape[1])) - - # Build input kwargs for critic/teacher - training_batch = self._build_distill_input_kwargs( - noisy_latent, timestep, training_batch.conditional_dict, - training_batch) - - # fake_score_transformer forward - current_fake_score_transformer = self._get_fake_score_transformer( - timestep) - fake_score_pred_noise = current_fake_score_transformer( - **training_batch.input_kwargs).permute(0, 2, 1, 3, 4) - - faker_score_pred_video = pred_noise_to_pred_video( - pred_noise=fake_score_pred_noise.flatten(0, 1), - noise_input_latent=noisy_latent.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler).unflatten( - 0, fake_score_pred_noise.shape[:2]) - - # real_score_transformer forward - current_real_score_transformer = self._get_real_score_transformer( - timestep) - real_score_pred_noise = current_real_score_transformer( - **training_batch.input_kwargs).permute(0, 2, 1, 3, 4) - - real_score_pred_video = pred_noise_to_pred_video( - pred_noise=real_score_pred_noise.flatten(0, 1), - noise_input_latent=noisy_latent.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler).unflatten( - 0, real_score_pred_noise.shape[:2]) - - # No CFG for WanGame - use real_score_pred_video directly - grad = (faker_score_pred_video - real_score_pred_video) / torch.abs( - original_latent - real_score_pred_video).mean() - grad = torch.nan_to_num(grad) - - dmd_loss = 0.5 * F.mse_loss( - original_latent.float(), - (original_latent.float() - grad.float()).detach()) - - training_batch.dmd_latent_vis_dict.update({ - "training_batch_dmd_fwd_clean_latent": - training_batch.latents, - "generator_pred_video": - original_latent.detach(), - "real_score_pred_video": - real_score_pred_video.detach(), - "faker_score_pred_video": - faker_score_pred_video.detach(), - "dmd_timestep": - timestep.detach(), - }) - - return dmd_loss - - def faker_score_forward( - self, training_batch: TrainingBatch - ) -> tuple[TrainingBatch, torch.Tensor]: - """Forward pass for critic training with WanGame action conditioning.""" - with torch.no_grad(), set_forward_context( - current_timestep=training_batch.timesteps, - attn_metadata=training_batch.attn_metadata_vsa): - if self.training_args.simulate_generator_forward: - generator_pred_video = self._generator_multi_step_simulation_forward( - training_batch) - else: - generator_pred_video = self._generator_forward(training_batch) - - fake_score_timestep = torch.randint(0, - self.num_train_timestep, [1], - device=self.device, - dtype=torch.long) - - fake_score_timestep = shift_timestep(fake_score_timestep, - self.timestep_shift, - self.num_train_timestep) - - fake_score_timestep = fake_score_timestep.clamp(self.min_timestep, - self.max_timestep) - - fake_score_noise = torch.randn(self.video_latent_shape, - device=self.device, - dtype=generator_pred_video.dtype) - - noisy_generator_pred_video = self.noise_scheduler.add_noise( - generator_pred_video.flatten(0, 1), - fake_score_noise.flatten(0, 1), fake_score_timestep).unflatten( - 0, (1, generator_pred_video.shape[1])) - - with set_forward_context(current_timestep=training_batch.timesteps, - attn_metadata=training_batch.attn_metadata): - training_batch = self._build_distill_input_kwargs( - noisy_generator_pred_video, fake_score_timestep, - training_batch.conditional_dict, training_batch) - - current_fake_score_transformer = self._get_fake_score_transformer( - fake_score_timestep) - fake_score_pred_noise = current_fake_score_transformer( - **training_batch.input_kwargs).permute(0, 2, 1, 3, 4) - - target = fake_score_noise - generator_pred_video - flow_matching_loss = torch.mean((fake_score_pred_noise - target)**2) - - training_batch.fake_score_latent_vis_dict = { - "training_batch_fakerscore_fwd_clean_latent": - training_batch.latents, - "generator_pred_video": generator_pred_video, - "fake_score_timestep": fake_score_timestep, - } - - return training_batch, flow_matching_loss - - def initialize_validation_pipeline(self, training_args: TrainingArgs): - logger.info("Initializing validation pipeline...") - args_copy = deepcopy(training_args) - args_copy.inference_mode = True - - validation_pipeline = WanGameActionImageToVideoPipeline.from_pretrained( - training_args.model_path, - args=args_copy, # type: ignore - inference_mode=True, - loaded_modules={ - "transformer": self.get_module("transformer"), - "vae": self.get_module("vae"), - }, - tp_size=training_args.tp_size, - sp_size=training_args.sp_size, - num_gpus=training_args.num_gpus, - pin_cpu_memory=training_args.pin_cpu_memory, - dit_cpu_offload=True) - - self.validation_pipeline = validation_pipeline - - def _prepare_validation_batch(self, sampling_param: SamplingParam, - training_args: TrainingArgs, - validation_batch: dict[str, Any], - num_inference_steps: int) -> ForwardBatch: - sampling_param.prompt = validation_batch['prompt'] - sampling_param.height = training_args.num_height - sampling_param.width = training_args.num_width - sampling_param.image_path = validation_batch.get( - 'image_path') or validation_batch.get('video_path') - sampling_param.num_inference_steps = num_inference_steps - sampling_param.data_type = "video" - assert self.seed is not None - sampling_param.seed = self.seed - - latents_size = [(sampling_param.num_frames - 1) // 4 + 1, - sampling_param.height // 8, sampling_param.width // 8] - n_tokens = latents_size[0] * latents_size[1] * latents_size[2] - temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (training_args.num_latent_t - - 1) * temporal_compression_factor + 1 - sampling_param.num_frames = num_frames - batch = ForwardBatch( - **shallow_asdict(sampling_param), - latents=None, - generator=torch.Generator(device="cpu").manual_seed(self.seed), - n_tokens=n_tokens, - eta=0.0, - VSA_sparsity=training_args.VSA_sparsity, - ) - if "image" in validation_batch and validation_batch["image"] is not None: - batch.pil_image = validation_batch["image"] - - if "keyboard_cond" in validation_batch and validation_batch[ - "keyboard_cond"] is not None: - keyboard_cond = validation_batch["keyboard_cond"] - if isinstance(keyboard_cond, torch.Tensor): - keyboard_cond = keyboard_cond.detach().clone().to(dtype=torch.bfloat16) - else: - keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) - keyboard_cond = keyboard_cond.unsqueeze(0) - batch.keyboard_cond = keyboard_cond[:num_frames] - - if "mouse_cond" in validation_batch and validation_batch[ - "mouse_cond"] is not None: - mouse_cond = validation_batch["mouse_cond"] - if isinstance(mouse_cond, torch.Tensor): - mouse_cond = mouse_cond.detach().clone().to(dtype=torch.bfloat16) - else: - mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) - mouse_cond = mouse_cond.unsqueeze(0) - batch.mouse_cond = mouse_cond[:num_frames] - - return batch - - def _post_process_validation_frames( - self, frames: list[np.ndarray], - batch: ForwardBatch) -> list[np.ndarray]: - """Apply action overlay to validation frames for WanGame. - - Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. - """ - # Check if action data is available - keyboard_cond = getattr(batch, 'keyboard_cond', None) - mouse_cond = getattr(batch, 'mouse_cond', None) - - if keyboard_cond is None and mouse_cond is None: - return frames - - # Import overlay functions - from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, - draw_mouse_on_frame) - - # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) - if keyboard_cond is not None: - keyboard_cond = keyboard_cond.squeeze( - 0).cpu().float().numpy() # (T, 6) - if mouse_cond is not None: - mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) - - # WanGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] - key_names = ["W", "S", "A", "D", "left", "right"] - - processed_frames = [] - for frame_idx, frame in enumerate(frames): - frame = np.ascontiguousarray(frame.copy()) - - # Draw keyboard overlay - if keyboard_cond is not None and frame_idx < len(keyboard_cond): - keys = { - key_names[i]: bool(keyboard_cond[frame_idx, i]) - for i in range(min(len(key_names), keyboard_cond.shape[1])) - } - draw_keys_on_frame(frame, keys, mode='universal') - - # Draw mouse overlay - if mouse_cond is not None and frame_idx < len(mouse_cond): - pitch = float(mouse_cond[frame_idx, 0]) - yaw = float(mouse_cond[frame_idx, 1]) - draw_mouse_on_frame(frame, pitch, yaw) - - processed_frames.append(frame) - - return processed_frames - - -def main(args) -> None: - logger.info("Starting WanGame DMD distillation pipeline...") - - pipeline = WanGameDistillationPipeline.from_pretrained( - args.pretrained_model_name_or_path, args=args) - - args = pipeline.training_args - pipeline.train() - logger.info("WanGame DMD distillation pipeline completed") - - -if __name__ == "__main__": - argv = sys.argv - from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.utils import FlexibleArgumentParser - parser = FlexibleArgumentParser() - parser = TrainingArgs.add_cli_args(parser) - parser = FastVideoArgs.add_cli_args(parser) - args = parser.parse_args() - main(args) diff --git a/fastvideo/training/wangame_lingbot_training_pipeline.py b/fastvideo/training/wangame_lingbot_training_pipeline.py deleted file mode 100644 index cb3eb8478..000000000 --- a/fastvideo/training/wangame_lingbot_training_pipeline.py +++ /dev/null @@ -1,418 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -import sys -from typing import Any - -import numpy as np -import torch - -from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame_lingbot -from fastvideo.distributed import get_local_torch_device -from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.logger import init_logger -from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( - FlowUniPCMultistepScheduler) -from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import WanLingBotImageToVideoPipeline -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch -from fastvideo.training.training_pipeline import TrainingPipeline -from fastvideo.utils import is_vsa_available, shallow_asdict - -vsa_available = is_vsa_available() - -logger = init_logger(__name__) - - -class WanLingBotTrainingPipeline(TrainingPipeline): - """ - A training pipeline for WanGame-2.1-Fun-1.3B-InP. - """ - _required_config_modules = ["scheduler", "transformer", "vae"] - - def initialize_pipeline(self, fastvideo_args: FastVideoArgs): - self.modules["scheduler"] = FlowUniPCMultistepScheduler( - shift=fastvideo_args.pipeline_config.flow_shift) - - def create_training_stages(self, training_args: TrainingArgs): - """ - May be used in future refactors. - """ - pass - - def set_schemas(self): - self.train_dataset_schema = pyarrow_schema_wangame_lingbot - - def set_trainable(self) -> None: - """ - Override to only train newly added action-related parameters for Lingbot: - - patch_embedding_wancamctrl: embeds camera Plucker coordinates - - blocks.*.cam_conditioner: injects camera conditioning into transformer blocks - """ - train_action_only = getattr(self.fastvideo_args, "train_action_only", - False) - - if not train_action_only: - # Default behavior: train all parameters - super().set_trainable() - return - - # Freeze all transformer parameters first - transformer = self.get_module("transformer") - transformer.train() - transformer.requires_grad_(False) - - # Define which parameter name patterns to train - action_param_patterns = [ - "patch_embedding_wancamctrl", - "cam_conditioner", - ] - - # Enable gradients for action-related parameters only - trainable_count = 0 - frozen_count = 0 - for name, param in transformer.named_parameters(): - should_train = any(pattern in name - for pattern in action_param_patterns) - if should_train: - param.requires_grad_(True) - trainable_count += 1 - logger.info(f"Trainable: {name} ({param.numel()} params)") - else: - frozen_count += 1 - - logger.info( - f"Action-only training: {trainable_count} trainable param groups, " - f"{frozen_count} frozen param groups") - - # ── Action module warmup ────────────────────────────────────────────── - # For the first `action_warmup_steps`, action modules (action_embedder, - # to_out_prope) have requires_grad=False so the base model stabilizes - # first. After warmup the gradients are re-enabled. - - _ACTION_PARAM_PATTERNS = [ - "patch_embedding_wancamctrl", - "cam_conditioner", - ] - - def _set_action_params_grad(self, requires_grad: bool) -> None: - """Toggle requires_grad for action-related parameters.""" - transformer = self.get_module("transformer") - count = 0 - for name, param in transformer.named_parameters(): - if any(p in name for p in self._ACTION_PARAM_PATTERNS): - param.requires_grad_(requires_grad) - count += 1 - state = "enabled" if requires_grad else "disabled" - logger.info("Gradients %s for %d action parameter groups", state, count) - - def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: - step = training_batch.current_timestep - warmup_steps = self.training_args.action_warmup_steps - - if warmup_steps > 0: - if step == 1: - # Freeze action params at the very first step - self._set_action_params_grad(False) - logger.info( - "Action warmup: freezing action modules for the first " - "%d steps to stabilize base model", warmup_steps) - elif step == warmup_steps + 1: - # Unfreeze action params once warmup is done - self._set_action_params_grad(True) - logger.info( - "Action warmup complete — action modules unfrozen at " - "step %d", step) - - return super().train_one_step(training_batch) - - def initialize_validation_pipeline(self, training_args: TrainingArgs): - logger.info("Initializing validation pipeline...") - # args_copy.pipeline_config.vae_config.load_encoder = False - # validation_pipeline = WanImageToVideoValidationPipeline.from_pretrained( - self.validation_pipeline = WanLingBotImageToVideoPipeline.from_pretrained( - training_args.model_path, - args=None, - inference_mode=True, - loaded_modules={ - "transformer": self.get_module("transformer"), - }, - tp_size=training_args.tp_size, - sp_size=training_args.sp_size, - num_gpus=training_args.num_gpus, - dit_cpu_offload=False) - - def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: - batch = next(self.train_loader_iter, None) # type: ignore - if batch is None: - self.current_epoch += 1 - logger.info("Starting epoch %s", self.current_epoch) - # Reset iterator for next epoch - self.train_loader_iter = iter(self.train_dataloader) - # Get first batch of new epoch - batch = next(self.train_loader_iter) - - latents = batch['vae_latent'] - latents = latents[:, :, :self.training_args.num_latent_t] - # encoder_hidden_states = batch['text_embedding'] - # encoder_attention_mask = batch['text_attention_mask'] - clip_features = batch['clip_feature'] - image_latents = batch['first_frame_latent'] - image_latents = image_latents[:, :, :self.training_args.num_latent_t] - pil_image = batch['pil_image'] - infos = batch['info_list'] - - training_batch.latents = latents.to(get_local_torch_device(), - dtype=torch.bfloat16) - training_batch.encoder_hidden_states = None - training_batch.encoder_attention_mask = None - training_batch.preprocessed_image = pil_image.to( - get_local_torch_device()) - training_batch.image_embeds = clip_features.to(get_local_torch_device()) - training_batch.image_latents = image_latents.to( - get_local_torch_device()) - training_batch.infos = infos - - # Action conditioning - if 'mouse_cond' in batch and batch['mouse_cond'].numel() > 0: - training_batch.mouse_cond = batch['mouse_cond'].to( - get_local_torch_device(), dtype=torch.bfloat16) - else: - training_batch.mouse_cond = None - - if 'keyboard_cond' in batch and batch['keyboard_cond'].numel() > 0: - training_batch.keyboard_cond = batch['keyboard_cond'].to( - get_local_torch_device(), dtype=torch.bfloat16) - else: - training_batch.keyboard_cond = None - - # Validate action temporal dimensions match video num_frames - expected_num_frames = (self.training_args.num_latent_t - 1) * 4 + 1 - if training_batch.keyboard_cond is not None: - assert training_batch.keyboard_cond.shape[1] == expected_num_frames, ( - f"keyboard_cond temporal dim {training_batch.keyboard_cond.shape[1]} " - f"!= expected {expected_num_frames} " - f"(num_latent_t={self.training_args.num_latent_t})") - if training_batch.mouse_cond is not None: - assert training_batch.mouse_cond.shape[1] == expected_num_frames, ( - f"mouse_cond temporal dim {training_batch.mouse_cond.shape[1]} " - f"!= expected {expected_num_frames} " - f"(num_latent_t={self.training_args.num_latent_t})") - - return training_batch - - def _prepare_dit_inputs(self, - training_batch: TrainingBatch) -> TrainingBatch: - """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" - - # First, call parent method to prepare noise, timesteps, etc. for video latents - training_batch = super()._prepare_dit_inputs(training_batch) - - assert isinstance(training_batch.image_latents, torch.Tensor) - image_latents = training_batch.image_latents.to( - get_local_torch_device(), dtype=torch.bfloat16) - - temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (self.training_args.num_latent_t - - 1) * temporal_compression_ratio + 1 - batch_size, num_channels, _, latent_height, latent_width = image_latents.shape - mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, - latent_width) - mask_lat_size[:, :, 1:] = 0 - - first_frame_mask = mask_lat_size[:, :, :1] - first_frame_mask = torch.repeat_interleave( - first_frame_mask, dim=2, repeats=temporal_compression_ratio) - mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], - dim=2) - mask_lat_size = mask_lat_size.view(batch_size, -1, - temporal_compression_ratio, - latent_height, latent_width) - mask_lat_size = mask_lat_size.transpose(1, 2) - mask_lat_size = mask_lat_size.to( - image_latents.device).to(dtype=torch.bfloat16) - - training_batch.noisy_model_input = torch.cat( - [training_batch.noisy_model_input, mask_lat_size, image_latents], - dim=1) - - return training_batch - - def _build_input_kwargs(self, - training_batch: TrainingBatch) -> TrainingBatch: - - # Image Embeds for conditioning - image_embeds = training_batch.image_embeds - assert torch.isnan(image_embeds).sum() == 0 - image_embeds = image_embeds.to(get_local_torch_device(), - dtype=torch.bfloat16) - encoder_hidden_states_image = image_embeds - - from fastvideo.models.dits.wangame_lingbot.cam_utils import process_custom_actions - - # Process actions for each batch sample - batch_size = training_batch.noisy_model_input.shape[0] - num_latent_t = training_batch.noisy_model_input.shape[2] - latent_height = training_batch.noisy_model_input.shape[3] - latent_width = training_batch.noisy_model_input.shape[4] - - temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (num_latent_t - 1) * temporal_compression_ratio + 1 - - c2ws_plucker_emb_list = [] - for b in range(batch_size): - # Lingbot's process_custom_actions returns [1, 6*spatial_scale^2, lat_f, H_lat, W_lat] - c2ws_plucker_emb = process_custom_actions( - num_frames=num_frames, - keyboard_cond=training_batch.keyboard_cond[b], - mouse_cond=training_batch.mouse_cond[b], - latent_height=latent_height, - latent_width=latent_width) - c2ws_plucker_emb_list.append(c2ws_plucker_emb) - - c2ws_plucker_emb = torch.cat(c2ws_plucker_emb_list, - dim=0).to(get_local_torch_device(), - dtype=torch.bfloat16) - - # c2ws_plucker_emb: [B, C, lat_f, H_lat, W_lat] - assert c2ws_plucker_emb.shape[2] == num_latent_t, ( - f"c2ws_plucker_emb temporal dim {c2ws_plucker_emb.shape[2]} != " - f"video latent temporal dim {num_latent_t}") - - training_batch.input_kwargs = { - "hidden_states": - training_batch.noisy_model_input, - "encoder_hidden_states": - training_batch.encoder_hidden_states, # None (no text conditioning) - "timestep": - training_batch.timesteps.to(get_local_torch_device(), - dtype=torch.bfloat16), - # "encoder_attention_mask": - # training_batch.encoder_attention_mask, - "encoder_hidden_states_image": - encoder_hidden_states_image, - # Action conditioning - "c2ws_plucker_emb": - c2ws_plucker_emb, - "return_dict": - False, - } - return training_batch - - def _prepare_validation_batch(self, sampling_param: SamplingParam, - training_args: TrainingArgs, - validation_batch: dict[str, Any], - num_inference_steps: int) -> ForwardBatch: - sampling_param.prompt = validation_batch['prompt'] - sampling_param.height = training_args.num_height - sampling_param.width = training_args.num_width - sampling_param.image_path = validation_batch.get( - 'image_path') or validation_batch.get('video_path') - sampling_param.num_inference_steps = num_inference_steps - sampling_param.data_type = "video" - assert self.seed is not None - sampling_param.seed = self.seed - - latents_size = [(sampling_param.num_frames - 1) // 4 + 1, - sampling_param.height // 8, sampling_param.width // 8] - n_tokens = latents_size[0] * latents_size[1] * latents_size[2] - temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (training_args.num_latent_t - - 1) * temporal_compression_factor + 1 - sampling_param.num_frames = num_frames - batch = ForwardBatch( - **shallow_asdict(sampling_param), - latents=None, - generator=torch.Generator(device="cpu").manual_seed(self.seed), - n_tokens=n_tokens, - eta=0.0, - VSA_sparsity=training_args.VSA_sparsity, - ) - if "image" in validation_batch and validation_batch["image"] is not None: - batch.pil_image = validation_batch["image"] - - if "keyboard_cond" in validation_batch and validation_batch[ - "keyboard_cond"] is not None: - keyboard_cond = validation_batch["keyboard_cond"] - keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) - keyboard_cond = keyboard_cond.unsqueeze(0) - batch.keyboard_cond = keyboard_cond - - if "mouse_cond" in validation_batch and validation_batch[ - "mouse_cond"] is not None: - mouse_cond = validation_batch["mouse_cond"] - mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) - mouse_cond = mouse_cond.unsqueeze(0) - batch.mouse_cond = mouse_cond - - return batch - - def _post_process_validation_frames( - self, frames: list[np.ndarray], - batch: ForwardBatch) -> list[np.ndarray]: - """Apply action overlay to validation frames for WanGame. - - Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. - """ - # Check if action data is available - keyboard_cond = getattr(batch, 'keyboard_cond', None) - mouse_cond = getattr(batch, 'mouse_cond', None) - - if keyboard_cond is None and mouse_cond is None: - return frames - - # Import overlay functions - from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, - draw_mouse_on_frame) - - # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) - if keyboard_cond is not None: - keyboard_cond = keyboard_cond.squeeze( - 0).cpu().float().numpy() # (T, 6) - if mouse_cond is not None: - mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) - - # MatrixGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] - key_names = ["W", "S", "A", "D", "left", "right"] - - processed_frames = [] - for frame_idx, frame in enumerate(frames): - frame = np.ascontiguousarray(frame.copy()) - - # Draw keyboard overlay - if keyboard_cond is not None and frame_idx < len(keyboard_cond): - keys = { - key_names[i]: bool(keyboard_cond[frame_idx, i]) - for i in range(min(len(key_names), keyboard_cond.shape[1])) - } - draw_keys_on_frame(frame, keys, mode='universal') - - # Draw mouse overlay - if mouse_cond is not None and frame_idx < len(mouse_cond): - pitch = float(mouse_cond[frame_idx, 0]) - yaw = float(mouse_cond[frame_idx, 1]) - draw_mouse_on_frame(frame, pitch, yaw) - - processed_frames.append(frame) - - return processed_frames - - -def main(args) -> None: - logger.info("Starting training pipeline...") - - pipeline = WanLingBotTrainingPipeline.from_pretrained( - args.pretrained_model_name_or_path, args=args) - args = pipeline.training_args - pipeline.train() - logger.info("Training pipeline done") - - -if __name__ == "__main__": - argv = sys.argv - from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.utils import FlexibleArgumentParser - parser = FlexibleArgumentParser() - parser = TrainingArgs.add_cli_args(parser) - parser = FastVideoArgs.add_cli_args(parser) - args = parser.parse_args() - args.dit_cpu_offload = False - main(args) diff --git a/fastvideo/training/wangame_ode_causal_pipeline.py b/fastvideo/training/wangame_ode_causal_pipeline.py deleted file mode 100644 index ae408282e..000000000 --- a/fastvideo/training/wangame_ode_causal_pipeline.py +++ /dev/null @@ -1,659 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -import sys -from copy import deepcopy -from typing import Any, cast - -import numpy as np -import torch -import torch.nn.functional as F - -from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.dataloader.schema import ( - pyarrow_schema_ode_trajectory_wangame) -from fastvideo.distributed import get_local_torch_device -from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.forward_context import set_forward_context -from fastvideo.logger import init_logger -from fastvideo.models.dits.hyworld.pose import process_custom_actions -from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( - SelfForcingFlowMatchScheduler) -from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( - WanGameCausalDMDPipeline) -from fastvideo.pipelines.stages.decoding import DecodingStage -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch -from fastvideo.training.training_pipeline import TrainingPipeline -from fastvideo.training.training_utils import ( - clip_grad_norm_while_handling_failing_dtensor_cases) -from fastvideo.utils import shallow_asdict - -logger = init_logger(__name__) - - -class WanGameODEInitTrainingPipeline(TrainingPipeline): - """ - Training pipeline for ODE-init using precomputed denoising trajectories. - - Supervision: predict the next latent in the stored trajectory by - - feeding current latent at timestep t into the transformer to predict noise - - stepping the scheduler with the predicted noise - - minimizing MSE to the stored next latent at timestep t_next - """ - - _required_config_modules = ["scheduler", "transformer", "vae"] - - def initialize_pipeline(self, fastvideo_args: FastVideoArgs): - # Match the preprocess/generation scheduler for consistent stepping - self.modules["scheduler"] = SelfForcingFlowMatchScheduler( - shift=fastvideo_args.pipeline_config.flow_shift, - sigma_min=0.0, - extra_one_step=True) - self.modules["scheduler"].set_timesteps(num_inference_steps=1000, - training=True) - - def set_schemas(self): - self.train_dataset_schema = pyarrow_schema_ode_trajectory_wangame - - def initialize_training_pipeline(self, training_args: TrainingArgs): - super().initialize_training_pipeline(training_args) - - self.noise_scheduler = self.get_module("scheduler") - self.vae = self.get_module("vae") - self.vae.requires_grad_(False) - - self.timestep_shift = self.training_args.pipeline_config.flow_shift - self.noise_scheduler = SelfForcingFlowMatchScheduler( - shift=self.timestep_shift, sigma_min=0.0, extra_one_step=True) - self.noise_scheduler.set_timesteps(num_inference_steps=1000, - training=True) - - self.training_args.pipeline_config.dmd_denoising_steps = [1000, 750, 500, 250, 0] - self.add_stage(stage_name="decoding_stage", - stage=DecodingStage(vae=self.get_module("vae"))) - - logger.info("dmd_denoising_steps: %s", - self.training_args.pipeline_config.dmd_denoising_steps) - self.dmd_denoising_steps = torch.tensor([1000, 750, 500, 250, 0], - dtype=torch.long, - device=get_local_torch_device()) - if training_args.warp_denoising_step: # Warp the denoising step according to the scheduler time shift - timesteps = torch.cat((self.noise_scheduler.timesteps.cpu(), - torch.tensor([0], - dtype=torch.float32))).cuda() - logger.info("timesteps: %s", timesteps) - self.dmd_denoising_steps = timesteps[1000 - - self.dmd_denoising_steps] - logger.info("warped self.dmd_denoising_steps: %s", - self.dmd_denoising_steps) - else: - raise ValueError("warp_denoising_step must be true") - - self.dmd_denoising_steps = self.dmd_denoising_steps.to( - get_local_torch_device()) - - logger.info("denoising_step_list: %s", self.dmd_denoising_steps) - - logger.info( - "Initialized ODE-init training pipeline with %s denoising steps", - len(self.dmd_denoising_steps)) - # Cache for nearest trajectory index per DMD step (computed lazily on first batch) - self._cached_closest_idx_per_dmd = None - self.num_train_timestep = self.noise_scheduler.num_train_timesteps - self.manual_idx = 0 - - def initialize_validation_pipeline(self, training_args: TrainingArgs): - logger.info("Initializing validation pipeline...") - args_copy = deepcopy(training_args) - args_copy.inference_mode = True - # Use the same flow-matching scheduler as training for consistent validation. - validation_scheduler = SelfForcingFlowMatchScheduler( - shift=args_copy.pipeline_config.flow_shift, - sigma_min=0.0, - extra_one_step=True) - validation_scheduler.set_timesteps(num_inference_steps=1000, - training=True) - # Warm start validation with current transformer - self.validation_pipeline = WanGameCausalDMDPipeline.from_pretrained( - training_args.model_path, - args=args_copy, # type: ignore - inference_mode=True, - loaded_modules={ - "transformer": self.get_module("transformer"), - "vae": self.get_module("vae"), - "scheduler": validation_scheduler, - }, - tp_size=training_args.tp_size, - sp_size=training_args.sp_size, - num_gpus=training_args.num_gpus, - pin_cpu_memory=training_args.pin_cpu_memory, - dit_cpu_offload=True) - - def _get_next_batch( - self, - training_batch) -> tuple[TrainingBatch, torch.Tensor, torch.Tensor]: - batch = next(self.train_loader_iter, None) # type: ignore - if batch is None: - self.current_epoch += 1 - logger.info("Starting epoch %s", self.current_epoch) - self.train_loader_iter = iter(self.train_dataloader) - batch = next(self.train_loader_iter) - - # Required fields from parquet (ODE trajectory schema) - clip_feature = batch['clip_feature'] - first_frame_latent = batch['first_frame_latent'] - keyboard_cond = batch.get('keyboard_cond', None) - # keyboard_cond = keyboard_cond[:, :, :3] # TODO: remove hardcode - mouse_cond = batch.get('mouse_cond', None) - infos = batch['info_list'] - - # Trajectory tensors may include a leading singleton batch dim per row - trajectory_latents = batch['trajectory_latents'] - if trajectory_latents.dim() == 7: - # [B, 1, S, C, T, H, W] -> [B, S, C, T, H, W] - trajectory_latents = trajectory_latents[:, 0] - elif trajectory_latents.dim() == 6: - # already [B, S, C, T, H, W] - pass - else: - raise ValueError( - f"Unexpected trajectory_latents dim: {trajectory_latents.dim()}" - ) - - trajectory_timesteps = batch['trajectory_timesteps'] - if trajectory_timesteps.dim() == 3: - # [B, 1, S] -> [B, S] - trajectory_timesteps = trajectory_timesteps[:, 0] - elif trajectory_timesteps.dim() == 2: - # [B, S] - pass - else: - raise ValueError( - f"Unexpected trajectory_timesteps dim: {trajectory_timesteps.dim()}" - ) - # [B, S, C, T, H, W] -> [B, S, T, C, H, W] to match self-forcing - trajectory_latents = trajectory_latents.permute(0, 1, 3, 2, 4, 5) - - # Move to device - device = get_local_torch_device() - training_batch.image_embeds = clip_feature.to(device, - dtype=torch.bfloat16) - training_batch.image_latents = first_frame_latent.to( - device, dtype=torch.bfloat16) - if keyboard_cond is not None and keyboard_cond.numel() > 0: - training_batch.keyboard_cond = keyboard_cond.to( - device, dtype=torch.bfloat16) - else: - training_batch.keyboard_cond = None - if mouse_cond is not None and mouse_cond.numel() > 0: - training_batch.mouse_cond = mouse_cond.to(device, - dtype=torch.bfloat16) - else: - training_batch.mouse_cond = None - training_batch.infos = infos - - # Validate action temporal dimensions match expected video frame count. - expected_num_frames = (self.training_args.num_latent_t - 1) * 4 + 1 - if training_batch.keyboard_cond is not None: - assert training_batch.keyboard_cond.shape[1] == expected_num_frames, ( - f"keyboard_cond temporal dim {training_batch.keyboard_cond.shape[1]} " - f"!= expected {expected_num_frames} " - f"(num_latent_t={self.training_args.num_latent_t})") - if training_batch.mouse_cond is not None: - assert training_batch.mouse_cond.shape[1] == expected_num_frames, ( - f"mouse_cond temporal dim {training_batch.mouse_cond.shape[1]} " - f"!= expected {expected_num_frames} " - f"(num_latent_t={self.training_args.num_latent_t})") - - return training_batch, trajectory_latents[:, :, :self.training_args. - num_latent_t].to( - device, - dtype=torch.bfloat16 - ), trajectory_timesteps.to( - device) - - def _get_timestep(self, - min_timestep: int, - max_timestep: int, - batch_size: int, - num_frame: int, - num_frame_per_block: int, - uniform_timestep: bool = False) -> torch.Tensor: - if uniform_timestep: - timestep = torch.randint(min_timestep, - max_timestep, [batch_size, 1], - device=self.device, - dtype=torch.long).repeat(1, num_frame) - return timestep - else: - timestep = torch.randint(min_timestep, - max_timestep, [batch_size, num_frame], - device=self.device, - dtype=torch.long) - # logger.info(f"individual timestep: {timestep}") - # make the noise level the same within every block - timestep = timestep.reshape(timestep.shape[0], -1, - num_frame_per_block) - timestep[:, :, 1:] = timestep[:, :, 0:1] - timestep = timestep.reshape(timestep.shape[0], -1) - return timestep - - def _prepare_dit_inputs(self, - training_batch: TrainingBatch) -> TrainingBatch: - """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" - - # First, call parent method to prepare noise, timesteps, etc. for video latents - training_batch = super()._prepare_dit_inputs(training_batch) - - assert isinstance(training_batch.image_latents, torch.Tensor) - image_latents = training_batch.image_latents.to( - get_local_torch_device(), dtype=torch.bfloat16) - - temporal_compression_ratio = 4 - num_frames = (self.training_args.num_latent_t - - 1) * temporal_compression_ratio + 1 - batch_size, num_channels, _, latent_height, latent_width = image_latents.shape - mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, - latent_width) - mask_lat_size[:, :, 1:] = 0 - - first_frame_mask = mask_lat_size[:, :, :1] - first_frame_mask = torch.repeat_interleave( - first_frame_mask, dim=2, repeats=temporal_compression_ratio) - mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], - dim=2) - mask_lat_size = mask_lat_size.view(batch_size, -1, - temporal_compression_ratio, - latent_height, latent_width) - mask_lat_size = mask_lat_size.transpose(1, 2) - mask_lat_size = mask_lat_size.to( - image_latents.device).to(dtype=torch.bfloat16) - - training_batch.noisy_model_input = torch.cat( - [training_batch.noisy_model_input, mask_lat_size, image_latents], - dim=1) - - return training_batch - - def _step_predict_next_latent( - self, traj_latents: torch.Tensor, traj_timesteps: torch.Tensor, - image_embeds: torch.Tensor, image_latents: torch.Tensor, - keyboard_cond: torch.Tensor | None, mouse_cond: torch.Tensor | None - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, dict[str, - torch.Tensor]]: - latent_vis_dict: dict[str, torch.Tensor] = {} - device = get_local_torch_device() - target_latent = traj_latents[:, -1] - - # Shapes: traj_latents [B, S, C, T, H, W], traj_timesteps [B, S] - B, S, num_frames, num_channels, height, width = traj_latents.shape - - # Lazily cache nearest trajectory index per DMD step based on the (fixed) S timesteps - if self._cached_closest_idx_per_dmd is None: - self._cached_closest_idx_per_dmd = torch.tensor( - [0, 12, 24, 36, S - 1], dtype=torch.long).cpu() - # [0, 1, 2, 3], dtype=torch.long).cpu() - logger.info("self._cached_closest_idx_per_dmd: %s", - self._cached_closest_idx_per_dmd) - logger.info( - "corresponding timesteps: %s", self.noise_scheduler.timesteps[ - self._cached_closest_idx_per_dmd]) - - # Select the K indexes from traj_latents using self._cached_closest_idx_per_dmd - # traj_latents: [B, S, C, T, H, W], self._cached_closest_idx_per_dmd: [K] - # Output: [B, K, C, T, H, W] - assert self._cached_closest_idx_per_dmd is not None - relevant_traj_latents = torch.index_select( - traj_latents, - dim=1, - index=self._cached_closest_idx_per_dmd.to(traj_latents.device)) - logger.info("relevant_traj_latents: %s", relevant_traj_latents.shape) - # assert relevant_traj_latents.shape[0] == 1 - - indexes = self._get_timestep( # [B, num_frames] - 0, - len(self.dmd_denoising_steps), - B, - num_frames, - 3, - uniform_timestep=False) - logger.info("indexes: %s", indexes.shape) - logger.info("indexes: %s", indexes) - # noisy_input = relevant_traj_latents[indexes] - noisy_input = torch.gather( - relevant_traj_latents, - dim=1, - index=indexes.reshape(B, 1, num_frames, 1, 1, - 1).expand(-1, -1, -1, num_channels, height, - width).to(self.device)).squeeze(1) - latent_model_input = noisy_input.permute(0, 2, 1, 3, 4) - if image_latents is not None: - latent_model_input = torch.cat([ - latent_model_input, - image_latents.to(latent_model_input.device, - latent_model_input.dtype), - ], - dim=1) - timestep = self.dmd_denoising_steps[indexes] - logger.info("selected timestep for rank %s: %s", - self.global_rank, - timestep, - local_main_process_only=False) - - # Prepare inputs for transformer - latent_vis_dict["noisy_input"] = noisy_input.permute( - 0, 2, 1, 3, 4).detach().clone().cpu() - latent_vis_dict["x0"] = target_latent.permute(0, 2, 1, 3, - 4).detach().clone().cpu() - - latent_model_input = latent_model_input.to(device, dtype=torch.bfloat16) - timestep = timestep.to(device, dtype=torch.bfloat16) - - logger.info("========== Transformer Input ==========") - logger.info("hidden_states (latent_model_input) shape: %s, dtype: %s", - latent_model_input.shape, latent_model_input.dtype) - logger.info("hidden_states min/max/mean: %.4f / %.4f / %.4f", - latent_model_input.min().item(), - latent_model_input.max().item(), - latent_model_input.mean().item()) - logger.info("encoder_hidden_states_image (image_embeds) shape: %s", - image_embeds.shape if image_embeds is not None else None) - logger.info("timestep shape: %s, dtype: %s", timestep.shape, - timestep.dtype) - logger.info("keyboard_cond: %s", - keyboard_cond.shape if keyboard_cond is not None else None) - logger.info("mouse_cond: %s", - mouse_cond.shape if mouse_cond is not None else None) - - if keyboard_cond is not None and mouse_cond is not None: - viewmats_list = [] - intrinsics_list = [] - action_labels_list = [] - for b in range(latent_model_input.shape[0]): - viewmats, intrinsics, action_labels = process_custom_actions( - keyboard_cond[b], mouse_cond[b]) - viewmats_list.append(viewmats) - intrinsics_list.append(intrinsics) - action_labels_list.append(action_labels) - viewmats = torch.stack(viewmats_list, - dim=0).to(device=device, - dtype=torch.bfloat16) - intrinsics = torch.stack(intrinsics_list, - dim=0).to(device=device, - dtype=torch.bfloat16) - action_labels = torch.stack(action_labels_list, - dim=0).to(device=device, - dtype=torch.bfloat16) - else: - viewmats = None - intrinsics = None - action_labels = None - - empty_text = torch.zeros( - (latent_model_input.shape[0], 0, self.transformer.hidden_size), - device=device, - dtype=torch.bfloat16) - - input_kwargs = { - "hidden_states": latent_model_input, - "encoder_hidden_states": empty_text, - "encoder_hidden_states_image": image_embeds, - "timestep": timestep, - "viewmats": viewmats, - "Ks": intrinsics, - "action": action_labels, - "return_dict": False, - } - # Predict noise and step the scheduler to obtain next latent - with set_forward_context(current_timestep=timestep, - attn_metadata=None, - forward_batch=None): - noise_pred = self.transformer(**input_kwargs).permute(0, 2, 1, 3, 4) - - logger.info("========== Transformer Output ==========") - logger.info("noise_pred shape: %s", noise_pred.shape) - logger.info("noise_pred min/max/mean: %.4f / %.4f / %.4f", - noise_pred.min().item(), - noise_pred.max().item(), - noise_pred.mean().item()) - - from fastvideo.models.utils import pred_noise_to_pred_video - pred_video = pred_noise_to_pred_video( - pred_noise=noise_pred.flatten(0, 1), - noise_input_latent=noisy_input.flatten(0, 1), - timestep=timestep.to(dtype=torch.bfloat16).flatten(0, 1), - scheduler=self.modules["scheduler"]).unflatten( - 0, noise_pred.shape[:2]) - latent_vis_dict["pred_video"] = pred_video.permute( - 0, 2, 1, 3, 4).detach().clone().cpu() - - return pred_video, target_latent, timestep, latent_vis_dict - - def train_one_step(self, training_batch): # type: ignore[override] - self.transformer.train() - self.optimizer.zero_grad() - training_batch.total_loss = 0.0 - args = cast(TrainingArgs, self.training_args) - - # Using cached nearest index per DMD step; computation happens in _step_predict_next_latent - - for _ in range(args.gradient_accumulation_steps): - training_batch, traj_latents, traj_timesteps = self._get_next_batch( - training_batch) - image_embeds = training_batch.image_embeds - image_latents = training_batch.image_latents - keyboard_cond = training_batch.keyboard_cond - mouse_cond = training_batch.mouse_cond - assert traj_latents.shape[0] == 1 - - # Shapes: traj_latents [B, S, C, T, H, W], traj_timesteps [B, S] - _, S = traj_latents.shape[0], traj_latents.shape[1] - if S < 2: - raise ValueError("Trajectory must contain at least 2 steps") - - # Forward to predict next latent by stepping scheduler with predicted noise - noise_pred, target_latent, t, latent_vis_dict = self._step_predict_next_latent( - traj_latents, traj_timesteps, image_embeds, image_latents, - keyboard_cond, mouse_cond) - - training_batch.latent_vis_dict.update(latent_vis_dict) - - mask = t != 0 - - # Compute loss - loss = F.mse_loss(noise_pred[mask], - target_latent[mask], - reduction="mean") - loss = loss / args.gradient_accumulation_steps - - with set_forward_context(current_timestep=t, - attn_metadata=None, - forward_batch=None): - loss.backward() - avg_loss = loss.detach().clone() - training_batch.total_loss += avg_loss.item() - - # Clip grad and step optimizers - grad_norm = clip_grad_norm_while_handling_failing_dtensor_cases( - [p for p in self.transformer.parameters() if p.requires_grad], - args.max_grad_norm if args.max_grad_norm is not None else 0.0) - - self.optimizer.step() - self.lr_scheduler.step() - - if grad_norm is None: - grad_value = 0.0 - else: - try: - if isinstance(grad_norm, torch.Tensor): - grad_value = float(grad_norm.detach().float().item()) - else: - grad_value = float(grad_norm) - except Exception: - grad_value = 0.0 - training_batch.grad_norm = grad_value - B, S, T, C, H, W = traj_latents.shape - training_batch.raw_latent_shape = (B, C, T, H, W) - return training_batch - - def _prepare_validation_batch(self, sampling_param: SamplingParam, - training_args: TrainingArgs, - validation_batch: dict[str, Any], - num_inference_steps: int) -> ForwardBatch: - sampling_param.prompt = validation_batch['prompt'] - sampling_param.height = training_args.num_height - sampling_param.width = training_args.num_width - sampling_param.image_path = validation_batch.get( - 'image_path') or validation_batch.get('video_path') - sampling_param.num_inference_steps = num_inference_steps - sampling_param.data_type = "video" - assert self.seed is not None - sampling_param.seed = self.seed - - latents_size = [(sampling_param.num_frames - 1) // 4 + 1, - sampling_param.height // 8, sampling_param.width // 8] - n_tokens = latents_size[0] * latents_size[1] * latents_size[2] - temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (training_args.num_latent_t - - 1) * temporal_compression_factor + 1 - sampling_param.num_frames = num_frames - batch = ForwardBatch( - **shallow_asdict(sampling_param), - latents=None, - generator=torch.Generator(device="cpu").manual_seed(self.seed), - n_tokens=n_tokens, - eta=0.0, - VSA_sparsity=training_args.VSA_sparsity, - ) - if "image" in validation_batch and validation_batch["image"] is not None: - batch.pil_image = validation_batch["image"] - - if "keyboard_cond" in validation_batch and validation_batch[ - "keyboard_cond"] is not None: - keyboard_cond = validation_batch["keyboard_cond"] - keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) - keyboard_cond = keyboard_cond.unsqueeze(0) - batch.keyboard_cond = keyboard_cond - - if "mouse_cond" in validation_batch and validation_batch[ - "mouse_cond"] is not None: - mouse_cond = validation_batch["mouse_cond"] - mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) - mouse_cond = mouse_cond.unsqueeze(0) - batch.mouse_cond = mouse_cond - - return batch - - def _post_process_validation_frames( - self, frames: list[np.ndarray], - batch: ForwardBatch) -> list[np.ndarray]: - """Apply action overlay to validation frames for WanGame. - - Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. - """ - # Check if action data is available - keyboard_cond = getattr(batch, 'keyboard_cond', None) - mouse_cond = getattr(batch, 'mouse_cond', None) - - if keyboard_cond is None and mouse_cond is None: - return frames - - # Import overlay functions - from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, - draw_mouse_on_frame) - - # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) - if keyboard_cond is not None: - keyboard_cond = keyboard_cond.squeeze( - 0).cpu().float().numpy() # (T, 6) - if mouse_cond is not None: - mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) - - # MatrixGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] - key_names = ["W", "S", "A", "D", "left", "right"] - - processed_frames = [] - for frame_idx, frame in enumerate(frames): - frame = np.ascontiguousarray(frame.copy()) - - # Draw keyboard overlay - if keyboard_cond is not None and frame_idx < len(keyboard_cond): - keys = { - key_names[i]: bool(keyboard_cond[frame_idx, i]) - for i in range(min(len(key_names), keyboard_cond.shape[1])) - } - draw_keys_on_frame(frame, keys, mode='universal') - - # Draw mouse overlay - if mouse_cond is not None and frame_idx < len(mouse_cond): - pitch = float(mouse_cond[frame_idx, 0]) - yaw = float(mouse_cond[frame_idx, 1]) - draw_mouse_on_frame(frame, pitch, yaw) - - processed_frames.append(frame) - - return processed_frames - - - def visualize_intermediate_latents(self, training_batch: TrainingBatch, - training_args: TrainingArgs, step: int): - tracker_loss_dict: dict[str, Any] = {} - latents_vis_dict = training_batch.latent_vis_dict - latent_log_keys = ['noisy_input', 'x0', 'pred_video'] - for latent_key in latent_log_keys: - assert latent_key in latents_vis_dict and latents_vis_dict[ - latent_key] is not None - latent = latents_vis_dict[latent_key] - pixel_latent = self.decoding_stage.decode(latent, training_args) - - video = pixel_latent.cpu().float() - video = video.permute(0, 2, 1, 3, 4) - video = (video * 255).numpy().astype(np.uint8) - - keyboard_cond = getattr(training_batch, "keyboard_cond", None) - mouse_cond = getattr(training_batch, "mouse_cond", None) - for batch_idx in range(video.shape[0]): - sample_batch = type("ValidationBatch", (), {})() - if keyboard_cond is not None and batch_idx < keyboard_cond.shape[0]: - sample_batch.keyboard_cond = keyboard_cond[batch_idx:batch_idx + 1] - if mouse_cond is not None and batch_idx < mouse_cond.shape[0]: - sample_batch.mouse_cond = mouse_cond[batch_idx:batch_idx + 1] - - video_frames = [ - np.transpose(video[batch_idx, frame_idx], (1, 2, 0)) - for frame_idx in range(video.shape[1]) - ] - video_frames = self._post_process_validation_frames( - video_frames, cast(ForwardBatch, sample_batch)) - video[batch_idx] = np.stack([ - np.transpose(frame, (2, 0, 1)) for frame in video_frames - ], axis=0) - - video_artifact = self.tracker.video( - video, fps=16, format="mp4") # change to 16 for Wan2.1 - if video_artifact is not None: - tracker_loss_dict[latent_key] = video_artifact - # Clean up references - del video, pixel_latent, latent - - if self.global_rank == 0 and tracker_loss_dict: - self.tracker.log_artifacts(tracker_loss_dict, step) - - -def main(args) -> None: - logger.info("Starting ODE-init training pipeline...") - pipeline = WanGameODEInitTrainingPipeline.from_pretrained( - args.pretrained_model_name_or_path, args=args) - args = pipeline.training_args - pipeline.train() - logger.info("ODE-init training pipeline done") - - -if __name__ == "__main__": - argv = sys.argv - from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.utils import FlexibleArgumentParser - parser = FlexibleArgumentParser() - parser = TrainingArgs.add_cli_args(parser) - parser = FastVideoArgs.add_cli_args(parser) - args = parser.parse_args() - args.dit_cpu_offload = False - main(args) diff --git a/fastvideo/training/wangame_self_forcing_distillation_pipeline.py b/fastvideo/training/wangame_self_forcing_distillation_pipeline.py deleted file mode 100644 index 0325826ac..000000000 --- a/fastvideo/training/wangame_self_forcing_distillation_pipeline.py +++ /dev/null @@ -1,952 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -import sys -from copy import deepcopy -from typing import Any - -import numpy as np -import torch -import torch.distributed as dist -import torch.nn.functional as F -from einops import rearrange - -from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.dataloader.schema import ( - pyarrow_schema_ode_trajectory_wangame) -from fastvideo.distributed import get_local_torch_device -from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.forward_context import set_forward_context -from fastvideo.logger import init_logger -from fastvideo.models.dits.hyworld.pose import process_custom_actions -from fastvideo.models.schedulers.scheduling_self_forcing_flow_match import ( - SelfForcingFlowMatchScheduler) -from fastvideo.models.utils import pred_noise_to_pred_video -from fastvideo.pipelines.basic.wan.wangame_causal_dmd_pipeline import ( - WanGameCausalDMDPipeline) -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch -from fastvideo.training.self_forcing_distillation_pipeline import ( - SelfForcingDistillationPipeline) -from fastvideo.training.training_utils import shift_timestep -from fastvideo.utils import is_vsa_available, shallow_asdict - -vsa_available = is_vsa_available() - -logger = init_logger(__name__) - - -class WanGameSelfForcingDistillationPipeline(SelfForcingDistillationPipeline): - """ - A self-forcing distillation pipeline for WanGame that uses the self-forcing methodology - with DMD for video generation. - """ - _required_config_modules = [ - "scheduler", - "transformer", - "vae", - ] - - def set_schemas(self): - self.train_dataset_schema = pyarrow_schema_ode_trajectory_wangame - - def _initialize_simulation_caches( - self, - batch_size: int, - dtype: torch.dtype, - device: torch.device, - *, - max_num_frames: int | None = None, - ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Initialize KV cache and cross-attention cache for multi-step simulation.""" - num_transformer_blocks = len(self.transformer.blocks) - latent_shape = self.video_latent_shape_sp - _, num_frames, _, height, width = latent_shape - - _, p_h, p_w = self.transformer.patch_size - post_patch_height = height // p_h - post_patch_width = width // p_w - - frame_seq_length = post_patch_height * post_patch_width - self.frame_seq_length = frame_seq_length - - # Get model configuration parameters - handle FSDP wrapping - num_attention_heads = getattr(self.transformer, 'num_attention_heads', - None) - attention_head_dim = getattr(self.transformer, 'attention_head_dim', - None) - - # 1 CLS token + 256 patch tokens = 257 - text_len = 257 - - if max_num_frames is None: - max_num_frames = num_frames - num_max_frames = max(max_num_frames, num_frames) - kv_cache_size = num_max_frames * frame_seq_length - # WanGame causal attention stores both RoPE and PRoPE branches in cache. - cache_head_dim = attention_head_dim * 2 - - kv_cache = [] - for _ in range(num_transformer_blocks): - kv_cache.append({ - "k": - torch.zeros([ - batch_size, kv_cache_size, num_attention_heads, - cache_head_dim - ], - dtype=dtype, - device=device), - "v": - torch.zeros([ - batch_size, kv_cache_size, num_attention_heads, - cache_head_dim - ], - dtype=dtype, - device=device), - "global_end_index": - torch.tensor([0], dtype=torch.long, device=device), - "local_end_index": - torch.tensor([0], dtype=torch.long, device=device) - }) - - # Initialize cross-attention cache - crossattn_cache = [] - for _ in range(num_transformer_blocks): - crossattn_cache.append({"is_init": False}) - - return kv_cache, crossattn_cache - - def _reset_simulation_caches( - self, kv_cache: list[dict[str, - Any]], crossattn_cache: list[dict[str, - Any]] - ) -> None: - """Reset KV cache and cross-attention cache to clean state.""" - if kv_cache is not None: - for cache_dict in kv_cache: - cache_dict["global_end_index"].fill_(0) - cache_dict["local_end_index"].fill_(0) - cache_dict["k"].zero_() - cache_dict["v"].zero_() - - if crossattn_cache is not None: - for cache_dict in crossattn_cache: - cache_dict["is_init"] = False - if "k" in cache_dict: - cache_dict["k"].zero_() - if "v" in cache_dict: - cache_dict["v"].zero_() - - def _generator_multi_step_simulation_forward( - self, - training_batch: TrainingBatch, - return_sim_steps: bool = False) -> torch.Tensor: - """Forward pass through student transformer matching inference procedure with KV cache management. - - This function is adapted from the reference self-forcing implementation's inference_with_trajectory - and includes gradient masking logic for dynamic frame generation. - """ - latents = training_batch.latents - dtype = latents.dtype - batch_size = latents.shape[0] - initial_latent = getattr(training_batch, 'image_latent', None) - - # Dynamic frame generation logic (adapted from _run_generator) - num_training_frames = getattr(self.training_args, 'num_latent_t', 21) - - # During training, the number of generated frames should be uniformly sampled from - # [21, self.num_training_frames], but still being a multiple of self.num_frame_per_block - min_num_frames = 20 if self.independent_first_frame else 21 - max_num_frames = num_training_frames - 1 if self.independent_first_frame else num_training_frames - assert max_num_frames % self.num_frame_per_block == 0 - assert min_num_frames % self.num_frame_per_block == 0 - max_num_blocks = max_num_frames // self.num_frame_per_block - min_num_blocks = min_num_frames // self.num_frame_per_block - - # Sample number of blocks and sync across processes - num_generated_blocks = torch.randint(min_num_blocks, - max_num_blocks + 1, (1, ), - device=self.device) - if dist.is_initialized(): - dist.broadcast(num_generated_blocks, src=0) - num_generated_blocks = num_generated_blocks.item() - num_generated_frames = num_generated_blocks * self.num_frame_per_block - if self.independent_first_frame and initial_latent is None: - num_generated_frames += 1 - min_num_frames += 1 - - # Create noise with dynamic shape - if initial_latent is not None: - noise_shape = [ - batch_size, num_generated_frames - 1, - *self.video_latent_shape[2:] - ] - else: - noise_shape = [ - batch_size, num_generated_frames, *self.video_latent_shape[2:] - ] - - noise = torch.randn(noise_shape, device=self.device, dtype=dtype) - if self.sp_world_size > 1: - noise = rearrange(noise, - "b (n t) c h w -> b n t c h w", - n=self.sp_world_size).contiguous() - noise = noise[:, self.rank_in_sp_group, :, :, :, :] - - batch_size, num_frames, num_channels, height, width = noise.shape - - # Block size calculation - if not self.independent_first_frame or (self.independent_first_frame - and initial_latent is not None): - assert num_frames % self.num_frame_per_block == 0 - num_blocks = num_frames // self.num_frame_per_block - else: - assert (num_frames - 1) % self.num_frame_per_block == 0 - num_blocks = (num_frames - 1) // self.num_frame_per_block - - num_input_frames = initial_latent.shape[ - 1] if initial_latent is not None else 0 - num_output_frames = num_frames + num_input_frames - output = torch.zeros( - [batch_size, num_output_frames, num_channels, height, width], - device=noise.device, - dtype=noise.dtype) - - def get_model_device(model): - if model is None: - return "None" - try: - return next(model.parameters()).device - except (StopIteration, AttributeError): - return "Unknown" - - # Step 1: Initialize KV cache to all zeros - cache_frames = num_generated_frames + num_input_frames - (self.kv_cache1, - self.crossattn_cache) = self._initialize_simulation_caches( - batch_size, dtype, self.device, max_num_frames=cache_frames) - - # Step 2: Cache context feature - current_start_frame = 0 - if initial_latent is not None: - timestep = torch.ones( - [batch_size, 1], device=noise.device, dtype=torch.int64) * 0 - output[:, :1] = initial_latent - with torch.no_grad(): - # Build input kwargs for initial latent - training_batch_temp = self._build_distill_input_kwargs( - initial_latent, - timestep * 0, - training_batch.conditional_dict, - training_batch, - frame_start=0, - frame_end=1, - num_frame_per_block=1) - - # we process the image latent with self.transformer_2 (low-noise expert) - current_model = self.transformer_2 if self.transformer_2 is not None else self.transformer - current_model( - **training_batch_temp.input_kwargs, - kv_cache=self.kv_cache1, - crossattn_cache=self.crossattn_cache, - current_start=current_start_frame * self.frame_seq_length, - start_frame=current_start_frame) - current_start_frame += 1 - - # Step 3: Temporal denoising loop - all_num_frames = [self.num_frame_per_block] * num_blocks - if self.independent_first_frame and initial_latent is None: - all_num_frames = [1] + all_num_frames - num_denoising_steps = len(self.denoising_step_list) - exit_flags = self.generate_and_sync_list(len(all_num_frames), - num_denoising_steps, - device=noise.device) - start_gradient_frame_index = max(0, num_output_frames - 21) - - for block_index, current_num_frames in enumerate(all_num_frames): - noisy_input = noise[:, current_start_frame - - num_input_frames:current_start_frame + - current_num_frames - num_input_frames] - - # Step 3.1: Spatial denoising loop - for index, current_timestep in enumerate(self.denoising_step_list): - if self.same_step_across_blocks: - exit_flag = (index == exit_flags[0]) - else: - exit_flag = (index == exit_flags[block_index]) - - timestep = torch.ones([batch_size, current_num_frames], - device=noise.device, - dtype=torch.int64) * current_timestep - - if self.boundary_timestep is not None and current_timestep < self.boundary_timestep and self.transformer_2 is not None: - current_model = self.transformer_2 - else: - current_model = self.transformer - - if not exit_flag: - with torch.no_grad(): - # Build input kwargs - training_batch_temp = self._build_distill_input_kwargs( - noisy_input, - timestep, - training_batch.conditional_dict, - training_batch, - frame_start=current_start_frame, - frame_end=current_start_frame + current_num_frames, - num_frame_per_block=current_num_frames) - - pred_flow = current_model( - **training_batch_temp.input_kwargs, - kv_cache=self.kv_cache1, - crossattn_cache=self.crossattn_cache, - current_start=current_start_frame * - self.frame_seq_length, - start_frame=current_start_frame).permute( - 0, 2, 1, 3, 4) - - denoised_pred = pred_noise_to_pred_video( - pred_noise=pred_flow.flatten(0, 1), - noise_input_latent=noisy_input.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler).unflatten( - 0, pred_flow.shape[:2]) - - next_timestep = self.denoising_step_list[index + 1] - noisy_input = self.noise_scheduler.add_noise( - denoised_pred.flatten(0, 1), - torch.randn_like(denoised_pred.flatten(0, 1)), - next_timestep * - torch.ones([batch_size * current_num_frames], - device=noise.device, - dtype=torch.long)).unflatten( - 0, denoised_pred.shape[:2]) - else: - # Final prediction with gradient control - if current_start_frame < start_gradient_frame_index: - with torch.no_grad(): - training_batch_temp = self._build_distill_input_kwargs( - noisy_input, - timestep, - training_batch.conditional_dict, - training_batch, - frame_start=current_start_frame, - frame_end=current_start_frame + - current_num_frames, - num_frame_per_block=current_num_frames) - - pred_flow = current_model( - **training_batch_temp.input_kwargs, - kv_cache=self.kv_cache1, - crossattn_cache=self.crossattn_cache, - current_start=current_start_frame * - self.frame_seq_length, - start_frame=current_start_frame).permute( - 0, 2, 1, 3, 4) - else: - training_batch_temp = self._build_distill_input_kwargs( - noisy_input, - timestep, - training_batch.conditional_dict, - training_batch, - frame_start=current_start_frame, - frame_end=current_start_frame + current_num_frames, - num_frame_per_block=current_num_frames) - - pred_flow = current_model( - **training_batch_temp.input_kwargs, - kv_cache=self.kv_cache1, - crossattn_cache=self.crossattn_cache, - current_start=current_start_frame * - self.frame_seq_length, - start_frame=current_start_frame).permute( - 0, 2, 1, 3, 4) - - denoised_pred = pred_noise_to_pred_video( - pred_noise=pred_flow.flatten(0, 1), - noise_input_latent=noisy_input.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler).unflatten( - 0, pred_flow.shape[:2]) - break - - # Step 3.2: record the model's output - output[:, current_start_frame:current_start_frame + - current_num_frames] = denoised_pred - - # Step 3.3: rerun with timestep zero to update the cache - context_timestep = torch.ones_like(timestep) * self.context_noise - denoised_pred = self.noise_scheduler.add_noise( - denoised_pred.flatten(0, 1), - torch.randn_like(denoised_pred.flatten(0, 1)), - context_timestep).unflatten(0, denoised_pred.shape[:2]) - - with torch.no_grad(): - training_batch_temp = self._build_distill_input_kwargs( - denoised_pred, - context_timestep, - training_batch.conditional_dict, - training_batch, - frame_start=current_start_frame, - frame_end=current_start_frame + current_num_frames, - num_frame_per_block=current_num_frames) - - # context_timestep is 0 so we use transformer_2 - current_model = self.transformer_2 if self.transformer_2 is not None else self.transformer - current_model( - **training_batch_temp.input_kwargs, - kv_cache=self.kv_cache1, - crossattn_cache=self.crossattn_cache, - current_start=current_start_frame * self.frame_seq_length, - start_frame=current_start_frame) - - # Step 3.4: update the start and end frame indices - current_start_frame += current_num_frames - - # Handle last 21 frames logic - pred_image_or_video = output - if num_input_frames > 0: - pred_image_or_video = output[:, num_input_frames:] - - # Slice last 21 frames if we generated more - gradient_mask = None - if pred_image_or_video.shape[1] > 21: - with torch.no_grad(): - # Re-encode to get image latent - latent_to_decode = pred_image_or_video[:, :-20, ...] - # Decode to video - latent_to_decode = latent_to_decode.permute( - 0, 2, 1, 3, 4) # [B, C, F, H, W] - - # Apply VAE scaling and shift factors - if isinstance(self.vae.scaling_factor, torch.Tensor): - latent_to_decode = latent_to_decode / self.vae.scaling_factor.to( - latent_to_decode.device, latent_to_decode.dtype) - else: - latent_to_decode = latent_to_decode / self.vae.scaling_factor - - if hasattr( - self.vae, - "shift_factor") and self.vae.shift_factor is not None: - if isinstance(self.vae.shift_factor, torch.Tensor): - latent_to_decode += self.vae.shift_factor.to( - latent_to_decode.device, latent_to_decode.dtype) - else: - latent_to_decode += self.vae.shift_factor - - # Decode to pixels - pixels = self.vae.decode(latent_to_decode) - frame = pixels[:, :, -1:, :, :].to( - dtype) # Last frame [B, C, 1, H, W] - - # Encode frame back to get image latent - image_latent = self.vae.encode(frame).to(dtype) - image_latent = image_latent.permute(0, 2, 1, 3, - 4) # [B, F, C, H, W] - - pred_image_or_video_last_21 = torch.cat( - [image_latent, pred_image_or_video[:, -20:, ...]], dim=1) - else: - pred_image_or_video_last_21 = pred_image_or_video - - # Set up gradient mask if we generated more than minimum frames - if num_generated_frames != min_num_frames: - # Currently, we do not use gradient for the first chunk, since it contains image latents - gradient_mask = torch.ones_like(pred_image_or_video_last_21, - dtype=torch.bool) - if self.independent_first_frame: - gradient_mask[:, :1] = False - else: - gradient_mask[:, :self.num_frame_per_block] = False - - # Apply gradient masking if needed - final_output = pred_image_or_video_last_21.to(dtype) - if gradient_mask is not None: - # Apply gradient masking: detach frames that shouldn't contribute gradients - final_output = torch.where( - gradient_mask, - pred_image_or_video_last_21, # Keep original values where gradient_mask is True - pred_image_or_video_last_21.detach( - ) # Detach where gradient_mask is False - ) - - # Store visualization data - training_batch.dmd_latent_vis_dict["generator_timestep"] = torch.tensor( - self.denoising_step_list[exit_flags[0]], - dtype=torch.float32, - device=self.device) - - # Store gradient mask information for debugging - if gradient_mask is not None: - training_batch.dmd_latent_vis_dict[ - "gradient_mask"] = gradient_mask.float() - training_batch.dmd_latent_vis_dict[ - "num_generated_frames"] = torch.tensor(num_generated_frames, - dtype=torch.float32, - device=self.device) - training_batch.dmd_latent_vis_dict["min_num_frames"] = torch.tensor( - min_num_frames, dtype=torch.float32, device=self.device) - - # Clean up caches - assert self.kv_cache1 is not None - assert self.crossattn_cache is not None - self._reset_simulation_caches(self.kv_cache1, self.crossattn_cache) - - return final_output if gradient_mask is not None else pred_image_or_video - - def initialize_validation_pipeline(self, training_args: TrainingArgs): - logger.info("Initializing validation pipeline...") - args_copy = deepcopy(training_args) - args_copy.inference_mode = True - # Use the same flow-matching scheduler as training for consistent validation. - validation_scheduler = SelfForcingFlowMatchScheduler( - shift=args_copy.pipeline_config.flow_shift, - sigma_min=0.0, - extra_one_step=True) - validation_scheduler.set_timesteps(num_inference_steps=1000, - training=True) - # Warm start validation with current transformer - self.validation_pipeline = WanGameCausalDMDPipeline.from_pretrained( - training_args.model_path, - args=args_copy, # type: ignore - inference_mode=True, - loaded_modules={ - "transformer": self.get_module("transformer"), - "vae": self.get_module("vae"), - "scheduler": validation_scheduler, - }, - tp_size=training_args.tp_size, - sp_size=training_args.sp_size, - num_gpus=training_args.num_gpus, - pin_cpu_memory=training_args.pin_cpu_memory, - dit_cpu_offload=True) - - def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: - batch = next(self.train_loader_iter, None) # type: ignore - if batch is None: - self.current_epoch += 1 - # Reset iterator for next epoch - self.train_loader_iter = iter(self.train_dataloader) - # Get first batch of new epoch - batch = next(self.train_loader_iter) - - clip_feature = batch['clip_feature'] - first_frame_latent = batch['first_frame_latent'] - keyboard_cond = batch.get('keyboard_cond', None) - mouse_cond = batch.get('mouse_cond', None) - infos = batch['info_list'] - - batch_size = clip_feature.shape[0] - vae_config = self.training_args.pipeline_config.vae_config.arch_config - num_channels = vae_config.z_dim - spatial_compression_ratio = vae_config.spatial_compression_ratio - - latent_height = self.training_args.num_height // spatial_compression_ratio - latent_width = self.training_args.num_width // spatial_compression_ratio - - latents = torch.randn(batch_size, num_channels, - self.training_args.num_latent_t, latent_height, - latent_width).to(get_local_torch_device(), - dtype=torch.bfloat16) - - training_batch.latents = latents.to(get_local_torch_device(), - dtype=torch.bfloat16) - training_batch.encoder_hidden_states = None - training_batch.encoder_attention_mask = None - training_batch.image_embeds = clip_feature.to(get_local_torch_device(), - dtype=torch.bfloat16) - training_batch.image_latents = first_frame_latent.to( - get_local_torch_device(), dtype=torch.bfloat16) - # Action conditioning - if keyboard_cond is not None and keyboard_cond.numel() > 0: - keyboard_cond_full = keyboard_cond.to(get_local_torch_device(), - dtype=torch.bfloat16) - training_batch.keyboard_cond = keyboard_cond_full # For Teacher/Critic (dim=6) - else: - training_batch.keyboard_cond = None - if mouse_cond is not None and mouse_cond.numel() > 0: - training_batch.mouse_cond = mouse_cond.to(get_local_torch_device(), - dtype=torch.bfloat16) - else: - training_batch.mouse_cond = None - training_batch.infos = infos - return training_batch - - def _prepare_dit_inputs(self, - training_batch: TrainingBatch) -> TrainingBatch: - """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" - # First, call parent method to prepare noise, timesteps, etc. for video latents - training_batch = super()._prepare_dit_inputs(training_batch) - - assert isinstance(training_batch.image_latents, torch.Tensor) - image_latents = training_batch.image_latents.to( - get_local_torch_device(), dtype=torch.bfloat16) - - # cond_concat = [mask(4), image_latent(16)] with 20 channels. - expected_cond_channels = 20 - if image_latents.shape[1] != expected_cond_channels: - raise ValueError( - "Unexpected first_frame_latent channels, " - "Expected {expected_cond_channels} (cond_concat), got {image_latents.shape[1]}." - ) - - if self.sp_world_size > 1: - total_frames = image_latents.shape[2] - # Split cond latents to local SP shard only when tensor is still global. - if total_frames == self.training_args.num_latent_t: - if total_frames % self.sp_world_size != 0: - raise ValueError( - "image_latents temporal dim is not divisible by SP world size: " - f"frames={total_frames}, sp_world_size={self.sp_world_size}" - ) - image_latents = rearrange(image_latents, - "b c (n t) h w -> b c n t h w", - n=self.sp_world_size).contiguous() - image_latents = image_latents[:, :, self.rank_in_sp_group, :, :, - :] - - training_batch.image_latents = image_latents - - return training_batch - - def _build_distill_input_kwargs( - self, - noise_input: torch.Tensor, - timestep: torch.Tensor, - text_dict: dict[str, torch.Tensor] | None, - training_batch: TrainingBatch, - frame_start: int | None = None, - frame_end: int | None = None, - num_frame_per_block: int | None = None) -> TrainingBatch: - # Image Embeds for conditioning - image_embeds = training_batch.image_embeds - assert torch.isnan(image_embeds).sum() == 0 - image_embeds = image_embeds.to(get_local_torch_device(), - dtype=torch.bfloat16) - - image_latents = training_batch.image_latents - if frame_start is not None and frame_end is not None: - image_latents = image_latents[:, :, frame_start:frame_end, :, :] - - vae_temporal_compression_ratio = 4 - if frame_start is not None and frame_end is not None: - action_frame_start = frame_start * vae_temporal_compression_ratio - action_frame_end = (frame_end - - 1) * vae_temporal_compression_ratio + 1 - if frame_start == 0: - action_frame_start = 0 - keyboard_cond_sliced = training_batch.keyboard_cond[:, - action_frame_start:action_frame_end, :] if training_batch.keyboard_cond is not None else None - mouse_cond_sliced = training_batch.mouse_cond[:, - action_frame_start:action_frame_end, :] if training_batch.mouse_cond is not None else None - else: - keyboard_cond_sliced = training_batch.keyboard_cond - mouse_cond_sliced = training_batch.mouse_cond - - if keyboard_cond_sliced is not None and mouse_cond_sliced is not None: - viewmats_list = [] - intrinsics_list = [] - action_labels_list = [] - for b in range(noise_input.shape[0]): - viewmats, intrinsics, action_labels = process_custom_actions( - keyboard_cond_sliced[b], mouse_cond_sliced[b]) - viewmats_list.append(viewmats) - intrinsics_list.append(intrinsics) - action_labels_list.append(action_labels) - - viewmats = torch.stack(viewmats_list, dim=0).to( - device=get_local_torch_device(), dtype=torch.bfloat16) - intrinsics = torch.stack(intrinsics_list, dim=0).to( - device=get_local_torch_device(), dtype=torch.bfloat16) - action_labels = torch.stack(action_labels_list, dim=0).to( - device=get_local_torch_device(), dtype=torch.bfloat16) - else: - viewmats = None - intrinsics = None - action_labels = None - - noisy_model_input = torch.cat( - [noise_input, image_latents.permute(0, 2, 1, 3, 4)], dim=2) - - training_batch.input_kwargs = { - "hidden_states": noisy_model_input.permute(0, 2, 1, 3, - 4), # bs, c, t, h, w - "encoder_hidden_states": None, - "timestep": timestep, - "encoder_hidden_states_image": image_embeds, - "viewmats": viewmats, - "Ks": intrinsics, - "action": action_labels, - "num_frame_per_block": num_frame_per_block if num_frame_per_block is not None else self.num_frame_per_block, - } - training_batch.noise_latents = noise_input - - return training_batch - - def _dmd_forward(self, generator_pred_video: torch.Tensor, - training_batch: TrainingBatch) -> torch.Tensor: - """Compute DMD (Diffusion Model Distillation) loss for WanGame.""" - original_latent = generator_pred_video - with torch.no_grad(): - timestep = torch.randint(0, - self.num_train_timestep, [1], - device=self.device, - dtype=torch.long) - - timestep = shift_timestep(timestep, self.timestep_shift, - self.num_train_timestep) - - timestep = timestep.clamp(self.min_timestep, self.max_timestep) - - noise = torch.randn(self.video_latent_shape, - device=self.device, - dtype=generator_pred_video.dtype) - - noisy_latent = self.noise_scheduler.add_noise( - generator_pred_video.flatten(0, 1), noise.flatten(0, 1), - timestep).detach().unflatten(0, (generator_pred_video.shape[0], - generator_pred_video.shape[1])) - - # Non-causal models expect 1D timestep (batch_size,) - critic_timestep = timestep.expand(noisy_latent.shape[0]) - - self._build_distill_input_kwargs( - noisy_latent, critic_timestep, None, training_batch - ) - - # fake_score_transformer forward - current_fake_score_transformer = self._get_fake_score_transformer( - timestep) - fake_score_pred_noise = current_fake_score_transformer( - **training_batch.input_kwargs - ).permute(0, 2, 1, 3, 4) - - faker_score_pred_video = pred_noise_to_pred_video( - pred_noise=fake_score_pred_noise.flatten(0, 1), - noise_input_latent=noisy_latent.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler).unflatten( - 0, fake_score_pred_noise.shape[:2]) - - # real_score_transformer forward - current_real_score_transformer = self._get_real_score_transformer( - timestep) - real_score_pred_noise = current_real_score_transformer( - **training_batch.input_kwargs - ).permute(0, 2, 1, 3, 4) - - real_score_pred_video = pred_noise_to_pred_video( - pred_noise=real_score_pred_noise.flatten(0, 1), - noise_input_latent=noisy_latent.flatten(0, 1), - timestep=timestep, - scheduler=self.noise_scheduler).unflatten( - 0, real_score_pred_noise.shape[:2]) - - # No CFG for WanGame - use real_score_pred_video directly - grad = (faker_score_pred_video - real_score_pred_video) / torch.abs( - original_latent - real_score_pred_video).mean() - grad = torch.nan_to_num(grad) - - dmd_loss = 0.5 * F.mse_loss( - original_latent.float(), - (original_latent.float() - grad.float()).detach()) - - training_batch.dmd_latent_vis_dict.update({ - "training_batch_dmd_fwd_clean_latent": - training_batch.latents, - "generator_pred_video": - original_latent.detach(), - "real_score_pred_video": - real_score_pred_video.detach(), - "faker_score_pred_video": - faker_score_pred_video.detach(), - "dmd_timestep": - timestep.detach(), - }) - - return dmd_loss - - def faker_score_forward( - self, training_batch: TrainingBatch - ) -> tuple[TrainingBatch, torch.Tensor]: - """Forward pass for critic training with WanGame action conditioning.""" - with torch.no_grad(), set_forward_context( - current_timestep=training_batch.timesteps, - attn_metadata=training_batch.attn_metadata_vsa): - if self.training_args.simulate_generator_forward: - generator_pred_video = self._generator_multi_step_simulation_forward( - training_batch) - else: - generator_pred_video = self._generator_forward(training_batch) - - fake_score_timestep = torch.randint(0, - self.num_train_timestep, [1], - device=self.device, - dtype=torch.long) - - fake_score_timestep = shift_timestep(fake_score_timestep, - self.timestep_shift, - self.num_train_timestep) - - fake_score_timestep = fake_score_timestep.clamp(self.min_timestep, - self.max_timestep) - - fake_score_noise = torch.randn(self.video_latent_shape, - device=self.device, - dtype=generator_pred_video.dtype) - - noisy_generator_pred_video = self.noise_scheduler.add_noise( - generator_pred_video.flatten(0, 1), - fake_score_noise.flatten(0, 1), fake_score_timestep).unflatten( - 0, - (generator_pred_video.shape[0], generator_pred_video.shape[1])) - - # Non-causal critic expects 1D timestep (batch_size,), not 2D (batch_size, num_frames). - expanded_fake_score_timestep = fake_score_timestep.expand( - noisy_generator_pred_video.shape[0]) - - self._build_distill_input_kwargs( - noisy_generator_pred_video, expanded_fake_score_timestep, None, training_batch - ) - - with set_forward_context(current_timestep=training_batch.timesteps, - attn_metadata=training_batch.attn_metadata): - current_fake_score_transformer = self._get_fake_score_transformer(fake_score_timestep) - fake_score_pred_noise = current_fake_score_transformer( - **training_batch.input_kwargs - ).permute(0, 2, 1, 3, 4) - - target = fake_score_noise - generator_pred_video - flow_matching_loss = torch.mean((fake_score_pred_noise - target)**2) - - training_batch.fake_score_latent_vis_dict = { - "training_batch_fakerscore_fwd_clean_latent": - training_batch.latents, - "generator_pred_video": generator_pred_video, - "fake_score_timestep": fake_score_timestep, - } - - return training_batch, flow_matching_loss - - def _prepare_validation_batch(self, sampling_param: SamplingParam, - training_args: TrainingArgs, - validation_batch: dict[str, Any], - num_inference_steps: int) -> ForwardBatch: - sampling_param.prompt = validation_batch['prompt'] - sampling_param.height = training_args.num_height - sampling_param.width = training_args.num_width - sampling_param.image_path = validation_batch.get( - 'image_path') or validation_batch.get('video_path') - sampling_param.num_inference_steps = num_inference_steps - sampling_param.data_type = "video" - assert self.seed is not None - sampling_param.seed = self.seed - - latents_size = [(sampling_param.num_frames - 1) // 4 + 1, - sampling_param.height // 8, sampling_param.width // 8] - n_tokens = latents_size[0] * latents_size[1] * latents_size[2] - temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (training_args.num_latent_t - - 1) * temporal_compression_factor + 1 - sampling_param.num_frames = num_frames - batch = ForwardBatch( - **shallow_asdict(sampling_param), - latents=None, - generator=torch.Generator(device="cpu").manual_seed(self.seed), - n_tokens=n_tokens, - eta=0.0, - VSA_sparsity=training_args.VSA_sparsity, - ) - if "image" in validation_batch and validation_batch["image"] is not None: - batch.pil_image = validation_batch["image"] - - if "keyboard_cond" in validation_batch and validation_batch[ - "keyboard_cond"] is not None: - keyboard_cond = validation_batch["keyboard_cond"] - if isinstance(keyboard_cond, torch.Tensor): - keyboard_cond = keyboard_cond.detach().clone().to(dtype=torch.bfloat16) - else: - keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) - keyboard_cond = keyboard_cond.unsqueeze(0) - batch.keyboard_cond = keyboard_cond - - if "mouse_cond" in validation_batch and validation_batch[ - "mouse_cond"] is not None: - mouse_cond = validation_batch["mouse_cond"] - if isinstance(mouse_cond, torch.Tensor): - mouse_cond = mouse_cond.detach().clone().to(dtype=torch.bfloat16) - else: - mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) - mouse_cond = mouse_cond.unsqueeze(0) - batch.mouse_cond = mouse_cond - - return batch - - def _post_process_validation_frames( - self, frames: list[np.ndarray], - batch: ForwardBatch) -> list[np.ndarray]: - """Apply action overlay to validation frames for WanGame. - - Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. - """ - # Check if action data is available - keyboard_cond = getattr(batch, 'keyboard_cond', None) - mouse_cond = getattr(batch, 'mouse_cond', None) - - if keyboard_cond is None and mouse_cond is None: - return frames - - # Import overlay functions - from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, - draw_mouse_on_frame) - - # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) - if keyboard_cond is not None: - keyboard_cond = keyboard_cond.squeeze( - 0).cpu().float().numpy() # (T, 6) - if mouse_cond is not None: - mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) - - # WanGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] - key_names = ["W", "S", "A", "D", "left", "right"] - - processed_frames = [] - for frame_idx, frame in enumerate(frames): - frame = np.ascontiguousarray(frame.copy()) - - # Draw keyboard overlay - if keyboard_cond is not None and frame_idx < len(keyboard_cond): - keys = { - key_names[i]: bool(keyboard_cond[frame_idx, i]) - for i in range(min(len(key_names), keyboard_cond.shape[1])) - } - draw_keys_on_frame(frame, keys, mode='universal') - - # Draw mouse overlay - if mouse_cond is not None and frame_idx < len(mouse_cond): - pitch = float(mouse_cond[frame_idx, 0]) - yaw = float(mouse_cond[frame_idx, 1]) - draw_mouse_on_frame(frame, pitch, yaw) - - processed_frames.append(frame) - - return processed_frames - - -def main(args) -> None: - logger.info("Starting WanGame self-forcing distillation pipeline...") - - pipeline = WanGameSelfForcingDistillationPipeline.from_pretrained( - args.pretrained_model_name_or_path, args=args) - - args = pipeline.training_args - pipeline.train() - logger.info("WanGame self-forcing distillation pipeline completed") - - -if __name__ == "__main__": - argv = sys.argv - from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.utils import FlexibleArgumentParser - parser = FlexibleArgumentParser() - parser = TrainingArgs.add_cli_args(parser) - parser = FastVideoArgs.add_cli_args(parser) - args = parser.parse_args() - main(args) diff --git a/fastvideo/training/wangame_training_pipeline.py b/fastvideo/training/wangame_training_pipeline.py deleted file mode 100644 index 68c626692..000000000 --- a/fastvideo/training/wangame_training_pipeline.py +++ /dev/null @@ -1,542 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -import sys -from typing import Any - -import numpy as np -import torch - -from fastvideo.configs.sample import SamplingParam -from fastvideo.dataset.dataloader.schema import pyarrow_schema_wangame -from fastvideo.distributed import get_local_torch_device -from fastvideo.fastvideo_args import FastVideoArgs, TrainingArgs -from fastvideo.logger import init_logger -from fastvideo.models.schedulers.scheduling_flow_unipc_multistep import ( - FlowUniPCMultistepScheduler) -from fastvideo.pipelines.basic.wan.wangame_i2v_pipeline import WanGameActionImageToVideoPipeline -from fastvideo.pipelines.pipeline_batch_info import ForwardBatch, TrainingBatch -from fastvideo.training.training_pipeline import TrainingPipeline -from fastvideo.training.training_utils import count_trainable, count_trainable_total -from fastvideo.utils import is_vsa_available, shallow_asdict - -vsa_available = is_vsa_available() - -logger = init_logger(__name__) - - -class WanGameTrainingPipeline(TrainingPipeline): - """ - A training pipeline for WanGame-2.1-Fun-1.3B-InP. - """ - _required_config_modules = ["scheduler", "transformer", "vae"] - - _FLOW_EVAL_SCALAR_KEYS = ( - "mf_epe_mean", - "mf_angle_err_mean", - "mf_cosine_mean", - "mf_mag_ratio_mean", - "pixel_epe_mean_mean", - "px_angle_rmse_mean", - "fl_all_mean", - "foe_dist_mean", - "flow_kl_2d_mean", - ) - - def initialize_pipeline(self, fastvideo_args: FastVideoArgs): - self.modules["scheduler"] = FlowUniPCMultistepScheduler( - shift=fastvideo_args.pipeline_config.flow_shift) - - def create_training_stages(self, training_args: TrainingArgs): - """ - May be used in future refactors. - """ - pass - - def set_schemas(self): - self.train_dataset_schema = pyarrow_schema_wangame - - def set_trainable(self) -> None: - """ - Override to only train newly added action-related parameters: - - condition_embedder.action_embedder: embeds action into timestep - - blocks.*.to_out_prope: projects PRoPE attention output - - This freezes the base model (q/k/v projections, FFN, etc.) while - allowing the action-conditioning path to be trained. - """ - train_action_only = getattr(self.fastvideo_args, "train_action_only", - False) - - if not train_action_only: - # Default behavior: train all parameters - super().set_trainable() - return - - # Freeze all transformer parameters first - transformer = self.get_module("transformer") - transformer.train() - transformer.requires_grad_(False) - - # Define which parameter name patterns to train - action_param_patterns = [ - "condition_embedder.action_embedder", # Action embedding MLP - "to_out_prope", # PRoPE output projections in each block - ] - - # Enable gradients for action-related parameters only - trainable_count = 0 - frozen_count = 0 - for name, param in transformer.named_parameters(): - should_train = any(pattern in name - for pattern in action_param_patterns) - if should_train: - param.requires_grad_(True) - trainable_count += 1 - logger.info(f"Trainable: {name} ({param.numel()} params)") - else: - frozen_count += 1 - - logger.info( - f"Action-only training: {trainable_count} trainable param groups, " - f"{frozen_count} frozen param groups") - - # ── Action module warmup ────────────────────────────────────────────── - # For the first `action_warmup_steps`, action modules (action_embedder, - # to_out_prope) have requires_grad=False so the base model stabilizes - # first. After warmup the gradients are re-enabled. - - _ACTION_PARAM_PATTERNS = [ - "condition_embedder.action_embedder", - "to_out_prope", - ] - - def _set_action_params_grad(self, requires_grad: bool) -> None: - """Toggle requires_grad for action-related parameters.""" - transformer = self.get_module("transformer") - count = 0 - for name, param in transformer.named_parameters(): - if any(p in name for p in self._ACTION_PARAM_PATTERNS): - param.requires_grad_(requires_grad) - count += 1 - state = "enabled" if requires_grad else "disabled" - logger.info("Gradients %s for %d action parameter groups", state, count) - - def train_one_step(self, training_batch: TrainingBatch) -> TrainingBatch: - step = training_batch.current_timestep - warmup_steps = self.training_args.action_warmup_steps - - if warmup_steps > 0: - if step == 1: - # Freeze action params at the very first step - self._set_action_params_grad(False) - local_trainable = count_trainable(self.transformer) - total_trainable = count_trainable_total( - self.transformer, get_local_torch_device()) - logger.info( - "Action warmup: freezing action modules for the first " - "%d steps to stabilize base model", warmup_steps) - logger.info( - "Trainable during warmup: %s B (total); this rank shard: %s B", - round(total_trainable / 1e9, 3), - round(local_trainable / 1e9, 3), - ) - elif step == warmup_steps + 1: - # Unfreeze action params once warmup is done - self._set_action_params_grad(True) - logger.info( - "Action warmup complete — action modules unfrozen at " - "step %d", step) - - return super().train_one_step(training_batch) - - def initialize_validation_pipeline(self, training_args: TrainingArgs): - logger.info("Initializing validation pipeline...") - # args_copy.pipeline_config.vae_config.load_encoder = False - # validation_pipeline = WanImageToVideoValidationPipeline.from_pretrained( - self.validation_pipeline = WanGameActionImageToVideoPipeline.from_pretrained( - training_args.model_path, - args=None, - inference_mode=True, - loaded_modules={ - "transformer": self.get_module("transformer"), - }, - tp_size=training_args.tp_size, - sp_size=training_args.sp_size, - num_gpus=training_args.num_gpus, - dit_cpu_offload=False) - - def _get_next_batch(self, training_batch: TrainingBatch) -> TrainingBatch: - batch = next(self.train_loader_iter, None) # type: ignore - if batch is None: - self.current_epoch += 1 - logger.info("Starting epoch %s", self.current_epoch) - # Reshuffle dataset order each epoch - self.train_dataset.sampler.set_epoch(self.current_epoch) - # Reset iterator for next epoch - self.train_loader_iter = iter(self.train_dataloader) - # Get first batch of new epoch - batch = next(self.train_loader_iter) - - latents = batch['vae_latent'] - latents = latents[:, :, :self.training_args.num_latent_t] - # encoder_hidden_states = batch['text_embedding'] - # encoder_attention_mask = batch['text_attention_mask'] - clip_features = batch['clip_feature'] - image_latents = batch['first_frame_latent'] - image_latents = image_latents[:, :, :self.training_args.num_latent_t] - pil_image = batch['pil_image'] - infos = batch['info_list'] - - training_batch.latents = latents.to(get_local_torch_device(), - dtype=torch.bfloat16) - training_batch.encoder_hidden_states = None - training_batch.encoder_attention_mask = None - training_batch.preprocessed_image = pil_image.to( - get_local_torch_device()) - training_batch.image_embeds = clip_features.to(get_local_torch_device()) - training_batch.image_latents = image_latents.to( - get_local_torch_device()) - training_batch.infos = infos - - # Action conditioning - if 'mouse_cond' in batch and batch['mouse_cond'].numel() > 0: - training_batch.mouse_cond = batch['mouse_cond'].to( - get_local_torch_device(), dtype=torch.bfloat16) - else: - training_batch.mouse_cond = None - - if 'keyboard_cond' in batch and batch['keyboard_cond'].numel() > 0: - training_batch.keyboard_cond = batch['keyboard_cond'].to( - get_local_torch_device(), dtype=torch.bfloat16) - else: - training_batch.keyboard_cond = None - - # Validate action temporal dimensions match video num_frames - expected_num_frames = (self.training_args.num_latent_t - 1) * 4 + 1 - if training_batch.keyboard_cond is not None: - assert training_batch.keyboard_cond.shape[1] == expected_num_frames, ( - f"keyboard_cond temporal dim {training_batch.keyboard_cond.shape[1]} " - f"!= expected {expected_num_frames} " - f"(num_latent_t={self.training_args.num_latent_t})") - if training_batch.mouse_cond is not None: - assert training_batch.mouse_cond.shape[1] == expected_num_frames, ( - f"mouse_cond temporal dim {training_batch.mouse_cond.shape[1]} " - f"!= expected {expected_num_frames} " - f"(num_latent_t={self.training_args.num_latent_t})") - - return training_batch - - def _prepare_dit_inputs(self, - training_batch: TrainingBatch) -> TrainingBatch: - """Override to properly handle I2V concatenation - call parent first, then concatenate image conditioning.""" - - # First, call parent method to prepare noise, timesteps, etc. for video latents - training_batch = super()._prepare_dit_inputs(training_batch) - - assert isinstance(training_batch.image_latents, torch.Tensor) - image_latents = training_batch.image_latents.to( - get_local_torch_device(), dtype=torch.bfloat16) - - temporal_compression_ratio = self.training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (self.training_args.num_latent_t - - 1) * temporal_compression_ratio + 1 - batch_size, num_channels, _, latent_height, latent_width = image_latents.shape - mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, - latent_width) - mask_lat_size[:, :, 1:] = 0 - - first_frame_mask = mask_lat_size[:, :, :1] - first_frame_mask = torch.repeat_interleave( - first_frame_mask, dim=2, repeats=temporal_compression_ratio) - mask_lat_size = torch.cat([first_frame_mask, mask_lat_size[:, :, 1:]], - dim=2) - mask_lat_size = mask_lat_size.view(batch_size, -1, - temporal_compression_ratio, - latent_height, latent_width) - mask_lat_size = mask_lat_size.transpose(1, 2) - mask_lat_size = mask_lat_size.to( - image_latents.device).to(dtype=torch.bfloat16) - - training_batch.noisy_model_input = torch.cat( - [training_batch.noisy_model_input, mask_lat_size, image_latents], - dim=1) - - return training_batch - - def _build_input_kwargs(self, - training_batch: TrainingBatch) -> TrainingBatch: - - # Image Embeds for conditioning - image_embeds = training_batch.image_embeds - assert torch.isnan(image_embeds).sum() == 0 - image_embeds = image_embeds.to(get_local_torch_device(), - dtype=torch.bfloat16) - encoder_hidden_states_image = image_embeds - - from fastvideo.models.dits.hyworld.pose import process_custom_actions - - # Process actions for each batch sample - batch_size = training_batch.noisy_model_input.shape[0] - viewmats_list, intrinsics_list, action_labels_list = [], [], [] - for b in range(batch_size): - v, i, a = process_custom_actions(training_batch.keyboard_cond[b], - training_batch.mouse_cond[b]) - viewmats_list.append(v) - intrinsics_list.append(i) - action_labels_list.append(a) - viewmats = torch.stack(viewmats_list, - dim=0).to(get_local_torch_device(), - dtype=torch.bfloat16) - intrinsics = torch.stack(intrinsics_list, - dim=0).to(get_local_torch_device(), - dtype=torch.bfloat16) - action_labels = torch.stack(action_labels_list, - dim=0).to(get_local_torch_device(), - dtype=torch.bfloat16) - - # Validate processed action latent dim matches video latent dim - num_latent_t = training_batch.noisy_model_input.shape[2] - assert action_labels.shape[1] == num_latent_t, ( - f"action_labels temporal dim {action_labels.shape[1]} != " - f"video latent temporal dim {num_latent_t}") - assert viewmats.shape[1] == num_latent_t, ( - f"viewmats temporal dim {viewmats.shape[1]} != " - f"video latent temporal dim {num_latent_t}") - - # NOTE: noisy_model_input already contains concatenated image_latents from _prepare_dit_inputs - training_batch.input_kwargs = { - "hidden_states": - training_batch.noisy_model_input, - "encoder_hidden_states": - training_batch.encoder_hidden_states, # None (no text conditioning) - "timestep": - training_batch.timesteps.to(get_local_torch_device(), - dtype=torch.bfloat16), - # "encoder_attention_mask": - # training_batch.encoder_attention_mask, - "encoder_hidden_states_image": - encoder_hidden_states_image, - # Action conditioning - "viewmats": - viewmats, - "Ks": - intrinsics, - "action": - action_labels, - "return_dict": - False, - } - return training_batch - - def _prepare_validation_batch(self, sampling_param: SamplingParam, - training_args: TrainingArgs, - validation_batch: dict[str, Any], - num_inference_steps: int) -> ForwardBatch: - sampling_param.prompt = validation_batch['prompt'] - sampling_param.height = training_args.num_height - sampling_param.width = training_args.num_width - sampling_param.image_path = validation_batch.get( - 'image_path') or validation_batch.get('video_path') - sampling_param.num_inference_steps = num_inference_steps - sampling_param.data_type = "video" - assert self.seed is not None - sampling_param.seed = self.seed - - latents_size = [(sampling_param.num_frames - 1) // 4 + 1, - sampling_param.height // 8, sampling_param.width // 8] - n_tokens = latents_size[0] * latents_size[1] * latents_size[2] - temporal_compression_factor = training_args.pipeline_config.vae_config.arch_config.temporal_compression_ratio - num_frames = (training_args.num_latent_t - - 1) * temporal_compression_factor + 1 - sampling_param.num_frames = num_frames - batch = ForwardBatch( - **shallow_asdict(sampling_param), - latents=None, - generator=torch.Generator(device="cpu").manual_seed(self.seed), - n_tokens=n_tokens, - eta=0.0, - VSA_sparsity=training_args.VSA_sparsity, - ) - if "image" in validation_batch and validation_batch["image"] is not None: - batch.pil_image = validation_batch["image"] - - if "keyboard_cond" in validation_batch and validation_batch[ - "keyboard_cond"] is not None: - keyboard_cond = validation_batch["keyboard_cond"] - keyboard_cond = torch.tensor(keyboard_cond, dtype=torch.bfloat16) - keyboard_cond = keyboard_cond.unsqueeze(0) - batch.keyboard_cond = keyboard_cond - - if "mouse_cond" in validation_batch and validation_batch[ - "mouse_cond"] is not None: - mouse_cond = validation_batch["mouse_cond"] - mouse_cond = torch.tensor(mouse_cond, dtype=torch.bfloat16) - mouse_cond = mouse_cond.unsqueeze(0) - batch.mouse_cond = mouse_cond - - return batch - - def _post_process_validation_frames( - self, frames: list[np.ndarray], - batch: ForwardBatch) -> list[np.ndarray]: - """Apply action overlay to validation frames for WanGame. - - Draws keyboard (WASD) and mouse (pitch/yaw) indicators on the video frames. - """ - # Check if action data is available - keyboard_cond = getattr(batch, 'keyboard_cond', None) - mouse_cond = getattr(batch, 'mouse_cond', None) - - if keyboard_cond is None and mouse_cond is None: - return frames - - # Import overlay functions - from fastvideo.models.dits.matrixgame.utils import (draw_keys_on_frame, - draw_mouse_on_frame) - - # Convert tensors to numpy if needed (bfloat16 -> float32 -> numpy) - if keyboard_cond is not None: - keyboard_cond = keyboard_cond.squeeze( - 0).cpu().float().numpy() # (T, 6) - if mouse_cond is not None: - mouse_cond = mouse_cond.squeeze(0).cpu().float().numpy() # (T, 2) - - # MatrixGame convention: keyboard [W, S, A, D, left, right], mouse [Pitch, Yaw] - key_names = ["W", "S", "A", "D", "left", "right"] - - processed_frames = [] - for frame_idx, frame in enumerate(frames): - frame = np.ascontiguousarray(frame.copy()) - - # Draw keyboard overlay - if keyboard_cond is not None and frame_idx < len(keyboard_cond): - keys = { - key_names[i]: bool(keyboard_cond[frame_idx, i]) - for i in range(min(len(key_names), keyboard_cond.shape[1])) - } - draw_keys_on_frame(frame, keys, mode='universal') - - # Draw mouse overlay - if mouse_cond is not None and frame_idx < len(mouse_cond): - pitch = float(mouse_cond[frame_idx, 0]) - yaw = float(mouse_cond[frame_idx, 1]) - draw_mouse_on_frame(frame, pitch, yaw) - - processed_frames.append(frame) - - return processed_frames - - def _init_flow_eval_module(self) -> None: - if getattr(self, "_flow_eval_init_done", False): - return - self._flow_eval_init_done = True - self._flow_eval_ready = False - - ptlflow_dir = Path("/mnt/weka/home/hao.zhang/mhuo/FastVideo/benchmarks/ptlflow") - - try: - ptlflow_dir_str = str(ptlflow_dir.resolve()) - if ptlflow_dir_str not in sys.path: - sys.path.insert(0, ptlflow_dir_str) - - from eval_flow_divergence import evaluate_pair_synthetic # type: ignore - - self._flow_eval_fn = evaluate_pair_synthetic - self._flow_eval_ckpt = str(ptlflow_dir / "dpflow-things-2012b5d6.ckpt") - self._flow_eval_calibration_path = str(ptlflow_dir / - "calibration.json") - self._flow_eval_ready = True - logger.info("Initialized flow divergence evaluator: %s", - ptlflow_dir) - except Exception as e: - logger.warning("Failed to initialize flow divergence evaluator: %s", - e) - - def _evaluate_validation_video( - self, - video_path: str, - caption: str, - action_path: str | None, - global_step: int, - num_inference_steps: int, - ) -> dict[str, float]: - del caption - self._init_flow_eval_module() - if not getattr(self, "_flow_eval_ready", False): - raise RuntimeError( - "ptlflow evaluator is not initialized; cannot compute flow metrics." - ) - - if not isinstance(action_path, str) or not os.path.isfile(action_path): - raise FileNotFoundError( - f"Validation sample is missing a valid action_path: {action_path}" - ) - - eval_output_dir = os.path.join( - self.training_args.output_dir, - "flow_eval", - f"step_{global_step}", - f"inference_steps_{num_inference_steps}", - Path(video_path).stem, - ) - - try: - summary = self._flow_eval_fn( - gen_video=video_path, - action_path=action_path, - calibration_path=self._flow_eval_calibration_path, - output_dir=eval_output_dir, - model_name="dpflow", - ckpt=self._flow_eval_ckpt, - no_viz=True, - use_depth=True, - ) - except Exception as e: - raise RuntimeError( - f"ptlflow synthetic evaluation failed for {video_path}") from e - - if not isinstance(summary, dict): - raise RuntimeError( - f"ptlflow returned invalid summary type: {type(summary)}" - ) - - metrics: dict[str, float] = {} - missing_or_invalid: list[str] = [] - for key in self._FLOW_EVAL_SCALAR_KEYS: - val = summary.get(key) - if not isinstance(val, (float, int, np.floating, np.integer)): - missing_or_invalid.append(key) - continue - val_float = float(val) - if not np.isfinite(val_float): - missing_or_invalid.append(key) - continue - metrics[key] = val_float - - if missing_or_invalid: - raise RuntimeError( - "ptlflow summary missing/invalid metrics: " - f"{', '.join(missing_or_invalid)}") - - return metrics - - -def main(args) -> None: - logger.info("Starting training pipeline...") - - pipeline = WanGameTrainingPipeline.from_pretrained( - args.pretrained_model_name_or_path, args=args) - args = pipeline.training_args - pipeline.train() - logger.info("Training pipeline done") - - -if __name__ == "__main__": - argv = sys.argv - from fastvideo.fastvideo_args import TrainingArgs - from fastvideo.utils import FlexibleArgumentParser - parser = FlexibleArgumentParser() - parser = TrainingArgs.add_cli_args(parser) - parser = FastVideoArgs.add_cli_args(parser) - args = parser.parse_args() - args.dit_cpu_offload = False - main(args) From a72bd9cd153e559a4137621d66a88848b5be70ad Mon Sep 17 00:00:00 2001 From: mignonjia Date: Sat, 7 Mar 2026 00:11:37 +0000 Subject: [PATCH 213/214] revise config --- ...2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml | 2 +- examples/train/rfc.md | 7 +- fastvideo/configs/models/dits/__init__.py | 4 +- fastvideo/configs/pipelines/__init__.py | 9 +- fastvideo/configs/pipelines/wan.py | 28 ------ fastvideo/dataset/dataloader/record_schema.py | 93 ------------------- fastvideo/dataset/dataloader/schema.py | 82 ---------------- fastvideo/models/loader/component_loader.py | 9 -- fastvideo/models/registry.py | 6 -- .../pipelines/preprocess/v1_preprocess.py | 13 +-- fastvideo/pipelines/stages/denoising.py | 2 +- .../pipelines/stages/matrixgame_denoising.py | 2 +- fastvideo/registry.py | 11 +-- fastvideo/train/callbacks/validation.py | 7 +- fastvideo/train/utils/dataloader.py | 23 ----- fastvideo/train/utils/instantiate.py | 2 +- 16 files changed, 16 insertions(+), 284 deletions(-) diff --git a/examples/train/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml b/examples/train/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml index 888ad2491..f4ac89599 100644 --- a/examples/train/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml +++ b/examples/train/finetune_wan2.1_t2v_1.3B_vsa_phase3.4_0.9sparsity.yaml @@ -51,7 +51,7 @@ training: checkpoints_total_limit: 3 tracker: - project_name: distillation_wangame_r + project_name: distillation_wan_r run_name: phase3.4_wan_finetune_vsa_0.9_v3 model: diff --git a/examples/train/rfc.md b/examples/train/rfc.md index 8ac81430d..35ed587c8 100644 --- a/examples/train/rfc.md +++ b/examples/train/rfc.md @@ -8,9 +8,6 @@ fastvideo/train/ base.py # BaseModel ABC: predict_x0, add_noise, backward, ... wan/ wan.py # Wan model loader - wangame/ - wangame.py # WanGame model loader - wangame_causal.py methods/ base.py # DistillMethod base; methods provide train_one_step distribution_matching/ @@ -46,9 +43,9 @@ By this design, we only need a YAML config to train different models using diffe Models declare `_target_` to select the model class; methods declare `_target_` to select the method class. Current code: https://github.com/FoundationResearch/FastVideo/tree/distill1/fastvideo/train -DMD2 Distillation, Self-Forcing, SFT, and DFSFT are tested on Wan / WanGame. +DMD2 Distillation, Self-Forcing, SFT, and DFSFT are tested on Wan. -Current supported models: Wan, WanGame. +Current supported models: Wan. Current supported methods: DMD2, Self-Forcing, SFT, DFSFT. Feedbacks are highly welcome! diff --git a/fastvideo/configs/models/dits/__init__.py b/fastvideo/configs/models/dits/__init__.py index 8caddca7e..d30d4e96d 100644 --- a/fastvideo/configs/models/dits/__init__.py +++ b/fastvideo/configs/models/dits/__init__.py @@ -3,17 +3,15 @@ from fastvideo.configs.models.dits.hunyuangamecraft import HunyuanGameCraftConfig from fastvideo.configs.models.dits.hunyuanvideo import HunyuanVideoConfig from fastvideo.configs.models.dits.hunyuanvideo15 import HunyuanVideo15Config +from fastvideo.configs.models.dits.lingbotworld import LingBotWorldVideoConfig from fastvideo.configs.models.dits.longcat import LongCatVideoConfig from fastvideo.configs.models.dits.ltx2 import LTX2VideoConfig from fastvideo.configs.models.dits.wanvideo import WanVideoConfig from fastvideo.configs.models.dits.hyworld import HYWorldConfig -from fastvideo.configs.models.dits.wangamevideo import (WanGameVideoConfig, - WanLingBotVideoConfig) __all__ = [ "HunyuanVideoConfig", "HunyuanVideo15Config", "WanVideoConfig", "StepVideoConfig", "CosmosVideoConfig", "Cosmos25VideoConfig", "LongCatVideoConfig", "LTX2VideoConfig", "HYWorldConfig", - "LingBotWorldVideoConfig", "WanGameVideoConfig", "WanLingBotVideoConfig", "LingBotWorldVideoConfig", "HunyuanGameCraftConfig", "WanVideoConfig" ] diff --git a/fastvideo/configs/pipelines/__init__.py b/fastvideo/configs/pipelines/__init__.py index b87c23102..4d7838638 100644 --- a/fastvideo/configs/pipelines/__init__.py +++ b/fastvideo/configs/pipelines/__init__.py @@ -5,14 +5,12 @@ from fastvideo.configs.pipelines.hunyuan15 import Hunyuan15T2V480PConfig, Hunyuan15T2V720PConfig from fastvideo.configs.pipelines.hunyuangamecraft import HunyuanGameCraftPipelineConfig from fastvideo.configs.pipelines.hyworld import HYWorldConfig +from fastvideo.configs.pipelines.lingbotworld import LingBotWorldI2V480PConfig from fastvideo.configs.pipelines.ltx2 import LTX2T2VConfig from fastvideo.registry import get_pipeline_config_cls_from_name from fastvideo.configs.pipelines.wan import (SelfForcingWanT2V480PConfig, - WanGameI2V480PConfig, WanI2V480PConfig, WanI2V720PConfig, - WanLingBotI2V480PConfig, - WanT2V480PConfig, WanT2V720PConfig, - SelfForcingWanGameI2V480PConfig) + WanT2V480PConfig, WanT2V720PConfig) __all__ = [ "HunyuanConfig", "FastHunyuanConfig", "HunyuanGameCraftPipelineConfig", @@ -20,7 +18,6 @@ "WanT2V480PConfig", "WanI2V480PConfig", "WanT2V720PConfig", "WanI2V720PConfig", "StepVideoT2VConfig", "SelfForcingWanT2V480PConfig", "CosmosConfig", "Cosmos25Config", "LTX2T2VConfig", "HYWorldConfig", - "SD35Config", "LingBotWorldI2V480PConfig", "WanGameI2V480PConfig", - "WanLingBotI2V480PConfig", "SelfForcingWanGameI2V480PConfig", + "SD35Config", "LingBotWorldI2V480PConfig", "get_pipeline_config_cls_from_name" ] diff --git a/fastvideo/configs/pipelines/wan.py b/fastvideo/configs/pipelines/wan.py index 996a734aa..434839dcc 100644 --- a/fastvideo/configs/pipelines/wan.py +++ b/fastvideo/configs/pipelines/wan.py @@ -7,8 +7,6 @@ from fastvideo.configs.models import DiTConfig, EncoderConfig, VAEConfig from fastvideo.configs.models.dits import WanVideoConfig from fastvideo.configs.models.dits.matrixgame import MatrixGameWanVideoConfig -from fastvideo.configs.models.dits.wangamevideo import (WanGameVideoConfig, - WanLingBotVideoConfig) from fastvideo.configs.models.encoders import (BaseEncoderOutput, CLIPVisionConfig, T5Config, WAN2_1ControlCLIPVisionConfig) @@ -114,23 +112,6 @@ class WANV2VConfig(WanI2V480PConfig): image_encoder_precision: str = 'bf16' -@dataclass -class WanLingBotI2V480PConfig(WanI2V480PConfig): - """Configuration for Wan LingBot image-to-video pipeline.""" - - dit_config: DiTConfig = field(default_factory=WanLingBotVideoConfig) - - -@dataclass -class WanGameI2V480PConfig(WanI2V480PConfig): - """Configuration for WanGame image-to-video pipeline.""" - - dit_config: DiTConfig = field(default_factory=WanGameVideoConfig) - flow_shift: float | None = 3.0 - dmd_denoising_steps: list[int] | None = field( - default_factory=lambda: [1000, 750, 500, 250, 0]) - - @dataclass class FastWan2_1_T2V_480P_Config(WanT2V480PConfig): """Base configuration for FastWan T2V 1.3B 480P pipeline architecture with DMD""" @@ -212,15 +193,6 @@ def __post_init__(self) -> None: self.vae_config.load_decoder = True -@dataclass -class SelfForcingWanGameI2V480PConfig(WanGameI2V480PConfig): - is_causal: bool = True - flow_shift: float | None = 3.0 - dmd_denoising_steps: list[int] | None = field( - default_factory=lambda: [1000, 750, 500, 250, 0]) - warp_denoising_step: bool = True - - # ============================================= # ============= Matrix Game =================== # ============================================= diff --git a/fastvideo/dataset/dataloader/record_schema.py b/fastvideo/dataset/dataloader/record_schema.py index 0eea39adf..1bc86dd7d 100644 --- a/fastvideo/dataset/dataloader/record_schema.py +++ b/fastvideo/dataset/dataloader/record_schema.py @@ -188,96 +188,3 @@ def text_only_record_creator(text_name: str, text_embedding: np.ndarray, "caption": caption, } return record - - -def wangame_ode_record_creator( - video_name: str, - clip_feature: np.ndarray, - first_frame_latent: np.ndarray, - trajectory_latents: np.ndarray, - trajectory_timesteps: np.ndarray, - pil_image: np.ndarray | None = None, - keyboard_cond: np.ndarray | None = None, - mouse_cond: np.ndarray | None = None, - caption: str = "") -> dict[str, Any]: - """Create a ODE trajectory record matching pyarrow_schema_wangame - """ - assert trajectory_latents is not None, "trajectory_latents is required" - assert trajectory_timesteps is not None, "trajectory_timesteps is required" - assert clip_feature is not None, "clip_feature is required" - assert first_frame_latent is not None, "first_frame_latent is required" - - record = { - "id": video_name, - "file_name": video_name, - "caption": caption, - "media_type": "video", - } - - # I2V features - record.update({ - "clip_feature_bytes": clip_feature.tobytes(), - "clip_feature_shape": list(clip_feature.shape), - "clip_feature_dtype": str(clip_feature.dtype), - }) - - record.update({ - "first_frame_latent_bytes": first_frame_latent.tobytes(), - "first_frame_latent_shape": list(first_frame_latent.shape), - "first_frame_latent_dtype": str(first_frame_latent.dtype), - }) - - # Optional PIL Image - if pil_image is not None: - record.update({ - "pil_image_bytes": pil_image.tobytes(), - "pil_image_shape": list(pil_image.shape), - "pil_image_dtype": str(pil_image.dtype), - }) - else: - record.update({ - "pil_image_bytes": b"", - "pil_image_shape": [], - "pil_image_dtype": "", - }) - - # Actions - if keyboard_cond is not None: - record.update({ - "keyboard_cond_bytes": keyboard_cond.tobytes(), - "keyboard_cond_shape": list(keyboard_cond.shape), - "keyboard_cond_dtype": str(keyboard_cond.dtype), - }) - else: - record.update({ - "keyboard_cond_bytes": b"", - "keyboard_cond_shape": [], - "keyboard_cond_dtype": "", - }) - - if mouse_cond is not None: - record.update({ - "mouse_cond_bytes": mouse_cond.tobytes(), - "mouse_cond_shape": list(mouse_cond.shape), - "mouse_cond_dtype": str(mouse_cond.dtype), - }) - else: - record.update({ - "mouse_cond_bytes": b"", - "mouse_cond_shape": [], - "mouse_cond_dtype": "", - }) - - record.update({ - "trajectory_latents_bytes": trajectory_latents.tobytes(), - "trajectory_latents_shape": list(trajectory_latents.shape), - "trajectory_latents_dtype": str(trajectory_latents.dtype), - }) - - record.update({ - "trajectory_timesteps_bytes": trajectory_timesteps.tobytes(), - "trajectory_timesteps_shape": list(trajectory_timesteps.shape), - "trajectory_timesteps_dtype": str(trajectory_timesteps.dtype), - }) - - return record diff --git a/fastvideo/dataset/dataloader/schema.py b/fastvideo/dataset/dataloader/schema.py index cdf9b42fb..048c7686c 100644 --- a/fastvideo/dataset/dataloader/schema.py +++ b/fastvideo/dataset/dataloader/schema.py @@ -157,85 +157,3 @@ pa.field("duration_sec", pa.float64()), pa.field("fps", pa.float64()), ]) - -pyarrow_schema_wangame = pa.schema([ - pa.field("id", pa.string()), - # --- Image/Video VAE latents --- - # Tensors are stored as raw bytes with shape and dtype info for loading - pa.field("vae_latent_bytes", pa.binary()), - # e.g., [C, T, H, W] or [C, H, W] - pa.field("vae_latent_shape", pa.list_(pa.int64())), - # e.g., 'float32' - pa.field("vae_latent_dtype", pa.string()), - #I2V - pa.field("clip_feature_bytes", pa.binary()), - pa.field("clip_feature_shape", pa.list_(pa.int64())), - pa.field("clip_feature_dtype", pa.string()), - pa.field("first_frame_latent_bytes", pa.binary()), - pa.field("first_frame_latent_shape", pa.list_(pa.int64())), - pa.field("first_frame_latent_dtype", pa.string()), - # --- Action --- - pa.field("mouse_cond_bytes", pa.binary()), - pa.field("mouse_cond_shape", pa.list_(pa.int64())), # [T, 2] - pa.field("mouse_cond_dtype", pa.string()), - pa.field("keyboard_cond_bytes", pa.binary()), - pa.field("keyboard_cond_shape", pa.list_(pa.int64())), # [T, 4] - pa.field("keyboard_cond_dtype", pa.string()), - # I2V Validation - pa.field("pil_image_bytes", pa.binary()), - pa.field("pil_image_shape", pa.list_(pa.int64())), - pa.field("pil_image_dtype", pa.string()), - # --- Metadata --- - pa.field("file_name", pa.string()), - pa.field("caption", pa.string()), - pa.field("media_type", pa.string()), # 'image' or 'video' - pa.field("width", pa.int64()), - pa.field("height", pa.int64()), - # -- Video-specific (can be null/default for images) --- - # Number of frames processed (e.g., 1 for image, N for video) - pa.field("num_frames", pa.int64()), - pa.field("duration_sec", pa.float64()), - pa.field("fps", pa.float64()), -]) - -pyarrow_schema_wangame_lingbot = pyarrow_schema_wangame - -pyarrow_schema_ode_trajectory_wangame = pa.schema([ - pa.field("id", pa.string()), - #I2V - pa.field("clip_feature_bytes", pa.binary()), - pa.field("clip_feature_shape", pa.list_(pa.int64())), - pa.field("clip_feature_dtype", pa.string()), - pa.field("first_frame_latent_bytes", pa.binary()), - pa.field("first_frame_latent_shape", pa.list_(pa.int64())), - pa.field("first_frame_latent_dtype", pa.string()), - # --- Action --- - pa.field("mouse_cond_bytes", pa.binary()), - pa.field("mouse_cond_shape", pa.list_(pa.int64())), # [T, 2] - pa.field("mouse_cond_dtype", pa.string()), - pa.field("keyboard_cond_bytes", pa.binary()), - pa.field("keyboard_cond_shape", pa.list_(pa.int64())), # [T, 4] - pa.field("keyboard_cond_dtype", pa.string()), - # I2V Validation - pa.field("pil_image_bytes", pa.binary()), - pa.field("pil_image_shape", pa.list_(pa.int64())), - pa.field("pil_image_dtype", pa.string()), - # --- ODE Trajectory --- - pa.field("trajectory_latents_bytes", pa.binary()), - pa.field("trajectory_latents_shape", pa.list_(pa.int64())), - pa.field("trajectory_latents_dtype", pa.string()), - pa.field("trajectory_timesteps_bytes", pa.binary()), - pa.field("trajectory_timesteps_shape", pa.list_(pa.int64())), - pa.field("trajectory_timesteps_dtype", pa.string()), - # --- Metadata --- - pa.field("file_name", pa.string()), - pa.field("caption", pa.string()), - pa.field("media_type", pa.string()), # 'image' or 'video' - pa.field("width", pa.int64()), - pa.field("height", pa.int64()), - # -- Video-specific (can be null/default for images) --- - # Number of frames processed (e.g., 1 for image, N for video) - pa.field("num_frames", pa.int64()), - pa.field("duration_sec", pa.float64()), - pa.field("fps", pa.float64()), -]) diff --git a/fastvideo/models/loader/component_loader.py b/fastvideo/models/loader/component_loader.py index 35dccb014..832d0d0d3 100644 --- a/fastvideo/models/loader/component_loader.py +++ b/fastvideo/models/loader/component_loader.py @@ -843,15 +843,6 @@ def load(self, model_path: str, fastvideo_args: FastVideoArgs): cls_name.startswith("Cosmos25") or cls_name == "Cosmos25Transformer3DModel" or getattr(fastvideo_args.pipeline_config, "prefix", "") == "Cosmos25" - ) and not ( - cls_name.startswith("WanGame") - or cls_name == "WanGameActionTransformer3DModel" - or cls_name.startswith("CausalWan") - or getattr(fastvideo_args.pipeline_config, "prefix", "") == "WanGame" - or cls_name.startswith("WanLingBot") - or cls_name == "WanLingBotTransformer3DModel" - or getattr(fastvideo_args.pipeline_config, "prefix", "") == "WanLingBot" - or cls_name.startswith("CausalWanGameActionTransformer3DModel") ) model = maybe_load_fsdp_model( model_cls=model_cls, diff --git a/fastvideo/models/registry.py b/fastvideo/models/registry.py index d9f2e8691..a22a582e0 100644 --- a/fastvideo/models/registry.py +++ b/fastvideo/models/registry.py @@ -46,12 +46,6 @@ # "HunyuanVideoTransformer3DModel": ("dits", "hunyuanvideo", "HunyuanVideoDiT"), "WanTransformer3DModel": ("dits", "wanvideo", "WanTransformer3DModel"), "CausalWanTransformer3DModel": ("dits", "causal_wanvideo", "CausalWanTransformer3DModel"), - "CausalWanGameTransformer3DModel": - ("dits", "wangame", "CausalWanGameActionTransformer3DModel"), - "CausalWanGameActionTransformer3DModel": - ("dits", "wangame", "CausalWanGameActionTransformer3DModel"), - "WanGameActionTransformer3DModel": ("dits", "wangame", "WanGameActionTransformer3DModel"), - "WanLingBotTransformer3DModel": ("dits", "wangame_lingbot", "WanLingBotTransformer3DModel"), "MatrixGameWanModel": ("dits", "matrixgame", "MatrixGameWanModel"), "CausalMatrixGameWanModel": ("dits", "matrixgame", "CausalMatrixGameWanModel"), } diff --git a/fastvideo/pipelines/preprocess/v1_preprocess.py b/fastvideo/pipelines/preprocess/v1_preprocess.py index e15e2239c..6e734877a 100644 --- a/fastvideo/pipelines/preprocess/v1_preprocess.py +++ b/fastvideo/pipelines/preprocess/v1_preprocess.py @@ -18,10 +18,6 @@ PreprocessPipeline_Text) from fastvideo.pipelines.preprocess.matrixgame.matrixgame_preprocess_pipeline import ( PreprocessPipeline_MatrixGame) -from fastvideo.pipelines.preprocess.wangame.wangame_preprocess_pipeline import ( - PreprocessPipeline_WanGame) -from fastvideo.pipelines.preprocess.wangame.wangame_preprocess_pipeline_ode_trajectory import ( - PreprocessPipeline_WanGame_ODE_Trajectory) from fastvideo.utils import maybe_download_model logger = init_logger(__name__) @@ -68,15 +64,10 @@ def main(args) -> None: PreprocessPipeline = PreprocessPipeline_ODE_Trajectory elif args.preprocess_task == "matrixgame": PreprocessPipeline = PreprocessPipeline_MatrixGame - elif args.preprocess_task == "wangame": - PreprocessPipeline = PreprocessPipeline_WanGame - elif args.preprocess_task == "wangame_ode_trajectory": - fastvideo_args.pipeline_config.flow_shift = args.flow_shift if args.flow_shift is not None else 5.0 - PreprocessPipeline = PreprocessPipeline_WanGame_ODE_Trajectory else: raise ValueError( f"Invalid preprocess task: {args.preprocess_task}. " - f"Valid options: t2v, i2v, ode_trajectory, text_only, matrixgame, wangame, wangame_ode_trajectory" + f"Valid options: t2v, i2v, ode_trajectory, text_only, matrixgame" ) logger.info("Preprocess task: %s using %s", args.preprocess_task, @@ -126,7 +117,7 @@ def main(args) -> None: default="t2v", choices=[ "t2v", "i2v", "text_only", "ode_trajectory", - "matrixgame", "wangame", "wangame_ode_trajectory" + "matrixgame" ], help="Type of preprocessing task to run") parser.add_argument("--train_fps", type=int, default=30) diff --git a/fastvideo/pipelines/stages/denoising.py b/fastvideo/pipelines/stages/denoising.py index 38cf77f14..49c40a7d5 100644 --- a/fastvideo/pipelines/stages/denoising.py +++ b/fastvideo/pipelines/stages/denoising.py @@ -175,7 +175,7 @@ def forward( dtype=target_dtype), }, ) - # from fastvideo.models.dits.wangame_lingbot.cam_utils import process_custom_actions as process_lingbot_actions + # Legacy action-conditioning helper removed. # num_frames = batch.num_frames # latent_height = batch.height // 8 # latent_width = batch.width // 8 diff --git a/fastvideo/pipelines/stages/matrixgame_denoising.py b/fastvideo/pipelines/stages/matrixgame_denoising.py index 85f42f6b9..5ae7bf7b7 100644 --- a/fastvideo/pipelines/stages/matrixgame_denoising.py +++ b/fastvideo/pipelines/stages/matrixgame_denoising.py @@ -725,7 +725,7 @@ def _update_context_cache( class MatrixGameCausalOdeDenoisingStage(MatrixGameCausalDenoisingStage): - """Causal ODE denoising for WanGame/MatrixGame. + """Causal ODE denoising for MatrixGame. This is the deterministic counterpart of `MatrixGameCausalDenoisingStage`. It performs block-by-block causal rollout, but uses the scheduler's ODE-style diff --git a/fastvideo/registry.py b/fastvideo/registry.py index 2d8c1f29f..99a3aad7f 100644 --- a/fastvideo/registry.py +++ b/fastvideo/registry.py @@ -36,7 +36,7 @@ MatrixGameI2V480PConfig, SelfForcingWan2_2_T2V480PConfig, SelfForcingWanT2V480PConfig, WANV2VConfig, Wan2_2_I2V_A14B_Config, Wan2_2_T2V_A14B_Config, Wan2_2_TI2V_5B_Config, WanI2V480PConfig, - WanI2V720PConfig, WanT2V480PConfig, WanT2V720PConfig, WanGameI2V480PConfig) + WanI2V720PConfig, WanT2V480PConfig, WanT2V720PConfig) from fastvideo.configs.pipelines.sd35 import SD35Config from fastvideo.configs.sample.base import SamplingParam from fastvideo.configs.sample.cosmos import ( @@ -549,15 +549,6 @@ def _register_configs() -> None: "FastVideo/SFWan2.2-I2V-A14B-Preview-Diffusers", ], ) - register_configs( - sampling_param_cls=Wan2_1_Fun_1_3B_InP_SamplingParam, - pipeline_config_cls=WanGameI2V480PConfig, - hf_model_paths=[ - "weizhou03/Wan2.1-Game-Fun-1.3B-InP-Diffusers", - ], - ) - # TODO: Need to add Lingbot - # SD3.5 register_configs( sampling_param_cls=SD35SamplingParam, diff --git a/fastvideo/train/callbacks/validation.py b/fastvideo/train/callbacks/validation.py index b0b152591..29f2edc9e 100644 --- a/fastvideo/train/callbacks/validation.py +++ b/fastvideo/train/callbacks/validation.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""Validation callback (unified replacement for WanValidator -and WanGameValidator). +"""Validation callback for inference-pipeline driven validation. All configuration is read from the YAML ``callbacks.validation`` section. The pipeline class is resolved from @@ -56,7 +55,7 @@ class ValidationCallback(Callback): Works with any pipeline that follows the ``PipelineCls.from_pretrained(...)`` + ``pipeline.forward()`` - contract (Wan, WanGame parallel, WanGame causal/DMD, etc.). + contract. """ def __init__( @@ -498,7 +497,7 @@ def _prepare_validation_batch( ) batch._inference_args = inference_args # type: ignore[attr-defined] - # Conditionally set I2V / WanGame fields. + # Conditionally set I2V / action-conditioning fields. if ( "image" in validation_batch and validation_batch["image"] is not None diff --git a/fastvideo/train/utils/dataloader.py b/fastvideo/train/utils/dataloader.py index 9771db6d2..a1b22d3ba 100644 --- a/fastvideo/train/utils/dataloader.py +++ b/fastvideo/train/utils/dataloader.py @@ -31,26 +31,3 @@ def build_parquet_t2v_train_dataloader( seed=int(data_config.seed or 0), )) return dataloader - - -def build_parquet_wangame_train_dataloader( - data_config: DataConfig, - *, - parquet_schema: Any, -) -> Any: - """Build a parquet dataloader for WanGame datasets.""" - - from fastvideo.dataset import ( - build_parquet_map_style_dataloader, ) - - _dataset, dataloader = (build_parquet_map_style_dataloader( - data_config.data_path, - data_config.train_batch_size, - num_data_workers=(data_config.dataloader_num_workers), - parquet_schema=parquet_schema, - cfg_rate=float(data_config.training_cfg_rate or 0.0), - drop_last=True, - text_padding_length=512, - seed=int(data_config.seed or 0), - )) - return dataloader diff --git a/fastvideo/train/utils/instantiate.py b/fastvideo/train/utils/instantiate.py index ed43122ee..1d1e3f306 100644 --- a/fastvideo/train/utils/instantiate.py +++ b/fastvideo/train/utils/instantiate.py @@ -20,7 +20,7 @@ def resolve_target(target: str) -> type: """Import and return the class (or callable) at *target*. *target* must be a fully-qualified dotted path, e.g. - ``"fastvideo.train.models.wangame.wangame.WanGameModel"``. + ``"fastvideo.train.models.wan.WanModel"``. """ if not isinstance(target, str) or not target.strip(): raise ValueError(f"_target_ must be a non-empty dotted path string, " From 2f29a795a32042b6d7ee9aea38fe76c3844e6471 Mon Sep 17 00:00:00 2001 From: Mingjia Huo Date: Fri, 6 Mar 2026 16:34:40 -0800 Subject: [PATCH 214/214] Delete visualize_trajectory.py --- visualize_trajectory.py | 224 ---------------------------------------- 1 file changed, 224 deletions(-) delete mode 100644 visualize_trajectory.py diff --git a/visualize_trajectory.py b/visualize_trajectory.py deleted file mode 100644 index a0eafd048..000000000 --- a/visualize_trajectory.py +++ /dev/null @@ -1,224 +0,0 @@ -import argparse -import os -import numpy as np -import pyarrow.parquet as pq -import torch -from tqdm import tqdm - -from fastvideo import PipelineConfig -from fastvideo.configs.models.vaes import WanVAEConfig -from fastvideo.fastvideo_args import FastVideoArgs -from fastvideo.models.loader.component_loader import VAELoader -from fastvideo.utils import maybe_download_model, save_decoded_latents_as_video - - -def _torch_dtype_from_precision(precision: str) -> torch.dtype: - precision = precision.lower() - if precision == "fp32": - return torch.float32 - if precision == "fp16": - return torch.float16 - if precision == "bf16": - return torch.bfloat16 - raise ValueError(f"Unsupported precision: {precision}") - - -def _denormalize_latents_for_vae(vae, latents: torch.Tensor) -> torch.Tensor: - if bool(getattr(vae, "handles_latent_denorm", False)): - return latents - - cfg = getattr(vae, "config", None) - - if cfg is not None and hasattr(cfg, "latents_mean") and hasattr( - cfg, "latents_std"): - latents_mean = torch.tensor(cfg.latents_mean, - device=latents.device, - dtype=latents.dtype).view(1, -1, 1, 1, 1) - latents_std = torch.tensor(cfg.latents_std, - device=latents.device, - dtype=latents.dtype).view(1, -1, 1, 1, 1) - return latents * latents_std + latents_mean - - if hasattr(vae, "scaling_factor"): - if isinstance(vae.scaling_factor, torch.Tensor): - latents = latents / vae.scaling_factor.to(latents.device, - latents.dtype) - else: - latents = latents / vae.scaling_factor - - if hasattr(vae, "shift_factor") and vae.shift_factor is not None: - if isinstance(vae.shift_factor, torch.Tensor): - latents = latents + vae.shift_factor.to(latents.device, - latents.dtype) - else: - latents = latents + vae.shift_factor - - return latents - - -@torch.inference_mode() -def _decode_with_vae(vae, latents: torch.Tensor, *, device: torch.device, - precision: str) -> torch.Tensor: - latents = latents.to(device=device) - target_dtype = _torch_dtype_from_precision(precision) - latents = latents.to(dtype=target_dtype) - - latents = _denormalize_latents_for_vae(vae, latents) - - use_autocast = (device.type == "cuda" and target_dtype != torch.float32) - with torch.autocast(device_type=device.type, - dtype=target_dtype, - enabled=use_autocast): - decoded = vae.decode(latents) - - return (decoded / 2 + 0.5).clamp(0, 1) - - -def main(): - parser = argparse.ArgumentParser( - description="Visualize Trajectory from Parquet file") - parser.add_argument("--parquet_path", - type=str, - required=True, - help="Path to the input parquet file") - parser.add_argument("--model_path", - type=str, - required=True, - help="Path to the model directory") - parser.add_argument("--output_dir", - type=str, - default="visualizations", - help="Directory to save output videos") - parser.add_argument("--num_samples", - type=int, - default=1, - help="Number of samples to visualize") - parser.add_argument("--device", - type=str, - default="cuda" if torch.cuda.is_available() else "cpu") - parser.add_argument("--vae_precision", - type=str, - default="fp32", - choices=["fp32", "fp16", "bf16"], - help="Precision for VAE decoding") - parser.add_argument("--vae_subfolder", - type=str, - default="vae", - help="Subfolder name containing VAE weights/config") - parser.add_argument("--fps", type=int, default=25, help="Output video FPS") - parser.add_argument( - "--decode_steps", - type=str, - default="last", - help= - "Which trajectory steps to decode: 'last', 'all', or comma-separated indices (e.g. '0,10,20')", - ) - - args = parser.parse_args() - - device = torch.device(args.device) - print(f"Using device: {device}, vae_precision: {args.vae_precision}") - - os.makedirs(args.output_dir, exist_ok=True) - - # Load VAE (must load weights; creating AutoencoderKLWan(config) alone leaves random weights) - print(f"Loading model from {args.model_path}...") - model_root = maybe_download_model(args.model_path) - pipeline_config = PipelineConfig.from_pretrained(model_root) - pipeline_config.update_config_from_dict({ - "vae_precision": - args.vae_precision, - "vae_config": - WanVAEConfig(load_encoder=False, load_decoder=True), - }) - fastvideo_args = FastVideoArgs( - model_path=model_root, - num_gpus=1, - dit_cpu_offload=False, - vae_cpu_offload=False, - text_encoder_cpu_offload=True, - pipeline_config=pipeline_config, - ) - - vae_path = os.path.join(model_root, args.vae_subfolder) - vae = VAELoader().load(vae_path, fastvideo_args) - vae.to(device) - - # Read Parquet - print(f"Reading parquet file: {args.parquet_path}") - table = pq.read_table(args.parquet_path) - - # Iterate over rows - num_visualized = 0 - - pbar = tqdm(total=min(args.num_samples, table.num_rows)) - - for i in range(table.num_rows): - if num_visualized >= args.num_samples: - break - - row = table.slice(i, length=1) - record = row.to_pydict() - - video_id = record["id"][0] - - # Parse Latents - shape = record["trajectory_latents_shape"][0] - dtype = record["trajectory_latents_dtype"][0] - dtype = np.dtype(dtype) - - latents_bytes = record["trajectory_latents_bytes"][0] - # Copy to avoid read-only warning - latents_np = np.copy( - np.frombuffer(latents_bytes, dtype=dtype).reshape(shape)) - - latents_tensor = torch.from_numpy(latents_np) - if latents_tensor.ndim == 6 and latents_tensor.shape[0] == 1: - latents_tensor = latents_tensor.squeeze(0) - - print(f"Decoding video {video_id} with shape {latents_tensor.shape}...") - - # create subfolder - vid_output_dir = os.path.join(args.output_dir, str(video_id)) - os.makedirs(vid_output_dir, exist_ok=True) - - # Pick steps to decode - steps = latents_tensor.shape[0] - if args.decode_steps == "last": - indices_to_decode = [steps - 1] - elif args.decode_steps == "all": - indices_to_decode = list(range(steps)) - else: - indices_to_decode = [ - int(x) for x in args.decode_steps.split(",") if x.strip() != "" - ] - indices_to_decode = [i for i in indices_to_decode if 0 <= i < steps] - if not indices_to_decode: - raise ValueError( - f"No valid indices selected for decode_steps='{args.decode_steps}' with steps={steps}" - ) - - for step in tqdm(indices_to_decode, - desc=f"Decoding {video_id}", - leave=False): - latent_step = latents_tensor[step].unsqueeze(0) # [1, C, T, H, W] - - decoded_video = _decode_with_vae(vae, - latent_step, - device=device, - precision=args.vae_precision) - - save_path = os.path.join(vid_output_dir, f"step_{step:03d}.mp4") - save_decoded_latents_as_video(decoded_video.float(), - save_path, - fps=args.fps) - - print(f"Saved {len(indices_to_decode)} step(s) to {vid_output_dir}") - num_visualized += 1 - pbar.update(1) - - pbar.close() - - -if __name__ == "__main__": - main()