fix: transcode non-AAC audio to AAC for MP4 container

lightx2v/models/runners/cosmos3/cosmos3_runner.py

@@ -26,6 +26,7 @@
 from lightx2v.models.runners.default_runner import DefaultRunner
 from lightx2v.models.schedulers.cosmos3.scheduler import Cosmos3Scheduler
 from lightx2v.models.video_encoders.hf.cosmos3.vae import Cosmos3WanVAE
+from lightx2v.utils.audio_mux import MP4_AAC_BITRATE
 from lightx2v.utils.envs import *
 from lightx2v.utils.profiler import *
 from lightx2v.utils.registry_factory import RUNNER_REGISTER
@@ -795,7 +796,7 @@ def _mux_generated_audio(self, video_path, audio):
                 "-c:a",
                 "aac",
                 "-b:a",
-                "192k",
+                MP4_AAC_BITRATE,
                 "-shortest",
                 "-f",
                 "mp4",

lightx2v/models/runners/wan/wan_infinitetalk_runner.py

@@ -20,6 +20,7 @@
 from lightx2v.models.schedulers.wan.infinitetalk.scheduler import InfiniteTalkScheduler
 from lightx2v.server.metrics import monitor_cli
 from lightx2v.utils.audio_io import load_audio_file
+from lightx2v.utils.audio_mux import MP4_AAC_BITRATE, mp4_audio_codec_args
 from lightx2v.utils.envs import GET_DTYPE, GET_RECORDER_MODE
 from lightx2v.utils.input_info import UNSET
 from lightx2v.utils.profiler import ProfilingContext4DebugL1, ProfilingContext4DebugL2
@@ -1197,25 +1198,22 @@ def _mux_audio(video_path, audio_path, timeout=600):
             "1:a:0",
             "-shortest",
         ]
-        cmd = [
-            *base_cmd,
-            "-c:a",
-            "copy",
-            tmp_path,
-        ]
+        audio_args = mp4_audio_codec_args(audio_path, ffmpeg.get_ffmpeg_exe())
+        cmd = [*base_cmd, *audio_args, tmp_path]
         try:
             res = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=timeout)
-            if res.returncode != 0:
-                # Fallback to aac re-encoding (e.g. for WAV/PCM inputs)
+            if res.returncode != 0 and audio_args == ["-c:a", "copy"]:
                 cmd = [
                     *base_cmd,
                     "-c:a",
                     "aac",
                     "-b:a",
-                    "192k",
+                    MP4_AAC_BITRATE,
                     tmp_path,
                 ]
                 subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=timeout)
+            elif res.returncode != 0:
+                res.check_returncode()
             os.replace(tmp_path, video_path)
             logger.info(f"Muxed audio from {audio_path}")
         except subprocess.TimeoutExpired as exc:

lightx2v/models/runners/wan/wan_s2v_runner.py

@@ -17,6 +17,7 @@
 from lightx2v.models.runners.wan.wan_runner import WanRunner
 from lightx2v.models.schedulers.wan.s2v.s2v_scheduler import WanS2VScheduler
 from lightx2v.server.metrics import monitor_cli
+from lightx2v.utils.audio_mux import MP4_AAC_BITRATE, mp4_audio_codec_args
 from lightx2v.utils.envs import GET_DTYPE
 from lightx2v.utils.profiler import *
 from lightx2v.utils.registry_factory import RUNNER_REGISTER
@@ -31,6 +32,7 @@
 
 def merge_video_audio(video_path: str, audio_path: str):
     tmp_path = video_path + ".tmp.mp4"
+    audio_args = mp4_audio_codec_args(audio_path, "ffmpeg")
     cmd = [
         "ffmpeg",
         "-y",
@@ -40,16 +42,31 @@ def merge_video_audio(video_path: str, audio_path: str):
         audio_path,
         "-c:v",
         "copy",
-        "-c:a",
-        "copy",
+        *audio_args,
         "-shortest",
         tmp_path,
     ]
     res = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    if res.returncode != 0:
-        # Fallback to aac re-encoding if stream copy fails (e.g., for WAV inputs)
-        cmd[9] = "aac"
+    if res.returncode != 0 and audio_args == ["-c:a", "copy"]:
+        cmd = [
+            "ffmpeg",
+            "-y",
+            "-i",
+            video_path,
+            "-i",
+            audio_path,
+            "-c:v",
+            "copy",
+            "-c:a",
+            "aac",
+            "-b:a",
+            MP4_AAC_BITRATE,
+            "-shortest",
+            tmp_path,
+        ]
         subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    elif res.returncode != 0:
+        res.check_returncode()
     os.replace(tmp_path, video_path)
 
 

lightx2v/utils/audio_mux.py

@@ -0,0 +1,55 @@
+import logging
+import os
+import shutil
+import subprocess
+from typing import Optional
+
+MP4_AAC_BITRATE = "256k"
+logger = logging.getLogger(__name__)
+
+
+def _find_ffprobe(ffmpeg_exe: Optional[str] = None) -> Optional[str]:
+    if ffmpeg_exe:
+        sibling = os.path.join(os.path.dirname(os.path.abspath(ffmpeg_exe)), "ffprobe")
+        if os.path.isfile(sibling) and os.access(sibling, os.X_OK):
+            return sibling
+    return shutil.which("ffprobe")
+
+
+def probe_audio_codec(source_path: str, ffmpeg_exe: Optional[str] = None) -> Optional[str]:
+    """Return the codec name of the first audio stream, if it can be probed."""
+    ffprobe_exe = _find_ffprobe(ffmpeg_exe)
+    if ffprobe_exe is None:
+        logger.warning("ffprobe was not found; audio will be transcoded to AAC for MP4 compatibility")
+        return None
+
+    cmd = [
+        ffprobe_exe,
+        "-v",
+        "error",
+        "-select_streams",
+        "a:0",
+        "-show_entries",
+        "stream=codec_name",
+        "-of",
+        "default=noprint_wrappers=1:nokey=1",
+        source_path,
+    ]
+    try:
+        result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False)
+    except OSError as exc:
+        logger.warning(f"Failed to run ffprobe for {source_path}: {exc}; audio will be transcoded to AAC")
+        return None
+
+    codec = result.stdout.strip().splitlines()[0].lower() if result.returncode == 0 and result.stdout.strip() else None
+    if codec is None:
+        stderr = result.stderr.strip() if result.stderr else "audio stream not found"
+        logger.warning(f"Failed to probe audio codec for {source_path}: {stderr}; audio will be transcoded to AAC")
+    return codec
+
+
+def mp4_audio_codec_args(source_path: str, ffmpeg_exe: Optional[str] = None) -> list[str]:
+    """Choose MP4-safe FFmpeg audio arguments without re-encoding AAC audio."""
+    if probe_audio_codec(source_path, ffmpeg_exe) == "aac":
+        return ["-c:a", "copy"]
+    return ["-c:a", "aac", "-b:a", MP4_AAC_BITRATE]

lightx2v/utils/utils.py

@@ -17,6 +17,7 @@
 from torchvision.transforms import InterpolationMode
 from torchvision.transforms.functional import resize
 
+from lightx2v.utils.audio_mux import MP4_AAC_BITRATE, mp4_audio_codec_args
 from lightx2v_platform.base.global_var import AI_DEVICE
 
 torch_device_module = getattr(torch, AI_DEVICE)
@@ -371,7 +372,8 @@ def mux_audio_from_video(
         source_video_path: Video file that contains the audio to copy.
         target_video_path: Video file that contains the video stream to keep.
         output_path: Optional output path. Defaults to target_video_path (in-place replace).
-        prefer_copy: If True, try stream copy for audio first, then fallback to AAC re-encode.
+        prefer_copy: If True, copy AAC audio and transcode other codecs to AAC. If False,
+            always transcode audio to AAC.
         trim_to_shortest: End the output when its shortest stream ends.
 
     Returns:
@@ -392,7 +394,7 @@ def mux_audio_from_video(
     if os.path.exists(tmp_path):
         os.remove(tmp_path)
 
-    def _run_mux(audio_codec: str, extra_args: Optional[list] = None) -> subprocess.CompletedProcess:
+    def _run_mux(audio_args: list[str]) -> subprocess.CompletedProcess:
         cmd = [
             ffmpeg_exe,
             "-y",
@@ -406,23 +408,21 @@ def _run_mux(audio_codec: str, extra_args: Optional[list] = None) -> subprocess.
             "1:a?",
             "-c:v",
             "copy",
-            "-c:a",
-            audio_codec,
+            *audio_args,
         ]
         if trim_to_shortest:
             cmd.append("-shortest")
         # Be explicit about container format in case ffmpeg can't infer it
         cmd += ["-f", "mp4"]
-        if extra_args:
-            cmd += extra_args
         cmd.append(tmp_path)
         return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
-    result = _run_mux("copy") if prefer_copy else _run_mux("aac", ["-b:a", "192k"])
+    audio_args = mp4_audio_codec_args(source_video_path, ffmpeg_exe) if prefer_copy else ["-c:a", "aac", "-b:a", MP4_AAC_BITRATE]
+    result = _run_mux(audio_args)
 
-    if result.returncode != 0 and prefer_copy:
-        # Fallback to AAC re-encode if stream copy fails
-        result = _run_mux("aac", ["-b:a", "192k"])
+    if result.returncode != 0 and audio_args == ["-c:a", "copy"]:
+        # An unusual AAC stream may still be unsuitable for MP4 stream copying.
+        result = _run_mux(["-c:a", "aac", "-b:a", MP4_AAC_BITRATE])
 
     if result.returncode != 0:
         stderr = result.stderr.decode(errors="ignore") if result.stderr else "Unknown error"