From d33d0ec516dbce8d09e438969a1ff1c46a25e785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=9E=E6=85=A7=E6=B1=80?= Date: Mon, 10 Aug 2026 18:35:28 +0800 Subject: [PATCH] fix: transcode non-AAC audio to AAC for MP4 container --- .../models/runners/cosmos3/cosmos3_runner.py | 3 +- .../runners/wan/wan_infinitetalk_runner.py | 16 +++--- lightx2v/models/runners/wan/wan_s2v_runner.py | 27 +++++++-- lightx2v/utils/audio_mux.py | 55 +++++++++++++++++++ lightx2v/utils/utils.py | 20 +++---- 5 files changed, 96 insertions(+), 25 deletions(-) create mode 100644 lightx2v/utils/audio_mux.py diff --git a/lightx2v/models/runners/cosmos3/cosmos3_runner.py b/lightx2v/models/runners/cosmos3/cosmos3_runner.py index 0c1b753eb..37ad365be 100644 --- a/lightx2v/models/runners/cosmos3/cosmos3_runner.py +++ b/lightx2v/models/runners/cosmos3/cosmos3_runner.py @@ -26,6 +26,7 @@ from lightx2v.models.runners.default_runner import DefaultRunner from lightx2v.models.schedulers.cosmos3.scheduler import Cosmos3Scheduler from lightx2v.models.video_encoders.hf.cosmos3.vae import Cosmos3WanVAE +from lightx2v.utils.audio_mux import MP4_AAC_BITRATE from lightx2v.utils.envs import * from lightx2v.utils.profiler import * from lightx2v.utils.registry_factory import RUNNER_REGISTER @@ -795,7 +796,7 @@ def _mux_generated_audio(self, video_path, audio): "-c:a", "aac", "-b:a", - "192k", + MP4_AAC_BITRATE, "-shortest", "-f", "mp4", diff --git a/lightx2v/models/runners/wan/wan_infinitetalk_runner.py b/lightx2v/models/runners/wan/wan_infinitetalk_runner.py index 225d0e736..53ce56208 100644 --- a/lightx2v/models/runners/wan/wan_infinitetalk_runner.py +++ b/lightx2v/models/runners/wan/wan_infinitetalk_runner.py @@ -20,6 +20,7 @@ from lightx2v.models.schedulers.wan.infinitetalk.scheduler import InfiniteTalkScheduler from lightx2v.server.metrics import monitor_cli from lightx2v.utils.audio_io import load_audio_file +from lightx2v.utils.audio_mux import MP4_AAC_BITRATE, mp4_audio_codec_args from lightx2v.utils.envs import GET_DTYPE, GET_RECORDER_MODE from lightx2v.utils.input_info import UNSET from lightx2v.utils.profiler import ProfilingContext4DebugL1, ProfilingContext4DebugL2 @@ -1197,25 +1198,22 @@ def _mux_audio(video_path, audio_path, timeout=600): "1:a:0", "-shortest", ] - cmd = [ - *base_cmd, - "-c:a", - "copy", - tmp_path, - ] + audio_args = mp4_audio_codec_args(audio_path, ffmpeg.get_ffmpeg_exe()) + cmd = [*base_cmd, *audio_args, tmp_path] try: res = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=timeout) - if res.returncode != 0: - # Fallback to aac re-encoding (e.g. for WAV/PCM inputs) + if res.returncode != 0 and audio_args == ["-c:a", "copy"]: cmd = [ *base_cmd, "-c:a", "aac", "-b:a", - "192k", + MP4_AAC_BITRATE, tmp_path, ] subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=timeout) + elif res.returncode != 0: + res.check_returncode() os.replace(tmp_path, video_path) logger.info(f"Muxed audio from {audio_path}") except subprocess.TimeoutExpired as exc: diff --git a/lightx2v/models/runners/wan/wan_s2v_runner.py b/lightx2v/models/runners/wan/wan_s2v_runner.py index 38e82410f..7749d51b9 100644 --- a/lightx2v/models/runners/wan/wan_s2v_runner.py +++ b/lightx2v/models/runners/wan/wan_s2v_runner.py @@ -17,6 +17,7 @@ from lightx2v.models.runners.wan.wan_runner import WanRunner from lightx2v.models.schedulers.wan.s2v.s2v_scheduler import WanS2VScheduler from lightx2v.server.metrics import monitor_cli +from lightx2v.utils.audio_mux import MP4_AAC_BITRATE, mp4_audio_codec_args from lightx2v.utils.envs import GET_DTYPE from lightx2v.utils.profiler import * from lightx2v.utils.registry_factory import RUNNER_REGISTER @@ -31,6 +32,7 @@ def merge_video_audio(video_path: str, audio_path: str): tmp_path = video_path + ".tmp.mp4" + audio_args = mp4_audio_codec_args(audio_path, "ffmpeg") cmd = [ "ffmpeg", "-y", @@ -40,16 +42,31 @@ def merge_video_audio(video_path: str, audio_path: str): audio_path, "-c:v", "copy", - "-c:a", - "copy", + *audio_args, "-shortest", tmp_path, ] res = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - if res.returncode != 0: - # Fallback to aac re-encoding if stream copy fails (e.g., for WAV inputs) - cmd[9] = "aac" + if res.returncode != 0 and audio_args == ["-c:a", "copy"]: + cmd = [ + "ffmpeg", + "-y", + "-i", + video_path, + "-i", + audio_path, + "-c:v", + "copy", + "-c:a", + "aac", + "-b:a", + MP4_AAC_BITRATE, + "-shortest", + tmp_path, + ] subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + elif res.returncode != 0: + res.check_returncode() os.replace(tmp_path, video_path) diff --git a/lightx2v/utils/audio_mux.py b/lightx2v/utils/audio_mux.py new file mode 100644 index 000000000..298d2cf4f --- /dev/null +++ b/lightx2v/utils/audio_mux.py @@ -0,0 +1,55 @@ +import logging +import os +import shutil +import subprocess +from typing import Optional + +MP4_AAC_BITRATE = "256k" +logger = logging.getLogger(__name__) + + +def _find_ffprobe(ffmpeg_exe: Optional[str] = None) -> Optional[str]: + if ffmpeg_exe: + sibling = os.path.join(os.path.dirname(os.path.abspath(ffmpeg_exe)), "ffprobe") + if os.path.isfile(sibling) and os.access(sibling, os.X_OK): + return sibling + return shutil.which("ffprobe") + + +def probe_audio_codec(source_path: str, ffmpeg_exe: Optional[str] = None) -> Optional[str]: + """Return the codec name of the first audio stream, if it can be probed.""" + ffprobe_exe = _find_ffprobe(ffmpeg_exe) + if ffprobe_exe is None: + logger.warning("ffprobe was not found; audio will be transcoded to AAC for MP4 compatibility") + return None + + cmd = [ + ffprobe_exe, + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "stream=codec_name", + "-of", + "default=noprint_wrappers=1:nokey=1", + source_path, + ] + try: + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + except OSError as exc: + logger.warning(f"Failed to run ffprobe for {source_path}: {exc}; audio will be transcoded to AAC") + return None + + codec = result.stdout.strip().splitlines()[0].lower() if result.returncode == 0 and result.stdout.strip() else None + if codec is None: + stderr = result.stderr.strip() if result.stderr else "audio stream not found" + logger.warning(f"Failed to probe audio codec for {source_path}: {stderr}; audio will be transcoded to AAC") + return codec + + +def mp4_audio_codec_args(source_path: str, ffmpeg_exe: Optional[str] = None) -> list[str]: + """Choose MP4-safe FFmpeg audio arguments without re-encoding AAC audio.""" + if probe_audio_codec(source_path, ffmpeg_exe) == "aac": + return ["-c:a", "copy"] + return ["-c:a", "aac", "-b:a", MP4_AAC_BITRATE] diff --git a/lightx2v/utils/utils.py b/lightx2v/utils/utils.py index e3de78105..baf4ccddf 100755 --- a/lightx2v/utils/utils.py +++ b/lightx2v/utils/utils.py @@ -17,6 +17,7 @@ from torchvision.transforms import InterpolationMode from torchvision.transforms.functional import resize +from lightx2v.utils.audio_mux import MP4_AAC_BITRATE, mp4_audio_codec_args from lightx2v_platform.base.global_var import AI_DEVICE torch_device_module = getattr(torch, AI_DEVICE) @@ -371,7 +372,8 @@ def mux_audio_from_video( source_video_path: Video file that contains the audio to copy. target_video_path: Video file that contains the video stream to keep. output_path: Optional output path. Defaults to target_video_path (in-place replace). - prefer_copy: If True, try stream copy for audio first, then fallback to AAC re-encode. + prefer_copy: If True, copy AAC audio and transcode other codecs to AAC. If False, + always transcode audio to AAC. trim_to_shortest: End the output when its shortest stream ends. Returns: @@ -392,7 +394,7 @@ def mux_audio_from_video( if os.path.exists(tmp_path): os.remove(tmp_path) - def _run_mux(audio_codec: str, extra_args: Optional[list] = None) -> subprocess.CompletedProcess: + def _run_mux(audio_args: list[str]) -> subprocess.CompletedProcess: cmd = [ ffmpeg_exe, "-y", @@ -406,23 +408,21 @@ def _run_mux(audio_codec: str, extra_args: Optional[list] = None) -> subprocess. "1:a?", "-c:v", "copy", - "-c:a", - audio_codec, + *audio_args, ] if trim_to_shortest: cmd.append("-shortest") # Be explicit about container format in case ffmpeg can't infer it cmd += ["-f", "mp4"] - if extra_args: - cmd += extra_args cmd.append(tmp_path) return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - result = _run_mux("copy") if prefer_copy else _run_mux("aac", ["-b:a", "192k"]) + audio_args = mp4_audio_codec_args(source_video_path, ffmpeg_exe) if prefer_copy else ["-c:a", "aac", "-b:a", MP4_AAC_BITRATE] + result = _run_mux(audio_args) - if result.returncode != 0 and prefer_copy: - # Fallback to AAC re-encode if stream copy fails - result = _run_mux("aac", ["-b:a", "192k"]) + if result.returncode != 0 and audio_args == ["-c:a", "copy"]: + # An unusual AAC stream may still be unsuitable for MP4 stream copying. + result = _run_mux(["-c:a", "aac", "-b:a", MP4_AAC_BITRATE]) if result.returncode != 0: stderr = result.stderr.decode(errors="ignore") if result.stderr else "Unknown error"