diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 6dc7d3966..1bdcbd510 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -563,6 +563,15 @@ interface Window { success: boolean; paths: string[]; startDelayMsByPath?: Record; + mediaInfoByPath?: Record< + string, + { + durationMs: number; + sampleRate: number | null; + channels: number | null; + hasAudioStream: boolean; + } + >; error?: string; }>; setRecordingState: (recording: boolean) => Promise; diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts index 9f7ddeb08..4a5992ff3 100644 --- a/electron/ipc/recording/diagnostics.test.ts +++ b/electron/ipc/recording/diagnostics.test.ts @@ -190,6 +190,76 @@ describe("getCompanionAudioFallbackPaths", () => { startDelayMsByPath: { [micPath]: 2750, }, + mediaInfoByPath: { + [micPath]: { + durationMs: 0, + sampleRate: null, + channels: null, + hasAudioStream: false, + }, + }, + }); + }); + + it("returns probed companion audio media info for pure mic sidecars", async () => { + const videoPath = path.join(tempRoot, "recording.mp4"); + const micPath = path.join(tempRoot, "recording.mic.wav"); + + await Promise.all([ + fs.writeFile(videoPath, "video"), + fs.writeFile(micPath, "mic"), + fs.writeFile(`${micPath}.json`, JSON.stringify({ startDelayMs: 143 })), + ]); + + execFileMock.mockImplementation( + ( + file: string, + args: string[], + _options: Record, + callback: ExecFileCallback, + ) => { + if ( + file === "ffprobe" && + args.includes("-select_streams") && + args.includes("a:0") + ) { + callback( + null, + JSON.stringify({ + streams: [ + { + duration: "147.360000", + sample_rate: "48000", + channels: 1, + }, + ], + }), + "", + ); + return; + } + + const error = new Error("ffmpeg probe failed") as Error & { stderr?: string }; + error.stderr = "Stream #0:0: Video: h264"; + callback(error, "", error.stderr); + }, + ); + + const { getCompanionAudioFallbackInfo } = await import("./diagnostics"); + + await expect(getCompanionAudioFallbackInfo(videoPath)).resolves.toEqual({ + paths: [micPath], + startDelayMsByPath: { + [micPath]: 143, + }, + mediaInfoByPath: { + [micPath]: { + durationMs: 147_360, + sampleRate: 48_000, + channels: 1, + hasAudioStream: true, + }, + }, }); }); diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index 1f0003a9a..d306d626b 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -37,6 +37,13 @@ export type VideoStreamDurationProbe = { frameRate: number | null; }; +export type CompanionAudioMediaInfo = { + durationMs: number; + sampleRate: number | null; + channels: number | null; + hasAudioStream: boolean; +}; + export type RecordingDiagnosticsSnapshot = { backend: NativeCaptureDiagnostics["backend"]; phase: @@ -305,6 +312,72 @@ export async function probeVideoStreamDurationSeconds(filePath: string): Promise : probeMediaDurationSeconds(filePath); } +function parseFfprobeAudioStreamInfo(output: string): CompanionAudioMediaInfo { + try { + const parsed = JSON.parse(output) as { + streams?: Array<{ + duration?: unknown; + sample_rate?: unknown; + channels?: unknown; + }>; + }; + const stream = parsed.streams?.[0]; + if (!stream) { + return { + durationMs: 0, + sampleRate: null, + channels: null, + hasAudioStream: false, + }; + } + + return { + durationMs: Math.round((parsePositiveNumber(stream.duration) ?? 0) * 1000), + sampleRate: parsePositiveInteger(stream.sample_rate), + channels: parsePositiveInteger(stream.channels), + hasAudioStream: true, + }; + } catch { + return { + durationMs: 0, + sampleRate: null, + channels: null, + hasAudioStream: false, + }; + } +} + +export async function probeCompanionAudioMediaInfo( + filePath: string, +): Promise { + try { + const result = await execFileAsync( + getFfprobeBinaryPath(), + [ + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "stream=duration,sample_rate,channels", + "-of", + "json", + filePath, + ], + { timeout: 30000, maxBuffer: 2 * 1024 * 1024 }, + ); + const stdout = typeof result === "string" ? result : result.stdout; + return parseFfprobeAudioStreamInfo(stdout); + } catch { + return { + durationMs: 0, + sampleRate: null, + channels: null, + hasAudioStream: false, + }; + } +} + export function getRecordingDiagnosticsPath(videoPath: string) { return `${videoPath.replace(/\.[^.]+$/u, "")}.recording-diagnostics.json`; } @@ -364,9 +437,11 @@ async function describeAudioFile(filePath: string | null | undefined) { } const startDelayMs = await getCompanionAudioStartDelayMs(filePath); + const stream = await probeCompanionAudioMediaInfo(filePath); return { ...media, startDelayMs, + stream, }; } @@ -503,7 +578,7 @@ export async function getCompanionAudioFallbackPaths(videoPath: string) { export async function getCompanionAudioFallbackInfo(videoPath: string) { const companionCandidates = await getUsableCompanionAudioCandidates(videoPath); if (companionCandidates.length === 0) { - return { paths: [], startDelayMsByPath: {} }; + return { paths: [], startDelayMsByPath: {}, mediaInfoByPath: {} }; } let paths: string[]; @@ -538,7 +613,7 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) { ), ); if (companionPaths.length === 0) { - return { paths: [], startDelayMsByPath: {} }; + return { paths: [], startDelayMsByPath: {}, mediaInfoByPath: {} }; } paths = [videoPath, ...companionPaths]; @@ -559,12 +634,26 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) { return [audioPath, startDelayMs] as const; }), ); + const mediaInfoEntries = await Promise.all( + paths.map(async (audioPath) => { + if (audioPath === videoPath) { + return null; + } + + return [audioPath, await probeCompanionAudioMediaInfo(audioPath)] as const; + }), + ); return { paths, startDelayMsByPath: Object.fromEntries( metadataEntries.filter((entry): entry is readonly [string, number] => entry !== null), ), + mediaInfoByPath: Object.fromEntries( + mediaInfoEntries.filter( + (entry): entry is readonly [string, CompanionAudioMediaInfo] => entry !== null, + ), + ), }; } diff --git a/electron/ipc/register/recording.test.ts b/electron/ipc/register/recording.test.ts new file mode 100644 index 000000000..f9ca494e4 --- /dev/null +++ b/electron/ipc/register/recording.test.ts @@ -0,0 +1,92 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const companionInfo = { + paths: ["C:\\Recordly\\recording.mic.wav"], + startDelayMsByPath: { + "C:\\Recordly\\recording.mic.wav": 143, + }, + mediaInfoByPath: { + "C:\\Recordly\\recording.mic.wav": { + durationMs: 147_360, + sampleRate: 48_000, + channels: 1, + hasAudioStream: true, + }, + }, +}; + +const getCompanionAudioFallbackInfoMock = vi.fn(); +const rememberApprovedLocalReadPathMock = vi.fn(async () => undefined); + +vi.mock("electron", () => ({ + app: { + getAppPath: () => process.cwd(), + getPath: () => process.cwd(), + isPackaged: false, + }, + BrowserWindow: { + getAllWindows: () => [], + }, + desktopCapturer: { + getSources: vi.fn(), + }, + dialog: { + showMessageBox: vi.fn(), + }, + ipcMain: { + handle: vi.fn(), + }, + shell: { + openExternal: vi.fn(), + }, + systemPreferences: { + getMediaAccessStatus: vi.fn(), + askForMediaAccess: vi.fn(), + }, +})); + +vi.mock("../recording/diagnostics", () => ({ + getCompanionAudioFallbackInfo: getCompanionAudioFallbackInfoMock, + getFileSizeIfPresent: vi.fn(), + recordNativeCaptureDiagnostics: vi.fn(), + summarizeMicrophoneChunkTiming: vi.fn(), + validateRecordedVideo: vi.fn(), + writeRecordingDiagnosticsSnapshot: vi.fn(), +})); + +vi.mock("../project/manager", () => ({ + rememberApprovedLocalReadPath: rememberApprovedLocalReadPathMock, +})); + +describe("resolveVideoAudioFallbackPathsForIpc", () => { + beforeEach(() => { + getCompanionAudioFallbackInfoMock.mockReset(); + rememberApprovedLocalReadPathMock.mockClear(); + }); + + it("returns companion media info so renderer playback and waveforms can use probed duration", async () => { + getCompanionAudioFallbackInfoMock.mockResolvedValue(companionInfo); + const videoPath = "C:\\Recordly\\recording.mp4"; + + const { resolveVideoAudioFallbackPathsForIpc } = await import("./recording"); + + await expect(resolveVideoAudioFallbackPathsForIpc(videoPath)).resolves.toEqual({ + success: true, + ...companionInfo, + }); + expect(rememberApprovedLocalReadPathMock).toHaveBeenCalledWith(videoPath); + expect(rememberApprovedLocalReadPathMock).toHaveBeenCalledWith(companionInfo.paths[0]); + }); + + it("returns a stable empty shape when no video path is available", async () => { + const { resolveVideoAudioFallbackPathsForIpc } = await import("./recording"); + + await expect(resolveVideoAudioFallbackPathsForIpc("")).resolves.toEqual({ + success: true, + paths: [], + startDelayMsByPath: {}, + mediaInfoByPath: {}, + }); + expect(getCompanionAudioFallbackInfoMock).not.toHaveBeenCalled(); + }); +}); diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index b13453c74..71e6b7b81 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -390,6 +390,31 @@ async function resolveExistingPath(...candidates: Array rememberApprovedLocalReadPath(fallbackPath)), + ]); + return { success: true, paths, startDelayMsByPath, mediaInfoByPath }; + } catch (error) { + console.error("Failed to resolve companion audio fallback paths:", error); + return { + success: false, + paths: [], + startDelayMsByPath: {}, + mediaInfoByPath: {}, + error: String(error), + }; + } +} + export function registerRecordingHandlers( onRecordingStateChange?: (recording: boolean, sourceName: string) => void, ) { @@ -1385,21 +1410,7 @@ export function registerRecordingHandlers( }); ipcMain.handle("get-video-audio-fallback-paths", async (_event, videoPath: string) => { - if (!videoPath) { - return { success: true, paths: [], startDelayMsByPath: {} }; - } - - try { - const { paths, startDelayMsByPath } = await getCompanionAudioFallbackInfo(videoPath); - await Promise.all([ - rememberApprovedLocalReadPath(videoPath), - ...paths.map((fallbackPath) => rememberApprovedLocalReadPath(fallbackPath)), - ]); - return { success: true, paths, startDelayMsByPath }; - } catch (error) { - console.error("Failed to resolve companion audio fallback paths:", error); - return { success: false, paths: [], startDelayMsByPath: {}, error: String(error) }; - } + return resolveVideoAudioFallbackPathsForIpc(videoPath); }); ipcMain.handle("mux-native-windows-recording", async (_event, expectedDurationMs?: number) => { diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index f18b931f7..636c555f2 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -6366,6 +6366,10 @@ export default function VideoEditor() { onSourceAudioTracksMetaChange={(tracks) => { audio.onSourceAudioTracksMetaChange(tracks); }} + sourceAudioFallbackPaths={audio.sourceAudioFallbackPaths} + sourceAudioFallbackMediaInfoByPath={ + audio.sourceAudioFallbackMediaInfoByPath + } /> diff --git a/src/components/video-editor/audio/audioTypes.ts b/src/components/video-editor/audio/audioTypes.ts index d20c224f0..9400e3c91 100644 --- a/src/components/video-editor/audio/audioTypes.ts +++ b/src/components/video-editor/audio/audioTypes.ts @@ -16,10 +16,21 @@ export interface SourceAudioTrackMetaItem { export type SourceAudioTrackMeta = SourceAudioTrackMetaItem[]; +export interface SourceAudioMediaInfo { + durationMs: number; + sampleRate: number | null; + channels: number | null; + hasAudioStream: boolean; +} + export interface SourceAudioTrackWithPeaks extends SourceAudioTrackMetaItem { - peaks: AudioPeaksData; + kind: "embedded" | "system" | "mic" | "mixed"; + resourcePath: string | null; + peaks: AudioPeaksData | null; + probedDurationMs: number | null; + waveformAvailable: boolean; + waveformCoverage?: "full" | "partial" | "none"; } export const SOURCE_AUDIO_FALLBACK_TOAST_ID = "source-audio-fallback-error"; export const SOURCE_AUDIO_NORMALIZE_GAIN = 1.35; - diff --git a/src/components/video-editor/audio/useAudioPreviewSync.test.ts b/src/components/video-editor/audio/useAudioPreviewSync.test.ts new file mode 100644 index 000000000..76351616f --- /dev/null +++ b/src/components/video-editor/audio/useAudioPreviewSync.test.ts @@ -0,0 +1,256 @@ +import { describe, expect, it } from "vitest"; +import { + decodeSourceWavAudioBuffer, + getDecodedSourcePreviewSyncAction, + getSourceAudioElementResourceKey, + getSourceAudioPreviewVolume, + isAudioResourceLoadCurrent, + shouldPlaySourceAudioElement, + shouldUseDecodedWavSourcePreview, + syncSourceAudioElementPlayback, +} from "./useAudioPreviewSync"; + +describe("getSourceAudioElementResourceKey", () => { + it("changes when probed companion audio media info arrives after an initial partial load", () => { + const audioPath = "C:\\Recordly\\recording.mic.wav"; + const mediaInfo = { + durationMs: 122_100, + sampleRate: 48_000, + channels: 1, + hasAudioStream: true, + }; + + const withoutProbe = getSourceAudioElementResourceKey(audioPath, undefined); + const withProbe = getSourceAudioElementResourceKey(audioPath, mediaInfo); + + expect(withoutProbe).not.toBe(withProbe); + expect(getSourceAudioElementResourceKey(audioPath, mediaInfo)).toBe(withProbe); + }); + + it("changes when the probed duration changes for the same sidecar path", () => { + const audioPath = "C:\\Recordly\\recording.mic.wav"; + + expect( + getSourceAudioElementResourceKey(audioPath, { + durationMs: 75_000, + sampleRate: 48_000, + channels: 1, + hasAudioStream: true, + }), + ).not.toBe( + getSourceAudioElementResourceKey(audioPath, { + durationMs: 122_100, + sampleRate: 48_000, + channels: 1, + hasAudioStream: true, + }), + ); + }); +}); + +describe("shouldUseDecodedWavSourcePreview", () => { + it("routes local wav companion audio through decoded Web Audio preview", () => { + expect( + shouldUseDecodedWavSourcePreview("C:\\Recordly\\recording.mic.wav", { + durationMs: 146_700, + sampleRate: 48_000, + channels: 1, + hasAudioStream: true, + }), + ).toBe(true); + }); + + it("keeps non-wav companion audio on the media element path", () => { + expect( + shouldUseDecodedWavSourcePreview("C:\\Recordly\\recording.mic.m4a", { + durationMs: 146_700, + sampleRate: 48_000, + channels: 1, + hasAudioStream: true, + }), + ).toBe(false); + }); + + it("does not decode a probed path without an audio stream", () => { + expect( + shouldUseDecodedWavSourcePreview("C:\\Recordly\\recording.mic.wav", { + durationMs: 0, + sampleRate: null, + channels: null, + hasAudioStream: false, + }), + ).toBe(false); + }); +}); + +describe("shouldPlaySourceAudioElement", () => { + it("plays only while the companion track is inside its active range", () => { + expect( + shouldPlaySourceAudioElement({ + isPlaying: true, + beforeAudioStart: false, + atEnd: false, + }), + ).toBe(true); + }); + + it("does not start a delayed companion track early or restart it after the end", () => { + expect( + shouldPlaySourceAudioElement({ + isPlaying: true, + beforeAudioStart: true, + atEnd: false, + }), + ).toBe(false); + expect( + shouldPlaySourceAudioElement({ + isPlaying: true, + beforeAudioStart: false, + atEnd: true, + }), + ).toBe(false); + }); +}); + +describe("syncSourceAudioElementPlayback", () => { + it("starts direct media playback immediately without waiting for Web Audio", () => { + let playCalls = 0; + const audio = { + paused: true, + play: () => { + playCalls += 1; + return Promise.resolve(); + }, + pause: () => undefined, + }; + + syncSourceAudioElementPlayback(audio, true); + + expect(playCalls).toBe(1); + }); + + it("pauses a direct media element when the current sync pass disallows playback", () => { + let pauseCalls = 0; + const audio = { + paused: false, + play: () => Promise.resolve(), + pause: () => { + pauseCalls += 1; + }, + }; + + syncSourceAudioElementPlayback(audio, false); + + expect(pauseCalls).toBe(1); + }); +}); + +describe("decodeSourceWavAudioBuffer", () => { + it("falls back to browser decoding when the custom WAV decoder rejects the subformat", async () => { + const arrayBuffer = new ArrayBuffer(8); + const browserDecodedBuffer = { duration: 1.5 } as AudioBuffer; + let fallbackCalls = 0; + const context = { + createBuffer: () => { + throw new Error("custom WAV conversion should not run"); + }, + decodeAudioData: async (input: ArrayBuffer) => { + fallbackCalls += 1; + expect(input).toBe(arrayBuffer); + return browserDecodedBuffer; + }, + } as unknown as AudioContext; + + await expect(decodeSourceWavAudioBuffer(context, arrayBuffer)).resolves.toBe( + browserDecodedBuffer, + ); + expect(fallbackCalls).toBe(1); + }); +}); + +describe("getDecodedSourcePreviewSyncAction", () => { + it("starts decoded playback when a wav buffer is ready and no source is active", () => { + expect( + getDecodedSourcePreviewSyncAction({ + isPlaying: true, + beforeAudioStart: false, + atEnd: false, + hasBuffer: true, + hasActiveSource: false, + timelineJumped: false, + targetTime: 115.8, + predictedTime: null, + playbackRate: 1, + activePlaybackRate: null, + }), + ).toBe("start"); + }); + + it("restarts decoded playback after a late seek drifts away from the active source", () => { + expect( + getDecodedSourcePreviewSyncAction({ + isPlaying: true, + beforeAudioStart: false, + atEnd: false, + hasBuffer: true, + hasActiveSource: true, + timelineJumped: true, + targetTime: 115.8, + predictedTime: 48.2, + playbackRate: 1, + activePlaybackRate: 1, + }), + ).toBe("restart"); + }); + + it("keeps decoded playback running while the predicted PCM time is in sync", () => { + expect( + getDecodedSourcePreviewSyncAction({ + isPlaying: true, + beforeAudioStart: false, + atEnd: false, + hasBuffer: true, + hasActiveSource: true, + timelineJumped: false, + targetTime: 115.8, + predictedTime: 115.86, + playbackRate: 1, + activePlaybackRate: 1, + }), + ).toBe("keep"); + }); + + it("stops decoded playback outside the companion audio range", () => { + expect( + getDecodedSourcePreviewSyncAction({ + isPlaying: true, + beforeAudioStart: false, + atEnd: true, + hasBuffer: true, + hasActiveSource: true, + timelineJumped: false, + targetTime: 146.7, + predictedTime: 146.7, + playbackRate: 1, + activePlaybackRate: 1, + }), + ).toBe("stop"); + }); +}); + +describe("async source audio resource loading", () => { + it("keeps an in-flight load valid across playback rerenders but rejects stale versions", () => { + const audioPath = "C:\\Recordly\\recording.mic.wav"; + const resources = new Map([[audioPath, `${audioPath}::v1`]]); + + expect(isAudioResourceLoadCurrent(resources, audioPath, `${audioPath}::v1`)).toBe(true); + + resources.set(audioPath, `${audioPath}::v2`); + expect(isAudioResourceLoadCurrent(resources, audioPath, `${audioPath}::v1`)).toBe(false); + }); + + it("applies the current preview volume as soon as a media element is created", () => { + expect(getSourceAudioPreviewVolume(0.5, 0.8, false)).toBe(0.4); + expect(getSourceAudioPreviewVolume(0.5, 0.8, true)).toBe(0); + }); +}); diff --git a/src/components/video-editor/audio/useAudioPreviewSync.ts b/src/components/video-editor/audio/useAudioPreviewSync.ts index c00ff9f6d..5c2bade8e 100644 --- a/src/components/video-editor/audio/useAudioPreviewSync.ts +++ b/src/components/video-editor/audio/useAudioPreviewSync.ts @@ -1,459 +1,923 @@ -import { useCallback, useEffect, useMemo, useRef } from "react"; +import { useCallback, useEffect, useMemo, useReducer, useRef } from "react"; +import type { SourceAudioMediaInfo } from "@/components/video-editor/audio/audioTypes"; import { buildResolvedAudioPlan } from "@/lib/exporter/audioRoutingEngine"; -import { resolveMediaElementSource } from "@/lib/exporter/localMediaSource"; import { - clampMediaTimeToDuration, - enablePitchPreservingPlayback, - estimateCompanionAudioStartDelaySeconds, - getMediaSyncPlaybackRate, + createReadableMediaResourceFile, + getLocalFilePath, + resolveMediaElementSource, +} from "@/lib/exporter/localMediaSource"; +import { + enablePitchPreservingPlayback, + getMediaSyncPlaybackRate, + resolveCompanionAudioPreviewTiming, } from "@/lib/mediaTiming"; import type { AudioRegion, SpeedRegion } from "../types"; +import { type DecodedWavAudio, decodeWavAudioData } from "./waveform/wavDecoder"; const SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS = 0.18; const SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS = 0.01; +const SOURCE_AUDIO_PREVIEW_PLAYBACK_RATE_EPSILON = 0.001; +const SOURCE_AUDIO_PREVIEW_REMOTE_RESOURCE_PATTERN = /^(https?:|blob:|data:)/i; interface UseAudioPreviewSyncParams { - audioRegions: AudioRegion[]; - previewVolume: number; - isPlaying: boolean; - currentTime: number; - timelineTime: number; - duration: number; - effectiveSpeedRegions: SpeedRegion[]; - previewSourceAudioFallbackPaths: string[]; - sourceAudioFallbackStartDelayMsByPath: Record; - isCurrentClipMuted: boolean; - getSourceTrackPreviewGain: (audioPath: string) => number; - onSourceFallbackLoadError: (error: unknown) => void; + audioRegions: AudioRegion[]; + previewVolume: number; + isPlaying: boolean; + currentTime: number; + timelineTime: number; + duration: number; + effectiveSpeedRegions: SpeedRegion[]; + previewSourceAudioFallbackPaths: string[]; + sourceAudioFallbackStartDelayMsByPath: Record; + sourceAudioFallbackMediaInfoByPath: Record; + isCurrentClipMuted: boolean; + getSourceTrackPreviewGain: (audioPath: string) => number; + onSourceFallbackLoadError: (error: unknown) => void; +} + +type DecodedSourcePreviewSyncAction = "start" | "restart" | "keep" | "stop"; + +interface DecodedSourcePreviewSyncInput { + isPlaying: boolean; + beforeAudioStart: boolean; + atEnd: boolean; + hasBuffer: boolean; + hasActiveSource: boolean; + timelineJumped: boolean; + targetTime: number; + predictedTime: number | null; + playbackRate: number; + activePlaybackRate: number | null; +} + +interface DecodedSourceAudioBufferEntry { + resourceKey: string; + buffer: AudioBuffer; +} + +interface ActiveDecodedSourceAudio { + source: AudioBufferSourceNode; + startedAtContextTime: number; + offsetSeconds: number; + playbackRate: number; +} + +export function getSourceAudioElementResourceKey( + audioPath: string, + mediaInfo?: SourceAudioMediaInfo | null, +) { + if (!mediaInfo) { + return `${audioPath}::unprobed`; + } + + return [ + audioPath, + Number.isFinite(mediaInfo.durationMs) ? Math.round(mediaInfo.durationMs) : "duration", + Number.isFinite(mediaInfo.sampleRate) ? mediaInfo.sampleRate : "sample-rate", + Number.isFinite(mediaInfo.channels) ? mediaInfo.channels : "channels", + mediaInfo.hasAudioStream ? "audio" : "no-audio", + ].join("::"); +} + +export function isAudioResourceLoadCurrent( + resources: ReadonlyMap, + audioPath: string, + expectedResourceKey: string, +) { + return resources.get(audioPath) === expectedResourceKey; +} + +export function getSourceAudioPreviewVolume( + trackGain: number, + previewVolume: number, + muted: boolean, +) { + if (muted) { + return 0; + } + return Math.max(0, Math.min(1, trackGain * previewVolume)); +} + +export function shouldPlaySourceAudioElement({ + isPlaying, + beforeAudioStart, + atEnd, +}: { + isPlaying: boolean; + beforeAudioStart: boolean; + atEnd: boolean; +}) { + return isPlaying && !beforeAudioStart && !atEnd; +} + +export function syncSourceAudioElementPlayback( + audio: Pick, + shouldPlay: boolean, +) { + if (shouldPlay) { + audio.play().catch(() => undefined); + } else if (!audio.paused) { + audio.pause(); + } +} + +export function shouldUseDecodedWavSourcePreview( + audioPath: string, + mediaInfo?: SourceAudioMediaInfo | null, +) { + if (mediaInfo?.hasAudioStream === false) { + return false; + } + + if ( + SOURCE_AUDIO_PREVIEW_REMOTE_RESOURCE_PATTERN.test(audioPath) && + !getLocalFilePath(audioPath) + ) { + return false; + } + + const resourcePath = getLocalFilePath(audioPath) ?? audioPath; + return resourcePath.split(/[?#]/)[0]?.toLowerCase().endsWith(".wav") ?? false; +} + +export function getDecodedSourcePreviewSyncAction({ + isPlaying, + beforeAudioStart, + atEnd, + hasBuffer, + hasActiveSource, + timelineJumped, + targetTime, + predictedTime, + playbackRate, + activePlaybackRate, +}: DecodedSourcePreviewSyncInput): DecodedSourcePreviewSyncAction { + if (!isPlaying || beforeAudioStart || atEnd || !hasBuffer) { + return "stop"; + } + + if (!hasActiveSource) { + return "start"; + } + + if (timelineJumped) { + return "restart"; + } + + if ( + Number.isFinite(playbackRate) && + Number.isFinite(activePlaybackRate) && + Math.abs(playbackRate - (activePlaybackRate ?? 1)) > + SOURCE_AUDIO_PREVIEW_PLAYBACK_RATE_EPSILON + ) { + return "restart"; + } + + if (!Number.isFinite(targetTime) || !Number.isFinite(predictedTime)) { + return "restart"; + } + + if ( + Math.abs(targetTime - (predictedTime ?? 0)) > + SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS + ) { + return "restart"; + } + + return "keep"; +} + +function createAudioBufferFromDecodedWav(context: AudioContext, decoded: DecodedWavAudio) { + const frameCount = decoded.channels[0]?.length ?? 0; + const buffer = context.createBuffer(decoded.channels.length, frameCount, decoded.sampleRate); + for (let channelIndex = 0; channelIndex < decoded.channels.length; channelIndex++) { + buffer.copyToChannel(new Float32Array(decoded.channels[channelIndex]), channelIndex); + } + return buffer; +} + +export async function decodeSourceWavAudioBuffer(context: AudioContext, arrayBuffer: ArrayBuffer) { + const decoded = decodeWavAudioData(arrayBuffer); + return decoded + ? createAudioBufferFromDecodedWav(context, decoded) + : context.decodeAudioData(arrayBuffer); +} + +function getDecodedSourcePredictedTime( + active: ActiveDecodedSourceAudio | null, + contextCurrentTime: number, +) { + if (!active) { + return null; + } + + return ( + active.offsetSeconds + + Math.max(0, contextCurrentTime - active.startedAtContextTime) * active.playbackRate + ); } export function useAudioPreviewSync({ - audioRegions, - previewVolume, - isPlaying, - currentTime, - timelineTime, - duration, - effectiveSpeedRegions, - previewSourceAudioFallbackPaths, - sourceAudioFallbackStartDelayMsByPath, - isCurrentClipMuted, - getSourceTrackPreviewGain, - onSourceFallbackLoadError, + audioRegions, + previewVolume, + isPlaying, + currentTime, + timelineTime, + duration, + effectiveSpeedRegions, + previewSourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + sourceAudioFallbackMediaInfoByPath, + isCurrentClipMuted, + getSourceTrackPreviewGain, + onSourceFallbackLoadError, }: UseAudioPreviewSyncParams) { - const resolvedPlan = useMemo( - () => - buildResolvedAudioPlan({ - videoResource: null, - sourceAudioFallbackPaths: previewSourceAudioFallbackPaths, - audioRegions, - }), - [audioRegions, previewSourceAudioFallbackPaths], - ); - const resolvedUserTracks = useMemo( - () => resolvedPlan.tracks.filter((track) => track.kind === "user"), - [resolvedPlan], - ); - const resolvedSourceTracks = useMemo( - () => resolvedPlan.tracks.filter((track) => track.kind !== "user"), - [resolvedPlan], - ); - - const audioElementsRef = useRef>(new Map()); - const audioElementRevokersRef = useRef void>>(new Map()); - const audioElementResourcesRef = useRef>(new Map()); - const sourceAudioElementsRef = useRef>(new Map()); - const sourceAudioMediaNodesRef = useRef>(new Map()); - const sourceAudioGainNodesRef = useRef>(new Map()); - const sourceAudioElementRevokersRef = useRef void>>(new Map()); - const sourceAudioElementResourcesRef = useRef>(new Map()); - const sourceAudioContextRef = useRef(null); - const sourceAudioMasterGainRef = useRef(null); - const sourceAudioResumePromiseRef = useRef | null>(null); - const lastSourceAudioSyncTimeRef = useRef(null); - - const ensureSourceAudioContext = useCallback(() => { - if (!sourceAudioContextRef.current) { - const context = new AudioContext({ latencyHint: "interactive" }); - const masterGain = context.createGain(); - masterGain.gain.value = 1; - masterGain.connect(context.destination); - sourceAudioContextRef.current = context; - sourceAudioMasterGainRef.current = masterGain; - } - return sourceAudioContextRef.current; - }, []); - - const ensureSourceAudioRunning = useCallback(() => { - const context = ensureSourceAudioContext(); - if (context.state === "running") { - return Promise.resolve(); - } - if (!sourceAudioResumePromiseRef.current) { - sourceAudioResumePromiseRef.current = context - .resume() - .catch(() => undefined) - .finally(() => { - sourceAudioResumePromiseRef.current = null; - }); - } - return sourceAudioResumePromiseRef.current; - }, [ensureSourceAudioContext]); - - const playSourceAudioPreview = useCallback(() => { - void ensureSourceAudioRunning(); - for (const audio of sourceAudioElementsRef.current.values()) { - if (!audio.src) continue; - audio.play().catch(() => undefined); - } - }, [ensureSourceAudioRunning]); - - useEffect(() => { - let cancelled = false; - const existing = audioElementsRef.current; - const currentIds = new Set(resolvedUserTracks.map((track) => track.id)); - - for (const [id, audio] of existing) { - if (!currentIds.has(id)) { - audio.pause(); - audio.src = ""; - audioElementRevokersRef.current.get(id)?.(); - audioElementRevokersRef.current.delete(id); - audioElementResourcesRef.current.delete(id); - existing.delete(id); - } - } - - for (const track of resolvedUserTracks) { - let audio = existing.get(track.id); - if (!audio) { - audio = new Audio(); - audio.preload = "auto"; - existing.set(track.id, audio); - } - - if (audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path) { - audio.pause(); - audio.src = ""; - audioElementRevokersRef.current.get(track.id)?.(); - audioElementRevokersRef.current.delete(track.id); - audioElementResourcesRef.current.set(track.id, track.sourceRef.path); - - void (async () => { - const resolved = await resolveMediaElementSource(track.sourceRef.path); - const latestAudio = existing.get(track.id); - - if ( - cancelled || - latestAudio !== audio || - audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path - ) { - resolved.revoke(); - return; - } - - audioElementRevokersRef.current.set(track.id, resolved.revoke); - latestAudio.src = resolved.src; - })(); - } - - audio.volume = Math.max(0, Math.min(1, track.gain * previewVolume)); - } - - return () => { - cancelled = true; - }; - }, [previewVolume, resolvedUserTracks]); - - useEffect(() => { - let cancelled = false; - const existing = sourceAudioElementsRef.current; - const currentIds = new Set(resolvedSourceTracks.map((track) => track.sourceRef.path)); - - for (const [id, audio] of existing) { - if (!currentIds.has(id)) { - audio.pause(); - audio.src = ""; - sourceAudioMediaNodesRef.current.get(id)?.disconnect(); - sourceAudioMediaNodesRef.current.delete(id); - sourceAudioGainNodesRef.current.get(id)?.disconnect(); - sourceAudioGainNodesRef.current.delete(id); - sourceAudioElementRevokersRef.current.get(id)?.(); - sourceAudioElementRevokersRef.current.delete(id); - sourceAudioElementResourcesRef.current.delete(id); - existing.delete(id); - } - } - - for (const track of resolvedSourceTracks) { - const audioPath = track.sourceRef.path; - let audio = existing.get(audioPath); - if (!audio) { - audio = new Audio(); - audio.preload = "auto"; - audio.crossOrigin = "anonymous"; - existing.set(audioPath, audio); - } - audio.volume = 1; - audio.dataset.sourceAudioPath = audioPath; - - // Web Audio API createMediaElementSource breaks preservesPitch on Chromium. - // We route directly through the HTMLAudioElement to ensure pitch preservation works - // during speed changes. Note: this limits maximum preview volume to 1.0 (100%). - - if (sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath) { - audio.pause(); - audio.src = ""; - sourceAudioElementRevokersRef.current.get(audioPath)?.(); - sourceAudioElementRevokersRef.current.delete(audioPath); - sourceAudioElementResourcesRef.current.set(audioPath, audioPath); - - void (async () => { - try { - const resolved = await resolveMediaElementSource(audioPath); - const latestAudio = existing.get(audioPath); - - if ( - cancelled || - latestAudio !== audio || - sourceAudioElementResourcesRef.current.get(audioPath) !== audioPath - ) { - resolved.revoke(); - return; - } - - sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); - latestAudio.src = resolved.src; - latestAudio.load(); - if (isPlaying) { - playSourceAudioPreview(); - } - } catch (error) { - if (cancelled) { - return; - } - - sourceAudioElementRevokersRef.current.get(audioPath)?.(); - sourceAudioElementRevokersRef.current.delete(audioPath); - sourceAudioElementResourcesRef.current.delete(audioPath); - const latestAudio = existing.get(audioPath); - if (latestAudio === audio) { - latestAudio.pause(); - latestAudio.src = ""; - } - onSourceFallbackLoadError(error); - } - })(); - } - - audio.volume = Math.max(0, Math.min(1, getSourceTrackPreviewGain(audioPath) * (isCurrentClipMuted ? 0 : previewVolume))); - } - - if (sourceAudioMasterGainRef.current) { - sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted - ? 0 - : Math.max(0, Math.min(1, previewVolume)); - } - - if (resolvedSourceTracks.length === 0) { - lastSourceAudioSyncTimeRef.current = null; - } - - return () => { - cancelled = true; - }; - }, [ - getSourceTrackPreviewGain, - isPlaying, - isCurrentClipMuted, - onSourceFallbackLoadError, - resolvedSourceTracks, - previewVolume, - playSourceAudioPreview, - ]); - - useEffect(() => { - return () => { - for (const audio of audioElementsRef.current.values()) { - audio.pause(); - audio.src = ""; - } - for (const revoke of audioElementRevokersRef.current.values()) { - revoke(); - } - audioElementsRef.current.clear(); - audioElementRevokersRef.current.clear(); - audioElementResourcesRef.current.clear(); - for (const audio of sourceAudioElementsRef.current.values()) { - audio.pause(); - audio.src = ""; - } - for (const node of sourceAudioMediaNodesRef.current.values()) { - node.disconnect(); - } - for (const node of sourceAudioGainNodesRef.current.values()) { - node.disconnect(); - } - for (const revoke of sourceAudioElementRevokersRef.current.values()) { - revoke(); - } - sourceAudioElementsRef.current.clear(); - sourceAudioMediaNodesRef.current.clear(); - sourceAudioGainNodesRef.current.clear(); - sourceAudioElementRevokersRef.current.clear(); - sourceAudioElementResourcesRef.current.clear(); - if (sourceAudioMasterGainRef.current) { - sourceAudioMasterGainRef.current.disconnect(); - sourceAudioMasterGainRef.current = null; - } - const context = sourceAudioContextRef.current; - sourceAudioContextRef.current = null; - sourceAudioResumePromiseRef.current = null; - if (context) { - void context.close(); - } - lastSourceAudioSyncTimeRef.current = null; - }; - }, []); - - useEffect(() => { - const currentTimeMs = timelineTime * 1000; - const activeSpeedRegion = effectiveSpeedRegions.find( - (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; - - for (const track of resolvedUserTracks) { - const audio = audioElementsRef.current.get(track.id); - if (!audio) continue; - - const startMs = track.timelineBinding.startMs; - const endMs = track.timelineBinding.endMs; - const isInRegion = currentTimeMs >= startMs && currentTimeMs < endMs; - - if (isPlaying && isInRegion) { - enablePitchPreservingPlayback(audio); - const audioOffset = (currentTimeMs - startMs) / 1000; - if (Math.abs(audio.currentTime - audioOffset) > 0.2) { - audio.currentTime = audioOffset; - } - const syncedPlaybackRate = getMediaSyncPlaybackRate({ - basePlaybackRate: targetPlaybackRate, - currentTime: audio.currentTime, - targetTime: audioOffset, - }); - if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { - audio.playbackRate = syncedPlaybackRate; - } - if (audio.paused) { - audio.play().catch(() => undefined); - } - } else if (!audio.paused) { - audio.pause(); - } - } - }, [effectiveSpeedRegions, isPlaying, resolvedUserTracks, timelineTime]); - - useEffect(() => { - if (resolvedSourceTracks.length === 0) { - lastSourceAudioSyncTimeRef.current = null; - return; - } - - const activeSpeedRegion = effectiveSpeedRegions.find( - (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs, - ); - const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; - const previousTimelineTime = lastSourceAudioSyncTimeRef.current; - const timelineJumped = - previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25; - const driftThreshold = isPlaying - ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS - : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS; - if (sourceAudioMasterGainRef.current) { - sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted - ? 0 - : Math.max(0, Math.min(1, previewVolume)); - } - - for (const audio of sourceAudioElementsRef.current.values()) { - const sourceAudioPath = audio.dataset.sourceAudioPath ?? ""; - audio.volume = Math.max(0, Math.min(1, getSourceTrackPreviewGain(sourceAudioPath) * (isCurrentClipMuted ? 0 : previewVolume))); - - enablePitchPreservingPlayback(audio); - const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; - const isMicCompanionTrack = /\.mic\./i.test(sourceAudioPath); - const rawStartDelaySeconds = estimateCompanionAudioStartDelaySeconds( - duration, - audioDuration, - sourceAudioFallbackStartDelayMsByPath[sourceAudioPath], - ); - const maxPreviewStartDelaySeconds = isMicCompanionTrack ? 2 : 5; - const startDelaySeconds = isMicCompanionTrack - ? 0 - : Number.isFinite(duration) && - (rawStartDelaySeconds >= Math.max(0, duration - 0.01) || - rawStartDelaySeconds > Math.max(maxPreviewStartDelaySeconds, duration * 0.9)) - ? 0 - : rawStartDelaySeconds; - const beforeAudioStart = currentTime + 0.001 < startDelaySeconds; - const targetTime = clampMediaTimeToDuration(currentTime - startDelaySeconds, audioDuration); - - const shouldSeek = - timelineJumped || - (!isPlaying && Math.abs(audio.currentTime - targetTime) > driftThreshold) || - (isPlaying && Math.abs(audio.currentTime - targetTime) > 0.9); - if (shouldSeek) { - try { - audio.currentTime = targetTime; - } catch { - // no-op - } - } - - // KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter - // from continuous micro-corrections on system audio. - const syncedPlaybackRate = targetPlaybackRate; - if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { - audio.playbackRate = syncedPlaybackRate; - } - - const atEnd = audioDuration !== null && targetTime >= audioDuration; - if (isPlaying && !beforeAudioStart && !atEnd) { - void ensureSourceAudioRunning().then(() => { - audio.play().catch(() => undefined); - }); - } else if (!audio.paused) { - audio.pause(); - } - } - - lastSourceAudioSyncTimeRef.current = currentTime; - }, [ - currentTime, - duration, - effectiveSpeedRegions, - getSourceTrackPreviewGain, - isCurrentClipMuted, - isPlaying, - previewVolume, - resolvedSourceTracks, - sourceAudioFallbackStartDelayMsByPath, - ensureSourceAudioRunning, - ]); - - useEffect(() => { - if (!isPlaying || resolvedSourceTracks.length === 0) { - return; - } - void ensureSourceAudioRunning().then(() => { - for (const audio of sourceAudioElementsRef.current.values()) { - if (audio.paused) { - audio.play().catch(() => undefined); - } - } - }); - }, [isPlaying, resolvedSourceTracks.length, ensureSourceAudioRunning]); - - return { playSourceAudioPreview }; + const resolvedPlan = useMemo( + () => + buildResolvedAudioPlan({ + videoResource: null, + sourceAudioFallbackPaths: previewSourceAudioFallbackPaths, + audioRegions, + }), + [audioRegions, previewSourceAudioFallbackPaths], + ); + const resolvedUserTracks = useMemo( + () => resolvedPlan.tracks.filter((track) => track.kind === "user"), + [resolvedPlan], + ); + const resolvedSourceTracks = useMemo( + () => resolvedPlan.tracks.filter((track) => track.kind !== "user"), + [resolvedPlan], + ); + const [decodedSourceAudioLoadVersion, bumpDecodedSourceAudioLoadVersion] = useReducer( + (value: number) => value + 1, + 0, + ); + const [sourceAudioElementLoadVersion, bumpSourceAudioElementLoadVersion] = useReducer( + (value: number) => value + 1, + 0, + ); + + const audioElementsRef = useRef>(new Map()); + const audioElementRevokersRef = useRef void>>(new Map()); + const audioElementResourcesRef = useRef>(new Map()); + const sourceAudioElementsRef = useRef>(new Map()); + const sourceAudioMediaNodesRef = useRef>(new Map()); + const sourceAudioGainNodesRef = useRef>(new Map()); + const sourceAudioElementRevokersRef = useRef void>>(new Map()); + const sourceAudioElementResourcesRef = useRef>(new Map()); + const sourceAudioContextRef = useRef(null); + const sourceAudioMasterGainRef = useRef(null); + const sourceAudioResumePromiseRef = useRef | null>(null); + const decodedSourceAudioBuffersRef = useRef>( + new Map(), + ); + const decodedSourceAudioResourcesRef = useRef>(new Map()); + const decodedSourceAudioActiveNodesRef = useRef>( + new Map(), + ); + const decodedSourceAudioGainNodesRef = useRef>(new Map()); + const lastSourceAudioSyncTimeRef = useRef(null); + const onSourceFallbackLoadErrorRef = useRef(onSourceFallbackLoadError); + const getSourceTrackPreviewGainRef = useRef(getSourceTrackPreviewGain); + const previewVolumeRef = useRef(previewVolume); + const isCurrentClipMutedRef = useRef(isCurrentClipMuted); + onSourceFallbackLoadErrorRef.current = onSourceFallbackLoadError; + getSourceTrackPreviewGainRef.current = getSourceTrackPreviewGain; + previewVolumeRef.current = previewVolume; + isCurrentClipMutedRef.current = isCurrentClipMuted; + + const ensureSourceAudioContext = useCallback(() => { + if (!sourceAudioContextRef.current) { + const context = new AudioContext({ latencyHint: "interactive" }); + const masterGain = context.createGain(); + masterGain.gain.value = 1; + masterGain.connect(context.destination); + sourceAudioContextRef.current = context; + sourceAudioMasterGainRef.current = masterGain; + } + return sourceAudioContextRef.current; + }, []); + + const ensureSourceAudioRunning = useCallback(() => { + const context = ensureSourceAudioContext(); + if (context.state === "running") { + return Promise.resolve(); + } + if (!sourceAudioResumePromiseRef.current) { + sourceAudioResumePromiseRef.current = context + .resume() + .catch(() => undefined) + .finally(() => { + sourceAudioResumePromiseRef.current = null; + }); + } + return sourceAudioResumePromiseRef.current; + }, [ensureSourceAudioContext]); + + const stopDecodedSourceAudioPreview = useCallback((audioPath: string) => { + const active = decodedSourceAudioActiveNodesRef.current.get(audioPath); + if (!active) { + return; + } + + try { + active.source.stop(); + } catch { + // The source may already have ended. + } + active.source.disconnect(); + decodedSourceAudioActiveNodesRef.current.delete(audioPath); + }, []); + + const disconnectDecodedSourceAudioPreview = useCallback( + (audioPath: string) => { + stopDecodedSourceAudioPreview(audioPath); + decodedSourceAudioBuffersRef.current.delete(audioPath); + decodedSourceAudioResourcesRef.current.delete(audioPath); + const gainNode = decodedSourceAudioGainNodesRef.current.get(audioPath); + if (gainNode) { + gainNode.disconnect(); + decodedSourceAudioGainNodesRef.current.delete(audioPath); + } + }, + [stopDecodedSourceAudioPreview], + ); + + const playSourceAudioPreview = useCallback(() => { + void ensureSourceAudioRunning(); + }, [ensureSourceAudioRunning]); + + const startDecodedSourceAudioPreview = useCallback( + ({ + audioPath, + buffer, + targetTime, + playbackRate, + gain, + }: { + audioPath: string; + buffer: AudioBuffer; + targetTime: number; + playbackRate: number; + gain: number; + }) => { + stopDecodedSourceAudioPreview(audioPath); + + const context = ensureSourceAudioContext(); + if (targetTime >= buffer.duration) { + return; + } + + let gainNode = decodedSourceAudioGainNodesRef.current.get(audioPath); + if (!gainNode) { + gainNode = context.createGain(); + decodedSourceAudioGainNodesRef.current.set(audioPath, gainNode); + } + gainNode.disconnect(); + gainNode.gain.value = gain; + gainNode.connect(sourceAudioMasterGainRef.current ?? context.destination); + + const source = context.createBufferSource(); + source.buffer = buffer; + source.playbackRate.value = + Number.isFinite(playbackRate) && playbackRate > 0 ? playbackRate : 1; + source.connect(gainNode); + const offsetSeconds = Math.max( + 0, + Math.min(targetTime, Math.max(0, buffer.duration - 0.001)), + ); + source.start(0, offsetSeconds); + + const active: ActiveDecodedSourceAudio = { + source, + startedAtContextTime: context.currentTime, + offsetSeconds, + playbackRate: source.playbackRate.value, + }; + decodedSourceAudioActiveNodesRef.current.set(audioPath, active); + source.onended = () => { + if (decodedSourceAudioActiveNodesRef.current.get(audioPath) === active) { + decodedSourceAudioActiveNodesRef.current.delete(audioPath); + } + }; + }, + [ensureSourceAudioContext, stopDecodedSourceAudioPreview], + ); + + useEffect(() => { + const existing = audioElementsRef.current; + const currentIds = new Set(resolvedUserTracks.map((track) => track.id)); + + for (const [id, audio] of existing) { + if (!currentIds.has(id)) { + audio.pause(); + audio.src = ""; + audioElementRevokersRef.current.get(id)?.(); + audioElementRevokersRef.current.delete(id); + audioElementResourcesRef.current.delete(id); + existing.delete(id); + } + } + + for (const track of resolvedUserTracks) { + let audio = existing.get(track.id); + if (!audio) { + audio = new Audio(); + audio.preload = "auto"; + existing.set(track.id, audio); + } + + if (audioElementResourcesRef.current.get(track.id) !== track.sourceRef.path) { + audio.pause(); + audio.src = ""; + audioElementRevokersRef.current.get(track.id)?.(); + audioElementRevokersRef.current.delete(track.id); + audioElementResourcesRef.current.set(track.id, track.sourceRef.path); + + void (async () => { + const resolved = await resolveMediaElementSource(track.sourceRef.path); + const latestAudio = existing.get(track.id); + + if ( + latestAudio !== audio || + !isAudioResourceLoadCurrent( + audioElementResourcesRef.current, + track.id, + track.sourceRef.path, + ) + ) { + resolved.revoke(); + return; + } + + audioElementRevokersRef.current.set(track.id, resolved.revoke); + latestAudio.src = resolved.src; + })(); + } + + audio.volume = Math.max(0, Math.min(1, track.gain * previewVolume)); + } + }, [previewVolume, resolvedUserTracks]); + + useEffect(() => { + const existing = sourceAudioElementsRef.current; + const currentIds = new Set(resolvedSourceTracks.map((track) => track.sourceRef.path)); + const cleanupMediaElement = (id: string, audio: HTMLAudioElement) => { + audio.pause(); + audio.src = ""; + sourceAudioMediaNodesRef.current.get(id)?.disconnect(); + sourceAudioMediaNodesRef.current.delete(id); + sourceAudioGainNodesRef.current.get(id)?.disconnect(); + sourceAudioGainNodesRef.current.delete(id); + sourceAudioElementRevokersRef.current.get(id)?.(); + sourceAudioElementRevokersRef.current.delete(id); + sourceAudioElementResourcesRef.current.delete(id); + existing.delete(id); + }; + + for (const [id, audio] of existing) { + if (!currentIds.has(id)) { + cleanupMediaElement(id, audio); + } + } + + for (const id of Array.from(decodedSourceAudioResourcesRef.current.keys())) { + if (!currentIds.has(id)) { + disconnectDecodedSourceAudioPreview(id); + } + } + + for (const track of resolvedSourceTracks) { + const audioPath = track.sourceRef.path; + const mediaInfo = sourceAudioFallbackMediaInfoByPath[audioPath]; + const resourceKey = getSourceAudioElementResourceKey(audioPath, mediaInfo); + const useDecodedWavPreview = shouldUseDecodedWavSourcePreview(audioPath, mediaInfo); + + if (useDecodedWavPreview) { + const existingAudio = existing.get(audioPath); + if (existingAudio) { + cleanupMediaElement(audioPath, existingAudio); + } + + if (decodedSourceAudioResourcesRef.current.get(audioPath) !== resourceKey) { + stopDecodedSourceAudioPreview(audioPath); + decodedSourceAudioBuffersRef.current.delete(audioPath); + decodedSourceAudioResourcesRef.current.set(audioPath, resourceKey); + + void (async () => { + try { + const file = await createReadableMediaResourceFile(audioPath); + const arrayBuffer = await file.arrayBuffer(); + + if ( + !isAudioResourceLoadCurrent( + decodedSourceAudioResourcesRef.current, + audioPath, + resourceKey, + ) + ) { + return; + } + + const context = ensureSourceAudioContext(); + const buffer = await decodeSourceWavAudioBuffer(context, arrayBuffer); + if ( + !isAudioResourceLoadCurrent( + decodedSourceAudioResourcesRef.current, + audioPath, + resourceKey, + ) + ) { + return; + } + + decodedSourceAudioBuffersRef.current.set(audioPath, { + resourceKey, + buffer, + }); + bumpDecodedSourceAudioLoadVersion(); + } catch (error) { + if ( + !isAudioResourceLoadCurrent( + decodedSourceAudioResourcesRef.current, + audioPath, + resourceKey, + ) + ) { + return; + } + + disconnectDecodedSourceAudioPreview(audioPath); + onSourceFallbackLoadErrorRef.current(error); + } + })(); + } + + continue; + } + + disconnectDecodedSourceAudioPreview(audioPath); + let audio = existing.get(audioPath); + if (!audio) { + audio = new Audio(); + audio.preload = "auto"; + audio.crossOrigin = "anonymous"; + existing.set(audioPath, audio); + } + audio.volume = getSourceAudioPreviewVolume( + getSourceTrackPreviewGainRef.current(audioPath), + previewVolumeRef.current, + isCurrentClipMutedRef.current, + ); + audio.dataset.sourceAudioPath = audioPath; + + // Web Audio API createMediaElementSource breaks preservesPitch on Chromium. + // We route directly through the HTMLAudioElement to ensure pitch preservation works + // during speed changes. Note: this limits maximum preview volume to 1.0 (100%). + + if (sourceAudioElementResourcesRef.current.get(audioPath) !== resourceKey) { + audio.pause(); + audio.src = ""; + sourceAudioElementRevokersRef.current.get(audioPath)?.(); + sourceAudioElementRevokersRef.current.delete(audioPath); + sourceAudioElementResourcesRef.current.set(audioPath, resourceKey); + + void (async () => { + try { + const resolved = await resolveMediaElementSource(audioPath); + const latestAudio = existing.get(audioPath); + + if ( + latestAudio !== audio || + !isAudioResourceLoadCurrent( + sourceAudioElementResourcesRef.current, + audioPath, + resourceKey, + ) + ) { + resolved.revoke(); + return; + } + + sourceAudioElementRevokersRef.current.set(audioPath, resolved.revoke); + latestAudio.src = resolved.src; + latestAudio.volume = getSourceAudioPreviewVolume( + getSourceTrackPreviewGainRef.current(audioPath), + previewVolumeRef.current, + isCurrentClipMutedRef.current, + ); + latestAudio.load(); + bumpSourceAudioElementLoadVersion(); + } catch (error) { + const latestAudio = existing.get(audioPath); + if ( + latestAudio !== audio || + !isAudioResourceLoadCurrent( + sourceAudioElementResourcesRef.current, + audioPath, + resourceKey, + ) + ) { + return; + } + + sourceAudioElementRevokersRef.current.get(audioPath)?.(); + sourceAudioElementRevokersRef.current.delete(audioPath); + sourceAudioElementResourcesRef.current.delete(audioPath); + latestAudio.pause(); + latestAudio.src = ""; + onSourceFallbackLoadErrorRef.current(error); + } + })(); + } + } + + if (resolvedSourceTracks.length === 0) { + lastSourceAudioSyncTimeRef.current = null; + } + }, [ + resolvedSourceTracks, + disconnectDecodedSourceAudioPreview, + ensureSourceAudioContext, + stopDecodedSourceAudioPreview, + sourceAudioFallbackMediaInfoByPath, + ]); + + useEffect(() => { + for (const track of resolvedSourceTracks) { + const audioPath = track.sourceRef.path; + const gain = Math.max(0, Math.min(1, getSourceTrackPreviewGain(audioPath))); + const decodedGainNode = decodedSourceAudioGainNodesRef.current.get(audioPath); + if (decodedGainNode) { + decodedGainNode.gain.value = gain; + } + + const audio = sourceAudioElementsRef.current.get(audioPath); + if (audio) { + audio.volume = getSourceAudioPreviewVolume(gain, previewVolume, isCurrentClipMuted); + } + } + + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted + ? 0 + : Math.max(0, Math.min(1, previewVolume)); + } + }, [getSourceTrackPreviewGain, isCurrentClipMuted, previewVolume, resolvedSourceTracks]); + + useEffect(() => { + return () => { + for (const audio of audioElementsRef.current.values()) { + audio.pause(); + audio.src = ""; + } + for (const revoke of audioElementRevokersRef.current.values()) { + revoke(); + } + audioElementsRef.current.clear(); + audioElementRevokersRef.current.clear(); + audioElementResourcesRef.current.clear(); + for (const audio of sourceAudioElementsRef.current.values()) { + audio.pause(); + audio.src = ""; + } + for (const node of sourceAudioMediaNodesRef.current.values()) { + node.disconnect(); + } + for (const node of sourceAudioGainNodesRef.current.values()) { + node.disconnect(); + } + for (const revoke of sourceAudioElementRevokersRef.current.values()) { + revoke(); + } + sourceAudioElementsRef.current.clear(); + sourceAudioMediaNodesRef.current.clear(); + sourceAudioGainNodesRef.current.clear(); + sourceAudioElementRevokersRef.current.clear(); + sourceAudioElementResourcesRef.current.clear(); + for (const active of decodedSourceAudioActiveNodesRef.current.values()) { + try { + active.source.stop(); + } catch { + // The source may already have ended. + } + active.source.disconnect(); + } + for (const gainNode of decodedSourceAudioGainNodesRef.current.values()) { + gainNode.disconnect(); + } + decodedSourceAudioBuffersRef.current.clear(); + decodedSourceAudioResourcesRef.current.clear(); + decodedSourceAudioActiveNodesRef.current.clear(); + decodedSourceAudioGainNodesRef.current.clear(); + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.disconnect(); + sourceAudioMasterGainRef.current = null; + } + const context = sourceAudioContextRef.current; + sourceAudioContextRef.current = null; + sourceAudioResumePromiseRef.current = null; + if (context) { + void context.close(); + } + lastSourceAudioSyncTimeRef.current = null; + }; + }, []); + + useEffect(() => { + const currentTimeMs = timelineTime * 1000; + const activeSpeedRegion = effectiveSpeedRegions.find( + (region) => currentTimeMs >= region.startMs && currentTimeMs < region.endMs, + ); + const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + + for (const track of resolvedUserTracks) { + const audio = audioElementsRef.current.get(track.id); + if (!audio) continue; + + const startMs = track.timelineBinding.startMs; + const endMs = track.timelineBinding.endMs; + const isInRegion = currentTimeMs >= startMs && currentTimeMs < endMs; + + if (isPlaying && isInRegion) { + enablePitchPreservingPlayback(audio); + const audioOffset = (currentTimeMs - startMs) / 1000; + if (Math.abs(audio.currentTime - audioOffset) > 0.2) { + audio.currentTime = audioOffset; + } + const syncedPlaybackRate = getMediaSyncPlaybackRate({ + basePlaybackRate: targetPlaybackRate, + currentTime: audio.currentTime, + targetTime: audioOffset, + }); + if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { + audio.playbackRate = syncedPlaybackRate; + } + if (audio.paused) { + audio.play().catch(() => undefined); + } + } else if (!audio.paused) { + audio.pause(); + } + } + }, [effectiveSpeedRegions, isPlaying, resolvedUserTracks, timelineTime]); + + useEffect(() => { + // Re-sync when an async media element resource finishes loading. + void sourceAudioElementLoadVersion; + + // Re-sync when an async decoded WAV buffer finishes loading. + void decodedSourceAudioLoadVersion; + + if (resolvedSourceTracks.length === 0) { + lastSourceAudioSyncTimeRef.current = null; + return; + } + + const activeSpeedRegion = effectiveSpeedRegions.find( + (region) => currentTime * 1000 >= region.startMs && currentTime * 1000 < region.endMs, + ); + const targetPlaybackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; + const previousTimelineTime = lastSourceAudioSyncTimeRef.current; + const timelineJumped = + previousTimelineTime === null || Math.abs(currentTime - previousTimelineTime) > 0.25; + const driftThreshold = isPlaying + ? SOURCE_AUDIO_PREVIEW_PLAYING_SEEK_DRIFT_SECONDS + : SOURCE_AUDIO_PREVIEW_PAUSED_SEEK_DRIFT_SECONDS; + if (sourceAudioMasterGainRef.current) { + sourceAudioMasterGainRef.current.gain.value = isCurrentClipMuted + ? 0 + : Math.max(0, Math.min(1, previewVolume)); + } + + for (const track of resolvedSourceTracks) { + const sourceAudioPath = track.sourceRef.path; + const mediaInfo = sourceAudioFallbackMediaInfoByPath[sourceAudioPath]; + const probedAudioDurationSeconds = Number.isFinite(mediaInfo?.durationMs) + ? (mediaInfo?.durationMs ?? 0) / 1000 + : null; + const useDecodedWavPreview = shouldUseDecodedWavSourcePreview( + sourceAudioPath, + mediaInfo, + ); + + if (useDecodedWavPreview) { + const bufferEntry = decodedSourceAudioBuffersRef.current.get(sourceAudioPath); + const buffer = bufferEntry?.buffer ?? null; + const { beforeAudioStart, targetTime, atEnd } = resolveCompanionAudioPreviewTiming({ + currentTimeSeconds: currentTime, + timelineDurationSeconds: duration, + audioDurationSeconds: buffer?.duration ?? null, + probedAudioDurationSeconds, + recordedStartDelayMs: sourceAudioFallbackStartDelayMsByPath[sourceAudioPath], + }); + const gain = Math.max(0, Math.min(1, getSourceTrackPreviewGain(sourceAudioPath))); + const gainNode = decodedSourceAudioGainNodesRef.current.get(sourceAudioPath); + if (gainNode) { + gainNode.gain.value = gain; + } + + const active = + decodedSourceAudioActiveNodesRef.current.get(sourceAudioPath) ?? null; + const predictedTime = getDecodedSourcePredictedTime( + active, + sourceAudioContextRef.current?.currentTime ?? 0, + ); + const action = getDecodedSourcePreviewSyncAction({ + isPlaying, + beforeAudioStart, + atEnd, + hasBuffer: buffer !== null, + hasActiveSource: active !== null, + timelineJumped, + targetTime, + predictedTime, + playbackRate: targetPlaybackRate, + activePlaybackRate: active?.playbackRate ?? null, + }); + + if (action === "stop") { + stopDecodedSourceAudioPreview(sourceAudioPath); + } else if (buffer && (action === "start" || action === "restart")) { + void ensureSourceAudioRunning(); + startDecodedSourceAudioPreview({ + audioPath: sourceAudioPath, + buffer, + targetTime, + playbackRate: targetPlaybackRate, + gain, + }); + } + + continue; + } + + const audio = sourceAudioElementsRef.current.get(sourceAudioPath); + if (!audio) continue; + + audio.volume = Math.max( + 0, + Math.min( + 1, + getSourceTrackPreviewGain(sourceAudioPath) * + (isCurrentClipMuted ? 0 : previewVolume), + ), + ); + + enablePitchPreservingPlayback(audio); + const audioDuration = Number.isFinite(audio.duration) ? audio.duration : null; + const { beforeAudioStart, targetTime, atEnd } = resolveCompanionAudioPreviewTiming({ + currentTimeSeconds: currentTime, + timelineDurationSeconds: duration, + audioDurationSeconds: audioDuration, + probedAudioDurationSeconds, + recordedStartDelayMs: sourceAudioFallbackStartDelayMsByPath[sourceAudioPath], + }); + + const shouldSeek = + timelineJumped || + (!isPlaying && Math.abs(audio.currentTime - targetTime) > driftThreshold) || + (isPlaying && Math.abs(audio.currentTime - targetTime) > 0.9); + if (shouldSeek) { + try { + audio.currentTime = targetTime; + } catch { + // no-op + } + } + + // KISS for companion source tracks: fixed playback rate avoids audible flutter/stutter + // from continuous micro-corrections on system audio. + const syncedPlaybackRate = targetPlaybackRate; + if (Math.abs(audio.playbackRate - syncedPlaybackRate) > 0.001) { + audio.playbackRate = syncedPlaybackRate; + } + + syncSourceAudioElementPlayback( + audio, + shouldPlaySourceAudioElement({ isPlaying, beforeAudioStart, atEnd }), + ); + } + + lastSourceAudioSyncTimeRef.current = currentTime; + }, [ + currentTime, + decodedSourceAudioLoadVersion, + duration, + effectiveSpeedRegions, + getSourceTrackPreviewGain, + isCurrentClipMuted, + isPlaying, + previewVolume, + resolvedSourceTracks, + sourceAudioElementLoadVersion, + sourceAudioFallbackStartDelayMsByPath, + sourceAudioFallbackMediaInfoByPath, + ensureSourceAudioRunning, + startDecodedSourceAudioPreview, + stopDecodedSourceAudioPreview, + ]); + + return { playSourceAudioPreview }; } diff --git a/src/components/video-editor/audio/useSourceAudioFallback.ts b/src/components/video-editor/audio/useSourceAudioFallback.ts index bdaf05408..5f4d73d6e 100644 --- a/src/components/video-editor/audio/useSourceAudioFallback.ts +++ b/src/components/video-editor/audio/useSourceAudioFallback.ts @@ -1,5 +1,6 @@ import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; +import type { SourceAudioMediaInfo } from "@/components/video-editor/audio/audioTypes"; import { SOURCE_AUDIO_FALLBACK_TOAST_ID } from "@/components/video-editor/audio/audioTypes"; interface UseSourceAudioFallbackParams { @@ -16,6 +17,9 @@ export function useSourceAudioFallback({ const [sourceAudioFallbackPaths, setSourceAudioFallbackPaths] = useState([]); const [sourceAudioFallbackStartDelayMsByPath, setSourceAudioFallbackStartDelayMsByPath] = useState>({}); + const [sourceAudioFallbackMediaInfoByPath, setSourceAudioFallbackMediaInfoByPath] = useState< + Record + >({}); const previousSourcePathRef = useRef(null); useEffect(() => { @@ -27,6 +31,7 @@ export function useSourceAudioFallback({ if (sourceChanged) { setSourceAudioFallbackPaths([]); setSourceAudioFallbackStartDelayMsByPath({}); + setSourceAudioFallbackMediaInfoByPath({}); } if (!currentSourcePath) { @@ -45,6 +50,7 @@ export function useSourceAudioFallback({ if (sourceChanged) { setSourceAudioFallbackPaths([]); setSourceAudioFallbackStartDelayMsByPath({}); + setSourceAudioFallbackMediaInfoByPath({}); } toast.warning( result.error @@ -58,11 +64,13 @@ export function useSourceAudioFallback({ toast.dismiss(SOURCE_AUDIO_FALLBACK_TOAST_ID); setSourceAudioFallbackPaths(result.paths ?? []); setSourceAudioFallbackStartDelayMsByPath(result.startDelayMsByPath ?? {}); + setSourceAudioFallbackMediaInfoByPath(result.mediaInfoByPath ?? {}); } catch (error) { if (!cancelled) { if (sourceChanged) { setSourceAudioFallbackPaths([]); setSourceAudioFallbackStartDelayMsByPath({}); + setSourceAudioFallbackMediaInfoByPath({}); } toast.warning( `Could not load companion audio sources: ${summarizeErrorMessage(String(error))}`, @@ -77,5 +85,9 @@ export function useSourceAudioFallback({ }; }, [currentSourcePath, refreshKey, summarizeErrorMessage]); - return { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath }; + return { + sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + sourceAudioFallbackMediaInfoByPath, + }; } diff --git a/src/components/video-editor/audio/useVideoEditorAudio.ts b/src/components/video-editor/audio/useVideoEditorAudio.ts index 9d61f850a..801b27006 100644 --- a/src/components/video-editor/audio/useVideoEditorAudio.ts +++ b/src/components/video-editor/audio/useVideoEditorAudio.ts @@ -74,12 +74,15 @@ export function useVideoEditorAudio({ [currentSourcePath], ); - const { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath } = - useSourceAudioFallback({ - currentSourcePath: fallbackLookupSourcePath, - refreshKey: sourceAudioFallbackRefreshKey, - summarizeErrorMessage, - }); + const { + sourceAudioFallbackPaths, + sourceAudioFallbackStartDelayMsByPath, + sourceAudioFallbackMediaInfoByPath, + } = useSourceAudioFallback({ + currentSourcePath: fallbackLookupSourcePath, + refreshKey: sourceAudioFallbackRefreshKey, + summarizeErrorMessage, + }); const sourceTrackRoutingPolicy = useMemo( () => resolveSourceTrackRoutingPolicy(currentSourcePath, sourceAudioFallbackPaths), @@ -126,6 +129,7 @@ export function useVideoEditorAudio({ effectiveSpeedRegions, previewSourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioFallbackMediaInfoByPath, isCurrentClipMuted, getSourceTrackPreviewGain, onSourceFallbackLoadError, @@ -134,6 +138,7 @@ export function useVideoEditorAudio({ return { sourceAudioFallbackPaths, sourceAudioFallbackStartDelayMsByPath, + sourceAudioFallbackMediaInfoByPath, previewSourceAudioFallbackPaths, shouldMutePreviewVideo, activeClipIdAtCurrentTime, diff --git a/src/components/video-editor/audio/waveform/WaveformGenerator.ts b/src/components/video-editor/audio/waveform/WaveformGenerator.ts index 9dd6c69f3..e5a528d02 100644 --- a/src/components/video-editor/audio/waveform/WaveformGenerator.ts +++ b/src/components/video-editor/audio/waveform/WaveformGenerator.ts @@ -1,6 +1,7 @@ import WorkerConstructor from "./waveform.worker?worker"; import type { AudioPeaksData } from "../../timeline/core/timelineTypes"; import { WAVEFORM_DEFAULT_PEAK_COUNT } from "../../timeline/core/constants"; +import { decodeWavAudioData } from "./wavDecoder"; const MAX_WAVEFORM_PEAKS = 200_000; @@ -60,8 +61,12 @@ export class WaveformGenerator { }); } - public async generate(url: string, peakCount = WAVEFORM_DEFAULT_PEAK_COUNT): Promise { - const cacheKey = `${url}::${peakCount}`; + public async generate( + url: string, + peakCount = WAVEFORM_DEFAULT_PEAK_COUNT, + cacheKeyVersion?: string | number | null, + ): Promise { + const cacheKey = `${url}::${peakCount}::${cacheKeyVersion ?? ""}`; const cached = this.peaksCache.get(cacheKey); if (cached) return cached; @@ -75,7 +80,14 @@ export class WaveformGenerator { } const arrayBuffer = await response.arrayBuffer(); - const decoded = await this.audioContext.decodeAudioData(arrayBuffer); + const wavDecoded = decodeWavAudioData(arrayBuffer); + const decoded = wavDecoded + ? { + duration: wavDecoded.durationSeconds, + numberOfChannels: wavDecoded.channels.length, + getChannelData: (index: number) => wavDecoded.channels[index], + } + : await this.audioContext.decodeAudioData(arrayBuffer); const adaptivePeakCount = Math.max( peakCount, Math.floor(decoded.duration * 500) diff --git a/src/components/video-editor/audio/waveform/wavDecoder.test.ts b/src/components/video-editor/audio/waveform/wavDecoder.test.ts new file mode 100644 index 000000000..1aa32c818 --- /dev/null +++ b/src/components/video-editor/audio/waveform/wavDecoder.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from "vitest"; +import { decodeWavAudioData } from "./wavDecoder"; + +function writeAscii(view: DataView, offset: number, value: string) { + for (let i = 0; i < value.length; i++) { + view.setUint8(offset + i, value.charCodeAt(i)); + } +} + +function encodeSample(value: number) { + return Math.max(-32768, Math.min(32767, Math.round(value * 32767))); +} + +function createPcm16Wav({ sampleRate, channels }: { sampleRate: number; channels: number[][] }) { + const channelCount = channels.length; + const frameCount = channels[0]?.length ?? 0; + const dataSize = frameCount * channelCount * 2; + const junkSize = 3; + const junkPaddedSize = junkSize + 1; + const totalSize = 12 + 8 + junkPaddedSize + 8 + 16 + 8 + dataSize; + const buffer = new ArrayBuffer(totalSize); + const view = new DataView(buffer); + + writeAscii(view, 0, "RIFF"); + view.setUint32(4, totalSize - 8, true); + writeAscii(view, 8, "WAVE"); + + let offset = 12; + writeAscii(view, offset, "JUNK"); + view.setUint32(offset + 4, junkSize, true); + offset += 8 + junkPaddedSize; + + writeAscii(view, offset, "fmt "); + view.setUint32(offset + 4, 16, true); + view.setUint16(offset + 8, 1, true); + view.setUint16(offset + 10, channelCount, true); + view.setUint32(offset + 12, sampleRate, true); + view.setUint32(offset + 16, sampleRate * channelCount * 2, true); + view.setUint16(offset + 20, channelCount * 2, true); + view.setUint16(offset + 22, 16, true); + offset += 24; + + writeAscii(view, offset, "data"); + view.setUint32(offset + 4, dataSize, true); + offset += 8; + + for (let frame = 0; frame < frameCount; frame++) { + for (let channel = 0; channel < channelCount; channel++) { + view.setInt16(offset, encodeSample(channels[channel][frame] ?? 0), true); + offset += 2; + } + } + + return buffer; +} + +function createSingleFrameWav({ + audioFormat, + bitsPerSample, +}: { + audioFormat: number; + bitsPerSample: number; +}) { + const channelCount = 1; + const sampleRate = 48_000; + const bytesPerSample = bitsPerSample / 8; + const dataSize = channelCount * bytesPerSample; + const totalSize = 44 + dataSize; + const buffer = new ArrayBuffer(totalSize); + const view = new DataView(buffer); + + writeAscii(view, 0, "RIFF"); + view.setUint32(4, totalSize - 8, true); + writeAscii(view, 8, "WAVE"); + writeAscii(view, 12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, audioFormat, true); + view.setUint16(22, channelCount, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * dataSize, true); + view.setUint16(32, dataSize, true); + view.setUint16(34, bitsPerSample, true); + writeAscii(view, 36, "data"); + view.setUint32(40, dataSize, true); + + return buffer; +} + +describe("decodeWavAudioData", () => { + it("decodes PCM wav channels, sample rate, and duration without using media elements", () => { + const wav = createPcm16Wav({ + sampleRate: 4, + channels: [ + [0, 0.5, -0.5, 1], + [1, -1, 0.25, -0.25], + ], + }); + + const decoded = decodeWavAudioData(wav); + + expect(decoded?.sampleRate).toBe(4); + expect(decoded?.durationSeconds).toBe(1); + expect(decoded?.channels).toHaveLength(2); + expect(Array.from(decoded?.channels[0] ?? [])).toEqual([ + 0, + expect.closeTo(0.5, 4), + expect.closeTo(-0.5, 4), + expect.closeTo(1, 4), + ]); + expect(Array.from(decoded?.channels[1] ?? [])).toEqual([ + expect.closeTo(1, 4), + expect.closeTo(-1, 4), + expect.closeTo(0.25, 4), + expect.closeTo(-0.25, 4), + ]); + }); + + it("rejects unsupported 64-bit float WAV data instead of decoding silence", () => { + expect( + decodeWavAudioData(createSingleFrameWav({ audioFormat: 3, bitsPerSample: 64 })), + ).toBe(null); + }); + + it("rejects unsupported 64-bit PCM WAV data instead of decoding silence", () => { + expect( + decodeWavAudioData(createSingleFrameWav({ audioFormat: 1, bitsPerSample: 64 })), + ).toBe(null); + }); +}); diff --git a/src/components/video-editor/audio/waveform/wavDecoder.ts b/src/components/video-editor/audio/waveform/wavDecoder.ts new file mode 100644 index 000000000..0aad3f1e6 --- /dev/null +++ b/src/components/video-editor/audio/waveform/wavDecoder.ts @@ -0,0 +1,125 @@ +export type DecodedWavAudio = { + durationSeconds: number; + sampleRate: number; + channels: Float32Array[]; +}; + +function readAscii(view: DataView, offset: number, length: number) { + let value = ""; + for (let i = 0; i < length; i++) { + value += String.fromCharCode(view.getUint8(offset + i)); + } + return value; +} + +function decodePcmSample(view: DataView, offset: number, bitsPerSample: number) { + switch (bitsPerSample) { + case 8: + return (view.getUint8(offset) - 128) / 128; + case 16: + return view.getInt16(offset, true) / 32768; + case 24: { + const byte0 = view.getUint8(offset); + const byte1 = view.getUint8(offset + 1); + const byte2 = view.getUint8(offset + 2); + let sample = byte0 | (byte1 << 8) | (byte2 << 16); + if (sample & 0x800000) { + sample |= ~0xffffff; + } + return sample / 8388608; + } + case 32: + return view.getInt32(offset, true) / 2147483648; + default: + return 0; + } +} + +export function decodeWavAudioData(arrayBuffer: ArrayBuffer): DecodedWavAudio | null { + const view = new DataView(arrayBuffer); + if (view.byteLength < 44) { + return null; + } + + if (readAscii(view, 0, 4) !== "RIFF" || readAscii(view, 8, 4) !== "WAVE") { + return null; + } + + let audioFormat: number | null = null; + let channelCount: number | null = null; + let sampleRate: number | null = null; + let bitsPerSample: number | null = null; + let dataOffset = 0; + let dataSize = 0; + let offset = 12; + + while (offset + 8 <= view.byteLength) { + const chunkId = readAscii(view, offset, 4); + const chunkSize = view.getUint32(offset + 4, true); + const chunkDataOffset = offset + 8; + + if (chunkId === "fmt " && chunkDataOffset + 16 <= view.byteLength) { + audioFormat = view.getUint16(chunkDataOffset, true); + channelCount = view.getUint16(chunkDataOffset + 2, true); + sampleRate = view.getUint32(chunkDataOffset + 4, true); + bitsPerSample = view.getUint16(chunkDataOffset + 14, true); + } else if (chunkId === "data") { + dataOffset = chunkDataOffset; + dataSize = Math.min(chunkSize, view.byteLength - chunkDataOffset); + break; + } + + offset = chunkDataOffset + chunkSize + (chunkSize % 2); + } + + if ( + (audioFormat !== 1 && audioFormat !== 3) || + !channelCount || + !sampleRate || + !bitsPerSample || + dataSize <= 0 + ) { + return null; + } + + if (audioFormat === 3 && bitsPerSample !== 32) { + return null; + } + if (audioFormat === 1 && ![8, 16, 24, 32].includes(bitsPerSample)) { + return null; + } + + const bytesPerSample = bitsPerSample / 8; + if (!Number.isInteger(bytesPerSample) || bytesPerSample <= 0) { + return null; + } + + const frameSize = channelCount * bytesPerSample; + if (!Number.isFinite(frameSize) || frameSize <= 0) { + return null; + } + + const frameCount = Math.floor(dataSize / frameSize); + if (frameCount <= 0) { + return null; + } + + const channels = Array.from({ length: channelCount }, () => new Float32Array(frameCount)); + + for (let frameIndex = 0; frameIndex < frameCount; frameIndex++) { + const frameOffset = dataOffset + frameIndex * frameSize; + for (let channelIndex = 0; channelIndex < channelCount; channelIndex++) { + const sampleOffset = frameOffset + channelIndex * bytesPerSample; + channels[channelIndex][frameIndex] = + audioFormat === 3 + ? view.getFloat32(sampleOffset, true) + : decodePcmSample(view, sampleOffset, bitsPerSample); + } + } + + return { + durationSeconds: frameCount / sampleRate, + sampleRate, + channels, + }; +} diff --git a/src/components/video-editor/timeline/Item.tsx b/src/components/video-editor/timeline/Item.tsx index 5d335b8e8..fc8be7738 100644 --- a/src/components/video-editor/timeline/Item.tsx +++ b/src/components/video-editor/timeline/Item.tsx @@ -32,8 +32,10 @@ interface ItemProps { speedValue?: number; waveformPeaks?: AudioPeaksData | null; waveformSegmentSpan?: Span; + waveformDurationMs?: number | null; waveformGain?: number; waveformNormalize?: boolean; + waveformCoverage?: "full" | "partial" | "none"; muted?: boolean; variant?: "zoom" | "trim" | "clip" | "annotation" | "speed" | "audio"; isLoading?: boolean; @@ -73,8 +75,10 @@ export default function Item({ speedValue, waveformPeaks = null, waveformSegmentSpan, + waveformDurationMs = null, waveformGain = 1, waveformNormalize = false, + waveformCoverage = "none", muted = false, variant = "zoom", isLoading = false, @@ -200,6 +204,7 @@ export default function Item({ {showAudioWaveform && waveformPeaks && ( )} + {isAudio && waveformCoverage === "partial" && ( +
+ )} {/* Muted overlay for source audio track items */} {isAudio && muted && (
diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index 041bc3183..c9eb0b646 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -2,11 +2,13 @@ import { Plus } from "@phosphor-icons/react"; import type { Span } from "dnd-timeline"; import { forwardRef, useEffect, useMemo, useRef, useState } from "react"; import type { + SourceAudioMediaInfo, SourceAudioTrackMeta, SourceAudioTrackSettings, } from "@/components/video-editor/audio/audioTypes"; import { useScopedT } from "@/contexts/I18nContext"; import { useShortcuts } from "@/contexts/ShortcutsContext"; +import { resolveSourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy"; import { fromFileUrl } from "../projectPersistence"; import type { AnnotationRegion, @@ -25,10 +27,7 @@ import { calculateTimelineScale } from "./core/time"; import { useTimelineAudioPeaks } from "./hooks/useTimelineAudioPeaks"; import { useTimelineEditorRuntime } from "./hooks/useTimelineEditorRuntime"; import { useTimelineRange } from "./hooks/useTimelineRange"; -import { - buildSourceSidecarPathCandidates, - buildTimelineSourceAudioTracks, -} from "./sourceAudioTracks"; +import { buildTimelineSourceAudioTracks } from "./sourceAudioTracks"; export interface TimelineEditorProps { videoDuration: number; @@ -76,6 +75,8 @@ export interface TimelineEditorProps { sourceAudioTrackSettings?: SourceAudioTrackSettings; getSourceAudioTrackSettingsForClip?: (clipId: string | null) => SourceAudioTrackSettings; onSourceAudioTracksMetaChange?: (tracks: SourceAudioTrackMeta) => void; + sourceAudioFallbackPaths?: string[]; + sourceAudioFallbackMediaInfoByPath?: Record; } function extractLocalPathFromMediaServerUrl(input: string | null | undefined): string | null { @@ -150,6 +151,8 @@ const TimelineEditor = forwardRef( sourceAudioTrackSettings = {}, getSourceAudioTrackSettingsForClip, onSourceAudioTracksMetaChange, + sourceAudioFallbackPaths = [], + sourceAudioFallbackMediaInfoByPath = {}, }, ref, ) { @@ -236,48 +239,78 @@ const TimelineEditor = forwardRef( (/^file:\/\//i.test(videoPath) ? fromFileUrl(videoPath) : videoPath) ); }, [videoPath]); - const micSidecarPaths = useMemo( - () => (localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "mic") : []), - [localSourcePath], - ); - const micSidecarFallbackPaths = useMemo(() => micSidecarPaths.slice(1), [micSidecarPaths]); - const systemSidecarPaths = useMemo( - () => - localSourcePath ? buildSourceSidecarPathCandidates(localSourcePath, "system") : [], - [localSourcePath], - ); - const systemSidecarFallbackPaths = useMemo( - () => systemSidecarPaths.slice(1), - [systemSidecarPaths], + const sourceTrackRoutingPolicy = useMemo( + () => resolveSourceTrackRoutingPolicy(videoPath, sourceAudioFallbackPaths), + [videoPath, sourceAudioFallbackPaths], ); + const micSidecarPath = sourceTrackRoutingPolicy.pathsByTrack.mic ?? null; + const systemSidecarPath = sourceTrackRoutingPolicy.pathsByTrack.system ?? null; + const mixedSidecarPath = sourceTrackRoutingPolicy.pathsByTrack.mixed ?? null; const { peaks: micSidecarPeaks, loading: micSidecarLoading } = useTimelineAudioPeaks( - micSidecarPaths[0] ?? null, - { fallbackResources: micSidecarFallbackPaths }, + micSidecarPath, + { + cacheKeyVersion: micSidecarPath + ? sourceAudioFallbackMediaInfoByPath[micSidecarPath]?.durationMs + : null, + }, ); const { peaks: systemSidecarPeaks, loading: systemSidecarLoading } = useTimelineAudioPeaks( - systemSidecarPaths[0] ?? null, + systemSidecarPath, + { + cacheKeyVersion: systemSidecarPath + ? sourceAudioFallbackMediaInfoByPath[systemSidecarPath]?.durationMs + : null, + }, + ); + const { peaks: mixedSidecarPeaks, loading: mixedSidecarLoading } = useTimelineAudioPeaks( + mixedSidecarPath, { - fallbackResources: systemSidecarFallbackPaths, + cacheKeyVersion: mixedSidecarPath + ? sourceAudioFallbackMediaInfoByPath[mixedSidecarPath]?.durationMs + : null, }, ); const sourceAudioTracks = useMemo( () => buildTimelineSourceAudioTracks({ + routingPolicy: sourceTrackRoutingPolicy, + videoResource: localSourcePath, sourceAudioPeaks, micSidecarPeaks, systemSidecarPeaks, + mixedSidecarPeaks, + probedDurationMsByPath: Object.fromEntries( + Object.entries(sourceAudioFallbackMediaInfoByPath).map( + ([audioPath, info]) => [audioPath, info.durationMs], + ), + ), labels: { system: t("audio.systemLabel", "Source System"), mic: t("audio.micLabel", "Source Mic"), mixed: t("audio.mixedLabel", "Source"), }, }), - [micSidecarPeaks, sourceAudioPeaks, systemSidecarPeaks, t], + [ + localSourcePath, + micSidecarPeaks, + mixedSidecarPeaks, + sourceAudioPeaks, + sourceAudioFallbackMediaInfoByPath, + sourceTrackRoutingPolicy, + systemSidecarPeaks, + t, + ], ); const isLoading = useMemo(() => { // If we are still actively trying to load audio peaks (main or sidecars) - if (videoPath && (sourceAudioLoading || micSidecarLoading || systemSidecarLoading)) + if ( + videoPath && + (sourceAudioLoading || + micSidecarLoading || + systemSidecarLoading || + mixedSidecarLoading) + ) return true; // Robust telemetry loading detection: @@ -289,6 +322,7 @@ const TimelineEditor = forwardRef( videoPath, videoSourcePath, cursorTelemetrySourcePath, + mixedSidecarLoading, sourceAudioLoading, micSidecarLoading, systemSidecarLoading, diff --git a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx index 6fac793fe..14fe1d61b 100644 --- a/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx +++ b/src/components/video-editor/timeline/components/viewport/TimelineCanvas.tsx @@ -409,9 +409,11 @@ const TimelineCanvasRows = memo(function TimelineCanvasRows({ onSelect={() => onSelectClip?.(item.id)} variant="audio" waveformPeaks={track.peaks} + waveformDurationMs={track.probedDurationMs} waveformSegmentSpan={item.sourceSpan ?? item.span} waveformGain={Math.max(0, Math.min(1, settings.volume))} waveformNormalize={Boolean(settings.normalize)} + waveformCoverage={track.waveformCoverage} muted={item.muted} > {track.label} diff --git a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx index 1b1926518..043b5b9b1 100644 --- a/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx +++ b/src/components/video-editor/timeline/components/waveform/AudioWaveform.tsx @@ -4,6 +4,7 @@ import type { AudioPeaksData } from "../../core/timelineTypes"; interface AudioWaveformProps { peaks: AudioPeaksData; + audioDurationMs?: number | null; segmentStartMs?: number; segmentEndMs?: number; gain?: number; @@ -18,6 +19,7 @@ interface AudioWaveformProps { */ function AudioWaveformComponent({ peaks, + audioDurationMs, segmentStartMs, segmentEndMs, gain = 1, @@ -48,6 +50,7 @@ function AudioWaveformComponent({ const canvas = canvasRef.current; if (!canvas) return; let rafId = 0; + void resizeKey; const draw = () => { const now = performance.now(); @@ -72,35 +75,45 @@ function AudioWaveformComponent({ ctx.clearRect(0, 0, width, height); - const { peaks: peakData, durationMs } = peaks; - if (durationMs <= 0 || peakData.length === 0) return; + const { peaks: peakData, durationMs: peaksDurationMs } = peaks; + const effectiveDurationMs = + Number.isFinite(audioDurationMs) && (audioDurationMs ?? 0) > 0 + ? (audioDurationMs ?? 0) + : peaksDurationMs; + if (effectiveDurationMs <= 0 || peakData.length === 0) return; // Use raw values for smooth zooming/panning (no snapping) const visibleStartMs = segmentStartMs ?? range.start; const visibleEndMs = segmentEndMs ?? range.end; const visibleDurationMs = visibleEndMs - visibleStartMs; - + if (visibleDurationMs <= 0) return; const midY = height / 2; ctx.beginPath(); - + for (let px = 0; px < width; px++) { const t = visibleStartMs + (px / width) * visibleDurationMs; - - // If the timeline time is beyond the actual audio duration, we draw nothing (flat line) - if (t < 0 || t > durationMs) continue; - const exactIndex = (t / durationMs) * (peakData.length - 1); + // If the timeline time is beyond the actual audio duration, we draw nothing (flat line) + if (t < 0 || t > effectiveDurationMs) continue; + if (t > peaksDurationMs) { + const guideHeight = midY * 0.18; + ctx.moveTo(px, midY - guideHeight); + ctx.lineTo(px, midY + guideHeight); + continue; + } + + const exactIndex = (t / peaksDurationMs) * (peakData.length - 1); const leftIndex = Math.floor(exactIndex); const rightIndex = Math.min(peakData.length - 1, leftIndex + 1); const mix = exactIndex - leftIndex; - + let amplitude = peakData[leftIndex] * (1 - mix) + peakData[rightIndex] * mix; - + if (normalize) amplitude = Math.sqrt(Math.max(0, amplitude)); amplitude = Math.max(0, Math.min(1, amplitude * gain)); - + const barHeight = amplitude * midY * 0.85; ctx.moveTo(px, midY - barHeight); @@ -113,7 +126,17 @@ function AudioWaveformComponent({ }; rafId = requestAnimationFrame(draw); return () => cancelAnimationFrame(rafId); - }, [gain, normalize, peaks, range.start, range.end, resizeKey, segmentStartMs, segmentEndMs]); + }, [ + audioDurationMs, + gain, + normalize, + peaks, + range.start, + range.end, + resizeKey, + segmentStartMs, + segmentEndMs, + ]); return ( ({ + resolveMediaResourceUrl: vi.fn(async (resource: string) => resource), +})); + +vi.mock("../../audio/waveform/WaveformGenerator", () => ({ + waveformGenerator: { + generate: vi.fn(), + }, +})); + +import { getTimelineAudioPeaksCacheKey } from "./useTimelineAudioPeaks"; + +describe("getTimelineAudioPeaksCacheKey", () => { + it("changes when sidecar media info changes for the same resource", () => { + const resource = "C:\\Recordly\\recording.mic.wav"; + + expect(getTimelineAudioPeaksCacheKey(resource, 75_000)).not.toBe( + getTimelineAudioPeaksCacheKey(resource, 122_100), + ); + }); + + it("keeps the cache key stable when media info has not changed", () => { + const resource = "C:\\Recordly\\recording.mic.wav"; + + expect(getTimelineAudioPeaksCacheKey(resource, 122_100)).toBe( + getTimelineAudioPeaksCacheKey(resource, 122_100), + ); + }); +}); diff --git a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts index cf5ca3937..f202ed5b9 100644 --- a/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts +++ b/src/components/video-editor/timeline/hooks/useTimelineAudioPeaks.ts @@ -43,6 +43,7 @@ interface TimelineAudioPeaksOptions { enableSourceSidecarFallback?: boolean; fallbackResources?: string[]; peakCount?: number; + cacheKeyVersion?: string | number | null; } export interface TimelineAudioPeaksResult { @@ -50,6 +51,13 @@ export interface TimelineAudioPeaksResult { loading: boolean; } +export function getTimelineAudioPeaksCacheKey( + resource: string, + cacheKeyVersion?: string | number | null, +) { + return `${resource}::${cacheKeyVersion ?? ""}`; +} + export function useTimelineAudioPeaks( mediaResource: string | null | undefined, options: TimelineAudioPeaksOptions = {}, @@ -60,6 +68,7 @@ export function useTimelineAudioPeaks( const enableSourceSidecarFallback = options.enableSourceSidecarFallback ?? false; const fallbackResources = options.fallbackResources ?? EMPTY_FALLBACK_RESOURCES; const peakCount = options.peakCount ?? WAVEFORM_DEFAULT_PEAK_COUNT; + const cacheKeyVersion = options.cacheKeyVersion ?? null; useEffect(() => { sourceRef.current = mediaResource; @@ -75,7 +84,11 @@ export function useTimelineAudioPeaks( const run = async () => { const tryGenerate = async (resource: string): Promise => { const resolvedUrl = await resolveMediaResourceUrl(resource); - return waveformGenerator.generate(resolvedUrl, peakCount); + return waveformGenerator.generate( + resolvedUrl, + peakCount, + getTimelineAudioPeaksCacheKey(resource, cacheKeyVersion), + ); }; try { @@ -135,7 +148,7 @@ export function useTimelineAudioPeaks( return () => { cancelled = true; }; - }, [mediaResource, enableSourceSidecarFallback, fallbackResources, peakCount]); + }, [mediaResource, enableSourceSidecarFallback, fallbackResources, peakCount, cacheKeyVersion]); return { peaks, loading }; } diff --git a/src/components/video-editor/timeline/sourceAudioTracks.test.ts b/src/components/video-editor/timeline/sourceAudioTracks.test.ts index f7c809e64..eeefc5c97 100644 --- a/src/components/video-editor/timeline/sourceAudioTracks.test.ts +++ b/src/components/video-editor/timeline/sourceAudioTracks.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import type { SourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy"; import type { AudioPeaksData } from "./core/timelineTypes"; import { buildSourceSidecarPathCandidates, @@ -18,6 +19,17 @@ const labels = { mixed: "Source", }; +function policy(overrides: Partial = {}): SourceTrackRoutingPolicy { + return { + hasEmbeddedSourceAudio: false, + pathsByTrack: {}, + playbackPaths: [], + muteEmbeddedPreview: false, + includeEmbeddedInExport: false, + ...overrides, + }; +} + describe("timeline source audio tracks", () => { it("builds candidates for Windows and macOS sidecar containers", () => { expect(buildSourceSidecarPathCandidates("C:\\Recordly\\recording-1.mp4", "mic")).toEqual([ @@ -33,45 +45,119 @@ describe("timeline source audio tracks", () => { expect( buildTimelineSourceAudioTracks({ + routingPolicy: policy({ + hasEmbeddedSourceAudio: true, + pathsByTrack: { mic: "/tmp/recording.mic.wav" }, + playbackPaths: ["/tmp/recording.mic.wav"], + includeEmbeddedInExport: true, + }), + videoResource: "/tmp/recording.mp4", sourceAudioPeaks: source, micSidecarPeaks: mic, systemSidecarPeaks: null, + mixedSidecarPeaks: null, labels, }), ).toEqual([ - { id: "system", label: "Source System", peaks: source }, - { id: "mic", label: "Source Mic", peaks: mic }, + { + id: "system", + label: "Source System", + kind: "embedded", + resourcePath: "/tmp/recording.mp4", + peaks: source, + probedDurationMs: null, + waveformAvailable: true, + waveformCoverage: "full", + }, + { + id: "mic", + label: "Source Mic", + kind: "mic", + resourcePath: "/tmp/recording.mic.wav", + peaks: mic, + probedDurationMs: null, + waveformAvailable: true, + waveformCoverage: "full", + }, ]); }); - it("does not invent a system track when only the mic sidecar exists", () => { + it("does not invent an embedded system track when the source has no embedded audio", () => { const mic = peaks(2); expect( buildTimelineSourceAudioTracks({ + routingPolicy: policy({ + pathsByTrack: { mic: "/tmp/recording.mic.wav" }, + playbackPaths: ["/tmp/recording.mic.wav"], + }), + videoResource: "/tmp/recording.mp4", sourceAudioPeaks: null, micSidecarPeaks: mic, systemSidecarPeaks: null, + mixedSidecarPeaks: null, + probedDurationMsByPath: { + "/tmp/recording.mic.wav": 147_360, + }, labels, }), - ).toEqual([{ id: "mic", label: "Source Mic", peaks: mic }]); + ).toEqual([ + { + id: "mic", + label: "Source Mic", + kind: "mic", + resourcePath: "/tmp/recording.mic.wav", + peaks: mic, + probedDurationMs: 147_360, + waveformAvailable: true, + waveformCoverage: "partial", + }, + ]); }); it("uses dedicated sidecars over the embedded track when both source tracks exist", () => { - const source = peaks(1); const system = peaks(2); const mic = peaks(3); expect( buildTimelineSourceAudioTracks({ - sourceAudioPeaks: source, + routingPolicy: policy({ + hasEmbeddedSourceAudio: true, + pathsByTrack: { + system: "/tmp/recording.system.wav", + mic: "/tmp/recording.mic.wav", + }, + playbackPaths: ["/tmp/recording.system.wav", "/tmp/recording.mic.wav"], + muteEmbeddedPreview: true, + }), + videoResource: "/tmp/recording.mp4", + sourceAudioPeaks: null, micSidecarPeaks: mic, systemSidecarPeaks: system, + mixedSidecarPeaks: null, labels, }), ).toEqual([ - { id: "system", label: "Source System", peaks: system }, - { id: "mic", label: "Source Mic", peaks: mic }, + { + id: "system", + label: "Source System", + kind: "system", + resourcePath: "/tmp/recording.system.wav", + peaks: system, + probedDurationMs: null, + waveformAvailable: true, + waveformCoverage: "full", + }, + { + id: "mic", + label: "Source Mic", + kind: "mic", + resourcePath: "/tmp/recording.mic.wav", + peaks: mic, + probedDurationMs: null, + waveformAvailable: true, + waveformCoverage: "full", + }, ]); }); @@ -80,11 +166,106 @@ describe("timeline source audio tracks", () => { expect( buildTimelineSourceAudioTracks({ + routingPolicy: policy({ + hasEmbeddedSourceAudio: true, + }), + videoResource: "/tmp/recording.mp4", sourceAudioPeaks: source, micSidecarPeaks: null, systemSidecarPeaks: null, + mixedSidecarPeaks: null, + labels, + }), + ).toEqual([ + { + id: "mixed", + label: "Source", + kind: "embedded", + resourcePath: "/tmp/recording.mp4", + peaks: source, + probedDurationMs: null, + waveformAvailable: true, + waveformCoverage: "full", + }, + ]); + }); + + it("keeps visible rows even when waveform peaks have not loaded yet", () => { + expect( + buildTimelineSourceAudioTracks({ + routingPolicy: policy({ + hasEmbeddedSourceAudio: true, + pathsByTrack: { mic: "/tmp/recording.mic.wav" }, + playbackPaths: ["/tmp/recording.mic.wav"], + includeEmbeddedInExport: true, + }), + videoResource: "/tmp/recording.mp4", + sourceAudioPeaks: null, + micSidecarPeaks: null, + systemSidecarPeaks: null, + mixedSidecarPeaks: null, + probedDurationMsByPath: { + "/tmp/recording.mic.wav": 147_360, + }, labels, }), - ).toEqual([{ id: "mixed", label: "Source", peaks: source }]); + ).toEqual([ + { + id: "system", + label: "Source System", + kind: "embedded", + resourcePath: "/tmp/recording.mp4", + peaks: null, + probedDurationMs: null, + waveformAvailable: false, + waveformCoverage: "none", + }, + { + id: "mic", + label: "Source Mic", + kind: "mic", + resourcePath: "/tmp/recording.mic.wav", + peaks: null, + probedDurationMs: 147_360, + waveformAvailable: false, + waveformCoverage: "none", + }, + ]); + }); + + it("marks mic waveforms as partial when peaks end before the probed sidecar duration", () => { + const mic = { + durationMs: 72_000, + peaks: new Float32Array([0.25, 0.5, 0.75]), + } satisfies AudioPeaksData; + + expect( + buildTimelineSourceAudioTracks({ + routingPolicy: policy({ + pathsByTrack: { mic: "/tmp/recording.mic.wav" }, + playbackPaths: ["/tmp/recording.mic.wav"], + }), + videoResource: "/tmp/recording.mp4", + sourceAudioPeaks: null, + micSidecarPeaks: mic, + systemSidecarPeaks: null, + mixedSidecarPeaks: null, + probedDurationMsByPath: { + "/tmp/recording.mic.wav": 147_360, + }, + labels, + }), + ).toEqual([ + { + id: "mic", + label: "Source Mic", + kind: "mic", + resourcePath: "/tmp/recording.mic.wav", + peaks: mic, + probedDurationMs: 147_360, + waveformAvailable: true, + waveformCoverage: "partial", + }, + ]); }); }); diff --git a/src/components/video-editor/timeline/sourceAudioTracks.ts b/src/components/video-editor/timeline/sourceAudioTracks.ts index 46a089ee8..414ae8773 100644 --- a/src/components/video-editor/timeline/sourceAudioTracks.ts +++ b/src/components/video-editor/timeline/sourceAudioTracks.ts @@ -1,4 +1,5 @@ import type { SourceAudioTrackWithPeaks } from "@/components/video-editor/audio/audioTypes"; +import type { SourceTrackRoutingPolicy } from "@/lib/exporter/sourceTrackRoutingPolicy"; import type { AudioPeaksData } from "./core/timelineTypes"; const SOURCE_SIDECAR_EXTENSIONS = [".wav", ".m4a", ".webm"] as const; @@ -17,51 +18,115 @@ export function buildSourceSidecarPathCandidates( } export function buildTimelineSourceAudioTracks({ + routingPolicy, + videoResource, sourceAudioPeaks, micSidecarPeaks, systemSidecarPeaks, + mixedSidecarPeaks, + probedDurationMsByPath = {}, labels, }: { + routingPolicy: SourceTrackRoutingPolicy; + videoResource: string | null; sourceAudioPeaks: AudioPeaksData | null; micSidecarPeaks: AudioPeaksData | null; systemSidecarPeaks: AudioPeaksData | null; + mixedSidecarPeaks: AudioPeaksData | null; + probedDurationMsByPath?: Record; labels: { system: string; mic: string; mixed: string; }; }): SourceAudioTrackWithPeaks[] { - if (systemSidecarPeaks || micSidecarPeaks) { + const getProbedDurationMs = (resourcePath: string | null) => { + if (!resourcePath) return null; + const durationMs = probedDurationMsByPath[resourcePath]; + return Number.isFinite(durationMs) && durationMs > 0 ? Math.round(durationMs) : null; + }; + const getWaveformCoverage = ( + peaks: AudioPeaksData | null, + probedDurationMs: number | null, + ): "full" | "partial" | "none" => { + if (!peaks) return "none"; + const durationMs = probedDurationMs ?? Number.NaN; + if (!Number.isFinite(durationMs)) return "full"; + return peaks.durationMs + 250 < durationMs ? "partial" : "full"; + }; + const systemResourcePath = routingPolicy.pathsByTrack.system ?? null; + const micResourcePath = routingPolicy.pathsByTrack.mic ?? null; + const mixedResourcePath = routingPolicy.pathsByTrack.mixed ?? null; + + if (systemResourcePath || micResourcePath) { const tracks: SourceAudioTrackWithPeaks[] = []; - if (systemSidecarPeaks) { + if (systemResourcePath) { + const probedDurationMs = getProbedDurationMs(systemResourcePath); tracks.push({ id: "system", label: labels.system, + kind: "system", + resourcePath: systemResourcePath, peaks: systemSidecarPeaks, + probedDurationMs, + waveformAvailable: systemSidecarPeaks !== null, + waveformCoverage: getWaveformCoverage(systemSidecarPeaks, probedDurationMs), }); - } else if (micSidecarPeaks && sourceAudioPeaks) { + } else if (routingPolicy.hasEmbeddedSourceAudio && routingPolicy.includeEmbeddedInExport) { tracks.push({ id: "system", label: labels.system, + kind: "embedded", + resourcePath: videoResource, peaks: sourceAudioPeaks, + probedDurationMs: null, + waveformAvailable: sourceAudioPeaks !== null, + waveformCoverage: getWaveformCoverage(sourceAudioPeaks, null), }); } - if (micSidecarPeaks) { + if (micResourcePath) { + const probedDurationMs = getProbedDurationMs(micResourcePath); tracks.push({ id: "mic", label: labels.mic, + kind: "mic", + resourcePath: micResourcePath, peaks: micSidecarPeaks, + probedDurationMs, + waveformAvailable: micSidecarPeaks !== null, + waveformCoverage: getWaveformCoverage(micSidecarPeaks, probedDurationMs), }); } return tracks; } - return sourceAudioPeaks + if (mixedResourcePath) { + const probedDurationMs = getProbedDurationMs(mixedResourcePath); + return [ + { + id: "mixed", + label: labels.mixed, + kind: "mixed", + resourcePath: mixedResourcePath, + peaks: mixedSidecarPeaks, + probedDurationMs, + waveformAvailable: mixedSidecarPeaks !== null, + waveformCoverage: getWaveformCoverage(mixedSidecarPeaks, probedDurationMs), + }, + ]; + } + + return (routingPolicy.hasEmbeddedSourceAudio || sourceAudioPeaks !== null) && videoResource ? [ { id: "mixed", label: labels.mixed, + kind: "embedded", + resourcePath: videoResource, peaks: sourceAudioPeaks, + probedDurationMs: null, + waveformAvailable: sourceAudioPeaks !== null, + waveformCoverage: getWaveformCoverage(sourceAudioPeaks, null), }, ] : []; diff --git a/src/lib/mediaTiming.test.ts b/src/lib/mediaTiming.test.ts index 4d6716ed2..9f34e6cfe 100644 --- a/src/lib/mediaTiming.test.ts +++ b/src/lib/mediaTiming.test.ts @@ -4,9 +4,11 @@ import { clampMediaTimeToDuration, enablePitchPreservingPlayback, estimateCompanionAudioStartDelaySeconds, + getCompanionAudioEndToleranceSeconds, getEffectiveRecordingDurationMs, getEffectiveVideoStreamDurationSeconds, getMediaSyncPlaybackRate, + resolveCompanionAudioPreviewTiming, } from "./mediaTiming"; describe("clampMediaTimeToDuration", () => { @@ -39,6 +41,72 @@ describe("estimateCompanionAudioStartDelaySeconds", () => { }); }); +describe("getCompanionAudioEndToleranceSeconds", () => { + it("uses a small default tail tolerance when durations already line up", () => { + expect( + getCompanionAudioEndToleranceSeconds({ + timelineDuration: 96.4, + audioDuration: 96.24, + recordedStartDelayMs: 134, + }), + ).toBeCloseTo(0.376, 2); + }); + + it("absorbs multi-second companion duration mismatches instead of ending immediately", () => { + expect( + getCompanionAudioEndToleranceSeconds({ + timelineDuration: 96.4, + audioDuration: 93, + recordedStartDelayMs: 134, + }), + ).toBeCloseTo(3.616, 2); + }); +}); + +describe("resolveCompanionAudioPreviewTiming", () => { + it("uses recorded microphone start delay instead of forcing mic preview to zero", () => { + expect( + resolveCompanionAudioPreviewTiming({ + currentTimeSeconds: 0.1, + timelineDurationSeconds: 96.4, + audioDurationSeconds: 96.24, + recordedStartDelayMs: 134, + }), + ).toMatchObject({ + startDelaySeconds: 0.134, + beforeAudioStart: true, + atEnd: false, + }); + }); + + it("does not mark a shorter-than-expected companion track as ended until its tail tolerance is exhausted", () => { + const result = resolveCompanionAudioPreviewTiming({ + currentTimeSeconds: 95.5, + timelineDurationSeconds: 96.4, + audioDurationSeconds: 93, + recordedStartDelayMs: 134, + }); + + expect(result.startDelaySeconds).toBeCloseTo(0.134, 2); + expect(result.targetTime).toBe(93); + expect(result.atEnd).toBe(false); + expect(result.endToleranceSeconds).toBeGreaterThan(3); + }); + + it("prefers a probed sidecar duration over a shorter browser media duration near the tail", () => { + const result = resolveCompanionAudioPreviewTiming({ + currentTimeSeconds: 146.8, + timelineDurationSeconds: 147.539, + audioDurationSeconds: 72, + recordedStartDelayMs: 143, + probedAudioDurationSeconds: 147.36, + }); + + expect(result.targetTime).toBeCloseTo(146.657, 2); + expect(result.atEnd).toBe(false); + }); +}); + describe("getEffectiveRecordingDurationMs", () => { it("subtracts accumulated paused time", () => { expect( diff --git a/src/lib/mediaTiming.ts b/src/lib/mediaTiming.ts index 38467098d..f44de4b70 100644 --- a/src/lib/mediaTiming.ts +++ b/src/lib/mediaTiming.ts @@ -9,6 +9,8 @@ export function clampMediaTimeToDuration(targetTime: number, duration?: number | const MIN_COMPANION_AUDIO_DELAY_SECONDS = 0.025; const MAX_INFERRED_COMPANION_AUDIO_DELAY_SECONDS = 0.5; +const DEFAULT_COMPANION_AUDIO_END_TOLERANCE_SECONDS = 0.35; +const MAX_COMPANION_AUDIO_END_TOLERANCE_SECONDS = 5; export function estimateCompanionAudioStartDelaySeconds( timelineDuration?: number | null, @@ -37,6 +39,83 @@ export function estimateCompanionAudioStartDelaySeconds( return estimatedDelaySeconds; } +export function getCompanionAudioEndToleranceSeconds({ + timelineDuration, + audioDuration, + recordedStartDelayMs, +}: { + timelineDuration?: number | null; + audioDuration?: number | null; + recordedStartDelayMs?: number | null; +}): number { + if (!Number.isFinite(timelineDuration) || !Number.isFinite(audioDuration)) { + return DEFAULT_COMPANION_AUDIO_END_TOLERANCE_SECONDS; + } + + const startDelaySeconds = estimateCompanionAudioStartDelaySeconds( + timelineDuration, + audioDuration, + recordedStartDelayMs, + ); + const expectedTailGapSeconds = Math.max( + 0, + Math.max(0, timelineDuration ?? 0) - startDelaySeconds - Math.max(0, audioDuration ?? 0), + ); + + return Math.min( + MAX_COMPANION_AUDIO_END_TOLERANCE_SECONDS, + Math.max( + DEFAULT_COMPANION_AUDIO_END_TOLERANCE_SECONDS, + expectedTailGapSeconds + DEFAULT_COMPANION_AUDIO_END_TOLERANCE_SECONDS, + ), + ); +} + +export function resolveCompanionAudioPreviewTiming({ + currentTimeSeconds, + timelineDurationSeconds, + audioDurationSeconds, + probedAudioDurationSeconds, + recordedStartDelayMs, +}: { + currentTimeSeconds: number; + timelineDurationSeconds?: number | null; + audioDurationSeconds?: number | null; + probedAudioDurationSeconds?: number | null; + recordedStartDelayMs?: number | null; +}) { + const effectiveAudioDurationSeconds = + Number.isFinite(probedAudioDurationSeconds) && (probedAudioDurationSeconds ?? 0) > 0 + ? Math.max(0, probedAudioDurationSeconds ?? 0) + : audioDurationSeconds; + const startDelaySeconds = estimateCompanionAudioStartDelaySeconds( + timelineDurationSeconds, + effectiveAudioDurationSeconds, + recordedStartDelayMs, + ); + const beforeAudioStart = currentTimeSeconds + 0.001 < startDelaySeconds; + const rawTargetTime = Math.max(0, currentTimeSeconds - startDelaySeconds); + const targetTime = clampMediaTimeToDuration(rawTargetTime, effectiveAudioDurationSeconds); + const endToleranceSeconds = getCompanionAudioEndToleranceSeconds({ + timelineDuration: timelineDurationSeconds, + audioDuration: effectiveAudioDurationSeconds, + recordedStartDelayMs, + }); + const atEnd = + Number.isFinite(effectiveAudioDurationSeconds) && + effectiveAudioDurationSeconds !== null && + rawTargetTime >= Math.max(0, effectiveAudioDurationSeconds ?? 0) + endToleranceSeconds; + + return { + startDelaySeconds, + beforeAudioStart, + rawTargetTime, + targetTime, + endToleranceSeconds, + atEnd, + }; +} + export function getMediaSyncPlaybackRate({ basePlaybackRate, currentTime,