Include extension click sounds in MP4 exports

EXTENSIONS.md

@@ -215,6 +215,10 @@ api.playSound("sounds/click.mp3", { volume: 0.8 });
 api.log("hello", payload);
 ```
 
+Sounds played by export-time cursor interaction handlers, such as `cursor:click`, are
+included in MP4 exports. Sounds played outside that capture window are not added
+automatically.
+
 ### Read-only Queries
 
 ```js

src/components/video-editor/VideoEditor.tsx

@@ -138,6 +138,7 @@ import {
 	saveEditorPresets,
 	serializeEditorPresetSnapshot,
 } from "./editorPreferences";
+import { collectExtensionAudioRegionsForExport } from "./extensionExportAudio";
 import ProjectBrowserDialog, { type ProjectLibraryEntry } from "./ProjectBrowserDialog";
 import { hasUnsavedProjectChanges } from "./projectDirtyState";
 import {
@@ -3347,17 +3348,17 @@ export default function VideoEditor() {
 	const handlePreviewSkipBack = useCallback(() => {
 		const currentMs = timelinePlayheadTime * 1000;
 		const keyframes = timelineRef.current?.keyframes ?? [];
-		const previous = [...keyframes].reverse().find((keyframe) => keyframe.time < currentMs - 50);
+		const previous = [...keyframes]
+			.reverse()
+			.find((keyframe) => keyframe.time < currentMs - 50);
 		handleSeek(previous ? previous.time / 1000 : Math.max(0, timelinePlayheadTime - 5));
 	}, [handleSeek, timelinePlayheadTime]);
 
 	const handlePreviewSkipForward = useCallback(() => {
 		const currentMs = timelinePlayheadTime * 1000;
 		const keyframes = timelineRef.current?.keyframes ?? [];
 		const next = keyframes.find((keyframe) => keyframe.time > currentMs + 50);
-		handleSeek(
-			next ? next.time / 1000 : Math.min(timelineDuration, timelinePlayheadTime + 5),
-		);
+		handleSeek(next ? next.time / 1000 : Math.min(timelineDuration, timelinePlayheadTime + 5));
 	}, [handleSeek, timelineDuration, timelinePlayheadTime]);
 
 	const handleSelectZoom = useCallback((id: string | null) => {
@@ -4356,6 +4357,16 @@ export default function VideoEditor() {
 						selectedClipId !== null
 							? audio.selectedClipSourceAudioTrackSettings
 							: audio.activeSourceAudioTrackSettings;
+					const extensionAudioRegions = collectExtensionAudioRegionsForExport(
+						extensionHost,
+						effectiveCursorTelemetry,
+					);
+					const audioRegionsForExport =
+						extensionAudioRegions.length > 0
+							? [...audioRegions, ...extensionAudioRegions].sort(
+									(a, b) => a.startMs - b.startMs || a.id.localeCompare(b.id),
+								)
+							: audioRegions;
 
 					const exporterConfig = {
 						videoUrl: videoPath,
@@ -4426,7 +4437,7 @@ export default function VideoEditor() {
 						cursorClickBounceDuration,
 						cursorSway,
 						frame,
-						audioRegions,
+						audioRegions: audioRegionsForExport,
 						clipRegions,
 						sourceAudioFallbackPaths: audio.sourceAudioFallbackPaths,
 						sourceAudioFallbackStartDelayMsByPath:
@@ -5148,10 +5159,7 @@ export default function VideoEditor() {
 			volume={
 				audio.shouldMutePreviewVideo || audio.isCurrentClipMuted
 					? 0
-					: Math.max(
-							0,
-							Math.min(1, previewVolume * audio.embeddedSourcePreviewGain),
-						)
+					: Math.max(0, Math.min(1, previewVolume * audio.embeddedSourcePreviewGain))
 			}
 			suspendRendering={suspendRendering}
 		/>

src/components/video-editor/extensionExportAudio.test.ts

@@ -0,0 +1,127 @@
+import { describe, expect, it, vi } from "vitest";
+import type { ExtensionExportAudioCue } from "@/lib/extensions/extensionHost";
+import {
+	collectExtensionAudioRegionsForExport,
+	extensionAudioCuesToRegions,
+	isExportableCursorInteraction,
+} from "./extensionExportAudio";
+import type { CursorTelemetryPoint } from "./types";
+
+describe("extension export audio", () => {
+	it("captures extension sounds for cursor interactions that preview treats as clicks", () => {
+		const cues: ExtensionExportAudioCue[] = [];
+		let activeTimeMs = 0;
+		const host = {
+			beginExportAudioCapture: vi.fn(),
+			setExportAudioCaptureTime: vi.fn((timeMs: number) => {
+				activeTimeMs = timeMs;
+			}),
+			emitEvent: vi.fn((event: { type: string }) => {
+				if (event.type === "cursor:click") {
+					cues.push({
+						id: `cue-${cues.length}`,
+						extensionId: "com.test.clicks",
+						timeMs: activeTimeMs,
+						audioPath: `file:///sounds/click-${cues.length}.mp3`,
+						volume: 0.8,
+					});
+				}
+			}),
+			finishExportAudioCapture: vi.fn(() => cues),
+			cancelExportAudioCapture: vi.fn(),
+		};
+		const telemetry: CursorTelemetryPoint[] = [
+			{ timeMs: 100, cx: 0.1, cy: 0.2, interactionType: "move" },
+			{ timeMs: 250, cx: 0.3, cy: 0.4, interactionType: "click" },
+			{ timeMs: 400, cx: 0.5, cy: 0.6, interactionType: "mouseup" },
+			{ timeMs: 550, cx: 0.7, cy: 0.8 },
+		];
+
+		const regions = collectExtensionAudioRegionsForExport(host, telemetry, 300);
+
+		expect(host.beginExportAudioCapture).toHaveBeenCalledTimes(1);
+		expect(host.setExportAudioCaptureTime).toHaveBeenNthCalledWith(1, 250);
+		expect(host.setExportAudioCaptureTime).toHaveBeenNthCalledWith(2, 400);
+		expect(host.emitEvent).toHaveBeenCalledTimes(2);
+		expect(host.emitEvent).toHaveBeenNthCalledWith(1, {
+			type: "cursor:click",
+			timeMs: 250,
+			data: { cx: 0.3, cy: 0.4, interactionType: "click" },
+		});
+		expect(host.cancelExportAudioCapture).not.toHaveBeenCalled();
+		expect(regions).toEqual([
+			{
+				id: "extension-audio-cue-0",
+				startMs: 250,
+				endMs: 550,
+				audioPath: "file:///sounds/click-0.mp3",
+				volume: 0.8,
+				normalize: false,
+			},
+			{
+				id: "extension-audio-cue-1",
+				startMs: 400,
+				endMs: 700,
+				audioPath: "file:///sounds/click-1.mp3",
+				volume: 0.8,
+				normalize: false,
+			},
+		]);
+	});
+
+	it("cancels capture if event collection fails", () => {
+		const error = new Error("event failed");
+		const host = {
+			beginExportAudioCapture: vi.fn(),
+			setExportAudioCaptureTime: vi.fn(),
+			emitEvent: vi.fn(() => {
+				throw error;
+			}),
+			finishExportAudioCapture: vi.fn(),
+			cancelExportAudioCapture: vi.fn(),
+		};
+
+		expect(() =>
+			collectExtensionAudioRegionsForExport(host, [
+				{ timeMs: 100, cx: 0.5, cy: 0.5, interactionType: "click" },
+			]),
+		).toThrow(error);
+		expect(host.cancelExportAudioCapture).toHaveBeenCalledTimes(1);
+		expect(host.finishExportAudioCapture).not.toHaveBeenCalled();
+	});
+
+	it("guards cursor interaction and cue duration boundaries", () => {
+		expect(isExportableCursorInteraction({ timeMs: 0, cx: 0, cy: 0 })).toBe(false);
+		expect(
+			isExportableCursorInteraction({
+				timeMs: 0,
+				cx: 0,
+				cy: 0,
+				interactionType: "move",
+			}),
+		).toBe(false);
+		expect(
+			isExportableCursorInteraction({
+				timeMs: 0,
+				cx: 0,
+				cy: 0,
+				interactionType: "right-click",
+			}),
+		).toBe(true);
+
+		expect(
+			extensionAudioCuesToRegions(
+				[
+					{
+						id: "cue",
+						extensionId: "com.test.clicks",
+						timeMs: 10,
+						audioPath: "file:///sounds/click.mp3",
+						volume: 0.5,
+					},
+				],
+				0,
+			)[0],
+		).toMatchObject({ startMs: 10, endMs: 11 });
+	});
+});

src/components/video-editor/extensionExportAudio.ts

@@ -0,0 +1,80 @@
+import type { ExtensionExportAudioCue, ExtensionHost } from "@/lib/extensions/extensionHost";
+import type { ExtensionEvent } from "@/lib/extensions/types";
+import type { AudioRegion, CursorTelemetryPoint } from "./types";
+
+export const DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS = 500;
+
+type ExtensionAudioCaptureHost = Pick<
+	ExtensionHost,
+	| "beginExportAudioCapture"
+	| "setExportAudioCaptureTime"
+	| "emitEvent"
+	| "finishExportAudioCapture"
+	| "cancelExportAudioCapture"
+>;
+
+/**
+ * Match preview playback by treating every non-move cursor interaction as a click event.
+ */
+export function isExportableCursorInteraction(point: CursorTelemetryPoint): boolean {
+	return Boolean(point.interactionType && point.interactionType !== "move");
+}
+
+/**
+ * Build the extension event payload emitted during export audio cue collection.
+ */
+function createCursorClickEvent(point: CursorTelemetryPoint): ExtensionEvent {
+	return {
+		type: "cursor:click",
+		timeMs: point.timeMs,
+		data: {
+			cx: point.cx,
+			cy: point.cy,
+			interactionType: point.interactionType,
+		},
+	};
+}
+
+/**
+ * Convert captured extension sound cues into temporary audio regions for the exporter.
+ */
+export function extensionAudioCuesToRegions(
+	cues: ExtensionExportAudioCue[],
+	durationMs = DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS,
+): AudioRegion[] {
+	const cueDurationMs = Number.isFinite(durationMs) && durationMs > 0 ? durationMs : 1;
+
+	return cues.map((cue) => ({
+		id: `extension-audio-${cue.id}`,
+		startMs: cue.timeMs,
+		endMs: cue.timeMs + cueDurationMs,
+		audioPath: cue.audioPath,
+		volume: cue.volume,
+		normalize: false,
+	}));
+}
+
+/**
+ * Emit export-time cursor interaction events and collect resulting extension audio regions.
+ */
+export function collectExtensionAudioRegionsForExport(
+	extensionHost: ExtensionAudioCaptureHost,
+	cursorTelemetry: CursorTelemetryPoint[],
+	durationMs = DEFAULT_EXTENSION_SOUND_CUE_DURATION_MS,
+): AudioRegion[] {
+	extensionHost.beginExportAudioCapture();
+
+	try {
+		for (const point of cursorTelemetry) {
+			if (!isExportableCursorInteraction(point)) continue;
+
+			extensionHost.setExportAudioCaptureTime(point.timeMs);
+			extensionHost.emitEvent(createCursorClickEvent(point));
+		}
+
+		return extensionAudioCuesToRegions(extensionHost.finishExportAudioCapture(), durationMs);
+	} catch (error) {
+		extensionHost.cancelExportAudioCapture();
+		throw error;
+	}
+}

src/components/video-editor/videoPlayback/zoomRegionUtils.ts

@@ -6,7 +6,7 @@ import {
 	ZOOM_OUT_EARLY_START_MS,
 } from "./constants";
 import { clampFocusToScale } from "./focusUtils";
-import { clamp01, cubicBezier, easeOutZoom } from "./mathUtils";
+import { clamp01, easeOutZoom } from "./mathUtils";
 
 const CHAINED_ZOOM_PAN_GAP_MS = 1350;
 const CONNECTED_ZOOM_PAN_DURATION_MS = 1000;
@@ -34,14 +34,6 @@ type ConnectedPanTransition = {
 	endScale: number;
 };
 
-function lerp(start: number, end: number, amount: number) {
-	return start + (end - start) * amount;
-}
-
-function easeConnectedPan(value: number) {
-	return cubicBezier(0.1, 0.0, 0.2, 1.0, value);
-}
-
 export function computeRegionStrength(
 	region: ZoomRegion,
 	timeMs: number,
@@ -79,13 +71,6 @@ export function computeRegionStrength(
 	return 1 - easeOutZoom(progress);
 }
 
-function getLinearFocus(start: ZoomFocus, end: ZoomFocus, amount: number): ZoomFocus {
-	return {
-		cx: lerp(start.cx, end.cx, amount),
-		cy: lerp(start.cy, end.cy, amount),
-	};
-}
-
 function getResolvedFocus(region: ZoomRegion, zoomScale: number): ZoomFocus {
 	return clampFocusToScale(region.focus, zoomScale);
 }
@@ -199,44 +184,6 @@ function getConnectedRegionHold(timeMs: number, connectedPairs: ConnectedRegionP
 	return null;
 }
 
-function getConnectedRegionTransition(connectedPairs: ConnectedRegionPair[], timeMs: number) {
-	for (const pair of connectedPairs) {
-		const { currentRegion, nextRegion, transitionStart, transitionEnd } = pair;
-
-		if (timeMs < transitionStart || timeMs > transitionEnd) {
-			continue;
-		}
-
-		const transitionProgress = easeConnectedPan(
-			clamp01((timeMs - transitionStart) / Math.max(1, transitionEnd - transitionStart)),
-		);
-		const currentScale = ZOOM_DEPTH_SCALES[currentRegion.depth];
-		const nextScale = ZOOM_DEPTH_SCALES[nextRegion.depth];
-		const transitionScale = lerp(currentScale, nextScale, transitionProgress);
-		const currentFocus = getResolvedFocus(currentRegion, currentScale);
-		const nextFocus = getResolvedFocus(nextRegion, nextScale);
-		const transitionFocus = getLinearFocus(currentFocus, nextFocus, transitionProgress);
-
-		return {
-			region: {
-				...nextRegion,
-				focus: transitionFocus,
-			},
-			strength: 1,
-			blendedScale: transitionScale,
-			transition: {
-				progress: transitionProgress,
-				startFocus: currentFocus,
-				endFocus: nextFocus,
-				startScale: currentScale,
-				endScale: nextScale,
-			},
-		};
-	}
-
-	return null;
-}
-
 export function findDominantRegion(
 	regions: ZoomRegion[],
 	timeMs: number,