From 9debf39576e51187439ed05ebedc21d94e31fbec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 13 Apr 2026 21:41:52 +0200 Subject: [PATCH 1/4] feat: add recording quality flag --- .../RunnerTests+CommandExecution.swift | 15 +- .../RunnerTests+Models.swift | 1 + .../RunnerTests+ScreenRecorder.swift | 29 ++- .../RunnerTests.swift | 2 + .../RecordingScripts/recording-resize.swift | 173 +++++++++++++++ ios-runner/RUNNER_PROTOCOL.md | 2 +- .../agent-device/references/verification.md | 1 + src/cli/commands/generic.ts | 1 + src/client-normalizers.ts | 1 + src/client-types.ts | 4 + src/daemon/__tests__/session-store.test.ts | 4 +- .../handlers/__tests__/record-trace.test.ts | 198 +++++++++++++++++- .../__tests__/session-replay-script.test.ts | 7 +- src/daemon/handlers/record-trace-android.ts | 64 +++++- src/daemon/handlers/record-trace-ios.ts | 2 + src/daemon/handlers/record-trace-recording.ts | 32 ++- src/daemon/handlers/session-replay-script.ts | 8 + src/daemon/script-utils.ts | 3 + src/daemon/session-store.ts | 1 + src/daemon/types.ts | 1 + .../ios/__tests__/recording-scripts.test.ts | 8 + .../ios/__tests__/runner-client.test.ts | 8 +- src/platforms/ios/runner-contract.ts | 1 + src/recording/overlay.ts | 20 ++ src/utils/__tests__/args.test.ts | 26 ++- src/utils/command-schema.ts | 16 +- website/docs/docs/client-api.md | 2 + website/docs/docs/commands.md | 2 + website/docs/docs/configuration.md | 2 +- website/docs/docs/introduction.md | 1 + 30 files changed, 612 insertions(+), 23 deletions(-) create mode 100644 ios-runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 7c85e0b0a..9c1601505 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -183,16 +183,25 @@ extension RunnerTests { if let requestedFps = command.fps, (requestedFps < minRecordingFps || requestedFps > maxRecordingFps) { return Response(ok: false, error: ErrorPayload(message: "recordStart fps must be between \(minRecordingFps) and \(maxRecordingFps)")) } + if let requestedQuality = command.quality, (requestedQuality < minRecordingQuality || requestedQuality > maxRecordingQuality) { + return Response(ok: false, error: ErrorPayload(message: "recordStart quality must be between \(minRecordingQuality) and \(maxRecordingQuality)")) + } do { let resolvedOutPath = resolveRecordingOutPath(requestedOutPath) let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps) + let qualityLabel = command.quality.map(String.init) ?? "native" NSLog( - "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@", + "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@ quality=%@", requestedOutPath, resolvedOutPath, - fpsLabel + fpsLabel, + qualityLabel + ) + let recorder = ScreenRecorder( + outputPath: resolvedOutPath, + fps: command.fps.map { Int32($0) }, + quality: command.quality ) - let recorder = ScreenRecorder(outputPath: resolvedOutPath, fps: command.fps.map { Int32($0) }) try recorder.start { [weak self] in return self?.captureRunnerFrame() } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 08eab5168..3f8cf72f7 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -52,6 +52,7 @@ struct Command: Codable { let scale: Double? let outPath: String? let fps: Int? + let quality: Int? let interactiveOnly: Bool? let compact: Bool? let depth: Int? diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift index 9c9db51f3..9d2b17971 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift @@ -7,6 +7,7 @@ extension RunnerTests { final class ScreenRecorder { private let outputPath: String private let fps: Int32? + private let quality: Int? private var effectiveFps: Int32 { max(1, fps ?? RunnerTests.defaultRecordingFps) } @@ -25,9 +26,10 @@ extension RunnerTests { private var startedSession = false private var startError: Error? - init(outputPath: String, fps: Int32?) { + init(outputPath: String, fps: Int32?, quality: Int?) { self.outputPath = outputPath self.fps = fps + self.quality = quality } func start(captureFrame: @escaping () -> RunnerImage?) throws { @@ -48,7 +50,7 @@ extension RunnerTests { while Date() < bootstrapDeadline { if let image = captureFrame(), let cgImage = runnerCGImage(from: image) { bootstrapImage = image - dimensions = CGSize(width: cgImage.width, height: cgImage.height) + dimensions = scaledDimensions(width: cgImage.width, height: cgImage.height) break } Thread.sleep(forTimeInterval: 0.05) @@ -240,11 +242,13 @@ extension RunnerTests { CVPixelBufferLockBaseAddress(pixelBuffer, []) defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) } + let width = CVPixelBufferGetWidth(pixelBuffer) + let height = CVPixelBufferGetHeight(pixelBuffer) guard let context = CGContext( data: CVPixelBufferGetBaseAddress(pixelBuffer), - width: image.width, - height: image.height, + width: width, + height: height, bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer), space: CGColorSpaceCreateDeviceRGB(), @@ -253,8 +257,23 @@ extension RunnerTests { else { return nil } - context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height)) + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) return pixelBuffer } + + private func scaledDimensions(width: Int, height: Int) -> CGSize { + guard let quality, quality < 10 else { + return CGSize(width: width, height: height) + } + let scale = Double(quality) / 10.0 + return CGSize( + width: scaledEvenDimension(width, scale: scale), + height: scaledEvenDimension(height, scale: scale) + ) + } + + private func scaledEvenDimension(_ value: Int, scale: Double) -> Int { + max(2, Int((Double(value) * scale / 2.0).rounded()) * 2) + } } } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index df3c983be..61a04afe3 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -48,6 +48,8 @@ final class RunnerTests: XCTestCase { let tvRemoteDoublePressDelayDefault: TimeInterval = 0.0 let minRecordingFps = 1 let maxRecordingFps = 120 + let minRecordingQuality = 5 + let maxRecordingQuality = 10 var needsPostSnapshotInteractionDelay = false var needsFirstInteractionDelay = false var activeRecording: ScreenRecorder? diff --git a/ios-runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift b/ios-runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift new file mode 100644 index 000000000..7efe1f568 --- /dev/null +++ b/ios-runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift @@ -0,0 +1,173 @@ +import AVFoundation +import Foundation + +enum ResizeError: Error, CustomStringConvertible { + case invalidArgs(String) + case missingVideoTrack + case exportFailed(String) + + var description: String { + switch self { + case .invalidArgs(let message): + return message + case .missingVideoTrack: + return "Input video does not contain a video track." + case .exportFailed(let message): + return message + } + } +} + +do { + try run() +} catch { + fputs("recording-resize: \(error)\n", stderr) + exit(1) +} + +func run() throws { + let arguments = Array(CommandLine.arguments.dropFirst()) + let parsedArgs = try parseArguments(arguments) + let inputURL = URL(fileURLWithPath: parsedArgs.inputPath) + let outputURL = URL(fileURLWithPath: parsedArgs.outputPath) + + if FileManager.default.fileExists(atPath: outputURL.path) { + try FileManager.default.removeItem(at: outputURL) + } + + let asset = AVURLAsset(url: inputURL) + guard let sourceVideoTrack = asset.tracks(withMediaType: .video).first else { + throw ResizeError.missingVideoTrack + } + + let renderSize = scaledRenderSize(for: sourceVideoTrack, quality: parsedArgs.quality) + let composition = AVMutableComposition() + let fullRange = CMTimeRange(start: .zero, duration: asset.duration) + + guard let compositionVideoTrack = composition.addMutableTrack( + withMediaType: .video, + preferredTrackID: kCMPersistentTrackID_Invalid + ) else { + throw ResizeError.exportFailed("Failed to create composition video track.") + } + try compositionVideoTrack.insertTimeRange(fullRange, of: sourceVideoTrack, at: .zero) + + if let sourceAudioTrack = asset.tracks(withMediaType: .audio).first, + let compositionAudioTrack = composition.addMutableTrack( + withMediaType: .audio, + preferredTrackID: kCMPersistentTrackID_Invalid + ) { + try? compositionAudioTrack.insertTimeRange(fullRange, of: sourceAudioTrack, at: .zero) + } + + let scale = CGFloat(parsedArgs.quality) / 10.0 + let videoComposition = AVMutableVideoComposition() + videoComposition.renderSize = renderSize + videoComposition.frameDuration = resolvedFrameDuration(for: sourceVideoTrack) + + let instruction = AVMutableVideoCompositionInstruction() + instruction.timeRange = fullRange + let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: compositionVideoTrack) + // Keep rotation/translation from the source track, then scale into the smaller render canvas. + let scaledTransform = sourceVideoTrack.preferredTransform.concatenating( + CGAffineTransform(scaleX: scale, y: scale) + ) + layerInstruction.setTransform(scaledTransform, at: .zero) + instruction.layerInstructions = [layerInstruction] + videoComposition.instructions = [instruction] + + guard let exporter = AVAssetExportSession(asset: composition, presetName: AVAssetExportPresetHighestQuality) else { + throw ResizeError.exportFailed("Failed to create export session.") + } + + exporter.outputURL = outputURL + exporter.outputFileType = .mp4 + exporter.videoComposition = videoComposition + exporter.shouldOptimizeForNetworkUse = true + + let semaphore = DispatchSemaphore(value: 0) + exporter.exportAsynchronously { + semaphore.signal() + } + if semaphore.wait(timeout: .now() + 120) == .timedOut { + exporter.cancelExport() + throw ResizeError.exportFailed("Resize export timed out.") + } + + if exporter.status != .completed { + throw ResizeError.exportFailed(exporter.error?.localizedDescription ?? "Resize export failed.") + } +} + +func parseArguments(_ arguments: [String]) throws -> (inputPath: String, outputPath: String, quality: Int) { + var inputPath: String? + var outputPath: String? + var quality: Int? + var index = 0 + + while index < arguments.count { + let argument = arguments[index] + let nextIndex = index + 1 + switch argument { + case "--input": + guard nextIndex < arguments.count else { throw ResizeError.invalidArgs("--input requires a value") } + inputPath = arguments[nextIndex] + index += 2 + case "--output": + guard nextIndex < arguments.count else { throw ResizeError.invalidArgs("--output requires a value") } + outputPath = arguments[nextIndex] + index += 2 + case "--quality": + guard nextIndex < arguments.count else { throw ResizeError.invalidArgs("--quality requires a value") } + guard let parsed = Int(arguments[nextIndex]), parsed >= 5, parsed <= 10 else { + throw ResizeError.invalidArgs("--quality must be an integer between 5 and 10") + } + quality = parsed + index += 2 + default: + throw ResizeError.invalidArgs("Unknown argument: \(argument)") + } + } + + guard let inputPath, let outputPath, let quality else { + throw ResizeError.invalidArgs( + "Usage: recording-resize.swift --input