From fea114935bbf4efaded00dcc57e1c73ab759c8bf Mon Sep 17 00:00:00 2001 From: huymobile Date: Sun, 30 Aug 2026 07:03:01 +0700 Subject: [PATCH] fix(ios): use connection rotation for frame coordinates --- .../ios/Delegates/FrameDelegate.swift | 4 +-- .../AVCaptureConnection+orientation.swift | 11 ++++++++ .../Image Types/HybridFrame.swift | 8 +++--- .../Outputs/HybridCameraFrameOutput.swift | 13 ++++++--- .../HybridCameraVideoFrameOutput.swift | 2 +- .../ios/Public/MediaSampleMetadata.swift | 28 +++++++++++++++++-- 6 files changed, 52 insertions(+), 14 deletions(-) diff --git a/packages/react-native-vision-camera/ios/Delegates/FrameDelegate.swift b/packages/react-native-vision-camera/ios/Delegates/FrameDelegate.swift index aa53c1d6a9..6a50da18d5 100644 --- a/packages/react-native-vision-camera/ios/Delegates/FrameDelegate.swift +++ b/packages/react-native-vision-camera/ios/Delegates/FrameDelegate.swift @@ -9,7 +9,7 @@ import AVFoundation import Foundation final class FrameDelegate: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate { - var onFrame: ((CMSampleBuffer, CMTime, CameraOrientation, Bool) -> Void)? + var onFrame: ((CMSampleBuffer, CMTime, CameraOrientation, Bool, CameraOrientation) -> Void)? var onFrameDropped: ((CMSampleBuffer) -> Void)? func captureOutput( @@ -28,7 +28,7 @@ final class FrameDelegate: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat if let onFrame { onFrame( sampleBuffer, sampleBuffer.presentationTimeStamp, connection.orientation, - connection.isVideoMirrored) + connection.isVideoMirrored, connection.physicalBufferRotation) } } } diff --git a/packages/react-native-vision-camera/ios/Extensions/AVFoundation/AVCaptureConnection+orientation.swift b/packages/react-native-vision-camera/ios/Extensions/AVFoundation/AVCaptureConnection+orientation.swift index 36181e9b49..46f995b0f0 100644 --- a/packages/react-native-vision-camera/ios/Extensions/AVFoundation/AVCaptureConnection+orientation.swift +++ b/packages/react-native-vision-camera/ios/Extensions/AVFoundation/AVCaptureConnection+orientation.swift @@ -13,6 +13,17 @@ extension AVCaptureConnection { return CameraOrientation(avOrientation: videoOrientation) } + var physicalBufferRotation: CameraOrientation { + #if os(iOS) + if #available(iOS 17.0, *) { + return CameraOrientation(degrees: Int(videoRotationAngle)) + } + #endif + // videoRotationAngle is unavailable before iOS 17 and on visionOS. + // Preserve the existing videoOrientation-based behavior on those targets. + return orientation + } + func setOrientation(_ orientation: CameraOrientation) throws { guard self.isVideoOrientationSupported else { throw RuntimeError.error( diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift index b985448a79..aef280ff2c 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift @@ -151,8 +151,8 @@ final class HybridFrame: HybridFrameSpec, NativeFrame, LazyLockableBuffer { } let matrix = FrameCoordinateSystemConverter.getCameraToFrameMatrix( pixelBuffer: pixelBuffer, - orientation: orientation, - isMirrored: isMirrored) + orientation: metadata.physicalBufferRotation, + isMirrored: metadata.isPhysicalBufferMirrored) return cameraPoint.applying(matrix) } @@ -162,8 +162,8 @@ final class HybridFrame: HybridFrameSpec, NativeFrame, LazyLockableBuffer { } let matrix = FrameCoordinateSystemConverter.getFrameToCameraMatrix( pixelBuffer: pixelBuffer, - orientation: orientation, - isMirrored: isMirrored) + orientation: metadata.physicalBufferRotation, + isMirrored: metadata.isPhysicalBufferMirrored) return framePoint.applying(matrix) } } diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift index 17c7a5f13c..9ee9274e8d 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift @@ -120,7 +120,8 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu private func getMediaSampleMetadata( at timestamp: CMTime, orientation bufferOrientation: CameraOrientation, - isMirrored isBufferMirrored: Bool + isMirrored isBufferMirrored: Bool, + physicalBufferRotation: CameraOrientation ) -> MediaSampleMetadata { // `isMirrored` is relative; if the buffer is already mirrored & we want mirror, good. // If not, we need to counter-mirror. @@ -142,7 +143,9 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu return MediaSampleMetadata( timestamp: timestamp, orientation: relativeOrientation, - isMirrored: isMirrored) + isMirrored: isMirrored, + physicalBufferRotation: physicalBufferRotation, + isPhysicalBufferMirrored: isBufferMirrored) } func setOnFrameCallback(onFrame: ((any HybridFrameSpec) -> Bool)?) throws { @@ -151,12 +154,14 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu withMessage: "setOnFrameCallback(...) must be called on the FrameOutput's `thread`!") } if let onFrame { - delegate.onFrame = { (sampleBuffer, timestamp, bufferOrientation, isBufferMirrored) in + delegate.onFrame = { + (sampleBuffer, timestamp, bufferOrientation, isBufferMirrored, physicalBufferRotation) in // Prepare Frame + Metadata let metadata = self.getMediaSampleMetadata( at: timestamp, orientation: bufferOrientation, - isMirrored: isBufferMirrored) + isMirrored: isBufferMirrored, + physicalBufferRotation: physicalBufferRotation) let frame = HybridFrame( buffer: sampleBuffer, metadata: metadata) diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraVideoFrameOutput.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraVideoFrameOutput.swift index ddf5895cd3..e88f5c232a 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraVideoFrameOutput.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraVideoFrameOutput.swift @@ -72,7 +72,7 @@ final class HybridCameraVideoFrameOutput: HybridCameraVideoOutputSpec, NativeCam output.preservesDynamicHDRMetadata = true } // set the delegate to append to the Recorder - delegate.onFrame = { [weak self] buffer, timestamp, orientation, isMirrored in + delegate.onFrame = { [weak self] buffer, _, _, _, _ in guard let self else { return } self.onFrame(buffer, type: .video) } diff --git a/packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift b/packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift index 4917891eef..3e6ebe59e5 100644 --- a/packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift +++ b/packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift @@ -14,6 +14,8 @@ public struct MediaSampleMetadata { let timestamp: CMTime let orientation: CameraOrientation let isMirrored: Bool + let physicalBufferRotation: CameraOrientation + let isPhysicalBufferMirrored: Bool init(timestamp: CMTime, orientationFromOutput output: AVCaptureOutput) throws { guard let connection = output.connection(with: .video) else { @@ -22,14 +24,34 @@ public struct MediaSampleMetadata { self.init(timestamp: timestamp, orientationFromConnection: connection) } init(timestamp: CMTime, orientationFromConnection connection: AVCaptureConnection) { - self.timestamp = timestamp - self.orientation = connection.orientation - self.isMirrored = connection.isVideoMirrored + let isMirrored = connection.isVideoMirrored + self.init( + timestamp: timestamp, + orientation: connection.orientation, + isMirrored: isMirrored, + physicalBufferRotation: connection.physicalBufferRotation, + isPhysicalBufferMirrored: isMirrored) } init(timestamp: CMTime, orientation: CameraOrientation, isMirrored: Bool) { + self.init( + timestamp: timestamp, + orientation: orientation, + isMirrored: isMirrored, + physicalBufferRotation: orientation, + isPhysicalBufferMirrored: isMirrored) + } + init( + timestamp: CMTime, + orientation: CameraOrientation, + isMirrored: Bool, + physicalBufferRotation: CameraOrientation, + isPhysicalBufferMirrored: Bool + ) { self.timestamp = timestamp self.orientation = orientation self.isMirrored = isMirrored + self.physicalBufferRotation = physicalBufferRotation + self.isPhysicalBufferMirrored = isPhysicalBufferMirrored } var uiImageOrientation: UIImage.Orientation {