From 8d4b1c2922e762c5dfbe30617a8098187676e24c Mon Sep 17 00:00:00 2001 From: huymobile Date: Thu, 27 Aug 2026 04:08:25 +0700 Subject: [PATCH] fix(ios): align frame coordinates with camera sensor Keep the physical buffer orientation separate from public output-relative metadata so Frame and Depth conversions use the same unrotated AVFoundation sensor space as PreviewView.\n\nFixes #4114. --- .../Image Types/HybridDepth.swift | 8 ++--- .../Image Types/HybridFrame.swift | 8 ++--- .../HybridCameraDepthFrameOutput.swift | 4 ++- .../Outputs/HybridCameraFrameOutput.swift | 4 ++- .../ios/Public/MediaSampleMetadata.swift | 31 +++++++++++++++++-- .../FrameCoordinateSystemConverter.swift | 28 ++++++++++------- 6 files changed, 58 insertions(+), 25 deletions(-) diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridDepth.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridDepth.swift index e783bca6d2..b92ecababa 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridDepth.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridDepth.swift @@ -152,8 +152,8 @@ final class HybridDepth: HybridDepthSpec, NativeDepth, LazyLockableBuffer { } let matrix = FrameCoordinateSystemConverter.getCameraToFrameMatrix( pixelBuffer: pixelBuffer, - orientation: orientation, - isMirrored: isMirrored) + bufferOrientation: metadata.bufferOrientation, + isBufferMirrored: metadata.isBufferMirrored) return cameraPoint.applying(matrix) } @@ -163,8 +163,8 @@ final class HybridDepth: HybridDepthSpec, NativeDepth, LazyLockableBuffer { } let matrix = FrameCoordinateSystemConverter.getFrameToCameraMatrix( pixelBuffer: pixelBuffer, - orientation: orientation, - isMirrored: isMirrored) + bufferOrientation: metadata.bufferOrientation, + isBufferMirrored: metadata.isBufferMirrored) return depthPoint.applying(matrix) } diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift index b985448a79..97d415fdb7 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift @@ -151,8 +151,8 @@ final class HybridFrame: HybridFrameSpec, NativeFrame, LazyLockableBuffer { } let matrix = FrameCoordinateSystemConverter.getCameraToFrameMatrix( pixelBuffer: pixelBuffer, - orientation: orientation, - isMirrored: isMirrored) + bufferOrientation: metadata.bufferOrientation, + isBufferMirrored: metadata.isBufferMirrored) return cameraPoint.applying(matrix) } @@ -162,8 +162,8 @@ final class HybridFrame: HybridFrameSpec, NativeFrame, LazyLockableBuffer { } let matrix = FrameCoordinateSystemConverter.getFrameToCameraMatrix( pixelBuffer: pixelBuffer, - orientation: orientation, - isMirrored: isMirrored) + bufferOrientation: metadata.bufferOrientation, + isBufferMirrored: metadata.isBufferMirrored) return framePoint.applying(matrix) } } diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraDepthFrameOutput.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraDepthFrameOutput.swift index 0be7d9d315..dc755806ba 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraDepthFrameOutput.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraDepthFrameOutput.swift @@ -106,7 +106,9 @@ final class HybridCameraDepthFrameOutput: HybridCameraDepthFrameOutputSpec, Nati return MediaSampleMetadata( timestamp: timestamp, orientation: relativeOrientation, - isMirrored: isMirrored) + isMirrored: isMirrored, + bufferOrientation: bufferOrientation, + isBufferMirrored: isBufferMirrored) } func setOnDepthFrameCallback(onDepthFrame: ((any HybridDepthSpec) -> Bool)?) throws { diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift index 17c7a5f13c..d09db59656 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift @@ -142,7 +142,9 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu return MediaSampleMetadata( timestamp: timestamp, orientation: relativeOrientation, - isMirrored: isMirrored) + isMirrored: isMirrored, + bufferOrientation: bufferOrientation, + isBufferMirrored: isBufferMirrored) } func setOnFrameCallback(onFrame: ((any HybridFrameSpec) -> Bool)?) throws { diff --git a/packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift b/packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift index 4917891eef..99244d6fa3 100644 --- a/packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift +++ b/packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift @@ -14,6 +14,8 @@ public struct MediaSampleMetadata { let timestamp: CMTime let orientation: CameraOrientation let isMirrored: Bool + let bufferOrientation: CameraOrientation + let isBufferMirrored: Bool init(timestamp: CMTime, orientationFromOutput output: AVCaptureOutput) throws { guard let connection = output.connection(with: .video) else { @@ -22,14 +24,37 @@ public struct MediaSampleMetadata { self.init(timestamp: timestamp, orientationFromConnection: connection) } init(timestamp: CMTime, orientationFromConnection connection: AVCaptureConnection) { - self.timestamp = timestamp - self.orientation = connection.orientation - self.isMirrored = connection.isVideoMirrored + let bufferOrientation = connection.orientation + let isBufferMirrored = connection.isVideoMirrored + self.init( + timestamp: timestamp, + orientation: bufferOrientation, + isMirrored: isBufferMirrored, + bufferOrientation: bufferOrientation, + isBufferMirrored: isBufferMirrored) } init(timestamp: CMTime, orientation: CameraOrientation, isMirrored: Bool) { + self.init( + timestamp: timestamp, + orientation: orientation, + isMirrored: isMirrored, + // This initializer has no AVCaptureConnection to expose the physical buffer state. + // Preserve the previous relative-orientation transform for callers such as Photo depth data. + bufferOrientation: orientation.rotatedBy(.left), + isBufferMirrored: isMirrored) + } + init( + timestamp: CMTime, + orientation: CameraOrientation, + isMirrored: Bool, + bufferOrientation: CameraOrientation, + isBufferMirrored: Bool + ) { self.timestamp = timestamp self.orientation = orientation self.isMirrored = isMirrored + self.bufferOrientation = bufferOrientation + self.isBufferMirrored = isBufferMirrored } var uiImageOrientation: UIImage.Orientation { diff --git a/packages/react-native-vision-camera/ios/Utils/FrameCoordinateSystemConverter.swift b/packages/react-native-vision-camera/ios/Utils/FrameCoordinateSystemConverter.swift index d3f051d762..d1d4c8d165 100644 --- a/packages/react-native-vision-camera/ios/Utils/FrameCoordinateSystemConverter.swift +++ b/packages/react-native-vision-camera/ios/Utils/FrameCoordinateSystemConverter.swift @@ -13,18 +13,22 @@ enum FrameCoordinateSystemConverter { * Get a Matrix that can convert a point in the * given `pixelBuffer` to normalized Camera coordinates * (`(cx, cy) ∈ [0, 1]²`). - * The `orientation` and `isMirrored` flags affect the - * Matrix if the `Frame` needs those to be adjusted. + * The buffer's physical orientation and mirroring affect + * the Matrix if the `Frame` needs those to be adjusted. */ static func getFrameToCameraMatrix( pixelBuffer: CVPixelBuffer, - orientation: CameraOrientation, - isMirrored: Bool + bufferOrientation: CameraOrientation, + isBufferMirrored: Bool ) -> CGAffineTransform { var matrix = CGAffineTransform.identity - // 1. Counter-rotate by the orientation to get it up-right - switch orientation { + // AVFoundation Camera coordinates use the unrotated sensor image, which is always + // landscape-right (home button on the right). In CameraOrientation, this is `.left`. + let sensorRelativeOrientation = bufferOrientation.relativeTo(.left) + + // 1. Rotate from the Pixel Buffer's orientation into the sensor's orientation + switch sensorRelativeOrientation { case .up: break case .down: @@ -44,8 +48,8 @@ enum FrameCoordinateSystemConverter { .rotated(by: -.pi / 2) } - // 2. If the Frame is mirrored, counter-mirror our Matrix - if isMirrored { + // 2. If the Pixel Buffer is mirrored, counter-mirror our Matrix + if isBufferMirrored { let mirror = CGAffineTransform.identity .translatedBy(x: 1, y: 0) .scaledBy(x: -1, y: 1) @@ -62,13 +66,13 @@ enum FrameCoordinateSystemConverter { static func getCameraToFrameMatrix( pixelBuffer: CVPixelBuffer, - orientation: CameraOrientation, - isMirrored: Bool + bufferOrientation: CameraOrientation, + isBufferMirrored: Bool ) -> CGAffineTransform { let frameToCameraMatrix = getFrameToCameraMatrix( pixelBuffer: pixelBuffer, - orientation: orientation, - isMirrored: isMirrored) + bufferOrientation: bufferOrientation, + isBufferMirrored: isBufferMirrored) return frameToCameraMatrix.inverted() } }