fix: align iOS frame coordinates with camera sensor

packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridDepth.swift

@@ -152,8 +152,8 @@ final class HybridDepth: HybridDepthSpec, NativeDepth, LazyLockableBuffer {
     }
     let matrix = FrameCoordinateSystemConverter.getCameraToFrameMatrix(
       pixelBuffer: pixelBuffer,
-      orientation: orientation,
-      isMirrored: isMirrored)
+      bufferOrientation: metadata.bufferOrientation,
+      isBufferMirrored: metadata.isBufferMirrored)
     return cameraPoint.applying(matrix)
   }
 
@@ -163,8 +163,8 @@ final class HybridDepth: HybridDepthSpec, NativeDepth, LazyLockableBuffer {
     }
     let matrix = FrameCoordinateSystemConverter.getFrameToCameraMatrix(
       pixelBuffer: pixelBuffer,
-      orientation: orientation,
-      isMirrored: isMirrored)
+      bufferOrientation: metadata.bufferOrientation,
+      isBufferMirrored: metadata.isBufferMirrored)
     return depthPoint.applying(matrix)
   }
 

packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift

@@ -151,8 +151,8 @@ final class HybridFrame: HybridFrameSpec, NativeFrame, LazyLockableBuffer {
     }
     let matrix = FrameCoordinateSystemConverter.getCameraToFrameMatrix(
       pixelBuffer: pixelBuffer,
-      orientation: orientation,
-      isMirrored: isMirrored)
+      bufferOrientation: metadata.bufferOrientation,
+      isBufferMirrored: metadata.isBufferMirrored)
     return cameraPoint.applying(matrix)
   }
 
@@ -162,8 +162,8 @@ final class HybridFrame: HybridFrameSpec, NativeFrame, LazyLockableBuffer {
     }
     let matrix = FrameCoordinateSystemConverter.getFrameToCameraMatrix(
       pixelBuffer: pixelBuffer,
-      orientation: orientation,
-      isMirrored: isMirrored)
+      bufferOrientation: metadata.bufferOrientation,
+      isBufferMirrored: metadata.isBufferMirrored)
     return framePoint.applying(matrix)
   }
 }

packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraDepthFrameOutput.swift

@@ -106,7 +106,9 @@ final class HybridCameraDepthFrameOutput: HybridCameraDepthFrameOutputSpec, Nati
     return MediaSampleMetadata(
       timestamp: timestamp,
       orientation: relativeOrientation,
-      isMirrored: isMirrored)
+      isMirrored: isMirrored,
+      bufferOrientation: bufferOrientation,
+      isBufferMirrored: isBufferMirrored)
   }
 
   func setOnDepthFrameCallback(onDepthFrame: ((any HybridDepthSpec) -> Bool)?) throws {

packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift

@@ -142,7 +142,9 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu
     return MediaSampleMetadata(
       timestamp: timestamp,
       orientation: relativeOrientation,
-      isMirrored: isMirrored)
+      isMirrored: isMirrored,
+      bufferOrientation: bufferOrientation,
+      isBufferMirrored: isBufferMirrored)
   }
 
   func setOnFrameCallback(onFrame: ((any HybridFrameSpec) -> Bool)?) throws {

packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift

@@ -14,6 +14,8 @@ public struct MediaSampleMetadata {
   let timestamp: CMTime
   let orientation: CameraOrientation
   let isMirrored: Bool
+  let bufferOrientation: CameraOrientation
+  let isBufferMirrored: Bool
 
   init(timestamp: CMTime, orientationFromOutput output: AVCaptureOutput) throws {
     guard let connection = output.connection(with: .video) else {
@@ -22,14 +24,37 @@ public struct MediaSampleMetadata {
     self.init(timestamp: timestamp, orientationFromConnection: connection)
   }
   init(timestamp: CMTime, orientationFromConnection connection: AVCaptureConnection) {
-    self.timestamp = timestamp
-    self.orientation = connection.orientation
-    self.isMirrored = connection.isVideoMirrored
+    let bufferOrientation = connection.orientation
+    let isBufferMirrored = connection.isVideoMirrored
+    self.init(
+      timestamp: timestamp,
+      orientation: bufferOrientation,
+      isMirrored: isBufferMirrored,
+      bufferOrientation: bufferOrientation,
+      isBufferMirrored: isBufferMirrored)
   }
   init(timestamp: CMTime, orientation: CameraOrientation, isMirrored: Bool) {
+    self.init(
+      timestamp: timestamp,
+      orientation: orientation,
+      isMirrored: isMirrored,
+      // This initializer has no AVCaptureConnection to expose the physical buffer state.
+      // Preserve the previous relative-orientation transform for callers such as Photo depth data.
+      bufferOrientation: orientation.rotatedBy(.left),
+      isBufferMirrored: isMirrored)
+  }
+  init(
+    timestamp: CMTime,
+    orientation: CameraOrientation,
+    isMirrored: Bool,
+    bufferOrientation: CameraOrientation,
+    isBufferMirrored: Bool
+  ) {
     self.timestamp = timestamp
     self.orientation = orientation
     self.isMirrored = isMirrored
+    self.bufferOrientation = bufferOrientation
+    self.isBufferMirrored = isBufferMirrored
   }
 
   var uiImageOrientation: UIImage.Orientation {

packages/react-native-vision-camera/ios/Utils/FrameCoordinateSystemConverter.swift

@@ -13,18 +13,22 @@ enum FrameCoordinateSystemConverter {
    * Get a Matrix that can convert a point in the
    * given `pixelBuffer` to normalized Camera coordinates
    * (`(cx, cy) ∈ [0, 1]²`).
-   * The `orientation` and `isMirrored` flags affect the
-   * Matrix if the `Frame` needs those to be adjusted.
+   * The buffer's physical orientation and mirroring affect
+   * the Matrix if the `Frame` needs those to be adjusted.
    */
   static func getFrameToCameraMatrix(
     pixelBuffer: CVPixelBuffer,
-    orientation: CameraOrientation,
-    isMirrored: Bool
+    bufferOrientation: CameraOrientation,
+    isBufferMirrored: Bool
   ) -> CGAffineTransform {
     var matrix = CGAffineTransform.identity
 
-    // 1. Counter-rotate by the orientation to get it up-right
-    switch orientation {
+    // AVFoundation Camera coordinates use the unrotated sensor image, which is always
+    // landscape-right (home button on the right). In CameraOrientation, this is `.left`.
+    let sensorRelativeOrientation = bufferOrientation.relativeTo(.left)
+
+    // 1. Rotate from the Pixel Buffer's orientation into the sensor's orientation
+    switch sensorRelativeOrientation {
     case .up:
       break
     case .down:
@@ -44,8 +48,8 @@ enum FrameCoordinateSystemConverter {
         .rotated(by: -.pi / 2)
     }
 
-    // 2. If the Frame is mirrored, counter-mirror our Matrix
-    if isMirrored {
+    // 2. If the Pixel Buffer is mirrored, counter-mirror our Matrix
+    if isBufferMirrored {
       let mirror = CGAffineTransform.identity
         .translatedBy(x: 1, y: 0)
         .scaledBy(x: -1, y: 1)
@@ -62,13 +66,13 @@ enum FrameCoordinateSystemConverter {
 
   static func getCameraToFrameMatrix(
     pixelBuffer: CVPixelBuffer,
-    orientation: CameraOrientation,
-    isMirrored: Bool
+    bufferOrientation: CameraOrientation,
+    isBufferMirrored: Bool
   ) -> CGAffineTransform {
     let frameToCameraMatrix = getFrameToCameraMatrix(
       pixelBuffer: pixelBuffer,
-      orientation: orientation,
-      isMirrored: isMirrored)
+      bufferOrientation: bufferOrientation,
+      isBufferMirrored: isBufferMirrored)
     return frameToCameraMatrix.inverted()
   }
 }