fix(ios): use connection rotation for frame coordinates

packages/react-native-vision-camera/ios/Delegates/FrameDelegate.swift

@@ -9,7 +9,7 @@ import AVFoundation
 import Foundation
 
 final class FrameDelegate: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
-  var onFrame: ((CMSampleBuffer, CMTime, CameraOrientation, Bool) -> Void)?
+  var onFrame: ((CMSampleBuffer, CMTime, CameraOrientation, Bool, CameraOrientation) -> Void)?
   var onFrameDropped: ((CMSampleBuffer) -> Void)?
 
   func captureOutput(
@@ -28,7 +28,7 @@ final class FrameDelegate: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
     if let onFrame {
       onFrame(
         sampleBuffer, sampleBuffer.presentationTimeStamp, connection.orientation,
-        connection.isVideoMirrored)
+        connection.isVideoMirrored, connection.physicalBufferRotation)
     }
   }
 }

packages/react-native-vision-camera/ios/Extensions/AVFoundation/AVCaptureConnection+orientation.swift

@@ -13,6 +13,17 @@ extension AVCaptureConnection {
     return CameraOrientation(avOrientation: videoOrientation)
   }
 
+  var physicalBufferRotation: CameraOrientation {
+    #if os(iOS)
+      if #available(iOS 17.0, *) {
+        return CameraOrientation(degrees: Int(videoRotationAngle))
+      }
+    #endif
+    // videoRotationAngle is unavailable before iOS 17 and on visionOS.
+    // Preserve the existing videoOrientation-based behavior on those targets.
+    return orientation
+  }
+
   func setOrientation(_ orientation: CameraOrientation) throws {
     guard self.isVideoOrientationSupported else {
       throw RuntimeError.error(

packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift

@@ -151,8 +151,8 @@ final class HybridFrame: HybridFrameSpec, NativeFrame, LazyLockableBuffer {
     }
     let matrix = FrameCoordinateSystemConverter.getCameraToFrameMatrix(
       pixelBuffer: pixelBuffer,
-      orientation: orientation,
-      isMirrored: isMirrored)
+      orientation: metadata.physicalBufferRotation,
+      isMirrored: metadata.isPhysicalBufferMirrored)
     return cameraPoint.applying(matrix)
   }
 
@@ -162,8 +162,8 @@ final class HybridFrame: HybridFrameSpec, NativeFrame, LazyLockableBuffer {
     }
     let matrix = FrameCoordinateSystemConverter.getFrameToCameraMatrix(
       pixelBuffer: pixelBuffer,
-      orientation: orientation,
-      isMirrored: isMirrored)
+      orientation: metadata.physicalBufferRotation,
+      isMirrored: metadata.isPhysicalBufferMirrored)
     return framePoint.applying(matrix)
   }
 }

packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift

@@ -120,7 +120,8 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu
   private func getMediaSampleMetadata(
     at timestamp: CMTime,
     orientation bufferOrientation: CameraOrientation,
-    isMirrored isBufferMirrored: Bool
+    isMirrored isBufferMirrored: Bool,
+    physicalBufferRotation: CameraOrientation
   ) -> MediaSampleMetadata {
     // `isMirrored` is relative; if the buffer is already mirrored & we want mirror, good.
     // If not, we need to counter-mirror.
@@ -142,7 +143,9 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu
     return MediaSampleMetadata(
       timestamp: timestamp,
       orientation: relativeOrientation,
-      isMirrored: isMirrored)
+      isMirrored: isMirrored,
+      physicalBufferRotation: physicalBufferRotation,
+      isPhysicalBufferMirrored: isBufferMirrored)
   }
 
   func setOnFrameCallback(onFrame: ((any HybridFrameSpec) -> Bool)?) throws {
@@ -151,12 +154,14 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu
         withMessage: "setOnFrameCallback(...) must be called on the FrameOutput's `thread`!")
     }
     if let onFrame {
-      delegate.onFrame = { (sampleBuffer, timestamp, bufferOrientation, isBufferMirrored) in
+      delegate.onFrame = {
+        (sampleBuffer, timestamp, bufferOrientation, isBufferMirrored, physicalBufferRotation) in
         // Prepare Frame + Metadata
         let metadata = self.getMediaSampleMetadata(
           at: timestamp,
           orientation: bufferOrientation,
-          isMirrored: isBufferMirrored)
+          isMirrored: isBufferMirrored,
+          physicalBufferRotation: physicalBufferRotation)
         let frame = HybridFrame(
           buffer: sampleBuffer,
           metadata: metadata)

packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraVideoFrameOutput.swift

@@ -72,7 +72,7 @@ final class HybridCameraVideoFrameOutput: HybridCameraVideoOutputSpec, NativeCam
       output.preservesDynamicHDRMetadata = true
     }
     // set the delegate to append to the Recorder
-    delegate.onFrame = { [weak self] buffer, timestamp, orientation, isMirrored in
+    delegate.onFrame = { [weak self] buffer, _, _, _, _ in
       guard let self else { return }
       self.onFrame(buffer, type: .video)
     }

packages/react-native-vision-camera/ios/Public/MediaSampleMetadata.swift

@@ -14,6 +14,8 @@ public struct MediaSampleMetadata {
   let timestamp: CMTime
   let orientation: CameraOrientation
   let isMirrored: Bool
+  let physicalBufferRotation: CameraOrientation
+  let isPhysicalBufferMirrored: Bool
 
   init(timestamp: CMTime, orientationFromOutput output: AVCaptureOutput) throws {
     guard let connection = output.connection(with: .video) else {
@@ -22,14 +24,34 @@ public struct MediaSampleMetadata {
     self.init(timestamp: timestamp, orientationFromConnection: connection)
   }
   init(timestamp: CMTime, orientationFromConnection connection: AVCaptureConnection) {
-    self.timestamp = timestamp
-    self.orientation = connection.orientation
-    self.isMirrored = connection.isVideoMirrored
+    let isMirrored = connection.isVideoMirrored
+    self.init(
+      timestamp: timestamp,
+      orientation: connection.orientation,
+      isMirrored: isMirrored,
+      physicalBufferRotation: connection.physicalBufferRotation,
+      isPhysicalBufferMirrored: isMirrored)
   }
   init(timestamp: CMTime, orientation: CameraOrientation, isMirrored: Bool) {
+    self.init(
+      timestamp: timestamp,
+      orientation: orientation,
+      isMirrored: isMirrored,
+      physicalBufferRotation: orientation,
+      isPhysicalBufferMirrored: isMirrored)
+  }
+  init(
+    timestamp: CMTime,
+    orientation: CameraOrientation,
+    isMirrored: Bool,
+    physicalBufferRotation: CameraOrientation,
+    isPhysicalBufferMirrored: Bool
+  ) {
     self.timestamp = timestamp
     self.orientation = orientation
     self.isMirrored = isMirrored
+    self.physicalBufferRotation = physicalBufferRotation
+    self.isPhysicalBufferMirrored = isPhysicalBufferMirrored
   }
 
   var uiImageOrientation: UIImage.Orientation {