feat: iOS video renderer improvements

ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformView.m

@@ -1,134 +1,38 @@
 #import "include/stream_webrtc_flutter/FlutterRTCVideoPlatformView.h"
 
 @implementation FlutterRTCVideoPlatformView {
-  CGSize _videoSize;
-  AVSampleBufferDisplayLayer* _videoLayer;
-  CGSize _remoteVideoSize;
-  CATransform3D _bufferTransform;
-  RTCVideoRotation _lastVideoRotation;
+  // The shared-Metal renderer, frames are handed straight to it (RTCVideoRenderer).
+  RTCVideoRenderingView* _renderingView;
 }
 
 - (instancetype)initWithFrame:(CGRect)frame {
   if (self = [super initWithFrame:frame]) {
-    _videoLayer = [[AVSampleBufferDisplayLayer alloc] init];
-    _videoLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;
-    _videoLayer.frame = CGRectZero;
-    _bufferTransform = CATransform3DIdentity;
-    _lastVideoRotation = RTCVideoRotation_0;
-    [self.layer addSublayer:_videoLayer];
+    // Mirror the Swift SDK's VideoRenderer configuration
+    _renderingView = [[RTCVideoRenderingView alloc] initWithFrame:self.bounds];
+    _renderingView.renderingBackend = RTCVideoRenderingBackendSharedMetal;
+    _renderingView.maxInFlightFrames = 2;
+    _renderingView.videoContentMode = UIViewContentModeScaleAspectFill;
+    _renderingView.enabled = YES;
+    _renderingView.autoresizingMask =
+        UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
+    [self addSubview:_renderingView];
     self.opaque = NO;
   }
   return self;
 }
 
 - (void)layoutSubviews {
-  _videoLayer.frame = self.bounds;
-  [_videoLayer removeAllAnimations];
+  [super layoutSubviews];
+  _renderingView.frame = self.bounds;
 }
 
 - (void)setSize:(CGSize)size {
-  _remoteVideoSize = size;
+  [_renderingView setSize:size];
 }
 
 - (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame {
-  CVPixelBufferRef pixelBuffer = nil;
-  if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
-    pixelBuffer = ((RTCCVPixelBuffer*)frame.buffer).pixelBuffer;
-    CFRetain(pixelBuffer);
-  } else if ([frame.buffer isKindOfClass:[RTCI420Buffer class]]) {
-    pixelBuffer = [self toCVPixelBuffer:frame];
-  }
-
-  if (_lastVideoRotation != frame.rotation) {
-    _bufferTransform = [self fromFrameRotation:frame.rotation];
-    _videoLayer.transform = _bufferTransform;
-    [_videoLayer layoutIfNeeded];
-    _lastVideoRotation = frame.rotation;
-  }
-
-  CMSampleBufferRef sampleBuffer = [self sampleBufferFromPixelBuffer:pixelBuffer];
-  if (sampleBuffer) {
-    if (@available(iOS 14.0, *)) {
-      if ([_videoLayer requiresFlushToResumeDecoding]) {
-        [_videoLayer flushAndRemoveImage];
-      }
-    } else {
-      // Fallback on earlier versions
-    }
-    [_videoLayer enqueueSampleBuffer:sampleBuffer];
-    CFRelease(sampleBuffer);
-  }
-
-  CFRelease(pixelBuffer);
-}
-
-- (CVPixelBufferRef)toCVPixelBuffer:(RTCVideoFrame*)frame {
-  CVPixelBufferRef outputPixelBuffer;
-  NSDictionary* pixelAttributes = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}};
-  CVPixelBufferCreate(kCFAllocatorDefault, frame.width, frame.height,
-                      kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
-                      (__bridge CFDictionaryRef)(pixelAttributes), &outputPixelBuffer);
-  id<RTCI420Buffer> i420Buffer = (RTCI420Buffer*)frame.buffer;
-
-  CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
-  // NV12
-  uint8_t* dstY = CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0);
-  const size_t dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0);
-  uint8_t* dstUV = CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1);
-  const size_t dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1);
-
-  [RTCYUVHelper I420ToNV12:i420Buffer.dataY
-                srcStrideY:i420Buffer.strideY
-                      srcU:i420Buffer.dataU
-                srcStrideU:i420Buffer.strideU
-                      srcV:i420Buffer.dataV
-                srcStrideV:i420Buffer.strideV
-                      dstY:dstY
-                dstStrideY:(int)dstYStride
-                     dstUV:dstUV
-               dstStrideUV:(int)dstUVStride
-                     width:i420Buffer.width
-                    height:i420Buffer.height];
-
-  CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0);
-  return outputPixelBuffer;
-}
-
-- (CMSampleBufferRef)sampleBufferFromPixelBuffer:(CVPixelBufferRef)pixelBuffer {
-  CMSampleBufferRef sampleBuffer = NULL;
-  OSStatus err = noErr;
-  CMVideoFormatDescriptionRef formatDesc = NULL;
-  err = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &formatDesc);
-  if (err != noErr) {
-    return nil;
-  }
-  CMSampleTimingInfo sampleTimingInfo = kCMTimingInfoInvalid;
-  err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBuffer, formatDesc,
-                                                 &sampleTimingInfo, &sampleBuffer);
-  if (sampleBuffer) {
-    CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES);
-    CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
-    CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
-  }
-  if (err != noErr) {
-    return nil;
-  }
-  formatDesc = nil;
-  return sampleBuffer;
-}
-
-- (CATransform3D)fromFrameRotation:(RTCVideoRotation)rotation {
-  switch (rotation) {
-    case RTCVideoRotation_0:
-      return CATransform3DIdentity;
-    case RTCVideoRotation_90:
-      return CATransform3DMakeRotation(M_PI / 2.0, 0, 0, 1);
-    case RTCVideoRotation_180:
-      return CATransform3DMakeRotation(M_PI, 0, 0, 1);
-    case RTCVideoRotation_270:
-      return CATransform3DMakeRotation(-M_PI / 0, 0, 0, 1);
-  }
-  return CATransform3DIdentity;
+  // Hand the frame straight to the shared-Metal renderer
+  [_renderingView renderFrame:frame];
 }
 
 @end

ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformViewController.m

@@ -46,7 +46,8 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack {
   }
   _videoTrack = videoTrack;
   _isFirstFrameRendered = false;
-  if (!oldValue) {
+  if (oldValue) {
+    // Detach from the previous track.
     [oldValue removeRenderer:(id<RTCVideoRenderer>)self];
     _videoView.frame = CGRectZero;
   }

ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m

@@ -20,6 +20,11 @@ @implementation FlutterRTCVideoRenderer {
   bool _isFirstFrameRendered;
   bool _frameAvailable;
   os_unfair_lock _lock;
+  id<RTCI420Buffer> _rotatedBuffer;  // reused across frames for rotation
+  CVPixelBufferRef _nv12Buffers[2];  // double-buffered NV12 crop output (ping-pong)
+  CVPixelBufferRef _currentBuffer;   // buffer handed to Flutter this frame
+  uint8_t* _cropTempBuffer;          // reused scratch for cropAndScaleTo
+  size_t _cropTempSize;
 }
 
 @synthesize textureId = _textureId;
@@ -54,14 +59,35 @@ - (instancetype)initWithTextureRegistry:(id<FlutterTextureRegistry>)registry
 - (CVPixelBufferRef)copyPixelBuffer {
   CVPixelBufferRef buffer = nil;
   os_unfair_lock_lock(&_lock);
-  if (_pixelBufferRef != nil && _frameAvailable) {
-    buffer = CVBufferRetain(_pixelBufferRef);
+  if (_currentBuffer != nil && _frameAvailable) {
+    buffer = CVBufferRetain(_currentBuffer);
     _frameAvailable = false;
   }
   os_unfair_lock_unlock(&_lock);
   return buffer;
 }
 
+// (re)create the two reused NV12 output buffers used by the crop path. They are
+// double-buffered (ping-pong) so the next crop never overwrites the buffer still
+// being read by Flutter. The pixel format is taken from the source frame so
+// video-range and full-range NV12 both keep their correct color range.
+- (void)ensureNV12BuffersWithWidth:(int)width height:(int)height pixelFormat:(OSType)pixelFormat {
+  NSDictionary* attrs = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}};
+  for (int i = 0; i < 2; i++) {
+    if (_nv12Buffers[i] != nil && CVPixelBufferGetWidth(_nv12Buffers[i]) == (size_t)width &&
+        CVPixelBufferGetHeight(_nv12Buffers[i]) == (size_t)height &&
+        CVPixelBufferGetPixelFormatType(_nv12Buffers[i]) == pixelFormat) {
+      continue;
+    }
+    if (_nv12Buffers[i] != nil) {
+      CVBufferRelease(_nv12Buffers[i]);
+      _nv12Buffers[i] = nil;
+    }
+    CVPixelBufferCreate(kCFAllocatorDefault, width, height, pixelFormat,
+                        (__bridge CFDictionaryRef)attrs, &_nv12Buffers[i]);
+  }
+}
+
 - (void)dispose {
   os_unfair_lock_lock(&_lock);
   [_registry unregisterTexture:_textureId];
@@ -70,6 +96,22 @@ - (void)dispose {
     CVBufferRelease(_pixelBufferRef);
     _pixelBufferRef = nil;
   }
+  if (_currentBuffer) {
+    CVBufferRelease(_currentBuffer);
+    _currentBuffer = nil;
+  }
+  for (int i = 0; i < 2; i++) {
+    if (_nv12Buffers[i]) {
+      CVBufferRelease(_nv12Buffers[i]);
+      _nv12Buffers[i] = nil;
+    }
+  }
+  if (_cropTempBuffer) {
+    free(_cropTempBuffer);
+    _cropTempBuffer = NULL;
+    _cropTempSize = 0;
+  }
+  _rotatedBuffer = nil;  // release the reused rotation buffer
   _frameAvailable = false;
   os_unfair_lock_unlock(&_lock);
 }
@@ -95,6 +137,11 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack {
 
 - (id<RTCI420Buffer>)correctRotation:(const id<RTCI420Buffer>)src
                         withRotation:(RTCVideoRotation)rotation {
+  // an unrotated frame needs no rotation
+  if (rotation == RTCVideoRotation_0) {
+    return src;
+  }
+
   int rotated_width = src.width;
   int rotated_height = src.height;
 
@@ -104,8 +151,13 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack {
     rotated_height = temp;
   }
 
-  id<RTCI420Buffer> buffer = [[RTCI420Buffer alloc] initWithWidth:rotated_width
-                                                           height:rotated_height];
+  // reuse one rotation buffer across frames; (re)allocate only when the
+  // rotated dimensions change, instead of allocating on every frame.
+  if (_rotatedBuffer == nil || _rotatedBuffer.width != rotated_width ||
+      _rotatedBuffer.height != rotated_height) {
+    _rotatedBuffer = [[RTCI420Buffer alloc] initWithWidth:rotated_width height:rotated_height];
+  }
+  id<RTCI420Buffer> buffer = _rotatedBuffer;
 
   [RTCYUVHelper I420Rotate:src.dataY
                 srcStrideY:src.strideY
@@ -128,8 +180,8 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack {
 
 - (void)copyI420ToCVPixelBuffer:(CVPixelBufferRef)outputPixelBuffer
                       withFrame:(RTCVideoFrame*)frame {
-  id<RTCI420Buffer> i420Buffer = [self correctRotation:[frame.buffer toI420]
-                                          withRotation:frame.rotation];
+  id<RTCI420Buffer> srcI420 = [frame.buffer toI420];
+  id<RTCI420Buffer> i420Buffer = [self correctRotation:srcI420 withRotation:frame.rotation];
   CVPixelBufferLockBaseAddress(outputPixelBuffer, 0);
 
   const OSType pixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer);
@@ -197,12 +249,60 @@ - (void)renderFrame:(RTCVideoFrame*)frame {
     os_unfair_lock_unlock(&_lock);
     return;
   }
-  if (!_frameAvailable && _pixelBufferRef) {
-    [self copyI420ToCVPixelBuffer:_pixelBufferRef withFrame:frame];
-    if (_textureId != -1) {
-      [_registry textureFrameAvailable:_textureId];
+  if (!_frameAvailable) {
+    CVPixelBufferRef output = NULL;  // becomes the buffer handed to Flutter
+
+    // hardware-decoded, upright frames are already NV12 in an IOSurface.
+    // Hand that to Flutter with no color conversion — zero-copy when the buffer
+    // is unpadded, otherwise a single crop-copy into a reused NV12 buffer.
+    if (frame.rotation == RTCVideoRotation_0 &&
+        [frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
+      RTCCVPixelBuffer* cvBuffer = (RTCCVPixelBuffer*)frame.buffer;
+      CVPixelBufferRef src = cvBuffer.pixelBuffer;
+      const OSType fmt = CVPixelBufferGetPixelFormatType(src);
+      if (fmt == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
+          fmt == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
+        if (![cvBuffer requiresCropping] && CVPixelBufferGetIOSurface(src) != NULL) {
+          output = src;  // true zero-copy
+        } else {
+          [self ensureNV12BuffersWithWidth:cvBuffer.cropWidth
+                                    height:cvBuffer.cropHeight
+                               pixelFormat:fmt];
+          int tmpSize = [cvBuffer bufferSizeForCroppingAndScalingToWidth:cvBuffer.cropWidth
+                                                                  height:cvBuffer.cropHeight];
+          if (tmpSize > 0 && (_cropTempBuffer == NULL || _cropTempSize < (size_t)tmpSize)) {
+            free(_cropTempBuffer);
+            _cropTempBuffer = malloc((size_t)tmpSize);
+            _cropTempSize = (size_t)tmpSize;
+          }
+          // Pick the buffer not currently published to Flutter, so the crop never
+          // overwrites the frame the compositor may still be reading.
+          CVPixelBufferRef dst =
+              (_nv12Buffers[0] != _currentBuffer) ? _nv12Buffers[0] : _nv12Buffers[1];
+          if (dst != nil && [cvBuffer cropAndScaleTo:dst withTempBuffer:_cropTempBuffer]) {
+            output = dst;
+          }
+        }
+      }
+    }
+
+    if (output == NULL && _pixelBufferRef != nil) {
+      // Fallback: software-decoded (I420) or rotated frames → BGRA convert path
+      [self copyI420ToCVPixelBuffer:_pixelBufferRef withFrame:frame];
+      output = _pixelBufferRef;
+    }
+
+    if (output != NULL) {
+      CVBufferRetain(output);
+      if (_currentBuffer != nil) {
+        CVBufferRelease(_currentBuffer);
+      }
+      _currentBuffer = output;
+      if (_textureId != -1) {
+        [_registry textureFrameAvailable:_textureId];
+      }
+      _frameAvailable = true;
     }
-    _frameAvailable = true;
   }
   os_unfair_lock_unlock(&_lock);