From 583b4919a0e38a7cab057686d5d1e031718edbc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Braz=CC=87ewicz?= Date: Tue, 21 Jul 2026 13:35:02 +0200 Subject: [PATCH 1/3] improved texture rendering path, platform view rewrote to shared-metal implementation from webrtc renderre --- .../FlutterRTCVideoPlatformView.m | 128 +++--------------- .../FlutterRTCVideoPlatformViewController.m | 3 +- .../FlutterRTCVideoRenderer.m | 110 +++++++++++++-- 3 files changed, 117 insertions(+), 124 deletions(-) diff --git a/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformView.m b/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformView.m index 83f8b5d78a..ef90306c9f 100644 --- a/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformView.m +++ b/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformView.m @@ -1,134 +1,38 @@ #import "include/stream_webrtc_flutter/FlutterRTCVideoPlatformView.h" @implementation FlutterRTCVideoPlatformView { - CGSize _videoSize; - AVSampleBufferDisplayLayer* _videoLayer; - CGSize _remoteVideoSize; - CATransform3D _bufferTransform; - RTCVideoRotation _lastVideoRotation; + // The shared-Metal renderer, frames are handed straight to it (RTCVideoRenderer). + RTCVideoRenderingView* _renderingView; } - (instancetype)initWithFrame:(CGRect)frame { if (self = [super initWithFrame:frame]) { - _videoLayer = [[AVSampleBufferDisplayLayer alloc] init]; - _videoLayer.videoGravity = AVLayerVideoGravityResizeAspectFill; - _videoLayer.frame = CGRectZero; - _bufferTransform = CATransform3DIdentity; - _lastVideoRotation = RTCVideoRotation_0; - [self.layer addSublayer:_videoLayer]; + // Mirror the Swift SDK's VideoRenderer configuration + _renderingView = [[RTCVideoRenderingView alloc] initWithFrame:self.bounds]; + _renderingView.renderingBackend = RTCVideoRenderingBackendSharedMetal; + _renderingView.maxInFlightFrames = 2; + _renderingView.videoContentMode = UIViewContentModeScaleAspectFill; + _renderingView.enabled = YES; + _renderingView.autoresizingMask = + UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; + [self addSubview:_renderingView]; self.opaque = NO; } return self; } - (void)layoutSubviews { - _videoLayer.frame = self.bounds; - [_videoLayer removeAllAnimations]; + [super layoutSubviews]; + _renderingView.frame = self.bounds; } - (void)setSize:(CGSize)size { - _remoteVideoSize = size; + [_renderingView setSize:size]; } - (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { - CVPixelBufferRef pixelBuffer = nil; - if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { - pixelBuffer = ((RTCCVPixelBuffer*)frame.buffer).pixelBuffer; - CFRetain(pixelBuffer); - } else if ([frame.buffer isKindOfClass:[RTCI420Buffer class]]) { - pixelBuffer = [self toCVPixelBuffer:frame]; - } - - if (_lastVideoRotation != frame.rotation) { - _bufferTransform = [self fromFrameRotation:frame.rotation]; - _videoLayer.transform = _bufferTransform; - [_videoLayer layoutIfNeeded]; - _lastVideoRotation = frame.rotation; - } - - CMSampleBufferRef sampleBuffer = [self sampleBufferFromPixelBuffer:pixelBuffer]; - if (sampleBuffer) { - if (@available(iOS 14.0, *)) { - if ([_videoLayer requiresFlushToResumeDecoding]) { - [_videoLayer flushAndRemoveImage]; - } - } else { - // Fallback on earlier versions - } - [_videoLayer enqueueSampleBuffer:sampleBuffer]; - CFRelease(sampleBuffer); - } - - CFRelease(pixelBuffer); -} - -- (CVPixelBufferRef)toCVPixelBuffer:(RTCVideoFrame*)frame { - CVPixelBufferRef outputPixelBuffer; - NSDictionary* pixelAttributes = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}}; - CVPixelBufferCreate(kCFAllocatorDefault, frame.width, frame.height, - kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, - (__bridge CFDictionaryRef)(pixelAttributes), &outputPixelBuffer); - id i420Buffer = (RTCI420Buffer*)frame.buffer; - - CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); - // NV12 - uint8_t* dstY = CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 0); - const size_t dstYStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 0); - uint8_t* dstUV = CVPixelBufferGetBaseAddressOfPlane(outputPixelBuffer, 1); - const size_t dstUVStride = CVPixelBufferGetBytesPerRowOfPlane(outputPixelBuffer, 1); - - [RTCYUVHelper I420ToNV12:i420Buffer.dataY - srcStrideY:i420Buffer.strideY - srcU:i420Buffer.dataU - srcStrideU:i420Buffer.strideU - srcV:i420Buffer.dataV - srcStrideV:i420Buffer.strideV - dstY:dstY - dstStrideY:(int)dstYStride - dstUV:dstUV - dstStrideUV:(int)dstUVStride - width:i420Buffer.width - height:i420Buffer.height]; - - CVPixelBufferUnlockBaseAddress(outputPixelBuffer, 0); - return outputPixelBuffer; -} - -- (CMSampleBufferRef)sampleBufferFromPixelBuffer:(CVPixelBufferRef)pixelBuffer { - CMSampleBufferRef sampleBuffer = NULL; - OSStatus err = noErr; - CMVideoFormatDescriptionRef formatDesc = NULL; - err = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &formatDesc); - if (err != noErr) { - return nil; - } - CMSampleTimingInfo sampleTimingInfo = kCMTimingInfoInvalid; - err = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBuffer, formatDesc, - &sampleTimingInfo, &sampleBuffer); - if (sampleBuffer) { - CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES); - CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0); - CFDictionarySetValue(dict, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue); - } - if (err != noErr) { - return nil; - } - formatDesc = nil; - return sampleBuffer; -} - -- (CATransform3D)fromFrameRotation:(RTCVideoRotation)rotation { - switch (rotation) { - case RTCVideoRotation_0: - return CATransform3DIdentity; - case RTCVideoRotation_90: - return CATransform3DMakeRotation(M_PI / 2.0, 0, 0, 1); - case RTCVideoRotation_180: - return CATransform3DMakeRotation(M_PI, 0, 0, 1); - case RTCVideoRotation_270: - return CATransform3DMakeRotation(-M_PI / 0, 0, 0, 1); - } - return CATransform3DIdentity; + // Hand the frame straight to the shared-Metal renderer + [_renderingView renderFrame:frame]; } @end diff --git a/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformViewController.m b/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformViewController.m index 51d60bfd1c..71187a5dec 100644 --- a/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformViewController.m +++ b/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoPlatformViewController.m @@ -46,7 +46,8 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack { } _videoTrack = videoTrack; _isFirstFrameRendered = false; - if (!oldValue) { + if (oldValue) { + // Detach from the previous track. [oldValue removeRenderer:(id)self]; _videoView.frame = CGRectZero; } diff --git a/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m b/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m index 41cbdbcc4b..fb1287dec8 100644 --- a/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m +++ b/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m @@ -20,6 +20,11 @@ @implementation FlutterRTCVideoRenderer { bool _isFirstFrameRendered; bool _frameAvailable; os_unfair_lock _lock; + id _rotatedBuffer; // reused across frames for rotation + CVPixelBufferRef _nv12BufferRef; // reused NV12 output for the crop path + CVPixelBufferRef _currentBuffer; // buffer handed to Flutter this frame + uint8_t* _cropTempBuffer; // reused scratch for cropAndScaleTo + size_t _cropTempSize; } @synthesize textureId = _textureId; @@ -54,14 +59,30 @@ - (instancetype)initWithTextureRegistry:(id)registry - (CVPixelBufferRef)copyPixelBuffer { CVPixelBufferRef buffer = nil; os_unfair_lock_lock(&_lock); - if (_pixelBufferRef != nil && _frameAvailable) { - buffer = CVBufferRetain(_pixelBufferRef); + if (_currentBuffer != nil && _frameAvailable) { + buffer = CVBufferRetain(_currentBuffer); _frameAvailable = false; } os_unfair_lock_unlock(&_lock); return buffer; } +// (re)create the reused NV12 output buffer used by the crop path. +- (void)ensureNV12BufferWithWidth:(int)width height:(int)height { + if (_nv12BufferRef != nil && CVPixelBufferGetWidth(_nv12BufferRef) == (size_t)width && + CVPixelBufferGetHeight(_nv12BufferRef) == (size_t)height) { + return; + } + if (_nv12BufferRef != nil) { + CVBufferRelease(_nv12BufferRef); + _nv12BufferRef = nil; + } + NSDictionary* attrs = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}}; + CVPixelBufferCreate(kCFAllocatorDefault, width, height, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + (__bridge CFDictionaryRef)attrs, &_nv12BufferRef); +} + - (void)dispose { os_unfair_lock_lock(&_lock); [_registry unregisterTexture:_textureId]; @@ -70,6 +91,20 @@ - (void)dispose { CVBufferRelease(_pixelBufferRef); _pixelBufferRef = nil; } + if (_currentBuffer) { + CVBufferRelease(_currentBuffer); + _currentBuffer = nil; + } + if (_nv12BufferRef) { + CVBufferRelease(_nv12BufferRef); + _nv12BufferRef = nil; + } + if (_cropTempBuffer) { + free(_cropTempBuffer); + _cropTempBuffer = NULL; + _cropTempSize = 0; + } + _rotatedBuffer = nil; // release the reused rotation buffer _frameAvailable = false; os_unfair_lock_unlock(&_lock); } @@ -95,6 +130,11 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack { - (id)correctRotation:(const id)src withRotation:(RTCVideoRotation)rotation { + // an unrotated frame needs no rotation + if (rotation == RTCVideoRotation_0) { + return src; + } + int rotated_width = src.width; int rotated_height = src.height; @@ -104,8 +144,13 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack { rotated_height = temp; } - id buffer = [[RTCI420Buffer alloc] initWithWidth:rotated_width - height:rotated_height]; + // reuse one rotation buffer across frames; (re)allocate only when the + // rotated dimensions change, instead of allocating on every frame. + if (_rotatedBuffer == nil || _rotatedBuffer.width != rotated_width || + _rotatedBuffer.height != rotated_height) { + _rotatedBuffer = [[RTCI420Buffer alloc] initWithWidth:rotated_width height:rotated_height]; + } + id buffer = _rotatedBuffer; [RTCYUVHelper I420Rotate:src.dataY srcStrideY:src.strideY @@ -128,8 +173,8 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack { - (void)copyI420ToCVPixelBuffer:(CVPixelBufferRef)outputPixelBuffer withFrame:(RTCVideoFrame*)frame { - id i420Buffer = [self correctRotation:[frame.buffer toI420] - withRotation:frame.rotation]; + id srcI420 = [frame.buffer toI420]; + id i420Buffer = [self correctRotation:srcI420 withRotation:frame.rotation]; CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); const OSType pixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer); @@ -197,12 +242,55 @@ - (void)renderFrame:(RTCVideoFrame*)frame { os_unfair_lock_unlock(&_lock); return; } - if (!_frameAvailable && _pixelBufferRef) { - [self copyI420ToCVPixelBuffer:_pixelBufferRef withFrame:frame]; - if (_textureId != -1) { - [_registry textureFrameAvailable:_textureId]; + if (!_frameAvailable) { + CVPixelBufferRef output = NULL; // becomes the buffer handed to Flutter + + // hardware-decoded, upright frames are already NV12 in an IOSurface. + // Hand that to Flutter with no color conversion — zero-copy when the buffer + // is unpadded, otherwise a single crop-copy into a reused NV12 buffer. + if (frame.rotation == RTCVideoRotation_0 && + [frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { + RTCCVPixelBuffer* cvBuffer = (RTCCVPixelBuffer*)frame.buffer; + CVPixelBufferRef src = cvBuffer.pixelBuffer; + const OSType fmt = CVPixelBufferGetPixelFormatType(src); + if (fmt == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || + fmt == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { + if (![cvBuffer requiresCropping] && CVPixelBufferGetIOSurface(src) != NULL) { + output = src; // true zero-copy + } else { + [self ensureNV12BufferWithWidth:cvBuffer.cropWidth height:cvBuffer.cropHeight]; + int tmpSize = [cvBuffer bufferSizeForCroppingAndScalingToWidth:cvBuffer.cropWidth + height:cvBuffer.cropHeight]; + if (tmpSize > 0 && (_cropTempBuffer == NULL || _cropTempSize < (size_t)tmpSize)) { + free(_cropTempBuffer); + _cropTempBuffer = malloc((size_t)tmpSize); + _cropTempSize = (size_t)tmpSize; + } + if (_nv12BufferRef != nil && [cvBuffer cropAndScaleTo:_nv12BufferRef + withTempBuffer:_cropTempBuffer]) { + output = _nv12BufferRef; + } + } + } + } + + if (output == NULL && _pixelBufferRef != nil) { + // Fallback: software-decoded (I420) or rotated frames → BGRA convert path + [self copyI420ToCVPixelBuffer:_pixelBufferRef withFrame:frame]; + output = _pixelBufferRef; + } + + if (output != NULL) { + CVBufferRetain(output); + if (_currentBuffer != nil) { + CVBufferRelease(_currentBuffer); + } + _currentBuffer = output; + if (_textureId != -1) { + [_registry textureFrameAvailable:_textureId]; + } + _frameAvailable = true; } - _frameAvailable = true; } os_unfair_lock_unlock(&_lock); From 1c8a212a4da636c70c10883dcd9a3dd5b2aeffe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Braz=CC=87ewicz?= Date: Wed, 22 Jul 2026 12:33:13 +0200 Subject: [PATCH 2/3] migrate improvements to macos tree --- .../FlutterRTCVideoRenderer.m | 110 ++++++++++++++++-- 1 file changed, 99 insertions(+), 11 deletions(-) diff --git a/macos/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m b/macos/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m index 41cbdbcc4b..fb1287dec8 100644 --- a/macos/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m +++ b/macos/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m @@ -20,6 +20,11 @@ @implementation FlutterRTCVideoRenderer { bool _isFirstFrameRendered; bool _frameAvailable; os_unfair_lock _lock; + id _rotatedBuffer; // reused across frames for rotation + CVPixelBufferRef _nv12BufferRef; // reused NV12 output for the crop path + CVPixelBufferRef _currentBuffer; // buffer handed to Flutter this frame + uint8_t* _cropTempBuffer; // reused scratch for cropAndScaleTo + size_t _cropTempSize; } @synthesize textureId = _textureId; @@ -54,14 +59,30 @@ - (instancetype)initWithTextureRegistry:(id)registry - (CVPixelBufferRef)copyPixelBuffer { CVPixelBufferRef buffer = nil; os_unfair_lock_lock(&_lock); - if (_pixelBufferRef != nil && _frameAvailable) { - buffer = CVBufferRetain(_pixelBufferRef); + if (_currentBuffer != nil && _frameAvailable) { + buffer = CVBufferRetain(_currentBuffer); _frameAvailable = false; } os_unfair_lock_unlock(&_lock); return buffer; } +// (re)create the reused NV12 output buffer used by the crop path. +- (void)ensureNV12BufferWithWidth:(int)width height:(int)height { + if (_nv12BufferRef != nil && CVPixelBufferGetWidth(_nv12BufferRef) == (size_t)width && + CVPixelBufferGetHeight(_nv12BufferRef) == (size_t)height) { + return; + } + if (_nv12BufferRef != nil) { + CVBufferRelease(_nv12BufferRef); + _nv12BufferRef = nil; + } + NSDictionary* attrs = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}}; + CVPixelBufferCreate(kCFAllocatorDefault, width, height, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + (__bridge CFDictionaryRef)attrs, &_nv12BufferRef); +} + - (void)dispose { os_unfair_lock_lock(&_lock); [_registry unregisterTexture:_textureId]; @@ -70,6 +91,20 @@ - (void)dispose { CVBufferRelease(_pixelBufferRef); _pixelBufferRef = nil; } + if (_currentBuffer) { + CVBufferRelease(_currentBuffer); + _currentBuffer = nil; + } + if (_nv12BufferRef) { + CVBufferRelease(_nv12BufferRef); + _nv12BufferRef = nil; + } + if (_cropTempBuffer) { + free(_cropTempBuffer); + _cropTempBuffer = NULL; + _cropTempSize = 0; + } + _rotatedBuffer = nil; // release the reused rotation buffer _frameAvailable = false; os_unfair_lock_unlock(&_lock); } @@ -95,6 +130,11 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack { - (id)correctRotation:(const id)src withRotation:(RTCVideoRotation)rotation { + // an unrotated frame needs no rotation + if (rotation == RTCVideoRotation_0) { + return src; + } + int rotated_width = src.width; int rotated_height = src.height; @@ -104,8 +144,13 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack { rotated_height = temp; } - id buffer = [[RTCI420Buffer alloc] initWithWidth:rotated_width - height:rotated_height]; + // reuse one rotation buffer across frames; (re)allocate only when the + // rotated dimensions change, instead of allocating on every frame. + if (_rotatedBuffer == nil || _rotatedBuffer.width != rotated_width || + _rotatedBuffer.height != rotated_height) { + _rotatedBuffer = [[RTCI420Buffer alloc] initWithWidth:rotated_width height:rotated_height]; + } + id buffer = _rotatedBuffer; [RTCYUVHelper I420Rotate:src.dataY srcStrideY:src.strideY @@ -128,8 +173,8 @@ - (void)setVideoTrack:(RTCVideoTrack*)videoTrack { - (void)copyI420ToCVPixelBuffer:(CVPixelBufferRef)outputPixelBuffer withFrame:(RTCVideoFrame*)frame { - id i420Buffer = [self correctRotation:[frame.buffer toI420] - withRotation:frame.rotation]; + id srcI420 = [frame.buffer toI420]; + id i420Buffer = [self correctRotation:srcI420 withRotation:frame.rotation]; CVPixelBufferLockBaseAddress(outputPixelBuffer, 0); const OSType pixelFormat = CVPixelBufferGetPixelFormatType(outputPixelBuffer); @@ -197,12 +242,55 @@ - (void)renderFrame:(RTCVideoFrame*)frame { os_unfair_lock_unlock(&_lock); return; } - if (!_frameAvailable && _pixelBufferRef) { - [self copyI420ToCVPixelBuffer:_pixelBufferRef withFrame:frame]; - if (_textureId != -1) { - [_registry textureFrameAvailable:_textureId]; + if (!_frameAvailable) { + CVPixelBufferRef output = NULL; // becomes the buffer handed to Flutter + + // hardware-decoded, upright frames are already NV12 in an IOSurface. + // Hand that to Flutter with no color conversion — zero-copy when the buffer + // is unpadded, otherwise a single crop-copy into a reused NV12 buffer. + if (frame.rotation == RTCVideoRotation_0 && + [frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { + RTCCVPixelBuffer* cvBuffer = (RTCCVPixelBuffer*)frame.buffer; + CVPixelBufferRef src = cvBuffer.pixelBuffer; + const OSType fmt = CVPixelBufferGetPixelFormatType(src); + if (fmt == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || + fmt == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { + if (![cvBuffer requiresCropping] && CVPixelBufferGetIOSurface(src) != NULL) { + output = src; // true zero-copy + } else { + [self ensureNV12BufferWithWidth:cvBuffer.cropWidth height:cvBuffer.cropHeight]; + int tmpSize = [cvBuffer bufferSizeForCroppingAndScalingToWidth:cvBuffer.cropWidth + height:cvBuffer.cropHeight]; + if (tmpSize > 0 && (_cropTempBuffer == NULL || _cropTempSize < (size_t)tmpSize)) { + free(_cropTempBuffer); + _cropTempBuffer = malloc((size_t)tmpSize); + _cropTempSize = (size_t)tmpSize; + } + if (_nv12BufferRef != nil && [cvBuffer cropAndScaleTo:_nv12BufferRef + withTempBuffer:_cropTempBuffer]) { + output = _nv12BufferRef; + } + } + } + } + + if (output == NULL && _pixelBufferRef != nil) { + // Fallback: software-decoded (I420) or rotated frames → BGRA convert path + [self copyI420ToCVPixelBuffer:_pixelBufferRef withFrame:frame]; + output = _pixelBufferRef; + } + + if (output != NULL) { + CVBufferRetain(output); + if (_currentBuffer != nil) { + CVBufferRelease(_currentBuffer); + } + _currentBuffer = output; + if (_textureId != -1) { + [_registry textureFrameAvailable:_textureId]; + } + _frameAvailable = true; } - _frameAvailable = true; } os_unfair_lock_unlock(&_lock); From 0ba15ef707ec4a43f4df45386cccd9ed996e29a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Braz=CC=87ewicz?= Date: Wed, 22 Jul 2026 22:03:31 +0200 Subject: [PATCH 3/3] tweak --- .../FlutterRTCVideoRenderer.m | 54 +++++++++++-------- .../FlutterRTCVideoRenderer.m | 54 +++++++++++-------- 2 files changed, 66 insertions(+), 42 deletions(-) diff --git a/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m b/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m index fb1287dec8..3365901f6d 100644 --- a/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m +++ b/ios/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m @@ -21,7 +21,7 @@ @implementation FlutterRTCVideoRenderer { bool _frameAvailable; os_unfair_lock _lock; id _rotatedBuffer; // reused across frames for rotation - CVPixelBufferRef _nv12BufferRef; // reused NV12 output for the crop path + CVPixelBufferRef _nv12Buffers[2]; // double-buffered NV12 crop output (ping-pong) CVPixelBufferRef _currentBuffer; // buffer handed to Flutter this frame uint8_t* _cropTempBuffer; // reused scratch for cropAndScaleTo size_t _cropTempSize; @@ -67,20 +67,25 @@ - (CVPixelBufferRef)copyPixelBuffer { return buffer; } -// (re)create the reused NV12 output buffer used by the crop path. -- (void)ensureNV12BufferWithWidth:(int)width height:(int)height { - if (_nv12BufferRef != nil && CVPixelBufferGetWidth(_nv12BufferRef) == (size_t)width && - CVPixelBufferGetHeight(_nv12BufferRef) == (size_t)height) { - return; - } - if (_nv12BufferRef != nil) { - CVBufferRelease(_nv12BufferRef); - _nv12BufferRef = nil; - } +// (re)create the two reused NV12 output buffers used by the crop path. They are +// double-buffered (ping-pong) so the next crop never overwrites the buffer still +// being read by Flutter. The pixel format is taken from the source frame so +// video-range and full-range NV12 both keep their correct color range. +- (void)ensureNV12BuffersWithWidth:(int)width height:(int)height pixelFormat:(OSType)pixelFormat { NSDictionary* attrs = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}}; - CVPixelBufferCreate(kCFAllocatorDefault, width, height, - kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, - (__bridge CFDictionaryRef)attrs, &_nv12BufferRef); + for (int i = 0; i < 2; i++) { + if (_nv12Buffers[i] != nil && CVPixelBufferGetWidth(_nv12Buffers[i]) == (size_t)width && + CVPixelBufferGetHeight(_nv12Buffers[i]) == (size_t)height && + CVPixelBufferGetPixelFormatType(_nv12Buffers[i]) == pixelFormat) { + continue; + } + if (_nv12Buffers[i] != nil) { + CVBufferRelease(_nv12Buffers[i]); + _nv12Buffers[i] = nil; + } + CVPixelBufferCreate(kCFAllocatorDefault, width, height, pixelFormat, + (__bridge CFDictionaryRef)attrs, &_nv12Buffers[i]); + } } - (void)dispose { @@ -95,9 +100,11 @@ - (void)dispose { CVBufferRelease(_currentBuffer); _currentBuffer = nil; } - if (_nv12BufferRef) { - CVBufferRelease(_nv12BufferRef); - _nv12BufferRef = nil; + for (int i = 0; i < 2; i++) { + if (_nv12Buffers[i]) { + CVBufferRelease(_nv12Buffers[i]); + _nv12Buffers[i] = nil; + } } if (_cropTempBuffer) { free(_cropTempBuffer); @@ -258,7 +265,9 @@ - (void)renderFrame:(RTCVideoFrame*)frame { if (![cvBuffer requiresCropping] && CVPixelBufferGetIOSurface(src) != NULL) { output = src; // true zero-copy } else { - [self ensureNV12BufferWithWidth:cvBuffer.cropWidth height:cvBuffer.cropHeight]; + [self ensureNV12BuffersWithWidth:cvBuffer.cropWidth + height:cvBuffer.cropHeight + pixelFormat:fmt]; int tmpSize = [cvBuffer bufferSizeForCroppingAndScalingToWidth:cvBuffer.cropWidth height:cvBuffer.cropHeight]; if (tmpSize > 0 && (_cropTempBuffer == NULL || _cropTempSize < (size_t)tmpSize)) { @@ -266,9 +275,12 @@ - (void)renderFrame:(RTCVideoFrame*)frame { _cropTempBuffer = malloc((size_t)tmpSize); _cropTempSize = (size_t)tmpSize; } - if (_nv12BufferRef != nil && [cvBuffer cropAndScaleTo:_nv12BufferRef - withTempBuffer:_cropTempBuffer]) { - output = _nv12BufferRef; + // Pick the buffer not currently published to Flutter, so the crop never + // overwrites the frame the compositor may still be reading. + CVPixelBufferRef dst = + (_nv12Buffers[0] != _currentBuffer) ? _nv12Buffers[0] : _nv12Buffers[1]; + if (dst != nil && [cvBuffer cropAndScaleTo:dst withTempBuffer:_cropTempBuffer]) { + output = dst; } } } diff --git a/macos/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m b/macos/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m index fb1287dec8..3365901f6d 100644 --- a/macos/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m +++ b/macos/stream_webrtc_flutter/Sources/stream_webrtc_flutter/FlutterRTCVideoRenderer.m @@ -21,7 +21,7 @@ @implementation FlutterRTCVideoRenderer { bool _frameAvailable; os_unfair_lock _lock; id _rotatedBuffer; // reused across frames for rotation - CVPixelBufferRef _nv12BufferRef; // reused NV12 output for the crop path + CVPixelBufferRef _nv12Buffers[2]; // double-buffered NV12 crop output (ping-pong) CVPixelBufferRef _currentBuffer; // buffer handed to Flutter this frame uint8_t* _cropTempBuffer; // reused scratch for cropAndScaleTo size_t _cropTempSize; @@ -67,20 +67,25 @@ - (CVPixelBufferRef)copyPixelBuffer { return buffer; } -// (re)create the reused NV12 output buffer used by the crop path. -- (void)ensureNV12BufferWithWidth:(int)width height:(int)height { - if (_nv12BufferRef != nil && CVPixelBufferGetWidth(_nv12BufferRef) == (size_t)width && - CVPixelBufferGetHeight(_nv12BufferRef) == (size_t)height) { - return; - } - if (_nv12BufferRef != nil) { - CVBufferRelease(_nv12BufferRef); - _nv12BufferRef = nil; - } +// (re)create the two reused NV12 output buffers used by the crop path. They are +// double-buffered (ping-pong) so the next crop never overwrites the buffer still +// being read by Flutter. The pixel format is taken from the source frame so +// video-range and full-range NV12 both keep their correct color range. +- (void)ensureNV12BuffersWithWidth:(int)width height:(int)height pixelFormat:(OSType)pixelFormat { NSDictionary* attrs = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}}; - CVPixelBufferCreate(kCFAllocatorDefault, width, height, - kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, - (__bridge CFDictionaryRef)attrs, &_nv12BufferRef); + for (int i = 0; i < 2; i++) { + if (_nv12Buffers[i] != nil && CVPixelBufferGetWidth(_nv12Buffers[i]) == (size_t)width && + CVPixelBufferGetHeight(_nv12Buffers[i]) == (size_t)height && + CVPixelBufferGetPixelFormatType(_nv12Buffers[i]) == pixelFormat) { + continue; + } + if (_nv12Buffers[i] != nil) { + CVBufferRelease(_nv12Buffers[i]); + _nv12Buffers[i] = nil; + } + CVPixelBufferCreate(kCFAllocatorDefault, width, height, pixelFormat, + (__bridge CFDictionaryRef)attrs, &_nv12Buffers[i]); + } } - (void)dispose { @@ -95,9 +100,11 @@ - (void)dispose { CVBufferRelease(_currentBuffer); _currentBuffer = nil; } - if (_nv12BufferRef) { - CVBufferRelease(_nv12BufferRef); - _nv12BufferRef = nil; + for (int i = 0; i < 2; i++) { + if (_nv12Buffers[i]) { + CVBufferRelease(_nv12Buffers[i]); + _nv12Buffers[i] = nil; + } } if (_cropTempBuffer) { free(_cropTempBuffer); @@ -258,7 +265,9 @@ - (void)renderFrame:(RTCVideoFrame*)frame { if (![cvBuffer requiresCropping] && CVPixelBufferGetIOSurface(src) != NULL) { output = src; // true zero-copy } else { - [self ensureNV12BufferWithWidth:cvBuffer.cropWidth height:cvBuffer.cropHeight]; + [self ensureNV12BuffersWithWidth:cvBuffer.cropWidth + height:cvBuffer.cropHeight + pixelFormat:fmt]; int tmpSize = [cvBuffer bufferSizeForCroppingAndScalingToWidth:cvBuffer.cropWidth height:cvBuffer.cropHeight]; if (tmpSize > 0 && (_cropTempBuffer == NULL || _cropTempSize < (size_t)tmpSize)) { @@ -266,9 +275,12 @@ - (void)renderFrame:(RTCVideoFrame*)frame { _cropTempBuffer = malloc((size_t)tmpSize); _cropTempSize = (size_t)tmpSize; } - if (_nv12BufferRef != nil && [cvBuffer cropAndScaleTo:_nv12BufferRef - withTempBuffer:_cropTempBuffer]) { - output = _nv12BufferRef; + // Pick the buffer not currently published to Flutter, so the crop never + // overwrites the frame the compositor may still be reading. + CVPixelBufferRef dst = + (_nv12Buffers[0] != _currentBuffer) ? _nv12Buffers[0] : _nv12Buffers[1]; + if (dst != nil && [cvBuffer cropAndScaleTo:dst withTempBuffer:_cropTempBuffer]) { + output = dst; } } }