diff --git a/docs/content/docs/pixel-formats-map.mdx b/docs/content/docs/pixel-formats-map.mdx index 5491e106f5..415944d1b3 100644 --- a/docs/content/docs/pixel-formats-map.mdx +++ b/docs/content/docs/pixel-formats-map.mdx @@ -7,10 +7,12 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs' ### Requesting a Pixel Format -In a [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput), you have three options that affects your [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame)'s [`pixelFormat`](/api/react-native-vision-camera/hybrid-objects/Frame#pixelformat): +In a [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput), you have multiple options that affect your [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame)'s [`pixelFormat`](/api/react-native-vision-camera/hybrid-objects/Frame#pixelformat): - [`'native'`](/api/react-native-vision-camera/type-aliases/TargetVideoPixelFormat): Uses the [`CameraSessionConfig`](/api/react-native-vision-camera/hybrid-objects/CameraSessionConfig)'s [`nativePixelFormat`](/api/react-native-vision-camera/hybrid-objects/CameraSessionConfig#nativepixelformat) - this requires zero conversion and is generally the most efficient. - [`'yuv'`](/api/react-native-vision-camera/type-aliases/TargetVideoPixelFormat): Uses the platform default YUV format - often [`yuv-420-8-bit-full`](/api/react-native-vision-camera/type-aliases/VideoPixelFormat). In most cases, this requires little to no conversion and is very efficient. +- [`'yuv-420-8-bit-full'`](/api/react-native-vision-camera/type-aliases/TargetVideoPixelFormat): Explicitly uses YUV 4:2:0 8-bit **full-range** ([`yuv-420-8-bit-full`](/api/react-native-vision-camera/type-aliases/VideoPixelFormat)). Full-range is often preferred for CPU-based processing (e.g. ML inference). +- [`'yuv-420-8-bit-video'`](/api/react-native-vision-camera/type-aliases/TargetVideoPixelFormat): Explicitly uses YUV 4:2:0 8-bit **video-range** ([`yuv-420-8-bit-video`](/api/react-native-vision-camera/type-aliases/VideoPixelFormat)). Some GPU video pipelines require video-range, for example WebGPU (Dawn) can only import video-range YUV buffers as external textures. On Android, the color range of the Camera output cannot be configured, so this resolves to the same format as [`'yuv'`](/api/react-native-vision-camera/type-aliases/TargetVideoPixelFormat). - [`'rgb'`](/api/react-native-vision-camera/type-aliases/TargetVideoPixelFormat): Uses the platform default RGB format - often [`rgb-rgba-8-bit`](/api/react-native-vision-camera/type-aliases/VideoPixelFormat) or [`rgb-bgra-8-bit`](/api/react-native-vision-camera/type-aliases/VideoPixelFormat). This always requires conversion as the Camera operates in YUV (or BayerRAW), and uses ~2.6x more memory. > [!TIP] diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt index 17513602c3..2aa124bc16 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt @@ -81,8 +81,14 @@ class HybridFrameOutput( setOutputImageRotationEnabled(options.enablePhysicalBufferRotation) when (options.pixelFormat) { - TargetVideoPixelFormat.YUV -> { - // Use YUV_420_888 (CPU) + TargetVideoPixelFormat.YUV, + TargetVideoPixelFormat.YUV_420_8_BIT_FULL, + TargetVideoPixelFormat.YUV_420_8_BIT_VIDEO, + -> { + // Use YUV_420_888 (CPU). + // Android does not allow configuring the color range of the + // Camera output, so the explicit full/video-range targets + // resolve to the same format as `yuv` here. setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888) } TargetVideoPixelFormat.RGB -> { diff --git a/packages/react-native-vision-camera/ios/Extensions/Converters/AV+TargetVideoPixelFormat.swift b/packages/react-native-vision-camera/ios/Extensions/Converters/AV+TargetVideoPixelFormat.swift index ab145afc8c..ea84814de2 100644 --- a/packages/react-native-vision-camera/ios/Extensions/Converters/AV+TargetVideoPixelFormat.swift +++ b/packages/react-native-vision-camera/ios/Extensions/Converters/AV+TargetVideoPixelFormat.swift @@ -15,6 +15,10 @@ extension TargetVideoPixelFormat { return .native case .yuv: return .specific(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) + case .yuv4208BitFull: + return .specific(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) + case .yuv4208BitVideo: + return .specific(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) case .rgb: return .specific(kCVPixelFormatType_32BGRA) } diff --git a/packages/react-native-vision-camera/nitrogen/generated/android/c++/JTargetVideoPixelFormat.hpp b/packages/react-native-vision-camera/nitrogen/generated/android/c++/JTargetVideoPixelFormat.hpp index 1fbc26a6ca..2fc7bd3a02 100644 --- a/packages/react-native-vision-camera/nitrogen/generated/android/c++/JTargetVideoPixelFormat.hpp +++ b/packages/react-native-vision-camera/nitrogen/generated/android/c++/JTargetVideoPixelFormat.hpp @@ -42,6 +42,12 @@ namespace margelo::nitro::camera { static jni::alias_ref fromCpp(TargetVideoPixelFormat value) { static const auto clazz = javaClassStatic(); switch (value) { + case TargetVideoPixelFormat::YUV_420_8_BIT_VIDEO: + static const auto fieldYUV_420_8_BIT_VIDEO = clazz->getStaticField("YUV_420_8_BIT_VIDEO"); + return clazz->getStaticFieldValue(fieldYUV_420_8_BIT_VIDEO); + case TargetVideoPixelFormat::YUV_420_8_BIT_FULL: + static const auto fieldYUV_420_8_BIT_FULL = clazz->getStaticField("YUV_420_8_BIT_FULL"); + return clazz->getStaticFieldValue(fieldYUV_420_8_BIT_FULL); case TargetVideoPixelFormat::NATIVE: static const auto fieldNATIVE = clazz->getStaticField("NATIVE"); return clazz->getStaticFieldValue(fieldNATIVE); diff --git a/packages/react-native-vision-camera/nitrogen/generated/android/kotlin/com/margelo/nitro/camera/TargetVideoPixelFormat.kt b/packages/react-native-vision-camera/nitrogen/generated/android/kotlin/com/margelo/nitro/camera/TargetVideoPixelFormat.kt index 0d0aac2b3a..5ae9365323 100644 --- a/packages/react-native-vision-camera/nitrogen/generated/android/kotlin/com/margelo/nitro/camera/TargetVideoPixelFormat.kt +++ b/packages/react-native-vision-camera/nitrogen/generated/android/kotlin/com/margelo/nitro/camera/TargetVideoPixelFormat.kt @@ -16,9 +16,11 @@ import com.facebook.proguard.annotations.DoNotStrip @DoNotStrip @Keep enum class TargetVideoPixelFormat(@DoNotStrip @Keep val value: Int) { - NATIVE(0), - YUV(1), - RGB(2); + YUV_420_8_BIT_VIDEO(0), + YUV_420_8_BIT_FULL(1), + NATIVE(2), + YUV(3), + RGB(4); companion object } diff --git a/packages/react-native-vision-camera/nitrogen/generated/ios/swift/TargetVideoPixelFormat.swift b/packages/react-native-vision-camera/nitrogen/generated/ios/swift/TargetVideoPixelFormat.swift index 26866240a2..9fc0698393 100644 --- a/packages/react-native-vision-camera/nitrogen/generated/ios/swift/TargetVideoPixelFormat.swift +++ b/packages/react-native-vision-camera/nitrogen/generated/ios/swift/TargetVideoPixelFormat.swift @@ -17,6 +17,10 @@ public extension TargetVideoPixelFormat { */ init?(fromString string: String) { switch string { + case "yuv-420-8-bit-video": + self = .yuv4208BitVideo + case "yuv-420-8-bit-full": + self = .yuv4208BitFull case "native": self = .native case "yuv": @@ -33,6 +37,10 @@ public extension TargetVideoPixelFormat { */ var stringValue: String { switch self { + case .yuv4208BitVideo: + return "yuv-420-8-bit-video" + case .yuv4208BitFull: + return "yuv-420-8-bit-full" case .native: return "native" case .yuv: diff --git a/packages/react-native-vision-camera/nitrogen/generated/shared/c++/TargetVideoPixelFormat.hpp b/packages/react-native-vision-camera/nitrogen/generated/shared/c++/TargetVideoPixelFormat.hpp index 193d063907..a4535c92c3 100644 --- a/packages/react-native-vision-camera/nitrogen/generated/shared/c++/TargetVideoPixelFormat.hpp +++ b/packages/react-native-vision-camera/nitrogen/generated/shared/c++/TargetVideoPixelFormat.hpp @@ -29,9 +29,11 @@ namespace margelo::nitro::camera { * An enum which can be represented as a JavaScript union (TargetVideoPixelFormat). */ enum class TargetVideoPixelFormat { - NATIVE SWIFT_NAME(native) = 0, - YUV SWIFT_NAME(yuv) = 1, - RGB SWIFT_NAME(rgb) = 2, + YUV_420_8_BIT_VIDEO SWIFT_NAME(yuv4208BitVideo) = 0, + YUV_420_8_BIT_FULL SWIFT_NAME(yuv4208BitFull) = 1, + NATIVE SWIFT_NAME(native) = 2, + YUV SWIFT_NAME(yuv) = 3, + RGB SWIFT_NAME(rgb) = 4, } CLOSED_ENUM; } // namespace margelo::nitro::camera @@ -44,6 +46,8 @@ namespace margelo::nitro { static inline margelo::nitro::camera::TargetVideoPixelFormat fromJSI(jsi::Runtime& runtime, const jsi::Value& arg) { std::string unionValue = JSIConverter::fromJSI(runtime, arg); switch (hashString(unionValue.c_str(), unionValue.size())) { + case hashString("yuv-420-8-bit-video"): return margelo::nitro::camera::TargetVideoPixelFormat::YUV_420_8_BIT_VIDEO; + case hashString("yuv-420-8-bit-full"): return margelo::nitro::camera::TargetVideoPixelFormat::YUV_420_8_BIT_FULL; case hashString("native"): return margelo::nitro::camera::TargetVideoPixelFormat::NATIVE; case hashString("yuv"): return margelo::nitro::camera::TargetVideoPixelFormat::YUV; case hashString("rgb"): return margelo::nitro::camera::TargetVideoPixelFormat::RGB; @@ -53,6 +57,8 @@ namespace margelo::nitro { } static inline jsi::Value toJSI(jsi::Runtime& runtime, margelo::nitro::camera::TargetVideoPixelFormat arg) { switch (arg) { + case margelo::nitro::camera::TargetVideoPixelFormat::YUV_420_8_BIT_VIDEO: return JSIConverter::toJSI(runtime, "yuv-420-8-bit-video"); + case margelo::nitro::camera::TargetVideoPixelFormat::YUV_420_8_BIT_FULL: return JSIConverter::toJSI(runtime, "yuv-420-8-bit-full"); case margelo::nitro::camera::TargetVideoPixelFormat::NATIVE: return JSIConverter::toJSI(runtime, "native"); case margelo::nitro::camera::TargetVideoPixelFormat::YUV: return JSIConverter::toJSI(runtime, "yuv"); case margelo::nitro::camera::TargetVideoPixelFormat::RGB: return JSIConverter::toJSI(runtime, "rgb"); @@ -67,6 +73,8 @@ namespace margelo::nitro { } std::string unionValue = JSIConverter::fromJSI(runtime, value); switch (hashString(unionValue.c_str(), unionValue.size())) { + case hashString("yuv-420-8-bit-video"): + case hashString("yuv-420-8-bit-full"): case hashString("native"): case hashString("yuv"): case hashString("rgb"): diff --git a/packages/react-native-vision-camera/src/specs/common-types/VideoPixelFormat.ts b/packages/react-native-vision-camera/src/specs/common-types/VideoPixelFormat.ts index 94e4e9aa31..8eb16517e3 100644 --- a/packages/react-native-vision-camera/src/specs/common-types/VideoPixelFormat.ts +++ b/packages/react-native-vision-camera/src/specs/common-types/VideoPixelFormat.ts @@ -56,7 +56,23 @@ export type VideoPixelFormat = * format ({@linkcode PixelFormat | 'private'}) and requires zero conversion. * - `'yuv'`: Choose the YUV format closest to the Camera's native format. Often YUV 4:2:0 * 8-bit full-range like {@linkcode VideoPixelFormat | 'yuv-420-8-bit-full'}. + * - `'yuv-420-8-bit-full'`: Explicitly choose YUV 4:2:0 8-bit **full-range** + * ({@linkcode VideoPixelFormat | 'yuv-420-8-bit-full'}). Full-range is often preferred + * for CPU-based processing (e.g. ML inference). + * - `'yuv-420-8-bit-video'`: Explicitly choose YUV 4:2:0 8-bit **video-range** + * ({@linkcode VideoPixelFormat | 'yuv-420-8-bit-video'}). Video-range is required by + * some GPU video pipelines, for example WebGPU (Dawn) can only import video-range + * YUV buffers as external textures. * - `'rgb'`: Choose an RGB format. Often 8-bit BGRA like * {@linkcode VideoPixelFormat | 'rgb-bgra-8-bit'}. + * + * Note: On Android, `'yuv-420-8-bit-full'` and `'yuv-420-8-bit-video'` both resolve + * to the same YUV format as `'yuv'` (`YUV_420_888`), as Android does not allow + * configuring the color range of the Camera output. */ -export type TargetVideoPixelFormat = 'native' | 'yuv' | 'rgb' +export type TargetVideoPixelFormat = + | 'native' + | 'yuv' + | 'yuv-420-8-bit-full' + | 'yuv-420-8-bit-video' + | 'rgb'