From 104da91f42a6932da3c1c04dddc85242539f5882 Mon Sep 17 00:00:00 2001 From: Test Date: Thu, 17 Sep 2026 16:42:09 +0200 Subject: [PATCH] fix: prefer a phase-detection camera format for receipt capture --- .../components/Camera/index.native.tsx | 20 ++--- .../index.android.ts | 18 +++++ .../index.ios.ts | 17 ++++ .../getReceiptCameraFormatFilters/index.ts | 9 +++ .../getReceiptCameraFormatFilters/types.ts | 10 +++ .../index.android.ts | 10 +++ .../preferPhaseDetectionFormat/index.ios.ts | 29 +++++++ .../utils/preferPhaseDetectionFormat/index.ts | 9 +++ .../utils/preferPhaseDetectionFormat/types.ts | 10 +++ .../unit/getReceiptCameraFormatFiltersTest.ts | 38 +++++++++ tests/unit/preferPhaseDetectionFormatTest.ts | 79 +++++++++++++++++++ 11 files changed, 234 insertions(+), 15 deletions(-) create mode 100644 src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.android.ts create mode 100644 src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.ios.ts create mode 100644 src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.ts create mode 100644 src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/types.ts create mode 100644 src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.android.ts create mode 100644 src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.ios.ts create mode 100644 src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.ts create mode 100644 src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/types.ts create mode 100644 tests/unit/getReceiptCameraFormatFiltersTest.ts create mode 100644 tests/unit/preferPhaseDetectionFormatTest.ts diff --git a/src/pages/iou/request/step/IOURequestStepScan/components/Camera/index.native.tsx b/src/pages/iou/request/step/IOURequestStepScan/components/Camera/index.native.tsx index 0f43073ebf27..e77f39a76c74 100644 --- a/src/pages/iou/request/step/IOURequestStepScan/components/Camera/index.native.tsx +++ b/src/pages/iou/request/step/IOURequestStepScan/components/Camera/index.native.tsx @@ -22,6 +22,8 @@ import ReceiptPreviews from '@pages/iou/request/step/IOURequestStepScan/componen import ScannerControlsBar from '@pages/iou/request/step/IOURequestStepScan/components/ScannerControlsBar'; import getCameraAspectRatio from '@pages/iou/request/step/IOURequestStepScan/getCameraAspectRatio'; import useCameraInitTelemetry from '@pages/iou/request/step/IOURequestStepScan/hooks/useCameraInitTelemetry'; +import getReceiptCameraFormatFilters from '@pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters'; +import preferPhaseDetectionFormat from '@pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat'; import startReceiptPrepareSpan from '@pages/iou/request/step/IOURequestStepScan/utils/startReceiptPrepareSpan'; import CONST from '@src/CONST'; @@ -30,7 +32,7 @@ import type {FileObject} from '@src/types/utils/Attachment'; import type {PhotoFile} from 'react-native-vision-camera'; import React, {useRef} from 'react'; -import {Alert, Platform, View} from 'react-native'; +import {Alert, View} from 'react-native'; import {RESULTS} from 'react-native-permissions'; import {useAnimatedStyle, useSharedValue, withSequence, withTiming} from 'react-native-reanimated'; import {useCameraFormat} from 'react-native-vision-camera'; @@ -83,20 +85,8 @@ function Camera({onCapture, onPicked, shouldAcceptMultipleFiles = false, onLayou cameraFocusIndicatorAnimatedStyle, } = useNativeCamera({onFocusStart, onFocusCleanup}); - // Prioritize photoResolution so the format selector picks the configured PHOTO_WIDTH/PHOTO_HEIGHT - // format. videoResolution is platform-specific: - // - iOS: match the photo target — `takeSnapshot` reads from the video pipeline, so a smaller - // video resolution would degrade the snapshot capture quality. - // - Android: keep screen dimensions — `takeSnapshot` is a GPU screenshot of the preview surface - // and doesn't depend on video resolution; constraining to screen size avoids burning GPU on a - // higher-than-needed preview. - const format = useCameraFormat(device, [ - {photoAspectRatio: CONST.RECEIPT_CAMERA.PHOTO_ASPECT_RATIO}, - {photoResolution: {width: CONST.RECEIPT_CAMERA.PHOTO_WIDTH, height: CONST.RECEIPT_CAMERA.PHOTO_HEIGHT}}, - Platform.OS === 'ios' - ? {videoResolution: {width: CONST.RECEIPT_CAMERA.PHOTO_WIDTH, height: CONST.RECEIPT_CAMERA.PHOTO_HEIGHT}} - : {videoResolution: {width: windowHeight, height: windowWidth}}, - ]); + const selectedFormat = useCameraFormat(device, getReceiptCameraFormatFilters({windowWidth, windowHeight})); + const format = preferPhaseDetectionFormat({device, format: selectedFormat}); const cameraAspectRatio = getCameraAspectRatio(format, isInLandscapeMode); const fps = format ? Math.min(Math.max(30, format.minFps), format.maxFps) : 30; diff --git a/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.android.ts b/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.android.ts new file mode 100644 index 000000000000..a86b2056d64a --- /dev/null +++ b/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.android.ts @@ -0,0 +1,18 @@ +/** + * Builds the filter list that picks which camera format the receipt scanner runs at. + */ +import CONST from '@src/CONST'; + +import type GetReceiptCameraFormatFilters from './types'; + +// Prioritize photoResolution so the format selector picks the configured PHOTO_WIDTH/PHOTO_HEIGHT +// format. videoResolution keeps screen dimensions because `takeSnapshot` is a GPU screenshot of the +// preview surface and doesn't depend on video resolution. Constraining it to screen size avoids +// burning GPU on a higher-than-needed preview. +const getReceiptCameraFormatFilters: GetReceiptCameraFormatFilters = ({windowWidth, windowHeight}) => [ + {photoAspectRatio: CONST.RECEIPT_CAMERA.PHOTO_ASPECT_RATIO}, + {photoResolution: {width: CONST.RECEIPT_CAMERA.PHOTO_WIDTH, height: CONST.RECEIPT_CAMERA.PHOTO_HEIGHT}}, + {videoResolution: {width: windowHeight, height: windowWidth}}, +]; + +export default getReceiptCameraFormatFilters; diff --git a/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.ios.ts b/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.ios.ts new file mode 100644 index 000000000000..a0ebaa3694ad --- /dev/null +++ b/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.ios.ts @@ -0,0 +1,17 @@ +/** + * Builds the filter list that picks which camera format the receipt scanner runs at. + */ +import CONST from '@src/CONST'; + +import type GetReceiptCameraFormatFilters from './types'; + +// Prioritize photoResolution so the format selector picks the configured PHOTO_WIDTH/PHOTO_HEIGHT +// format. videoResolution matches the photo target because `takeSnapshot` reads from the video +// pipeline, so a smaller video resolution would degrade the snapshot capture quality. +const getReceiptCameraFormatFilters: GetReceiptCameraFormatFilters = () => [ + {photoAspectRatio: CONST.RECEIPT_CAMERA.PHOTO_ASPECT_RATIO}, + {photoResolution: {width: CONST.RECEIPT_CAMERA.PHOTO_WIDTH, height: CONST.RECEIPT_CAMERA.PHOTO_HEIGHT}}, + {videoResolution: {width: CONST.RECEIPT_CAMERA.PHOTO_WIDTH, height: CONST.RECEIPT_CAMERA.PHOTO_HEIGHT}}, +]; + +export default getReceiptCameraFormatFilters; diff --git a/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.ts b/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.ts new file mode 100644 index 000000000000..a54c5a3a0a8b --- /dev/null +++ b/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.ts @@ -0,0 +1,9 @@ +/** + * Builds the filter list that picks which camera format the receipt scanner runs at. + */ +import type GetReceiptCameraFormatFilters from './types'; + +// Only the native camera reads these, so there is nothing to choose between on web. +const getReceiptCameraFormatFilters: GetReceiptCameraFormatFilters = () => []; + +export default getReceiptCameraFormatFilters; diff --git a/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/types.ts b/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/types.ts new file mode 100644 index 000000000000..10cd962ca991 --- /dev/null +++ b/src/pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/types.ts @@ -0,0 +1,10 @@ +import type {FormatFilter} from 'react-native-vision-camera'; + +type GetReceiptCameraFormatFiltersParams = { + windowWidth: number; + windowHeight: number; +}; + +type GetReceiptCameraFormatFilters = (params: GetReceiptCameraFormatFiltersParams) => FormatFilter[]; + +export default GetReceiptCameraFormatFilters; diff --git a/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.android.ts b/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.android.ts new file mode 100644 index 000000000000..3d63bb5fbc81 --- /dev/null +++ b/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.android.ts @@ -0,0 +1,10 @@ +/** + * Upgrades an already chosen camera format to a phase-detection equivalent where the device has one. + */ +import type PreferPhaseDetectionFormat from './types'; + +// Vision Camera reports every focus-capable Android camera as contrast-detection, so there is never a +// phase-detection format to upgrade to. +const preferPhaseDetectionFormat: PreferPhaseDetectionFormat = ({format}) => format; + +export default preferPhaseDetectionFormat; diff --git a/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.ios.ts b/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.ios.ts new file mode 100644 index 000000000000..ccd9fbcbddc7 --- /dev/null +++ b/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.ios.ts @@ -0,0 +1,29 @@ +/** + * Upgrades an already chosen camera format to a phase-detection equivalent where the device has one. + */ +import type PreferPhaseDetectionFormat from './types'; + +// This can't be a format filter. `useCameraFormat` scores filters additively, so adding one changes the +// weights of the rest and can flip a resolution or aspect ratio match. Running after selection keeps +// that choice, and matching on dimensions and frame rate means the swap only changes the autofocus +// system. +const preferPhaseDetectionFormat: PreferPhaseDetectionFormat = ({device, format}) => { + if (!device || !format || format.autoFocusSystem === 'phase-detection') { + return format; + } + + const equivalentPhaseDetectionFormat = device.formats.find( + (candidate) => + candidate.autoFocusSystem === 'phase-detection' && + candidate.photoWidth === format.photoWidth && + candidate.photoHeight === format.photoHeight && + candidate.videoWidth === format.videoWidth && + candidate.videoHeight === format.videoHeight && + candidate.minFps === format.minFps && + candidate.maxFps === format.maxFps, + ); + + return equivalentPhaseDetectionFormat ?? format; +}; + +export default preferPhaseDetectionFormat; diff --git a/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.ts b/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.ts new file mode 100644 index 000000000000..363ac4ad3ffd --- /dev/null +++ b/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.ts @@ -0,0 +1,9 @@ +/** + * Upgrades an already chosen camera format to a phase-detection equivalent where the device has one. + */ +import type PreferPhaseDetectionFormat from './types'; + +// Only the native camera picks a format, so there is nothing to upgrade on web. +const preferPhaseDetectionFormat: PreferPhaseDetectionFormat = ({format}) => format; + +export default preferPhaseDetectionFormat; diff --git a/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/types.ts b/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/types.ts new file mode 100644 index 000000000000..5e8e4870e1c0 --- /dev/null +++ b/src/pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/types.ts @@ -0,0 +1,10 @@ +import type {CameraDevice, CameraDeviceFormat} from 'react-native-vision-camera'; + +type PreferPhaseDetectionFormatParams = { + device: Pick | undefined; + format: CameraDeviceFormat | undefined; +}; + +type PreferPhaseDetectionFormat = (params: PreferPhaseDetectionFormatParams) => CameraDeviceFormat | undefined; + +export default PreferPhaseDetectionFormat; diff --git a/tests/unit/getReceiptCameraFormatFiltersTest.ts b/tests/unit/getReceiptCameraFormatFiltersTest.ts new file mode 100644 index 000000000000..3b8760cff871 --- /dev/null +++ b/tests/unit/getReceiptCameraFormatFiltersTest.ts @@ -0,0 +1,38 @@ +import getReceiptCameraFormatFiltersAndroid from '@pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.android'; +import getReceiptCameraFormatFiltersIOS from '@pages/iou/request/step/IOURequestStepScan/utils/getReceiptCameraFormatFilters/index.ios'; + +import CONST from '@src/CONST'; + +const WINDOW_WIDTH = 390; +const WINDOW_HEIGHT = 844; + +describe('getReceiptCameraFormatFilters', () => { + it('ranks aspect ratio and photo resolution above everything else', () => { + const filters = getReceiptCameraFormatFiltersIOS({windowWidth: WINDOW_WIDTH, windowHeight: WINDOW_HEIGHT}); + + expect(filters.at(0)).toEqual({photoAspectRatio: CONST.RECEIPT_CAMERA.PHOTO_ASPECT_RATIO}); + expect(filters.at(1)).toEqual({photoResolution: {width: CONST.RECEIPT_CAMERA.PHOTO_WIDTH, height: CONST.RECEIPT_CAMERA.PHOTO_HEIGHT}}); + }); + + it('matches the iOS video resolution to the photo target so snapshots keep their quality', () => { + const filters = getReceiptCameraFormatFiltersIOS({windowWidth: WINDOW_WIDTH, windowHeight: WINDOW_HEIGHT}); + + expect(filters.at(2)).toEqual({videoResolution: {width: CONST.RECEIPT_CAMERA.PHOTO_WIDTH, height: CONST.RECEIPT_CAMERA.PHOTO_HEIGHT}}); + }); + + it('caps the Android video resolution to the screen, which is all its preview screenshot needs', () => { + const filters = getReceiptCameraFormatFiltersAndroid({windowWidth: WINDOW_WIDTH, windowHeight: WINDOW_HEIGHT}); + + expect(filters.at(2)).toEqual({videoResolution: {width: WINDOW_HEIGHT, height: WINDOW_WIDTH}}); + }); + + it('carries no autoFocusSystem filter, which would change the weights of every other filter', () => { + const iosFilters = getReceiptCameraFormatFiltersIOS({windowWidth: WINDOW_WIDTH, windowHeight: WINDOW_HEIGHT}); + const androidFilters = getReceiptCameraFormatFiltersAndroid({windowWidth: WINDOW_WIDTH, windowHeight: WINDOW_HEIGHT}); + + expect(iosFilters).toHaveLength(3); + expect(androidFilters).toHaveLength(3); + expect(iosFilters.some((filter) => 'autoFocusSystem' in filter)).toBe(false); + expect(androidFilters.some((filter) => 'autoFocusSystem' in filter)).toBe(false); + }); +}); diff --git a/tests/unit/preferPhaseDetectionFormatTest.ts b/tests/unit/preferPhaseDetectionFormatTest.ts new file mode 100644 index 000000000000..e5654cab44ef --- /dev/null +++ b/tests/unit/preferPhaseDetectionFormatTest.ts @@ -0,0 +1,79 @@ +import preferPhaseDetectionFormatAndroid from '@pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.android'; +import preferPhaseDetectionFormatIOS from '@pages/iou/request/step/IOURequestStepScan/utils/preferPhaseDetectionFormat/index.ios'; + +import type {CameraDevice, CameraDeviceFormat} from 'react-native-vision-camera'; + +type FormatOverrides = Partial>; + +function createFormat(overrides: FormatOverrides = {}): CameraDeviceFormat { + return { + photoWidth: 2880, + photoHeight: 2160, + videoWidth: 2880, + videoHeight: 2160, + minFps: 30, + maxFps: 30, + autoFocusSystem: 'contrast-detection', + minISO: 0, + maxISO: 0, + fieldOfView: 0, + supportsVideoHdr: false, + supportsPhotoHdr: false, + supportsDepthCapture: false, + videoStabilizationModes: [], + ...overrides, + }; +} + +function createDevice(formats: CameraDeviceFormat[]): Pick { + return {formats}; +} + +const CONTRAST_FORMAT = createFormat(); +const EQUIVALENT_PHASE_FORMAT = createFormat({autoFocusSystem: 'phase-detection'}); + +describe('preferPhaseDetectionFormat', () => { + it('swaps to a phase-detection format with identical dimensions and frame rate', () => { + const device = createDevice([CONTRAST_FORMAT, EQUIVALENT_PHASE_FORMAT]); + + expect(preferPhaseDetectionFormatIOS({device, format: CONTRAST_FORMAT})).toBe(EQUIVALENT_PHASE_FORMAT); + }); + + it('keeps the selected format when the only phase-detection candidate has a different resolution', () => { + const lowerResolutionPhaseFormat = createFormat({autoFocusSystem: 'phase-detection', photoWidth: 1920, photoHeight: 1440}); + const device = createDevice([CONTRAST_FORMAT, lowerResolutionPhaseFormat]); + + expect(preferPhaseDetectionFormatIOS({device, format: CONTRAST_FORMAT})).toBe(CONTRAST_FORMAT); + }); + + it('keeps the selected format when the only phase-detection candidate has a different frame rate range', () => { + const highFrameRatePhaseFormat = createFormat({autoFocusSystem: 'phase-detection', maxFps: 60}); + const device = createDevice([CONTRAST_FORMAT, highFrameRatePhaseFormat]); + + expect(preferPhaseDetectionFormatIOS({device, format: CONTRAST_FORMAT})).toBe(CONTRAST_FORMAT); + }); + + it('keeps the selected format when no phase-detection candidate exists', () => { + const device = createDevice([CONTRAST_FORMAT, createFormat({videoWidth: 1920, videoHeight: 1440})]); + + expect(preferPhaseDetectionFormatIOS({device, format: CONTRAST_FORMAT})).toBe(CONTRAST_FORMAT); + }); + + it('leaves a format that already uses phase detection alone', () => { + const device = createDevice([EQUIVALENT_PHASE_FORMAT, createFormat({autoFocusSystem: 'phase-detection'})]); + + expect(preferPhaseDetectionFormatIOS({device, format: EQUIVALENT_PHASE_FORMAT})).toBe(EQUIVALENT_PHASE_FORMAT); + }); + + it('passes an undefined format through', () => { + const device = createDevice([EQUIVALENT_PHASE_FORMAT]); + + expect(preferPhaseDetectionFormatIOS({device, format: undefined})).toBeUndefined(); + }); + + it('never swaps on Android, where no format reports phase detection', () => { + const device = createDevice([CONTRAST_FORMAT, EQUIVALENT_PHASE_FORMAT]); + + expect(preferPhaseDetectionFormatAndroid({device, format: CONTRAST_FORMAT})).toBe(CONTRAST_FORMAT); + }); +});