From 14b51530d495a1c7e48eaf95413755aeb01767f5 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:34:55 +0100 Subject: [PATCH] iOS: scan dense codes at 1080p, from raw frames, with close-range focus Dense PDF417 codes were hard to read on iPhones while Android read them easily. Three causes in the iOS camera, none of them new: - A session opened with no preview size (every Vision scanner) ran at 640x480. It now gets 1080p, falling back to 720p. - Frames were always re-encoded as JPEG at quality 0.8, whatever format the listener asked for. A raw request now gets the pixels as RGBA8888 (iOS has no NV21), which Apple Vision already accepts; the JPEG is encoded lazily only if asked for. - Nothing configured focus for close range, and recent iPhone Pro main cameras cannot focus closer than ~20cm. While frames are delivered the camera now uses continuous near-range autofocus and zooms to the lens' minimum focus distance (Apple's AVCamBarcode formula); the application's zoom and focus mode are put back when delivery stops. Co-Authored-By: Claude Opus 5.5 --- .../src/com/codename1/camera/CameraFrame.java | 5 +- Ports/iOSPort/nativeSources/CN1Camera.h | 3 +- Ports/iOSPort/nativeSources/CN1Camera.m | 142 ++++++++++++++++-- .../com/codename1/impl/ios/IOSCameraImpl.java | 53 ++++++- .../src/com/codename1/impl/ios/IOSNative.java | 2 +- 5 files changed, 187 insertions(+), 18 deletions(-) diff --git a/CodenameOne/src/com/codename1/camera/CameraFrame.java b/CodenameOne/src/com/codename1/camera/CameraFrame.java index a3c4cf389e3..c909e8bef35 100644 --- a/CodenameOne/src/com/codename1/camera/CameraFrame.java +++ b/CodenameOne/src/com/codename1/camera/CameraFrame.java @@ -91,7 +91,10 @@ public synchronized byte[] getJpegBytes() { /// `CameraSessionOptions#frameFormat(FrameFormat)`. This is `null` for /// JPEG requests and may also be `null` when the current port cannot /// expose the requested raw format. In that case - /// {@link #getJpegBytes()} remains available as a fallback. + /// {@link #getJpegBytes()} remains available as a fallback. A port may + /// also answer a raw request in another raw format -- iOS captures BGRA + /// and has no NV21, so it delivers RGBA8888 -- so read {@link #getFormat()} + /// before interpreting these bytes. public byte[] getRawBytes() { return rawBytes; } diff --git a/Ports/iOSPort/nativeSources/CN1Camera.h b/Ports/iOSPort/nativeSources/CN1Camera.h index c4c012158a4..1c805588cca 100644 --- a/Ports/iOSPort/nativeSources/CN1Camera.h +++ b/Ports/iOSPort/nativeSources/CN1Camera.h @@ -52,6 +52,7 @@ @property (nonatomic, strong) CN1View *previewView; @property (nonatomic, strong) dispatch_queue_t videoQueue; @property (nonatomic, assign) BOOL frameDeliveryEnabled; +@property (nonatomic, assign) BOOL frameDeliveryRaw; @property (nonatomic, assign) int frameMaxFps; @property (nonatomic, assign) NSTimeInterval lastFrameTime; @property (nonatomic, assign) BOOL frameInFlight; @@ -68,7 +69,7 @@ callbackId:(int)callbackId; - (BOOL)startVideoToPath:(NSString *)path captureAudio:(BOOL)audio; - (void)stopVideoWithCallback:(int)callbackId; -- (void)setFrameDeliveryEnabled:(BOOL)enabled maxFps:(int)maxFps; +- (void)setFrameDeliveryEnabled:(BOOL)enabled maxFps:(int)maxFps raw:(BOOL)raw; - (void)setFlashMode:(int)mode; - (void)setZoomRatio:(float)ratio; - (void)focusAtX:(float)xNorm y:(float)yNorm; diff --git a/Ports/iOSPort/nativeSources/CN1Camera.m b/Ports/iOSPort/nativeSources/CN1Camera.m index 93343eb5e39..68e5c2ff910 100644 --- a/Ports/iOSPort/nativeSources/CN1Camera.m +++ b/Ports/iOSPort/nativeSources/CN1Camera.m @@ -67,6 +67,16 @@ @interface CN1Camera () @property (nonatomic, copy) NSString *pendingPhotoFilePath; @property (nonatomic, assign) int pendingPhotoCallbackId; @property (nonatomic, assign) int pendingPhotoQuality; +// What configureForCloseRangeAnalysis: changed, so turning it off puts back +// the application's own zoom and focus mode -- the zoom only when nothing has +// zoomed since, so a setZoom made during analysis is kept. +@property (nonatomic, assign) CGFloat analysisZoom; +@property (nonatomic, assign) CGFloat zoomBeforeAnalysis; +@property (nonatomic, assign) NSInteger focusModeBeforeAnalysis; +// One RGBA buffer per session, reused across raw frames. A 1080p frame is +// 8MB; allocating one per frame on top of the Java array it is copied into +// was a large-object churn the GC had to absorb at the frame rate. +@property (nonatomic, strong) NSMutableData *rawFrameBuffer; @end @implementation CN1Camera @@ -173,8 +183,17 @@ - (BOOL)openWithCameraId:(NSString *)cameraId self.session = [[AVCaptureSession alloc] init]; [self.session beginConfiguration]; - // Preset selection: prefer 1280x720 for previewW > 800, 640x480 otherwise. - if (previewW >= 1280 && [self.session canSetSessionPreset:AVCaptureSessionPreset1280x720]) { + // Preset selection. No requested size (0, "let the platform pick") gets + // 1080p -- for every session, not only analysis, which also lifts default + // still photos off 640x480, the ceiling the preset puts on them: it used to fall through to 640x480, and that is too coarse for a + // dense PDF417 to decode -- every Vision scanner ran on it, since + // VisionCameraView requests no size. An explicit request keeps the old + // mapping: 1280x720 for a width of 1280 or more, 640x480 otherwise. + if (previewW <= 0 && [self.session canSetSessionPreset:AVCaptureSessionPreset1920x1080]) { + self.session.sessionPreset = AVCaptureSessionPreset1920x1080; + } else if (previewW <= 0 && [self.session canSetSessionPreset:AVCaptureSessionPreset1280x720]) { + self.session.sessionPreset = AVCaptureSessionPreset1280x720; + } else if (previewW >= 1280 && [self.session canSetSessionPreset:AVCaptureSessionPreset1280x720]) { self.session.sessionPreset = AVCaptureSessionPreset1280x720; } else if ([self.session canSetSessionPreset:AVCaptureSessionPreset640x480]) { self.session.sessionPreset = AVCaptureSessionPreset640x480; @@ -275,6 +294,11 @@ - (void)closeSession { self.photoOutput = nil; self.movieOutput = nil; self.session = nil; + // On the delivery queue, because a frame may still be rendering into the + // buffer there; the block keeps self alive until it has run. + dispatch_async(self.videoQueue, ^{ + self.rawFrameBuffer = nil; + }); } #pragma mark - Photo @@ -363,9 +387,69 @@ - (void)captureOutput:(AVCaptureFileOutput *)output #pragma mark - Frame delivery -- (void)setFrameDeliveryEnabled:(BOOL)enabled maxFps:(int)maxFps { +- (void)setFrameDeliveryEnabled:(BOOL)enabled maxFps:(int)maxFps raw:(BOOL)raw { + BOOL wasEnabled = self.frameDeliveryEnabled; self.frameDeliveryEnabled = enabled; + self.frameDeliveryRaw = raw; if (maxFps > 0) self.frameMaxFps = maxFps; + if (enabled != wasEnabled) { + [self configureForCloseRangeAnalysis:enabled]; + } +} + +// A frame listener is analysis -- codes, documents, text -- and that is held +// close to the lens. The defaults are tuned for a scene at a distance, and a +// newer iPhone's main camera cannot focus closer than ~20cm at all: a PDF417 +// held where it fills the screen stays blurred however long autofocus hunts. +// So while frames are delivered, focus continuously with a near-range bias, +// and zoom in just enough that a 20mm code filling 80% of the preview sits at +// the lens' minimum focus distance (Apple's AVCamBarcode formula). Focus mode, +// range and zoom are all put back when delivery stops, so a plain photo or +// video session is untouched. +- (void)configureForCloseRangeAnalysis:(BOOL)on { +#if !TARGET_OS_OSX + AVCaptureDevice *d = self.device; + if (!d || ![d lockForConfiguration:nil]) return; + if (on) { + self.focusModeBeforeAnalysis = d.focusMode; + if ([d isFocusModeSupported:AVCaptureFocusModeContinuousAutoFocus]) { + d.focusMode = AVCaptureFocusModeContinuousAutoFocus; + } + } else if ([d isFocusModeSupported:(AVCaptureFocusMode)self.focusModeBeforeAnalysis]) { + d.focusMode = (AVCaptureFocusMode)self.focusModeBeforeAnalysis; + } + if (d.isAutoFocusRangeRestrictionSupported) { + d.autoFocusRangeRestriction = on ? AVCaptureAutoFocusRangeRestrictionNear + : AVCaptureAutoFocusRangeRestrictionNone; + } + if (@available(iOS 15.0, *)) { + if (on) { + CGFloat zoom = d.videoZoomFactor; + self.zoomBeforeAnalysis = zoom; + NSInteger minFocusMm = d.minimumFocusDistance; + float fov = d.activeFormat.videoFieldOfView; + if (minFocusMm > 0 && fov > 0) { + double halfFov = (fov / 2.0) * M_PI / 180.0; + double minSubjectMm = (20.0 / 0.8) / tan(halfFov); + if ((double)minFocusMm > minSubjectMm) { + // Never zoom OUT of what the application already chose. + zoom = MAX(zoom, (CGFloat)((double)minFocusMm / minSubjectMm)); + } + } + d.videoZoomFactor = MAX(1.0, MIN(zoom, d.activeFormat.videoMaxZoomFactor)); + self.analysisZoom = d.videoZoomFactor; + } else { + if (self.analysisZoom > 0 && d.videoZoomFactor == self.analysisZoom) { + d.videoZoomFactor = MAX(1.0, MIN(self.zoomBeforeAnalysis, + d.activeFormat.videoMaxZoomFactor)); + } + self.analysisZoom = 0; + } + } + [d unlockForConfiguration]; +#else + (void)on; +#endif } - (void)captureOutput:(AVCaptureOutput *)output @@ -400,13 +484,6 @@ - (void)captureOutput:(AVCaptureOutput *)output [ctx retain]; #endif } - CGImageRef cg = [ctx createCGImage:ci fromRect:rect]; - if (!cg) return; - CN1Image *img = [CN1Image imageWithCGImage:cg]; - CGImageRelease(cg); - NSData *jpeg = UIImageJPEGRepresentation(img, 0.8); - if (!jpeg) return; - int rotation = 0; if (connection.isVideoOrientationSupported) { switch (connection.videoOrientation) { @@ -416,8 +493,37 @@ - (void)captureOutput:(AVCaptureOutput *)output case AVCaptureVideoOrientationLandscapeRight: rotation = 0; break; } } - int64_t ts = (int64_t)(now * 1e9); + + if (self.frameDeliveryRaw) { + // A raw format was asked for, so hand over the pixels themselves as + // RGBA8888. They used to go through a JPEG at quality 0.8 whatever was + // requested, and the artifacts ate the thin bars of a PDF417 before + // Vision saw them. The capture is BGRA; Core Image swizzles it. + if (w <= 0 || h <= 0 || w > 16384 || h > 16384) return; + NSUInteger need = (NSUInteger)w * (NSUInteger)h * 4; + if (self.rawFrameBuffer.length != need) { + self.rawFrameBuffer = [NSMutableData dataWithLength:need]; + } + NSMutableData *rgba = self.rawFrameBuffer; + if (!rgba) return; + CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); + [ctx render:ci toBitmap:rgba.mutableBytes rowBytes:(ptrdiff_t)w * 4 + bounds:rect format:kCIFormatRGBA8 colorSpace:cs]; + CGColorSpaceRelease(cs); + self.frameInFlight = YES; + [self fireRawFrame:rgba width:w height:h rotation:rotation timestamp:ts]; + self.frameInFlight = NO; + return; + } + + CGImageRef cg = [ctx createCGImage:ci fromRect:rect]; + if (!cg) return; + CN1Image *img = [CN1Image imageWithCGImage:cg]; + CGImageRelease(cg); + NSData *jpeg = UIImageJPEGRepresentation(img, 0.8); + if (!jpeg) return; + self.frameInFlight = YES; [self fireFrame:jpeg width:w height:h rotation:rotation timestamp:ts]; self.frameInFlight = NO; @@ -482,6 +588,13 @@ - (void)fireFrame:(NSData *)jpeg width:(int)w height:(int)h CN1_THREAD_GET_STATE_PASS_ARG bytes, w, h, rotation, (JAVA_LONG)ts); } +- (void)fireRawFrame:(NSData *)rgba width:(int)w height:(int)h + rotation:(int)rotation timestamp:(int64_t)ts { + JAVA_OBJECT bytes = nsDataToByteArr(rgba); + com_codename1_impl_ios_IOSCameraImpl_onRawFrameDelivered___byte_1ARRAY_int_int_int_long( + CN1_THREAD_GET_STATE_PASS_ARG bytes, w, h, rotation, (JAVA_LONG)ts); +} + - (void)firePhotoCaptured:(int)cbId bytes:(NSData *)jpeg path:(NSString *)path width:(int)w height:(int)h { JAVA_OBJECT b = nsDataToByteArr(jpeg); @@ -698,16 +811,17 @@ void com_codename1_impl_ios_IOSNative_cn1CameraStopVideo___long_int( #endif } -void com_codename1_impl_ios_IOSNative_cn1CameraSetFrameDelivery___long_boolean_int( +void com_codename1_impl_ios_IOSNative_cn1CameraSetFrameDelivery___long_boolean_int_boolean( CN1_THREAD_STATE_MULTI_ARG JAVA_OBJECT instanceObject, - JAVA_LONG sessionPeer, JAVA_BOOLEAN enabled, JAVA_INT maxFps) { + JAVA_LONG sessionPeer, JAVA_BOOLEAN enabled, JAVA_INT maxFps, + JAVA_BOOLEAN raw) { #if defined(INCLUDE_CN1_CAMERA) && !TARGET_OS_TV #ifndef CN1_USE_ARC CN1Camera *cam = (CN1Camera *)sessionPeer; #else CN1Camera *cam = (__bridge CN1Camera *)(void *)sessionPeer; #endif - [cam setFrameDeliveryEnabled:enabled maxFps:maxFps]; + [cam setFrameDeliveryEnabled:enabled maxFps:maxFps raw:raw]; #endif } diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSCameraImpl.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSCameraImpl.java index 0f56ec22bbe..f2a8ffe161d 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSCameraImpl.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSCameraImpl.java @@ -34,10 +34,13 @@ import com.codename1.impl.CameraImpl; import com.codename1.io.Log; import com.codename1.ui.Display; +import com.codename1.ui.Image; import com.codename1.ui.PeerComponent; import com.codename1.ui.geom.Dimension; +import com.codename1.ui.util.ImageIO; import com.codename1.util.AsyncResource; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -182,7 +185,8 @@ public void setFrameListener(FrameListener listener, FrameFormat format, int max this.frameFormat = format == null ? FrameFormat.JPEG : format; if (sessionPeer != 0) { IOSImplementation.nativeInstance.cn1CameraSetFrameDelivery( - sessionPeer, listener != null, Math.max(1, maxFps)); + sessionPeer, listener != null, Math.max(1, maxFps), + this.frameFormat != FrameFormat.JPEG); } } @@ -268,6 +272,53 @@ public static void onFrameDelivered(byte[] jpegBytes, int width, int height, } } + /// Called from native code with a single video frame as RGBA8888, when the + /// listener asked for a raw format. iOS captures BGRA and has no NV21, so an + /// NV21 request is answered with RGBA8888 and the frame says so in + /// `getFormat()`. The JPEG is encoded only if somebody asks for it. + public static void onRawFrameDelivered(final byte[] rgba, final int width, + final int height, int rotationDegrees, + long timestampNanos) { + IOSCameraImpl self = ACTIVE; + if (self == null) return; + FrameListener l = self.frameListener; + if (l == null) return; + if (!self.listenerBusy.compareAndSet(false, true)) { + return; // drop frame; previous still in flight + } + try { + CameraFrame f = new CameraFrame(null, rgba, width, height, + rotationDegrees, timestampNanos, FrameFormat.RGBA8888, + new CameraFrame.JpegEncoder() { + public byte[] encode() { + return encodeRgbaAsJpeg(rgba, width, height); + } + }); + l.onFrame(f); + } catch (Throwable t) { + Log.e(t); + } finally { + self.listenerBusy.set(false); + } + } + + private static byte[] encodeRgbaAsJpeg(byte[] rgba, int width, int height) { + int[] argb = new int[width * height]; + for (int i = 0, p = 0; i < argb.length; i++, p += 4) { + argb[i] = 0xff000000 | ((rgba[p] & 0xff) << 16) + | ((rgba[p + 1] & 0xff) << 8) | (rgba[p + 2] & 0xff); + } + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ImageIO.getImageIO().save(Image.createImage(argb, width, height), + out, ImageIO.FORMAT_JPEG, 0.8f); + return out.toByteArray(); + } catch (IOException e) { + Log.e(e); + return null; + } + } + /// Called from native code when a still photo capture completes successfully. public static void onPhotoCaptured(int callbackId, byte[] jpeg, String filePath, int width, int height) { diff --git a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java index 9295389bab5..eebcd8ec37f 100644 --- a/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java +++ b/Ports/iOSPort/src/com/codename1/impl/ios/IOSNative.java @@ -727,7 +727,7 @@ native long createFairPlayVideoComponent(String url, String certificateUrl, Stri native void cn1CameraTakePhoto(long sessionPeer, int width, int height, int jpegQuality, String filePath, int callbackId); native boolean cn1CameraStartVideo(long sessionPeer, String filePath, boolean captureAudio); native void cn1CameraStopVideo(long sessionPeer, int callbackId); - native void cn1CameraSetFrameDelivery(long sessionPeer, boolean enabled, int maxFps); + native void cn1CameraSetFrameDelivery(long sessionPeer, boolean enabled, int maxFps, boolean raw); native void cn1CameraSetFlash(long sessionPeer, int mode); native void cn1CameraSetZoom(long sessionPeer, float ratio); native void cn1CameraFocus(long sessionPeer, float xNorm, float yNorm);