From b660514a8ae69832244900cd4cdd9d44e2ffe12e Mon Sep 17 00:00:00 2001 From: OpenSec <323596076+opensec-intelligence@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:28:59 +0530 Subject: [PATCH 1/2] perf(coding-agent): images within the limits skip photon; resizes run in Bun.Image Pi-Bolt runs photon's WebAssembly in the interpreter. Reading a 1800x1100 PNG that needs no resize cost 1.16 s of CPU (a full decode to learn its size), and a 2442x1438 screenshot 36.7 s (decode, Lanczos3 resize, a PNG and five JPEG encodes, then a second decode when the tool result was normalized). - Read width and height from the PNG, JPEG, GIF or WebP structure. An image within the limits is returned as it is, without photon or a worker. The parsers accept only intact files (every PNG chunk CRC, end markers, frame bounds); anything else takes the decode path as before. - Encode candidates one at a time and stop at the first that fits: the JPEG qualities are encoded only when the PNG is too large. Same output. - The read tool marks the image blocks it produced; tool-result normalization skips them when the options match instead of processing them again. - When the runtime has Bun.Image (Pi-Bolt's runtime, Bun 1.4), resize there: native decoders, SIMD Lanczos3, libspng/libjpeg-turbo encoders, off the main thread. Same target sizes, candidate order and size limit; different bytes. Photon remains the path under Node and the fallback when it fails. CPU per `pi -p` read (0.9.5 runtime, medians of 3): 1800x1100 PNG 1162 -> 56 ms (a text read is 57 ms), 1600x1200 JPEG 3651 -> 68 ms, 2442x1438 PNG 36.7 -> 0.52 s, 3024x1964 PNG 39.9 -> 0.70 s, 4032x3024 JPEG 75.5 -> 1.09 s. --- packages/coding-agent/src/core/tools/read.ts | 18 +- .../src/utils/exif-orientation.ts | 3 +- .../coding-agent/src/utils/image-header.ts | 306 ++++++++++++ .../coding-agent/src/utils/image-process.ts | 47 ++ .../src/utils/image-resize-core.ts | 230 +++++---- .../src/utils/image-resize-native.ts | 83 ++++ .../coding-agent/src/utils/image-resize.ts | 28 +- .../src/utils/tool-result-images.ts | 14 +- .../coding-agent/test/image-header.test.ts | 459 ++++++++++++++++++ .../test/image-resize-encode.test.ts | 139 ++++++ .../suite/agent-session-read-images.test.ts | 101 ++++ 11 files changed, 1315 insertions(+), 113 deletions(-) create mode 100644 packages/coding-agent/src/utils/image-header.ts create mode 100644 packages/coding-agent/src/utils/image-resize-native.ts create mode 100644 packages/coding-agent/test/image-header.test.ts create mode 100644 packages/coding-agent/test/image-resize-encode.test.ts create mode 100644 packages/coding-agent/test/suite/agent-session-read-images.test.ts diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index 45b9e85c5..921d22116 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -3,7 +3,7 @@ import type { Api, ImageContent, Model, ModelImageResizeOptions, TextContent } f import { constants } from "fs"; import { access as fsAccess, readFile as fsReadFile } from "fs/promises"; import { type Static, Type } from "typebox"; -import { processImage } from "../../utils/image-process.ts"; +import { markProcessedImageBlock, processImage } from "../../utils/image-process.ts"; import { detectSupportedImageMimeTypeFromFile } from "../../utils/mime.ts"; import { detachString } from "../../utils/text.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; @@ -163,10 +163,11 @@ export function createReadToolDefinition( if (mimeType) { // Read image as binary. const buffer = await ops.readFile(absolutePath); - const processed = await processImage(buffer, mimeType, { + const processOptions = { autoResizeImages, resizeOptions: ctx?.model?.inputLimits?.images?.resize ?? fallbackResizeOptions, - }); + }; + const processed = await processImage(buffer, mimeType, processOptions); if (!processed.ok) { let textNote = `Read image file [${mimeType}]\n${processed.message}`; if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; @@ -175,10 +176,13 @@ export function createReadToolDefinition( let textNote = `Read image file [${processed.mimeType}]`; if (processed.hints.length > 0) textNote += `\n${processed.hints.join("\n")}`; if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; - content = [ - { type: "text", text: textNote }, - { type: "image", data: processed.data, mimeType: processed.mimeType }, - ]; + const image: ImageContent = { + type: "image", + data: processed.data, + mimeType: processed.mimeType, + }; + markProcessedImageBlock(image, processOptions); + content = [{ type: "text", text: textNote }, image]; } } else { // Read text content. diff --git a/packages/coding-agent/src/utils/exif-orientation.ts b/packages/coding-agent/src/utils/exif-orientation.ts index d1c47df1b..44c24e4ee 100644 --- a/packages/coding-agent/src/utils/exif-orientation.ts +++ b/packages/coding-agent/src/utils/exif-orientation.ts @@ -94,7 +94,8 @@ function hasExifHeader(bytes: Uint8Array, offset: number): boolean { ); } -function getExifOrientation(bytes: Uint8Array): number { +/** EXIF orientation (1-8) of a JPEG or WebP file; 1 when there is none. */ +export function getExifOrientation(bytes: Uint8Array): number { let tiffOffset = -1; // JPEG: starts with FF D8 diff --git a/packages/coding-agent/src/utils/image-header.ts b/packages/coding-agent/src/utils/image-header.ts new file mode 100644 index 000000000..1e0e2209a --- /dev/null +++ b/packages/coding-agent/src/utils/image-header.ts @@ -0,0 +1,306 @@ +/** + * Width and height of an image, read from its file structure without decoding pixels. + * + * Resizing needs the dimensions only to decide whether an image already fits the limits. Decoding a + * whole screenshot for that costs about a second in photon's WebAssembly when it runs interpreted. + * + * The parsers are strict on purpose: they return undefined for anything they do not fully + * understand (unknown chunk layout, truncation, bad PNG checksums, uncommon JPEG codings), and the + * caller then takes the full decode path as before. A file is accepted only when its structure is + * intact from the signature to the end marker. + */ +export interface ImageHeader { + width: number; + height: number; +} + +// node:zlib loads on first use, not at every start. +function crc32(data: Uint8Array): number { + return process.getBuiltinModule("node:zlib").crc32(data); +} + +function u16be(bytes: Uint8Array, offset: number): number { + return (bytes[offset] << 8) | bytes[offset + 1]; +} + +function u32be(bytes: Uint8Array, offset: number): number { + return ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) >>> 0; +} + +function u16le(bytes: Uint8Array, offset: number): number { + return bytes[offset] | (bytes[offset + 1] << 8); +} + +function u24le(bytes: Uint8Array, offset: number): number { + return bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16); +} + +function u32le(bytes: Uint8Array, offset: number): number { + return (bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16) | (bytes[offset + 3] << 24)) >>> 0; +} + +function asciiAt(bytes: Uint8Array, offset: number, text: string): boolean { + if (offset + text.length > bytes.length) return false; + for (let i = 0; i < text.length; i++) { + if (bytes[offset + i] !== text.charCodeAt(i)) return false; + } + return true; +} + +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + +/** Bit depths the PNG specification allows for each colour type. */ +const PNG_BIT_DEPTHS: Record = { + 0: [1, 2, 4, 8, 16], + 2: [8, 16], + 3: [1, 2, 4, 8], + 4: [8, 16], + 6: [8, 16], +}; + +function isChunkTypeByte(byte: number): boolean { + return (byte >= 0x41 && byte <= 0x5a) || (byte >= 0x61 && byte <= 0x7a); +} + +/** + * PNG: every chunk is walked and its CRC checked (decoders reject a bad CRC), IHDR must come first + * with valid fields, IDAT must be present (after PLTE for palette images), and IEND must end the file. + */ +export function readPngHeader(bytes: Uint8Array): ImageHeader | undefined { + if (bytes.length < PNG_SIGNATURE.length + 25 + 12 + 12) return undefined; + for (let i = 0; i < PNG_SIGNATURE.length; i++) { + if (bytes[i] !== PNG_SIGNATURE[i]) return undefined; + } + + let header: ImageHeader | undefined; + let colorType = -1; + let sawPalette = false; + let sawData = false; + let offset = PNG_SIGNATURE.length; + while (offset + 12 <= bytes.length) { + const length = u32be(bytes, offset); + if (length > 0x7fffffff) return undefined; + const dataStart = offset + 8; + const dataEnd = dataStart + length; + if (dataEnd + 4 > bytes.length) return undefined; + for (let i = 4; i < 8; i++) { + if (!isChunkTypeByte(bytes[offset + i])) return undefined; + } + if (crc32(bytes.subarray(offset + 4, dataEnd)) !== u32be(bytes, dataEnd)) return undefined; + + const isFirst = offset === PNG_SIGNATURE.length; + const isHeader = asciiAt(bytes, offset + 4, "IHDR"); + if (isFirst !== isHeader) return undefined; + if (isHeader) { + if (length !== 13) return undefined; + const width = u32be(bytes, dataStart); + const height = u32be(bytes, dataStart + 4); + const bitDepth = bytes[dataStart + 8]; + colorType = bytes[dataStart + 9]; + const compression = bytes[dataStart + 10]; + const filter = bytes[dataStart + 11]; + const interlace = bytes[dataStart + 12]; + if (width === 0 || height === 0 || width > 0x7fffffff || height > 0x7fffffff) return undefined; + if (!PNG_BIT_DEPTHS[colorType]?.includes(bitDepth)) return undefined; + if (compression !== 0 || filter !== 0 || interlace > 1) return undefined; + header = { width, height }; + } else if (asciiAt(bytes, offset + 4, "PLTE")) { + if (sawData) return undefined; + sawPalette = true; + } else if (asciiAt(bytes, offset + 4, "IDAT")) { + if (colorType === 3 && !sawPalette) return undefined; + sawData = true; + } else if (asciiAt(bytes, offset + 4, "IEND")) { + // IEND must be empty and end the file. + if (length !== 0 || dataEnd + 4 !== bytes.length || !sawData) return undefined; + return header; + } + offset = dataEnd + 4; + } + return undefined; +} + +/** + * JPEG: the marker segments before the first scan are walked, there must be exactly one baseline, + * extended or progressive Huffman frame header (SOF0-SOF2) with 8-bit samples and 1 or 3 + * components, and the file must end with the EOI marker. Lossless, arithmetic-coded, 12-bit and + * CMYK files take the decode path. + */ +export function readJpegHeader(bytes: Uint8Array): ImageHeader | undefined { + if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return undefined; + if (bytes[bytes.length - 2] !== 0xff || bytes[bytes.length - 1] !== 0xd9) return undefined; + + let header: ImageHeader | undefined; + let offset = 2; + while (offset < bytes.length) { + if (bytes[offset] !== 0xff) return undefined; + // Any number of 0xff fill bytes may precede a marker. + while (offset < bytes.length && bytes[offset] === 0xff) offset++; + if (offset >= bytes.length) return undefined; + const marker = bytes[offset]; + offset++; + // Markers without a segment (TEM, RSTn, SOI, EOI) cannot appear before the first scan. + if (marker === 0x00 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) return undefined; + if (offset + 2 > bytes.length) return undefined; + const length = u16be(bytes, offset); + if (length < 2 || offset + length > bytes.length) return undefined; + + if (marker === 0xda) { + // Start of scan: the frame header must have come first. + return header; + } + // C0-CF other than DHT (C4) are frame headers, JPG (C8) or DAC (CC). Only SOF0-SOF2 are accepted. + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4) { + if (header || marker > 0xc2 || length < 8) return undefined; + const precision = bytes[offset + 2]; + const height = u16be(bytes, offset + 3); + const width = u16be(bytes, offset + 5); + const components = bytes[offset + 7]; + if (precision !== 8 || width === 0 || height === 0) return undefined; + if (components !== 1 && components !== 3) return undefined; + if (length !== 8 + components * 3) return undefined; + header = { width, height }; + } + offset += length; + } + return undefined; +} + +function skipGifSubBlocks(bytes: Uint8Array, offset: number): number { + while (offset < bytes.length) { + const size = bytes[offset]; + offset += 1 + size; + if (size === 0) return offset; + } + return -1; +} + +/** + * GIF: the logical screen size, after walking every block to the trailer. Each frame must lie + * within the logical screen, and the trailer must end the file. + */ +export function readGifHeader(bytes: Uint8Array): ImageHeader | undefined { + if (bytes.length < 14 || !(asciiAt(bytes, 0, "GIF87a") || asciiAt(bytes, 0, "GIF89a"))) return undefined; + const width = u16le(bytes, 6); + const height = u16le(bytes, 8); + if (width === 0 || height === 0) return undefined; + const screenFlags = bytes[10]; + let offset = 13; + if (screenFlags & 0x80) offset += 3 * (1 << ((screenFlags & 0x07) + 1)); + + let frames = 0; + while (offset < bytes.length) { + const block = bytes[offset]; + if (block === 0x3b) { + return frames > 0 && offset + 1 === bytes.length ? { width, height } : undefined; + } + if (block === 0x21) { + if (offset + 2 > bytes.length) return undefined; + offset = skipGifSubBlocks(bytes, offset + 2); + } else if (block === 0x2c) { + if (offset + 10 > bytes.length) return undefined; + const left = u16le(bytes, offset + 1); + const top = u16le(bytes, offset + 3); + const frameWidth = u16le(bytes, offset + 5); + const frameHeight = u16le(bytes, offset + 7); + const frameFlags = bytes[offset + 9]; + if (frameWidth === 0 || frameHeight === 0 || left + frameWidth > width || top + frameHeight > height) { + return undefined; + } + offset += 10; + if (frameFlags & 0x80) offset += 3 * (1 << ((frameFlags & 0x07) + 1)); + if (offset >= bytes.length) return undefined; + const minCodeSize = bytes[offset]; + if (minCodeSize < 2 || minCodeSize > 8) return undefined; + offset = skipGifSubBlocks(bytes, offset + 1); + frames++; + } else { + return undefined; + } + if (offset < 0) return undefined; + } + return undefined; +} + +/** + * WebP: the RIFF size must match the file and every chunk must fit. The file is either one lossy + * (VP8) or lossless (VP8L) bitstream, or an extended (VP8X) canvas with one bitstream of the canvas + * size. Animated files take the decode path. + */ +export function readWebpHeader(bytes: Uint8Array): ImageHeader | undefined { + if (bytes.length < 20 || !asciiAt(bytes, 0, "RIFF") || !asciiAt(bytes, 8, "WEBP")) return undefined; + if (u32le(bytes, 4) + 8 !== bytes.length) return undefined; + + let canvas: ImageHeader | undefined; + let frame: ImageHeader | undefined; + let offset = 12; + while (offset < bytes.length) { + if (offset + 8 > bytes.length) return undefined; + const size = u32le(bytes, offset + 4); + const dataStart = offset + 8; + const dataEnd = dataStart + size; + if (dataEnd > bytes.length) return undefined; + const isFirst = offset === 12; + // Without a VP8X header, the file is a single VP8 or VP8L chunk. + if (!isFirst && !canvas) return undefined; + + if (asciiAt(bytes, offset, "VP8 ") || asciiAt(bytes, offset, "VP8L")) { + if (frame) return undefined; + frame = readWebpBitstreamSize(bytes, offset, dataStart, size); + if (!frame) return undefined; + } else if (asciiAt(bytes, offset, "VP8X")) { + if (!isFirst || size < 10) return undefined; + canvas = { width: u24le(bytes, dataStart + 4) + 1, height: u24le(bytes, dataStart + 7) + 1 }; + } else if (isFirst || asciiAt(bytes, offset, "ANIM") || asciiAt(bytes, offset, "ANMF")) { + return undefined; + } + // Chunks are padded to an even size. + offset = dataEnd + (size & 1); + } + if (offset !== bytes.length || !frame) return undefined; + // An extended file's one bitstream must fill the canvas. + if (canvas && (frame.width !== canvas.width || frame.height !== canvas.height)) return undefined; + return frame; +} + +function readWebpBitstreamSize( + bytes: Uint8Array, + chunkOffset: number, + dataStart: number, + size: number, +): ImageHeader | undefined { + if (asciiAt(bytes, chunkOffset, "VP8L")) { + // Signature 0x2f, then 14-bit width-1 and height-1, an alpha hint bit and a 3-bit version (0). + if (size < 5 || bytes[dataStart] !== 0x2f) return undefined; + const bits = u32le(bytes, dataStart + 1); + if (bits >>> 29 !== 0) return undefined; + return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 }; + } + // VP8: 3-byte frame tag (key frame flag clear), start code 9d 01 2a, then 14-bit width and height. + if (size < 10 || (bytes[dataStart] & 0x01) !== 0) return undefined; + if (bytes[dataStart + 3] !== 0x9d || bytes[dataStart + 4] !== 0x01 || bytes[dataStart + 5] !== 0x2a) { + return undefined; + } + const width = u16le(bytes, dataStart + 6) & 0x3fff; + const height = u16le(bytes, dataStart + 8) & 0x3fff; + return width === 0 || height === 0 ? undefined : { width, height }; +} + +/** + * Dimensions from the header of a well-formed PNG, JPEG, GIF or WebP file, as stored (before any + * EXIF orientation). Undefined when the format is unknown or the structure is not fully understood. + */ +export function readImageHeader(bytes: Uint8Array, mimeType: string): ImageHeader | undefined { + switch (mimeType) { + case "image/png": + return readPngHeader(bytes); + case "image/jpeg": + return readJpegHeader(bytes); + case "image/gif": + return readGifHeader(bytes); + case "image/webp": + return readWebpHeader(bytes); + default: + return undefined; + } +} diff --git a/packages/coding-agent/src/utils/image-process.ts b/packages/coding-agent/src/utils/image-process.ts index a461c8b21..f54407739 100644 --- a/packages/coding-agent/src/utils/image-process.ts +++ b/packages/coding-agent/src/utils/image-process.ts @@ -1,5 +1,6 @@ import { convertImageBytesToPng } from "./image-convert.ts"; import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize.ts"; +import { resolveImageResizeOptions } from "./image-resize-core.ts"; export interface ProcessImageOptions { /** Whether to resize images to inline provider limits. Default: true */ @@ -117,3 +118,49 @@ export async function processImage( hints, }; } + +interface ProcessedImageBlock { + data: string; + mimeType: string; + autoResizeImages: boolean; + limits: Required; +} + +/** Image blocks built from processImage results, with what they were processed for. */ +const processedImageBlocks = new WeakMap(); + +/** + * Record that `block` holds a processImage result for `options`. Processing such a block again with + * the same options returns it unchanged (the result is already within the limits and in a supported + * format), so tool-result normalization can skip it: see isProcessedImageBlock. + */ +export function markProcessedImageBlock( + block: { data: string; mimeType: string }, + options: ProcessImageOptions | undefined, +): void { + processedImageBlocks.set(block, { + data: block.data, + mimeType: block.mimeType, + autoResizeImages: options?.autoResizeImages ?? true, + limits: resolveImageResizeOptions(options?.resizeOptions), + }); +} + +/** Whether `block` is unchanged since markProcessedImageBlock recorded it for the same options. */ +export function isProcessedImageBlock( + block: { data: string; mimeType: string }, + options: ProcessImageOptions | undefined, +): boolean { + const processed = processedImageBlocks.get(block); + if (!processed || processed.data !== block.data || processed.mimeType !== block.mimeType) return false; + const autoResizeImages = options?.autoResizeImages ?? true; + if (processed.autoResizeImages !== autoResizeImages) return false; + if (!autoResizeImages) return true; + const limits = resolveImageResizeOptions(options?.resizeOptions); + return ( + processed.limits.maxWidth === limits.maxWidth && + processed.limits.maxHeight === limits.maxHeight && + processed.limits.maxBytes === limits.maxBytes && + processed.limits.jpegQuality === limits.jpegQuality + ); +} diff --git a/packages/coding-agent/src/utils/image-resize-core.ts b/packages/coding-agent/src/utils/image-resize-core.ts index 3f80111ff..83bfb745d 100644 --- a/packages/coding-agent/src/utils/image-resize-core.ts +++ b/packages/coding-agent/src/utils/image-resize-core.ts @@ -1,4 +1,5 @@ -import { applyExifOrientation } from "./exif-orientation.ts"; +import { applyExifOrientation, getExifOrientation } from "./exif-orientation.ts"; +import { type ImageHeader, readImageHeader } from "./image-header.ts"; import { loadPhoton } from "./photon.ts"; export interface ImageResizeOptions { @@ -38,41 +39,143 @@ const DEFAULT_OPTIONS: Required = { jpegQuality: 80, }; -interface EncodedCandidate { - data: string; - encodedSize: number; - mimeType: string; +/** The limits with defaults applied. */ +export function resolveImageResizeOptions(options?: ImageResizeOptions): Required { + return { ...DEFAULT_OPTIONS, ...options }; +} + +/** + * Width and height as displayed, read from the file header: EXIF orientations 5-8 rotate the image + * by 90 degrees. Undefined when the header cannot be read; the caller then decodes the image. + */ +export function readDisplayedImageSize(inputBytes: Uint8Array, mimeType: string): ImageHeader | undefined { + // The header is checked first: it also validates the structure the EXIF reader walks. + const header = readImageHeader(inputBytes, mimeType); + if (!header) return undefined; + return getExifOrientation(inputBytes) >= 5 ? { width: header.height, height: header.width } : header; +} + +function base64Size(byteLength: number): number { + return Math.ceil(byteLength / 3) * 4; } -function encodeCandidate(buffer: Uint8Array, mimeType: string): EncodedCandidate { - const data = Buffer.from(buffer).toString("base64"); +/** + * The input as it is when it is already within the dimension and size limits, else undefined. + * `width` and `height` are the displayed size (after EXIF orientation). + */ +export function keepIfWithinLimits( + inputBytes: Uint8Array, + mimeType: string, + width: number, + height: number, + opts: Required, +): ResizedImage | undefined { + if (width > opts.maxWidth || height > opts.maxHeight || base64Size(inputBytes.byteLength) >= opts.maxBytes) { + return undefined; + } + const format = mimeType.split("/")[1] ?? "png"; return { - data, - encodedSize: Buffer.byteLength(data, "utf-8"), - mimeType, + data: Buffer.from(inputBytes.buffer, inputBytes.byteOffset, inputBytes.byteLength).toString("base64"), + mimeType: mimeType || `image/${format}`, + originalWidth: width, + originalHeight: height, + width, + height, + wasResized: false, + }; +} + +/** One resized version of the image, encoded on demand. */ +export interface ResizedFrame { + png(): Uint8Array | Promise; + jpeg(quality: number): Uint8Array | Promise; + free(): void; +} + +/** + * Resize to fit `maxWidth`/`maxHeight`, then find the first encoding under `maxBytes`: PNG, then JPEG + * at each quality step, then the same at 75% of the size, down to 1x1. Encodings are produced one at + * a time and the first that fits wins, so later candidates are not encoded at all. + * Returns null when nothing fits. + */ +export async function encodeWithinLimits( + originalWidth: number, + originalHeight: number, + opts: Required, + resizeTo: (width: number, height: number) => ResizedFrame, +): Promise { + let currentWidth = originalWidth; + let currentHeight = originalHeight; + if (currentWidth > opts.maxWidth) { + currentHeight = Math.round((currentHeight * opts.maxWidth) / currentWidth); + currentWidth = opts.maxWidth; + } + if (currentHeight > opts.maxHeight) { + currentWidth = Math.round((currentWidth * opts.maxHeight) / currentHeight); + currentHeight = opts.maxHeight; + } + + const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40])); + const result = (bytes: Uint8Array, mimeType: string): ResizedImage | undefined => { + const data = Buffer.from(bytes).toString("base64"); + if (Buffer.byteLength(data, "utf-8") >= opts.maxBytes) return undefined; + return { + data, + mimeType, + originalWidth, + originalHeight, + width: currentWidth, + height: currentHeight, + wasResized: true, + }; }; + + while (true) { + const frame = resizeTo(currentWidth, currentHeight); + try { + const png = result(await frame.png(), "image/png"); + if (png) return png; + for (const quality of qualitySteps) { + const jpeg = result(await frame.jpeg(quality), "image/jpeg"); + if (jpeg) return jpeg; + } + } finally { + frame.free(); + } + + if (currentWidth === 1 && currentHeight === 1) { + return null; + } + + const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75)); + const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75)); + if (nextWidth === currentWidth && nextHeight === currentHeight) { + return null; + } + + currentWidth = nextWidth; + currentHeight = nextHeight; + } } /** * Resize an image to fit within the specified max dimensions and encoded file size. * Returns null if the image cannot be resized below maxBytes. * - * Uses Photon (Rust/WASM) for image processing. If Photon is not available, - * returns null. - * - * Strategy for staying under maxBytes: - * 1. First resize to maxWidth/maxHeight - * 2. Try both PNG and JPEG formats, pick the smaller one - * 3. If still too large, try JPEG with decreasing quality - * 4. If still too large, progressively reduce dimensions until 1x1 + * An image whose header shows it is already within the limits is returned as it is, without + * decoding. Otherwise Photon (Rust/WASM) decodes it once and resizes it (see encodeWithinLimits). + * If Photon is not available, returns null. */ export async function resizeImageInProcess( inputBytes: Uint8Array, mimeType: string, options?: ImageResizeOptions, ): Promise { - const opts = { ...DEFAULT_OPTIONS, ...options }; - const inputBase64Size = Math.ceil(inputBytes.byteLength / 3) * 4; + const opts = resolveImageResizeOptions(options); + + const size = readDisplayedImageSize(inputBytes, mimeType); + const unchanged = size && keepIfWithinLimits(inputBytes, mimeType, size.width, size.height, opts); + if (unchanged) return unchanged; const photon = await loadPhoton(); if (!photon) { @@ -84,86 +187,21 @@ export async function resizeImageInProcess( const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes); image = applyExifOrientation(photon, rawImage, inputBytes); if (image !== rawImage) rawImage.free(); + const decoded = image; - const originalWidth = image.get_width(); - const originalHeight = image.get_height(); - const format = mimeType.split("/")[1] ?? "png"; + const originalWidth = decoded.get_width(); + const originalHeight = decoded.get_height(); + const decodedUnchanged = keepIfWithinLimits(inputBytes, mimeType, originalWidth, originalHeight, opts); + if (decodedUnchanged) return decodedUnchanged; - // Check if already within all limits (dimensions AND encoded size) - if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) { + return await encodeWithinLimits(originalWidth, originalHeight, opts, (width, height) => { + const resized = photon.resize(decoded, width, height, photon.SamplingFilter.Lanczos3); return { - data: Buffer.from(inputBytes).toString("base64"), - mimeType: mimeType || `image/${format}`, - originalWidth, - originalHeight, - width: originalWidth, - height: originalHeight, - wasResized: false, + png: () => resized.get_bytes(), + jpeg: (quality) => resized.get_bytes_jpeg(quality), + free: () => resized.free(), }; - } - - // Calculate initial dimensions respecting max limits - let targetWidth = originalWidth; - let targetHeight = originalHeight; - - if (targetWidth > opts.maxWidth) { - targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth); - targetWidth = opts.maxWidth; - } - if (targetHeight > opts.maxHeight) { - targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight); - targetHeight = opts.maxHeight; - } - - function tryEncodings(width: number, height: number, jpegQualities: number[]): EncodedCandidate[] { - const resized = photon!.resize(image!, width, height, photon!.SamplingFilter.Lanczos3); - - try { - const candidates: EncodedCandidate[] = [encodeCandidate(resized.get_bytes(), "image/png")]; - for (const quality of jpegQualities) { - candidates.push(encodeCandidate(resized.get_bytes_jpeg(quality), "image/jpeg")); - } - return candidates; - } finally { - resized.free(); - } - } - - const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40])); - let currentWidth = targetWidth; - let currentHeight = targetHeight; - - while (true) { - const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps); - for (const candidate of candidates) { - if (candidate.encodedSize < opts.maxBytes) { - return { - data: candidate.data, - mimeType: candidate.mimeType, - originalWidth, - originalHeight, - width: currentWidth, - height: currentHeight, - wasResized: true, - }; - } - } - - if (currentWidth === 1 && currentHeight === 1) { - break; - } - - const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75)); - const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75)); - if (nextWidth === currentWidth && nextHeight === currentHeight) { - break; - } - - currentWidth = nextWidth; - currentHeight = nextHeight; - } - - return null; + }); } catch { return null; } finally { diff --git a/packages/coding-agent/src/utils/image-resize-native.ts b/packages/coding-agent/src/utils/image-resize-native.ts new file mode 100644 index 000000000..699e0452b --- /dev/null +++ b/packages/coding-agent/src/utils/image-resize-native.ts @@ -0,0 +1,83 @@ +import { getExifOrientation } from "./exif-orientation.ts"; +import { encodeWithinLimits, type ImageResizeOptions, type ResizedImage } from "./image-resize-core.ts"; + +/** + * The part of Bun's `Bun.Image` (Bun 1.4+) used here. Pi does not depend on Bun's type package, + * so the shape is declared locally; it matches `packages/bun-types/bun.d.ts` of the pinned runtime. + */ +interface NativeImagePipeline { + resize(width: number, height: number, options: { filter: "lanczos3"; fit: "fill" }): NativeImagePipeline; + rotate(degrees: number): NativeImagePipeline; + flip(): NativeImagePipeline; + flop(): NativeImagePipeline; + png(): NativeImagePipeline; + jpeg(options: { quality: number }): NativeImagePipeline; + bytes(): Promise; +} + +type NativeImageConstructor = new (input: Uint8Array, options: { autoOrient: boolean }) => NativeImagePipeline; + +/** `Bun.Image` when the runtime has it (Bun, including Pi-Bolt's runtime), else undefined (Node). */ +export function getNativeImage(): NativeImageConstructor | undefined { + const image = (globalThis as { Bun?: { Image?: unknown } }).Bun?.Image; + return typeof image === "function" ? (image as NativeImageConstructor) : undefined; +} + +/** + * The EXIF orientation as the transforms Bun.Image applies in its fixed order (rotate, then flip + * or flop), matching applyExifOrientation's photon transforms pixel for pixel. + */ +function orient(pipeline: NativeImagePipeline, orientation: number): NativeImagePipeline { + switch (orientation) { + case 2: + return pipeline.flop(); + case 3: + return pipeline.rotate(180); + case 4: + return pipeline.flip(); + case 5: + return pipeline.rotate(90).flop(); + case 6: + return pipeline.rotate(90); + case 7: + return pipeline.rotate(270).flop(); + case 8: + return pipeline.rotate(270); + default: + return pipeline; + } +} + +/** + * Resize with the runtime's native image code (libspng, libjpeg-turbo and libwebp decoders, SIMD + * Lanczos3 resize, libspng and libjpeg-turbo encoders) instead of photon's WebAssembly, which the + * JIT-off Pi-Bolt runtime interprets: a 2442x1438 screenshot takes about 0.5 s of CPU instead of + * about 15 s. It runs on Bun's own worker threads, so the event loop stays free. + * + * Same output rules as the photon path (same target sizes, candidate order and size limit), but + * not the same bytes: the resampling rounds differently and the encoders differ. + * + * `width` and `height` are the displayed size of the input (after EXIF orientation), from its + * header. Throws when the native decoder or encoder fails; the caller then uses photon. + */ +export function resizeImageNative( + NativeImage: NativeImageConstructor, + inputBytes: Uint8Array, + width: number, + height: number, + opts: Required, +): Promise { + const orientation = getExifOrientation(inputBytes); + return encodeWithinLimits(width, height, opts, (targetWidth, targetHeight) => { + const pipeline = () => + orient(new NativeImage(inputBytes, { autoOrient: false }), orientation).resize(targetWidth, targetHeight, { + filter: "lanczos3", + fit: "fill", + }); + return { + png: () => pipeline().png().bytes(), + jpeg: (quality) => pipeline().jpeg({ quality }).bytes(), + free: () => {}, + }; + }); +} diff --git a/packages/coding-agent/src/utils/image-resize.ts b/packages/coding-agent/src/utils/image-resize.ts index 9de941ba3..57f9154ca 100644 --- a/packages/coding-agent/src/utils/image-resize.ts +++ b/packages/coding-agent/src/utils/image-resize.ts @@ -2,10 +2,14 @@ import type { Worker } from "node:worker_threads"; import { IMAGE_RESIZE_WORKER_RESPONSE_TYPE, type ImageResizeOptions, + keepIfWithinLimits, type ResizedImage, type ResizeImageWorkerResponse, + readDisplayedImageSize, resizeImageInProcess, + resolveImageResizeOptions, } from "./image-resize-core.ts"; +import { getNativeImage, resizeImageNative } from "./image-resize-native.ts"; export type { ImageResizeOptions, ResizedImage } from "./image-resize-core.ts"; @@ -77,16 +81,32 @@ async function resizeImageInWorker( /** * Resize an image to fit within the specified max dimensions and encoded file size. - * Runs Photon in a worker thread so WASM decoding, resizing, and encoding do not - * block the TUI event loop. If the worker cannot be loaded (for example in some - * Bun compiled executable layouts), fall back to in-process resizing so image - * reads still work. + * + * - An image whose header shows it is within the limits is returned as it is, without decoding. + * - Otherwise, when the runtime has a native image pipeline (`Bun.Image`), it resizes there. + * - Otherwise, or when the header cannot be read or the native pipeline fails, Photon runs in a + * worker thread so WASM decoding, resizing, and encoding do not block the TUI event loop. If the + * worker cannot be loaded (for example in some Bun compiled executable layouts), fall back to + * in-process resizing so image reads still work. */ export async function resizeImage( inputBytes: Uint8Array, mimeType: string, options?: ImageResizeOptions, ): Promise { + const opts = resolveImageResizeOptions(options); + const size = readDisplayedImageSize(inputBytes, mimeType); + if (size) { + const unchanged = keepIfWithinLimits(inputBytes, mimeType, size.width, size.height, opts); + if (unchanged) return unchanged; + const NativeImage = getNativeImage(); + if (NativeImage) { + try { + return await resizeImageNative(NativeImage, inputBytes, size.width, size.height, opts); + } catch {} + } + } + const isTypeScriptRuntime = import.meta.url.endsWith(".ts"); const workerUrl = new URL( isTypeScriptRuntime ? "./image-resize-worker.ts" : "./image-resize-worker.js", diff --git a/packages/coding-agent/src/utils/tool-result-images.ts b/packages/coding-agent/src/utils/tool-result-images.ts index 702d98772..e035d5e99 100644 --- a/packages/coding-agent/src/utils/tool-result-images.ts +++ b/packages/coding-agent/src/utils/tool-result-images.ts @@ -1,5 +1,5 @@ import type { ImageContent, ModelImageResizeOptions, TextContent } from "@earendil-works/pi-ai"; -import { processImage } from "./image-process.ts"; +import { isProcessedImageBlock, processImage } from "./image-process.ts"; export type ToolResultContent = TextContent | ImageContent; @@ -39,10 +39,14 @@ export async function normalizeToolResultImages( continue; } - const processed = await processImage(Buffer.from(block.data, "base64"), block.mimeType, { - autoResizeImages, - resizeOptions: options?.resizeOptions, - }); + const processOptions = { autoResizeImages, resizeOptions: options?.resizeOptions }; + // The read tool's images were processed with these options already; a second pass returns them unchanged. + if (isProcessedImageBlock(block, processOptions)) { + normalized.push(block); + continue; + } + + const processed = await processImage(Buffer.from(block.data, "base64"), block.mimeType, processOptions); if (!processed.ok) { // Unlike `read`, keep the original block. The tool already produced this image and the // failure may just be an unavailable image backend, so passing it through preserves the diff --git a/packages/coding-agent/test/image-header.test.ts b/packages/coding-agent/test/image-header.test.ts new file mode 100644 index 000000000..1b41fee8b --- /dev/null +++ b/packages/coding-agent/test/image-header.test.ts @@ -0,0 +1,459 @@ +import { crc32, deflateSync } from "node:zlib"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type * as PhotonModule from "../src/utils/photon.ts"; + +const photonLoads = vi.hoisted(() => ({ count: 0 })); +vi.mock("../src/utils/photon.ts", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + loadPhoton: () => { + photonLoads.count++; + return actual.loadPhoton(); + }, + }; +}); + +import { + readGifHeader, + readImageHeader, + readJpegHeader, + readPngHeader, + readWebpHeader, +} from "../src/utils/image-header.ts"; +import { resizeImageInProcess } from "../src/utils/image-resize-core.ts"; +import { loadPhoton } from "../src/utils/photon.ts"; + +// Small 2x2 blue JPEG image (baseline) - generated with ImageMagick +const JPEG_2X2 = + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAACAAIDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAVAQEBAAAAAAAAAAAAAAAAAAAGCf/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AD3VTB3/2Q=="; + +const JPEG_PROGRESSIVE_5X3 = + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wgARCAADAAUDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAL/xAAVAQEBAAAAAAAAAAAAAAAAAAAFBv/aAAwDAQACEAMQAAABsHUX/8QAFhABAQEAAAAAAAAAAAAAAAAAAgER/9oACAEBAAEFAk7n/8QAGhEAAgIDAAAAAAAAAAAAAAAAAQMABBMikf/aAAgBAwEBPwFFVGMaDk//xAAYEQEAAwEAAAAAAAAAAAAAAAABAANxsf/aAAgBAgEBPwGxRMOT/8QAFhAAAwAAAAAAAAAAAAAAAAAAAAEh/9oACAEBAAY/AlT/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/IRH/2gAMAwEAAgADAAAAEA//xAAWEQADAAAAAAAAAAAAAAAAAAAAwdH/2gAIAQMBAT8QWdD/xAAWEQEBAQAAAAAAAAAAAAAAAAABAPH/2gAIAQIBAT8QABwX/8QAGBABAQADAAAAAAAAAAAAAAAAAREAQWH/2gAIAQEAAT8QK9Vujmf/2Q=="; + +const WEBP_LOSSY_5X3 = + "UklGRmwAAABXRUJQVlA4IGAAAADwAQCdASoFAAMAAUAmJbACdLoAArl9WmgA/fzw9op6bHEwjhOVx/wF//v//yHC9hWy7jAdxkLZm35O/zZfR5/8m3wfmv+c4Nh/5N//UMf+w/6cWv9zf/10HG9vn+MAAAA="; + +const WEBP_LOSSLESS_5X3 = "UklGRjoAAABXRUJQVlA4TC0AAAAvBIAAAHcgFkz8eQfQmcJETNP8Z7SBgkaSmu+FrgVPaMETWhAW0f9YW87N9wcA"; + +// VP8X canvas with an ALPH chunk and a VP8 bitstream. +const WEBP_ALPHA_5X3 = + "UklGRpQAAABXRUJQVlA4WAoAAAAQAAAABAAAAgAAQUxQSA0AAAABD/DA/4iIMAER/Y8AAFZQOCBgAAAA8AEAnQEqBQADAAFAJiWwAnS6AAK5fVpoAP388PaKemxxMI4Tlcf318/v/97ZLZm3/CpcMWRAgV+Tv82X0ef/Jt8H5r/nODYf+Tf/1DH/sP+nFr/c3/9h10Fke/mQAAAA"; + +function bytes(base64: string): Buffer { + return Buffer.from(base64, "base64"); +} + +function pngChunk(type: string, body: Buffer): Buffer { + const header = Buffer.alloc(8); + header.writeUInt32BE(body.length, 0); + header.write(type, 4, "ascii"); + const checksum = Buffer.alloc(4); + checksum.writeUInt32BE(crc32(Buffer.concat([header.subarray(4), body])), 0); + return Buffer.concat([header, body, checksum]); +} + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +function ihdr(width: number, height: number, bitDepth = 8, colorType = 0): Buffer { + const body = Buffer.alloc(13); + body.writeUInt32BE(width, 0); + body.writeUInt32BE(height, 4); + body[8] = bitDepth; + body[9] = colorType; + return pngChunk("IHDR", body); +} + +/** An 8-bit grayscale PNG. */ +function createPng(width: number, height: number): Buffer { + const raw = Buffer.alloc((width + 1) * height); + for (let row = 0; row < height; row++) { + raw.fill((row * 7) % 256, row * (width + 1) + 1, (row + 1) * (width + 1)); + } + return Buffer.concat([ + PNG_SIGNATURE, + ihdr(width, height), + pngChunk("tEXt", Buffer.from("Comment\0test")), + pngChunk("IDAT", deflateSync(raw)), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +/** + * LZW image data for `pixels` pixels of colour 0: minimum code size 2, so codes are 3 bits wide + * (clear = 4, end = 5). A clear code after every two pixels keeps the code width at 3 bits. + */ +function gifImageData(pixels: number): Buffer { + const codes: number[] = []; + for (let i = 0; i < pixels; i++) { + if (i % 2 === 0) codes.push(4); + codes.push(0); + } + codes.push(5); + const packed: number[] = []; + let bits = 0; + let bitCount = 0; + for (const code of codes) { + bits |= code << bitCount; + bitCount += 3; + while (bitCount >= 8) { + packed.push(bits & 0xff); + bits >>= 8; + bitCount -= 8; + } + } + if (bitCount > 0) packed.push(bits & 0xff); + const blocks: number[] = [0x02]; + for (let i = 0; i < packed.length; i += 255) { + const block = packed.slice(i, i + 255); + blocks.push(block.length, ...block); + } + blocks.push(0); + return Buffer.from(blocks); +} + +/** A GIF with a 2-colour global table and the given frames ([left, top, width, height]). */ +function createGif(width: number, height: number, frames: [number, number, number, number][]): Buffer { + const parts: Buffer[] = [Buffer.from("GIF89a", "ascii")]; + const screen = Buffer.alloc(7); + screen.writeUInt16LE(width, 0); + screen.writeUInt16LE(height, 2); + screen[4] = 0x80; // global colour table, 2 entries + parts.push(screen, Buffer.from([0, 0, 0, 255, 255, 255])); + // Graphic control extension. + parts.push(Buffer.from([0x21, 0xf9, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00])); + for (const [left, top, frameWidth, frameHeight] of frames) { + const descriptor = Buffer.alloc(10); + descriptor[0] = 0x2c; + descriptor.writeUInt16LE(left, 1); + descriptor.writeUInt16LE(top, 3); + descriptor.writeUInt16LE(frameWidth, 5); + descriptor.writeUInt16LE(frameHeight, 7); + parts.push(descriptor, gifImageData(frameWidth * frameHeight)); + } + parts.push(Buffer.from([0x3b])); + return Buffer.concat(parts); +} + +function riff(chunks: Buffer[]): Buffer { + const body = Buffer.concat([Buffer.from("WEBP", "ascii"), ...chunks]); + const header = Buffer.alloc(8); + header.write("RIFF", 0, "ascii"); + header.writeUInt32LE(body.length, 4); + return Buffer.concat([header, body]); +} + +function webpChunk(type: string, body: Buffer): Buffer { + const header = Buffer.alloc(8); + header.write(type, 0, "ascii"); + header.writeUInt32LE(body.length, 4); + return Buffer.concat([header, body, body.length % 2 ? Buffer.alloc(1) : Buffer.alloc(0)]); +} + +function vp8x(width: number, height: number, flags = 0): Buffer { + const body = Buffer.alloc(10); + body[0] = flags; + body.writeUIntLE(width - 1, 4, 3); + body.writeUIntLE(height - 1, 7, 3); + return webpChunk("VP8X", body); +} + +/** The chunks of a simple WebP file (everything after "RIFF" size "WEBP"). */ +function webpChunks(base64: string): Buffer { + return bytes(base64).subarray(12); +} + +function app1Exif(orientation: number): Buffer { + const tiff = Buffer.from("49492a00080000000100120103000100000000000000000000000000", "hex"); + tiff.writeUInt16LE(orientation, 18); + const payload = Buffer.concat([Buffer.from("Exif\0\0", "latin1"), tiff]); + const segment = Buffer.alloc(payload.length + 4); + segment[0] = 0xff; + segment[1] = 0xe1; + segment.writeUInt16BE(payload.length + 2, 2); + payload.copy(segment, 4); + return segment; +} + +/** Replace the JPEG segment with `marker` by `replace(segment)`. */ +function editJpegSegment(jpeg: Buffer, marker: number, replace: (segment: Buffer) => Buffer): Buffer { + let offset = 2; + while (offset < jpeg.length) { + const length = jpeg.readUInt16BE(offset + 2); + if (jpeg[offset + 1] === marker) { + const segment = jpeg.subarray(offset, offset + 2 + length); + return Buffer.concat([ + jpeg.subarray(0, offset), + replace(Buffer.from(segment)), + jpeg.subarray(offset + 2 + length), + ]); + } + offset += 2 + length; + } + throw new Error(`no marker ${marker.toString(16)}`); +} + +describe("image header parsers", () => { + const valid: [string, string, Buffer, number, number][] = [ + ["PNG", "image/png", createPng(37, 21), 37, 21], + ["baseline JPEG", "image/jpeg", bytes(JPEG_2X2), 2, 2], + ["progressive JPEG", "image/jpeg", bytes(JPEG_PROGRESSIVE_5X3), 5, 3], + ["GIF", "image/gif", createGif(6, 4, [[0, 0, 6, 4]]), 6, 4], + [ + "animated GIF with a partial frame", + "image/gif", + createGif(6, 4, [ + [0, 0, 6, 4], + [1, 1, 2, 2], + ]), + 6, + 4, + ], + ["lossy WebP", "image/webp", bytes(WEBP_LOSSY_5X3), 5, 3], + ["lossless WebP", "image/webp", bytes(WEBP_LOSSLESS_5X3), 5, 3], + ["extended WebP with alpha", "image/webp", bytes(WEBP_ALPHA_5X3), 5, 3], + ]; + + it.each(valid)( + "reads the size of a %s, and photon decodes the same size", + async (_name, mimeType, data, width, height) => { + expect(readImageHeader(data, mimeType)).toEqual({ width, height }); + const photon = (await loadPhoton())!; + const image = photon.PhotonImage.new_from_byteslice(data); + expect([image.get_width(), image.get_height()]).toEqual([width, height]); + image.free(); + }, + ); + + it("returns undefined for a MIME type it does not parse", () => { + expect(readImageHeader(createPng(2, 2), "image/bmp")).toBeUndefined(); + expect(readImageHeader(createPng(2, 2), "image/jpeg")).toBeUndefined(); + }); + + describe("rejects malformed PNG files", () => { + const png = createPng(4, 3); + const cases: [string, Buffer][] = [ + ["an empty file", Buffer.alloc(0)], + ["a truncated file", png.subarray(0, png.length - 5)], + ["trailing bytes after IEND", Buffer.concat([png, Buffer.from([0])])], + ["a bad signature", Buffer.concat([Buffer.from([0x88]), png.subarray(1)])], + [ + "a bad chunk CRC", + (() => { + const copy = Buffer.from(png); + copy[copy.length - 13] ^= 0xff; // last byte of the IDAT CRC + return copy; + })(), + ], + [ + "a flipped pixel-data byte", + (() => { + const copy = Buffer.from(png); + copy[png.indexOf("IDAT") + 6] ^= 0x01; + return copy; + })(), + ], + ["IHDR not first", Buffer.concat([PNG_SIGNATURE, pngChunk("tEXt", Buffer.from("a\0b")), png.subarray(8)])], + [ + "no IDAT", + Buffer.concat([ + PNG_SIGNATURE, + ihdr(4, 3), + pngChunk("IEND", Buffer.alloc(0)), + pngChunk("IEND", Buffer.alloc(0)), + ]), + ], + ["zero width", Buffer.concat([PNG_SIGNATURE, ihdr(0, 3), png.subarray(8 + 25)])], + ["an invalid bit depth", Buffer.concat([PNG_SIGNATURE, ihdr(4, 3, 3, 0), png.subarray(8 + 25)])], + ["a palette image without PLTE", Buffer.concat([PNG_SIGNATURE, ihdr(4, 3, 8, 3), png.subarray(8 + 25)])], + [ + "a chunk length past the end", + Buffer.concat([png.subarray(0, 8), Buffer.from([0x7f, 0xff, 0xff, 0xff]), png.subarray(12)]), + ], + ]; + it.each(cases)("%s", (_name, data) => { + expect(readPngHeader(data)).toBeUndefined(); + }); + }); + + describe("rejects malformed or uncommon JPEG files", () => { + const jpeg = bytes(JPEG_2X2); + const cases: [string, Buffer][] = [ + ["a missing EOI marker", jpeg.subarray(0, jpeg.length - 2)], + ["a truncated file", Buffer.concat([jpeg.subarray(0, 100), Buffer.from([0xff, 0xd9])])], + ["no frame header before the scan", editJpegSegment(jpeg, 0xc0, () => Buffer.alloc(0))], + [ + "a lossless frame (SOF3)", + editJpegSegment(jpeg, 0xc0, (s) => Buffer.concat([Buffer.from([0xff, 0xc3]), s.subarray(2)])), + ], + [ + "an arithmetic-coded frame (SOF9)", + editJpegSegment(jpeg, 0xc0, (s) => Buffer.concat([Buffer.from([0xff, 0xc9]), s.subarray(2)])), + ], + [ + "12-bit samples", + editJpegSegment(jpeg, 0xc0, (s) => { + s[4] = 12; + return s; + }), + ], + [ + "zero height", + editJpegSegment(jpeg, 0xc0, (s) => { + s.writeUInt16BE(0, 5); + return s; + }), + ], + ["two frame headers", editJpegSegment(jpeg, 0xc0, (s) => Buffer.concat([s, s]))], + [ + "a segment length past the end", + Buffer.concat([jpeg.subarray(0, 4), Buffer.from([0xff, 0xff]), Buffer.from([0xff, 0xd9])]), + ], + [ + "a restart marker before the scan", + Buffer.concat([jpeg.subarray(0, 2), Buffer.from([0xff, 0xd0]), jpeg.subarray(2)]), + ], + ["not a JPEG", Buffer.from("not a jpeg at all\xff\xd9", "latin1")], + ]; + it.each(cases)("%s", (_name, data) => { + expect(readJpegHeader(data)).toBeUndefined(); + }); + + it("CMYK (4 components)", () => { + const cmyk = editJpegSegment(jpeg, 0xc0, (s) => { + const body = Buffer.alloc(8 + 4 * 3); + s.copy(body, 0, 2, 9); + body.writeUInt16BE(body.length, 0); + body[7] = 4; + return Buffer.concat([Buffer.from([0xff, 0xc0]), body]); + }); + expect(readJpegHeader(cmyk)).toBeUndefined(); + }); + }); + + describe("rejects malformed GIF files", () => { + const gif = createGif(6, 4, [[0, 0, 6, 4]]); + const cases: [string, Buffer][] = [ + ["a missing trailer", gif.subarray(0, gif.length - 1)], + ["trailing bytes after the trailer", Buffer.concat([gif, Buffer.from([0])])], + ["no frames", createGif(6, 4, [])], + ["a frame outside the screen", createGif(6, 4, [[4, 0, 6, 4]])], + ["a zero-size screen", createGif(0, 4, [[0, 0, 1, 1]])], + ["an unknown block", Buffer.concat([gif.subarray(0, 19), Buffer.from([0x99]), gif.subarray(19)])], + ["a bad signature", Buffer.concat([Buffer.from("GIF88a", "ascii"), gif.subarray(6)])], + ]; + it.each(cases)("%s", (_name, data) => { + expect(readGifHeader(data)).toBeUndefined(); + }); + }); + + describe("WebP files", () => { + const lossless = webpChunks(WEBP_LOSSLESS_5X3); + + it("reads an extended still file whose bitstream fills the canvas", () => { + expect(readWebpHeader(riff([vp8x(5, 3), lossless]))).toEqual({ width: 5, height: 3 }); + }); + + const cases: [string, Buffer][] = [ + ["a RIFF size that does not match the file", Buffer.concat([bytes(WEBP_LOSSLESS_5X3), Buffer.alloc(2)])], + ["a truncated file", bytes(WEBP_LOSSY_5X3).subarray(0, 40)], + ["an extended canvas larger than its bitstream", riff([vp8x(6, 3), lossless])], + ["an extended file without image data", riff([vp8x(5, 3)])], + [ + "an animated file", + riff([vp8x(9, 7, 0x02), webpChunk("ANIM", Buffer.alloc(6)), webpChunk("ANMF", Buffer.alloc(16))]), + ], + ["two bitstreams in a simple file", riff([lossless, lossless])], + ["an unknown first chunk", riff([webpChunk("XXXX", Buffer.alloc(4)), lossless])], + [ + "a lossless bitstream with a bad signature", + (() => { + const copy = Buffer.from(bytes(WEBP_LOSSLESS_5X3)); + copy[20] = 0x2e; + return copy; + })(), + ], + [ + "a lossy bitstream with a bad start code", + (() => { + const copy = Buffer.from(bytes(WEBP_LOSSY_5X3)); + copy[23] = 0; + return copy; + })(), + ], + ]; + it.each(cases)("rejects %s", (_name, data) => { + expect(readWebpHeader(data)).toBeUndefined(); + }); + }); +}); + +describe("resizeImageInProcess without decoding", () => { + beforeEach(() => { + photonLoads.count = 0; + }); + afterEach(() => { + photonLoads.count = 0; + }); + + it("returns an image within the limits as it is, without loading photon", async () => { + const png = createPng(300, 200); + const result = await resizeImageInProcess(png, "image/png"); + expect(result).toEqual({ + data: png.toString("base64"), + mimeType: "image/png", + originalWidth: 300, + originalHeight: 200, + width: 300, + height: 200, + wasResized: false, + }); + expect(photonLoads.count).toBe(0); + }); + + it("uses the displayed size of an EXIF-rotated JPEG", async () => { + // 2x1 stored, orientation 6 (rotate 90 degrees): displayed as 1x2. + const jpeg2x1 = editJpegSegment(bytes(JPEG_2X2), 0xc0, (s) => { + s.writeUInt16BE(1, 5); + return s; + }); + const rotated = Buffer.concat([jpeg2x1.subarray(0, 2), app1Exif(6), jpeg2x1.subarray(2)]); + expect(readJpegHeader(rotated)).toEqual({ width: 2, height: 1 }); + + const result = await resizeImageInProcess(rotated, "image/jpeg", { maxWidth: 1, maxHeight: 2 }); + expect(result).toMatchObject({ data: rotated.toString("base64"), width: 1, height: 2, wasResized: false }); + expect(photonLoads.count).toBe(0); + }); + + it("falls back to decoding when the header cannot be read", async () => { + // Trailing bytes after IEND: the header parser gives up, photon still decodes the file. + const png = Buffer.concat([createPng(30, 20), Buffer.from("trailer")]); + expect(readPngHeader(png)).toBeUndefined(); + + const result = await resizeImageInProcess(png, "image/png"); + expect(result).toMatchObject({ data: png.toString("base64"), width: 30, height: 20, wasResized: false }); + expect(photonLoads.count).toBe(1); + }); + + it("falls back to decoding, and omits the image, when a PNG checksum is wrong", async () => { + const png = createPng(30, 20); + png[png.length - 13] ^= 0xff; + + expect(await resizeImageInProcess(png, "image/png")).toBeNull(); + expect(photonLoads.count).toBe(1); + }); + + it("decodes an image over the limits and encodes only until a candidate fits", async () => { + const png = createPng(400, 100); + const result = await resizeImageInProcess(png, "image/png", { maxWidth: 200, maxHeight: 200 }); + expect(result).toMatchObject({ + mimeType: "image/png", + originalWidth: 400, + originalHeight: 100, + width: 200, + height: 50, + wasResized: true, + }); + expect(photonLoads.count).toBe(1); + }); +}); diff --git a/packages/coding-agent/test/image-resize-encode.test.ts b/packages/coding-agent/test/image-resize-encode.test.ts new file mode 100644 index 000000000..bd66ecb1b --- /dev/null +++ b/packages/coding-agent/test/image-resize-encode.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from "vitest"; +import { encodeWithinLimits, type ResizedFrame, resolveImageResizeOptions } from "../src/utils/image-resize-core.ts"; +import { resizeImageNative } from "../src/utils/image-resize-native.ts"; + +/** A frame whose encodings have the given byte sizes, recording what was encoded. */ +function fakeFrames(sizes: { png: number; jpeg: Record }, log: string[]) { + return (width: number, height: number): ResizedFrame => { + log.push(`resize ${width}x${height}`); + return { + png: () => { + log.push("png"); + return new Uint8Array(sizes.png); + }, + jpeg: (quality) => { + log.push(`jpeg ${quality}`); + return new Uint8Array(sizes.jpeg[quality] ?? 1_000_000); + }, + free: () => log.push("free"), + }; + }; +} + +describe("encodeWithinLimits", () => { + // maxBytes counts base64: n bytes encode to ceil(n / 3) * 4 characters. + const opts = resolveImageResizeOptions({ maxWidth: 100, maxHeight: 100, maxBytes: 4000 }); + + it("encodes no JPEG when the PNG fits", async () => { + const log: string[] = []; + const result = await encodeWithinLimits(400, 200, opts, fakeFrames({ png: 2000, jpeg: {} }, log)); + expect(log).toEqual(["resize 100x50", "png", "free"]); + expect(result).toMatchObject({ mimeType: "image/png", width: 100, height: 50, wasResized: true }); + }); + + it("tries JPEG qualities in order and stops at the first that fits", async () => { + const log: string[] = []; + const result = await encodeWithinLimits( + 400, + 200, + opts, + fakeFrames({ png: 5000, jpeg: { 80: 3500, 85: 3600, 70: 2900 } }, log), + ); + expect(log).toEqual(["resize 100x50", "png", "jpeg 80", "jpeg 85", "jpeg 70", "free"]); + expect(result).toMatchObject({ mimeType: "image/jpeg", width: 100, height: 50 }); + }); + + it("shrinks by 75% when no encoding fits, and gives up at 1x1", async () => { + const log: string[] = []; + expect(await encodeWithinLimits(2, 2, opts, fakeFrames({ png: 5000, jpeg: {} }, log))).toBeNull(); + expect(log.filter((entry) => entry.startsWith("resize"))).toEqual(["resize 2x2", "resize 1x1"]); + }); +}); + +describe("resizeImageNative", () => { + /** A stand-in for Bun.Image that records the pipeline and encodes to `size` bytes. */ + function fakeNativeImage(log: string[], size: (format: string) => number) { + return class { + constructor(_input: Uint8Array, options: { autoOrient: boolean }) { + log.push(`new autoOrient=${options.autoOrient}`); + } + resize(width: number, height: number, options: { filter: string; fit: string }) { + log.push(`resize ${width}x${height} ${options.filter} ${options.fit}`); + return this; + } + rotate(degrees: number) { + log.push(`rotate ${degrees}`); + return this; + } + flip() { + log.push("flip"); + return this; + } + flop() { + log.push("flop"); + return this; + } + private format = ""; + png() { + this.format = "png"; + return this; + } + jpeg(options: { quality: number }) { + this.format = `jpeg ${options.quality}`; + return this; + } + async bytes(): Promise { + log.push(`encode ${this.format}`); + return new Uint8Array(size(this.format)); + } + }; + } + + it("applies the EXIF orientation itself, resizes with Lanczos3 and stops at the first fit", async () => { + // A JPEG start with an APP1 Exif segment, orientation 6 (rotate 90 degrees clockwise). + const exif = Buffer.from("457869660000" + "49492a00080000000100120103000100000006000000000000000000", "hex"); + const app1 = Buffer.concat([Buffer.from([0xff, 0xe1, 0, exif.length + 2]), exif]); + const input = Buffer.concat([Buffer.from([0xff, 0xd8]), app1, Buffer.from([0xff, 0xd9])]); + const log: string[] = []; + const NativeImage = fakeNativeImage(log, (format) => (format === "png" ? 9000 : 1000)); + + const result = await resizeImageNative( + NativeImage, + input, + 300, + 600, + resolveImageResizeOptions({ maxWidth: 200, maxHeight: 200, maxBytes: 4000 }), + ); + + expect(log).toEqual([ + "new autoOrient=false", + "rotate 90", + "resize 100x200 lanczos3 fill", + "encode png", + "new autoOrient=false", + "rotate 90", + "resize 100x200 lanczos3 fill", + "encode jpeg 80", + ]); + expect(result).toMatchObject({ + mimeType: "image/jpeg", + originalWidth: 300, + originalHeight: 600, + width: 100, + height: 200, + wasResized: true, + }); + }); + + it("passes native failures to the caller", async () => { + const log: string[] = []; + const Failing = class extends fakeNativeImage(log, () => 0) { + override async bytes(): Promise { + throw new Error("ERR_IMAGE_DECODE_FAILED"); + } + }; + await expect( + resizeImageNative(Failing, new Uint8Array([0xff, 0xd8, 0xff, 0xd9]), 300, 300, resolveImageResizeOptions()), + ).rejects.toThrow("ERR_IMAGE_DECODE_FAILED"); + }); +}); diff --git a/packages/coding-agent/test/suite/agent-session-read-images.test.ts b/packages/coding-agent/test/suite/agent-session-read-images.test.ts new file mode 100644 index 000000000..0e6dc96c0 --- /dev/null +++ b/packages/coding-agent/test/suite/agent-session-read-images.test.ts @@ -0,0 +1,101 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { crc32, deflateSync } from "node:zlib"; +import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type * as ImageProcessModule from "../../src/utils/image-process.ts"; +import { createHarness, getToolResult, type Harness } from "./harness.ts"; + +const processImageCalls = vi.hoisted(() => ({ count: 0 })); +vi.mock("../../src/utils/image-process.ts", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + processImage: (...args: Parameters) => { + processImageCalls.count++; + return actual.processImage(...args); + }, + }; +}); + +function pngChunk(type: string, body: Buffer): Buffer { + const header = Buffer.alloc(8); + header.writeUInt32BE(body.length, 0); + header.write(type, 4, "ascii"); + const checksum = Buffer.alloc(4); + checksum.writeUInt32BE(crc32(Buffer.concat([header.subarray(4), body])), 0); + return Buffer.concat([header, body, checksum]); +} + +/** An 8-bit grayscale PNG with horizontal stripes (small enough to stay PNG after resizing). */ +function createPng(width: number, height: number): Buffer { + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(width, 0); + ihdr.writeUInt32BE(height, 4); + ihdr[8] = 8; + const raw = Buffer.alloc((width + 1) * height); + for (let row = 0; row < height; row++) { + raw.fill(row % 256, row * (width + 1) + 1, (row + 1) * (width + 1)); + } + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", ihdr), + pngChunk("IDAT", deflateSync(raw)), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +describe("AgentSession read of images", () => { + const harnesses: Harness[] = []; + + afterEach(() => { + processImageCalls.count = 0; + while (harnesses.length > 0) { + harnesses.pop()?.cleanup(); + } + }); + + async function readImage(name: string, data: Buffer): Promise<(TextContent | ImageContent)[]> { + const harness = await createHarness(); + harnesses.push(harness); + writeFileSync(join(harness.tempDir, name), data); + harness.setResponses([ + fauxAssistantMessage([fauxToolCall("read", { path: name })], { stopReason: "toolUse" }), + fauxAssistantMessage("done"), + ]); + await harness.session.prompt("look at the image"); + return getToolResult(harness, "read").content; + } + + it("sends an image within the limits byte for byte, processed once", async () => { + const png = createPng(1800, 1100); + + const content = await readImage("small.png", png); + + expect(content).toEqual([ + { type: "text", text: "Read image file [image/png]" }, + { type: "image", data: png.toString("base64"), mimeType: "image/png" }, + ]); + // The read tool processes it; tool-result normalization recognizes the result and skips it. + expect(processImageCalls.count).toBe(1); + }); + + it("resizes an image over the limits once, with the dimension note", async () => { + const png = createPng(2442, 1438); + + const content = await readImage("shot.png", png); + + expect(content).toHaveLength(2); + const [note, image] = content; + expect(note).toEqual({ + type: "text", + text: "Read image file [image/png]\n[Image: original 2442x1438, displayed at 2000x1178. Multiply coordinates by 1.22 to map to original image.]", + }); + if (image.type !== "image") throw new Error("Expected an image"); + expect(image.mimeType).toBe("image/png"); + const resized = Buffer.from(image.data, "base64"); + expect([resized.readUInt32BE(16), resized.readUInt32BE(20)]).toEqual([2000, 1178]); + expect(processImageCalls.count).toBe(1); + }); +}); From 25014f777ab8039df2533ea1d66fa4ca04cb47d0 Mon Sep 17 00:00:00 2001 From: OpenSec <323596076+opensec-intelligence@users.noreply.github.com> Date: Tue, 20 Oct 2026 06:41:35 +0530 Subject: [PATCH 2/2] fix(coding-agent): images within the limits are decoded before they are sent as they are The header check alone let through files whose structure is intact but whose image data is damaged, for example a JPEG with corrupt scan data ending in FF D9. 0.9.5 decoded every image with photon and omitted those; the previous commit sent them as they were, which a provider can reject for the whole request. - PNG and WebP within the limits skip photon only when Bun.Image decodes them (a full native decode to 1x1: about 5 ms for an 1800x1100 PNG, 25 ms for an 1800x1100 WebP). When it fails, photon decodes them as in 0.9.5. - JPEG and GIF within the limits always go through photon: libjpeg-turbo and Bun's GIF decoder accept damaged data that photon rejects. - Without Bun.Image (Node, photon with a JIT), every format goes through photon, as in 0.9.5. - resizeImageInProcess (photon path) no longer skips the decode. CPU per `pi -p` read against 0.9.5 (medians of 3): 1800x1100 PNG 996 -> 47 ms, 1800x1100 WebP 6328 -> 68 ms, 1600x1200 JPEG 2094 -> 1052 ms (one photon decode instead of two). Requests are identical to 0.9.5 for these files and for a damaged JPEG ("Image omitted") and a damaged WebP. --- .../src/utils/image-resize-core.ts | 25 ++- .../src/utils/image-resize-native.ts | 20 +- .../coding-agent/src/utils/image-resize.ts | 51 ++++- .../coding-agent/test/image-header.test.ts | 198 ++++++++++++++++-- 4 files changed, 257 insertions(+), 37 deletions(-) diff --git a/packages/coding-agent/src/utils/image-resize-core.ts b/packages/coding-agent/src/utils/image-resize-core.ts index 83bfb745d..168616123 100644 --- a/packages/coding-agent/src/utils/image-resize-core.ts +++ b/packages/coding-agent/src/utils/image-resize-core.ts @@ -59,6 +59,16 @@ function base64Size(byteLength: number): number { return Math.ceil(byteLength / 3) * 4; } +/** Whether an image of this displayed size (after EXIF orientation) and byte length needs no change. */ +export function isWithinLimits( + width: number, + height: number, + byteLength: number, + opts: Required, +): boolean { + return width <= opts.maxWidth && height <= opts.maxHeight && base64Size(byteLength) < opts.maxBytes; +} + /** * The input as it is when it is already within the dimension and size limits, else undefined. * `width` and `height` are the displayed size (after EXIF orientation). @@ -70,9 +80,7 @@ export function keepIfWithinLimits( height: number, opts: Required, ): ResizedImage | undefined { - if (width > opts.maxWidth || height > opts.maxHeight || base64Size(inputBytes.byteLength) >= opts.maxBytes) { - return undefined; - } + if (!isWithinLimits(width, height, inputBytes.byteLength, opts)) return undefined; const format = mimeType.split("/")[1] ?? "png"; return { data: Buffer.from(inputBytes.buffer, inputBytes.byteOffset, inputBytes.byteLength).toString("base64"), @@ -162,9 +170,9 @@ export async function encodeWithinLimits( * Resize an image to fit within the specified max dimensions and encoded file size. * Returns null if the image cannot be resized below maxBytes. * - * An image whose header shows it is already within the limits is returned as it is, without - * decoding. Otherwise Photon (Rust/WASM) decodes it once and resizes it (see encodeWithinLimits). - * If Photon is not available, returns null. + * Photon (Rust/WASM) decodes the image once; an image within the limits is returned as it is, and + * one over them is resized (see encodeWithinLimits). If Photon is not available or cannot decode + * the image, returns null. resizeImage skips this decode when it can show the image is valid. */ export async function resizeImageInProcess( inputBytes: Uint8Array, @@ -172,11 +180,6 @@ export async function resizeImageInProcess( options?: ImageResizeOptions, ): Promise { const opts = resolveImageResizeOptions(options); - - const size = readDisplayedImageSize(inputBytes, mimeType); - const unchanged = size && keepIfWithinLimits(inputBytes, mimeType, size.width, size.height, opts); - if (unchanged) return unchanged; - const photon = await loadPhoton(); if (!photon) { return null; diff --git a/packages/coding-agent/src/utils/image-resize-native.ts b/packages/coding-agent/src/utils/image-resize-native.ts index 699e0452b..8d898ac0c 100644 --- a/packages/coding-agent/src/utils/image-resize-native.ts +++ b/packages/coding-agent/src/utils/image-resize-native.ts @@ -6,7 +6,7 @@ import { encodeWithinLimits, type ImageResizeOptions, type ResizedImage } from " * so the shape is declared locally; it matches `packages/bun-types/bun.d.ts` of the pinned runtime. */ interface NativeImagePipeline { - resize(width: number, height: number, options: { filter: "lanczos3"; fit: "fill" }): NativeImagePipeline; + resize(width: number, height: number, options: { filter: "lanczos3" | "box"; fit: "fill" }): NativeImagePipeline; rotate(degrees: number): NativeImagePipeline; flip(): NativeImagePipeline; flop(): NativeImagePipeline; @@ -15,7 +15,7 @@ interface NativeImagePipeline { bytes(): Promise; } -type NativeImageConstructor = new (input: Uint8Array, options: { autoOrient: boolean }) => NativeImagePipeline; +export type NativeImageConstructor = new (input: Uint8Array, options: { autoOrient: boolean }) => NativeImagePipeline; /** `Bun.Image` when the runtime has it (Bun, including Pi-Bolt's runtime), else undefined (Node). */ export function getNativeImage(): NativeImageConstructor | undefined { @@ -81,3 +81,19 @@ export function resizeImageNative( }; }); } + +/** + * Whether the native decoder decodes the whole image without an error. A full decode to a 1x1 box + * average: about 4-6 ms for a 1600x1200 JPEG or a 1800x1100 PNG, 25-35 ms for WebP and GIF. + */ +export async function decodesNatively(NativeImage: NativeImageConstructor, inputBytes: Uint8Array): Promise { + try { + await new NativeImage(inputBytes, { autoOrient: false }) + .resize(1, 1, { filter: "box", fit: "fill" }) + .png() + .bytes(); + return true; + } catch { + return false; + } +} diff --git a/packages/coding-agent/src/utils/image-resize.ts b/packages/coding-agent/src/utils/image-resize.ts index 57f9154ca..e15d82430 100644 --- a/packages/coding-agent/src/utils/image-resize.ts +++ b/packages/coding-agent/src/utils/image-resize.ts @@ -2,6 +2,7 @@ import type { Worker } from "node:worker_threads"; import { IMAGE_RESIZE_WORKER_RESPONSE_TYPE, type ImageResizeOptions, + isWithinLimits, keepIfWithinLimits, type ResizedImage, type ResizeImageWorkerResponse, @@ -9,7 +10,12 @@ import { resizeImageInProcess, resolveImageResizeOptions, } from "./image-resize-core.ts"; -import { getNativeImage, resizeImageNative } from "./image-resize-native.ts"; +import { + decodesNatively, + getNativeImage, + type NativeImageConstructor, + resizeImageNative, +} from "./image-resize-native.ts"; export type { ImageResizeOptions, ResizedImage } from "./image-resize-core.ts"; @@ -79,11 +85,36 @@ async function resizeImageInWorker( } } +/** + * The input as it is, when its header shows it is within the limits and its data is known to + * decode, else undefined (the caller then decodes it with photon, which also rejects damaged data). + * + * - PNG and WebP, with a native image pipeline: the native decoder must decode it. + * - Not JPEG or GIF: the native decoders accept damaged data photon rejects (libjpeg-turbo decodes + * corrupt or truncated scan data with only a warning), and such an image must still be omitted. + * - Not without a native pipeline (Node, where photon runs with a JIT): photon decodes it as before. + */ +export async function keepValidImageWithinLimits( + inputBytes: Uint8Array, + mimeType: string, + width: number, + height: number, + opts: Required, + NativeImage: NativeImageConstructor | undefined, +): Promise { + if (!isWithinLimits(width, height, inputBytes.byteLength, opts)) return undefined; + if (!NativeImage || (mimeType !== "image/png" && mimeType !== "image/webp")) return undefined; + if (!(await decodesNatively(NativeImage, inputBytes))) return undefined; + return keepIfWithinLimits(inputBytes, mimeType, width, height, opts); +} + /** * Resize an image to fit within the specified max dimensions and encoded file size. * - * - An image whose header shows it is within the limits is returned as it is, without decoding. - * - Otherwise, when the runtime has a native image pipeline (`Bun.Image`), it resizes there. + * - An image within the limits whose data is known to decode is returned as it is, without photon + * (see keepValidImageWithinLimits). + * - An image over the limits, when the runtime has a native image pipeline (`Bun.Image`), is + * resized there. * - Otherwise, or when the header cannot be read or the native pipeline fails, Photon runs in a * worker thread so WASM decoding, resizing, and encoding do not block the TUI event loop. If the * worker cannot be loaded (for example in some Bun compiled executable layouts), fall back to @@ -97,10 +128,18 @@ export async function resizeImage( const opts = resolveImageResizeOptions(options); const size = readDisplayedImageSize(inputBytes, mimeType); if (size) { - const unchanged = keepIfWithinLimits(inputBytes, mimeType, size.width, size.height, opts); - if (unchanged) return unchanged; const NativeImage = getNativeImage(); - if (NativeImage) { + if (isWithinLimits(size.width, size.height, inputBytes.byteLength, opts)) { + const unchanged = await keepValidImageWithinLimits( + inputBytes, + mimeType, + size.width, + size.height, + opts, + NativeImage, + ); + if (unchanged) return unchanged; + } else if (NativeImage) { try { return await resizeImageNative(NativeImage, inputBytes, size.width, size.height, opts); } catch {} diff --git a/packages/coding-agent/test/image-header.test.ts b/packages/coding-agent/test/image-header.test.ts index 1b41fee8b..2932fd402 100644 --- a/packages/coding-agent/test/image-header.test.ts +++ b/packages/coding-agent/test/image-header.test.ts @@ -21,7 +21,13 @@ import { readPngHeader, readWebpHeader, } from "../src/utils/image-header.ts"; -import { resizeImageInProcess } from "../src/utils/image-resize-core.ts"; +import { keepValidImageWithinLimits, resizeImage } from "../src/utils/image-resize.ts"; +import { + readDisplayedImageSize, + resizeImageInProcess, + resolveImageResizeOptions, +} from "../src/utils/image-resize-core.ts"; +import type { NativeImageConstructor } from "../src/utils/image-resize-native.ts"; import { loadPhoton } from "../src/utils/photon.ts"; // Small 2x2 blue JPEG image (baseline) - generated with ImageMagick @@ -388,7 +394,76 @@ describe("image header parsers", () => { }); }); -describe("resizeImageInProcess without decoding", () => { +/** Copy of `data` with byte `offset` set to `value`. */ +function withByte(data: Buffer, offset: number, value: number): Buffer { + const copy = Buffer.from(data); + copy[offset] = value; + return copy; +} + +// Damaged image data with an intact structure: the header parsers accept these, photon rejects them. +const DAMAGED_JPEG = withByte(bytes(JPEG_PROGRESSIVE_5X3), 555, 0xff); +const DAMAGED_WEBP_LOSSLESS = withByte(bytes(WEBP_LOSSLESS_5X3), 59, 0xff); +const DAMAGED_WEBP_LOSSY = withByte(bytes(WEBP_LOSSY_5X3), 31, 0x55); +/** A GIF whose first LZW code after the clear code (4) is 7, a code not defined yet. */ +function damagedGif(): Buffer { + const gif = createGif(6, 4, [[0, 0, 6, 4]]); + // Image data starts after the header (13), colour table (6), GCE (8) and descriptor (10): the + // minimum code size, the sub-block size, then the codes packed 3 bits each, least significant first. + const codes = 13 + 6 + 8 + 10 + 2; + gif[codes] = 4 | (7 << 3); + return gif; +} + +/** A PNG whose chunk CRCs are valid but whose compressed pixel data is cut short. */ +function damagedPng(): Buffer { + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(30, 0); + ihdr.writeUInt32BE(20, 4); + ihdr[8] = 8; + const raw = deflateSync(Buffer.alloc(31 * 20, 7)); + return Buffer.concat([ + PNG_SIGNATURE, + pngChunk("IHDR", ihdr), + pngChunk("IDAT", raw.subarray(0, 4)), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +/** A stand-in for Bun.Image whose decode succeeds or fails, recording what it was given. */ +function fakeNativeImage(decodes: boolean, log: Uint8Array[]): NativeImageConstructor { + return class { + constructor(input: Uint8Array) { + log.push(input); + } + resize() { + return this; + } + rotate() { + return this; + } + flip() { + return this; + } + flop() { + return this; + } + png() { + return this; + } + jpeg() { + return this; + } + async bytes(): Promise { + if (!decodes) throw new Error("ERR_IMAGE_DECODE_FAILED"); + return new Uint8Array(1); + } + }; +} + +describe("images within the limits", () => { + const opts = resolveImageResizeOptions(); + beforeEach(() => { photonLoads.count = 0; }); @@ -396,22 +471,71 @@ describe("resizeImageInProcess without decoding", () => { photonLoads.count = 0; }); - it("returns an image within the limits as it is, without loading photon", async () => { - const png = createPng(300, 200); - const result = await resizeImageInProcess(png, "image/png"); + it("are returned as they are when the native decoder decodes them (PNG, WebP)", async () => { + const log: Uint8Array[] = []; + const webp = bytes(WEBP_LOSSY_5X3); + const result = await keepValidImageWithinLimits(webp, "image/webp", 5, 3, opts, fakeNativeImage(true, log)); expect(result).toEqual({ - data: png.toString("base64"), - mimeType: "image/png", - originalWidth: 300, - originalHeight: 200, - width: 300, - height: 200, + data: WEBP_LOSSY_5X3, + mimeType: "image/webp", + originalWidth: 5, + originalHeight: 3, + width: 5, + height: 3, wasResized: false, }); + expect(log).toEqual([webp]); + const png = createPng(300, 200); + expect( + await keepValidImageWithinLimits(png, "image/png", 300, 200, opts, fakeNativeImage(true, [])), + ).toMatchObject({ + data: png.toString("base64"), + }); expect(photonLoads.count).toBe(0); }); - it("uses the displayed size of an EXIF-rotated JPEG", async () => { + it("take the photon path when the native decoder fails", async () => { + expect( + await keepValidImageWithinLimits(DAMAGED_WEBP_LOSSY, "image/webp", 5, 3, opts, fakeNativeImage(false, [])), + ).toBeUndefined(); + expect( + await keepValidImageWithinLimits(damagedPng(), "image/png", 30, 20, opts, fakeNativeImage(false, [])), + ).toBeUndefined(); + }); + + it.each([ + ["JPEG", "image/jpeg", bytes(JPEG_2X2), 2, 2], + ["GIF", "image/gif", createGif(6, 4, [[0, 0, 6, 4]]), 6, 4], + ])( + "take the photon path for %s even when the native decoder accepts it", + async (_name, mimeType, data, width, height) => { + const log: Uint8Array[] = []; + expect( + await keepValidImageWithinLimits(data, mimeType, width, height, opts, fakeNativeImage(true, log)), + ).toBeUndefined(); + expect(log).toEqual([]); + }, + ); + + it("take the photon path without a native decoder", async () => { + expect( + await keepValidImageWithinLimits(createPng(300, 200), "image/png", 300, 200, opts, undefined), + ).toBeUndefined(); + expect(await keepValidImageWithinLimits(bytes(JPEG_2X2), "image/jpeg", 2, 2, opts, undefined)).toBeUndefined(); + expect( + await keepValidImageWithinLimits(bytes(WEBP_LOSSY_5X3), "image/webp", 5, 3, opts, undefined), + ).toBeUndefined(); + }); + + it("are not kept when over the limits", async () => { + const png = createPng(300, 200); + const small = resolveImageResizeOptions({ maxWidth: 299 }); + expect( + await keepValidImageWithinLimits(png, "image/png", 300, 200, small, fakeNativeImage(true, [])), + ).toBeUndefined(); + }); + + it("use the displayed size of an EXIF-rotated JPEG", () => { // 2x1 stored, orientation 6 (rotate 90 degrees): displayed as 1x2. const jpeg2x1 = editJpegSegment(bytes(JPEG_2X2), 0xc0, (s) => { s.writeUInt16BE(1, 5); @@ -419,13 +543,51 @@ describe("resizeImageInProcess without decoding", () => { }); const rotated = Buffer.concat([jpeg2x1.subarray(0, 2), app1Exif(6), jpeg2x1.subarray(2)]); expect(readJpegHeader(rotated)).toEqual({ width: 2, height: 1 }); + expect(readDisplayedImageSize(rotated, "image/jpeg")).toEqual({ width: 1, height: 2 }); + }); +}); - const result = await resizeImageInProcess(rotated, "image/jpeg", { maxWidth: 1, maxHeight: 2 }); - expect(result).toMatchObject({ data: rotated.toString("base64"), width: 1, height: 2, wasResized: false }); - expect(photonLoads.count).toBe(0); +describe("resizeImage with damaged image data (as in 0.9.5: photon rejects it, the image is omitted)", () => { + const damaged: [string, string, Buffer][] = [ + ["JPEG", "image/jpeg", DAMAGED_JPEG], + ["lossless WebP", "image/webp", DAMAGED_WEBP_LOSSLESS], + ["lossy WebP", "image/webp", DAMAGED_WEBP_LOSSY], + ["GIF", "image/gif", damagedGif()], + ["PNG with valid CRCs", "image/png", damagedPng()], + ]; + + it.each(damaged)( + "%s: the header is accepted, photon rejects the data, the result is null", + async (_name, mimeType, data) => { + expect(readImageHeader(data, mimeType)).toBeDefined(); + const photon = (await loadPhoton())!; + expect(() => photon.PhotonImage.new_from_byteslice(data)).toThrow(); + expect(await resizeImage(data, mimeType)).toBeNull(); + }, + ); + + it("PNG with a wrong checksum: the header is rejected, the result is null", async () => { + const png = createPng(30, 20); + png[png.length - 13] ^= 0xff; + expect(readPngHeader(png)).toBeUndefined(); + expect(await resizeImage(png, "image/png")).toBeNull(); }); - it("falls back to decoding when the header cannot be read", async () => { + it.each([ + ["PNG", "image/png", createPng(30, 20).toString("base64")], + ["JPEG", "image/jpeg", JPEG_PROGRESSIVE_5X3], + ["WebP", "image/webp", WEBP_LOSSY_5X3], + ])("an intact %s is returned as it is", async (_name, mimeType, base64) => { + expect(await resizeImage(bytes(base64), mimeType)).toMatchObject({ data: base64, wasResized: false }); + }); +}); + +describe("resizeImageInProcess", () => { + beforeEach(() => { + photonLoads.count = 0; + }); + + it("decodes when the header cannot be read", async () => { // Trailing bytes after IEND: the header parser gives up, photon still decodes the file. const png = Buffer.concat([createPng(30, 20), Buffer.from("trailer")]); expect(readPngHeader(png)).toBeUndefined(); @@ -435,12 +597,12 @@ describe("resizeImageInProcess without decoding", () => { expect(photonLoads.count).toBe(1); }); - it("falls back to decoding, and omits the image, when a PNG checksum is wrong", async () => { + it("omits the image when a PNG checksum is wrong", async () => { const png = createPng(30, 20); png[png.length - 13] ^= 0xff; + expect(readPngHeader(png)).toBeUndefined(); expect(await resizeImageInProcess(png, "image/png")).toBeNull(); - expect(photonLoads.count).toBe(1); }); it("decodes an image over the limits and encodes only until a candidate fits", async () => {