diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index 45b9e85c5..921d22116 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -3,7 +3,7 @@ import type { Api, ImageContent, Model, ModelImageResizeOptions, TextContent } f import { constants } from "fs"; import { access as fsAccess, readFile as fsReadFile } from "fs/promises"; import { type Static, Type } from "typebox"; -import { processImage } from "../../utils/image-process.ts"; +import { markProcessedImageBlock, processImage } from "../../utils/image-process.ts"; import { detectSupportedImageMimeTypeFromFile } from "../../utils/mime.ts"; import { detachString } from "../../utils/text.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; @@ -163,10 +163,11 @@ export function createReadToolDefinition( if (mimeType) { // Read image as binary. const buffer = await ops.readFile(absolutePath); - const processed = await processImage(buffer, mimeType, { + const processOptions = { autoResizeImages, resizeOptions: ctx?.model?.inputLimits?.images?.resize ?? fallbackResizeOptions, - }); + }; + const processed = await processImage(buffer, mimeType, processOptions); if (!processed.ok) { let textNote = `Read image file [${mimeType}]\n${processed.message}`; if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; @@ -175,10 +176,13 @@ export function createReadToolDefinition( let textNote = `Read image file [${processed.mimeType}]`; if (processed.hints.length > 0) textNote += `\n${processed.hints.join("\n")}`; if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; - content = [ - { type: "text", text: textNote }, - { type: "image", data: processed.data, mimeType: processed.mimeType }, - ]; + const image: ImageContent = { + type: "image", + data: processed.data, + mimeType: processed.mimeType, + }; + markProcessedImageBlock(image, processOptions); + content = [{ type: "text", text: textNote }, image]; } } else { // Read text content. diff --git a/packages/coding-agent/src/utils/exif-orientation.ts b/packages/coding-agent/src/utils/exif-orientation.ts index d1c47df1b..44c24e4ee 100644 --- a/packages/coding-agent/src/utils/exif-orientation.ts +++ b/packages/coding-agent/src/utils/exif-orientation.ts @@ -94,7 +94,8 @@ function hasExifHeader(bytes: Uint8Array, offset: number): boolean { ); } -function getExifOrientation(bytes: Uint8Array): number { +/** EXIF orientation (1-8) of a JPEG or WebP file; 1 when there is none. */ +export function getExifOrientation(bytes: Uint8Array): number { let tiffOffset = -1; // JPEG: starts with FF D8 diff --git a/packages/coding-agent/src/utils/image-header.ts b/packages/coding-agent/src/utils/image-header.ts new file mode 100644 index 000000000..1e0e2209a --- /dev/null +++ b/packages/coding-agent/src/utils/image-header.ts @@ -0,0 +1,306 @@ +/** + * Width and height of an image, read from its file structure without decoding pixels. + * + * Resizing needs the dimensions only to decide whether an image already fits the limits. Decoding a + * whole screenshot for that costs about a second in photon's WebAssembly when it runs interpreted. + * + * The parsers are strict on purpose: they return undefined for anything they do not fully + * understand (unknown chunk layout, truncation, bad PNG checksums, uncommon JPEG codings), and the + * caller then takes the full decode path as before. A file is accepted only when its structure is + * intact from the signature to the end marker. + */ +export interface ImageHeader { + width: number; + height: number; +} + +// node:zlib loads on first use, not at every start. +function crc32(data: Uint8Array): number { + return process.getBuiltinModule("node:zlib").crc32(data); +} + +function u16be(bytes: Uint8Array, offset: number): number { + return (bytes[offset] << 8) | bytes[offset + 1]; +} + +function u32be(bytes: Uint8Array, offset: number): number { + return ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) >>> 0; +} + +function u16le(bytes: Uint8Array, offset: number): number { + return bytes[offset] | (bytes[offset + 1] << 8); +} + +function u24le(bytes: Uint8Array, offset: number): number { + return bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16); +} + +function u32le(bytes: Uint8Array, offset: number): number { + return (bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16) | (bytes[offset + 3] << 24)) >>> 0; +} + +function asciiAt(bytes: Uint8Array, offset: number, text: string): boolean { + if (offset + text.length > bytes.length) return false; + for (let i = 0; i < text.length; i++) { + if (bytes[offset + i] !== text.charCodeAt(i)) return false; + } + return true; +} + +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + +/** Bit depths the PNG specification allows for each colour type. */ +const PNG_BIT_DEPTHS: Record = { + 0: [1, 2, 4, 8, 16], + 2: [8, 16], + 3: [1, 2, 4, 8], + 4: [8, 16], + 6: [8, 16], +}; + +function isChunkTypeByte(byte: number): boolean { + return (byte >= 0x41 && byte <= 0x5a) || (byte >= 0x61 && byte <= 0x7a); +} + +/** + * PNG: every chunk is walked and its CRC checked (decoders reject a bad CRC), IHDR must come first + * with valid fields, IDAT must be present (after PLTE for palette images), and IEND must end the file. + */ +export function readPngHeader(bytes: Uint8Array): ImageHeader | undefined { + if (bytes.length < PNG_SIGNATURE.length + 25 + 12 + 12) return undefined; + for (let i = 0; i < PNG_SIGNATURE.length; i++) { + if (bytes[i] !== PNG_SIGNATURE[i]) return undefined; + } + + let header: ImageHeader | undefined; + let colorType = -1; + let sawPalette = false; + let sawData = false; + let offset = PNG_SIGNATURE.length; + while (offset + 12 <= bytes.length) { + const length = u32be(bytes, offset); + if (length > 0x7fffffff) return undefined; + const dataStart = offset + 8; + const dataEnd = dataStart + length; + if (dataEnd + 4 > bytes.length) return undefined; + for (let i = 4; i < 8; i++) { + if (!isChunkTypeByte(bytes[offset + i])) return undefined; + } + if (crc32(bytes.subarray(offset + 4, dataEnd)) !== u32be(bytes, dataEnd)) return undefined; + + const isFirst = offset === PNG_SIGNATURE.length; + const isHeader = asciiAt(bytes, offset + 4, "IHDR"); + if (isFirst !== isHeader) return undefined; + if (isHeader) { + if (length !== 13) return undefined; + const width = u32be(bytes, dataStart); + const height = u32be(bytes, dataStart + 4); + const bitDepth = bytes[dataStart + 8]; + colorType = bytes[dataStart + 9]; + const compression = bytes[dataStart + 10]; + const filter = bytes[dataStart + 11]; + const interlace = bytes[dataStart + 12]; + if (width === 0 || height === 0 || width > 0x7fffffff || height > 0x7fffffff) return undefined; + if (!PNG_BIT_DEPTHS[colorType]?.includes(bitDepth)) return undefined; + if (compression !== 0 || filter !== 0 || interlace > 1) return undefined; + header = { width, height }; + } else if (asciiAt(bytes, offset + 4, "PLTE")) { + if (sawData) return undefined; + sawPalette = true; + } else if (asciiAt(bytes, offset + 4, "IDAT")) { + if (colorType === 3 && !sawPalette) return undefined; + sawData = true; + } else if (asciiAt(bytes, offset + 4, "IEND")) { + // IEND must be empty and end the file. + if (length !== 0 || dataEnd + 4 !== bytes.length || !sawData) return undefined; + return header; + } + offset = dataEnd + 4; + } + return undefined; +} + +/** + * JPEG: the marker segments before the first scan are walked, there must be exactly one baseline, + * extended or progressive Huffman frame header (SOF0-SOF2) with 8-bit samples and 1 or 3 + * components, and the file must end with the EOI marker. Lossless, arithmetic-coded, 12-bit and + * CMYK files take the decode path. + */ +export function readJpegHeader(bytes: Uint8Array): ImageHeader | undefined { + if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return undefined; + if (bytes[bytes.length - 2] !== 0xff || bytes[bytes.length - 1] !== 0xd9) return undefined; + + let header: ImageHeader | undefined; + let offset = 2; + while (offset < bytes.length) { + if (bytes[offset] !== 0xff) return undefined; + // Any number of 0xff fill bytes may precede a marker. + while (offset < bytes.length && bytes[offset] === 0xff) offset++; + if (offset >= bytes.length) return undefined; + const marker = bytes[offset]; + offset++; + // Markers without a segment (TEM, RSTn, SOI, EOI) cannot appear before the first scan. + if (marker === 0x00 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) return undefined; + if (offset + 2 > bytes.length) return undefined; + const length = u16be(bytes, offset); + if (length < 2 || offset + length > bytes.length) return undefined; + + if (marker === 0xda) { + // Start of scan: the frame header must have come first. + return header; + } + // C0-CF other than DHT (C4) are frame headers, JPG (C8) or DAC (CC). Only SOF0-SOF2 are accepted. + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4) { + if (header || marker > 0xc2 || length < 8) return undefined; + const precision = bytes[offset + 2]; + const height = u16be(bytes, offset + 3); + const width = u16be(bytes, offset + 5); + const components = bytes[offset + 7]; + if (precision !== 8 || width === 0 || height === 0) return undefined; + if (components !== 1 && components !== 3) return undefined; + if (length !== 8 + components * 3) return undefined; + header = { width, height }; + } + offset += length; + } + return undefined; +} + +function skipGifSubBlocks(bytes: Uint8Array, offset: number): number { + while (offset < bytes.length) { + const size = bytes[offset]; + offset += 1 + size; + if (size === 0) return offset; + } + return -1; +} + +/** + * GIF: the logical screen size, after walking every block to the trailer. Each frame must lie + * within the logical screen, and the trailer must end the file. + */ +export function readGifHeader(bytes: Uint8Array): ImageHeader | undefined { + if (bytes.length < 14 || !(asciiAt(bytes, 0, "GIF87a") || asciiAt(bytes, 0, "GIF89a"))) return undefined; + const width = u16le(bytes, 6); + const height = u16le(bytes, 8); + if (width === 0 || height === 0) return undefined; + const screenFlags = bytes[10]; + let offset = 13; + if (screenFlags & 0x80) offset += 3 * (1 << ((screenFlags & 0x07) + 1)); + + let frames = 0; + while (offset < bytes.length) { + const block = bytes[offset]; + if (block === 0x3b) { + return frames > 0 && offset + 1 === bytes.length ? { width, height } : undefined; + } + if (block === 0x21) { + if (offset + 2 > bytes.length) return undefined; + offset = skipGifSubBlocks(bytes, offset + 2); + } else if (block === 0x2c) { + if (offset + 10 > bytes.length) return undefined; + const left = u16le(bytes, offset + 1); + const top = u16le(bytes, offset + 3); + const frameWidth = u16le(bytes, offset + 5); + const frameHeight = u16le(bytes, offset + 7); + const frameFlags = bytes[offset + 9]; + if (frameWidth === 0 || frameHeight === 0 || left + frameWidth > width || top + frameHeight > height) { + return undefined; + } + offset += 10; + if (frameFlags & 0x80) offset += 3 * (1 << ((frameFlags & 0x07) + 1)); + if (offset >= bytes.length) return undefined; + const minCodeSize = bytes[offset]; + if (minCodeSize < 2 || minCodeSize > 8) return undefined; + offset = skipGifSubBlocks(bytes, offset + 1); + frames++; + } else { + return undefined; + } + if (offset < 0) return undefined; + } + return undefined; +} + +/** + * WebP: the RIFF size must match the file and every chunk must fit. The file is either one lossy + * (VP8) or lossless (VP8L) bitstream, or an extended (VP8X) canvas with one bitstream of the canvas + * size. Animated files take the decode path. + */ +export function readWebpHeader(bytes: Uint8Array): ImageHeader | undefined { + if (bytes.length < 20 || !asciiAt(bytes, 0, "RIFF") || !asciiAt(bytes, 8, "WEBP")) return undefined; + if (u32le(bytes, 4) + 8 !== bytes.length) return undefined; + + let canvas: ImageHeader | undefined; + let frame: ImageHeader | undefined; + let offset = 12; + while (offset < bytes.length) { + if (offset + 8 > bytes.length) return undefined; + const size = u32le(bytes, offset + 4); + const dataStart = offset + 8; + const dataEnd = dataStart + size; + if (dataEnd > bytes.length) return undefined; + const isFirst = offset === 12; + // Without a VP8X header, the file is a single VP8 or VP8L chunk. + if (!isFirst && !canvas) return undefined; + + if (asciiAt(bytes, offset, "VP8 ") || asciiAt(bytes, offset, "VP8L")) { + if (frame) return undefined; + frame = readWebpBitstreamSize(bytes, offset, dataStart, size); + if (!frame) return undefined; + } else if (asciiAt(bytes, offset, "VP8X")) { + if (!isFirst || size < 10) return undefined; + canvas = { width: u24le(bytes, dataStart + 4) + 1, height: u24le(bytes, dataStart + 7) + 1 }; + } else if (isFirst || asciiAt(bytes, offset, "ANIM") || asciiAt(bytes, offset, "ANMF")) { + return undefined; + } + // Chunks are padded to an even size. + offset = dataEnd + (size & 1); + } + if (offset !== bytes.length || !frame) return undefined; + // An extended file's one bitstream must fill the canvas. + if (canvas && (frame.width !== canvas.width || frame.height !== canvas.height)) return undefined; + return frame; +} + +function readWebpBitstreamSize( + bytes: Uint8Array, + chunkOffset: number, + dataStart: number, + size: number, +): ImageHeader | undefined { + if (asciiAt(bytes, chunkOffset, "VP8L")) { + // Signature 0x2f, then 14-bit width-1 and height-1, an alpha hint bit and a 3-bit version (0). + if (size < 5 || bytes[dataStart] !== 0x2f) return undefined; + const bits = u32le(bytes, dataStart + 1); + if (bits >>> 29 !== 0) return undefined; + return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 }; + } + // VP8: 3-byte frame tag (key frame flag clear), start code 9d 01 2a, then 14-bit width and height. + if (size < 10 || (bytes[dataStart] & 0x01) !== 0) return undefined; + if (bytes[dataStart + 3] !== 0x9d || bytes[dataStart + 4] !== 0x01 || bytes[dataStart + 5] !== 0x2a) { + return undefined; + } + const width = u16le(bytes, dataStart + 6) & 0x3fff; + const height = u16le(bytes, dataStart + 8) & 0x3fff; + return width === 0 || height === 0 ? undefined : { width, height }; +} + +/** + * Dimensions from the header of a well-formed PNG, JPEG, GIF or WebP file, as stored (before any + * EXIF orientation). Undefined when the format is unknown or the structure is not fully understood. + */ +export function readImageHeader(bytes: Uint8Array, mimeType: string): ImageHeader | undefined { + switch (mimeType) { + case "image/png": + return readPngHeader(bytes); + case "image/jpeg": + return readJpegHeader(bytes); + case "image/gif": + return readGifHeader(bytes); + case "image/webp": + return readWebpHeader(bytes); + default: + return undefined; + } +} diff --git a/packages/coding-agent/src/utils/image-process.ts b/packages/coding-agent/src/utils/image-process.ts index a461c8b21..f54407739 100644 --- a/packages/coding-agent/src/utils/image-process.ts +++ b/packages/coding-agent/src/utils/image-process.ts @@ -1,5 +1,6 @@ import { convertImageBytesToPng } from "./image-convert.ts"; import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize.ts"; +import { resolveImageResizeOptions } from "./image-resize-core.ts"; export interface ProcessImageOptions { /** Whether to resize images to inline provider limits. Default: true */ @@ -117,3 +118,49 @@ export async function processImage( hints, }; } + +interface ProcessedImageBlock { + data: string; + mimeType: string; + autoResizeImages: boolean; + limits: Required; +} + +/** Image blocks built from processImage results, with what they were processed for. */ +const processedImageBlocks = new WeakMap(); + +/** + * Record that `block` holds a processImage result for `options`. Processing such a block again with + * the same options returns it unchanged (the result is already within the limits and in a supported + * format), so tool-result normalization can skip it: see isProcessedImageBlock. + */ +export function markProcessedImageBlock( + block: { data: string; mimeType: string }, + options: ProcessImageOptions | undefined, +): void { + processedImageBlocks.set(block, { + data: block.data, + mimeType: block.mimeType, + autoResizeImages: options?.autoResizeImages ?? true, + limits: resolveImageResizeOptions(options?.resizeOptions), + }); +} + +/** Whether `block` is unchanged since markProcessedImageBlock recorded it for the same options. */ +export function isProcessedImageBlock( + block: { data: string; mimeType: string }, + options: ProcessImageOptions | undefined, +): boolean { + const processed = processedImageBlocks.get(block); + if (!processed || processed.data !== block.data || processed.mimeType !== block.mimeType) return false; + const autoResizeImages = options?.autoResizeImages ?? true; + if (processed.autoResizeImages !== autoResizeImages) return false; + if (!autoResizeImages) return true; + const limits = resolveImageResizeOptions(options?.resizeOptions); + return ( + processed.limits.maxWidth === limits.maxWidth && + processed.limits.maxHeight === limits.maxHeight && + processed.limits.maxBytes === limits.maxBytes && + processed.limits.jpegQuality === limits.jpegQuality + ); +} diff --git a/packages/coding-agent/src/utils/image-resize-core.ts b/packages/coding-agent/src/utils/image-resize-core.ts index 3f80111ff..168616123 100644 --- a/packages/coding-agent/src/utils/image-resize-core.ts +++ b/packages/coding-agent/src/utils/image-resize-core.ts @@ -1,4 +1,5 @@ -import { applyExifOrientation } from "./exif-orientation.ts"; +import { applyExifOrientation, getExifOrientation } from "./exif-orientation.ts"; +import { type ImageHeader, readImageHeader } from "./image-header.ts"; import { loadPhoton } from "./photon.ts"; export interface ImageResizeOptions { @@ -38,42 +39,147 @@ const DEFAULT_OPTIONS: Required = { jpegQuality: 80, }; -interface EncodedCandidate { - data: string; - encodedSize: number; - mimeType: string; +/** The limits with defaults applied. */ +export function resolveImageResizeOptions(options?: ImageResizeOptions): Required { + return { ...DEFAULT_OPTIONS, ...options }; +} + +/** + * Width and height as displayed, read from the file header: EXIF orientations 5-8 rotate the image + * by 90 degrees. Undefined when the header cannot be read; the caller then decodes the image. + */ +export function readDisplayedImageSize(inputBytes: Uint8Array, mimeType: string): ImageHeader | undefined { + // The header is checked first: it also validates the structure the EXIF reader walks. + const header = readImageHeader(inputBytes, mimeType); + if (!header) return undefined; + return getExifOrientation(inputBytes) >= 5 ? { width: header.height, height: header.width } : header; +} + +function base64Size(byteLength: number): number { + return Math.ceil(byteLength / 3) * 4; } -function encodeCandidate(buffer: Uint8Array, mimeType: string): EncodedCandidate { - const data = Buffer.from(buffer).toString("base64"); +/** Whether an image of this displayed size (after EXIF orientation) and byte length needs no change. */ +export function isWithinLimits( + width: number, + height: number, + byteLength: number, + opts: Required, +): boolean { + return width <= opts.maxWidth && height <= opts.maxHeight && base64Size(byteLength) < opts.maxBytes; +} + +/** + * The input as it is when it is already within the dimension and size limits, else undefined. + * `width` and `height` are the displayed size (after EXIF orientation). + */ +export function keepIfWithinLimits( + inputBytes: Uint8Array, + mimeType: string, + width: number, + height: number, + opts: Required, +): ResizedImage | undefined { + if (!isWithinLimits(width, height, inputBytes.byteLength, opts)) return undefined; + const format = mimeType.split("/")[1] ?? "png"; return { - data, - encodedSize: Buffer.byteLength(data, "utf-8"), - mimeType, + data: Buffer.from(inputBytes.buffer, inputBytes.byteOffset, inputBytes.byteLength).toString("base64"), + mimeType: mimeType || `image/${format}`, + originalWidth: width, + originalHeight: height, + width, + height, + wasResized: false, }; } +/** One resized version of the image, encoded on demand. */ +export interface ResizedFrame { + png(): Uint8Array | Promise; + jpeg(quality: number): Uint8Array | Promise; + free(): void; +} + +/** + * Resize to fit `maxWidth`/`maxHeight`, then find the first encoding under `maxBytes`: PNG, then JPEG + * at each quality step, then the same at 75% of the size, down to 1x1. Encodings are produced one at + * a time and the first that fits wins, so later candidates are not encoded at all. + * Returns null when nothing fits. + */ +export async function encodeWithinLimits( + originalWidth: number, + originalHeight: number, + opts: Required, + resizeTo: (width: number, height: number) => ResizedFrame, +): Promise { + let currentWidth = originalWidth; + let currentHeight = originalHeight; + if (currentWidth > opts.maxWidth) { + currentHeight = Math.round((currentHeight * opts.maxWidth) / currentWidth); + currentWidth = opts.maxWidth; + } + if (currentHeight > opts.maxHeight) { + currentWidth = Math.round((currentWidth * opts.maxHeight) / currentHeight); + currentHeight = opts.maxHeight; + } + + const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40])); + const result = (bytes: Uint8Array, mimeType: string): ResizedImage | undefined => { + const data = Buffer.from(bytes).toString("base64"); + if (Buffer.byteLength(data, "utf-8") >= opts.maxBytes) return undefined; + return { + data, + mimeType, + originalWidth, + originalHeight, + width: currentWidth, + height: currentHeight, + wasResized: true, + }; + }; + + while (true) { + const frame = resizeTo(currentWidth, currentHeight); + try { + const png = result(await frame.png(), "image/png"); + if (png) return png; + for (const quality of qualitySteps) { + const jpeg = result(await frame.jpeg(quality), "image/jpeg"); + if (jpeg) return jpeg; + } + } finally { + frame.free(); + } + + if (currentWidth === 1 && currentHeight === 1) { + return null; + } + + const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75)); + const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75)); + if (nextWidth === currentWidth && nextHeight === currentHeight) { + return null; + } + + currentWidth = nextWidth; + currentHeight = nextHeight; + } +} + /** * Resize an image to fit within the specified max dimensions and encoded file size. * Returns null if the image cannot be resized below maxBytes. * - * Uses Photon (Rust/WASM) for image processing. If Photon is not available, - * returns null. - * - * Strategy for staying under maxBytes: - * 1. First resize to maxWidth/maxHeight - * 2. Try both PNG and JPEG formats, pick the smaller one - * 3. If still too large, try JPEG with decreasing quality - * 4. If still too large, progressively reduce dimensions until 1x1 + * Photon (Rust/WASM) decodes the image once; an image within the limits is returned as it is, and + * one over them is resized (see encodeWithinLimits). If Photon is not available or cannot decode + * the image, returns null. resizeImage skips this decode when it can show the image is valid. */ export async function resizeImageInProcess( inputBytes: Uint8Array, mimeType: string, options?: ImageResizeOptions, ): Promise { - const opts = { ...DEFAULT_OPTIONS, ...options }; - const inputBase64Size = Math.ceil(inputBytes.byteLength / 3) * 4; - + const opts = resolveImageResizeOptions(options); const photon = await loadPhoton(); if (!photon) { return null; @@ -84,86 +190,21 @@ export async function resizeImageInProcess( const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes); image = applyExifOrientation(photon, rawImage, inputBytes); if (image !== rawImage) rawImage.free(); + const decoded = image; - const originalWidth = image.get_width(); - const originalHeight = image.get_height(); - const format = mimeType.split("/")[1] ?? "png"; + const originalWidth = decoded.get_width(); + const originalHeight = decoded.get_height(); + const decodedUnchanged = keepIfWithinLimits(inputBytes, mimeType, originalWidth, originalHeight, opts); + if (decodedUnchanged) return decodedUnchanged; - // Check if already within all limits (dimensions AND encoded size) - if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) { + return await encodeWithinLimits(originalWidth, originalHeight, opts, (width, height) => { + const resized = photon.resize(decoded, width, height, photon.SamplingFilter.Lanczos3); return { - data: Buffer.from(inputBytes).toString("base64"), - mimeType: mimeType || `image/${format}`, - originalWidth, - originalHeight, - width: originalWidth, - height: originalHeight, - wasResized: false, + png: () => resized.get_bytes(), + jpeg: (quality) => resized.get_bytes_jpeg(quality), + free: () => resized.free(), }; - } - - // Calculate initial dimensions respecting max limits - let targetWidth = originalWidth; - let targetHeight = originalHeight; - - if (targetWidth > opts.maxWidth) { - targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth); - targetWidth = opts.maxWidth; - } - if (targetHeight > opts.maxHeight) { - targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight); - targetHeight = opts.maxHeight; - } - - function tryEncodings(width: number, height: number, jpegQualities: number[]): EncodedCandidate[] { - const resized = photon!.resize(image!, width, height, photon!.SamplingFilter.Lanczos3); - - try { - const candidates: EncodedCandidate[] = [encodeCandidate(resized.get_bytes(), "image/png")]; - for (const quality of jpegQualities) { - candidates.push(encodeCandidate(resized.get_bytes_jpeg(quality), "image/jpeg")); - } - return candidates; - } finally { - resized.free(); - } - } - - const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40])); - let currentWidth = targetWidth; - let currentHeight = targetHeight; - - while (true) { - const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps); - for (const candidate of candidates) { - if (candidate.encodedSize < opts.maxBytes) { - return { - data: candidate.data, - mimeType: candidate.mimeType, - originalWidth, - originalHeight, - width: currentWidth, - height: currentHeight, - wasResized: true, - }; - } - } - - if (currentWidth === 1 && currentHeight === 1) { - break; - } - - const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75)); - const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75)); - if (nextWidth === currentWidth && nextHeight === currentHeight) { - break; - } - - currentWidth = nextWidth; - currentHeight = nextHeight; - } - - return null; + }); } catch { return null; } finally { diff --git a/packages/coding-agent/src/utils/image-resize-native.ts b/packages/coding-agent/src/utils/image-resize-native.ts new file mode 100644 index 000000000..8d898ac0c --- /dev/null +++ b/packages/coding-agent/src/utils/image-resize-native.ts @@ -0,0 +1,99 @@ +import { getExifOrientation } from "./exif-orientation.ts"; +import { encodeWithinLimits, type ImageResizeOptions, type ResizedImage } from "./image-resize-core.ts"; + +/** + * The part of Bun's `Bun.Image` (Bun 1.4+) used here. Pi does not depend on Bun's type package, + * so the shape is declared locally; it matches `packages/bun-types/bun.d.ts` of the pinned runtime. + */ +interface NativeImagePipeline { + resize(width: number, height: number, options: { filter: "lanczos3" | "box"; fit: "fill" }): NativeImagePipeline; + rotate(degrees: number): NativeImagePipeline; + flip(): NativeImagePipeline; + flop(): NativeImagePipeline; + png(): NativeImagePipeline; + jpeg(options: { quality: number }): NativeImagePipeline; + bytes(): Promise; +} + +export type NativeImageConstructor = new (input: Uint8Array, options: { autoOrient: boolean }) => NativeImagePipeline; + +/** `Bun.Image` when the runtime has it (Bun, including Pi-Bolt's runtime), else undefined (Node). */ +export function getNativeImage(): NativeImageConstructor | undefined { + const image = (globalThis as { Bun?: { Image?: unknown } }).Bun?.Image; + return typeof image === "function" ? (image as NativeImageConstructor) : undefined; +} + +/** + * The EXIF orientation as the transforms Bun.Image applies in its fixed order (rotate, then flip + * or flop), matching applyExifOrientation's photon transforms pixel for pixel. + */ +function orient(pipeline: NativeImagePipeline, orientation: number): NativeImagePipeline { + switch (orientation) { + case 2: + return pipeline.flop(); + case 3: + return pipeline.rotate(180); + case 4: + return pipeline.flip(); + case 5: + return pipeline.rotate(90).flop(); + case 6: + return pipeline.rotate(90); + case 7: + return pipeline.rotate(270).flop(); + case 8: + return pipeline.rotate(270); + default: + return pipeline; + } +} + +/** + * Resize with the runtime's native image code (libspng, libjpeg-turbo and libwebp decoders, SIMD + * Lanczos3 resize, libspng and libjpeg-turbo encoders) instead of photon's WebAssembly, which the + * JIT-off Pi-Bolt runtime interprets: a 2442x1438 screenshot takes about 0.5 s of CPU instead of + * about 15 s. It runs on Bun's own worker threads, so the event loop stays free. + * + * Same output rules as the photon path (same target sizes, candidate order and size limit), but + * not the same bytes: the resampling rounds differently and the encoders differ. + * + * `width` and `height` are the displayed size of the input (after EXIF orientation), from its + * header. Throws when the native decoder or encoder fails; the caller then uses photon. + */ +export function resizeImageNative( + NativeImage: NativeImageConstructor, + inputBytes: Uint8Array, + width: number, + height: number, + opts: Required, +): Promise { + const orientation = getExifOrientation(inputBytes); + return encodeWithinLimits(width, height, opts, (targetWidth, targetHeight) => { + const pipeline = () => + orient(new NativeImage(inputBytes, { autoOrient: false }), orientation).resize(targetWidth, targetHeight, { + filter: "lanczos3", + fit: "fill", + }); + return { + png: () => pipeline().png().bytes(), + jpeg: (quality) => pipeline().jpeg({ quality }).bytes(), + free: () => {}, + }; + }); +} + +/** + * Whether the native decoder decodes the whole image without an error. A full decode to a 1x1 box + * average: about 4-6 ms for a 1600x1200 JPEG or a 1800x1100 PNG, 25-35 ms for WebP and GIF. + */ +export async function decodesNatively(NativeImage: NativeImageConstructor, inputBytes: Uint8Array): Promise { + try { + await new NativeImage(inputBytes, { autoOrient: false }) + .resize(1, 1, { filter: "box", fit: "fill" }) + .png() + .bytes(); + return true; + } catch { + return false; + } +} diff --git a/packages/coding-agent/src/utils/image-resize.ts b/packages/coding-agent/src/utils/image-resize.ts index 9de941ba3..e15d82430 100644 --- a/packages/coding-agent/src/utils/image-resize.ts +++ b/packages/coding-agent/src/utils/image-resize.ts @@ -2,10 +2,20 @@ import type { Worker } from "node:worker_threads"; import { IMAGE_RESIZE_WORKER_RESPONSE_TYPE, type ImageResizeOptions, + isWithinLimits, + keepIfWithinLimits, type ResizedImage, type ResizeImageWorkerResponse, + readDisplayedImageSize, resizeImageInProcess, + resolveImageResizeOptions, } from "./image-resize-core.ts"; +import { + decodesNatively, + getNativeImage, + type NativeImageConstructor, + resizeImageNative, +} from "./image-resize-native.ts"; export type { ImageResizeOptions, ResizedImage } from "./image-resize-core.ts"; @@ -75,18 +85,67 @@ async function resizeImageInWorker( } } +/** + * The input as it is, when its header shows it is within the limits and its data is known to + * decode, else undefined (the caller then decodes it with photon, which also rejects damaged data). + * + * - PNG and WebP, with a native image pipeline: the native decoder must decode it. + * - Not JPEG or GIF: the native decoders accept damaged data photon rejects (libjpeg-turbo decodes + * corrupt or truncated scan data with only a warning), and such an image must still be omitted. + * - Not without a native pipeline (Node, where photon runs with a JIT): photon decodes it as before. + */ +export async function keepValidImageWithinLimits( + inputBytes: Uint8Array, + mimeType: string, + width: number, + height: number, + opts: Required, + NativeImage: NativeImageConstructor | undefined, +): Promise { + if (!isWithinLimits(width, height, inputBytes.byteLength, opts)) return undefined; + if (!NativeImage || (mimeType !== "image/png" && mimeType !== "image/webp")) return undefined; + if (!(await decodesNatively(NativeImage, inputBytes))) return undefined; + return keepIfWithinLimits(inputBytes, mimeType, width, height, opts); +} + /** * Resize an image to fit within the specified max dimensions and encoded file size. - * Runs Photon in a worker thread so WASM decoding, resizing, and encoding do not - * block the TUI event loop. If the worker cannot be loaded (for example in some - * Bun compiled executable layouts), fall back to in-process resizing so image - * reads still work. + * + * - An image within the limits whose data is known to decode is returned as it is, without photon + * (see keepValidImageWithinLimits). + * - An image over the limits, when the runtime has a native image pipeline (`Bun.Image`), is + * resized there. + * - Otherwise, or when the header cannot be read or the native pipeline fails, Photon runs in a + * worker thread so WASM decoding, resizing, and encoding do not block the TUI event loop. If the + * worker cannot be loaded (for example in some Bun compiled executable layouts), fall back to + * in-process resizing so image reads still work. */ export async function resizeImage( inputBytes: Uint8Array, mimeType: string, options?: ImageResizeOptions, ): Promise { + const opts = resolveImageResizeOptions(options); + const size = readDisplayedImageSize(inputBytes, mimeType); + if (size) { + const NativeImage = getNativeImage(); + if (isWithinLimits(size.width, size.height, inputBytes.byteLength, opts)) { + const unchanged = await keepValidImageWithinLimits( + inputBytes, + mimeType, + size.width, + size.height, + opts, + NativeImage, + ); + if (unchanged) return unchanged; + } else if (NativeImage) { + try { + return await resizeImageNative(NativeImage, inputBytes, size.width, size.height, opts); + } catch {} + } + } + const isTypeScriptRuntime = import.meta.url.endsWith(".ts"); const workerUrl = new URL( isTypeScriptRuntime ? "./image-resize-worker.ts" : "./image-resize-worker.js", diff --git a/packages/coding-agent/src/utils/tool-result-images.ts b/packages/coding-agent/src/utils/tool-result-images.ts index 702d98772..e035d5e99 100644 --- a/packages/coding-agent/src/utils/tool-result-images.ts +++ b/packages/coding-agent/src/utils/tool-result-images.ts @@ -1,5 +1,5 @@ import type { ImageContent, ModelImageResizeOptions, TextContent } from "@earendil-works/pi-ai"; -import { processImage } from "./image-process.ts"; +import { isProcessedImageBlock, processImage } from "./image-process.ts"; export type ToolResultContent = TextContent | ImageContent; @@ -39,10 +39,14 @@ export async function normalizeToolResultImages( continue; } - const processed = await processImage(Buffer.from(block.data, "base64"), block.mimeType, { - autoResizeImages, - resizeOptions: options?.resizeOptions, - }); + const processOptions = { autoResizeImages, resizeOptions: options?.resizeOptions }; + // The read tool's images were processed with these options already; a second pass returns them unchanged. + if (isProcessedImageBlock(block, processOptions)) { + normalized.push(block); + continue; + } + + const processed = await processImage(Buffer.from(block.data, "base64"), block.mimeType, processOptions); if (!processed.ok) { // Unlike `read`, keep the original block. The tool already produced this image and the // failure may just be an unavailable image backend, so passing it through preserves the diff --git a/packages/coding-agent/test/image-header.test.ts b/packages/coding-agent/test/image-header.test.ts new file mode 100644 index 000000000..2932fd402 --- /dev/null +++ b/packages/coding-agent/test/image-header.test.ts @@ -0,0 +1,621 @@ +import { crc32, deflateSync } from "node:zlib"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type * as PhotonModule from "../src/utils/photon.ts"; + +const photonLoads = vi.hoisted(() => ({ count: 0 })); +vi.mock("../src/utils/photon.ts", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + loadPhoton: () => { + photonLoads.count++; + return actual.loadPhoton(); + }, + }; +}); + +import { + readGifHeader, + readImageHeader, + readJpegHeader, + readPngHeader, + readWebpHeader, +} from "../src/utils/image-header.ts"; +import { keepValidImageWithinLimits, resizeImage } from "../src/utils/image-resize.ts"; +import { + readDisplayedImageSize, + resizeImageInProcess, + resolveImageResizeOptions, +} from "../src/utils/image-resize-core.ts"; +import type { NativeImageConstructor } from "../src/utils/image-resize-native.ts"; +import { loadPhoton } from "../src/utils/photon.ts"; + +// Small 2x2 blue JPEG image (baseline) - generated with ImageMagick +const JPEG_2X2 = + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAACAAIDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAVAQEBAAAAAAAAAAAAAAAAAAAGCf/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AD3VTB3/2Q=="; + +const JPEG_PROGRESSIVE_5X3 = + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wgARCAADAAUDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAL/xAAVAQEBAAAAAAAAAAAAAAAAAAAFBv/aAAwDAQACEAMQAAABsHUX/8QAFhABAQEAAAAAAAAAAAAAAAAAAgER/9oACAEBAAEFAk7n/8QAGhEAAgIDAAAAAAAAAAAAAAAAAQMABBMikf/aAAgBAwEBPwFFVGMaDk//xAAYEQEAAwEAAAAAAAAAAAAAAAABAANxsf/aAAgBAgEBPwGxRMOT/8QAFhAAAwAAAAAAAAAAAAAAAAAAAAEh/9oACAEBAAY/AlT/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/IRH/2gAMAwEAAgADAAAAEA//xAAWEQADAAAAAAAAAAAAAAAAAAAAwdH/2gAIAQMBAT8QWdD/xAAWEQEBAQAAAAAAAAAAAAAAAAABAPH/2gAIAQIBAT8QABwX/8QAGBABAQADAAAAAAAAAAAAAAAAAREAQWH/2gAIAQEAAT8QK9Vujmf/2Q=="; + +const WEBP_LOSSY_5X3 = + "UklGRmwAAABXRUJQVlA4IGAAAADwAQCdASoFAAMAAUAmJbACdLoAArl9WmgA/fzw9op6bHEwjhOVx/wF//v//yHC9hWy7jAdxkLZm35O/zZfR5/8m3wfmv+c4Nh/5N//UMf+w/6cWv9zf/10HG9vn+MAAAA="; + +const WEBP_LOSSLESS_5X3 = "UklGRjoAAABXRUJQVlA4TC0AAAAvBIAAAHcgFkz8eQfQmcJETNP8Z7SBgkaSmu+FrgVPaMETWhAW0f9YW87N9wcA"; + +// VP8X canvas with an ALPH chunk and a VP8 bitstream. +const WEBP_ALPHA_5X3 = + "UklGRpQAAABXRUJQVlA4WAoAAAAQAAAABAAAAgAAQUxQSA0AAAABD/DA/4iIMAER/Y8AAFZQOCBgAAAA8AEAnQEqBQADAAFAJiWwAnS6AAK5fVpoAP388PaKemxxMI4Tlcf318/v/97ZLZm3/CpcMWRAgV+Tv82X0ef/Jt8H5r/nODYf+Tf/1DH/sP+nFr/c3/9h10Fke/mQAAAA"; + +function bytes(base64: string): Buffer { + return Buffer.from(base64, "base64"); +} + +function pngChunk(type: string, body: Buffer): Buffer { + const header = Buffer.alloc(8); + header.writeUInt32BE(body.length, 0); + header.write(type, 4, "ascii"); + const checksum = Buffer.alloc(4); + checksum.writeUInt32BE(crc32(Buffer.concat([header.subarray(4), body])), 0); + return Buffer.concat([header, body, checksum]); +} + +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +function ihdr(width: number, height: number, bitDepth = 8, colorType = 0): Buffer { + const body = Buffer.alloc(13); + body.writeUInt32BE(width, 0); + body.writeUInt32BE(height, 4); + body[8] = bitDepth; + body[9] = colorType; + return pngChunk("IHDR", body); +} + +/** An 8-bit grayscale PNG. */ +function createPng(width: number, height: number): Buffer { + const raw = Buffer.alloc((width + 1) * height); + for (let row = 0; row < height; row++) { + raw.fill((row * 7) % 256, row * (width + 1) + 1, (row + 1) * (width + 1)); + } + return Buffer.concat([ + PNG_SIGNATURE, + ihdr(width, height), + pngChunk("tEXt", Buffer.from("Comment\0test")), + pngChunk("IDAT", deflateSync(raw)), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +/** + * LZW image data for `pixels` pixels of colour 0: minimum code size 2, so codes are 3 bits wide + * (clear = 4, end = 5). A clear code after every two pixels keeps the code width at 3 bits. + */ +function gifImageData(pixels: number): Buffer { + const codes: number[] = []; + for (let i = 0; i < pixels; i++) { + if (i % 2 === 0) codes.push(4); + codes.push(0); + } + codes.push(5); + const packed: number[] = []; + let bits = 0; + let bitCount = 0; + for (const code of codes) { + bits |= code << bitCount; + bitCount += 3; + while (bitCount >= 8) { + packed.push(bits & 0xff); + bits >>= 8; + bitCount -= 8; + } + } + if (bitCount > 0) packed.push(bits & 0xff); + const blocks: number[] = [0x02]; + for (let i = 0; i < packed.length; i += 255) { + const block = packed.slice(i, i + 255); + blocks.push(block.length, ...block); + } + blocks.push(0); + return Buffer.from(blocks); +} + +/** A GIF with a 2-colour global table and the given frames ([left, top, width, height]). */ +function createGif(width: number, height: number, frames: [number, number, number, number][]): Buffer { + const parts: Buffer[] = [Buffer.from("GIF89a", "ascii")]; + const screen = Buffer.alloc(7); + screen.writeUInt16LE(width, 0); + screen.writeUInt16LE(height, 2); + screen[4] = 0x80; // global colour table, 2 entries + parts.push(screen, Buffer.from([0, 0, 0, 255, 255, 255])); + // Graphic control extension. + parts.push(Buffer.from([0x21, 0xf9, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00])); + for (const [left, top, frameWidth, frameHeight] of frames) { + const descriptor = Buffer.alloc(10); + descriptor[0] = 0x2c; + descriptor.writeUInt16LE(left, 1); + descriptor.writeUInt16LE(top, 3); + descriptor.writeUInt16LE(frameWidth, 5); + descriptor.writeUInt16LE(frameHeight, 7); + parts.push(descriptor, gifImageData(frameWidth * frameHeight)); + } + parts.push(Buffer.from([0x3b])); + return Buffer.concat(parts); +} + +function riff(chunks: Buffer[]): Buffer { + const body = Buffer.concat([Buffer.from("WEBP", "ascii"), ...chunks]); + const header = Buffer.alloc(8); + header.write("RIFF", 0, "ascii"); + header.writeUInt32LE(body.length, 4); + return Buffer.concat([header, body]); +} + +function webpChunk(type: string, body: Buffer): Buffer { + const header = Buffer.alloc(8); + header.write(type, 0, "ascii"); + header.writeUInt32LE(body.length, 4); + return Buffer.concat([header, body, body.length % 2 ? Buffer.alloc(1) : Buffer.alloc(0)]); +} + +function vp8x(width: number, height: number, flags = 0): Buffer { + const body = Buffer.alloc(10); + body[0] = flags; + body.writeUIntLE(width - 1, 4, 3); + body.writeUIntLE(height - 1, 7, 3); + return webpChunk("VP8X", body); +} + +/** The chunks of a simple WebP file (everything after "RIFF" size "WEBP"). */ +function webpChunks(base64: string): Buffer { + return bytes(base64).subarray(12); +} + +function app1Exif(orientation: number): Buffer { + const tiff = Buffer.from("49492a00080000000100120103000100000000000000000000000000", "hex"); + tiff.writeUInt16LE(orientation, 18); + const payload = Buffer.concat([Buffer.from("Exif\0\0", "latin1"), tiff]); + const segment = Buffer.alloc(payload.length + 4); + segment[0] = 0xff; + segment[1] = 0xe1; + segment.writeUInt16BE(payload.length + 2, 2); + payload.copy(segment, 4); + return segment; +} + +/** Replace the JPEG segment with `marker` by `replace(segment)`. */ +function editJpegSegment(jpeg: Buffer, marker: number, replace: (segment: Buffer) => Buffer): Buffer { + let offset = 2; + while (offset < jpeg.length) { + const length = jpeg.readUInt16BE(offset + 2); + if (jpeg[offset + 1] === marker) { + const segment = jpeg.subarray(offset, offset + 2 + length); + return Buffer.concat([ + jpeg.subarray(0, offset), + replace(Buffer.from(segment)), + jpeg.subarray(offset + 2 + length), + ]); + } + offset += 2 + length; + } + throw new Error(`no marker ${marker.toString(16)}`); +} + +describe("image header parsers", () => { + const valid: [string, string, Buffer, number, number][] = [ + ["PNG", "image/png", createPng(37, 21), 37, 21], + ["baseline JPEG", "image/jpeg", bytes(JPEG_2X2), 2, 2], + ["progressive JPEG", "image/jpeg", bytes(JPEG_PROGRESSIVE_5X3), 5, 3], + ["GIF", "image/gif", createGif(6, 4, [[0, 0, 6, 4]]), 6, 4], + [ + "animated GIF with a partial frame", + "image/gif", + createGif(6, 4, [ + [0, 0, 6, 4], + [1, 1, 2, 2], + ]), + 6, + 4, + ], + ["lossy WebP", "image/webp", bytes(WEBP_LOSSY_5X3), 5, 3], + ["lossless WebP", "image/webp", bytes(WEBP_LOSSLESS_5X3), 5, 3], + ["extended WebP with alpha", "image/webp", bytes(WEBP_ALPHA_5X3), 5, 3], + ]; + + it.each(valid)( + "reads the size of a %s, and photon decodes the same size", + async (_name, mimeType, data, width, height) => { + expect(readImageHeader(data, mimeType)).toEqual({ width, height }); + const photon = (await loadPhoton())!; + const image = photon.PhotonImage.new_from_byteslice(data); + expect([image.get_width(), image.get_height()]).toEqual([width, height]); + image.free(); + }, + ); + + it("returns undefined for a MIME type it does not parse", () => { + expect(readImageHeader(createPng(2, 2), "image/bmp")).toBeUndefined(); + expect(readImageHeader(createPng(2, 2), "image/jpeg")).toBeUndefined(); + }); + + describe("rejects malformed PNG files", () => { + const png = createPng(4, 3); + const cases: [string, Buffer][] = [ + ["an empty file", Buffer.alloc(0)], + ["a truncated file", png.subarray(0, png.length - 5)], + ["trailing bytes after IEND", Buffer.concat([png, Buffer.from([0])])], + ["a bad signature", Buffer.concat([Buffer.from([0x88]), png.subarray(1)])], + [ + "a bad chunk CRC", + (() => { + const copy = Buffer.from(png); + copy[copy.length - 13] ^= 0xff; // last byte of the IDAT CRC + return copy; + })(), + ], + [ + "a flipped pixel-data byte", + (() => { + const copy = Buffer.from(png); + copy[png.indexOf("IDAT") + 6] ^= 0x01; + return copy; + })(), + ], + ["IHDR not first", Buffer.concat([PNG_SIGNATURE, pngChunk("tEXt", Buffer.from("a\0b")), png.subarray(8)])], + [ + "no IDAT", + Buffer.concat([ + PNG_SIGNATURE, + ihdr(4, 3), + pngChunk("IEND", Buffer.alloc(0)), + pngChunk("IEND", Buffer.alloc(0)), + ]), + ], + ["zero width", Buffer.concat([PNG_SIGNATURE, ihdr(0, 3), png.subarray(8 + 25)])], + ["an invalid bit depth", Buffer.concat([PNG_SIGNATURE, ihdr(4, 3, 3, 0), png.subarray(8 + 25)])], + ["a palette image without PLTE", Buffer.concat([PNG_SIGNATURE, ihdr(4, 3, 8, 3), png.subarray(8 + 25)])], + [ + "a chunk length past the end", + Buffer.concat([png.subarray(0, 8), Buffer.from([0x7f, 0xff, 0xff, 0xff]), png.subarray(12)]), + ], + ]; + it.each(cases)("%s", (_name, data) => { + expect(readPngHeader(data)).toBeUndefined(); + }); + }); + + describe("rejects malformed or uncommon JPEG files", () => { + const jpeg = bytes(JPEG_2X2); + const cases: [string, Buffer][] = [ + ["a missing EOI marker", jpeg.subarray(0, jpeg.length - 2)], + ["a truncated file", Buffer.concat([jpeg.subarray(0, 100), Buffer.from([0xff, 0xd9])])], + ["no frame header before the scan", editJpegSegment(jpeg, 0xc0, () => Buffer.alloc(0))], + [ + "a lossless frame (SOF3)", + editJpegSegment(jpeg, 0xc0, (s) => Buffer.concat([Buffer.from([0xff, 0xc3]), s.subarray(2)])), + ], + [ + "an arithmetic-coded frame (SOF9)", + editJpegSegment(jpeg, 0xc0, (s) => Buffer.concat([Buffer.from([0xff, 0xc9]), s.subarray(2)])), + ], + [ + "12-bit samples", + editJpegSegment(jpeg, 0xc0, (s) => { + s[4] = 12; + return s; + }), + ], + [ + "zero height", + editJpegSegment(jpeg, 0xc0, (s) => { + s.writeUInt16BE(0, 5); + return s; + }), + ], + ["two frame headers", editJpegSegment(jpeg, 0xc0, (s) => Buffer.concat([s, s]))], + [ + "a segment length past the end", + Buffer.concat([jpeg.subarray(0, 4), Buffer.from([0xff, 0xff]), Buffer.from([0xff, 0xd9])]), + ], + [ + "a restart marker before the scan", + Buffer.concat([jpeg.subarray(0, 2), Buffer.from([0xff, 0xd0]), jpeg.subarray(2)]), + ], + ["not a JPEG", Buffer.from("not a jpeg at all\xff\xd9", "latin1")], + ]; + it.each(cases)("%s", (_name, data) => { + expect(readJpegHeader(data)).toBeUndefined(); + }); + + it("CMYK (4 components)", () => { + const cmyk = editJpegSegment(jpeg, 0xc0, (s) => { + const body = Buffer.alloc(8 + 4 * 3); + s.copy(body, 0, 2, 9); + body.writeUInt16BE(body.length, 0); + body[7] = 4; + return Buffer.concat([Buffer.from([0xff, 0xc0]), body]); + }); + expect(readJpegHeader(cmyk)).toBeUndefined(); + }); + }); + + describe("rejects malformed GIF files", () => { + const gif = createGif(6, 4, [[0, 0, 6, 4]]); + const cases: [string, Buffer][] = [ + ["a missing trailer", gif.subarray(0, gif.length - 1)], + ["trailing bytes after the trailer", Buffer.concat([gif, Buffer.from([0])])], + ["no frames", createGif(6, 4, [])], + ["a frame outside the screen", createGif(6, 4, [[4, 0, 6, 4]])], + ["a zero-size screen", createGif(0, 4, [[0, 0, 1, 1]])], + ["an unknown block", Buffer.concat([gif.subarray(0, 19), Buffer.from([0x99]), gif.subarray(19)])], + ["a bad signature", Buffer.concat([Buffer.from("GIF88a", "ascii"), gif.subarray(6)])], + ]; + it.each(cases)("%s", (_name, data) => { + expect(readGifHeader(data)).toBeUndefined(); + }); + }); + + describe("WebP files", () => { + const lossless = webpChunks(WEBP_LOSSLESS_5X3); + + it("reads an extended still file whose bitstream fills the canvas", () => { + expect(readWebpHeader(riff([vp8x(5, 3), lossless]))).toEqual({ width: 5, height: 3 }); + }); + + const cases: [string, Buffer][] = [ + ["a RIFF size that does not match the file", Buffer.concat([bytes(WEBP_LOSSLESS_5X3), Buffer.alloc(2)])], + ["a truncated file", bytes(WEBP_LOSSY_5X3).subarray(0, 40)], + ["an extended canvas larger than its bitstream", riff([vp8x(6, 3), lossless])], + ["an extended file without image data", riff([vp8x(5, 3)])], + [ + "an animated file", + riff([vp8x(9, 7, 0x02), webpChunk("ANIM", Buffer.alloc(6)), webpChunk("ANMF", Buffer.alloc(16))]), + ], + ["two bitstreams in a simple file", riff([lossless, lossless])], + ["an unknown first chunk", riff([webpChunk("XXXX", Buffer.alloc(4)), lossless])], + [ + "a lossless bitstream with a bad signature", + (() => { + const copy = Buffer.from(bytes(WEBP_LOSSLESS_5X3)); + copy[20] = 0x2e; + return copy; + })(), + ], + [ + "a lossy bitstream with a bad start code", + (() => { + const copy = Buffer.from(bytes(WEBP_LOSSY_5X3)); + copy[23] = 0; + return copy; + })(), + ], + ]; + it.each(cases)("rejects %s", (_name, data) => { + expect(readWebpHeader(data)).toBeUndefined(); + }); + }); +}); + +/** Copy of `data` with byte `offset` set to `value`. */ +function withByte(data: Buffer, offset: number, value: number): Buffer { + const copy = Buffer.from(data); + copy[offset] = value; + return copy; +} + +// Damaged image data with an intact structure: the header parsers accept these, photon rejects them. +const DAMAGED_JPEG = withByte(bytes(JPEG_PROGRESSIVE_5X3), 555, 0xff); +const DAMAGED_WEBP_LOSSLESS = withByte(bytes(WEBP_LOSSLESS_5X3), 59, 0xff); +const DAMAGED_WEBP_LOSSY = withByte(bytes(WEBP_LOSSY_5X3), 31, 0x55); +/** A GIF whose first LZW code after the clear code (4) is 7, a code not defined yet. */ +function damagedGif(): Buffer { + const gif = createGif(6, 4, [[0, 0, 6, 4]]); + // Image data starts after the header (13), colour table (6), GCE (8) and descriptor (10): the + // minimum code size, the sub-block size, then the codes packed 3 bits each, least significant first. + const codes = 13 + 6 + 8 + 10 + 2; + gif[codes] = 4 | (7 << 3); + return gif; +} + +/** A PNG whose chunk CRCs are valid but whose compressed pixel data is cut short. */ +function damagedPng(): Buffer { + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(30, 0); + ihdr.writeUInt32BE(20, 4); + ihdr[8] = 8; + const raw = deflateSync(Buffer.alloc(31 * 20, 7)); + return Buffer.concat([ + PNG_SIGNATURE, + pngChunk("IHDR", ihdr), + pngChunk("IDAT", raw.subarray(0, 4)), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +/** A stand-in for Bun.Image whose decode succeeds or fails, recording what it was given. */ +function fakeNativeImage(decodes: boolean, log: Uint8Array[]): NativeImageConstructor { + return class { + constructor(input: Uint8Array) { + log.push(input); + } + resize() { + return this; + } + rotate() { + return this; + } + flip() { + return this; + } + flop() { + return this; + } + png() { + return this; + } + jpeg() { + return this; + } + async bytes(): Promise { + if (!decodes) throw new Error("ERR_IMAGE_DECODE_FAILED"); + return new Uint8Array(1); + } + }; +} + +describe("images within the limits", () => { + const opts = resolveImageResizeOptions(); + + beforeEach(() => { + photonLoads.count = 0; + }); + afterEach(() => { + photonLoads.count = 0; + }); + + it("are returned as they are when the native decoder decodes them (PNG, WebP)", async () => { + const log: Uint8Array[] = []; + const webp = bytes(WEBP_LOSSY_5X3); + const result = await keepValidImageWithinLimits(webp, "image/webp", 5, 3, opts, fakeNativeImage(true, log)); + expect(result).toEqual({ + data: WEBP_LOSSY_5X3, + mimeType: "image/webp", + originalWidth: 5, + originalHeight: 3, + width: 5, + height: 3, + wasResized: false, + }); + expect(log).toEqual([webp]); + const png = createPng(300, 200); + expect( + await keepValidImageWithinLimits(png, "image/png", 300, 200, opts, fakeNativeImage(true, [])), + ).toMatchObject({ + data: png.toString("base64"), + }); + expect(photonLoads.count).toBe(0); + }); + + it("take the photon path when the native decoder fails", async () => { + expect( + await keepValidImageWithinLimits(DAMAGED_WEBP_LOSSY, "image/webp", 5, 3, opts, fakeNativeImage(false, [])), + ).toBeUndefined(); + expect( + await keepValidImageWithinLimits(damagedPng(), "image/png", 30, 20, opts, fakeNativeImage(false, [])), + ).toBeUndefined(); + }); + + it.each([ + ["JPEG", "image/jpeg", bytes(JPEG_2X2), 2, 2], + ["GIF", "image/gif", createGif(6, 4, [[0, 0, 6, 4]]), 6, 4], + ])( + "take the photon path for %s even when the native decoder accepts it", + async (_name, mimeType, data, width, height) => { + const log: Uint8Array[] = []; + expect( + await keepValidImageWithinLimits(data, mimeType, width, height, opts, fakeNativeImage(true, log)), + ).toBeUndefined(); + expect(log).toEqual([]); + }, + ); + + it("take the photon path without a native decoder", async () => { + expect( + await keepValidImageWithinLimits(createPng(300, 200), "image/png", 300, 200, opts, undefined), + ).toBeUndefined(); + expect(await keepValidImageWithinLimits(bytes(JPEG_2X2), "image/jpeg", 2, 2, opts, undefined)).toBeUndefined(); + expect( + await keepValidImageWithinLimits(bytes(WEBP_LOSSY_5X3), "image/webp", 5, 3, opts, undefined), + ).toBeUndefined(); + }); + + it("are not kept when over the limits", async () => { + const png = createPng(300, 200); + const small = resolveImageResizeOptions({ maxWidth: 299 }); + expect( + await keepValidImageWithinLimits(png, "image/png", 300, 200, small, fakeNativeImage(true, [])), + ).toBeUndefined(); + }); + + it("use the displayed size of an EXIF-rotated JPEG", () => { + // 2x1 stored, orientation 6 (rotate 90 degrees): displayed as 1x2. + const jpeg2x1 = editJpegSegment(bytes(JPEG_2X2), 0xc0, (s) => { + s.writeUInt16BE(1, 5); + return s; + }); + const rotated = Buffer.concat([jpeg2x1.subarray(0, 2), app1Exif(6), jpeg2x1.subarray(2)]); + expect(readJpegHeader(rotated)).toEqual({ width: 2, height: 1 }); + expect(readDisplayedImageSize(rotated, "image/jpeg")).toEqual({ width: 1, height: 2 }); + }); +}); + +describe("resizeImage with damaged image data (as in 0.9.5: photon rejects it, the image is omitted)", () => { + const damaged: [string, string, Buffer][] = [ + ["JPEG", "image/jpeg", DAMAGED_JPEG], + ["lossless WebP", "image/webp", DAMAGED_WEBP_LOSSLESS], + ["lossy WebP", "image/webp", DAMAGED_WEBP_LOSSY], + ["GIF", "image/gif", damagedGif()], + ["PNG with valid CRCs", "image/png", damagedPng()], + ]; + + it.each(damaged)( + "%s: the header is accepted, photon rejects the data, the result is null", + async (_name, mimeType, data) => { + expect(readImageHeader(data, mimeType)).toBeDefined(); + const photon = (await loadPhoton())!; + expect(() => photon.PhotonImage.new_from_byteslice(data)).toThrow(); + expect(await resizeImage(data, mimeType)).toBeNull(); + }, + ); + + it("PNG with a wrong checksum: the header is rejected, the result is null", async () => { + const png = createPng(30, 20); + png[png.length - 13] ^= 0xff; + expect(readPngHeader(png)).toBeUndefined(); + expect(await resizeImage(png, "image/png")).toBeNull(); + }); + + it.each([ + ["PNG", "image/png", createPng(30, 20).toString("base64")], + ["JPEG", "image/jpeg", JPEG_PROGRESSIVE_5X3], + ["WebP", "image/webp", WEBP_LOSSY_5X3], + ])("an intact %s is returned as it is", async (_name, mimeType, base64) => { + expect(await resizeImage(bytes(base64), mimeType)).toMatchObject({ data: base64, wasResized: false }); + }); +}); + +describe("resizeImageInProcess", () => { + beforeEach(() => { + photonLoads.count = 0; + }); + + it("decodes when the header cannot be read", async () => { + // Trailing bytes after IEND: the header parser gives up, photon still decodes the file. + const png = Buffer.concat([createPng(30, 20), Buffer.from("trailer")]); + expect(readPngHeader(png)).toBeUndefined(); + + const result = await resizeImageInProcess(png, "image/png"); + expect(result).toMatchObject({ data: png.toString("base64"), width: 30, height: 20, wasResized: false }); + expect(photonLoads.count).toBe(1); + }); + + it("omits the image when a PNG checksum is wrong", async () => { + const png = createPng(30, 20); + png[png.length - 13] ^= 0xff; + + expect(readPngHeader(png)).toBeUndefined(); + expect(await resizeImageInProcess(png, "image/png")).toBeNull(); + }); + + it("decodes an image over the limits and encodes only until a candidate fits", async () => { + const png = createPng(400, 100); + const result = await resizeImageInProcess(png, "image/png", { maxWidth: 200, maxHeight: 200 }); + expect(result).toMatchObject({ + mimeType: "image/png", + originalWidth: 400, + originalHeight: 100, + width: 200, + height: 50, + wasResized: true, + }); + expect(photonLoads.count).toBe(1); + }); +}); diff --git a/packages/coding-agent/test/image-resize-encode.test.ts b/packages/coding-agent/test/image-resize-encode.test.ts new file mode 100644 index 000000000..bd66ecb1b --- /dev/null +++ b/packages/coding-agent/test/image-resize-encode.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from "vitest"; +import { encodeWithinLimits, type ResizedFrame, resolveImageResizeOptions } from "../src/utils/image-resize-core.ts"; +import { resizeImageNative } from "../src/utils/image-resize-native.ts"; + +/** A frame whose encodings have the given byte sizes, recording what was encoded. */ +function fakeFrames(sizes: { png: number; jpeg: Record }, log: string[]) { + return (width: number, height: number): ResizedFrame => { + log.push(`resize ${width}x${height}`); + return { + png: () => { + log.push("png"); + return new Uint8Array(sizes.png); + }, + jpeg: (quality) => { + log.push(`jpeg ${quality}`); + return new Uint8Array(sizes.jpeg[quality] ?? 1_000_000); + }, + free: () => log.push("free"), + }; + }; +} + +describe("encodeWithinLimits", () => { + // maxBytes counts base64: n bytes encode to ceil(n / 3) * 4 characters. + const opts = resolveImageResizeOptions({ maxWidth: 100, maxHeight: 100, maxBytes: 4000 }); + + it("encodes no JPEG when the PNG fits", async () => { + const log: string[] = []; + const result = await encodeWithinLimits(400, 200, opts, fakeFrames({ png: 2000, jpeg: {} }, log)); + expect(log).toEqual(["resize 100x50", "png", "free"]); + expect(result).toMatchObject({ mimeType: "image/png", width: 100, height: 50, wasResized: true }); + }); + + it("tries JPEG qualities in order and stops at the first that fits", async () => { + const log: string[] = []; + const result = await encodeWithinLimits( + 400, + 200, + opts, + fakeFrames({ png: 5000, jpeg: { 80: 3500, 85: 3600, 70: 2900 } }, log), + ); + expect(log).toEqual(["resize 100x50", "png", "jpeg 80", "jpeg 85", "jpeg 70", "free"]); + expect(result).toMatchObject({ mimeType: "image/jpeg", width: 100, height: 50 }); + }); + + it("shrinks by 75% when no encoding fits, and gives up at 1x1", async () => { + const log: string[] = []; + expect(await encodeWithinLimits(2, 2, opts, fakeFrames({ png: 5000, jpeg: {} }, log))).toBeNull(); + expect(log.filter((entry) => entry.startsWith("resize"))).toEqual(["resize 2x2", "resize 1x1"]); + }); +}); + +describe("resizeImageNative", () => { + /** A stand-in for Bun.Image that records the pipeline and encodes to `size` bytes. */ + function fakeNativeImage(log: string[], size: (format: string) => number) { + return class { + constructor(_input: Uint8Array, options: { autoOrient: boolean }) { + log.push(`new autoOrient=${options.autoOrient}`); + } + resize(width: number, height: number, options: { filter: string; fit: string }) { + log.push(`resize ${width}x${height} ${options.filter} ${options.fit}`); + return this; + } + rotate(degrees: number) { + log.push(`rotate ${degrees}`); + return this; + } + flip() { + log.push("flip"); + return this; + } + flop() { + log.push("flop"); + return this; + } + private format = ""; + png() { + this.format = "png"; + return this; + } + jpeg(options: { quality: number }) { + this.format = `jpeg ${options.quality}`; + return this; + } + async bytes(): Promise { + log.push(`encode ${this.format}`); + return new Uint8Array(size(this.format)); + } + }; + } + + it("applies the EXIF orientation itself, resizes with Lanczos3 and stops at the first fit", async () => { + // A JPEG start with an APP1 Exif segment, orientation 6 (rotate 90 degrees clockwise). + const exif = Buffer.from("457869660000" + "49492a00080000000100120103000100000006000000000000000000", "hex"); + const app1 = Buffer.concat([Buffer.from([0xff, 0xe1, 0, exif.length + 2]), exif]); + const input = Buffer.concat([Buffer.from([0xff, 0xd8]), app1, Buffer.from([0xff, 0xd9])]); + const log: string[] = []; + const NativeImage = fakeNativeImage(log, (format) => (format === "png" ? 9000 : 1000)); + + const result = await resizeImageNative( + NativeImage, + input, + 300, + 600, + resolveImageResizeOptions({ maxWidth: 200, maxHeight: 200, maxBytes: 4000 }), + ); + + expect(log).toEqual([ + "new autoOrient=false", + "rotate 90", + "resize 100x200 lanczos3 fill", + "encode png", + "new autoOrient=false", + "rotate 90", + "resize 100x200 lanczos3 fill", + "encode jpeg 80", + ]); + expect(result).toMatchObject({ + mimeType: "image/jpeg", + originalWidth: 300, + originalHeight: 600, + width: 100, + height: 200, + wasResized: true, + }); + }); + + it("passes native failures to the caller", async () => { + const log: string[] = []; + const Failing = class extends fakeNativeImage(log, () => 0) { + override async bytes(): Promise { + throw new Error("ERR_IMAGE_DECODE_FAILED"); + } + }; + await expect( + resizeImageNative(Failing, new Uint8Array([0xff, 0xd8, 0xff, 0xd9]), 300, 300, resolveImageResizeOptions()), + ).rejects.toThrow("ERR_IMAGE_DECODE_FAILED"); + }); +}); diff --git a/packages/coding-agent/test/suite/agent-session-read-images.test.ts b/packages/coding-agent/test/suite/agent-session-read-images.test.ts new file mode 100644 index 000000000..0e6dc96c0 --- /dev/null +++ b/packages/coding-agent/test/suite/agent-session-read-images.test.ts @@ -0,0 +1,101 @@ +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { crc32, deflateSync } from "node:zlib"; +import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type * as ImageProcessModule from "../../src/utils/image-process.ts"; +import { createHarness, getToolResult, type Harness } from "./harness.ts"; + +const processImageCalls = vi.hoisted(() => ({ count: 0 })); +vi.mock("../../src/utils/image-process.ts", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + processImage: (...args: Parameters) => { + processImageCalls.count++; + return actual.processImage(...args); + }, + }; +}); + +function pngChunk(type: string, body: Buffer): Buffer { + const header = Buffer.alloc(8); + header.writeUInt32BE(body.length, 0); + header.write(type, 4, "ascii"); + const checksum = Buffer.alloc(4); + checksum.writeUInt32BE(crc32(Buffer.concat([header.subarray(4), body])), 0); + return Buffer.concat([header, body, checksum]); +} + +/** An 8-bit grayscale PNG with horizontal stripes (small enough to stay PNG after resizing). */ +function createPng(width: number, height: number): Buffer { + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(width, 0); + ihdr.writeUInt32BE(height, 4); + ihdr[8] = 8; + const raw = Buffer.alloc((width + 1) * height); + for (let row = 0; row < height; row++) { + raw.fill(row % 256, row * (width + 1) + 1, (row + 1) * (width + 1)); + } + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", ihdr), + pngChunk("IDAT", deflateSync(raw)), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +describe("AgentSession read of images", () => { + const harnesses: Harness[] = []; + + afterEach(() => { + processImageCalls.count = 0; + while (harnesses.length > 0) { + harnesses.pop()?.cleanup(); + } + }); + + async function readImage(name: string, data: Buffer): Promise<(TextContent | ImageContent)[]> { + const harness = await createHarness(); + harnesses.push(harness); + writeFileSync(join(harness.tempDir, name), data); + harness.setResponses([ + fauxAssistantMessage([fauxToolCall("read", { path: name })], { stopReason: "toolUse" }), + fauxAssistantMessage("done"), + ]); + await harness.session.prompt("look at the image"); + return getToolResult(harness, "read").content; + } + + it("sends an image within the limits byte for byte, processed once", async () => { + const png = createPng(1800, 1100); + + const content = await readImage("small.png", png); + + expect(content).toEqual([ + { type: "text", text: "Read image file [image/png]" }, + { type: "image", data: png.toString("base64"), mimeType: "image/png" }, + ]); + // The read tool processes it; tool-result normalization recognizes the result and skips it. + expect(processImageCalls.count).toBe(1); + }); + + it("resizes an image over the limits once, with the dimension note", async () => { + const png = createPng(2442, 1438); + + const content = await readImage("shot.png", png); + + expect(content).toHaveLength(2); + const [note, image] = content; + expect(note).toEqual({ + type: "text", + text: "Read image file [image/png]\n[Image: original 2442x1438, displayed at 2000x1178. Multiply coordinates by 1.22 to map to original image.]", + }); + if (image.type !== "image") throw new Error("Expected an image"); + expect(image.mimeType).toBe("image/png"); + const resized = Buffer.from(image.data, "base64"); + expect([resized.readUInt32BE(16), resized.readUInt32BE(20)]).toEqual([2000, 1178]); + expect(processImageCalls.count).toBe(1); + }); +});