From f0079acee5ed96de1a313b1bc093a48485592c71 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Thu, 18 Jun 2026 14:48:10 -0700 Subject: [PATCH 1/3] feat(sdk): image-alpha hit-test phase 1 (WS-G) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the WS-A1 iframe adapter with image-alpha hit-testing: - Replace `elementFromPoint` with `elementsFromPoint` (z-stack) so a transparent-image hit falls through to the layer behind. - For `` hits: map client point → natural-pixel coords via a pure `mapPointToImagePixel` fn (object-fit cover/contain/fill aware); draw to an offscreen canvas once (cached by `currentSrc`); sample alpha via pure `alphaIsOpaque`. Transparent pixel → miss, continue the stack. - Cross-origin images that taint the canvas → SecurityError fallback → treat pixel as opaque (never drop an unverifiable hit). - Phase 2 (per-pixel alpha via `drawElement`) NOT built; gated on a perf spike per plan. Tests: alphaIsOpaque thresholds, mapPointToImagePixel (fill/cover/contain + out-of-box→null), z-stack fallthrough, taint→opaque fallback, non-image WS-A1 regression. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/sdk/src/adapters/iframe.test.ts | 339 ++++++++++++++++++++++- packages/sdk/src/adapters/iframe.ts | 304 +++++++++++++++++++- 2 files changed, 637 insertions(+), 6 deletions(-) diff --git a/packages/sdk/src/adapters/iframe.test.ts b/packages/sdk/src/adapters/iframe.test.ts index c01861900..7d7cd97eb 100644 --- a/packages/sdk/src/adapters/iframe.test.ts +++ b/packages/sdk/src/adapters/iframe.test.ts @@ -8,13 +8,23 @@ * applyDraft / commitPreview / cancelPreview require HTMLElement.style + querySelector * which are also browser-only. They are tested via a lightweight fake-DOM helper * that simulates style.setProperty / getAttribute / removeProperty. + * + * WS-G image-alpha tests cover: + * - alphaIsOpaque (pure predicate) + * - mapPointToImagePixel (pure coordinate mapping) + * - z-stack fallthrough via mock elementsFromPoint + * - canvas taint → opaque fallback + * - non-image regression (WS-A1 opacity behavior unchanged) */ -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect, vi, beforeEach } from "vitest"; import { resolveNearestHfElement, computeDraftPosition, createIframePreviewAdapter, + alphaIsOpaque, + mapPointToImagePixel, + _imgCanvasCache, } from "./iframe.js"; import type { ElementAtPointResult } from "./types.js"; import type { EditOp } from "../types.js"; @@ -372,3 +382,330 @@ describe("IframePreviewAdapter draft / commit / cancel", () => { expect(dispatch).toHaveBeenCalledTimes(1); }); }); + +// ─── WS-G: alphaIsOpaque ────────────────────────────────────────────────────── + +describe("alphaIsOpaque", () => { + function makeImageData(alpha: number): ImageData { + const data = new Uint8ClampedArray([255, 0, 0, alpha]); + return { data, width: 1, height: 1, colorSpace: "srgb" } as unknown as ImageData; + } + + it("returns false for a fully transparent pixel (a=0)", () => { + expect(alphaIsOpaque(makeImageData(0))).toBe(false); + }); + + it("returns true for a fully opaque pixel (a=255)", () => { + expect(alphaIsOpaque(makeImageData(255))).toBe(true); + }); + + it("returns false for alpha below default threshold (a=0 < 1)", () => { + expect(alphaIsOpaque(makeImageData(0), 1)).toBe(false); + }); + + it("returns true for alpha at default threshold (a=1 >= 1)", () => { + expect(alphaIsOpaque(makeImageData(1), 1)).toBe(true); + }); + + it("respects a custom threshold: a=100 < threshold=128 → false", () => { + expect(alphaIsOpaque(makeImageData(100), 128)).toBe(false); + }); + + it("respects a custom threshold: a=200 >= threshold=128 → true", () => { + expect(alphaIsOpaque(makeImageData(200), 128)).toBe(true); + }); + + it("threshold edge: a === threshold → true", () => { + expect(alphaIsOpaque(makeImageData(64), 64)).toBe(true); + }); +}); + +// ─── WS-G: mapPointToImagePixel ─────────────────────────────────────────────── + +describe("mapPointToImagePixel", () => { + // A 200×100 CSS box displaying a 400×200 natural image. + const rect = { left: 10, top: 20, width: 200, height: 100 }; + const natural = { width: 400, height: 200 }; + + it("fill: maps center of box to center of natural image", () => { + const result = mapPointToImagePixel(rect, natural, "fill", "50% 50%", { + x: rect.left + rect.width / 2, + y: rect.top + rect.height / 2, + }); + expect(result).toEqual({ px: 200, py: 100 }); + }); + + it("fill: maps top-left corner to natural (0, 0)", () => { + const result = mapPointToImagePixel(rect, natural, "fill", "50% 50%", { + x: rect.left, + y: rect.top, + }); + expect(result).toEqual({ px: 0, py: 0 }); + }); + + it("fill: maps bottom-right corner to natural (399, 199)", () => { + const result = mapPointToImagePixel(rect, natural, "fill", "50% 50%", { + x: rect.left + rect.width, + y: rect.top + rect.height, + }); + // rect.width maps to naturalWidth → px = floor(200/200 * 400) = 400, clamped to 399 + expect(result).toEqual({ px: 399, py: 199 }); + }); + + it("returns null when point is outside the CSS box (left)", () => { + const result = mapPointToImagePixel(rect, natural, "fill", "50% 50%", { + x: rect.left - 1, + y: rect.top + 10, + }); + expect(result).toBeNull(); + }); + + it("returns null when point is outside the CSS box (above)", () => { + const result = mapPointToImagePixel(rect, natural, "fill", "50% 50%", { + x: rect.left + 10, + y: rect.top - 1, + }); + expect(result).toBeNull(); + }); + + describe("cover", () => { + // 200×100 box, 100×100 natural → scale = max(2, 1) = 2 (letterbox on x) + // rendered: 200×200; centered by default (50% 50%) + // imgTop = (100 - 200)/2 = -50; imgLeft = (200-200)/2 = 0 + const coverRect = { left: 0, top: 0, width: 200, height: 100 }; + const coverNatural = { width: 100, height: 100 }; + + it("cover: point in center maps to center of natural image", () => { + // x=100, y=50 → rx=100-0=100, ry=50-(-50)=100; px=100/2=50, py=100/2=50 + const result = mapPointToImagePixel(coverRect, coverNatural, "cover", "50% 50%", { + x: 100, + y: 50, + }); + expect(result).toEqual({ px: 50, py: 50 }); + }); + }); + + describe("contain", () => { + // 200×100 box, 400×100 natural → scale = min(0.5, 1) = 0.5 + // rendered: 200×50; centered by default + // imgLeft = (200-200)/2 = 0; imgTop = (100-50)/2 = 25 + const containRect = { left: 0, top: 0, width: 200, height: 100 }; + const containNatural = { width: 400, height: 100 }; + + it("contain: point in rendered area maps to natural pixel", () => { + // y=50 → ry = 50-25 = 25; py = 25/0.5 = 50 + // x=100 → rx = 100-0 = 100; px = 100/0.5 = 200 + const result = mapPointToImagePixel(containRect, containNatural, "contain", "50% 50%", { + x: 100, + y: 50, + }); + expect(result).toEqual({ px: 200, py: 50 }); + }); + + it("contain: point in letterbox region (above image) → null", () => { + // imgTop = 25; y=10 → ry=10-25 = -15 → null + const result = mapPointToImagePixel(containRect, containNatural, "contain", "50% 50%", { + x: 100, + y: 10, + }); + expect(result).toBeNull(); + }); + + it("contain: point in letterbox region (below image) → null", () => { + // imgTop = 25; rendered height = 50 → imgBottom = 75; y=80 > 75 → null + const result = mapPointToImagePixel(containRect, containNatural, "contain", "50% 50%", { + x: 100, + y: 80, + }); + expect(result).toBeNull(); + }); + }); + + it("out-of-box point returns null regardless of object-fit", () => { + const r = { left: 50, top: 50, width: 100, height: 100 }; + const n = { width: 200, height: 200 }; + // Point to the left of the CSS box + expect(mapPointToImagePixel(r, n, "cover", "50% 50%", { x: 49, y: 100 })).toBeNull(); + }); +}); + +// ─── WS-G: z-stack fallthrough (mock elementsFromPoint) ────────────────────── + +/** + * WS-G z-stack tests. + * + * The adapter's elementAtPoint uses elementsFromPoint (z-stack) and checks + * `candidate instanceof win.HTMLImageElement`. Since happy-dom / the Bun test + * runner doesn't expose a global HTMLImageElement, we supply a local stub + * class and wire it into the fake contentWindow so the instanceof check works. + * + * Canvas pixel reads (getImageData) are unavailable in the test environment, so + * we patch globalThis.OffscreenCanvas to control alpha values: + * - opaque: getImageData → alpha=255 + * - transparent: getImageData → alpha=0 + * - tainted: getImageData → throws SecurityError + */ + +// ─── Stub HTMLImageElement (no global in Node/Bun) ─────────────────────────── + +class FakeHTMLImageElement { + attrs: Record; + tagName: string; + parentElement: FakeHTMLImageElement | null; + naturalWidth: number; + naturalHeight: number; + currentSrc: string; + src: string; + + constructor(id: string, parent: FakeHTMLImageElement | null = null) { + this.attrs = { "data-hf-id": id }; + this.tagName = "IMG"; + this.parentElement = parent; + this.naturalWidth = 100; + this.naturalHeight = 100; + this.currentSrc = `http://example.com/img-${id}.png`; + this.src = `http://example.com/img-${id}.png`; + } + + getBoundingClientRect(): DOMRect { + return { left: 0, top: 0, width: 100, height: 100, right: 100, bottom: 100 } as DOMRect; + } + + getAttribute(name: string): string | null { + return Object.prototype.hasOwnProperty.call(this.attrs, name) ? this.attrs[name] : null; + } + + hasAttribute(name: string): boolean { + return Object.prototype.hasOwnProperty.call(this.attrs, name); + } +} + +// WS-G tests reuse the existing FakeEl / fakeEl helper from above. +// buildFakeIframeWithStack accepts FakeEl or FakeHTMLImageElement. + +function buildFakeIframeWithStack(stack: Array) { + return { + contentDocument: { + elementsFromPoint(_x: number, _y: number) { + return stack; + }, + }, + contentWindow: { + // Supply our stub class so `candidate instanceof win.HTMLImageElement` works + HTMLImageElement: FakeHTMLImageElement, + getComputedStyle(_el: Element) { + return { opacity: "1" } as CSSStyleDeclaration; + }, + }, + } as unknown as HTMLIFrameElement; +} + +// ─── OffscreenCanvas stub helpers ──────────────────────────────────────────── + +type CanvasAlphaBehavior = "opaque" | "transparent" | "tainted"; + +function stubOffscreenCanvas(behavior: CanvasAlphaBehavior): () => void { + const orig = globalThis.OffscreenCanvas as typeof OffscreenCanvas | undefined; + globalThis.OffscreenCanvas = class { + width: number; + height: number; + constructor(w: number, h: number) { + this.width = w; + this.height = h; + } + getContext(_type: string) { + return { + drawImage() {}, + getImageData() { + if (behavior === "tainted") { + throw new DOMException("Tainted canvases may not be exported.", "SecurityError"); + } + const alpha = behavior === "opaque" ? 255 : 0; + return { data: new Uint8ClampedArray([255, 0, 0, alpha]), width: 1, height: 1 }; + }, + }; + } + } as unknown as typeof OffscreenCanvas; + return () => { + if (orig === undefined) { + delete (globalThis as Record).OffscreenCanvas; + } else { + globalThis.OffscreenCanvas = orig; + } + }; +} + +/** + * Run a z-stack image-alpha test with a controlled canvas behavior. + * Abstracts the restore/try/finally boilerplate shared across multiple tests. + */ +function withCanvasStub( + behavior: CanvasAlphaBehavior, + fn: ( + makeImgStack: ( + imgId: string, + behind?: Array, + ) => HTMLIFrameElement, + ) => void, +): void { + const restore = stubOffscreenCanvas(behavior); + try { + fn((imgId, behind = []) => { + const img = new FakeHTMLImageElement(imgId); + return buildFakeIframeWithStack([img, ...behind]); + }); + } finally { + restore(); + } +} + +describe("WS-G: z-stack fallthrough via mock elementsFromPoint", () => { + // Clear the module-level canvas cache before each test so stubs from a + // previous test don't affect the next one. + beforeEach(() => { + _imgCanvasCache.clear(); + }); + + it("non-image hit resolves normally (WS-A1 regression)", () => { + // No images in the stack — should behave exactly like WS-A1. + const div = fakeEl({ "data-hf-id": "hf-div" }, "DIV"); + const iframe = buildFakeIframeWithStack([div]); + const adapter = createIframePreviewAdapter(iframe); + const result = adapter.elementAtPoint(50, 50); + expect(result).toEqual({ id: "hf-div", tag: "div" }); + }); + + it("opaque image hit resolves to the image element", () => { + withCanvasStub("opaque", (makeImgStack) => { + const iframe = makeImgStack("hf-img", [fakeEl({ "data-hf-id": "hf-behind" }, "DIV")]); + const result = createIframePreviewAdapter(iframe).elementAtPoint(50, 50); + expect(result).toEqual({ id: "hf-img", tag: "img" }); + }); + }); + + it("transparent image pixel falls through to the element behind", () => { + withCanvasStub("transparent", (makeImgStack) => { + const iframe = makeImgStack("hf-img", [fakeEl({ "data-hf-id": "hf-behind" }, "DIV")]); + const result = createIframePreviewAdapter(iframe).elementAtPoint(50, 50); + expect(result).toEqual({ id: "hf-behind", tag: "div" }); + }); + }); + + it("tainted canvas (SecurityError) falls back to treating pixel as opaque", () => { + // Taint fallback → opaque → hit the image, not the behind-layer element + withCanvasStub("tainted", (makeImgStack) => { + const iframe = makeImgStack("hf-tainted", [fakeEl({ "data-hf-id": "hf-behind" }, "DIV")]); + const result = createIframePreviewAdapter(iframe).elementAtPoint(50, 50); + expect(result).toEqual({ id: "hf-tainted", tag: "img" }); + }); + }); + + it("transparent image with no element behind returns null", () => { + // No behind element in stack — transparent hit returns null. + withCanvasStub("transparent", (makeImgStack) => { + const iframe = makeImgStack("hf-img"); + const result = createIframePreviewAdapter(iframe).elementAtPoint(50, 50); + expect(result).toBeNull(); + }); + }); +}); diff --git a/packages/sdk/src/adapters/iframe.ts b/packages/sdk/src/adapters/iframe.ts index a0af067f9..24b8bc2cf 100644 --- a/packages/sdk/src/adapters/iframe.ts +++ b/packages/sdk/src/adapters/iframe.ts @@ -1,11 +1,26 @@ /** * Same-origin iframe PreviewAdapter — WS-A1 (hit-test + selection) + - * WS-A2 (applyDraft / commitPreview / cancelPreview → moveElement). + * WS-A2 (applyDraft / commitPreview / cancelPreview → moveElement) + + * WS-G (image-alpha hit-test, phase 1). * * Requirements: * - The iframe MUST be same-origin (srcdoc / blob URL). Cross-origin access to * contentDocument throws a DOMException; this adapter does not guard that — * the caller is responsible for ensuring same-origin. + * + * Image-alpha (phase 1): + * - Replaces elementFromPoint with elementsFromPoint (z-stack) so transparent + * image hits fall through to the element behind. + * - For hits, maps the client point to the natural-pixel coordinate + * (object-fit/object-position aware), draws to an offscreen canvas (cached + * by src), samples alpha. Transparent pixel → miss, continue the stack. + * - Cross-origin images taint the canvas → getImageData throws SecurityError + * → falls back to treating the pixel as OPAQUE (never drop an unverifiable + * hit). Callers must ensure CORS headers or accept the fallback behavior. + * - Limitation: animated (gif) or src swaps invalidate the cache only + * when currentSrc changes. Phase 1 is optimized for static images. + * - Phase 2 (full per-pixel alpha via drawElement rasterization) is NOT built + * here — gated on a perf spike. */ import type { PreviewAdapter, ElementAtPointResult, DraftProps } from "./types.js"; @@ -67,6 +82,162 @@ export function computeDraftPosition( return { x: baseX + dx, y: baseY + dy }; } +// ─── Image-alpha pure helpers (WS-G phase 1) ────────────────────────────────── + +/** + * Returns true when the first pixel in `imageData` has alpha >= threshold. + * + * Pure — no DOM access; unit-testable with a plain Uint8ClampedArray. + * threshold defaults to 1 so a fully-transparent pixel (a=0) is a miss. + */ +export function alphaIsOpaque(imageData: ImageData, threshold = 1): boolean { + // ImageData.data is [R, G, B, A, R, G, B, A, ...] + const alpha = imageData.data[3] ?? 0; + return alpha >= threshold; +} + +/** + * Map a client-space point to the natural-pixel coordinates of the image. + * + * Handles object-fit: fill | cover | contain (default=fill when unset). + * object-position is parsed as two percentage/px values (default "50% 50%"). + * + * Returns null when the point falls outside the rendered image area (e.g. + * the letterbox region of a contain-fitted image). A null result means the + * image does not own this pixel — the caller should continue the z-stack. + * + * Pure — no DOM/window access; unit-testable with plain objects. + */ +// fallow-ignore-next-line complexity +export function mapPointToImagePixel( + rect: { left: number; top: number; width: number; height: number }, + natural: { width: number; height: number }, + objectFit: string, + objectPosition: string, + point: { x: number; y: number }, +): { px: number; py: number } | null { + // Local coords within the CSS box + const lx = point.x - rect.left; + const ly = point.y - rect.top; + + if (lx < 0 || ly < 0 || lx > rect.width || ly > rect.height) return null; + + const fit = objectFit || "fill"; + + // For fill (or unrecognized values): the natural image is stretched to the + // box; direct linear mapping. + if (fit === "fill" || (fit !== "cover" && fit !== "contain" && fit !== "none")) { + if (rect.width === 0 || rect.height === 0) return null; + const px = Math.floor((lx / rect.width) * natural.width); + const py = Math.floor((ly / rect.height) * natural.height); + return { px: clamp(px, 0, natural.width - 1), py: clamp(py, 0, natural.height - 1) }; + } + + // For none: image is drawn at its natural size; no scaling. + if (fit === "none") { + const pos = parseObjectPosition(objectPosition, rect, natural); + const ox = pos.x; + const oy = pos.y; + const px = Math.floor(lx - ox); + const py = Math.floor(ly - oy); + if (px < 0 || py < 0 || px >= natural.width || py >= natural.height) return null; + return { px, py }; + } + + // cover: scale uniformly so the image covers the box; may clip edges. + // contain: scale uniformly so the image fits within the box; may letterbox. + const scaleX = rect.width / natural.width; + const scaleY = rect.height / natural.height; + const scale = fit === "cover" ? Math.max(scaleX, scaleY) : Math.min(scaleX, scaleY); + + const renderedW = natural.width * scale; + const renderedH = natural.height * scale; + + const pos = parseObjectPosition(objectPosition, rect, { + width: renderedW, + height: renderedH, + }); + + // Offset of the rendered image's top-left within the CSS box + const imgLeft = pos.x; + const imgTop = pos.y; + + // Local coords relative to the rendered image's top-left + const rx = lx - imgLeft; + const ry = ly - imgTop; + + if (rx < 0 || ry < 0 || rx > renderedW || ry > renderedH) return null; + + if (scale === 0) return null; + + const px = Math.floor(rx / scale); + const py = Math.floor(ry / scale); + return { px: clamp(px, 0, natural.width - 1), py: clamp(py, 0, natural.height - 1) }; +} + +// ─── object-position parser (pure) ─────────────────────────────────────────── + +/** + * Parse a CSS object-position value into x/y offsets (top-left of the + * rendered content relative to the CSS box top-left). + * + * Supports the common subset: keyword pairs, percentage pairs, pixel pairs, + * and single-value shorthand. Mixed units (e.g. "50% 10px") are supported. + * + * Pure — no DOM access. + */ +function parseObjectPosition( + objectPosition: string, + box: { width: number; height: number }, + content: { width: number; height: number }, +): { x: number; y: number } { + const raw = (objectPosition || "50% 50%").trim(); + const parts = raw.split(/\s+/); + + // Resolve a single token into a pixel offset along the given axis. + // `available` is the "slack" (box dimension - content dimension). + // fallow-ignore-next-line complexity + function resolveToken(token: string, available: number, _isX: boolean): number { + if (token === "left" || token === "top") return 0; + if (token === "right" || token === "bottom") return available; + if (token === "center") return available / 2; + if (token.endsWith("%")) { + const pct = parseFloat(token) / 100; + return isNaN(pct) ? available / 2 : pct * available; + } + if (token.endsWith("px")) { + const px = parseFloat(token); + return isNaN(px) ? available / 2 : px; + } + // Bare number — treat as px + const n = parseFloat(token); + return isNaN(n) ? available / 2 : n; + } + + const availX = box.width - content.width; + const availY = box.height - content.height; + + if (parts.length === 1) { + const tokenX = parts[0] ?? "50%"; + // Single value: if it's a vertical keyword the x defaults to center + if (tokenX === "top" || tokenX === "bottom") { + return { x: availX / 2, y: resolveToken(tokenX, availY, false) }; + } + return { x: resolveToken(tokenX, availX, true), y: availY / 2 }; + } + + const t0 = parts[0] ?? "50%"; + const t1 = parts[1] ?? "50%"; + return { + x: resolveToken(t0, availX, true), + y: resolveToken(t1, availY, false), + }; +} + +function clamp(v: number, min: number, max: number): number { + return v < min ? min : v > max ? max : v; +} + // ─── Visibility check ───────────────────────────────────────────────────────── /** @@ -89,6 +260,105 @@ function isOpacityVisible(el: Element, win: Window & typeof globalThis): boolean return true; } +// ─── Image-alpha canvas cache (WS-G phase 1) ───────────────────────────────── + +/** + * Cache of offscreen canvases keyed by image currentSrc. + * + * Canvases are drawn once; the same canvas is reused across hit-tests. + * Animated images (gif) or dynamic src swaps are NOT tracked — this is a + * phase-1 static-image optimization. A tainted entry stores null to record + * that the image is cross-origin and all pixels should be treated as opaque. + * + * Exported for tests that need to reset the cache between runs. + */ +export const _imgCanvasCache = new Map(); + +/** + * Sample the alpha at (clientX, clientY) for an element. + * + * Returns true (opaque) when: + * - The image has not finished loading (naturalWidth/naturalHeight === 0) + * - The point maps outside the rendered image area (not this image's pixel) + * - The canvas is tainted (cross-origin, SecurityError) — fallback: opaque + * - Alpha >= 1 + * + * Returns false (transparent/miss) only when the canvas is readable AND the + * alpha at the mapped pixel is 0. + * + * `win` is the iframe's contentWindow, used to call getComputedStyle on the + * element which lives in the iframe's document. + */ +// fallow-ignore-next-line complexity +function imageAlphaOpaqueAt( + img: HTMLImageElement, + clientX: number, + clientY: number, + win: Window & typeof globalThis, +): boolean { + // Not loaded yet — treat as opaque (safe fallback) + if (img.naturalWidth === 0 || img.naturalHeight === 0) return true; + + const src = img.currentSrc || img.src; + if (!src) return true; + + const rect = img.getBoundingClientRect(); + const style = win.getComputedStyle(img); + const objectFit = style.objectFit || "fill"; + const objectPosition = style.objectPosition || "50% 50%"; + + const mapped = mapPointToImagePixel( + { left: rect.left, top: rect.top, width: rect.width, height: rect.height }, + { width: img.naturalWidth, height: img.naturalHeight }, + objectFit, + objectPosition, + { x: clientX, y: clientY }, + ); + + // Point is outside the rendered image area — not this image's pixel. + // Continue the z-stack (return false = miss on this element). + if (mapped === null) return false; + + // Retrieve or build the offscreen canvas for this src. + let canvas: OffscreenCanvas | null | undefined = _imgCanvasCache.get(src); + if (canvas === undefined) { + // First time: draw to an offscreen canvas and cache. + try { + const oc = new OffscreenCanvas(img.naturalWidth, img.naturalHeight); + const ctx = oc.getContext("2d"); + if (!ctx) { + // OffscreenCanvas 2D unavailable — treat as opaque. + _imgCanvasCache.set(src, null); + return true; + } + ctx.drawImage(img, 0, 0, img.naturalWidth, img.naturalHeight); + // Attempt a pixel read immediately to detect taint at draw time. + // Some browsers taint lazily (on getImageData), so we also guard below. + ctx.getImageData(0, 0, 1, 1); + _imgCanvasCache.set(src, oc); + canvas = oc; + } catch { + // SecurityError from tainted canvas — record null and fall back opaque. + _imgCanvasCache.set(src, null); + return true; + } + } + + // null means we already know this src is tainted — treat as opaque. + if (canvas === null) return true; + + try { + const ctx = canvas.getContext("2d"); + if (!ctx) return true; + const data = ctx.getImageData(mapped.px, mapped.py, 1, 1); + return alphaIsOpaque(data); + } catch { + // Taint discovered on getImageData — update cache and fall back opaque. + _imgCanvasCache.set(src, null); + return true; + } +} + // ─── IframePreviewAdapter ───────────────────────────────────────────────────── type SelectionHandler = (ids: string[]) => void; @@ -112,10 +382,15 @@ class IframePreviewAdapter implements PreviewAdapter { /** * Synchronous hit-test. Returns the nearest `[data-hf-id]` element under * (x, y) in the iframe's coordinate space, or null for a transparent hit - * (root, opacity-0, or nothing at all). + * (root, opacity-0, nothing at all, or a transparent image pixel). + * + * WS-G phase 1: uses elementsFromPoint (z-stack) so a transparent-image hit + * falls through to the layer behind. For elements, the alpha at the + * mapped natural pixel is sampled from an offscreen canvas. Cross-origin + * images that taint the canvas are treated as opaque (safe fallback). * * atTime: reflects the GSAP state at the playhead when this is called. - * Seeking to a different time to check visibility is WS-G scope. + * Seeking to a different time to check visibility is WS-G follow-on. */ elementAtPoint(x: number, y: number, _opts?: { atTime?: number }): ElementAtPointResult | null { const doc = this.iframe.contentDocument; @@ -123,8 +398,27 @@ class IframePreviewAdapter implements PreviewAdapter { const win = this.iframe.contentWindow as (Window & typeof globalThis) | null; if (!win) return null; - const hit = doc.elementFromPoint(x, y); - return resolveNearestHfElement(hit, (el) => isOpacityVisible(el, win)); + const stack = doc.elementsFromPoint(x, y); + const isVisible = (el: Element) => isOpacityVisible(el, win); + + for (const candidate of stack) { + // Check opacity visibility first — skip entirely invisible branches. + if (!isOpacityVisible(candidate, win)) continue; + + // Image-alpha check: if this is an , verify the pixel is opaque. + if (candidate instanceof win.HTMLImageElement) { + if (!imageAlphaOpaqueAt(candidate, x, y, win)) { + // Transparent pixel — fall through to the next element in the stack. + continue; + } + } + + // Resolve the nearest hf element from this candidate. + const result = resolveNearestHfElement(candidate, isVisible); + if (result !== null) return result; + } + + return null; } /** From 823e9fbf42540c58b2715d4fe5d291f6f7210fd8 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Thu, 18 Jun 2026 19:01:36 -0700 Subject: [PATCH 2/3] fix(sdk): correct image-alpha hit-test edge cases (WS-G review) - map within the content box, not the border box (object-fit positions the image inside border+padding; getBoundingClientRect was off for a bordered/padded ) - normalize vertical-first object-position keyword pairs ("bottom left") - guard natural.width/height===0 in cover/contain (Infinity/NaN scale) - fall back to elementFromPoint when elementsFromPoint is unavailable - guard `instanceof win.HTMLImageElement` when the constructor is absent - bound _imgCanvasCache with a FIFO cap so it can't leak one canvas per src - drop the redundant taint-probe getImageData; the real pixel read already surfaces lazy taint - one opacity walk per candidate: the hf node is on the already-checked ancestor chain, so the resolver no longer re-walks for visibility - remove dead `fit==="fill"||` clause and unused resolveToken param - tests: none-fit, object-position keyword/px/reversed-pair, zero-natural, no-throw-without-HTMLImageElement + elementsFromPoint fallback Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/sdk/src/adapters/iframe.test.ts | 88 ++++++++++++++++++++- packages/sdk/src/adapters/iframe.ts | 97 ++++++++++++++++++------ 2 files changed, 161 insertions(+), 24 deletions(-) diff --git a/packages/sdk/src/adapters/iframe.test.ts b/packages/sdk/src/adapters/iframe.test.ts index 7d7cd97eb..40fff47b8 100644 --- a/packages/sdk/src/adapters/iframe.test.ts +++ b/packages/sdk/src/adapters/iframe.test.ts @@ -469,7 +469,8 @@ describe("mapPointToImagePixel", () => { }); describe("cover", () => { - // 200×100 box, 100×100 natural → scale = max(2, 1) = 2 (letterbox on x) + // 200×100 box, 100×100 natural → scale = max(2, 1) = 2 (cover clips Y; + // the 200×200 rendered image overflows top/bottom — cover never letterboxes) // rendered: 200×200; centered by default (50% 50%) // imgTop = (100 - 200)/2 = -50; imgLeft = (200-200)/2 = 0 const coverRect = { left: 0, top: 0, width: 200, height: 100 }; @@ -527,6 +528,70 @@ describe("mapPointToImagePixel", () => { // Point to the left of the CSS box expect(mapPointToImagePixel(r, n, "cover", "50% 50%", { x: 49, y: 100 })).toBeNull(); }); + + describe("none", () => { + // object-fit:none draws the image at natural size, positioned in the box. + // box 100×100, natural 50×50, "50% 50%" → availX=availY=50, offset=25. + const r = { left: 0, top: 0, width: 100, height: 100 }; + const n = { width: 50, height: 50 }; + + it("none: box center maps to image center", () => { + // click (50,50): px=floor(50-25)=25, py=25 + expect(mapPointToImagePixel(r, n, "none", "50% 50%", { x: 50, y: 50 })).toEqual({ + px: 25, + py: 25, + }); + }); + + it("none: point inside the box but outside the natural-size image → null", () => { + // image occupies 25..75; click at (10,10) is left/above it + expect(mapPointToImagePixel(r, n, "none", "50% 50%", { x: 10, y: 10 })).toBeNull(); + }); + }); + + describe("object-position", () => { + // contain so object-position has an effect on the vertical axis. + // box 200×100, natural 400×100 → scale 0.5, rendered 200×50; availX=0, availY=50. + const r = { left: 0, top: 0, width: 200, height: 100 }; + const n = { width: 400, height: 100 }; + + it("'left top' aligns the image to the top of the box", () => { + expect(mapPointToImagePixel(r, n, "contain", "left top", { x: 0, y: 0 })).toEqual({ + px: 0, + py: 0, + }); + }); + + it("reversed keyword pair 'top left' resolves identically to 'left top'", () => { + const reversed = mapPointToImagePixel(r, n, "contain", "top left", { x: 0, y: 0 }); + const canonical = mapPointToImagePixel(r, n, "contain", "left top", { x: 0, y: 0 }); + expect(reversed).toEqual(canonical); + expect(reversed).toEqual({ px: 0, py: 0 }); + }); + + it("'bottom left' (vertical-first) and 'left bottom' both align bottom-left", () => { + // bottom → imgTop=availY=50; bottom edge click y=100 → ry=50 → py=floor(50/0.5)=100→clamp 99 + const vh = mapPointToImagePixel(r, n, "contain", "bottom left", { x: 0, y: 100 }); + const hv = mapPointToImagePixel(r, n, "contain", "left bottom", { x: 0, y: 100 }); + expect(vh).toEqual(hv); + expect(vh).toEqual({ px: 0, py: 99 }); + }); + + it("supports pixel object-position values", () => { + // "0px 10px" → imgTop=10; click (0,10) → ry=0 → py=0 + expect(mapPointToImagePixel(r, n, "contain", "0px 10px", { x: 0, y: 10 })).toEqual({ + px: 0, + py: 0, + }); + }); + }); + + it("cover/contain returns null for a zero-dimension natural image", () => { + const r = { left: 0, top: 0, width: 200, height: 100 }; + expect( + mapPointToImagePixel(r, { width: 0, height: 100 }, "cover", "50% 50%", { x: 10, y: 10 }), + ).toBeNull(); + }); }); // ─── WS-G: z-stack fallthrough (mock elementsFromPoint) ────────────────────── @@ -708,4 +773,25 @@ describe("WS-G: z-stack fallthrough via mock elementsFromPoint", () => { expect(result).toBeNull(); }); }); + + it("survives a window without HTMLImageElement and a doc without elementsFromPoint", () => { + const div = fakeEl({ "data-hf-id": "hf-div" }, "DIV"); + const fakeIframe = (doc: unknown, win: unknown) => + ({ contentDocument: doc, contentWindow: win }) as unknown as HTMLIFrameElement; + const expected = { id: "hf-div", tag: "div" }; + + // No HTMLImageElement on the window — `instanceof` must not throw. + const noCtor = fakeIframe( + { elementsFromPoint: () => [div] }, + { getComputedStyle: () => ({ opacity: "1" }) }, + ); + expect(createIframePreviewAdapter(noCtor).elementAtPoint(50, 50)).toEqual(expected); + + // No elementsFromPoint on the doc — fall back to elementFromPoint. + const noStack = fakeIframe( + { elementFromPoint: () => div }, + { HTMLImageElement: FakeHTMLImageElement, getComputedStyle: () => ({ opacity: "1" }) }, + ); + expect(createIframePreviewAdapter(noStack).elementAtPoint(50, 50)).toEqual(expected); + }); }); diff --git a/packages/sdk/src/adapters/iframe.ts b/packages/sdk/src/adapters/iframe.ts index 24b8bc2cf..cb055980f 100644 --- a/packages/sdk/src/adapters/iframe.ts +++ b/packages/sdk/src/adapters/iframe.ts @@ -126,7 +126,7 @@ export function mapPointToImagePixel( // For fill (or unrecognized values): the natural image is stretched to the // box; direct linear mapping. - if (fit === "fill" || (fit !== "cover" && fit !== "contain" && fit !== "none")) { + if (fit !== "cover" && fit !== "contain" && fit !== "none") { if (rect.width === 0 || rect.height === 0) return null; const px = Math.floor((lx / rect.width) * natural.width); const py = Math.floor((ly / rect.height) * natural.height); @@ -146,6 +146,7 @@ export function mapPointToImagePixel( // cover: scale uniformly so the image covers the box; may clip edges. // contain: scale uniformly so the image fits within the box; may letterbox. + if (natural.width === 0 || natural.height === 0) return null; const scaleX = rect.width / natural.width; const scaleY = rect.height / natural.height; const scale = fit === "cover" ? Math.max(scaleX, scaleY) : Math.min(scaleX, scaleY); @@ -197,7 +198,7 @@ function parseObjectPosition( // Resolve a single token into a pixel offset along the given axis. // `available` is the "slack" (box dimension - content dimension). // fallow-ignore-next-line complexity - function resolveToken(token: string, available: number, _isX: boolean): number { + function resolveToken(token: string, available: number): number { if (token === "left" || token === "top") return 0; if (token === "right" || token === "bottom") return available; if (token === "center") return available / 2; @@ -217,20 +218,28 @@ function parseObjectPosition( const availX = box.width - content.width; const availY = box.height - content.height; + const isVert = (t: string) => t === "top" || t === "bottom"; + const isHoriz = (t: string) => t === "left" || t === "right"; + if (parts.length === 1) { const tokenX = parts[0] ?? "50%"; // Single value: if it's a vertical keyword the x defaults to center - if (tokenX === "top" || tokenX === "bottom") { - return { x: availX / 2, y: resolveToken(tokenX, availY, false) }; + if (isVert(tokenX)) { + return { x: availX / 2, y: resolveToken(tokenX, availY) }; } - return { x: resolveToken(tokenX, availX, true), y: availY / 2 }; + return { x: resolveToken(tokenX, availX), y: availY / 2 }; } - const t0 = parts[0] ?? "50%"; - const t1 = parts[1] ?? "50%"; + // Keyword pairs may be given vertical-first ("bottom left"); normalize so the + // first token addresses the x-axis and the second the y-axis. + let xToken = parts[0] ?? "50%"; + let yToken = parts[1] ?? "50%"; + if (isVert(xToken) || isHoriz(yToken)) { + [xToken, yToken] = [yToken, xToken]; + } return { - x: resolveToken(t0, availX, true), - y: resolveToken(t1, availY, false), + x: resolveToken(xToken, availX), + y: resolveToken(yToken, availY), }; } @@ -274,6 +283,21 @@ function isOpacityVisible(el: Element, win: Window & typeof globalThis): boolean */ export const _imgCanvasCache = new Map(); +/** + * Bounded cap so a long session can't accumulate one full-resolution + * OffscreenCanvas per image src indefinitely. + * ponytail: FIFO eviction, upgrade to LRU if cache hit-rate matters. + */ +const _IMG_CANVAS_CACHE_MAX = 64; + +function cacheCanvas(src: string, value: OffscreenCanvas | null): void { + if (!_imgCanvasCache.has(src) && _imgCanvasCache.size >= _IMG_CANVAS_CACHE_MAX) { + const oldest = _imgCanvasCache.keys().next().value; + if (oldest !== undefined) _imgCanvasCache.delete(oldest); + } + _imgCanvasCache.set(src, value); +} + /** * Sample the alpha at (clientX, clientY) for an element. * @@ -302,13 +326,29 @@ function imageAlphaOpaqueAt( const src = img.currentSrc || img.src; if (!src) return true; + // object-fit/object-position lay the image out within the CONTENT box, not + // the border box that getBoundingClientRect() returns. Inset by border + + // padding so the mapping is correct for an that has a border or padding. const rect = img.getBoundingClientRect(); const style = win.getComputedStyle(img); + const borderL = parseFloat(style.borderLeftWidth) || 0; + const borderT = parseFloat(style.borderTopWidth) || 0; + const borderR = parseFloat(style.borderRightWidth) || 0; + const borderB = parseFloat(style.borderBottomWidth) || 0; + const padL = parseFloat(style.paddingLeft) || 0; + const padT = parseFloat(style.paddingTop) || 0; + const padR = parseFloat(style.paddingRight) || 0; + const padB = parseFloat(style.paddingBottom) || 0; const objectFit = style.objectFit || "fill"; const objectPosition = style.objectPosition || "50% 50%"; const mapped = mapPointToImagePixel( - { left: rect.left, top: rect.top, width: rect.width, height: rect.height }, + { + left: rect.left + borderL + padL, + top: rect.top + borderT + padT, + width: rect.width - borderL - borderR - padL - padR, + height: rect.height - borderT - borderB - padT - padB, + }, { width: img.naturalWidth, height: img.naturalHeight }, objectFit, objectPosition, @@ -328,18 +368,15 @@ function imageAlphaOpaqueAt( const ctx = oc.getContext("2d"); if (!ctx) { // OffscreenCanvas 2D unavailable — treat as opaque. - _imgCanvasCache.set(src, null); + cacheCanvas(src, null); return true; } ctx.drawImage(img, 0, 0, img.naturalWidth, img.naturalHeight); - // Attempt a pixel read immediately to detect taint at draw time. - // Some browsers taint lazily (on getImageData), so we also guard below. - ctx.getImageData(0, 0, 1, 1); - _imgCanvasCache.set(src, oc); + cacheCanvas(src, oc); canvas = oc; } catch { // SecurityError from tainted canvas — record null and fall back opaque. - _imgCanvasCache.set(src, null); + cacheCanvas(src, null); return true; } } @@ -350,17 +387,29 @@ function imageAlphaOpaqueAt( try { const ctx = canvas.getContext("2d"); if (!ctx) return true; + // The mapped-pixel read also surfaces lazy canvas taint (SecurityError), + // so no separate taint probe is needed. const data = ctx.getImageData(mapped.px, mapped.py, 1, 1); return alphaIsOpaque(data); } catch { // Taint discovered on getImageData — update cache and fall back opaque. - _imgCanvasCache.set(src, null); + cacheCanvas(src, null); return true; } } // ─── IframePreviewAdapter ───────────────────────────────────────────────────── +/** + * The hit-test z-stack at (x, y): the full elementsFromPoint stack, or a + * single-element fallback for hosts that lack elementsFromPoint. + */ +function hitStack(doc: Document, x: number, y: number): Element[] { + if (typeof doc.elementsFromPoint === "function") return doc.elementsFromPoint(x, y); + const top = doc.elementFromPoint(x, y); + return top ? [top] : []; +} + type SelectionHandler = (ids: string[]) => void; class IframePreviewAdapter implements PreviewAdapter { @@ -398,23 +447,25 @@ class IframePreviewAdapter implements PreviewAdapter { const win = this.iframe.contentWindow as (Window & typeof globalThis) | null; if (!win) return null; - const stack = doc.elementsFromPoint(x, y); - const isVisible = (el: Element) => isOpacityVisible(el, win); + const stack = hitStack(doc, x, y); for (const candidate of stack) { - // Check opacity visibility first — skip entirely invisible branches. + // One opacity walk per candidate (candidate → root). An opacity:0 element + // is skipped, so the click falls through to the layer painted behind it. if (!isOpacityVisible(candidate, win)) continue; // Image-alpha check: if this is an , verify the pixel is opaque. - if (candidate instanceof win.HTMLImageElement) { + if (win.HTMLImageElement && candidate instanceof win.HTMLImageElement) { if (!imageAlphaOpaqueAt(candidate, x, y, win)) { // Transparent pixel — fall through to the next element in the stack. continue; } } - // Resolve the nearest hf element from this candidate. - const result = resolveNearestHfElement(candidate, isVisible); + // The candidate's whole ancestor chain is already known visible (the walk + // above covers it, and the hf node is on that chain), so the resolver + // needs no second visibility walk. + const result = resolveNearestHfElement(candidate, () => true); if (result !== null) return result; } From 9162ede1adf6a6ef37a4972d191eebadb0f599b0 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Fri, 19 Jun 2026 00:03:42 -0700 Subject: [PATCH 3/3] =?UTF-8?q?fix(sdk):=20image-alpha=20review=20fixes=20?= =?UTF-8?q?(WS-G)=20=E2=80=94=20transform/taint/cache-key/memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the phase-1 gaps flagged in review (was documentation-only): - CSS rotation/skew on the image or an ancestor now fails safe to opaque instead of sampling the wrong pixel (getBoundingClientRect is axis-aligned). Full transform-inverse mapping stays phase 2. No-op where DOMMatrix is unavailable. - Cross-origin canvas taint now warns once per src (was silent) so the fall-back-to-opaque path is visible, not "hit-test feels wrong". - Canvas cache keyed on src + natural dimensions (was src only) so a srcset/responsive re-render of the same URL doesn't reuse a stale canvas. - Pathological-size guard: images above a pixel budget skip alpha-testing (opaque) to bound OffscreenCanvas memory. - Docs: border/padding clicks fall through (intentional) noted on imageAlphaOpaqueAt. Tests: removed the duplicate a=0 threshold case; added transparent-over- transparent-over-div fallthrough. iframe.test.ts 59/59. tsc/oxlint/oxfmt green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/sdk/src/adapters/iframe.test.ts | 15 +++- packages/sdk/src/adapters/iframe.ts | 98 ++++++++++++++++++++---- 2 files changed, 95 insertions(+), 18 deletions(-) diff --git a/packages/sdk/src/adapters/iframe.test.ts b/packages/sdk/src/adapters/iframe.test.ts index 40fff47b8..e274800ae 100644 --- a/packages/sdk/src/adapters/iframe.test.ts +++ b/packages/sdk/src/adapters/iframe.test.ts @@ -399,10 +399,6 @@ describe("alphaIsOpaque", () => { expect(alphaIsOpaque(makeImageData(255))).toBe(true); }); - it("returns false for alpha below default threshold (a=0 < 1)", () => { - expect(alphaIsOpaque(makeImageData(0), 1)).toBe(false); - }); - it("returns true for alpha at default threshold (a=1 >= 1)", () => { expect(alphaIsOpaque(makeImageData(1), 1)).toBe(true); }); @@ -765,6 +761,17 @@ describe("WS-G: z-stack fallthrough via mock elementsFromPoint", () => { }); }); + it("transparent image over transparent image falls through to the div behind both", () => { + // Two consecutive transparent fallthroughs — exercises the loop iterating + // past more than one transparent image before hitting an opaque layer. + withCanvasStub("transparent", (makeImgStack) => { + const img2 = new FakeHTMLImageElement("hf-img2"); + const iframe = makeImgStack("hf-img1", [img2, fakeEl({ "data-hf-id": "hf-behind" }, "DIV")]); + const result = createIframePreviewAdapter(iframe).elementAtPoint(50, 50); + expect(result).toEqual({ id: "hf-behind", tag: "div" }); + }); + }); + it("transparent image with no element behind returns null", () => { // No behind element in stack — transparent hit returns null. withCanvasStub("transparent", (makeImgStack) => { diff --git a/packages/sdk/src/adapters/iframe.ts b/packages/sdk/src/adapters/iframe.ts index cb055980f..e45c7acd9 100644 --- a/packages/sdk/src/adapters/iframe.ts +++ b/packages/sdk/src/adapters/iframe.ts @@ -13,12 +13,17 @@ * image hits fall through to the element behind. * - For hits, maps the client point to the natural-pixel coordinate * (object-fit/object-position aware), draws to an offscreen canvas (cached - * by src), samples alpha. Transparent pixel → miss, continue the stack. + * by src + natural dimensions, so a srcset re-render gets a fresh canvas), + * samples alpha. Transparent pixel → miss, continue the stack. * - Cross-origin images taint the canvas → getImageData throws SecurityError * → falls back to treating the pixel as OPAQUE (never drop an unverifiable - * hit). Callers must ensure CORS headers or accept the fallback behavior. - * - Limitation: animated (gif) or src swaps invalidate the cache only - * when currentSrc changes. Phase 1 is optimized for static images. + * hit) and warns once per src. Callers must ensure CORS or accept the fallback. + * - A CSS rotation/skew on the image or an ancestor also falls back to opaque + * (the axis-aligned rect mapping can't sample a rotated image correctly); + * transform-inverse mapping is phase 2. + * - Images above a pixel budget skip alpha-testing (opaque) to bound canvas + * memory. Limitation: animated (gif) or src swaps invalidate the cache + * only when currentSrc/dimensions change. Phase 1 is optimized for static images. * - Phase 2 (full per-pixel alpha via drawElement rasterization) is NOT built * here — gated on a perf spike. */ @@ -290,12 +295,61 @@ export const _imgCanvasCache = new Map(); */ const _IMG_CANVAS_CACHE_MAX = 64; -function cacheCanvas(src: string, value: OffscreenCanvas | null): void { - if (!_imgCanvasCache.has(src) && _imgCanvasCache.size >= _IMG_CANVAS_CACHE_MAX) { +function cacheCanvas(key: string, value: OffscreenCanvas | null): void { + if (!_imgCanvasCache.has(key) && _imgCanvasCache.size >= _IMG_CANVAS_CACHE_MAX) { const oldest = _imgCanvasCache.keys().next().value; if (oldest !== undefined) _imgCanvasCache.delete(oldest); } - _imgCanvasCache.set(src, value); + _imgCanvasCache.set(key, value); +} + +/** + * Skip alpha-testing images above this pixel budget — allocating a full-res + * OffscreenCanvas for one hit-test is memory-prohibitive (a 4000×3000 image is + * ~46 MB). Above the cap we fail safe to opaque. + * ponytail: per-image pixel cap; a byte-budget cache is phase 2. + */ +const _MAX_ALPHA_TEST_PIXELS = 16_000_000; + +/** + * Srcs we've already warned about taint for — keeps the cross-origin warning to + * once per image instead of once per hit-test. Cleared with `_imgCanvasCache` + * is not necessary; suppressing duplicate warnings across resets is harmless. + */ +const _warnedTaintSrcs = new Set(); + +function warnTaintOnce(src: string): void { + if (_warnedTaintSrcs.has(src)) return; + _warnedTaintSrcs.add(src); + // Visibility for the silent-failure path: a cross-origin / uncorsed image + // taints the canvas, so alpha hit-test is unavailable and we fall back to + // opaque. Without this, the fall-back is invisible ("hit-test feels wrong"). + console.warn( + `[hyperframes] image-alpha hit-test unavailable for cross-origin/tainted image; treating as opaque: ${src}`, + ); +} + +/** + * True when the element or any ancestor carries a CSS rotation or skew. Such a + * transform makes getBoundingClientRect() return the axis-aligned bounding box, + * so the rect→natural-pixel mapping would sample the wrong pixel. Pure translate + * / scale keep the matrix b and c components at 0 and map correctly. No-op + * (returns false) when DOMMatrix is unavailable (e.g. the test env), preserving + * existing behavior there. + */ +function hasRotationOrSkew(el: Element | null, win: Window & typeof globalThis): boolean { + if (typeof win.DOMMatrix !== "function") return false; + for (let node: Element | null = el; node; node = node.parentElement) { + const t = win.getComputedStyle(node).transform; + if (!t || t === "none") continue; + try { + const m = new win.DOMMatrix(t); + if (Math.abs(m.b) > 1e-6 || Math.abs(m.c) > 1e-6) return true; + } catch { + return true; // unparseable transform — fail safe (treat as non-axis-aligned) + } + } + return false; } /** @@ -308,7 +362,9 @@ function cacheCanvas(src: string, value: OffscreenCanvas | null): void { * - Alpha >= 1 * * Returns false (transparent/miss) only when the canvas is readable AND the - * alpha at the mapped pixel is 0. + * alpha at the mapped pixel is 0. A click on the element's border/padding maps + * outside the content box → also false (the click falls through to the layer + * behind), since border/padding pixels aren't part of the image — intentional. * * `win` is the iframe's contentWindow, used to call getComputedStyle on the * element which lives in the iframe's document. @@ -326,6 +382,15 @@ function imageAlphaOpaqueAt( const src = img.currentSrc || img.src; if (!src) return true; + // CSS rotation/skew on the image or an ancestor breaks the axis-aligned + // rect→natural-pixel mapping below (getBoundingClientRect returns the AABB), + // so we'd sample the wrong pixel. Fail safe to opaque rather than guess. + // Full transform-inverse mapping is phase 2. + if (hasRotationOrSkew(img, win)) return true; + + // Pathological-size guard: don't allocate a huge canvas for one hit-test. + if (img.naturalWidth * img.naturalHeight > _MAX_ALPHA_TEST_PIXELS) return true; + // object-fit/object-position lay the image out within the CONTENT box, not // the border box that getBoundingClientRect() returns. Inset by border + // padding so the mapping is correct for an that has a border or padding. @@ -359,8 +424,11 @@ function imageAlphaOpaqueAt( // Continue the z-stack (return false = miss on this element). if (mapped === null) return false; - // Retrieve or build the offscreen canvas for this src. - let canvas: OffscreenCanvas | null | undefined = _imgCanvasCache.get(src); + // Retrieve or build the offscreen canvas. Key on src + natural dimensions: a + // srcset/responsive layout can serve the same URL at a different natural size, + // and keying on src alone would reuse a canvas drawn at the prior dimensions. + const cacheKey = `${src}@${img.naturalWidth}x${img.naturalHeight}`; + let canvas: OffscreenCanvas | null | undefined = _imgCanvasCache.get(cacheKey); if (canvas === undefined) { // First time: draw to an offscreen canvas and cache. try { @@ -368,15 +436,16 @@ function imageAlphaOpaqueAt( const ctx = oc.getContext("2d"); if (!ctx) { // OffscreenCanvas 2D unavailable — treat as opaque. - cacheCanvas(src, null); + cacheCanvas(cacheKey, null); return true; } ctx.drawImage(img, 0, 0, img.naturalWidth, img.naturalHeight); - cacheCanvas(src, oc); + cacheCanvas(cacheKey, oc); canvas = oc; } catch { // SecurityError from tainted canvas — record null and fall back opaque. - cacheCanvas(src, null); + warnTaintOnce(src); + cacheCanvas(cacheKey, null); return true; } } @@ -393,7 +462,8 @@ function imageAlphaOpaqueAt( return alphaIsOpaque(data); } catch { // Taint discovered on getImageData — update cache and fall back opaque. - cacheCanvas(src, null); + warnTaintOnce(src); + cacheCanvas(cacheKey, null); return true; } }