From 1abf3005c2ffa1c3697135ebf2822edc8766286b Mon Sep 17 00:00:00 2001 From: Logan Kuzyk Date: Sun, 11 Oct 2026 10:28:34 -0700 Subject: [PATCH 1/2] feat(photos): AI-generated alt text via Workers AI Photos uploaded with a blank alt are described by a vision model (@cf/moondream/moondream3.1-9B-A2B on Cloudflare Workers AI, over REST). - Generation runs inline during the upload, from the rendition Payload has just built in memory, so the edit form that loads afterwards already holds the text and the create's own cache revalidation covers it. - If the model call fails, the upload still succeeds and a job on a dedicated `alt-text` queue retries it; `jobs.autoRun` drains only that queue. - New server-maintained `altSource` field (ai | manual) so regeneration never replaces hand-written text. - "Regenerate alt text" button in the photo edit view (POST /api/photos/:id/alt-text). - `npm run generate:alt-text` backfills existing photos. Off entirely unless CLOUDFLARE_ACCOUNT_ID and WORKERS_AI_API_TOKEN are set. Co-Authored-By: Claude Opus 5.5 --- .env.example | 10 + CMS.md | 14 +- package.json | 1 + scripts/generate-alt-text.ts | 201 +++ src/collections/Photos.ts | 46 +- .../Photos/altText/GenerateAltTextButton.tsx | 91 + src/collections/Photos/altText/endpoint.ts | 49 + src/collections/Photos/altText/generate.ts | 148 ++ src/collections/Photos/altText/hooks.ts | 108 ++ src/collections/Photos/altText/image.ts | 67 + src/collections/Photos/altText/prompt.ts | 73 + src/collections/Photos/altText/task.ts | 29 + src/collections/Photos/altText/workersAI.ts | 80 + src/payload-types.ts | 25 +- src/payload.config.ts | 6 +- tests/int/photoAltText.int.spec.ts | 1560 +++++++++++++++++ 16 files changed, 2502 insertions(+), 6 deletions(-) create mode 100644 scripts/generate-alt-text.ts create mode 100644 src/collections/Photos/altText/GenerateAltTextButton.tsx create mode 100644 src/collections/Photos/altText/endpoint.ts create mode 100644 src/collections/Photos/altText/generate.ts create mode 100644 src/collections/Photos/altText/hooks.ts create mode 100644 src/collections/Photos/altText/image.ts create mode 100644 src/collections/Photos/altText/prompt.ts create mode 100644 src/collections/Photos/altText/task.ts create mode 100644 src/collections/Photos/altText/workersAI.ts create mode 100644 tests/int/photoAltText.int.spec.ts diff --git a/.env.example b/.env.example index 96a7b47..cc1e28b 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,16 @@ PREVIEW_SECRET=YOUR_SECRET_HERE # (e.g. a local MinIO mock). Real Cloudflare R2 does not need it. # R2_FORCE_PATH_STYLE= +# Cloudflare Workers AI: writes alt text for photos (model @cf/moondream/moondream3.1-9B-A2B, called over REST). +# With both set, a photo uploaded with a blank alt gets one generated, the photo edit view shows a +# "Regenerate alt text" button, and `npm run generate:alt-text` can backfill existing photos. +# Leave either unset and the feature is off entirely: uploads keep a blank alt, nothing is sent to Cloudflare. +# CLOUDFLARE_ACCOUNT_ID is the account ID shown in the Cloudflare dashboard. +# WORKERS_AI_API_TOKEN is an API token for that account; a custom token needs the "Workers AI - Read" +# and "Workers AI - Edit" permissions. Each generated alt text is a billed model call. +# CLOUDFLARE_ACCOUNT_ID= +# WORKERS_AI_API_TOKEN= + # --- Local R2 mock (MinIO) -------------------------------------------------------------- # Start it with: npm run dev:r2 (console at http://localhost:9001, minioadmin/minioadmin) # Then run the app with the storage vars pointed at MinIO. Easiest via the bundled launch diff --git a/CMS.md b/CMS.md index 7a3b3bf..f39ae9c 100644 --- a/CMS.md +++ b/CMS.md @@ -18,6 +18,8 @@ This document describes the environment variables and Payload CMS data structure | `R2_REGION` | No | Typically `auto` for R2. | | `STORAGE_URL` | No | Public URL for R2 (e.g. `https://media.example.com`). No trailing slash. | | `NEXT_PUBLIC_STORAGE_URL` | No | Same as `STORAGE_URL`. Exposed to the browser so images load directly from the CDN instead of proxying through Next.js. | +| `CLOUDFLARE_ACCOUNT_ID` | No | Cloudflare account ID for Workers AI. With `WORKERS_AI_API_TOKEN`, enables AI-generated photo alt text; if either is unset the feature is off entirely. | +| `WORKERS_AI_API_TOKEN` | No | Cloudflare API token used to call Workers AI over REST. A custom token needs the "Workers AI - Read" and "Workers AI - Edit" permissions. | ### Storage Behavior @@ -75,7 +77,8 @@ Slug: `media` · Folders: Yes |-------|------|-------------| | tags | relationship → tags | Tags for photo gallery filtering | | displayOrder | number | Order in photos gallery (lower = earlier) | -| alt | text | Alt text for accessibility | +| alt | text | Alt text for accessibility. Left blank on upload, it is written by a vision model when Workers AI is configured (see **Alt text** below) | +| altSource | select | `ai`, `manual` or unset: who wrote `alt`. Read-only, maintained by a hook | | caption | richText | Image caption | | (upload) | — | File upload with focal point, image sizes: thumbnail, square, small, medium, large, xlarge, og | @@ -92,6 +95,15 @@ Slug: `media` · Folders: Yes - **Delivery**: `ImageMedia` builds a real `srcSet` from `sizes.*` even when images are served straight from the CDN (`NEXT_PUBLIC_STORAGE_URL`), via a Next.js image `loader` that maps each requested width to the closest rendition. Callers that render into a known box (e.g. `PhotoGrid` tiles) pass an explicit `size`. The rendition ladder is capped at `xlarge` (1920px WebP); only `priority` (LCP) images may fall back to the raw original, so a missing/inaccurate `size` can never silently serve a multi-MB file. - **Backfill**: existing photos are reprocessed with `npm run reprocess:photos` (`-- --dry-run` / `-- --limit N` / `-- --force` / `-- --continue-on-error`). It re-downloads each original, has Payload rebuild `sizes.*`, then verifies every object resolves from storage before moving on and stops on the first failure. AVIF/HEIC sources decode slowly, so large batches take time. Run it against the local MinIO mock (`npm run dev:r2`) first and confirm bucket object counts before pointing it at R2. +**Alt text** (`src/collections/Photos/altText/`): optional, enabled by setting both `CLOUDFLARE_ACCOUNT_ID` and `WORKERS_AI_API_TOKEN`. Descriptions come from the Workers AI vision model `@cf/moondream/moondream3.1-9B-A2B`, called over REST; the photo's tags and location are passed along as hints. + +- **On upload**: a photo created with a blank `alt` is described during the upload itself, so the edit form already holds the text. Text typed at upload is kept as is. Existing photos are never changed by an ordinary save. +- **`altSource`**: `ai` when the model wrote `alt`, `manual` once it is typed or edited by hand, unset while `alt` is blank and on photos whose text predates the feature (those count as hand-written). It is what lets a regeneration pass replace machine-written text without touching hand-written text. +- **Regenerate**: the edit view has a "Regenerate alt text" button under the alt field (hidden when Workers AI is not configured). It POSTs to `/api/photos/:id/alt-text` (authenticated), which replaces whatever is there and saves it immediately; the button asks first when the current text is hand-written. +- **Retry queue**: if the model call fails during an upload, the upload still succeeds with a blank `alt` and a `generateAltText` job is queued on the `alt-text` queue. `jobs.autoRun` in `payload.config.ts` drains that queue every 2 minutes inside the server process; the job only ever fills a blank `alt` and retries with exponential backoff. +- **Backfill**: `npm run generate:alt-text` (`-- --dry-run` / `-- --limit N` / `-- --regenerate` / `-- --continue-on-error`) fills photos whose `alt` is blank; `--regenerate` also redoes `ai` text. Hand-written text is never touched. It makes one billed model call per photo and stops on the first failure. +- **Caches**: writes made outside a Next.js request (the backfill script, the retry job) cannot revalidate the photos sitemap or the new-tab manifest. Those pick the text up on the next photo change in the admin, or the manifest's 24h backstop. + ### Categories Slug: `categories` diff --git a/package.json b/package.json index e74aec5..e65b362 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "dev:db": "docker compose -f docker-compose.yml -f docker-compose.dev.yml up mongo -d", "dev:r2": "docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile r2 up minio minio-setup -d", "dev:prod": "cross-env NODE_OPTIONS=--no-deprecation rm -rf .next && npm run build && npm run start", + "generate:alt-text": "cross-env NODE_OPTIONS=--no-deprecation tsx scripts/generate-alt-text.ts", "generate:importmap": "cross-env NODE_OPTIONS=--no-deprecation payload generate:importmap", "generate:types": "cross-env NODE_OPTIONS=--no-deprecation payload generate:types", "ii": "cross-env NODE_OPTIONS=--no-deprecation npm install", diff --git a/scripts/generate-alt-text.ts b/scripts/generate-alt-text.ts new file mode 100644 index 0000000..2f8fa9f --- /dev/null +++ b/scripts/generate-alt-text.ts @@ -0,0 +1,201 @@ +/** + * Backfill script: write AI-generated alt text for existing Photos that have none (see + * src/collections/Photos/altText). New uploads get theirs during the upload itself; this + * covers everything uploaded before that existed, or while Workers AI was not configured. + * + * Each photo is one request to Cloudflare Workers AI (`@cf/moondream/moondream3.1-9B-A2B`), + * so it needs CLOUDFLARE_ACCOUNT_ID and WORKERS_AI_API_TOKEN (see .env.example). + * + * Runs standalone (outside Next.js), same pattern as scripts/reprocess-photos.ts. + * + * ───────────────────────────────────────────────────────────────────────────────────────── + * WHAT IT WILL AND WON'T TOUCH + * + * - Default: only photos whose `alt` is blank (missing, null or ''). + * - `--regenerate`: also photos whose `altSource` is 'ai', replacing text an earlier run (or + * the upload hook) generated. + * - Never hand-written text. `generatePhotoAltText` re-reads each photo right before the + * model call and skips it if it has since been given alt text it may not replace, so text + * typed in the admin while this runs is safe. + * + * Only `alt` and `altSource` are written; no file is uploaded and `sizes.*` is not rebuilt. + * + * The run STOPS on the first failure (pass `--continue-on-error` to override): a bad token + * or an exhausted quota fails every photo the same way, and there is no point finding that + * out several hundred requests later. + * + * COST: every generated alt text is a billed model call. Cloudflare's model page lists + * $0.30 per million input tokens and $1.00 per million output tokens; `--dry-run` shows how + * many photos a run would send, and `--limit` lets you try a handful and read the real usage + * off the Workers AI dashboard before committing to the lot. + * ───────────────────────────────────────────────────────────────────────────────────────── + * + * Usage: + * npm run generate:alt-text -- --dry-run # list what would be processed, call nothing + * npm run generate:alt-text -- --limit 25 # process at most 25 photos, then stop + * npm run generate:alt-text -- --regenerate # also redo AI-written alt text + * npm run generate:alt-text -- --continue-on-error # don't stop on the first failed photo + * npm run generate:alt-text # process every photo with blank alt text + */ +import 'dotenv/config' +import { createLocalReq, getPayload, type Where } from 'payload' +import config from '../src/payload.config' +import { generatePhotoAltText } from '../src/collections/Photos/altText/generate' +import { ALT_TEXT_MODEL, isAltTextConfigured } from '../src/collections/Photos/altText/workersAI' + +const FLAGS = ['--dry-run', '--regenerate', '--continue-on-error', '--limit'] + +const args = process.argv.slice(2) +const DRY_RUN = args.includes('--dry-run') +const REGENERATE = args.includes('--regenerate') +const CONTINUE_ON_ERROR = args.includes('--continue-on-error') +const limitArg = args.indexOf('--limit') +const LIMIT = limitArg >= 0 ? Number(args[limitArg + 1]) : Infinity +const DELAY_MS = 500 // be polite: one model call per photo, strictly one at a time + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +// A mistyped `--dry-run` must not turn into a real, billed run. +const unknown = args.filter( + (arg, i) => !FLAGS.includes(arg) && !(limitArg >= 0 && i === limitArg + 1), +) +if (unknown.length) { + console.error(`Unknown argument(s): ${unknown.join(' ')}\nKnown flags: ${FLAGS.join(' ')}`) + process.exit(1) +} +if (limitArg >= 0 && !(Number.isInteger(LIMIT) && LIMIT > 0)) { + console.error('--limit needs a positive whole number, e.g. --limit 25') + process.exit(1) +} + +/** + * Photos this run may write to. + * + * `exists: false` is the clause doing the work: for a text field the Mongo adapter expands + * it to "missing OR null OR ''", which are the three ways a blank `alt` is stored (never set, + * cleared through the API, cleared in the admin form). The explicit `equals: ''` is redundant + * with it today and only there so an adapter change can't silently drop the empty-string case. + * + * Not matched: an `alt` of nothing but whitespace. No query operator expresses that, and it + * takes someone typing only spaces into the field to create one. + */ +const candidatesWhere: Where = { + or: [ + { alt: { exists: false } }, + { alt: { equals: '' } }, + ...(REGENERATE ? [{ altSource: { equals: 'ai' } }] : []), + ], +} + +async function main() { + if (!DRY_RUN && !isAltTextConfigured()) { + console.error( + 'Workers AI is not configured: set CLOUDFLARE_ACCOUNT_ID and WORKERS_AI_API_TOKEN ' + + '(see .env.example).\nNothing was changed. Use --dry-run to list the photos without them.', + ) + process.exit(1) + } + + const payload = await getPayload({ config }) + + // Collect every candidate before the first write. Paging through this query while filling + // in `alt` would shrink the result set under the cursor and skip photos. + const { docs: candidates } = await payload.find({ + collection: 'photos', + depth: 0, + overrideAccess: true, + pagination: false, + select: { alt: true, altSource: true, filename: true }, + sort: 'createdAt', + where: candidatesWhere, + }) + const batch = candidates.slice(0, LIMIT) + + console.log( + `${candidates.length} photo(s) ${REGENERATE ? 'with blank or AI-written' : 'with blank'} alt text` + + (batch.length < candidates.length ? `, taking the first ${batch.length} (--limit)` : '') + + (DRY_RUN ? '' : `. Model: ${ALT_TEXT_MODEL}`), + ) + + const label = (photo: (typeof batch)[number]) => + `${photo.id} ${photo.filename ?? '(no filename)'}` + + if (DRY_RUN) { + for (const photo of batch) { + const current = photo.alt?.trim() + console.log( + current + ? `would regenerate: ${label(photo)} (currently ${JSON.stringify(current)})` + : `would generate: ${label(photo)}`, + ) + } + if (!isAltTextConfigured()) { + console.log( + '\nNote: CLOUDFLARE_ACCOUNT_ID / WORKERS_AI_API_TOKEN are not set, so a real run would refuse to start.', + ) + } + console.log(`\nDone. (dry run) would process=${batch.length}`) + process.exit(0) + } + + const { docs: users } = await payload.find({ + collection: 'users', + limit: 1, + overrideAccess: true, + }) + if (!users[0]) throw new Error('no admin user found to attribute the alt text writes to') + const req = await createLocalReq({ user: users[0] }, payload) + + let generated = 0 + let skipped = 0 + let failed = 0 + + for (const photo of batch) { + try { + const result = await generatePhotoAltText({ + id: photo.id, + // 'ai' still fills blanks, and still refuses hand-written text. + overwrite: REGENERATE ? 'ai' : 'empty', + // Skip the `revalidatePhoto` afterChange hook: `revalidateTag()` throws outside a Next.js + // request. The hook catches that now, so without this flag each photo would just log two + // warnings, but a throw there is what once skipped the R2 upload hook for a whole + // backfill (see scripts/reprocess-photos.ts), so scripts opt out explicitly. + // Consequence: this script does NOT refresh the photos sitemap or the new-tab manifest + // caches. They pick the new text up on the next photo change made in the admin, or, for + // the manifest, its 24h backstop. + context: { disableRevalidate: true }, + req, + }) + + if (result.status === 'generated') { + generated++ + console.log(`${label(photo)} -> ${JSON.stringify(result.alt)}`) + } else { + // Deleted, or given alt text by someone else, since the candidates were collected. + skipped++ + console.log(`skipped (${result.reason}): ${label(photo)}`) + } + } catch (err) { + failed++ + console.error(`FAILED ${label(photo)}:`, err instanceof Error ? err.message : err) + if (!CONTINUE_ON_ERROR) { + console.error('\nStopping on first failure (pass --continue-on-error to override).') + break + } + } + + await sleep(DELAY_MS) + } + + const remaining = candidates.length - generated - skipped - failed + console.log( + `\nDone. generated=${generated} skipped=${skipped} failed=${failed}` + + (remaining > 0 ? ` remaining=${remaining}` : ''), + ) + process.exit(failed > 0 ? 1 : 0) +} + +main().catch((err) => { + console.error('generate-alt-text failed:', err) + process.exit(1) +}) diff --git a/src/collections/Photos.ts b/src/collections/Photos.ts index e03d2a0..6cab5b3 100644 --- a/src/collections/Photos.ts +++ b/src/collections/Photos.ts @@ -11,6 +11,9 @@ import { fileURLToPath } from 'url' import { anyone } from '../access/anyone' import { authenticated } from '../access/authenticated' import { appendPrefixToCollectionBeforeChangeHook } from '../hooks/uploadPrefixed' +import { generateAltTextEndpoint } from './Photos/altText/endpoint' +import { generateAltTextOnUpload, queueAltTextRetry } from './Photos/altText/hooks' +import { isAltTextConfigured } from './Photos/altText/workersAI' import { revalidateDelete, revalidatePhoto } from './Photos/hooks/revalidatePhoto' import { transcodeHeicUpload } from './Photos/hooks/transcodeHeicUpload' @@ -24,10 +27,11 @@ export const Photos: CollectionConfig = { defaultColumns: ['filename', 'alt', 'tags', 'updatedAt'], }, folders: true, + endpoints: [generateAltTextEndpoint], hooks: { beforeOperation: [transcodeHeicUpload], - beforeChange: [appendPrefixToCollectionBeforeChangeHook('photos')], - afterChange: [revalidatePhoto], + beforeChange: [appendPrefixToCollectionBeforeChangeHook('photos'), generateAltTextOnUpload], + afterChange: [revalidatePhoto, queueAltTextRetry], afterDelete: [revalidateDelete], }, access: { @@ -58,6 +62,44 @@ export const Photos: CollectionConfig = { name: 'alt', type: 'text', //required: true, + admin: { + description: + 'Left blank on upload, this is written by a vision model (when Workers AI is configured).', + }, + }, + { + name: 'generateAltText', + type: 'ui', + admin: { + components: { + Field: { + // Env vars never reach the browser, so the server decides whether to show the button. + clientProps: { enabled: isAltTextConfigured() }, + path: '@/collections/Photos/altText/GenerateAltTextButton#GenerateAltTextButton', + }, + }, + // A `ui` field would otherwise be offered as an (empty) list column. + disableListColumn: true, + }, + }, + { + // Maintained by the `generateAltTextOnUpload` hook, never by the client: it is what lets + // a regeneration pass replace machine-written text without touching hand-written text. + name: 'altSource', + type: 'select', + options: [ + { label: 'AI-generated', value: 'ai' }, + { label: 'Hand-written', value: 'manual' }, + ], + access: { + create: () => false, + update: () => false, + }, + admin: { + position: 'sidebar', + readOnly: true, + description: 'Who wrote the alt text. Editing the alt text marks it hand-written.', + }, }, { name: 'caption', diff --git a/src/collections/Photos/altText/GenerateAltTextButton.tsx b/src/collections/Photos/altText/GenerateAltTextButton.tsx new file mode 100644 index 0000000..e574c49 --- /dev/null +++ b/src/collections/Photos/altText/GenerateAltTextButton.tsx @@ -0,0 +1,91 @@ +'use client' + +import { Button, toast, useConfig, useDocumentInfo, useForm } from '@payloadcms/ui' +import { formatAdminURL } from 'payload/shared' +import React, { useCallback, useState } from 'react' + +type Props = { + /** Whether Workers AI is configured, resolved on the server (see the `ui` field in Photos.ts). */ + enabled?: boolean +} + +export const GenerateAltTextButton: React.FC = ({ enabled }) => { + const { id } = useDocumentInfo() + const { + config: { + routes: { api }, + serverURL, + }, + } = useConfig() + // Read at click time through the form's getters instead of subscribing to field state, so + // typing in the form never re-renders this button. + const { dispatchFields, getField } = useForm() + const [loading, setLoading] = useState(false) + + const handleClick = useCallback(async () => { + const field = getField('alt') + const current = typeof field?.value === 'string' ? field.value.trim() : '' + // Ask before replacing anything a person wrote: saved hand-written text, or unsaved typing. + const isHandWritten = + current !== '' && + (getField('altSource')?.value === 'manual' || field?.value !== field?.initialValue) + if ( + isHandWritten && + !window.confirm('Replace the hand-written alt text with a generated description?') + ) { + return + } + + setLoading(true) + try { + const res = await fetch( + formatAdminURL({ apiRoute: api, path: `/photos/${id}/alt-text`, serverURL }), + { method: 'POST', credentials: 'include' }, + ) + const body = (await res.json().catch(() => null)) as { + alt?: unknown + error?: unknown + } | null + + if (!res.ok || typeof body?.alt !== 'string') { + throw new Error( + typeof body?.error === 'string' ? body.error : `Request failed (HTTP ${res.status})`, + ) + } + + // The endpoint has already saved this, so mirror it into the form as the new baseline: + // dispatching directly (rather than `useField().setValue`) leaves `modified` untouched. + dispatchFields({ type: 'UPDATE', initialValue: body.alt, path: 'alt', value: body.alt }) + dispatchFields({ type: 'UPDATE', initialValue: 'ai', path: 'altSource', value: 'ai' }) + toast.success('Alt text regenerated.') + } catch (err) { + toast.error( + `Could not regenerate alt text: ${err instanceof Error ? err.message : String(err)}`, + ) + } finally { + setLoading(false) + } + }, [api, dispatchFields, getField, id, serverURL]) + + // Nothing to describe until the photo exists (create view, bulk upload). + if (!enabled || !id) return null + + return ( + // `field-type` gives this the same bottom spacing as a real field; the negative top margin + // pulls it up under the alt input it belongs to. +
+ +
+ ) +} diff --git a/src/collections/Photos/altText/endpoint.ts b/src/collections/Photos/altText/endpoint.ts new file mode 100644 index 0000000..b45967f --- /dev/null +++ b/src/collections/Photos/altText/endpoint.ts @@ -0,0 +1,49 @@ +import type { Endpoint } from 'payload' + +import { generatePhotoAltText } from './generate' +import { isAltTextConfigured } from './workersAI' + +/** + * `POST /api/photos/:id/alt-text` — the admin "Regenerate alt text" button. Unlike the other + * entry points this replaces whatever is there, hand-written text included, and saves it: the + * click is the consent, so the response is the stored value rather than a suggestion. + */ +export const generateAltTextEndpoint: Endpoint = { + path: '/:id/alt-text', + method: 'post', + handler: async (req) => { + // Same bar as the collection's `update` access (any authenticated user). + if (!req.user) return Response.json({ error: 'Unauthorized' }, { status: 401 }) + + if (!isAltTextConfigured()) { + return Response.json( + { + error: + 'Alt text generation is not configured: set CLOUDFLARE_ACCOUNT_ID and WORKERS_AI_API_TOKEN.', + }, + { status: 503 }, + ) + } + + const id = req.routeParams?.id + if (typeof id !== 'string' || !id) { + return Response.json({ error: 'Photo not found' }, { status: 404 }) + } + + try { + const result = await generatePhotoAltText({ id, overwrite: 'always', req }) + // With `overwrite: 'always'` the only way to be skipped is a missing photo. + if (result.status === 'skipped') { + return Response.json({ error: 'Photo not found' }, { status: 404 }) + } + return Response.json({ alt: result.alt, altSource: 'ai' }) + } catch (err) { + // The stack goes to the log; the client only gets the message. + req.payload.logger.error({ err, msg: `alt text: regeneration failed for photo ${id}` }) + return Response.json( + { error: err instanceof Error ? err.message : String(err) }, + { status: 502 }, + ) + } + }, +} diff --git a/src/collections/Photos/altText/generate.ts b/src/collections/Photos/altText/generate.ts new file mode 100644 index 0000000..188caff --- /dev/null +++ b/src/collections/Photos/altText/generate.ts @@ -0,0 +1,148 @@ +import type { Payload, PayloadRequest, RequestContext } from 'payload' + +import { isolateObjectProperty } from 'payload' + +import type { Location, Photo, PhotoTag } from '@/payload-types' + +import { loadStoredPhoto, toModelJpeg } from './image' +import { type AltTextHints, buildAltTextQuestion, cleanAltText } from './prompt' +import { queryImage } from './workersAI' + +/** + * Which existing alt text a run may replace: + * - `empty` only fill photos that have none + * - `ai` also replace text a previous run generated (never hand-written text) + * - `always` replace whatever is there (an explicit "Regenerate" click) + */ +export type AltTextOverwrite = 'ai' | 'always' | 'empty' + +export type AltTextResult = + | { alt: string; status: 'generated' } + | { reason: 'has-alt' | 'not-found'; status: 'skipped' } + +const mayReplace = ( + photo: Pick, + overwrite: AltTextOverwrite, +): boolean => + !photo.alt?.trim() || overwrite === 'always' || (overwrite === 'ai' && photo.altSource === 'ai') + +type Named = Pick +type Ref = Named | null | string | undefined + +/** + * Turn the photo's tag/location relationships into names for the prompt. Accepts ids (hook + * data) or populated docs. Hints are a nice-to-have, so any lookup failure just drops them. + */ +export async function resolveAltTextHints( + payload: Payload, + { location, tags }: { location?: Ref; tags?: null | Ref[] }, + req?: PayloadRequest, +): Promise { + try { + const tagRefs = (tags ?? []).filter((tag): tag is Named | string => Boolean(tag)) + const tagIds = tagRefs.filter((tag): tag is string => typeof tag === 'string') + const tagNames = tagRefs.flatMap((tag) => (typeof tag === 'string' ? [] : [tag.name])) + if (tagIds.length) { + const { docs } = await payload.find({ + collection: 'photo-tags', + depth: 0, + limit: tagIds.length, + pagination: false, + req, + where: { id: { in: tagIds } }, + }) + tagNames.push(...docs.map((tag) => tag.name)) + } + + const locationDoc = + typeof location === 'string' + ? await payload.findByID({ + collection: 'locations', + id: location, + depth: 0, + disableErrors: true, + req, + }) + : location + + return { location: locationDoc?.name ?? null, tags: tagNames.filter(Boolean) } + } catch (err) { + payload.logger.warn(`alt text: could not resolve prompt hints: ${String(err)}`) + return {} + } +} + +/** Describe an image in any format sharp can read. Throws when the model gives nothing usable. */ +export async function describeImage(image: Buffer, hints?: AltTextHints): Promise { + const answer = await queryImage({ + jpeg: await toModelJpeg(image), + question: buildAltTextQuestion(hints), + }) + const alt = cleanAltText(answer) + if (!alt) throw new Error(`model answer was not usable as alt text: ${JSON.stringify(answer)}`) + return alt +} + +/** + * Generate and save alt text for a photo that is already stored. Used by everything except + * the upload itself (which describes the in-memory file, see `generateAltTextOnUpload`): + * the retry task, the admin "Regenerate" endpoint and the backfill script. + */ +export async function generatePhotoAltText({ + id, + context, + overwrite, + req, +}: { + /** Extra hook context for the write, e.g. `disableRevalidate` outside a Next request. */ + context?: RequestContext + id: string + overwrite: AltTextOverwrite + req: PayloadRequest +}): Promise { + const { payload } = req + + const photo = await payload.findByID({ + collection: 'photos', + id, + depth: 1, + disableErrors: true, + req, + }) + if (!photo) return { reason: 'not-found', status: 'skipped' } + + if (!mayReplace(photo, overwrite)) return { reason: 'has-alt', status: 'skipped' } + + const hints = await resolveAltTextHints(payload, photo, req) + const alt = await describeImage(await loadStoredPhoto(payload, photo), hints) + + // The model call takes seconds. Look again before writing so text someone saved in the + // meantime is not replaced. + const current = await payload.findByID({ + collection: 'photos', + id, + depth: 0, + disableErrors: true, + req, + select: { alt: true, altSource: true }, + }) + if (!current) return { reason: 'not-found', status: 'skipped' } + if (!mayReplace(current, overwrite)) return { reason: 'has-alt', status: 'skipped' } + + await payload.update({ + collection: 'photos', + id, + // `altTextGenerated` tells `generateAltTextOnUpload` this write is ours, not a hand edit. + context: { ...context, altTextGenerated: true }, + data: { alt, altSource: 'ai' }, + depth: 0, + // Payload merges `context` into the req it is given, and ours is shared (the jobs of one + // run execute in parallel on one req; the backfill reuses one). Give this write its own + // `context` so the flag can't reach, or be cleared under, any other write. + // Relies on the Local API default `overrideAccess: true`: field access makes `altSource` + // read-only for everyone else. + req: isolateObjectProperty(req, 'context'), + }) + + return { alt, status: 'generated' } +} diff --git a/src/collections/Photos/altText/hooks.ts b/src/collections/Photos/altText/hooks.ts new file mode 100644 index 0000000..eef9fbf --- /dev/null +++ b/src/collections/Photos/altText/hooks.ts @@ -0,0 +1,108 @@ +import type { CollectionAfterChangeHook, CollectionBeforeChangeHook } from 'payload' + +import type { Photo } from '@/payload-types' + +import { describeImage, resolveAltTextHints } from './generate' +import { PREFERRED_SIZES } from './image' +import { ALT_TEXT_QUEUE, ALT_TEXT_TASK } from './task' +import { isAltTextConfigured, WorkersAIUnavailableError } from './workersAI' + +// After Workers AI itself fails, skip the inline attempt for a while and go straight to the +// retry queue, so an outage doesn't add a timeout to every file of a bulk upload. +const INLINE_COOLDOWN_MS = 2 * 60_000 +let inlineDisabledUntil = 0 + +// Give the upload (and its R2 write, which runs in a later afterChange hook) time to finish +// before the retry task goes looking for the stored file. +const RETRY_DELAY_MS = 60_000 + +/** + * Keeps `alt` / `altSource` consistent on every write, and on upload fills a blank `alt` from + * the vision model. + * + * The upload is described inline, from the rendition Payload has just generated in memory, + * rather than from a queued job. That way the alt text is part of the create itself: the edit + * form the admin lands on already holds it (a job finishing later would be overwritten by the + * next save of that stale form), and the create's own `revalidatePhoto` covers it. Only when + * the inline attempt fails does `queueAltTextRetry` hand the photo to the jobs queue. + */ +export const generateAltTextOnUpload: CollectionBeforeChangeHook = async ({ + data, + operation, + originalDoc, + req, +}) => { + // `generatePhotoAltText` wrote this and set `altSource` itself. + if (req.context.altTextGenerated) return data + + const alt = typeof data.alt === 'string' ? data.alt.trim() : '' + + if (operation === 'update') { + if ('alt' in data && alt !== (originalDoc?.alt ?? '').trim()) { + data.altSource = alt ? 'manual' : null + } + return data + } + + // A duplicate arrives with its source's text (`originalDoc` is the source); keep that + // text's provenance instead of calling a copied AI description hand-written. + const copied = alt !== '' && alt === (originalDoc?.alt ?? '').trim() + data.altSource = copied ? (originalDoc?.altSource ?? null) : alt ? 'manual' : null + if (alt || !isAltTextConfigured()) return data + + const sizes = req.payloadUploadSizes ?? {} + const image = + PREFERRED_SIZES.map((size) => sizes[size]).find((buffer) => buffer?.length) ?? req.file?.data + + if (image?.length && Date.now() >= inlineDisabledUntil) { + try { + const hints = await resolveAltTextHints(req.payload, data, req) + data.alt = await describeImage(image, hints) + data.altSource = 'ai' + return data + } catch (err) { + if (err instanceof WorkersAIUnavailableError) { + inlineDisabledUntil = Date.now() + INLINE_COOLDOWN_MS + } + req.payload.logger.warn( + `alt text: generation failed during upload, queueing a retry: ${err instanceof Error ? err.message : String(err)}`, + ) + } + } + + // No usable bytes, in cooldown, or the attempt failed. Set on `req.context`, not a `context` + // captured earlier: every nested Local API call above replaces `req.context` with a new + // object, and the afterChange hook is handed the current one. + req.context.altTextPending = true + return data +} + +/** Hands a photo whose inline generation didn't happen to the jobs queue. */ +export const queueAltTextRetry: CollectionAfterChangeHook = async ({ + doc, + operation, + req, +}) => { + if (operation !== 'create' || !req.context.altTextPending) return doc + // One-shot: a `req` reused for several creates (the seed, import scripts) must not queue a + // job for every later photo. + delete req.context.altTextPending + + // Never throw from here: @payloadcms/plugin-cloud-storage registers the R2 upload as a later + // afterChange hook, so a throw would leave the document without its files. + try { + await req.payload.jobs.queue({ + input: { photoId: String(doc.id) }, + queue: ALT_TEXT_QUEUE, + req, + task: ALT_TEXT_TASK, + waitUntil: new Date(Date.now() + RETRY_DELAY_MS), + }) + } catch (err) { + req.payload.logger.error( + `alt text: could not queue a retry for photo ${doc.id}: ${String(err)}`, + ) + } + + return doc +} diff --git a/src/collections/Photos/altText/image.ts b/src/collections/Photos/altText/image.ts new file mode 100644 index 0000000..375cf38 --- /dev/null +++ b/src/collections/Photos/altText/image.ts @@ -0,0 +1,67 @@ +import type { Payload } from 'payload' + +import fs from 'fs/promises' +import path from 'path' +import sharp from 'sharp' + +import type { Photo } from '@/payload-types' + +import { getServerSideURL } from '@/utilities/getURL' + +// Smallest rendition that still shows enough detail to describe; larger ones only cost tokens. +// Not every size exists for every photo (small sources skip the bigger renditions). +export const PREFERRED_SIZES = ['medium', 'large', 'small', 'xlarge'] as const + +// A custom domain in front of R2 with bot protection answers bare clients with 403. +const ASSET_HEADERS = { 'User-Agent': 'photo-alt-text/1.0' } + +/** + * Re-encode whatever we have (WebP rendition, AVIF/TIFF original, animated GIF…) as a modest + * JPEG, so the model only ever sees one well-supported format at a predictable size. + */ +export const toModelJpeg = (input: Buffer): Promise => + sharp(input) + .rotate() + .resize({ width: 1024, height: 1024, fit: 'inside', withoutEnlargement: true }) + .flatten({ background: '#ffffff' }) + .jpeg({ quality: 85 }) + .toBuffer() + +type StoredFile = { filename?: string | null; url?: string | null } + +/** Read a stored photo from local disk when it is there, otherwise from its public URL. */ +export async function loadStoredPhoto(payload: Payload, photo: Photo): Promise { + const candidates: StoredFile[] = [ + ...PREFERRED_SIZES.map((size) => photo.sizes?.[size]), + photo, + ].filter((file): file is StoredFile => Boolean(file?.filename)) + + const staticDir = payload.collections.photos.config.upload.staticDir + const errors: string[] = [] + + for (const { filename, url } of candidates) { + if (staticDir && filename) { + try { + return await fs.readFile(path.join(staticDir, filename)) + } catch { + /* not stored locally (R2) — fall through to HTTP */ + } + } + if (!url) continue + const absolute = url.startsWith('http') ? url : `${getServerSideURL().replace(/\/$/, '')}${url}` + try { + const res = await fetch(absolute, { + headers: ASSET_HEADERS, + signal: AbortSignal.timeout(20_000), + }) + if (res.ok) return Buffer.from(await res.arrayBuffer()) + errors.push(`GET ${absolute} -> ${res.status}`) + } catch (err) { + errors.push(`GET ${absolute} -> ${err instanceof Error ? err.message : String(err)}`) + } + } + + throw new Error( + `could not read photo ${photo.id} from storage: ${errors.join(', ') || 'no readable file'}`, + ) +} diff --git a/src/collections/Photos/altText/prompt.ts b/src/collections/Photos/altText/prompt.ts new file mode 100644 index 0000000..1a06aed --- /dev/null +++ b/src/collections/Photos/altText/prompt.ts @@ -0,0 +1,73 @@ +/** What we already know about a photo that the model can't see in the pixels. */ +export type AltTextHints = { + location?: string | null + tags?: string[] +} + +const MAX_ALT_LENGTH = 200 + +export const buildAltTextQuestion = ({ location, tags }: AltTextHints = {}): string => { + const hints = [ + location && `Location: ${location}.`, + tags?.length && `Tags: ${tags.join(', ')}.`, + ].filter(Boolean) + + return [ + 'Write alt text for this photograph for a screen reader user.', + 'Describe the main subject and setting in one plain sentence of at most 25 words.', + 'Do not start with "Image of" or "Photo of". Do not mention the camera or photographer.', + hints.length && `Known context, use it only if it matches what is visible: ${hints.join(' ')}`, + ] + .filter(Boolean) + .join(' ') +} + +// "A photo of", "This image shows", "The picture depicts" … screen readers already announce +// "image", so the lead-in is noise. +const LEAD_IN = + /^(?:this is\s+)?(?:(?:this|the|an?)\s+)?(?:image|photo|photograph|picture)\s+(?:is\s+)?(?:of|shows|showing|depicts|depicting|features|featuring|captures|capturing)\s+/i + +// Only a quote pair around the whole answer. Double quotes inside mean the outer two are not a +// pair (`"Open" sign next to one reading "Closed"`). +const unquote = (text: string) => + text + .replace(/^["“]([^"“”]*)["”]$/, '$1') + .replace(/^['‘](.*)['’]$/, '$1') + .trim() + +// "I'm sorry, but I can't describe this image." is an answer, but not a description. +const REFUSAL = + /^(?:i['’]?m sorry|i am sorry|sorry[,.]|i (?:cannot|can['’]?t|am unable|am not able)\b)/i + +// "Alt text:", "Alt-text:", "**Alt:**" … +const LABEL = /^\**\s*alt(?:ernative)?(?:[\s-]*text)?\s*:\**\s*/i + +// A sentence cut shorter than this is more likely an abbreviation ("Mt.") than a description. +const MIN_SENTENCE_CUT = 60 + +const truncate = (text: string): string => { + if (text.length <= MAX_ALT_LENGTH) return text + + // Prefer whole sentences: the last sentence end inside the limit that is followed by a space. + const head = text.slice(0, MAX_ALT_LENGTH) + let cut = -1 + for (const match of head.matchAll(/[.!?](?=\s)/g)) cut = match.index + 1 + if (text[MAX_ALT_LENGTH] === ' ' && /[.!?]$/.test(head)) cut = MAX_ALT_LENGTH + if (cut >= MIN_SENTENCE_CUT) return text.slice(0, cut) + + // Otherwise a word boundary, leaving room for the ellipsis. + return `${text.slice(0, MAX_ALT_LENGTH - 1).replace(/\s+\S*$/, '')}…` +} + +/** Normalise a raw model answer into alt text. Returns '' when nothing usable is left. */ +export const cleanAltText = (raw: string): string => { + let text = unquote(raw.replace(/\s+/g, ' ').trim()) + // The label can sit inside or outside the quotes, so unquote on both sides of removing it. + text = unquote(text.replace(LABEL, '')) + text = text.replace(LEAD_IN, '') + // Stray punctuation or quotes are not a description. + if (!/[\p{L}\p{N}]/u.test(text) || REFUSAL.test(text)) return '' + + text = truncate(text) + return text.charAt(0).toUpperCase() + text.slice(1) +} diff --git a/src/collections/Photos/altText/task.ts b/src/collections/Photos/altText/task.ts new file mode 100644 index 0000000..6783895 --- /dev/null +++ b/src/collections/Photos/altText/task.ts @@ -0,0 +1,29 @@ +import type { TaskConfig } from 'payload' + +import { generatePhotoAltText } from './generate' + +export const ALT_TEXT_TASK = 'generateAltText' + +// A dedicated queue so `jobs.autoRun` (payload.config.ts) can drain these without also picking +// up the default queue, which carries scheduled publishes. +export const ALT_TEXT_QUEUE = 'alt-text' + +/** + * Retry path for uploads whose inline alt text generation failed (see `generateAltTextOnUpload`). + * Only ever fills a blank `alt`: anything written since the job was queued wins. + * + * Caveat: the autoRun cron is not a Next request, so the `revalidatePhoto` hook may not be able + * to revalidate cache tags (it logs a warning when it can't). The sitemap and new-tab manifest + * then pick the text up on the next photo change. + */ +export const generateAltTextTask: TaskConfig = { + slug: ALT_TEXT_TASK, + label: 'Generate photo alt text', + inputSchema: [{ name: 'photoId', type: 'text', required: true }], + outputSchema: [{ name: 'status', type: 'text' }], + retries: { attempts: 4, backoff: { type: 'exponential', delay: 2 * 60_000 } }, + handler: async ({ input, req }) => { + const result = await generatePhotoAltText({ id: input.photoId, overwrite: 'empty', req }) + return { output: { status: result.status } } + }, +} diff --git a/src/collections/Photos/altText/workersAI.ts b/src/collections/Photos/altText/workersAI.ts new file mode 100644 index 0000000..4c10220 --- /dev/null +++ b/src/collections/Photos/altText/workersAI.ts @@ -0,0 +1,80 @@ +// Workers AI over REST: the site runs in Docker, not on Workers, so there is no `env.AI` binding. +// https://developers.cloudflare.com/workers-ai/models/moondream3.1-9B-A2B/ +export const ALT_TEXT_MODEL = '@cf/moondream/moondream3.1-9B-A2B' + +const REQUEST_TIMEOUT_MS = 20_000 + +type WorkersAIResponse = { + errors?: { code?: number; message?: string }[] + result?: { answer?: string | null; finish_reason?: string } + success?: boolean +} + +/** The service is down, timing out or rate limiting, as opposed to rejecting one request. */ +export class WorkersAIUnavailableError extends Error {} + +/** Alt text generation is opt-in: without both vars every entry point is a no-op. */ +export const isAltTextConfigured = (): boolean => + Boolean(process.env.CLOUDFLARE_ACCOUNT_ID && process.env.WORKERS_AI_API_TOKEN) + +/** Ask the vision model one question about a JPEG. Returns the raw answer; throws on any failure. */ +export async function queryImage({ + jpeg, + question, +}: { + jpeg: Buffer + question: string +}): Promise { + if (!isAltTextConfigured()) { + throw new Error('Workers AI is not configured (CLOUDFLARE_ACCOUNT_ID, WORKERS_AI_API_TOKEN)') + } + + let res: Response + try { + res = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${process.env.CLOUDFLARE_ACCOUNT_ID}/ai/run/${ALT_TEXT_MODEL}`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env.WORKERS_AI_API_TOKEN}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + task: 'query', + image: `data:image/jpeg;base64,${jpeg.toString('base64')}`, + question, + // The reasoning trace defaults to on; it costs output tokens and latency we don't need. + reasoning: false, + temperature: 0.2, + max_tokens: 120, + }), + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }, + ) + } catch (err) { + // Network failure or timeout. Report the error's name and code, never its message: a + // malformed token makes fetch throw with the whole Authorization header in the message. + const reason = + err instanceof Error + ? [err.name, (err.cause as { code?: string } | undefined)?.code].filter(Boolean).join(' ') + : 'unknown error' + throw new WorkersAIUnavailableError(`Workers AI request failed: ${reason}`) + } + + const body = (await res.json().catch(() => null)) as WorkersAIResponse | null + if (!res.ok || !body?.success) { + const detail = body?.errors + ?.map((e) => e.message) + .filter(Boolean) + .join('; ') + const message = `Workers AI request failed (HTTP ${res.status})${detail ? `: ${detail}` : ''}` + // A 4xx is about this request (bad token, rejected image), not an outage. + throw res.status === 429 || res.status >= 500 + ? new WorkersAIUnavailableError(message) + : new Error(message) + } + + const answer = body.result?.answer?.trim() + if (!answer) throw new Error('Workers AI returned no answer') + return answer +} diff --git a/src/payload-types.ts b/src/payload-types.ts index a929e33..159c58b 100644 --- a/src/payload-types.ts +++ b/src/payload-types.ts @@ -135,6 +135,7 @@ export interface Config { user: User; jobs: { tasks: { + generateAltText: TaskGenerateAltText; schedulePublish: TaskSchedulePublish; inline: { input: unknown; @@ -267,7 +268,14 @@ export interface Photo { * Location where the photo was taken. Used for filtering in collections. */ location?: (string | null) | Location; + /** + * Left blank on upload, this is written by a vision model (when Workers AI is configured). + */ alt?: string | null; + /** + * Who wrote the alt text. Editing the alt text marks it hand-written. + */ + altSource?: ('ai' | 'manual') | null; caption?: { root: { type: string; @@ -1398,7 +1406,7 @@ export interface PayloadJob { | { executedAt: string; completedAt: string; - taskSlug: 'inline' | 'schedulePublish'; + taskSlug: 'inline' | 'generateAltText' | 'schedulePublish'; taskID: string; input?: | { @@ -1431,7 +1439,7 @@ export interface PayloadJob { id?: string | null; }[] | null; - taskSlug?: ('inline' | 'schedulePublish') | null; + taskSlug?: ('inline' | 'generateAltText' | 'schedulePublish') | null; queue?: string | null; waitUntil?: string | null; processing?: boolean | null; @@ -1776,6 +1784,7 @@ export interface PhotosSelect { tags?: T; location?: T; alt?: T; + altSource?: T; caption?: T; showInNewTab?: T; exif?: @@ -2475,6 +2484,18 @@ export interface LlmsTxtSettingsSelect { createdAt?: T; globalType?: T; } +/** + * This interface was referenced by `Config`'s JSON-Schema + * via the `definition` "TaskGenerateAltText". + */ +export interface TaskGenerateAltText { + input: { + photoId: string; + }; + output: { + status?: string | null; + }; +} /** * This interface was referenced by `Config`'s JSON-Schema * via the `definition` "TaskSchedulePublish". diff --git a/src/payload.config.ts b/src/payload.config.ts index 0ce881a..4a3f770 100644 --- a/src/payload.config.ts +++ b/src/payload.config.ts @@ -8,6 +8,7 @@ import { Career } from './collections/Career' import { Categories } from './collections/Categories' import { Locations } from './collections/Locations' import { Photos } from './collections/Photos' +import { ALT_TEXT_QUEUE, generateAltTextTask } from './collections/Photos/altText/task' import { PhotoCollections } from './collections/PhotoCollections' import { Tags } from './collections/Tags' import { Pages } from './collections/Pages' @@ -125,6 +126,9 @@ export default buildConfig({ return authHeader === `Bearer ${secret}` }, }, - tasks: [], + tasks: [generateAltTextTask], + // Drains only the alt text retry queue. Runs in-process, which suits the long-lived + // Docker server (and is skipped during `next build`). + autoRun: [{ cron: '*/2 * * * *', limit: 10, queue: ALT_TEXT_QUEUE }], }, }) diff --git a/tests/int/photoAltText.int.spec.ts b/tests/int/photoAltText.int.spec.ts new file mode 100644 index 0000000..4d95c19 --- /dev/null +++ b/tests/int/photoAltText.int.spec.ts @@ -0,0 +1,1560 @@ +// @vitest-environment node +import type { Payload, PayloadRequest } from 'payload' + +import fs from 'fs/promises' +import os from 'os' +import path from 'path' +import sharp from 'sharp' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { Photo } from '@/payload-types' + +import { generatePhotoAltText, resolveAltTextHints } from '@/collections/Photos/altText/generate' +import { loadStoredPhoto, toModelJpeg } from '@/collections/Photos/altText/image' +import { buildAltTextQuestion, cleanAltText } from '@/collections/Photos/altText/prompt' +import { + ALT_TEXT_QUEUE, + ALT_TEXT_TASK, + generateAltTextTask, +} from '@/collections/Photos/altText/task' +import { + ALT_TEXT_MODEL, + isAltTextConfigured, + queryImage, + WorkersAIUnavailableError, +} from '@/collections/Photos/altText/workersAI' + +const ACCOUNT = 'acct-123' +const TOKEN = 'tok-abc' +const SITE_URL = 'https://site.example.test' +const MEDIA = 'https://media.example.test/photos/abc' +const MAX_ALT_LENGTH = 200 + +const fetchMock = vi.fn<(url: string, init?: RequestInit) => Promise>() +const logger = { error: vi.fn(), info: vi.fn(), warn: vi.fn() } + +const configure = () => { + vi.stubEnv('CLOUDFLARE_ACCOUNT_ID', ACCOUNT) + vi.stubEnv('WORKERS_AI_API_TOKEN', TOKEN) +} + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) + +const modelAnswers = (answer: null | string) => + fetchMock.mockImplementation(async () => json({ result: { answer }, success: true })) + +const modelFails = (status = 500, message = 'upstream exploded') => + fetchMock.mockImplementation(async () => + json({ errors: [{ code: 5000, message }], result: null, success: false }, status), + ) + +const sent = (call = 0) => { + const [url, init] = fetchMock.mock.calls[call] ?? [] + return { body: JSON.parse(String(init?.body)) as Record, init, url } +} + +const sentImage = (call = 0) => + Buffer.from(String(sent(call).body.image).replace(/^data:image\/jpeg;base64,/, ''), 'base64') + +const describeBuffer = async (image: Buffer) => { + const { format, height, width } = await sharp(image).metadata() + const [r, g, b] = (await sharp(image).stats()).channels.map((c) => Math.round(c.mean)) + return { b, format, g, height, r, width } +} + +const solid = (width: number, height: number, background: string) => + sharp({ create: { width, height, background, channels: 3 } }) + +const words = (count: number) => Array.from({ length: count }, (_, i) => `word${i}`).join(' ') + +/** A single "sentence" of exactly `length` characters, ending in a full stop. */ +const sentenceOfLength = (length: number) => + `${words(60) + .slice(0, length - 1) + .replace(/\s$/, 'x')}.` + +type FakePhoto = Partial> & { + location?: unknown + tags?: unknown +} + +type LocalArgs = { context?: Record; req?: PayloadRequest } + +// What Payload's createLocalReq does on every Local API call: `req.context` becomes a NEW +// object, merged with the call's `context`. Anything holding the old object goes stale. +const replaceContext = ({ context, req }: LocalArgs) => { + if (req) req.context = { ...req.context, ...context } +} + +const fakePayload = ({ + photo = null, + staticDir, +}: { photo?: FakePhoto | null; staticDir?: string } = {}) => ({ + collections: { photos: { config: { upload: { staticDir } } } }, + find: vi.fn(async (args: LocalArgs) => { + replaceContext(args) + return { + docs: [ + { id: 'tag-1', name: 'Surfing' }, + { id: 'tag-2', name: 'Winter' }, + ], + } + }), + findByID: vi.fn(async (args: { collection: string; id: string } & LocalArgs) => { + replaceContext(args) + if (args.collection === 'photos') return photo + if (args.collection === 'locations') return { id: args.id, name: 'Tofino' } + return null + }), + jobs: { queue: vi.fn(async (_args: Record) => ({ id: 'job-1' })) }, + logger, + update: vi.fn(async (args: LocalArgs) => { + replaceContext(args) + return {} + }), +}) + +type FakePayload = ReturnType + +const fakeReq = (payload: FakePayload = fakePayload(), extra: Record = {}) => + ({ context: {}, payload, ...extra }) as unknown as PayloadRequest + +let tmp: string +let staticDir: string +let emptyDir: string +let original: Buffer +let medium: Buffer + +beforeAll(async () => { + tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'photo-alt-text-')) + staticDir = path.join(tmp, 'photos') + emptyDir = path.join(tmp, 'empty') + await fs.mkdir(staticDir) + await fs.mkdir(emptyDir) + + // Distinguishable by colour and by size: the original is blue, the medium rendition red. + original = await solid(1600, 1200, '#0000ff').jpeg().toBuffer() + medium = await solid(900, 675, '#ff0000').webp().toBuffer() + await fs.writeFile(path.join(staticDir, 'pic.jpg'), original) + await fs.writeFile(path.join(staticDir, 'pic-900.webp'), medium) +}) + +afterAll(async () => { + await fs.rm(tmp, { force: true, recursive: true }) +}) + +beforeEach(() => { + vi.clearAllMocks() + fetchMock.mockReset() + fetchMock.mockRejectedValue(new Error('unexpected fetch')) + vi.stubGlobal('fetch', fetchMock) + // `.env` is loaded by the setup file, so start every test unconfigured. + vi.stubEnv('CLOUDFLARE_ACCOUNT_ID', '') + vi.stubEnv('WORKERS_AI_API_TOKEN', '') + vi.stubEnv('NEXT_PUBLIC_SERVER_URL', SITE_URL) +}) + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllEnvs() + vi.unstubAllGlobals() +}) + +describe('cleanAltText', () => { + it('collapses whitespace', () => { + expect(cleanAltText(' A dog\n runs\t across a beach. \n')).toBe('A dog runs across a beach.') + expect(cleanAltText('A dog on a beach.\n\nThe dog is brown.')).toBe( + 'A dog on a beach. The dog is brown.', + ) + }) + + it.each([ + '"A dog on a beach."', + "'A dog on a beach.'", + '“A dog on a beach.”', + '‘A dog on a beach.’', + ' "A dog on a beach." ', + ])('strips wrapping quotes: %s', (raw) => { + expect(cleanAltText(raw)).toBe('A dog on a beach.') + }) + + it.each([ + 'A neon sign that reads "Open"', + '"Open" sign hanging in a shop window', + 'A sign reading "Open" hangs on a door.', + "A dog's bowl beside the neighbours'", + ])('keeps quotes and apostrophes that belong to the sentence: %s', (raw) => { + expect(cleanAltText(raw)).toBe(raw) + }) + + it('does not strip a leading and trailing quote that are not a pair', () => { + const raw = '"Open" sign next to one reading "Closed"' + expect(cleanAltText(raw)).toBe(raw) + }) + + it.each([ + 'Alt text: A dog on a beach.', + 'ALT TEXT:A dog on a beach.', + 'alt text : A dog on a beach.', + 'Alternative text: A dog on a beach.', + 'Alt text: "A dog on a beach."', + '"Alt text: A dog on a beach."', + '**Alt text:** A dog on a beach.', + 'Alt-text: A dog on a beach.', + 'Alt: A dog on a beach.', + ])('strips the label: %s', (raw) => { + expect(cleanAltText(raw)).toBe('A dog on a beach.') + }) + + it.each([ + 'A photo of a dog on a beach.', + 'Photo of a dog on a beach.', + 'An image of a dog on a beach.', + 'This image shows a dog on a beach.', + 'The picture depicts a dog on a beach.', + 'The photograph captures a dog on a beach.', + 'THIS IMAGE SHOWS a dog on a beach.', + 'This is a photo of a dog on a beach.', + 'The image is of a dog on a beach.', + 'Alt text: "A photo of a dog on a beach."', + ])('strips the lead-in and re-capitalises: %s', (raw) => { + expect(cleanAltText(raw)).toBe('A dog on a beach.') + }) + + it('capitalises the first letter only', () => { + expect(cleanAltText('a dog chases a Frisbee near Tofino.')).toBe( + 'A dog chases a Frisbee near Tofino.', + ) + }) + + it.each([ + 'A photographer on a ridge at dawn.', + 'Photographs of birds hang in a gallery.', + 'Picture frames cover a hallway wall.', + 'The photo booth at a county fair.', + 'An imageless grey wall.', + 'Two hikers picture the summit ahead of them.', + 'Alto saxophone: a player on a street corner.', + ])('leaves an ordinary sentence alone: %s', (raw) => { + expect(cleanAltText(raw)).toBe(raw) + }) + + it('keeps text at the length limit as it is', () => { + const raw = sentenceOfLength(MAX_ALT_LENGTH) + expect(raw).toHaveLength(MAX_ALT_LENGTH) + expect(cleanAltText(raw).toLowerCase()).toBe(raw) + }) + + it('cuts over-long text back to whole sentences', () => { + const first = `A hiker crosses ${words(10)}.` + const second = `Behind them ${words(10)}!` + const third = `Clouds gather ${words(10)}.` + const raw = `${first} ${second} ${third}` + expect(raw.length).toBeGreaterThan(MAX_ALT_LENGTH) + + expect(cleanAltText(raw)).toBe(`${first} ${second}`) + }) + + it('keeps a sentence that ends exactly at the limit', () => { + const first = sentenceOfLength(MAX_ALT_LENGTH) + expect(cleanAltText(`${first} More words follow here.`).toLowerCase()).toBe(first) + }) + + it('cuts over-long text without a sentence break at a word boundary', () => { + const raw = words(60) + const alt = cleanAltText(raw) + const kept = alt.slice(0, -1) + + expect(alt.endsWith('…')).toBe(true) + expect(alt.length).toBeLessThanOrEqual(MAX_ALT_LENGTH) + expect(kept.length).toBeGreaterThan(MAX_ALT_LENGTH - 20) + expect(kept).toBe(kept.trimEnd()) + // Whole words only: the cut is followed by a space in the source text. + expect(raw.toLowerCase().startsWith(`${kept.toLowerCase()} `)).toBe(true) + }) + + it.each([ + ['a 201-character sentence', sentenceOfLength(MAX_ALT_LENGTH + 1)], + [ + 'a sentence ending on character 201, then more', + `${sentenceOfLength(MAX_ALT_LENGTH + 1)} More words follow here.`, + ], + ])('never returns more than 200 characters: %s', (_name, raw) => { + expect(cleanAltText(raw).length).toBeLessThanOrEqual(MAX_ALT_LENGTH) + }) + + it('never returns more than 200 characters: one unbroken token', () => { + expect(cleanAltText('x'.repeat(250)).length).toBeLessThanOrEqual(MAX_ALT_LENGTH) + }) + + it('does not cut an over-long answer down to a leading abbreviation', () => { + const alt = cleanAltText(`Mt. Rainier rises above ${words(40)}`) + expect(alt.startsWith('Mt. Rainier rises above')).toBe(true) + }) + + it.each(['', ' ', '\n\t', '""', "''", '“”', '" "', 'Alt text:', 'Alt text: ""'])( + 'returns an empty string when nothing usable is left: %j', + (raw) => { + expect(cleanAltText(raw)).toBe('') + }, + ) + + it.each([ + "I'm sorry, but I can't describe this image.", + 'I’m sorry, I cannot help with that.', + 'Sorry, I am unable to view images.', + 'I cannot describe people in images.', + "I can't see an image.", + ])('returns an empty string for a refusal: %s', (raw) => { + expect(cleanAltText(raw)).toBe('') + }) + + it('keeps a description that merely starts with "I"', () => { + expect(cleanAltText('Icy peaks above a frozen lake.')).toBe('Icy peaks above a frozen lake.') + }) + + it.each(['"', '”', "'", '...', '**'])('returns an empty string for a lone quote: %j', (raw) => { + expect(cleanAltText(raw)).toBe('') + }) +}) + +describe('buildAltTextQuestion', () => { + it('includes the location and tags as context', () => { + const question = buildAltTextQuestion({ location: 'Tofino', tags: ['surfing', 'winter'] }) + + expect(question).toContain('Known context') + expect(question).toContain('Tofino') + expect(question).toContain('surfing') + expect(question).toContain('winter') + }) + + it('includes whichever hint is present', () => { + const onlyLocation = buildAltTextQuestion({ location: 'Tofino', tags: [] }) + expect(onlyLocation).toContain('Known context') + expect(onlyLocation).toContain('Tofino') + expect(onlyLocation).not.toContain('Tags') + + const onlyTags = buildAltTextQuestion({ location: null, tags: ['surfing'] }) + expect(onlyTags).toContain('Known context') + expect(onlyTags).toContain('surfing') + expect(onlyTags).not.toContain('Location') + }) + + it.each([ + ['no argument', undefined], + ['an empty object', {}], + ['null location and no tags', { location: null, tags: [] }], + ['an empty location', { location: '', tags: [] }], + ])('omits the context sentence for %s', (_name, hints) => { + const question = buildAltTextQuestion(hints) + + expect(question).toContain('alt text') + expect(question).not.toContain('Known context') + expect(question).not.toContain('Location') + expect(question).not.toContain('Tags') + // Nothing falsy leaked out of the `&&` chains. + expect(question).not.toMatch(/\b(?:0|false|null|undefined)\b/) + expect(question).toBe(question.trim()) + }) +}) + +describe('isAltTextConfigured', () => { + it('needs both the account id and the token', () => { + expect(isAltTextConfigured()).toBe(false) + + vi.stubEnv('CLOUDFLARE_ACCOUNT_ID', ACCOUNT) + expect(isAltTextConfigured()).toBe(false) + + vi.stubEnv('CLOUDFLARE_ACCOUNT_ID', '') + vi.stubEnv('WORKERS_AI_API_TOKEN', TOKEN) + expect(isAltTextConfigured()).toBe(false) + + configure() + expect(isAltTextConfigured()).toBe(true) + }) +}) + +describe('queryImage', () => { + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3, 4]) + + it('posts the image and question to the Workers AI REST endpoint', async () => { + configure() + modelAnswers('A dog on a beach.') + + await queryImage({ jpeg, question: 'What is this?' }) + + expect(fetchMock).toHaveBeenCalledTimes(1) + const { body, init, url } = sent() + expect(url).toBe( + `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT}/ai/run/${ALT_TEXT_MODEL}`, + ) + expect(ALT_TEXT_MODEL).toMatch(/^@cf\/moondream\//) + expect(init?.method).toBe('POST') + + const headers = new Headers(init?.headers) + expect(headers.get('Authorization')).toBe(`Bearer ${TOKEN}`) + expect(headers.get('Content-Type')).toBe('application/json') + + expect(body).toMatchObject({ question: 'What is this?', reasoning: false, task: 'query' }) + expect(body.image).toBe(`data:image/jpeg;base64,${jpeg.toString('base64')}`) + // A hung request must not hang the upload it is part of. + expect(init?.signal).toBeInstanceOf(AbortSignal) + }) + + it('returns the trimmed answer', async () => { + configure() + modelAnswers(' \n A dog on a beach. \n') + + await expect(queryImage({ jpeg, question: 'q' })).resolves.toBe('A dog on a beach.') + }) + + it.each([401, 429, 500])('throws on HTTP %i', async (status) => { + configure() + fetchMock.mockImplementation(async () => + json({ result: { answer: 'ignored' }, success: true }, status), + ) + + await expect(queryImage({ jpeg, question: 'q' })).rejects.toThrow(String(status)) + }) + + it('throws on a non-JSON error response', async () => { + configure() + fetchMock.mockImplementation( + async () => new Response('Bad gateway', { status: 502 }), + ) + + await expect(queryImage({ jpeg, question: 'q' })).rejects.toThrow('502') + }) + + it('throws on a 2xx response that is not JSON', async () => { + configure() + fetchMock.mockImplementation(async () => new Response('ok', { status: 200 })) + + await expect(queryImage({ jpeg, question: 'q' })).rejects.toThrow() + }) + + it('throws with the API error messages on success: false', async () => { + configure() + fetchMock.mockImplementation(async () => + json({ + errors: [ + { code: 7003, message: 'No route for that URI' }, + { code: 10000, message: 'Authentication error' }, + ], + success: false, + }), + ) + + const failure = queryImage({ jpeg, question: 'q' }) + await expect(failure).rejects.toThrow('No route for that URI') + await expect(failure).rejects.toThrow('Authentication error') + }) + + it.each([ + ['an empty answer', { result: { answer: '' }, success: true }], + ['a whitespace answer', { result: { answer: ' \n ' }, success: true }], + ['a null answer', { result: { answer: null }, success: true }], + ['a missing answer', { result: {}, success: true }], + ['a missing result', { success: true }], + ])('throws on %s', async (_name, body) => { + configure() + fetchMock.mockImplementation(async () => json(body)) + + await expect(queryImage({ jpeg, question: 'q' })).rejects.toThrow(/no answer/i) + }) + + it('propagates a network failure', async () => { + configure() + fetchMock.mockRejectedValue(new TypeError('fetch failed')) + + await expect(queryImage({ jpeg, question: 'q' })).rejects.toThrow( + 'Workers AI request failed: TypeError', + ) + }) + + it.each([ + [400, false], + [401, false], + [403, false], + [429, true], + [500, true], + [503, true], + ])('HTTP %i counts as the service being unavailable: %s', async (status, unavailable) => { + configure() + modelFails(status) + + const error = await queryImage({ jpeg, question: 'q' }).catch((err: Error) => err) + expect(error).toBeInstanceOf(Error) + expect(error instanceof WorkersAIUnavailableError).toBe(unavailable) + }) + + it('counts a network failure as the service being unavailable', async () => { + configure() + fetchMock.mockRejectedValue(new TypeError('fetch failed')) + + await expect(queryImage({ jpeg, question: 'q' })).rejects.toBeInstanceOf( + WorkersAIUnavailableError, + ) + }) + + it('never repeats a fetch error message, which can contain the Authorization header', async () => { + configure() + fetchMock.mockRejectedValue( + new TypeError('Headers.append: "Bearer secret-token\n" is an invalid header value.'), + ) + + const error = await queryImage({ jpeg, question: 'q' }).catch((err: Error) => err) + expect(String(error)).not.toContain('secret-token') + }) + + it.each([ + ['nothing', '', ''], + ['only the account id', ACCOUNT, ''], + ['only the token', '', TOKEN], + ])('throws without calling fetch when %s is configured', async (_name, account, token) => { + vi.stubEnv('CLOUDFLARE_ACCOUNT_ID', account) + vi.stubEnv('WORKERS_AI_API_TOKEN', token) + + await expect(queryImage({ jpeg, question: 'q' })).rejects.toThrow(/not configured/i) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) + +describe('toModelJpeg', () => { + it('bounds a large image to 1024px on its longest side', async () => { + const landscape = await toModelJpeg(await solid(3000, 2000, '#336699').jpeg().toBuffer()) + expect(await describeBuffer(landscape)).toMatchObject({ + format: 'jpeg', + height: 683, + width: 1024, + }) + + const portrait = await toModelJpeg(await solid(2000, 3000, '#336699').jpeg().toBuffer()) + expect(await describeBuffer(portrait)).toMatchObject({ + format: 'jpeg', + height: 1024, + width: 683, + }) + }) + + it('does not enlarge a small image', async () => { + const jpeg = await toModelJpeg(await solid(320, 200, '#336699').png().toBuffer()) + expect(await describeBuffer(jpeg)).toMatchObject({ format: 'jpeg', height: 200, width: 320 }) + }) + + it('converts a WebP rendition', async () => { + expect(await describeBuffer(await toModelJpeg(medium))).toMatchObject({ + format: 'jpeg', + height: 675, + width: 900, + }) + }) + + it('converts a PNG with alpha, flattening transparency onto white', async () => { + const png = await sharp({ + create: { width: 64, height: 48, background: { alpha: 0, b: 0, g: 0, r: 0 }, channels: 4 }, + }) + .png() + .toBuffer() + + const jpeg = await toModelJpeg(png) + const { b, format, g, r } = await describeBuffer(jpeg) + expect(format).toBe('jpeg') + expect((await sharp(jpeg).metadata()).channels).toBe(3) + expect(Math.min(r ?? 0, g ?? 0, b ?? 0)).toBeGreaterThan(250) + }) + + it('applies EXIF orientation before measuring', async () => { + const rotated = await solid(2000, 1000, '#336699') + .jpeg() + .withMetadata({ orientation: 6 }) + .toBuffer() + + expect(await describeBuffer(await toModelJpeg(rotated))).toMatchObject({ + height: 1024, + width: 512, + }) + }) + + it('rejects bytes that are not an image', async () => { + await expect(toModelJpeg(Buffer.from('not an image'))).rejects.toThrow() + }) +}) + +describe('loadStoredPhoto', () => { + const stored = (overrides: FakePhoto = {}) => + ({ + id: 'p1', + filename: 'pic.jpg', + sizes: { + medium: { filename: 'pic-900.webp', url: `${MEDIA}/pic-900.webp` }, + thumbnail: { filename: 'pic-300.webp', url: `${MEDIA}/pic-300.webp` }, + }, + url: `${MEDIA}/pic.jpg`, + ...overrides, + }) as Photo + + const load = (dir: string | undefined, photo: Photo) => + loadStoredPhoto(fakePayload({ staticDir: dir }) as unknown as Payload, photo) + + it('reads the medium rendition from disk when it is there', async () => { + const buffer = await load(staticDir, stored()) + + expect(buffer.equals(medium)).toBe(true) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('reads the original from disk when there are no usable renditions', async () => { + const photo = stored({ sizes: { medium: { filename: null, url: null } }, url: null }) + + expect((await load(staticDir, photo)).equals(original)).toBe(true) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('falls back to HTTP, medium rendition first, when the file is not on disk', async () => { + fetchMock.mockImplementation(async () => new Response(new Uint8Array(medium))) + + const buffer = await load(emptyDir, stored()) + + expect(buffer.equals(medium)).toBe(true) + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(fetchMock.mock.calls[0]?.[0]).toBe(`${MEDIA}/pic-900.webp`) + expect(fetchMock.mock.calls[0]?.[1]?.signal).toBeInstanceOf(AbortSignal) + }) + + it('uses HTTP when the collection has no static dir', async () => { + fetchMock.mockImplementation(async () => new Response(new Uint8Array(medium))) + + expect((await load(undefined, stored())).equals(medium)).toBe(true) + expect(fetchMock.mock.calls[0]?.[0]).toBe(`${MEDIA}/pic-900.webp`) + }) + + it('resolves a relative URL against the server URL', async () => { + vi.stubEnv('NEXT_PUBLIC_SERVER_URL', `${SITE_URL}/`) + fetchMock.mockImplementation(async () => new Response(new Uint8Array(original))) + const photo = stored({ sizes: {}, url: '/api/photos/file/pic.jpg' }) + + expect((await load(emptyDir, photo)).equals(original)).toBe(true) + expect(fetchMock.mock.calls[0]?.[0]).toBe(`${SITE_URL}/api/photos/file/pic.jpg`) + }) + + it('moves on to the next rendition when one cannot be fetched', async () => { + fetchMock.mockImplementation(async (url) => + url.endsWith('pic-900.webp') + ? new Response('gone', { status: 404 }) + : new Response(new Uint8Array(original)), + ) + const photo = stored({ + sizes: { + large: { filename: 'pic-1400.webp', url: `${MEDIA}/pic-1400.webp` }, + medium: { filename: 'pic-900.webp', url: `${MEDIA}/pic-900.webp` }, + }, + }) + + expect((await load(emptyDir, photo)).equals(original)).toBe(true) + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + `${MEDIA}/pic-900.webp`, + `${MEDIA}/pic-1400.webp`, + ]) + }) + + it('throws an error naming the photo and every failed request', async () => { + fetchMock.mockImplementation(async (url) => { + if (url.endsWith('pic.jpg')) throw new Error('socket hang up') + return new Response('forbidden', { status: 403 }) + }) + + const failure = load(emptyDir, stored()) + + await expect(failure).rejects.toThrow(/photo p1\b/) + await expect(failure).rejects.toThrow(`${MEDIA}/pic-900.webp -> 403`) + await expect(failure).rejects.toThrow(`${MEDIA}/pic.jpg -> socket hang up`) + }) + + it('throws when the photo has no file at all', async () => { + const photo = stored({ filename: null, sizes: {}, url: null }) + + await expect(load(emptyDir, photo)).rejects.toThrow(/photo p1\b.*no readable file/) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) + +describe('resolveAltTextHints', () => { + it('looks up tag and location ids, passing req along', async () => { + const payload = fakePayload() + const req = fakeReq(payload) + + const hints = await resolveAltTextHints( + payload as unknown as Payload, + { location: 'loc-1', tags: ['tag-1', 'tag-2'] }, + req, + ) + + expect(hints.location).toBe('Tofino') + expect(hints.tags?.sort()).toEqual(['Surfing', 'Winter']) + expect(payload.find).toHaveBeenCalledWith( + expect.objectContaining({ + collection: 'photo-tags', + req, + where: { id: { in: ['tag-1', 'tag-2'] } }, + }), + ) + expect(payload.findByID).toHaveBeenCalledWith( + expect.objectContaining({ id: 'loc-1', collection: 'locations', req }), + ) + }) + + it('uses populated docs without any lookup', async () => { + const payload = fakePayload() + + const hints = await resolveAltTextHints(payload as unknown as Payload, { + location: { id: 'loc-1', name: 'Ucluelet' }, + tags: [{ id: 'tag-9', name: 'Storm' }, null], + }) + + expect(hints).toEqual({ location: 'Ucluelet', tags: ['Storm'] }) + expect(payload.find).not.toHaveBeenCalled() + expect(payload.findByID).not.toHaveBeenCalled() + }) + + it('returns no hints for a photo without tags or location', async () => { + const payload = fakePayload() + + const hints = await resolveAltTextHints(payload as unknown as Payload, { + location: null, + tags: null, + }) + + expect(hints.location ?? null).toBeNull() + expect(hints.tags ?? []).toEqual([]) + expect(payload.find).not.toHaveBeenCalled() + }) + + it('drops the hints instead of throwing when a lookup fails', async () => { + const payload = fakePayload() + payload.find.mockRejectedValue(new Error('db down')) + + const hints = await resolveAltTextHints(payload as unknown as Payload, { + location: 'loc-1', + tags: ['tag-1'], + }) + + expect(buildAltTextQuestion(hints)).not.toContain('Known context') + expect(logger.warn).toHaveBeenCalled() + }) +}) + +describe('generateAltTextOnUpload', () => { + // The inline cooldown is module state, so every test gets a fresh copy of the module. + const loadHook = async () => { + vi.resetModules() + const { generateAltTextOnUpload } = await import('@/collections/Photos/altText/hooks') + + return async ({ + context = {}, + data, + operation = 'create', + originalDoc, + req = uploadReq(), + }: { + context?: Record + data: Record + operation?: 'create' | 'update' + originalDoc?: FakePhoto + req?: PayloadRequest + }) => { + // Payload hands hooks `req.context`, and afterChange hooks see whatever it is by then. + req.context = context + const result = await generateAltTextOnUpload({ + collection: {}, + context, + data, + operation, + originalDoc, + req, + } as never) + return { context: req.context, data: result as Record } + } + } + + const uploadReq = (payload: FakePayload = fakePayload(), extra: Record = {}) => + fakeReq(payload, { + file: { name: 'pic.jpg', data: original, mimetype: 'image/jpeg', size: original.length }, + payloadUploadSizes: { medium }, + ...extra, + }) + + it('leaves a write flagged altTextGenerated untouched', async () => { + configure() + const run = await loadHook() + const input = { alt: 'A red barn in a field.', altSource: 'ai' } + + for (const operation of ['create', 'update'] as const) { + const { context, data } = await run({ + context: { altTextGenerated: true }, + data: input, + operation, + originalDoc: { alt: 'Hand-written', altSource: 'manual' }, + }) + + expect(data).toBe(input) + expect(data).toEqual({ alt: 'A red barn in a field.', altSource: 'ai' }) + expect(context.altTextPending).toBeUndefined() + } + expect(fetchMock).not.toHaveBeenCalled() + }) + + describe('on update', () => { + it.each([ + { + name: 'changes the alt', + alt: 'New words', + expected: 'manual', + originalDoc: { alt: 'Old words', altSource: 'ai' }, + }, + { + name: 'writes an alt where there was none', + alt: 'New words', + expected: 'manual', + originalDoc: { alt: null, altSource: null }, + }, + { + name: 'blanks the alt', + alt: '', + expected: null, + originalDoc: { alt: 'Old words', altSource: 'ai' }, + }, + { + name: 'blanks the alt with whitespace', + alt: ' ', + expected: null, + originalDoc: { alt: 'Old words', altSource: 'manual' }, + }, + { + name: 'nulls the alt', + alt: null, + expected: null, + originalDoc: { alt: 'Old words', altSource: 'manual' }, + }, + ] as const)( + 'sets altSource to $expected when it $name', + async ({ alt, expected, originalDoc }) => { + const run = await loadHook() + + const { data } = await run({ data: { alt }, operation: 'update', originalDoc }) + + expect(data.altSource).toBe(expected) + expect(data.alt).toBe(alt) + }, + ) + + it.each([ + { name: 'does not include alt', data: { focalX: 10 } }, + { name: 'sends the same alt', data: { alt: 'Old words' } }, + { name: 'sends the same alt with extra whitespace', data: { alt: ' Old words \n' } }, + ])('leaves altSource alone when the update $name', async ({ data: input }) => { + const run = await loadHook() + + const { data } = await run({ + data: { ...input }, + operation: 'update', + originalDoc: { alt: ' Old words', altSource: 'ai' }, + }) + + expect(data).toEqual(input) + expect('altSource' in data).toBe(false) + }) + + it('leaves altSource alone when a blank alt stays blank', async () => { + const run = await loadHook() + + const { data } = await run({ + data: { alt: '' }, + operation: 'update', + originalDoc: { alt: null, altSource: null }, + }) + + expect('altSource' in data).toBe(false) + }) + + it('never asks the model or queues a retry, even when the alt is blanked', async () => { + configure() + modelAnswers('A red barn in a field.') + const run = await loadHook() + + const { context, data } = await run({ + data: { alt: '' }, + operation: 'update', + originalDoc: { alt: 'Old words', altSource: 'manual' }, + }) + + expect(data.alt).toBe('') + expect(fetchMock).not.toHaveBeenCalled() + expect(context.altTextPending).toBeUndefined() + }) + }) + + describe('on create', () => { + it('marks a supplied alt as hand-written without asking the model', async () => { + configure() + modelAnswers('A red barn in a field.') + const run = await loadHook() + + // `altSource` is hook-owned: whatever the client sent is replaced. + const { context, data } = await run({ data: { alt: 'My own words', altSource: 'ai' } }) + + expect(data).toMatchObject({ alt: 'My own words', altSource: 'manual' }) + expect(fetchMock).not.toHaveBeenCalled() + expect(context.altTextPending).toBeUndefined() + }) + + it.each([{ alt: '' }, { alt: ' ' }, { alt: null }, {}])( + 'does nothing for a blank alt (%j) when Workers AI is not configured', + async (input) => { + const run = await loadHook() + + const { context, data } = await run({ data: { ...input } }) + + expect(data.altSource).toBeNull() + expect(data.alt?.toString().trim() ?? '').toBe('') + expect(fetchMock).not.toHaveBeenCalled() + expect(context.altTextPending).toBeUndefined() + }, + ) + + it('fills a blank alt with the cleaned model answer', async () => { + configure() + modelAnswers(' "A photo of a red barn\n in a field." ') + const run = await loadHook() + + const { context, data } = await run({ data: { alt: '' } }) + + expect(data).toMatchObject({ alt: 'A red barn in a field.', altSource: 'ai' }) + expect(context.altTextPending).toBeUndefined() + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it.each([{ alt: ' ' }, { alt: null }, {}])('treats %j as a blank alt', async (input) => { + configure() + modelAnswers('A red barn in a field.') + const run = await loadHook() + + const { data } = await run({ data: { ...input } }) + + expect(data).toMatchObject({ alt: 'A red barn in a field.', altSource: 'ai' }) + }) + + it('describes the medium rendition rather than the original upload', async () => { + configure() + modelAnswers('A red barn in a field.') + const run = await loadHook() + + await run({ data: {} }) + + const image = await describeBuffer(sentImage()) + expect(image).toMatchObject({ format: 'jpeg', height: 675, width: 900 }) + expect(image.r).toBeGreaterThan(200) + expect(image.b).toBeLessThan(60) + }) + + it('falls back to the uploaded file when no rendition was generated', async () => { + configure() + modelAnswers('A blue lake.') + const run = await loadHook() + + const { data } = await run({ + data: {}, + req: uploadReq(fakePayload(), { payloadUploadSizes: undefined }), + }) + + expect(data.alt).toBe('A blue lake.') + const image = await describeBuffer(sentImage()) + expect(image).toMatchObject({ format: 'jpeg', height: 768, width: 1024 }) + expect(image.b).toBeGreaterThan(200) + expect(image.r).toBeLessThan(60) + }) + + it('gives the model the names behind the tag and location ids', async () => { + configure() + modelAnswers('A surfer on a winter wave.') + const payload = fakePayload() + const req = uploadReq(payload) + const run = await loadHook() + + await run({ data: { location: 'loc-1', tags: ['tag-1', 'tag-2'] }, req }) + + const question = String(sent().body.question) + expect(question).toContain('Tofino') + expect(question).toContain('Surfing') + expect(question).toContain('Winter') + // Lookups made inside a hook must stay in the request's transaction. + expect(payload.find).toHaveBeenCalledWith(expect.objectContaining({ req })) + expect(payload.findByID).toHaveBeenCalledWith(expect.objectContaining({ req })) + }) + + it('still generates when the hint lookup fails', async () => { + configure() + modelAnswers('A red barn in a field.') + const payload = fakePayload() + payload.find.mockRejectedValue(new Error('db down')) + const run = await loadHook() + + const { data } = await run({ data: { tags: ['tag-1'] }, req: uploadReq(payload) }) + + expect(data).toMatchObject({ alt: 'A red barn in a field.', altSource: 'ai' }) + }) + + it('marks the photo pending when there are no bytes to describe', async () => { + configure() + modelAnswers('A red barn in a field.') + const run = await loadHook() + + const { context, data } = await run({ + data: { alt: '' }, + req: fakeReq(fakePayload(), { file: { name: 'pic.jpg', data: Buffer.alloc(0) } }), + }) + + expect(data).toMatchObject({ alt: '', altSource: null }) + expect(context.altTextPending).toBe(true) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it.each([ + ['an HTTP error', () => modelFails(500)], + ['a network error', () => fetchMock.mockRejectedValue(new TypeError('fetch failed'))], + ['an empty answer', () => modelAnswers('')], + ['an answer that cleans to nothing', () => modelAnswers('""')], + ])('does not throw on %s: the photo is saved blank and marked pending', async (_name, fail) => { + configure() + fail() + const run = await loadHook() + + const { context, data } = await run({ data: { alt: '' } }) + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(data).toMatchObject({ alt: '', altSource: null }) + expect(context.altTextPending).toBe(true) + expect(logger.warn).toHaveBeenCalled() + }) + + it('does not throw when the upload cannot be decoded', async () => { + configure() + modelAnswers('A red barn in a field.') + const run = await loadHook() + + const { context, data } = await run({ + data: {}, + req: uploadReq(fakePayload(), { payloadUploadSizes: { medium: Buffer.from('junk') } }), + }) + + expect(data.altSource).toBeNull() + expect(context.altTextPending).toBe(true) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('skips the model during the cooldown after a failure, then tries again', async () => { + vi.useFakeTimers({ toFake: ['Date'] }) + vi.setSystemTime(new Date('2026-10-11T12:00:00.000Z')) + configure() + modelFails(500) + const run = await loadHook() + + const failed = await run({ data: { alt: '' } }) + expect(failed.context.altTextPending).toBe(true) + expect(fetchMock).toHaveBeenCalledTimes(1) + + // The service is back, but the next uploads go straight to the retry queue. + modelAnswers('A red barn in a field.') + vi.setSystemTime(new Date('2026-10-11T12:01:00.000Z')) + const cooling = await run({ data: { alt: '' } }) + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(cooling.data).toMatchObject({ alt: '', altSource: null }) + expect(cooling.context.altTextPending).toBe(true) + + // A hand-written alt is unaffected by the cooldown. + const manual = await run({ data: { alt: 'My own words' } }) + expect(manual.data.altSource).toBe('manual') + expect(manual.context.altTextPending).toBeUndefined() + + vi.setSystemTime(new Date('2026-10-11T12:02:00.001Z')) + const recovered = await run({ data: { alt: '' } }) + + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(recovered.data).toMatchObject({ alt: 'A red barn in a field.', altSource: 'ai' }) + expect(recovered.context.altTextPending).toBeUndefined() + }) + + it('still marks the photo pending when hints were looked up before the failure', async () => { + // Looking up tag and location names replaces `req.context`; the flag must land on the + // object the afterChange hook will be given, not on the one this hook started with. + configure() + modelFails(500) + const req = uploadReq() + const started = req.context + const run = await loadHook() + + await run({ context: started, data: { alt: '', location: 'loc-1', tags: ['tag-1'] }, req }) + + expect(req.context).not.toBe(started) + expect(req.context.altTextPending).toBe(true) + }) + + it('does not start a cooldown for a rejected request (4xx)', async () => { + configure() + modelFails(401) + const run = await loadHook() + + const rejected = await run({ data: { alt: '' } }) + expect(rejected.context.altTextPending).toBe(true) + + await run({ data: { alt: '' } }) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it.each([ + { name: 'a copied AI description stays AI', alt: 'A red barn.', source: 'ai', want: 'ai' }, + { + name: 'a copied hand-written one stays manual', + alt: 'A red barn.', + source: 'manual', + want: 'manual', + }, + { name: 'a copied legacy one stays unset', alt: 'A red barn.', source: null, want: null }, + { + name: 'text changed while duplicating is manual', + alt: 'My barn.', + source: 'ai', + want: 'manual', + }, + ])('duplicate: $name', async ({ alt, source, want }) => { + configure() + const run = await loadHook() + + const { data } = await run({ + data: { alt }, + originalDoc: { alt: 'A red barn.', altSource: source as FakePhoto['altSource'] }, + }) + + expect(data.altSource).toBe(want) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('does not start a cooldown when the service answered, just not usefully', async () => { + configure() + modelAnswers('""') + const run = await loadHook() + + const unusable = await run({ data: { alt: '' } }) + expect(unusable.context.altTextPending).toBe(true) + + modelAnswers('A red barn in a field.') + const next = await run({ data: { alt: '' } }) + + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(next.data).toMatchObject({ alt: 'A red barn in a field.', altSource: 'ai' }) + }) + + it('starts without a cooldown in a freshly loaded module', async () => { + configure() + modelFails(500) + await ( + await loadHook() + )({ data: {} }) + expect(fetchMock).toHaveBeenCalledTimes(1) + + await ( + await loadHook() + )({ data: {} }) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + }) +}) + +describe('queueAltTextRetry', () => { + const loadHook = async () => { + vi.resetModules() + const { queueAltTextRetry } = await import('@/collections/Photos/altText/hooks') + + return ({ + context = {}, + operation = 'create', + req, + }: { + context?: Record + operation?: 'create' | 'update' + req: PayloadRequest + }) => { + req.context = context + return queueAltTextRetry({ + collection: {}, + context, + data: {}, + doc, + operation, + previousDoc: {}, + req, + } as never) + } + } + + const doc = { id: 'p1', alt: '', filename: 'pic.jpg' } as Photo + + it('queues once: a req reused for the next create does not queue again', async () => { + const payload = fakePayload() + const req = fakeReq(payload) + const run = await loadHook() + + await run({ context: { altTextPending: true }, req }) + expect(req.context.altTextPending).toBeUndefined() + + await run({ context: req.context, req }) + expect(payload.jobs.queue).toHaveBeenCalledTimes(1) + }) + + it('queues the alt text task for a create that was left pending', async () => { + const payload = fakePayload() + const req = fakeReq(payload) + const run = await loadHook() + const before = Date.now() + + const result = await run({ context: { altTextPending: true }, req }) + + expect(result).toBe(doc) + expect(payload.jobs.queue).toHaveBeenCalledTimes(1) + const args = payload.jobs.queue.mock.calls[0]?.[0] as { + input: unknown + queue: string + req: unknown + task: string + waitUntil: Date + } + expect(args.task).toBe('generateAltText') + expect(args.queue).toBe('alt-text') + expect(args.input).toEqual({ photoId: 'p1' }) + expect(args.req).toBe(req) + expect(args.waitUntil).toBeInstanceOf(Date) + expect(args.waitUntil.getTime()).toBeGreaterThan(before) + }) + + it.each([ + { name: 'a create that is not pending', context: {}, operation: 'create' }, + { + name: 'an update, even with the flag set', + context: { altTextPending: true }, + operation: 'update', + }, + ] as const)('does nothing for $name', async ({ context, operation }) => { + const payload = fakePayload() + const run = await loadHook() + + const result = await run({ context, operation, req: fakeReq(payload) }) + + expect(result).toBe(doc) + expect(payload.jobs.queue).not.toHaveBeenCalled() + }) + + // A throw here would skip the R2 upload hook that runs after this one. + it.each([ + ['rejects', () => Promise.reject(new Error('db down'))], + [ + 'throws synchronously', + () => { + throw new Error('no such task') + }, + ], + ])('swallows a jobs.queue that %s and still returns the doc', async (_name, fail) => { + const payload = fakePayload() + payload.jobs.queue.mockImplementation(fail) + const run = await loadHook() + + await expect(run({ context: { altTextPending: true }, req: fakeReq(payload) })).resolves.toBe( + doc, + ) + expect(payload.jobs.queue).toHaveBeenCalledTimes(1) + expect(logger.error).toHaveBeenCalledWith(expect.stringContaining('p1')) + }) +}) + +describe('generatePhotoAltText', () => { + const storedPhoto = (overrides: FakePhoto = {}): FakePhoto => ({ + id: 'p1', + alt: null, + altSource: null, + filename: 'pic.jpg', + sizes: { medium: { filename: 'pic-900.webp', url: '/api/photos/file/pic-900.webp' } }, + url: '/api/photos/file/pic.jpg', + ...overrides, + }) + + const setup = (photo: FakePhoto | null) => { + const payload = fakePayload({ photo, staticDir }) + return { payload, req: fakeReq(payload) } + } + + it.each([ + { alt: null, altSource: null, generated: true, overwrite: 'empty' }, + { alt: '', altSource: null, generated: true, overwrite: 'empty' }, + { alt: ' ', altSource: 'manual', generated: true, overwrite: 'empty' }, + { alt: 'Old AI text', altSource: 'ai', generated: false, overwrite: 'empty' }, + { alt: 'Hand-written', altSource: 'manual', generated: false, overwrite: 'empty' }, + { alt: 'Legacy text', altSource: null, generated: false, overwrite: 'empty' }, + { alt: null, altSource: null, generated: true, overwrite: 'ai' }, + { alt: 'Old AI text', altSource: 'ai', generated: true, overwrite: 'ai' }, + { alt: 'Hand-written', altSource: 'manual', generated: false, overwrite: 'ai' }, + { alt: 'Legacy text', altSource: null, generated: false, overwrite: 'ai' }, + { alt: null, altSource: null, generated: true, overwrite: 'always' }, + { alt: 'Old AI text', altSource: 'ai', generated: true, overwrite: 'always' }, + { alt: 'Hand-written', altSource: 'manual', generated: true, overwrite: 'always' }, + { alt: 'Legacy text', altSource: null, generated: true, overwrite: 'always' }, + ] as const)( + "overwrite '$overwrite' with alt $alt / altSource $altSource → generated: $generated", + async ({ alt, altSource, generated, overwrite }) => { + configure() + modelAnswers('A red barn in a field.') + const { payload, req } = setup(storedPhoto({ alt, altSource })) + + const result = await generatePhotoAltText({ id: 'p1', overwrite, req }) + + if (generated) { + expect(result).toEqual({ alt: 'A red barn in a field.', status: 'generated' }) + expect(payload.update).toHaveBeenCalledTimes(1) + } else { + expect(result).toEqual({ reason: 'has-alt', status: 'skipped' }) + expect(payload.update).not.toHaveBeenCalled() + expect(fetchMock).not.toHaveBeenCalled() + } + }, + ) + + it('reports a missing photo without asking the model', async () => { + configure() + modelAnswers('A red barn in a field.') + const { payload, req } = setup(null) + + const result = await generatePhotoAltText({ id: 'gone', overwrite: 'always', req }) + + expect(result).toEqual({ reason: 'not-found', status: 'skipped' }) + expect(payload.findByID).toHaveBeenCalledWith( + expect.objectContaining({ id: 'gone', collection: 'photos', req }), + ) + expect(fetchMock).not.toHaveBeenCalled() + expect(payload.update).not.toHaveBeenCalled() + }) + + it('saves the cleaned answer as AI-written, flagged so the hook leaves it alone', async () => { + configure() + modelAnswers('"This image shows a red barn in a field."') + const { payload, req } = setup(storedPhoto()) + + const result = await generatePhotoAltText({ + id: 'p1', + context: { disableRevalidate: true }, + overwrite: 'empty', + req, + }) + + expect(result).toEqual({ alt: 'A red barn in a field.', status: 'generated' }) + expect(payload.update).toHaveBeenCalledTimes(1) + const args = payload.update.mock.calls[0]?.[0] as Record + expect(args).toMatchObject({ + id: 'p1', + collection: 'photos', + data: { alt: 'A red barn in a field.', altSource: 'ai' }, + }) + expect(args.context).toEqual({ altTextGenerated: true, disableRevalidate: true }) + // Written through a req with its own `context`, so the flag never touches the shared one. + expect(args.req).not.toBe(req) + expect((args.req as PayloadRequest).payload).toBe(payload) + expect(req.context.altTextGenerated).toBeUndefined() + }) + + it('does not replace text saved while the model was answering', async () => { + configure() + modelAnswers('A red barn in a field.') + const { payload, req } = setup(storedPhoto()) + + for (const overwrite of ['empty', 'ai'] as const) { + payload.findByID + .mockResolvedValueOnce(storedPhoto()) + .mockResolvedValueOnce({ id: 'p1', alt: 'Typed meanwhile', altSource: 'manual' }) + expect(await generatePhotoAltText({ id: 'p1', overwrite, req })).toEqual({ + reason: 'has-alt', + status: 'skipped', + }) + } + expect(fetchMock).toHaveBeenCalledTimes(2) + expect(payload.update).not.toHaveBeenCalled() + }) + + it('reports a photo deleted while the model was answering', async () => { + configure() + modelAnswers('A red barn in a field.') + const { payload, req } = setup(storedPhoto()) + payload.findByID.mockResolvedValueOnce(storedPhoto()).mockResolvedValueOnce(null) + + expect(await generatePhotoAltText({ id: 'p1', overwrite: 'always', req })).toEqual({ + reason: 'not-found', + status: 'skipped', + }) + expect(payload.update).not.toHaveBeenCalled() + }) + + it('does not let caller context switch the altTextGenerated flag off', async () => { + configure() + modelAnswers('A red barn in a field.') + const { payload, req } = setup(storedPhoto()) + + await generatePhotoAltText({ + id: 'p1', + context: { altTextGenerated: false }, + overwrite: 'empty', + req, + }) + + const args = payload.update.mock.calls[0]?.[0] as Record + expect(args.context).toEqual({ altTextGenerated: true }) + }) + + it('does not leave altTextGenerated on the shared req after the write', async () => { + configure() + modelAnswers('A red barn in a field.') + const { payload, req } = setup(storedPhoto()) + + await generatePhotoAltText({ id: 'p1', overwrite: 'empty', req }) + expect(req.context.altTextGenerated).toBeUndefined() + + payload.update.mockImplementationOnce(async (args) => { + if (args.req) args.req.context = { ...args.req.context, ...args.context } + throw new Error('write failed') + }) + await expect(generatePhotoAltText({ id: 'p1', overwrite: 'empty', req })).rejects.toThrow( + 'write failed', + ) + expect(req.context.altTextGenerated).toBeUndefined() + }) + + it('describes the stored medium rendition, read from disk', async () => { + configure() + modelAnswers('A red barn in a field.') + const { req } = setup(storedPhoto()) + + await generatePhotoAltText({ id: 'p1', overwrite: 'empty', req }) + + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(sent().url).toContain('api.cloudflare.com') + const image = await describeBuffer(sentImage()) + expect(image).toMatchObject({ format: 'jpeg', height: 675, width: 900 }) + expect(image.r).toBeGreaterThan(200) + }) + + it('passes the populated tag and location names to the model', async () => { + configure() + modelAnswers('A surfer on a winter wave.') + const { payload, req } = setup( + storedPhoto({ + location: { id: 'loc-1', name: 'Ucluelet' }, + tags: [{ id: 'tag-9', name: 'Storm' }], + }), + ) + + await generatePhotoAltText({ id: 'p1', overwrite: 'empty', req }) + + const question = String(sent().body.question) + expect(question).toContain('Ucluelet') + expect(question).toContain('Storm') + expect(payload.find).not.toHaveBeenCalled() + }) + + it.each([ + ['the model call fails', () => modelFails(500), /500/], + ['the model answer is unusable', () => modelAnswers('""'), /not usable/], + ])('throws and writes nothing when %s', async (_name, fail, message) => { + configure() + fail() + const { payload, req } = setup(storedPhoto()) + + await expect(generatePhotoAltText({ id: 'p1', overwrite: 'always', req })).rejects.toThrow( + message, + ) + expect(payload.update).not.toHaveBeenCalled() + }) + + it('throws and writes nothing when Workers AI is not configured', async () => { + const { payload, req } = setup(storedPhoto()) + + await expect(generatePhotoAltText({ id: 'p1', overwrite: 'empty', req })).rejects.toThrow( + /not configured/i, + ) + expect(fetchMock).not.toHaveBeenCalled() + expect(payload.update).not.toHaveBeenCalled() + }) + + it('throws and writes nothing when the stored file cannot be read', async () => { + configure() + fetchMock.mockImplementation(async () => new Response('gone', { status: 404 })) + const payload = fakePayload({ photo: storedPhoto(), staticDir: emptyDir }) + + await expect( + generatePhotoAltText({ id: 'p1', overwrite: 'empty', req: fakeReq(payload) }), + ).rejects.toThrow(/could not read photo p1/) + expect(payload.update).not.toHaveBeenCalled() + }) +}) + +describe('generateAltTextTask', () => { + const handler = generateAltTextTask.handler as unknown as (args: { + input: { photoId: string } + req: PayloadRequest + }) => Promise<{ output: { status: string } }> + + const photo = (alt: null | string): FakePhoto => ({ + id: 'p1', + alt, + altSource: alt ? 'ai' : null, + filename: 'pic.jpg', + }) + + it('uses the slug and queue the retry hook queues to', () => { + expect(generateAltTextTask.slug).toBe(ALT_TEXT_TASK) + expect(ALT_TEXT_TASK).toBe('generateAltText') + expect(ALT_TEXT_QUEUE).toBe('alt-text') + }) + + it('fills a blank alt', async () => { + configure() + modelAnswers('A blue lake.') + const payload = fakePayload({ photo: photo(null), staticDir }) + + const result = await handler({ input: { photoId: 'p1' }, req: fakeReq(payload) }) + + expect(result).toEqual({ output: { status: 'generated' } }) + expect(payload.update).toHaveBeenCalledWith( + expect.objectContaining({ data: { alt: 'A blue lake.', altSource: 'ai' } }), + ) + }) + + it('never replaces text written since the job was queued, not even AI text', async () => { + configure() + modelAnswers('A blue lake.') + const payload = fakePayload({ photo: photo('Written in the meantime'), staticDir }) + + const result = await handler({ input: { photoId: 'p1' }, req: fakeReq(payload) }) + + expect(result).toEqual({ output: { status: 'skipped' } }) + expect(payload.update).not.toHaveBeenCalled() + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('throws on a model failure so the job is retried', async () => { + configure() + modelFails(500) + const payload = fakePayload({ photo: photo(null), staticDir }) + + await expect(handler({ input: { photoId: 'p1' }, req: fakeReq(payload) })).rejects.toThrow() + }) +}) From 2df9a7069531e01b60abc943beb478836d8db25b Mon Sep 17 00:00:00 2001 From: Logan Kuzyk Date: Sun, 11 Oct 2026 21:48:16 -0700 Subject: [PATCH 2/2] fix(photos): use Gemma 4 for alt text, Moondream returns empty over REST Moondream 3.1 returned `result: {}` for every non-streaming request against the real Workers AI API, so every upload failed to "no answer" and queued a retry that could never succeed. It only answers with `stream: true`. Switch to @cf/google/gemma-4-26b-a4b-it: standard chat format, works without streaming, ~300 input tokens per photo vs 740, $0.10/$0.30 per M tokens (about $0.00003 a photo vs $0.00024). Thinking is disabled via chat_template_kwargs; left on it spends the whole max_tokens budget on a reasoning trace and returns empty content. Verified against the live API: a blank-alt upload now gets alt text inline with altSource "ai" and no queued job. Co-Authored-By: Claude Sonnet 5.5 --- .env.example | 2 +- CMS.md | 2 +- scripts/generate-alt-text.ts | 4 +- src/collections/Photos/altText/workersAI.ts | 32 ++++++--- tests/int/photoAltText.int.spec.ts | 74 ++++++++++++++++----- 5 files changed, 86 insertions(+), 28 deletions(-) diff --git a/.env.example b/.env.example index cc1e28b..ce858d4 100644 --- a/.env.example +++ b/.env.example @@ -49,7 +49,7 @@ PREVIEW_SECRET=YOUR_SECRET_HERE # (e.g. a local MinIO mock). Real Cloudflare R2 does not need it. # R2_FORCE_PATH_STYLE= -# Cloudflare Workers AI: writes alt text for photos (model @cf/moondream/moondream3.1-9B-A2B, called over REST). +# Cloudflare Workers AI: writes alt text for photos (model @cf/google/gemma-4-26b-a4b-it, called over REST). # With both set, a photo uploaded with a blank alt gets one generated, the photo edit view shows a # "Regenerate alt text" button, and `npm run generate:alt-text` can backfill existing photos. # Leave either unset and the feature is off entirely: uploads keep a blank alt, nothing is sent to Cloudflare. diff --git a/CMS.md b/CMS.md index f39ae9c..a148f40 100644 --- a/CMS.md +++ b/CMS.md @@ -95,7 +95,7 @@ Slug: `media` · Folders: Yes - **Delivery**: `ImageMedia` builds a real `srcSet` from `sizes.*` even when images are served straight from the CDN (`NEXT_PUBLIC_STORAGE_URL`), via a Next.js image `loader` that maps each requested width to the closest rendition. Callers that render into a known box (e.g. `PhotoGrid` tiles) pass an explicit `size`. The rendition ladder is capped at `xlarge` (1920px WebP); only `priority` (LCP) images may fall back to the raw original, so a missing/inaccurate `size` can never silently serve a multi-MB file. - **Backfill**: existing photos are reprocessed with `npm run reprocess:photos` (`-- --dry-run` / `-- --limit N` / `-- --force` / `-- --continue-on-error`). It re-downloads each original, has Payload rebuild `sizes.*`, then verifies every object resolves from storage before moving on and stops on the first failure. AVIF/HEIC sources decode slowly, so large batches take time. Run it against the local MinIO mock (`npm run dev:r2`) first and confirm bucket object counts before pointing it at R2. -**Alt text** (`src/collections/Photos/altText/`): optional, enabled by setting both `CLOUDFLARE_ACCOUNT_ID` and `WORKERS_AI_API_TOKEN`. Descriptions come from the Workers AI vision model `@cf/moondream/moondream3.1-9B-A2B`, called over REST; the photo's tags and location are passed along as hints. +**Alt text** (`src/collections/Photos/altText/`): optional, enabled by setting both `CLOUDFLARE_ACCOUNT_ID` and `WORKERS_AI_API_TOKEN`. Descriptions come from the Workers AI vision model `@cf/google/gemma-4-26b-a4b-it`, called over REST; the photo's tags and location are passed along as hints. - **On upload**: a photo created with a blank `alt` is described during the upload itself, so the edit form already holds the text. Text typed at upload is kept as is. Existing photos are never changed by an ordinary save. - **`altSource`**: `ai` when the model wrote `alt`, `manual` once it is typed or edited by hand, unset while `alt` is blank and on photos whose text predates the feature (those count as hand-written). It is what lets a regeneration pass replace machine-written text without touching hand-written text. diff --git a/scripts/generate-alt-text.ts b/scripts/generate-alt-text.ts index 2f8fa9f..4921268 100644 --- a/scripts/generate-alt-text.ts +++ b/scripts/generate-alt-text.ts @@ -3,7 +3,7 @@ * src/collections/Photos/altText). New uploads get theirs during the upload itself; this * covers everything uploaded before that existed, or while Workers AI was not configured. * - * Each photo is one request to Cloudflare Workers AI (`@cf/moondream/moondream3.1-9B-A2B`), + * Each photo is one request to Cloudflare Workers AI (`@cf/google/gemma-4-26b-a4b-it`), * so it needs CLOUDFLARE_ACCOUNT_ID and WORKERS_AI_API_TOKEN (see .env.example). * * Runs standalone (outside Next.js), same pattern as scripts/reprocess-photos.ts. @@ -25,7 +25,7 @@ * out several hundred requests later. * * COST: every generated alt text is a billed model call. Cloudflare's model page lists - * $0.30 per million input tokens and $1.00 per million output tokens; `--dry-run` shows how + * $0.10 per million input tokens and $0.30 per million output tokens (about $0.00003 a photo); `--dry-run` shows how * many photos a run would send, and `--limit` lets you try a handful and read the real usage * off the Workers AI dashboard before committing to the lot. * ───────────────────────────────────────────────────────────────────────────────────────── diff --git a/src/collections/Photos/altText/workersAI.ts b/src/collections/Photos/altText/workersAI.ts index 4c10220..b912db1 100644 --- a/src/collections/Photos/altText/workersAI.ts +++ b/src/collections/Photos/altText/workersAI.ts @@ -1,12 +1,16 @@ // Workers AI over REST: the site runs in Docker, not on Workers, so there is no `env.AI` binding. -// https://developers.cloudflare.com/workers-ai/models/moondream3.1-9B-A2B/ -export const ALT_TEXT_MODEL = '@cf/moondream/moondream3.1-9B-A2B' +// https://developers.cloudflare.com/workers-ai/models/gemma-4-26b-a4b-it/ +// +// Chosen over `@cf/moondream/moondream3.1-9B-A2B`, which over REST returns `result: {}` unless +// `stream: true` is set, and which costs roughly 7x more per photo (740 vs ~300 input tokens, +// $0.30 vs $0.10 per M). Gemma is a chat model: image in an OpenAI-style `image_url` part. +export const ALT_TEXT_MODEL = '@cf/google/gemma-4-26b-a4b-it' const REQUEST_TIMEOUT_MS = 20_000 type WorkersAIResponse = { errors?: { code?: number; message?: string }[] - result?: { answer?: string | null; finish_reason?: string } + result?: { choices?: { finish_reason?: string; message?: { content?: string | null } }[] } success?: boolean } @@ -40,11 +44,21 @@ export async function queryImage({ 'Content-Type': 'application/json', }, body: JSON.stringify({ - task: 'query', - image: `data:image/jpeg;base64,${jpeg.toString('base64')}`, - question, - // The reasoning trace defaults to on; it costs output tokens and latency we don't need. - reasoning: false, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: question }, + { + type: 'image_url', + image_url: { url: `data:image/jpeg;base64,${jpeg.toString('base64')}` }, + }, + ], + }, + ], + // Thinking defaults to on: the reasoning trace is billed as output, takes ~20s, and with + // a small `max_tokens` eats the whole budget and leaves `content` empty. + chat_template_kwargs: { enable_thinking: false }, temperature: 0.2, max_tokens: 120, }), @@ -74,7 +88,7 @@ export async function queryImage({ : new Error(message) } - const answer = body.result?.answer?.trim() + const answer = body.result?.choices?.[0]?.message?.content?.trim() if (!answer) throw new Error('Workers AI returned no answer') return answer } diff --git a/tests/int/photoAltText.int.spec.ts b/tests/int/photoAltText.int.spec.ts index 4d95c19..4ceb88a 100644 --- a/tests/int/photoAltText.int.spec.ts +++ b/tests/int/photoAltText.int.spec.ts @@ -44,21 +44,38 @@ const json = (body: unknown, status = 200) => headers: { 'Content-Type': 'application/json' }, }) +const completion = (content: null | string) => ({ + result: { choices: [{ finish_reason: 'stop', message: { content, role: 'assistant' } }] }, + success: true, +}) + const modelAnswers = (answer: null | string) => - fetchMock.mockImplementation(async () => json({ result: { answer }, success: true })) + fetchMock.mockImplementation(async () => json(completion(answer))) const modelFails = (status = 500, message = 'upstream exploded') => fetchMock.mockImplementation(async () => json({ errors: [{ code: 5000, message }], result: null, success: false }, status), ) +type ChatPart = { image_url?: { url: string }; text?: string; type: string } + const sent = (call = 0) => { const [url, init] = fetchMock.mock.calls[call] ?? [] - return { body: JSON.parse(String(init?.body)) as Record, init, url } + const body = JSON.parse(String(init?.body)) as Record & { + messages?: { content: ChatPart[] }[] + } + const parts = body.messages?.[0]?.content ?? [] + return { + body, + image: parts.find((p) => p.type === 'image_url')?.image_url?.url, + init, + question: parts.find((p) => p.type === 'text')?.text, + url, + } } const sentImage = (call = 0) => - Buffer.from(String(sent(call).body.image).replace(/^data:image\/jpeg;base64,/, ''), 'base64') + Buffer.from(String(sent(call).image).replace(/^data:image\/jpeg;base64,/, ''), 'base64') const describeBuffer = async (image: Buffer) => { const { format, height, width } = await sharp(image).metadata() @@ -392,15 +409,30 @@ describe('queryImage', () => { expect(url).toBe( `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT}/ai/run/${ALT_TEXT_MODEL}`, ) - expect(ALT_TEXT_MODEL).toMatch(/^@cf\/moondream\//) + expect(ALT_TEXT_MODEL).toMatch(/^@cf\/google\/gemma-/) expect(init?.method).toBe('POST') const headers = new Headers(init?.headers) expect(headers.get('Authorization')).toBe(`Bearer ${TOKEN}`) expect(headers.get('Content-Type')).toBe('application/json') - expect(body).toMatchObject({ question: 'What is this?', reasoning: false, task: 'query' }) - expect(body.image).toBe(`data:image/jpeg;base64,${jpeg.toString('base64')}`) + expect(body).toMatchObject({ + chat_template_kwargs: { enable_thinking: false }, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'What is this?' }, + { + type: 'image_url', + image_url: { url: `data:image/jpeg;base64,${jpeg.toString('base64')}` }, + }, + ], + }, + ], + }) + // Not the (billed) streaming/Moondream shape. + expect(body).not.toHaveProperty('stream') // A hung request must not hang the upload it is part of. expect(init?.signal).toBeInstanceOf(AbortSignal) }) @@ -414,9 +446,7 @@ describe('queryImage', () => { it.each([401, 429, 500])('throws on HTTP %i', async (status) => { configure() - fetchMock.mockImplementation(async () => - json({ result: { answer: 'ignored' }, success: true }, status), - ) + fetchMock.mockImplementation(async () => json(completion('ignored'), status)) await expect(queryImage({ jpeg, question: 'q' })).rejects.toThrow(String(status)) }) @@ -455,10 +485,24 @@ describe('queryImage', () => { }) it.each([ - ['an empty answer', { result: { answer: '' }, success: true }], - ['a whitespace answer', { result: { answer: ' \n ' }, success: true }], - ['a null answer', { result: { answer: null }, success: true }], - ['a missing answer', { result: {}, success: true }], + ['an empty answer', completion('')], + ['a whitespace answer', completion(' \n ')], + ['a null answer', completion(null)], + [ + 'a reasoning-only answer cut off by max_tokens', + { + result: { + choices: [ + { finish_reason: 'length', message: { content: '', reasoning_content: 'hmm' } }, + ], + }, + success: true, + }, + ], + [ + 'an empty result (what a non-streaming Moondream call returns)', + { result: {}, success: true }, + ], ['a missing result', { success: true }], ])('throws on %s', async (_name, body) => { configure() @@ -987,7 +1031,7 @@ describe('generateAltTextOnUpload', () => { await run({ data: { location: 'loc-1', tags: ['tag-1', 'tag-2'] }, req }) - const question = String(sent().body.question) + const question = String(sent().question) expect(question).toContain('Tofino') expect(question).toContain('Surfing') expect(question).toContain('Winter') @@ -1464,7 +1508,7 @@ describe('generatePhotoAltText', () => { await generatePhotoAltText({ id: 'p1', overwrite: 'empty', req }) - const question = String(sent().body.question) + const question = String(sent().question) expect(question).toContain('Ucluelet') expect(question).toContain('Storm') expect(payload.find).not.toHaveBeenCalled()