From 2833a9ddbaf3dc34d503e6c808117d837dcb3673 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 10 Aug 2026 12:46:47 +0000 Subject: [PATCH 1/3] feat: follow an athlete past the fragment you pointed at MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Identifying a child only ever labels them where the user happened to look, and re-identification could only confirm a binding where one already existed. On a real game that left an athlete known for 31.7s of 300s across six fragments, all inside the one 32-second window originally clicked. Every signal that follows the athlete was dark for the other 90%, and the moments that survived were scene-wide ones with nothing to do with them. A jersey is the one thing about a child a detector can see that stays the same all afternoon, so the worker gains an `appearance` command: a coarse HSV histogram of the torso, from one pass over the 540p proxy. Boxes arrive on stdin because there are thousands of them. Colour is a veto, not an identifier, and that is the whole design. Measured against the production game, three ways: colour only (>= 0.55) 661 of 1152 tracks, 2306s of "athlete" in a 300s video continuity only 56 tracks, 120.3s both 14 tracks, 51.2s (from 6 tracks, 31.7s) Teammates wear the same shirt, so colour alone selects a *team* — and the children it wrongly volunteers are exactly the ones standing next to yours. Continuity alone links whoever happens to be nearby. The identity claim therefore rests on continuity — a fragment that begins where and when another ended, within 2s and a distance a child could actually run — with colour able only to rule a link out. At a 4s gap the accepted links reached 894px of a 1920-wide frame for eight more seconds of coverage, which is where that trade stops being worth taking. Nothing is assigned. Matches are pre-selected in the picker with the evidence behind each one — the gap, the distance, the colour agreement — and a human confirms, because the cost of a confident wrong answer is another family's child in your highlight reel. Also: `run` can write to a child's stdin, which the box list needs. Verified against production via scripts/appearance-probe.mjs and scripts/stitch-probe.mjs, both read-only; the numbers above are theirs. Co-Authored-By: Claude Opus 5 (1M context) --- apps/cv-worker/src/appearance.test.ts | 135 ++++++++ apps/cv-worker/src/appearance.ts | 160 ++++++++++ apps/cv-worker/src/index.ts | 78 ++++- apps/cv-worker/src/signatures.ts | 115 +++++++ apps/web/src/actions.ts | 50 +++ apps/web/src/client/identify.tsx | 117 ++++++- packages/core/src/appearance.test.ts | 193 ++++++++++++ packages/core/src/appearance.ts | 427 ++++++++++++++++++++++++++ packages/core/src/ffmpeg.ts | 27 +- packages/core/src/index.ts | 1 + scripts/appearance-probe.mjs | 121 ++++++++ scripts/stitch-probe.mjs | 143 +++++++++ 12 files changed, 1546 insertions(+), 21 deletions(-) create mode 100644 apps/cv-worker/src/appearance.test.ts create mode 100644 apps/cv-worker/src/appearance.ts create mode 100644 apps/cv-worker/src/signatures.ts create mode 100644 packages/core/src/appearance.test.ts create mode 100644 packages/core/src/appearance.ts create mode 100644 scripts/appearance-probe.mjs create mode 100644 scripts/stitch-probe.mjs diff --git a/apps/cv-worker/src/appearance.test.ts b/apps/cv-worker/src/appearance.test.ts new file mode 100644 index 0000000..6831d08 --- /dev/null +++ b/apps/cv-worker/src/appearance.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from 'vitest'; + +import { + accumulate, + BIN_COUNT, + binFor, + normalize, + similarity, + toHsv, + torsoRect, +} from './appearance.js'; + +/** + * The safety property under all of this: a signature that cannot tell two teams + * apart is useless, and a signature that confidently equates them is dangerous. + * These pin both directions. + */ + +/** A solid-colour BGR frame, so a crop's expected histogram is known exactly. */ +const solidFrame = (width: number, height: number, [b, g, r]: [number, number, number]): Buffer => { + const buffer = Buffer.alloc(width * height * 3); + for (let i = 0; i < width * height; i += 1) { + buffer[i * 3] = b; + buffer[i * 3 + 1] = g; + buffer[i * 3 + 2] = r; + } + return buffer; +}; + +const signatureOf = (frame: Buffer, width: number, height: number): number[] => { + const bins = new Float64Array(BIN_COUNT); + const rect = torsoRect({ x: 0, y: 0, w: width, h: height }, 1, width, height); + expect(rect).not.toBeNull(); + accumulate(frame, width, rect!, bins); + return normalize(bins); +}; + +describe('colour conversion', () => { + it('reads primaries at the hues they belong to', () => { + expect(toHsv(255, 0, 0).h).toBeCloseTo(0); + expect(toHsv(0, 255, 0).h).toBeCloseTo(120); + expect(toHsv(0, 0, 255).h).toBeCloseTo(240); + }); + + it('reports grey as unsaturated whatever its lightness', () => { + expect(toHsv(128, 128, 128).s).toBe(0); + expect(toHsv(255, 255, 255).s).toBe(0); + expect(toHsv(0, 0, 0).s).toBe(0); + }); +}); + +describe('binning', () => { + it('sends washed-out and near-black pixels to the lightness bins, not a random hue', () => { + // A white shirt has a hue, arithmetically; it means nothing. Binning it by + // hue would scatter white jerseys across the spectrum at random. + const white = binFor(toHsv(250, 250, 250)); + const black = binFor(toHsv(4, 4, 6)); + expect(white).toBeGreaterThanOrEqual(24); + expect(black).toBeGreaterThanOrEqual(24); + expect(white).not.toBe(black); + }); + + it('keeps saturated colours apart', () => { + expect(binFor(toHsv(255, 0, 0))).not.toBe(binFor(toHsv(0, 0, 255))); + }); + + it('never returns a bin outside the histogram', () => { + for (let r = 0; r <= 255; r += 17) { + for (let g = 0; g <= 255; g += 17) { + for (let b = 0; b <= 255; b += 17) { + const bin = binFor(toHsv(r, g, b)); + expect(bin).toBeGreaterThanOrEqual(0); + expect(bin).toBeLessThan(BIN_COUNT); + } + } + } + }); +}); + +describe('the torso crop', () => { + it('takes the shirt, not the head, the legs or the air beside them', () => { + const rect = torsoRect({ x: 100, y: 200, w: 100, h: 200 }, 1, 1920, 1080); + expect(rect).toEqual({ x0: 120, x1: 180, y0: 230, y1: 300 }); + }); + + it('scales into decoded-frame pixels', () => { + // Boxes arrive in source pixels; the frame is decoded smaller. + const rect = torsoRect({ x: 100, y: 200, w: 100, h: 200 }, 0.5, 960, 540); + expect(rect).toEqual({ x0: 60, x1: 90, y0: 115, y1: 150 }); + }); + + it('refuses a box with nothing left in frame rather than inventing a sliver', () => { + expect(torsoRect({ x: -500, y: 0, w: 100, h: 200 }, 1, 960, 540)).toBeNull(); + expect(torsoRect({ x: 0, y: 0, w: 1, h: 1 }, 1, 960, 540)).toBeNull(); + }); +}); + +describe('comparing two players', () => { + it('matches a shirt against itself', () => { + const red = signatureOf(solidFrame(40, 80, [30, 30, 200]), 40, 80); + expect(similarity(red, red)).toBeCloseTo(1); + }); + + it('separates two teams wearing different colours', () => { + const red = signatureOf(solidFrame(40, 80, [30, 30, 200]), 40, 80); + const blue = signatureOf(solidFrame(40, 80, [200, 30, 30]), 40, 80); + // The whole point. If this ever creeps up, the wrong child ends up in a reel. + expect(similarity(red, blue)).toBeLessThan(0.1); + }); + + it('separates a white shirt from a black one', () => { + const white = signatureOf(solidFrame(40, 80, [245, 245, 245]), 40, 80); + const black = signatureOf(solidFrame(40, 80, [12, 12, 12]), 40, 80); + expect(similarity(white, black)).toBeLessThan(0.1); + }); + + it('still recognises a shirt through a shading change', () => { + // Same jersey, one player in sun and one in shadow: value drops, hue holds. + const lit = signatureOf(solidFrame(40, 80, [40, 40, 220]), 40, 80); + const shaded = signatureOf(solidFrame(40, 80, [26, 26, 140]), 40, 80); + expect(similarity(lit, shaded)).toBeGreaterThan(0.8); + }); + + it('normalises away crop size, so a close-up matches a distant shot', () => { + const near = signatureOf(solidFrame(80, 160, [30, 180, 40]), 80, 160); + const far = signatureOf(solidFrame(12, 24, [30, 180, 40]), 12, 24); + expect(similarity(near, far)).toBeCloseTo(1, 1); + }); + + it('gives an empty signature no similarity to anything', () => { + const nothing = normalize(new Float64Array(BIN_COUNT)); + const red = signatureOf(solidFrame(40, 80, [30, 30, 200]), 40, 80); + expect(similarity(nothing, red)).toBe(0); + }); +}); diff --git a/apps/cv-worker/src/appearance.ts b/apps/cv-worker/src/appearance.ts new file mode 100644 index 0000000..3569b2f --- /dev/null +++ b/apps/cv-worker/src/appearance.ts @@ -0,0 +1,160 @@ +/** + * What a player looks like, reduced to something two tracks can be compared on. + * + * Re-identification matched on box overlap alone, which by construction only + * finds an athlete where they were already known to be: it can confirm a + * binding across a re-detection, but it can never discover the same child + * somewhere else in the game. On real footage that left an athlete identified + * for 31.7s of a 300s match, with every signal that follows them dark for the + * other 90%. + * + * A jersey is the one thing about a child that a detector can see and that + * stays the same all afternoon, so that is what this measures: a coarse colour + * histogram of the torso. Deliberately coarse — the point is to tell one team's + * shirt from the other's and one shirt from the floor, not to recognise a face. + * Anything finer would invite false confidence, and the cost of a confident + * wrong answer here is another family's child in your highlight reel. + */ + +/** Where the shirt is, as fractions of a player's box. */ +const TORSO = { x0: 0.2, x1: 0.8, y0: 0.15, y1: 0.5 }; + +/** 12 hues x 2 saturations for colour, plus 4 lightness bins for grey. */ +export const HUE_BINS = 12; +export const SAT_BINS = 2; +export const GREY_BINS = 4; +export const BIN_COUNT = HUE_BINS * SAT_BINS + GREY_BINS; + +/** + * Below these a pixel has no usable hue — a white shirt, a black shoe, a shadow + * — and binning it by hue would scatter it at random across the spectrum. Those + * pixels carry their lightness instead, which is what actually distinguishes a + * white jersey from a dark one. + */ +const MIN_SATURATION = 0.2; +const MIN_VALUE = 0.15; + +export interface Box { + x: number; + y: number; + w: number; + h: number; +} + +export interface Rect { + x0: number; + y0: number; + x1: number; + y1: number; +} + +/** + * The torso of a box, scaled into decoded-frame pixels and clamped to the + * frame. Returns null when nothing usable is left — an off-screen or + * sub-pixel box has no appearance to measure, and inventing one from a + * clamped sliver would be worse than skipping it. + */ +export const torsoRect = ( + box: Box, + scale: number, + frameWidth: number, + frameHeight: number, +): Rect | null => { + const x0 = Math.round((box.x + box.w * TORSO.x0) * scale); + const x1 = Math.round((box.x + box.w * TORSO.x1) * scale); + const y0 = Math.round((box.y + box.h * TORSO.y0) * scale); + const y1 = Math.round((box.y + box.h * TORSO.y1) * scale); + + const clamped: Rect = { + x0: Math.max(0, Math.min(frameWidth, x0)), + x1: Math.max(0, Math.min(frameWidth, x1)), + y0: Math.max(0, Math.min(frameHeight, y0)), + y1: Math.max(0, Math.min(frameHeight, y1)), + }; + if (clamped.x1 - clamped.x0 < 2 || clamped.y1 - clamped.y0 < 2) return null; + return clamped; +}; + +export interface Hsv { + /** Degrees, 0..360. */ + h: number; + s: number; + v: number; +} + +/** Standard conversion, on 0..255 channels. */ +export const toHsv = (r: number, g: number, b: number): Hsv => { + const rn = r / 255; + const gn = g / 255; + const bn = b / 255; + const max = Math.max(rn, gn, bn); + const min = Math.min(rn, gn, bn); + const delta = max - min; + + let h = 0; + if (delta > 0) { + if (max === rn) h = 60 * (((gn - bn) / delta) % 6); + else if (max === gn) h = 60 * ((bn - rn) / delta + 2); + else h = 60 * ((rn - gn) / delta + 4); + } + if (h < 0) h += 360; + return { h, s: max === 0 ? 0 : delta / max, v: max }; +}; + +/** Which bin a colour belongs in; grey and near-black go to the lightness bins. */ +export const binFor = ({ h, s, v }: Hsv): number => { + if (s < MIN_SATURATION || v < MIN_VALUE) { + const bin = Math.min(GREY_BINS - 1, Math.floor(v * GREY_BINS)); + return HUE_BINS * SAT_BINS + Math.max(0, bin); + } + const hue = Math.min(HUE_BINS - 1, Math.floor(h / (360 / HUE_BINS))); + const sat = s < 0.5 ? 0 : 1; + return hue * SAT_BINS + sat; +}; + +/** + * Adds one crop's colours into an accumulator. BGR because that is the order + * the frame decoder emits, matching what the detector was trained on. + */ +export const accumulate = ( + pixels: Buffer | Uint8Array, + frameWidth: number, + rect: Rect, + into: Float64Array, +): number => { + let counted = 0; + for (let y = rect.y0; y < rect.y1; y += 1) { + const row = y * frameWidth; + for (let x = rect.x0; x < rect.x1; x += 1) { + const at = (row + x) * 3; + const b = pixels[at] ?? 0; + const g = pixels[at + 1] ?? 0; + const r = pixels[at + 2] ?? 0; + const bin = binFor(toHsv(r, g, b)); + into[bin] = (into[bin] ?? 0) + 1; + counted += 1; + } + } + return counted; +}; + +/** Sum to one, so signatures from crops of different sizes are comparable. */ +export const normalize = (histogram: Float64Array | number[]): number[] => { + let total = 0; + for (const value of histogram) total += value; + if (total <= 0) return Array.from({ length: histogram.length }, () => 0); + return Array.from(histogram, (value) => value / total); +}; + +/** + * Histogram intersection: 1 for identical signatures, 0 for no shared colour at + * all. Chosen over a Euclidean distance because it degrades gracefully when a + * crop catches some background — the extra mass simply fails to overlap, + * rather than dominating the distance. + */ +export const similarity = (a: number[], b: number[]): number => { + const length = Math.min(a.length, b.length); + let shared = 0; + for (let i = 0; i < length; i += 1) shared += Math.min(a[i] ?? 0, b[i] ?? 0); + return shared; +}; diff --git a/apps/cv-worker/src/index.ts b/apps/cv-worker/src/index.ts index 214a7b3..1064d94 100644 --- a/apps/cv-worker/src/index.ts +++ b/apps/cv-worker/src/index.ts @@ -3,7 +3,7 @@ import { existsSync, readFileSync, realpathSync } from 'node:fs'; import { availableParallelism, cpus } from 'node:os'; import { fileURLToPath } from 'node:url'; -import { probe } from '@reeleel/core'; +import { probe, requireBinary } from '@reeleel/core'; import { COCO_TO_SPORT, @@ -13,6 +13,8 @@ import { } from './classes.js'; import { DEFAULT_MODEL, defaultModelPath, fetchModel, resolveModelPath } from './models.js'; import { runPipeline } from './pipeline.js'; +import { computeSignatures } from './signatures.js'; +import type { SignatureBox } from './signatures.js'; /** Minimal flag parsing — the protocol is fixed and this has no users but us. */ export const parseArgs = (argv: string[]): { command: string; flags: Record; bare: Set } => { @@ -194,6 +196,75 @@ const detectAndTrack = async (flags: Record): Promise => { } }; +/** Reads the whole of stdin, which is how a box list arrives. */ +const readStdin = async (): Promise => { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) chunks.push(chunk as Buffer); + return Buffer.concat(chunks).toString('utf8'); +}; + +/** + * Colour signatures for a list of boxes, so the host can tell whether two + * tracks are the same child. Boxes arrive on stdin because there can be + * thousands of them and an argv has limits. + */ +const appearance = async (flags: Record): Promise => { + const input = flags['input']; + if (input === undefined) { + emit({ error: '--input is required.' }); + return; + } + + let request: { boxes?: SignatureBox[] }; + try { + request = JSON.parse(await readStdin()) as { boxes?: SignatureBox[] }; + } catch (cause) { + emit({ error: `stdin was not the JSON box list this expects: ${String(cause)}` }); + return; + } + const boxes = request.boxes ?? []; + if (boxes.length === 0) { + emit({ error: 'No boxes were given, so there is nothing to measure.' }); + return; + } + + const media = await probe(input); + const width = media.video?.width ?? 0; + const height = media.video?.height ?? 0; + if (width <= 0 || height <= 0) { + emit({ error: `${input} has no readable video stream.` }); + return; + } + + const controller = new AbortController(); + const onSignal = (): void => controller.abort(); + process.once('SIGINT', onSignal); + process.once('SIGTERM', onSignal); + + try { + const started = Date.now(); + const result = await computeSignatures({ + input, + ffmpegPath: requireBinary('ffmpeg'), + sourceWidth: width, + sourceHeight: height, + fps: media.video?.fps ?? 0, + boxes, + samplesPerSecond: number(flags['samples-per-second'], 2), + decodeWidth: number(flags['decode-width'], 960), + signal: controller.signal, + }); + process.stderr.write( + `signatures: ${Object.keys(result.signatures).length} track(s) from ` + + `${result.framesRead} frames in ${Math.round((Date.now() - started) / 1000)}s\n`, + ); + emit({ signatures: result.signatures, pixels: result.pixels }); + } finally { + process.off('SIGINT', onSignal); + process.off('SIGTERM', onSignal); + } +}; + const fetch = async (flags: Record): Promise => { const sport = flags['sport'] ?? 'soccer'; const url = flags['url'] ?? DEFAULT_MODEL.url; @@ -225,6 +296,9 @@ export const main = async (argv: string[]): Promise => { case 'detect-and-track': await detectAndTrack(flags); return 0; + case 'appearance': + await appearance(flags); + return 0; case 'capabilities': capabilities(); return 0; @@ -233,7 +307,7 @@ export const main = async (argv: string[]): Promise => { return 0; default: process.stderr.write( - 'usage: reeleel-cv [options]\n' + + 'usage: reeleel-cv [options]\n' + 'see workers/cv/README.md for the protocol\n', ); return command === 'help' ? 0 : 1; diff --git a/apps/cv-worker/src/signatures.ts b/apps/cv-worker/src/signatures.ts new file mode 100644 index 0000000..baa0f5e --- /dev/null +++ b/apps/cv-worker/src/signatures.ts @@ -0,0 +1,115 @@ +import { accumulate, BIN_COUNT, normalize, torsoRect } from './appearance.js'; +import { frameStream } from './frames.js'; + +/** One box to look at, in source-video pixels. */ +export interface SignatureBox { + track: string; + ts: number; + x: number; + y: number; + w: number; + h: number; +} + +export interface SignatureRequest { + input: string; + /** + * Injected rather than resolved here, so this module depends on nothing but + * a frame decoder — which is what lets it be run standalone against a real + * video to check the matching before trusting it with anyone's reel. + */ + ffmpegPath: string; + sourceWidth: number; + sourceHeight: number; + fps: number; + boxes: SignatureBox[]; + /** Frames to look at per second of footage. */ + samplesPerSecond?: number; + /** Width to decode at; a jersey needs colour, not detail. */ + decodeWidth?: number; + signal?: AbortSignal; +} + +export interface SignatureResult { + /** Track id → normalized colour histogram. */ + signatures: Record; + /** Track id → how many pixels went into it, so thin evidence can be refused. */ + pixels: Record; + framesRead: number; +} + +/** + * Which decoded frame a box should be measured on. + * + * Decoding every frame to sample a handful would cost minutes for nothing, and + * seeking to each box individually costs more than reading straight through. + * Instead the video is read once at a coarse stride and every box is snapped to + * its nearest decoded frame — a player has not moved meaningfully in the eighth + * of a second that costs, and their shirt has not changed colour at all. + */ +export const frameIndexFor = (ts: number, fps: number, stride: number): number => + Math.max(0, Math.round((ts * fps) / stride) * stride); + +/** + * Colour signatures for a set of tracks, from one pass over the video. + * + * Runs on whatever file it is given — the 540p proxy is deliberate and + * sufficient: this measures the colour of a shirt, and downscaling averages + * noise out of it rather than destroying anything that matters. + */ +export const computeSignatures = async ( + request: SignatureRequest, +): Promise => { + const fps = request.fps > 0 ? request.fps : 30; + const perSecond = request.samplesPerSecond ?? 2; + const stride = Math.max(1, Math.round(fps / perSecond)); + + const decodeWidth = Math.min(request.decodeWidth ?? 960, request.sourceWidth); + const scale = request.sourceWidth > 0 ? decodeWidth / request.sourceWidth : 1; + const decodeHeight = Math.max(2, Math.round(request.sourceHeight * scale)); + + // Boxes bucketed by the frame they will be measured on, so each decoded frame + // is a single lookup rather than a scan of every box. + const byFrame = new Map(); + for (const box of request.boxes) { + const index = frameIndexFor(box.ts, fps, stride); + const existing = byFrame.get(index); + if (existing === undefined) byFrame.set(index, [box]); + else existing.push(box); + } + + const bins = new Map(); + const pixels: Record = {}; + let framesRead = 0; + + for await (const frame of frameStream({ + input: request.input, + ffmpegPath: request.ffmpegPath, + sourceWidth: request.sourceWidth, + sourceHeight: request.sourceHeight, + targetWidth: decodeWidth, + targetHeight: decodeHeight, + frameStride: stride, + fps, + ...(request.signal === undefined ? {} : { signal: request.signal }), + })) { + framesRead += 1; + const due = byFrame.get(frame.index); + if (due === undefined) continue; + + for (const box of due) { + const rect = torsoRect(box, scale, decodeWidth, decodeHeight); + if (rect === null) continue; + let accumulator = bins.get(box.track); + if (accumulator === undefined) { + accumulator = new Float64Array(BIN_COUNT); + bins.set(box.track, accumulator); + } + pixels[box.track] = (pixels[box.track] ?? 0) + accumulate(frame.pixels, decodeWidth, rect, accumulator); + } + } + + const signatures: Record = {}; + for (const [track, accumulator] of bins) signatures[track] = normalize(accumulator); + return { signatures, pixels, framesRead }; +}; diff --git a/apps/web/src/actions.ts b/apps/web/src/actions.ts index d49698e..31fb56b 100644 --- a/apps/web/src/actions.ts +++ b/apps/web/src/actions.ts @@ -24,6 +24,7 @@ import { loadTrackSeries, isReelEelError, listAthleteCandidates, + proposeAthleteTracks, listAthletes, listExports, listJobLogsSince, @@ -627,6 +628,55 @@ export const registerActions = (app: Hono): void => { * happened — seconds rather than another full pass. Identifying your athlete * should not cost another minute of inference. */ + /** + * The rest of the game, found by what the athlete looks like. + * + * Picking an athlete by hand only ever labels them where the user happened to + * look, and re-identification can only confirm a binding where it already + * exists — production had a child known for 31.7s of a 300s game. This + * proposes the other tracks that match their shirt, with the score, and + * assigns nothing: the grid shows them for confirmation. A wrong answer here + * puts somebody else's child in the reel, so it is deliberately a suggestion. + */ + app.get('/projects/:ref/athletes/:id/suggestions', async (c) => { + try { + const root = await rootOf(c); + const videoId = c.req.query('videoId'); + const found = await proposeAthleteTracks(root, c.req.param('id') ?? '', { + ...(videoId === undefined ? {} : { videoId }), + }); + + // Reuse the picker's preview geometry so a proposal renders as the same + // crop the user is already choosing from. + const previews = new Map( + (await listAthleteCandidates(root, { limit: 10_000, minSeconds: 0 })).map( + (candidate) => [candidate.trackId, candidate], + ), + ); + + return c.json({ + ok: true, + considered: found.considered, + referenceTrackIds: found.referenceTrackIds, + proposals: found.proposals.flatMap((proposal) => { + const preview = previews.get(proposal.trackId); + return preview === undefined + ? [] + : [ + { + ...preview, + score: proposal.score, + gapSeconds: proposal.gapSeconds, + distancePx: proposal.distancePx, + }, + ]; + }), + }); + } catch (error) { + return uploadJson(c, error); + } + }); + app.post('/projects/:ref/athletes/:id/track', async (c) => { const bad = await guard(c); if (bad !== null) return bad; diff --git a/apps/web/src/client/identify.tsx b/apps/web/src/client/identify.tsx index 27c91c0..f742b19 100644 --- a/apps/web/src/client/identify.tsx +++ b/apps/web/src/client/identify.tsx @@ -28,6 +28,13 @@ interface Candidate { sourceHeight: number; } +/** Why the server thinks a fragment continues the athlete. */ +interface Match { + score: number; + gapSeconds: number; + distancePx: number; +} + interface Athlete { id: string; name: string | null; @@ -77,6 +84,14 @@ const Identify = ({ base }: { base: string }) => { const [busy, setBusy] = useState(null); const [error, setError] = useState(null); const [loaded, setLoaded] = useState(false); + /** + * Appearance scores for tracks the server thinks are the same child, keyed by + * track id. Picking by hand only labels the athlete where the user happened + * to look — on a real game that was 31.7s out of 300. + */ + const [scores, setScores] = useState>({}); + const [finding, setFinding] = useState(false); + const [found, setFound] = useState(null); const load = async (): Promise => { try { @@ -118,6 +133,53 @@ const Identify = ({ base }: { base: string }) => { ); }; + /** + * Ask the server to find this child in the rest of the footage by the colour + * of their shirt. Matches are selected, not applied: the grid shows them + * ticked so a human confirms before anything is bound, because the cost of a + * confident wrong answer is another family's child in the reel. + */ + const findRest = async (): Promise => { + setFinding(true); + setError(null); + setFound(null); + try { + const response = await fetch(`${base}/athletes/${athleteId}/suggestions`, { + headers: { accept: 'application/json' }, + }); + const body = (await response.json()) as { + ok: boolean; + proposals?: (Candidate & Match)[]; + considered?: number; + error?: string; + }; + if (!response.ok || !body.ok) throw new Error(body.error ?? 'Could not search the footage.'); + + const proposals = body.proposals ?? []; + setScores( + Object.fromEntries( + proposals.map((p) => [ + p.trackId, + { score: p.score, gapSeconds: p.gapSeconds, distancePx: p.distancePx }, + ]), + ), + ); + setPicked((current) => [ + ...current, + ...proposals.map((p) => p.trackId).filter((id) => !current.includes(id)), + ]); + setFound( + proposals.length === 0 + ? `Nothing else followed on from where your athlete was, out of ${body.considered ?? 0} tracks checked. Pick any more you recognise by hand.` + : `Followed your athlete into ${proposals.length} more fragment(s), out of ${body.considered ?? 0} checked, and selected them. Each one carries on from where a fragment you already have left off, in a matching shirt — check them and untick anything that is not them.`, + ); + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setFinding(false); + } + }; + const save = async (): Promise => { /** * No athlete yet is not a reason to refuse — it is the ordinary first-run @@ -205,22 +267,36 @@ const Identify = ({ base }: { base: string }) => { every crop that is your athlete — the same child usually appears more than once, and each one you add is more of the game they are followed through.

+ {found === null ? null :

{found}

}
- {candidates.map((candidate) => ( - - ))} + {/* Matches first, so the ones needing a decision are not buried + halfway down a grid of forty strangers. */} + {[...candidates] + .sort((a, b) => (scores[b.trackId]?.score ?? -1) - (scores[a.trackId]?.score ?? -1)) + .map((candidate) => { + const match = scores[candidate.trackId]; + return ( + + ); + })}
+ {/* Only useful once they are bound somewhere: the search compares + against the tracks already saved for this athlete. */} + {coverage}
diff --git a/packages/core/src/appearance.test.ts b/packages/core/src/appearance.test.ts new file mode 100644 index 0000000..f5f806f --- /dev/null +++ b/packages/core/src/appearance.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from 'vitest'; + +import { + COLOUR_FLOOR, + linkBetween, + mergeSignatures, + overlapsInTime, + sampleBoxes, + similarity, + spanOf, +} from './appearance.js'; +import type { TrackSeries } from './scoring.js'; + +/** + * Appearance matching exists because box overlap can only confirm an athlete + * where they were already known — on production it left a child identified for + * 31.7s of a 300s game. The risk it introduces is the mirror image: a confident + * wrong match puts another family's child in the reel. These tests are mostly + * about the guards, not the matching. + */ + +const track = (id: string, from: number, to: number, step = 0.25): TrackSeries => ({ + id, + className: 'player', + samples: Array.from({ length: Math.round((to - from) / step) + 1 }, (_unused, i) => ({ + ts: Number((from + i * step).toFixed(3)), + x: 100 + i, + y: 200, + w: 40, + h: 100, + confidence: 0.9, + })), +}); + +describe('a track’s span', () => { + it('reads first and last sample', () => { + expect(spanOf(track('a', 10, 20))).toEqual({ start: 10, end: 20 }); + }); + + it('does not throw on an empty track', () => { + expect(spanOf({ id: 'empty', className: 'player', samples: [] })).toEqual({ start: 0, end: 0 }); + }); +}); + +describe('one child cannot be in two places at once', () => { + it('rejects a candidate that shares any moment with a known track', () => { + // The most dangerous false match there is: a teammate in the same kit, + // standing next to them. Colour cannot separate those; time can. + expect(overlapsInTime(track('known', 10, 20), track('teammate', 15, 25))).toBe(true); + expect(overlapsInTime(track('known', 10, 20), track('teammate', 19.9, 40))).toBe(true); + }); + + it('allows a candidate from a different part of the game', () => { + expect(overlapsInTime(track('known', 10, 20), track('later', 20.5, 40))).toBe(false); + expect(overlapsInTime(track('known', 100, 120), track('earlier', 10, 40))).toBe(false); + }); + + it('treats touching spans as overlapping', () => { + expect(overlapsInTime(track('a', 10, 20), track('b', 20, 30))).toBe(true); + }); +}); + +describe('choosing which boxes to look at', () => { + it('spreads samples across the track rather than taking consecutive frames', () => { + const boxes = sampleBoxes(track('a', 0, 10, 0.05), 0.5, 12); + expect(boxes.length).toBeLessThanOrEqual(12); + const gaps = boxes.slice(1).map((box, i) => box.ts - (boxes[i]?.ts ?? 0)); + for (const gap of gaps) expect(gap).toBeGreaterThan(0.3); + }); + + it('caps a long track without dropping its tail', () => { + const boxes = sampleBoxes(track('a', 0, 120, 0.25), 0.5, 10); + expect(boxes).toHaveLength(10); + // A child who crosses to the other end of the court is still that child. + expect(boxes[boxes.length - 1]?.ts).toBeGreaterThan(90); + }); + + it('keeps a short track whole', () => { + const boxes = sampleBoxes(track('a', 0, 2, 0.25), 0.5, 12); + expect(boxes).toHaveLength(5); + }); + + it('returns nothing for an empty track instead of throwing', () => { + expect(sampleBoxes({ id: 'e', className: 'player', samples: [] })).toEqual([]); + }); +}); + +describe('building the reference signature', () => { + it('weights by evidence, so a glimpse cannot outvote a long look', () => { + const merged = mergeSignatures([ + { signature: [1, 0, 0], weight: 1000 }, + { signature: [0, 1, 0], weight: 1 }, + ]); + expect(merged[0]).toBeGreaterThan(0.99); + }); + + it('sums to one, so it is comparable to any candidate', () => { + const merged = mergeSignatures([ + { signature: [0.5, 0.5, 0], weight: 3 }, + { signature: [0, 0.25, 0.75], weight: 7 }, + ]); + expect(merged.reduce((a, b) => a + b, 0)).toBeCloseTo(1); + }); + + it('ignores tracks no frames could be read for', () => { + const merged = mergeSignatures([ + { signature: [1, 0, 0], weight: 500 }, + { signature: [], weight: 0 }, + ]); + expect(merged).toEqual([1, 0, 0]); + }); + + it('returns nothing when there is no evidence at all', () => { + expect(mergeSignatures([])).toEqual([]); + expect(mergeSignatures([{ signature: [], weight: 0 }])).toEqual([]); + }); +}); + +describe('the colour veto', () => { + it('sits clear of two different jerseys and below the same one', () => { + // Mirrors the worker's measured behaviour: different colours share almost + // nothing, the same colour under different light shares most of it. + const different = similarity([1, 0, 0, 0], [0, 0, 1, 0]); + const sameThroughShade = similarity([0.7, 0.3, 0, 0], [0.55, 0.45, 0, 0]); + expect(different).toBeLessThan(COLOUR_FLOOR); + expect(sameThroughShade).toBeGreaterThan(COLOUR_FLOOR); + }); + + it('is a proper intersection, capped at one', () => { + expect(similarity([0.5, 0.5], [0.5, 0.5])).toBeCloseTo(1); + expect(similarity([1, 0], [0, 1])).toBe(0); + }); +}); + +/** + * Continuity is the part that actually claims "this is the same child", so it + * is the part that has to be mean. Measured on a real game, continuity with no + * colour veto pulled in 50 extra fragments and 120s of "athlete"; with it, 8 + * and 51s. + */ +describe('linking one fragment to the next', () => { + const FRAME = 1920; + /** A track sitting still at (x, y) between two times. */ + const at = (id: string, from: number, to: number, x: number, y = 400): TrackSeries => ({ + id, + className: 'player', + samples: [ + { ts: from, x, y, w: 40, h: 100, confidence: 0.9 }, + { ts: to, x, y, w: 40, h: 100, confidence: 0.9 }, + ], + }); + + it('links a fragment that resumes moments later, close by', () => { + const link = linkBetween(at('known', 10, 20, 500), at('next', 20.5, 25, 560), FRAME); + expect(link).not.toBeNull(); + expect(link?.gapSeconds).toBeCloseTo(0.5); + expect(link?.distancePx).toBeCloseTo(60); + }); + + it('links backwards, so a fragment can extend the athlete earlier', () => { + const link = linkBetween(at('known', 10, 20, 500), at('before', 5, 9.5, 520), FRAME); + expect(link).not.toBeNull(); + expect(link?.gapSeconds).toBeCloseTo(0.5); + }); + + it('refuses a silence longer than a child can be vouched for', () => { + expect(linkBetween(at('known', 10, 20, 500), at('later', 25, 30, 505), FRAME)).toBeNull(); + }); + + it('refuses a jump no child could have run', () => { + // Half a second, most of the way across the court: that is a different kid. + expect(linkBetween(at('known', 10, 20, 200), at('far', 20.5, 25, 1800), FRAME)).toBeNull(); + }); + + it('allows further travel when more time has passed', () => { + const brief = linkBetween(at('known', 10, 20, 200), at('far', 20.2, 25, 600), FRAME); + const longer = linkBetween(at('known', 10, 20, 200), at('far', 21.5, 25, 600), FRAME); + expect(brief).toBeNull(); + expect(longer).not.toBeNull(); + }); + + it('refuses two fragments that are on screen together', () => { + // Overlapping in time means no gap in either direction, so no link at all. + expect(linkBetween(at('known', 10, 20, 500), at('same-time', 15, 25, 505), FRAME)).toBeNull(); + }); + + it('scales its reach with the frame, not with a pixel count', () => { + const wide = linkBetween(at('known', 10, 20, 500), at('next', 20.5, 25, 900), 3840); + const narrow = linkBetween(at('known', 10, 20, 500), at('next', 20.5, 25, 900), 640); + expect(wide).not.toBeNull(); + expect(narrow).toBeNull(); + }); +}); diff --git a/packages/core/src/appearance.ts b/packages/core/src/appearance.ts new file mode 100644 index 0000000..62be5d1 --- /dev/null +++ b/packages/core/src/appearance.ts @@ -0,0 +1,427 @@ +import { existsSync } from 'node:fs'; + +import { getAthlete } from './athletes.js'; +import { resolveCvWorker } from './analyze.js'; +import { ReelEelError } from './errors.js'; +import { run } from './ffmpeg.js'; +import { loadTrackSeries, tracksForAthlete } from './tracks.js'; +import type { TrackSeries } from './scoring.js'; +import { listVideos } from './videos.js'; + +/** + * Finding the same child in the rest of the game. + * + * Re-identification matched on box overlap, which can only ever confirm an + * athlete where they were already known — it recovers a binding across a + * re-detection and cannot do anything else. Measured on production, that left + * an athlete identified for 31.7s of a 300s game across six fragments, all of + * them inside the one 32-second window the user had originally pointed at. + * Every signal that follows the athlete was therefore dark for 90% of the + * footage, and the moments that survived were scene-wide ones that had nothing + * to do with them. + * + * The missing ingredient is appearance. Nothing here decides anything: it + * ranks, and a human confirms. A wrong answer puts another family's child in + * your highlight reel, so the design point throughout is that a weak match is + * dropped rather than guessed. + */ + +export interface AthleteProposal { + trackId: string; + /** Colour-signature agreement with the athlete's known tracks, 0..1. */ + score: number; + startTs: number; + endTs: number; + seconds: number; + samples: number; + /** The link that justified it, so a person can judge the claim. */ + gapSeconds: number; + distancePx: number; +} + +/** + * Colour is a veto, never an identifier. + * + * Measured on a real game: at a 0.55 colour match, 661 of 1152 candidate tracks + * qualified — 2,306 seconds of "athlete" in a 300-second video. That is not a + * tuning failure, it is what a shirt means. Teammates wear the same one, so a + * colour signature identifies a *team*, and the children it wrongly volunteers + * are precisely the ones standing next to yours. + * + * So the identity claim rests on continuity, and colour only ever rules a link + * out. The same measurement, three ways: continuity alone recovered 120.3s + * across 56 tracks (too permissive — it links whoever is nearby); colour alone + * 2,306s; both together 51.2s across 14 tracks, up from 31.7s across 6, with + * every link under a second of gap and a few hundred pixels of travel. + */ +export const COLOUR_FLOOR = 0.7; + +/** Longest silence a link may be drawn across. */ +export const MAX_LINK_SECONDS = 2; + +/** + * How far a child may travel between two fragments, as a fraction of frame + * width per second, plus a fixed allowance for the tracker's own jitter. + * + * At four seconds and this speed the accepted links reached 894 pixels of a + * 1920-wide frame — most of the way across a court — for eight more seconds of + * coverage. The gap limit above is where that trade stops being worth taking. + */ +export const LINK_SPEED_FRACTION = 0.31; +export const LINK_SLACK_FRACTION = 0.03; + +/** Time span a track occupies. */ +export const spanOf = (series: TrackSeries): { start: number; end: number } => { + const first = series.samples[0]; + const last = series.samples[series.samples.length - 1]; + if (first === undefined || last === undefined) return { start: 0, end: 0 }; + return { start: first.ts, end: last.ts }; +}; + +/** + * Whether two tracks are ever on screen at the same moment. + * + * The hardest constraint available and the cheapest: one child cannot be in two + * places at once, so a candidate that coexists with a track already known to be + * the athlete is definitively somebody else — however similar their shirt. + * Teammates wear the same colour, so without this the strongest false matches + * would be exactly the children standing next to them. + */ +export const overlapsInTime = (a: TrackSeries, b: TrackSeries): boolean => { + const first = spanOf(a); + const second = spanOf(b); + return first.start <= second.end && second.start <= first.end; +}; + +/** + * A handful of boxes spread across a track's life, rather than all of them. + * + * A signature wants variety — different moments, poses and lighting — not + * volume. Sampling every half-second and capping keeps a thirty-second track + * from drowning out a three-second one in the reference average. + */ +export const sampleBoxes = ( + series: TrackSeries, + everySeconds = 0.5, + cap = 12, +): { ts: number; x: number; y: number; w: number; h: number }[] => { + const picked: { ts: number; x: number; y: number; w: number; h: number }[] = []; + let nextTs = Number.NEGATIVE_INFINITY; + for (const sample of series.samples) { + if (sample.ts < nextTs) continue; + picked.push({ ts: sample.ts, x: sample.x, y: sample.y, w: sample.w, h: sample.h }); + nextTs = sample.ts + everySeconds; + } + if (picked.length <= cap) return picked; + + // Thin evenly rather than truncating, so the tail of a long track is still + // represented — a child who changes ends of the court is still that child. + const step = picked.length / cap; + return Array.from({ length: cap }, (_unused, i) => picked[Math.floor(i * step)]).filter( + (box): box is { ts: number; x: number; y: number; w: number; h: number } => box !== undefined, + ); +}; + +/** Weighted mean of several signatures, renormalized. */ +export const mergeSignatures = ( + parts: { signature: number[]; weight: number }[], +): number[] => { + const usable = parts.filter((part) => part.weight > 0 && part.signature.length > 0); + const first = usable[0]; + if (first === undefined) return []; + + const totals = new Array(first.signature.length).fill(0); + let weightSum = 0; + for (const part of usable) { + weightSum += part.weight; + for (let i = 0; i < totals.length; i += 1) { + totals[i] = (totals[i] ?? 0) + (part.signature[i] ?? 0) * part.weight; + } + } + if (weightSum <= 0) return []; + const scaled = totals.map((value) => value / weightSum); + const sum = scaled.reduce((a, b) => a + b, 0); + return sum > 0 ? scaled.map((value) => value / sum) : scaled; +}; + +const centreOf = (sample: { x: number; y: number; w: number; h: number }): { x: number; y: number } => ({ + x: sample.x + sample.w / 2, + y: sample.y + sample.h / 2, +}); + +export interface Link { + gapSeconds: number; + distancePx: number; +} + +/** + * Whether a candidate plausibly continues a known track — picking up where it + * left off, or leading into where it began. + * + * This is the part that actually claims identity, so it is deliberately mean: + * a short silence, and a distance a child could really have covered in it. Both + * directions, because a fragment can extend an athlete backwards just as + * usefully as forwards. + */ +export const linkBetween = ( + known: TrackSeries, + candidate: TrackSeries, + frameWidth: number, + maxSeconds = MAX_LINK_SECONDS, +): Link | null => { + const knownFirst = known.samples[0]; + const knownLast = known.samples[known.samples.length - 1]; + const otherFirst = candidate.samples[0]; + const otherLast = candidate.samples[candidate.samples.length - 1]; + if ( + knownFirst === undefined || + knownLast === undefined || + otherFirst === undefined || + otherLast === undefined + ) { + return null; + } + + const reach = (gap: number): number => + frameWidth * LINK_SPEED_FRACTION * gap + frameWidth * LINK_SLACK_FRACTION; + + const forward = otherFirst.ts - knownLast.ts; + if (forward > 0 && forward <= maxSeconds) { + const distance = Math.hypot( + centreOf(knownLast).x - centreOf(otherFirst).x, + centreOf(knownLast).y - centreOf(otherFirst).y, + ); + if (distance <= reach(forward)) return { gapSeconds: forward, distancePx: distance }; + } + + const backward = knownFirst.ts - otherLast.ts; + if (backward > 0 && backward <= maxSeconds) { + const distance = Math.hypot( + centreOf(knownFirst).x - centreOf(otherLast).x, + centreOf(knownFirst).y - centreOf(otherLast).y, + ); + if (distance <= reach(backward)) return { gapSeconds: backward, distancePx: distance }; + } + + return null; +}; + +/** Histogram intersection, mirroring the worker's own comparison. */ +export const similarity = (a: number[], b: number[]): number => { + const length = Math.min(a.length, b.length); + let shared = 0; + for (let i = 0; i < length; i += 1) shared += Math.min(a[i] ?? 0, b[i] ?? 0); + return shared; +}; + +export interface ProposalOptions { + videoId?: string; + /** Ignore candidates shorter than this. Default 1.5s. */ + minSeconds?: number; + /** Minimum agreement to propose at all. Default {@link PROPOSAL_THRESHOLD}. */ + threshold?: number; + /** Most proposals to return. Default 40. */ + limit?: number; + signal?: AbortSignal; +} + +export interface ProposalResult { + proposals: AthleteProposal[]; + /** Tracks already assigned to this athlete, which are never proposed again. */ + referenceTrackIds: string[]; + /** How many tracks were compared, so "none found" can be told from "none tried". */ + considered: number; +} + +interface WorkerSignatures { + signatures?: Record; + pixels?: Record; + error?: string; +} + +/** + * Ranks the tracks most likely to be this athlete, elsewhere in the video. + * + * Nothing is assigned. The caller shows these to a human, because this cannot + * tell twins apart and should not pretend to. + */ +export const proposeAthleteTracks = async ( + root: string, + athleteId: string, + options: ProposalOptions = {}, +): Promise => { + const athlete = await getAthlete(root, athleteId); + const videos = await listVideos(root); + const video = + options.videoId === undefined + ? videos[0] + : videos.find((candidate) => candidate.id === options.videoId); + if (video === undefined) { + throw new ReelEelError('NOT_FOUND', 'This project has no video to search.'); + } + + const series = await loadTrackSeries(root, video.id); + const assigned = new Set(await tracksForAthlete(root, athlete.id)); + if (athlete.focalTrackId !== null) assigned.add(athlete.focalTrackId); + + const reference = series.filter((track) => assigned.has(track.id)); + if (reference.length === 0) { + throw new ReelEelError( + 'NOT_FOUND', + `${athlete.name} is not bound to any track yet, so there is nothing to compare against.`, + { hint: 'Identify them on one clip first, then search for the rest.' }, + ); + } + + const minSeconds = options.minSeconds ?? 1.5; + const candidates = series.filter((track) => { + if (assigned.has(track.id)) return false; + if (track.className !== 'player') return false; + const { start, end } = spanOf(track); + if (end - start < minSeconds) return false; + // One child, one place at a time. + return !reference.some((known) => overlapsInTime(known, track)); + }); + + if (candidates.length === 0) { + return { proposals: [], referenceTrackIds: [...assigned], considered: 0 }; + } + + const worker = resolveCvWorker(); + if (worker === null) { + throw new ReelEelError('WORKER_MISSING', 'The ReelEel CV worker is not installed.', { + hint: 'Appearance matching reads frames through the worker; install it as for detection.', + }); + } + + const boxes = [...reference, ...candidates].flatMap((track) => + sampleBoxes(track).map((box) => ({ track: track.id, ...box })), + ); + + /** + * The proxy is the right input here, unlike detection: this measures the + * colour of a shirt, and 540p carries that perfectly well while decoding in a + * fraction of the time. + */ + const input = + video.proxyPath !== null && existsSync(video.proxyPath) ? video.proxyPath : video.path; + + const result = await run( + worker.command, + [...worker.args, 'appearance', '--input', input], + { + stdin: JSON.stringify({ boxes }), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }, + ); + if (result.code !== 0) { + throw new ReelEelError('WORKER_CRASHED', 'The CV worker could not read this video.', { + hint: result.stderr.trim().split('\n').at(-1) ?? undefined, + }); + } + + let parsed: WorkerSignatures; + try { + parsed = JSON.parse(result.stdout) as WorkerSignatures; + } catch (cause) { + throw new ReelEelError('WORKER_CRASHED', 'The CV worker returned output we could not parse.', { + cause, + }); + } + if (parsed.error !== undefined) { + throw new ReelEelError('WORKER_CRASHED', parsed.error); + } + + const signatures = parsed.signatures ?? {}; + const pixels = parsed.pixels ?? {}; + + const referenceSignature = mergeSignatures( + reference.map((track) => ({ + signature: signatures[track.id] ?? [], + weight: pixels[track.id] ?? 0, + })), + ); + if (referenceSignature.length === 0) { + throw new ReelEelError( + 'NOT_FOUND', + `No frames could be read for ${athlete.name}'s existing tracks, so there is nothing to match.`, + ); + } + + /** + * Grow the athlete one fragment at a time, re-deriving their appearance after + * each addition. + * + * Iterative rather than a single pass because coverage compounds: the + * fragment that continues the athlete's *new* last track was not adjacent to + * anything before it was accepted. Re-averaging the signature as it goes also + * lets the reference follow a genuine change in lighting down the court, + * which a signature frozen at the first binding cannot. + */ + const threshold = options.threshold ?? COLOUR_FLOOR; + const frameWidth = video.probe?.video?.width ?? 1920; + const limit = options.limit ?? 40; + + const chosen = [...reference]; + const accepted: AthleteProposal[] = []; + const remaining = new Set(candidates); + + while (accepted.length < limit) { + const current = mergeSignatures( + chosen.map((track) => ({ + signature: signatures[track.id] ?? [], + weight: pixels[track.id] ?? 0, + })), + ); + if (current.length === 0) break; + + let best: { track: TrackSeries; colour: number; link: Link } | null = null; + for (const track of remaining) { + // One child, one place at a time — re-checked against everything accepted + // so far, not only the original binding. + if (chosen.some((known) => overlapsInTime(known, track))) { + remaining.delete(track); + continue; + } + + let link: Link | null = null; + for (const known of chosen) { + const found = linkBetween(known, track, frameWidth); + if (found !== null && (link === null || found.gapSeconds < link.gapSeconds)) link = found; + } + if (link === null) continue; + + const signature = signatures[track.id]; + if (signature === undefined || signature.length === 0) continue; + const colour = similarity(signature, current); + if (colour < threshold) continue; + + // Prefer the closest, cleanest link; colour has already done its only job. + const score = colour - link.distancePx / (frameWidth * 2); + const bestScore = + best === null ? -Infinity : best.colour - best.link.distancePx / (frameWidth * 2); + if (score > bestScore) best = { track, colour, link }; + } + + if (best === null) break; + remaining.delete(best.track); + chosen.push(best.track); + const { start, end } = spanOf(best.track); + accepted.push({ + trackId: best.track.id, + score: best.colour, + startTs: start, + endTs: end, + seconds: end - start, + samples: best.track.samples.length, + gapSeconds: best.link.gapSeconds, + distancePx: Math.round(best.link.distancePx), + }); + } + + return { + proposals: accepted, + referenceTrackIds: [...assigned], + considered: candidates.length, + }; +}; diff --git a/packages/core/src/ffmpeg.ts b/packages/core/src/ffmpeg.ts index 5231cbc..e23493a 100644 --- a/packages/core/src/ffmpeg.ts +++ b/packages/core/src/ffmpeg.ts @@ -76,22 +76,41 @@ export interface RunResult { export const run = ( binary: string, args: readonly string[], - options: { signal?: AbortSignal; onStderr?: (chunk: string) => void } = {}, + options: { + signal?: AbortSignal; + onStderr?: (chunk: string) => void; + /** + * Written to the child's stdin, then closed. Some worker commands take a + * list of thousands of boxes, which is far past what an argv can carry. + */ + stdin?: string; + } = {}, ): Promise => new Promise((resolve, reject) => { - const child = spawn(binary, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + const child = spawn(binary, args, { + stdio: [options.stdin === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'], + }); let stdout = ''; let stderr = ''; + if (options.stdin !== undefined && child.stdin !== null) { + // A child that dies before reading it all would otherwise raise EPIPE and + // lose the real error, which is whatever it wrote to stderr. + child.stdin.on('error', () => undefined); + child.stdin.end(options.stdin); + } + const abort = (): void => { child.kill('SIGTERM'); }; options.signal?.addEventListener('abort', abort, { once: true }); - child.stdout.on('data', (chunk: Buffer) => { + // Optional chaining because the stdio tuple is now computed, which costs + // TypeScript the overload that guaranteed these were non-null. + child.stdout?.on('data', (chunk: Buffer) => { stdout += chunk.toString(); }); - child.stderr.on('data', (chunk: Buffer) => { + child.stderr?.on('data', (chunk: Buffer) => { const text = chunk.toString(); stderr += text; options.onStderr?.(text); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 84052ac..70108bb 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -6,6 +6,7 @@ * the CLI honest about being the same product. */ export * from './analyze.js'; +export * from './appearance.js'; export * from './athletes.js'; export * from './camera.js'; export * from './candidates.js'; diff --git a/scripts/appearance-probe.mjs b/scripts/appearance-probe.mjs new file mode 100644 index 0000000..b4835a6 --- /dev/null +++ b/scripts/appearance-probe.mjs @@ -0,0 +1,121 @@ +/** + * Checks appearance matching against a real project, read-only. + * + * Synthetic tests prove a red shirt is not a blue one. They cannot tell you + * whether two teams of children on an actual gym floor separate at all, which + * is the only question that matters before this is allowed to suggest anyone. + * + * node --experimental-sqlite scripts/appearance-probe.mjs